An AEvolution-only Prometheus 3.13.3 and Alertmanager 0.34.1, pinned by digest and delivered by Argo (ADR-0072), scrape each region's operations Service through static targets (no API access, no RBAC); the region namespaces admit port 9100 from Prometheus only. Sixteen alert rules cover authority, lease, commits, backlog and object capacity and are proven by promtool injection tests in CI with tools copied from the same pinned images. Two new gauges expose object volume free space and the reserve. Alerts route to a null receiver until a destination is chosen; no broker is adopted and the database outbox stays. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
61 lines
2.9 KiB
C++
61 lines
2.9 KiB
C++
#pragma once
|
|
#include <array>
|
|
#include <cstddef>
|
|
#include <cstdint>
|
|
#include <string>
|
|
|
|
// Region host status and its bounded operational exposition (M7.8, ADR-0066).
|
|
// Portable: no sockets, clocks or persistence types. Liveness is not authority:
|
|
// only `ready` means the host may originate output.
|
|
namespace evo::server {
|
|
enum class HostPhase : std::uint8_t {
|
|
acquiring, // Waiting for the lease (busy/unavailable) or recovering.
|
|
running, // Authoritative: simulating and committing.
|
|
resolving, // A commit outcome is unknown: paused until resolved.
|
|
draining, // Shutdown: committing pending work, then releasing.
|
|
stopped, // Drained (lease released when possible).
|
|
quarantined, // Fail closed: committed state or history cannot be trusted.
|
|
};
|
|
inline constexpr std::size_t host_phase_count = 6;
|
|
// Why an accepted input will never be acknowledged by this host.
|
|
enum class Unacknowledged : std::uint8_t {
|
|
refused, // The simulation rejected it; no tick consumed it.
|
|
drained, // Still queued at shutdown.
|
|
fenced, // Authority was lost before its tick committed; the next owner never saw it.
|
|
quarantined,
|
|
};
|
|
inline constexpr std::size_t unacknowledged_count = 4;
|
|
|
|
struct HostStatus {
|
|
HostPhase phase{HostPhase::acquiring};
|
|
bool ready{}; // Authoritative now: recovered, running, lease may originate.
|
|
std::uint64_t epoch{};
|
|
std::uint64_t simulated_tick{}, committed_tick{}, committed_sequence{};
|
|
std::size_t queued{}, queued_bytes{};
|
|
std::uint64_t tick_debt{}, max_tick_debt{}; // Due minus simulated since running.
|
|
std::uint64_t commits{}, store_failures{}, unknown_outcomes{}, fences{};
|
|
// Local lease safety: time until this host must stop originating (0 = none).
|
|
std::uint64_t lease_remaining_ms{};
|
|
};
|
|
|
|
// Process-level counters and bounds the exposition adds to the host status.
|
|
struct OpsCounters {
|
|
std::size_t queue_capacity_inputs{}, queue_capacity_bytes{};
|
|
std::uint64_t checkpoint_age_ms{}; // Since the last confirmed commit or recovery.
|
|
std::uint64_t acks{}, outcomes{};
|
|
std::array<std::uint64_t, unacknowledged_count> dropped{}; // Indexed by Unacknowledged.
|
|
// M8.5: object volume free space and the store's reserve (ADR-0070), so capacity
|
|
// is alerted on before new objects are refused.
|
|
std::uint64_t objects_available_bytes{}, objects_reserve_bytes{};
|
|
};
|
|
|
|
[[nodiscard]] const char* phase_name(HostPhase phase) noexcept;
|
|
[[nodiscard]] const char* unacknowledged_name(Unacknowledged reason) noexcept;
|
|
|
|
// Prometheus text format 0.0.4. Fixed metric names; the only labels are `phase`
|
|
// (6 values) and `reason` (4 values), so cardinality is bounded by construction.
|
|
// No world/region/entity/request labels: scrape targets carry region identity.
|
|
[[nodiscard]] std::string render_metrics(const HostStatus& status, const OpsCounters& counters);
|
|
inline constexpr std::size_t max_metrics_bytes = 8192;
|
|
} // namespace evo::server
|