An AEvolution-only Prometheus 3.13.3 and Alertmanager 0.34.1, pinned by digest and delivered by Argo (ADR-0072), scrape each region's operations Service through static targets (no API access, no RBAC); the region namespaces admit port 9100 from Prometheus only. Sixteen alert rules cover authority, lease, commits, backlog and object capacity and are proven by promtool injection tests in CI with tools copied from the same pinned images. Two new gauges expose object volume free space and the reserve. Alerts route to a null receiver until a destination is chosen; no broker is adopted and the database outbox stays. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
109 lines
5.0 KiB
C++
109 lines
5.0 KiB
C++
#include "evo/server/ops.hpp"
|
|
#include <string_view>
|
|
|
|
namespace evo::server {
|
|
const char* phase_name(HostPhase phase) noexcept {
|
|
switch (phase) {
|
|
case HostPhase::acquiring:
|
|
return "acquiring";
|
|
case HostPhase::running:
|
|
return "running";
|
|
case HostPhase::resolving:
|
|
return "resolving";
|
|
case HostPhase::draining:
|
|
return "draining";
|
|
case HostPhase::stopped:
|
|
return "stopped";
|
|
case HostPhase::quarantined:
|
|
return "quarantined";
|
|
}
|
|
return "unknown";
|
|
}
|
|
const char* unacknowledged_name(Unacknowledged reason) noexcept {
|
|
switch (reason) {
|
|
case Unacknowledged::refused:
|
|
return "refused";
|
|
case Unacknowledged::drained:
|
|
return "drained";
|
|
case Unacknowledged::fenced:
|
|
return "fenced";
|
|
case Unacknowledged::quarantined:
|
|
return "quarantined";
|
|
}
|
|
return "unknown";
|
|
}
|
|
|
|
namespace {
|
|
void metric(std::string& out, std::string_view name, std::string_view type, std::string_view help,
|
|
std::uint64_t value) {
|
|
out.append("# HELP ").append(name).append(" ").append(help).append("\n");
|
|
out.append("# TYPE ").append(name).append(" ").append(type).append("\n");
|
|
out.append(name).append(" ").append(std::to_string(value)).append("\n");
|
|
}
|
|
} // namespace
|
|
|
|
std::string render_metrics(const HostStatus& s, const OpsCounters& c) {
|
|
std::string out;
|
|
out.reserve(4096);
|
|
metric(out, "evo_region_ready", "gauge",
|
|
"1 only while this host holds current authority and may originate output", s.ready);
|
|
out.append("# HELP evo_region_phase Current host phase (one-hot)\n");
|
|
out.append("# TYPE evo_region_phase gauge\n");
|
|
for (std::size_t i = 0; i < host_phase_count; ++i) {
|
|
const auto phase = static_cast<HostPhase>(i);
|
|
out.append("evo_region_phase{phase=\"")
|
|
.append(phase_name(phase))
|
|
.append("\"} ")
|
|
.append(phase == s.phase ? "1" : "0")
|
|
.append("\n");
|
|
}
|
|
metric(out, "evo_region_lease_epoch", "gauge", "Lease epoch held or last held", s.epoch);
|
|
metric(out, "evo_region_lease_remaining_milliseconds", "gauge",
|
|
"Local time until this host must stop originating (0 without a lease)",
|
|
s.lease_remaining_ms);
|
|
metric(out, "evo_region_simulated_tick", "gauge", "Last simulated tick", s.simulated_tick);
|
|
metric(out, "evo_region_committed_tick", "gauge", "End tick of the committed head",
|
|
s.committed_tick);
|
|
metric(out, "evo_region_committed_sequence", "gauge", "Commit sequence of the head",
|
|
s.committed_sequence);
|
|
metric(out, "evo_region_commit_lag_ticks", "gauge",
|
|
"Simulated but not yet durably committed ticks",
|
|
s.simulated_tick >= s.committed_tick ? s.simulated_tick - s.committed_tick : 0);
|
|
metric(out, "evo_region_checkpoint_age_milliseconds", "gauge",
|
|
"Time since the last confirmed commit or recovery", c.checkpoint_age_ms);
|
|
metric(out, "evo_region_tick_debt", "gauge", "Due ticks not yet simulated", s.tick_debt);
|
|
metric(out, "evo_region_tick_debt_max", "gauge", "Largest tick debt since recovery",
|
|
s.max_tick_debt);
|
|
metric(out, "evo_region_queued_inputs", "gauge", "Admitted inputs awaiting a tick", s.queued);
|
|
metric(out, "evo_region_queued_bytes", "gauge", "Accounted bytes of queued inputs",
|
|
s.queued_bytes);
|
|
metric(out, "evo_region_queue_capacity_inputs", "gauge", "Input backlog bound",
|
|
c.queue_capacity_inputs);
|
|
metric(out, "evo_region_queue_capacity_bytes", "gauge", "Input backlog byte bound",
|
|
c.queue_capacity_bytes);
|
|
metric(out, "evo_region_objects_available_bytes", "gauge",
|
|
"Free bytes on the object volume available to the host", c.objects_available_bytes);
|
|
metric(out, "evo_region_objects_reserve_bytes", "gauge",
|
|
"Free-space reserve below which new objects are refused", c.objects_reserve_bytes);
|
|
metric(out, "evo_region_commits_total", "counter", "Confirmed commits", s.commits);
|
|
metric(out, "evo_region_store_failures_total", "counter",
|
|
"Storage or database failures that paused progress", s.store_failures);
|
|
metric(out, "evo_region_unknown_outcomes_total", "counter",
|
|
"Commits whose outcome had to be resolved", s.unknown_outcomes);
|
|
metric(out, "evo_region_fences_total", "counter", "Times this host lost authority", s.fences);
|
|
metric(out, "evo_region_acknowledgements_total", "counter", "Final acknowledgements emitted",
|
|
c.acks);
|
|
metric(out, "evo_region_outcomes_total", "counter", "Final permanent outcomes emitted",
|
|
c.outcomes);
|
|
out.append("# HELP evo_region_dropped_total Accepted inputs that will never be acknowledged\n");
|
|
out.append("# TYPE evo_region_dropped_total counter\n");
|
|
for (std::size_t i = 0; i < unacknowledged_count; ++i)
|
|
out.append("evo_region_dropped_total{reason=\"")
|
|
.append(unacknowledged_name(static_cast<Unacknowledged>(i)))
|
|
.append("\"} ")
|
|
.append(std::to_string(c.dropped[i]))
|
|
.append("\n");
|
|
return out;
|
|
}
|
|
} // namespace evo::server
|