Foundation / checks (deploy) (push) Successful in 3m20s
Foundation / checks (ecs) (push) Successful in 1m59s
Foundation / checks (client) (push) Failing after 10m0s
Foundation / checks (network) (push) Successful in 13m9s
Foundation / checks (tsan) (push) Successful in 8m1s
Foundation / checks (checks) (push) Successful in 32m2s
Foundation / checks (postgres) (push) Failing after 26m39s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
ADR-0097 (M10.9 load and abuse qualification; only evolution-dev reference-worker runs qualify). evo-server measures each simulated tick's work (host step plus the M10 bridge, with the session layer's share kept separately) and feeds a portable LoadGovernor: grant logins are refused at a 2 s p95 of 15 ms or any tick debt, replication is shed in up to three levels at 20 ms or debt >= 2 (radius cap 7/7/5/3, far updates x1/x2/x2/x4, no fresh baselines, own markers unaffected), and both relax after 5 s of calm. Handoff tickets are still admitted. /metrics gains evo_m10_* gauges when the listener is enabled. Adds the missing ADR index entries for ADR-0094 to ADR-0096. Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
127 lines
6.3 KiB
C++
127 lines
6.3 KiB
C++
#include "evo/server/ops.hpp"
|
|
#include <string_view>
|
|
|
|
namespace evo::server {
|
|
const char* phase_name(HostPhase phase) noexcept {
|
|
switch (phase) {
|
|
case HostPhase::acquiring:
|
|
return "acquiring";
|
|
case HostPhase::running:
|
|
return "running";
|
|
case HostPhase::resolving:
|
|
return "resolving";
|
|
case HostPhase::draining:
|
|
return "draining";
|
|
case HostPhase::stopped:
|
|
return "stopped";
|
|
case HostPhase::quarantined:
|
|
return "quarantined";
|
|
}
|
|
return "unknown";
|
|
}
|
|
const char* unacknowledged_name(Unacknowledged reason) noexcept {
|
|
switch (reason) {
|
|
case Unacknowledged::refused:
|
|
return "refused";
|
|
case Unacknowledged::drained:
|
|
return "drained";
|
|
case Unacknowledged::fenced:
|
|
return "fenced";
|
|
case Unacknowledged::quarantined:
|
|
return "quarantined";
|
|
}
|
|
return "unknown";
|
|
}
|
|
|
|
namespace {
|
|
void metric(std::string& out, std::string_view name, std::string_view type, std::string_view help,
|
|
std::uint64_t value) {
|
|
out.append("# HELP ").append(name).append(" ").append(help).append("\n");
|
|
out.append("# TYPE ").append(name).append(" ").append(type).append("\n");
|
|
out.append(name).append(" ").append(std::to_string(value)).append("\n");
|
|
}
|
|
} // namespace
|
|
|
|
std::string render_metrics(const HostStatus& s, const OpsCounters& c) {
|
|
std::string out;
|
|
out.reserve(4096);
|
|
metric(out, "evo_region_ready", "gauge",
|
|
"1 only while this host holds current authority and may originate output", s.ready);
|
|
out.append("# HELP evo_region_phase Current host phase (one-hot)\n");
|
|
out.append("# TYPE evo_region_phase gauge\n");
|
|
for (std::size_t i = 0; i < host_phase_count; ++i) {
|
|
const auto phase = static_cast<HostPhase>(i);
|
|
out.append("evo_region_phase{phase=\"")
|
|
.append(phase_name(phase))
|
|
.append("\"} ")
|
|
.append(phase == s.phase ? "1" : "0")
|
|
.append("\n");
|
|
}
|
|
metric(out, "evo_region_lease_epoch", "gauge", "Lease epoch held or last held", s.epoch);
|
|
metric(out, "evo_region_lease_remaining_milliseconds", "gauge",
|
|
"Local time until this host must stop originating (0 without a lease)",
|
|
s.lease_remaining_ms);
|
|
metric(out, "evo_region_simulated_tick", "gauge", "Last simulated tick", s.simulated_tick);
|
|
metric(out, "evo_region_committed_tick", "gauge", "End tick of the committed head",
|
|
s.committed_tick);
|
|
metric(out, "evo_region_committed_sequence", "gauge", "Commit sequence of the head",
|
|
s.committed_sequence);
|
|
metric(out, "evo_region_commit_lag_ticks", "gauge",
|
|
"Simulated but not yet durably committed ticks",
|
|
s.simulated_tick >= s.committed_tick ? s.simulated_tick - s.committed_tick : 0);
|
|
metric(out, "evo_region_checkpoint_age_milliseconds", "gauge",
|
|
"Time since the last confirmed commit or recovery", c.checkpoint_age_ms);
|
|
metric(out, "evo_region_tick_debt", "gauge", "Due ticks not yet simulated", s.tick_debt);
|
|
metric(out, "evo_region_tick_debt_max", "gauge", "Largest tick debt since recovery",
|
|
s.max_tick_debt);
|
|
metric(out, "evo_region_queued_inputs", "gauge", "Admitted inputs awaiting a tick", s.queued);
|
|
metric(out, "evo_region_queued_bytes", "gauge", "Accounted bytes of queued inputs",
|
|
s.queued_bytes);
|
|
metric(out, "evo_region_queue_capacity_inputs", "gauge", "Input backlog bound",
|
|
c.queue_capacity_inputs);
|
|
metric(out, "evo_region_queue_capacity_bytes", "gauge", "Input backlog byte bound",
|
|
c.queue_capacity_bytes);
|
|
metric(out, "evo_region_objects_available_bytes", "gauge",
|
|
"Free bytes on the object volume available to the host", c.objects_available_bytes);
|
|
metric(out, "evo_region_objects_reserve_bytes", "gauge",
|
|
"Free-space reserve below which new objects are refused", c.objects_reserve_bytes);
|
|
metric(out, "evo_region_commits_total", "counter", "Confirmed commits", s.commits);
|
|
metric(out, "evo_region_store_failures_total", "counter",
|
|
"Storage or database failures that paused progress", s.store_failures);
|
|
metric(out, "evo_region_unknown_outcomes_total", "counter",
|
|
"Commits whose outcome had to be resolved", s.unknown_outcomes);
|
|
metric(out, "evo_region_fences_total", "counter", "Times this host lost authority", s.fences);
|
|
metric(out, "evo_region_acknowledgements_total", "counter", "Final acknowledgements emitted",
|
|
c.acks);
|
|
metric(out, "evo_region_outcomes_total", "counter", "Final permanent outcomes emitted",
|
|
c.outcomes);
|
|
out.append("# HELP evo_region_dropped_total Accepted inputs that will never be acknowledged\n");
|
|
out.append("# TYPE evo_region_dropped_total counter\n");
|
|
for (std::size_t i = 0; i < unacknowledged_count; ++i)
|
|
out.append("evo_region_dropped_total{reason=\"")
|
|
.append(unacknowledged_name(static_cast<Unacknowledged>(i)))
|
|
.append("\"} ")
|
|
.append(std::to_string(c.dropped[i]))
|
|
.append("\n");
|
|
if (c.m10_enabled) {
|
|
metric(out, "evo_m10_sessions", "gauge", "Open M10 sessions", c.m10_sessions);
|
|
metric(out, "evo_m10_refusing", "gauge", "1 while new logins are refused for load",
|
|
c.m10_load.refusing);
|
|
metric(out, "evo_m10_shed_level", "gauge", "Replication shed level (0-3)", c.m10_load.shed);
|
|
metric(out, "evo_m10_tick_work_p95_microseconds", "gauge",
|
|
"p95 per-tick region work over the last 40 ticks", c.m10_load.p95_us);
|
|
metric(out, "evo_m10_tick_work_p99_microseconds", "gauge",
|
|
"p99 per-tick region work over the last 40 ticks", c.m10_load.p99_us);
|
|
metric(out, "evo_m10_tick_work_max_microseconds", "gauge",
|
|
"Largest per-tick region work over the last 40 ticks", c.m10_load.max_us);
|
|
metric(out, "evo_m10_session_work_p95_microseconds", "gauge",
|
|
"p95 per-tick session-layer work over the last 40 ticks", c.m10_load.session_p95_us);
|
|
metric(out, "evo_m10_session_work_p99_microseconds", "gauge",
|
|
"p99 per-tick session-layer work over the last 40 ticks", c.m10_load.session_p99_us);
|
|
metric(out, "evo_m10_refused_logins_total", "counter", "Logins refused for load",
|
|
c.m10_refused_logins);
|
|
}
|
|
return out;
|
|
}
|
|
} // namespace evo::server
|