Add M10.9a load measurement, governor and overload shedding
Foundation / checks (deploy) (push) Successful in 3m20s
Foundation / checks (ecs) (push) Successful in 1m59s
Foundation / checks (client) (push) Failing after 10m0s
Foundation / checks (network) (push) Successful in 13m9s
Foundation / checks (tsan) (push) Successful in 8m1s
Foundation / checks (checks) (push) Successful in 32m2s
Foundation / checks (postgres) (push) Failing after 26m39s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped

ADR-0097 (M10.9 load and abuse qualification; only evolution-dev
reference-worker runs qualify). evo-server measures each simulated tick's
work (host step plus the M10 bridge, with the session layer's share kept
separately) and feeds a portable LoadGovernor: grant logins are refused
at a 2 s p95 of 15 ms or any tick debt, replication is shed in up to
three levels at 20 ms or debt >= 2 (radius cap 7/7/5/3, far updates
x1/x2/x2/x4, no fresh baselines, own markers unaffected), and both relax
after 5 s of calm. Handoff tickets are still admitted. /metrics gains
evo_m10_* gauges when the listener is enabled.

Adds the missing ADR index entries for ADR-0094 to ADR-0096. Refreshes
the dev session window.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-07 06:34:40 -04:00
co-authored by Claude Opus 5.5
parent 265163a9a6
commit b8cb9db297
22 changed files with 676 additions and 34 deletions
+96 -1
View File
@@ -52,6 +52,17 @@ std::vector<Sample> parse(const std::string& text) {
return out;
}
// Every host counter at its largest value (the M10 section stays disabled).
OpsCounters worst_counters() {
constexpr auto most = std::numeric_limits<std::uint64_t>::max();
OpsCounters c;
c.queue_capacity_inputs = c.queue_capacity_bytes = most;
c.checkpoint_age_ms = c.acks = c.outcomes = most;
c.dropped = {most, most, most, most};
c.objects_available_bytes = c.objects_reserve_bytes = most;
return c;
}
void exposition_is_bounded_and_well_formed() {
HostStatus s;
s.phase = HostPhase::running;
@@ -93,7 +104,7 @@ void exposition_is_bounded_and_well_formed() {
big.max_tick_debt = big.commits = big.store_failures = big.unknown_outcomes = most;
big.fences = big.lease_remaining_ms = most;
big.simulated_tick = 1; // Lag saturates at zero rather than wrapping.
OpsCounters bigc{most, most, most, most, most, {most, most, most, most}, most, most};
const auto bigc = worst_counters();
const auto large = render_metrics(big, bigc);
CHECK(large.size() <= max_metrics_bytes);
CHECK(large.find("evo_region_commit_lag_ticks 0\n") != std::string::npos);
@@ -105,11 +116,95 @@ void exposition_is_bounded_and_well_formed() {
CHECK(std::string(unacknowledged_name(static_cast<Unacknowledged>(i))) != "unknown");
std::printf("ops_metrics_bytes typical=%zu worst=%zu\n", text.size(), large.size());
}
// M10.9a (ADR-0097): the listener's governor families appear only when it is enabled.
void m10_load_metrics() {
HostStatus s;
OpsCounters c;
const auto without = parse(render_metrics(s, c)).size();
c.m10_enabled = true;
c.m10_sessions = 21;
c.m10_refused_logins = 3;
c.m10_load = {true, 2, 17'000, 19'500, 23'000, 40, 900, 1'100};
const auto text = render_metrics(s, c);
CHECK(parse(text).size() == without + 9);
CHECK(text.find("evo_m10_session_work_p99_microseconds 1100\n") != std::string::npos);
CHECK(text.find("evo_m10_sessions 21\n") != std::string::npos);
CHECK(text.find("evo_m10_refusing 1\n") != std::string::npos);
CHECK(text.find("evo_m10_shed_level 2\n") != std::string::npos);
CHECK(text.find("evo_m10_tick_work_p99_microseconds 19500\n") != std::string::npos);
CHECK(text.find("evo_m10_refused_logins_total 3\n") != std::string::npos);
constexpr auto most = std::numeric_limits<std::uint64_t>::max();
HostStatus big;
big.epoch = big.committed_tick = big.committed_sequence = big.tick_debt = most;
big.max_tick_debt = big.commits = big.store_failures = big.unknown_outcomes = most;
big.fences = big.lease_remaining_ms = most;
auto bigc = worst_counters();
bigc.m10_enabled = true;
bigc.m10_sessions = bigc.m10_refused_logins = most;
bigc.m10_load = {true, 3, most, most, most, 40, most, most};
CHECK(render_metrics(big, bigc).size() <= max_metrics_bytes);
}
// M10.9a (ADR-0097): refuse, then shed one level per second; relax one level per 5 s of
// calm, then accept again. Samples are 50 ms ticks.
void load_governor() {
CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 15'000, 1'000'000, 5'000'000, 3}));
CHECK(!LoadGovernor::create({25'000, 21'000, 20'000, 10'000, 1'000'000, 5'000'000, 3}));
CHECK(!LoadGovernor::create({19'000, 15'000, 20'000, 10'000, 1'000'000, 5'000'000, 3}));
CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 10'000, 0, 5'000'000, 3}));
CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 10'000, 1'000'000, 5'000'000, 4}));
auto made = LoadGovernor::create({});
CHECK(made.has_value());
auto& g = *made;
std::uint64_t now = 0;
const auto run = [&](std::size_t ticks, std::uint64_t work, std::uint64_t debt = 0) {
for (std::size_t i = 0; i < ticks; ++i) {
now += 50'000;
g.observe(work, work / 4, debt, now);
}
return g.state();
};
// Quiet: nothing changes; the window reports its percentiles.
CHECK(run(60, 5'000) == LoadState{false, 0, 5'000, 5'000, 5'000, 40, 1'250, 1'250});
// A single slow tick is not the p95 of 40.
CHECK(run(1, 30'000).max_us == 30'000 && !g.state().refusing && g.state().p95_us == 5'000);
// Sustained 16 ms: refuse, no shedding.
auto st = run(40, 16'000);
CHECK(st.refusing && st.shed == 0 && st.p95_us == 16'000);
// 22 ms: one shed level per second, never past 3.
// The p95 (nearest rank, 38th of 40) reaches the threshold with the third slow tick.
CHECK(run(2, 22'000).shed == 0 && run(1, 22'000).shed == 1);
CHECK(run(18, 22'000).shed == 1 && run(2, 22'000).shed == 2);
CHECK(run(100, 22'000).shed == 3);
// Calm only once the window's p95 is below 10 ms; then one level per 5 s.
st = run(37, 4'000);
CHECK(st.shed == 3 && st.p95_us == 22'000);
CHECK(run(1, 4'000).p95_us == 4'000); // Calm starts now.
CHECK(run(99, 4'000).shed == 3 && run(1, 4'000).shed == 2);
// A non-calm window (p95 >= 10 ms, still under the refuse threshold) restarts the timer.
CHECK(run(50, 4'000).shed == 2 && run(3, 12'000).p95_us == 12'000);
CHECK(run(37, 4'000).p95_us == 12'000 && run(1, 4'000).p95_us == 4'000);
CHECK(run(99, 4'000).shed == 2 && run(1, 4'000).shed == 1);
// Down to 0 after another 5 s, then logins after 5 s more.
CHECK(run(100, 4'000).shed == 0 && g.state().refusing);
CHECK(run(99, 4'000).refusing && !run(1, 4'000).refusing);
// Tick debt alone: refuse at 1, shed at 2, even with light work.
CHECK(run(1, 1'000, 1).refusing && g.state().shed == 0);
CHECK(run(1, 1'000, 2).shed == 1);
CHECK(!g.observe(1'000, 250, 2, now + 10)); // Within the step interval: no change.
CHECK(g.state().session_p95_us == 1'000); // The window is mostly 4 ms ticks.
// Debt gone and calm (from the next sample): recovers.
CHECK(run(100, 1'000).shed == 1 && run(1, 1'000).shed == 0);
CHECK(run(99, 1'000).refusing && !run(1, 1'000).refusing);
}
} // namespace
int main() {
try {
exposition_is_bounded_and_well_formed();
m10_load_metrics();
load_governor();
std::puts("ops contracts passed");
return 0;
} catch (const std::exception& e) {