Add M10.9a load measurement, governor and overload shedding
Foundation / checks (deploy) (push) Successful in 3m20s
Foundation / checks (ecs) (push) Successful in 1m59s
Foundation / checks (client) (push) Failing after 10m0s
Foundation / checks (network) (push) Successful in 13m9s
Foundation / checks (tsan) (push) Successful in 8m1s
Foundation / checks (checks) (push) Successful in 32m2s
Foundation / checks (postgres) (push) Failing after 26m39s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
Foundation / checks (deploy) (push) Successful in 3m20s
Foundation / checks (ecs) (push) Successful in 1m59s
Foundation / checks (client) (push) Failing after 10m0s
Foundation / checks (network) (push) Successful in 13m9s
Foundation / checks (tsan) (push) Successful in 8m1s
Foundation / checks (checks) (push) Successful in 32m2s
Foundation / checks (postgres) (push) Failing after 26m39s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
ADR-0097 (M10.9 load and abuse qualification; only evolution-dev reference-worker runs qualify). evo-server measures each simulated tick's work (host step plus the M10 bridge, with the session layer's share kept separately) and feeds a portable LoadGovernor: grant logins are refused at a 2 s p95 of 15 ms or any tick debt, replication is shed in up to three levels at 20 ms or debt >= 2 (radius cap 7/7/5/3, far updates x1/x2/x2/x4, no fresh baselines, own markers unaffected), and both relax after 5 s of calm. Handoff tickets are still admitted. /metrics gains evo_m10_* gauges when the listener is enabled. Adds the missing ADR index entries for ADR-0094 to ADR-0096. Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+96
-1
@@ -52,6 +52,17 @@ std::vector<Sample> parse(const std::string& text) {
|
||||
return out;
|
||||
}
|
||||
|
||||
// Every host counter at its largest value (the M10 section stays disabled).
|
||||
OpsCounters worst_counters() {
|
||||
constexpr auto most = std::numeric_limits<std::uint64_t>::max();
|
||||
OpsCounters c;
|
||||
c.queue_capacity_inputs = c.queue_capacity_bytes = most;
|
||||
c.checkpoint_age_ms = c.acks = c.outcomes = most;
|
||||
c.dropped = {most, most, most, most};
|
||||
c.objects_available_bytes = c.objects_reserve_bytes = most;
|
||||
return c;
|
||||
}
|
||||
|
||||
void exposition_is_bounded_and_well_formed() {
|
||||
HostStatus s;
|
||||
s.phase = HostPhase::running;
|
||||
@@ -93,7 +104,7 @@ void exposition_is_bounded_and_well_formed() {
|
||||
big.max_tick_debt = big.commits = big.store_failures = big.unknown_outcomes = most;
|
||||
big.fences = big.lease_remaining_ms = most;
|
||||
big.simulated_tick = 1; // Lag saturates at zero rather than wrapping.
|
||||
OpsCounters bigc{most, most, most, most, most, {most, most, most, most}, most, most};
|
||||
const auto bigc = worst_counters();
|
||||
const auto large = render_metrics(big, bigc);
|
||||
CHECK(large.size() <= max_metrics_bytes);
|
||||
CHECK(large.find("evo_region_commit_lag_ticks 0\n") != std::string::npos);
|
||||
@@ -105,11 +116,95 @@ void exposition_is_bounded_and_well_formed() {
|
||||
CHECK(std::string(unacknowledged_name(static_cast<Unacknowledged>(i))) != "unknown");
|
||||
std::printf("ops_metrics_bytes typical=%zu worst=%zu\n", text.size(), large.size());
|
||||
}
|
||||
|
||||
// M10.9a (ADR-0097): the listener's governor families appear only when it is enabled.
|
||||
void m10_load_metrics() {
|
||||
HostStatus s;
|
||||
OpsCounters c;
|
||||
const auto without = parse(render_metrics(s, c)).size();
|
||||
c.m10_enabled = true;
|
||||
c.m10_sessions = 21;
|
||||
c.m10_refused_logins = 3;
|
||||
c.m10_load = {true, 2, 17'000, 19'500, 23'000, 40, 900, 1'100};
|
||||
const auto text = render_metrics(s, c);
|
||||
CHECK(parse(text).size() == without + 9);
|
||||
CHECK(text.find("evo_m10_session_work_p99_microseconds 1100\n") != std::string::npos);
|
||||
CHECK(text.find("evo_m10_sessions 21\n") != std::string::npos);
|
||||
CHECK(text.find("evo_m10_refusing 1\n") != std::string::npos);
|
||||
CHECK(text.find("evo_m10_shed_level 2\n") != std::string::npos);
|
||||
CHECK(text.find("evo_m10_tick_work_p99_microseconds 19500\n") != std::string::npos);
|
||||
CHECK(text.find("evo_m10_refused_logins_total 3\n") != std::string::npos);
|
||||
constexpr auto most = std::numeric_limits<std::uint64_t>::max();
|
||||
HostStatus big;
|
||||
big.epoch = big.committed_tick = big.committed_sequence = big.tick_debt = most;
|
||||
big.max_tick_debt = big.commits = big.store_failures = big.unknown_outcomes = most;
|
||||
big.fences = big.lease_remaining_ms = most;
|
||||
auto bigc = worst_counters();
|
||||
bigc.m10_enabled = true;
|
||||
bigc.m10_sessions = bigc.m10_refused_logins = most;
|
||||
bigc.m10_load = {true, 3, most, most, most, 40, most, most};
|
||||
CHECK(render_metrics(big, bigc).size() <= max_metrics_bytes);
|
||||
}
|
||||
|
||||
// M10.9a (ADR-0097): refuse, then shed one level per second; relax one level per 5 s of
|
||||
// calm, then accept again. Samples are 50 ms ticks.
|
||||
void load_governor() {
|
||||
CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 15'000, 1'000'000, 5'000'000, 3}));
|
||||
CHECK(!LoadGovernor::create({25'000, 21'000, 20'000, 10'000, 1'000'000, 5'000'000, 3}));
|
||||
CHECK(!LoadGovernor::create({19'000, 15'000, 20'000, 10'000, 1'000'000, 5'000'000, 3}));
|
||||
CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 10'000, 0, 5'000'000, 3}));
|
||||
CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 10'000, 1'000'000, 5'000'000, 4}));
|
||||
auto made = LoadGovernor::create({});
|
||||
CHECK(made.has_value());
|
||||
auto& g = *made;
|
||||
std::uint64_t now = 0;
|
||||
const auto run = [&](std::size_t ticks, std::uint64_t work, std::uint64_t debt = 0) {
|
||||
for (std::size_t i = 0; i < ticks; ++i) {
|
||||
now += 50'000;
|
||||
g.observe(work, work / 4, debt, now);
|
||||
}
|
||||
return g.state();
|
||||
};
|
||||
// Quiet: nothing changes; the window reports its percentiles.
|
||||
CHECK(run(60, 5'000) == LoadState{false, 0, 5'000, 5'000, 5'000, 40, 1'250, 1'250});
|
||||
// A single slow tick is not the p95 of 40.
|
||||
CHECK(run(1, 30'000).max_us == 30'000 && !g.state().refusing && g.state().p95_us == 5'000);
|
||||
// Sustained 16 ms: refuse, no shedding.
|
||||
auto st = run(40, 16'000);
|
||||
CHECK(st.refusing && st.shed == 0 && st.p95_us == 16'000);
|
||||
// 22 ms: one shed level per second, never past 3.
|
||||
// The p95 (nearest rank, 38th of 40) reaches the threshold with the third slow tick.
|
||||
CHECK(run(2, 22'000).shed == 0 && run(1, 22'000).shed == 1);
|
||||
CHECK(run(18, 22'000).shed == 1 && run(2, 22'000).shed == 2);
|
||||
CHECK(run(100, 22'000).shed == 3);
|
||||
// Calm only once the window's p95 is below 10 ms; then one level per 5 s.
|
||||
st = run(37, 4'000);
|
||||
CHECK(st.shed == 3 && st.p95_us == 22'000);
|
||||
CHECK(run(1, 4'000).p95_us == 4'000); // Calm starts now.
|
||||
CHECK(run(99, 4'000).shed == 3 && run(1, 4'000).shed == 2);
|
||||
// A non-calm window (p95 >= 10 ms, still under the refuse threshold) restarts the timer.
|
||||
CHECK(run(50, 4'000).shed == 2 && run(3, 12'000).p95_us == 12'000);
|
||||
CHECK(run(37, 4'000).p95_us == 12'000 && run(1, 4'000).p95_us == 4'000);
|
||||
CHECK(run(99, 4'000).shed == 2 && run(1, 4'000).shed == 1);
|
||||
// Down to 0 after another 5 s, then logins after 5 s more.
|
||||
CHECK(run(100, 4'000).shed == 0 && g.state().refusing);
|
||||
CHECK(run(99, 4'000).refusing && !run(1, 4'000).refusing);
|
||||
// Tick debt alone: refuse at 1, shed at 2, even with light work.
|
||||
CHECK(run(1, 1'000, 1).refusing && g.state().shed == 0);
|
||||
CHECK(run(1, 1'000, 2).shed == 1);
|
||||
CHECK(!g.observe(1'000, 250, 2, now + 10)); // Within the step interval: no change.
|
||||
CHECK(g.state().session_p95_us == 1'000); // The window is mostly 4 ms ticks.
|
||||
// Debt gone and calm (from the next sample): recovers.
|
||||
CHECK(run(100, 1'000).shed == 1 && run(1, 1'000).shed == 0);
|
||||
CHECK(run(99, 1'000).refusing && !run(1, 1'000).refusing);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
try {
|
||||
exposition_is_bounded_and_well_formed();
|
||||
m10_load_metrics();
|
||||
load_governor();
|
||||
std::puts("ops contracts passed");
|
||||
return 0;
|
||||
} catch (const std::exception& e) {
|
||||
|
||||
Reference in New Issue
Block a user