#include "evo/server/ops.hpp" #include #include #include #include #include #include #include #include #include namespace { using namespace evo::server; void check(bool ok, std::source_location at = std::source_location::current()) { if (!ok) throw std::runtime_error("ops line " + std::to_string(at.line())); } #define CHECK(...) check(static_cast(__VA_ARGS__)) struct Sample { std::string name, labels, value; }; // Parses exposition lines; every family must be introduced by HELP and TYPE. std::vector parse(const std::string& text) { std::vector out; std::set typed, helped; std::istringstream in(text); for (std::string line; std::getline(in, line);) { CHECK(!line.empty()); if (line.rfind("# HELP ", 0) == 0) { helped.insert(line.substr(7, line.find(' ', 7) - 7)); continue; } if (line.rfind("# TYPE ", 0) == 0) { const auto name = line.substr(7, line.find(' ', 7) - 7); const auto type = line.substr(line.rfind(' ') + 1); CHECK(type == "gauge" || type == "counter"); CHECK(helped.contains(name) && !typed.contains(name)); // One family, documented. typed.insert(name); continue; } const auto space = line.rfind(' '); const auto brace = line.find('{'); Sample s{line.substr(0, std::min(space, brace)), "", line.substr(space + 1)}; if (brace < space) s.labels = line.substr(brace, space - brace); CHECK(typed.contains(s.name)); CHECK(!s.value.empty() && s.value.find_first_not_of("0123456789") == std::string::npos); out.push_back(s); } return out; } // Every host counter at its largest value (the M10 section stays disabled). OpsCounters worst_counters() { constexpr auto most = std::numeric_limits::max(); OpsCounters c; c.queue_capacity_inputs = c.queue_capacity_bytes = most; c.checkpoint_age_ms = c.acks = c.outcomes = most; c.dropped = {most, most, most, most}; c.objects_available_bytes = c.objects_reserve_bytes = most; return c; } void exposition_is_bounded_and_well_formed() { HostStatus s; s.phase = HostPhase::running; s.ready = true; s.epoch = 3; s.simulated_tick = 50; s.committed_tick = 41; s.lease_remaining_ms = 1234; s.ecological_pending_ticks = 64; OpsCounters c; c.queue_capacity_inputs = 64; c.queue_capacity_bytes = 65536; c.dropped[2] = 5; const auto text = render_metrics(s, c); const auto samples = parse(text); const std::set label_names{"", "phase", "reason"}; std::size_t phases_on = 0; for (const auto& x : samples) { CHECK(x.name.rfind("evo_region_", 0) == 0); const auto key = x.labels.empty() ? std::string{} : x.labels.substr(1, x.labels.find('=') - 1); CHECK(label_names.contains(key)); // No world/region/entity/request labels. if (x.name == "evo_region_phase") phases_on += x.value == "1"; } CHECK(phases_on == 1); CHECK(text.find("evo_region_phase{phase=\"running\"} 1\n") != std::string::npos); CHECK(text.find("evo_region_ready 1\n") != std::string::npos); CHECK(text.find("evo_region_commit_lag_ticks 9\n") != std::string::npos); CHECK(text.find("evo_region_ecological_pending_ticks 64\n") != std::string::npos); CHECK(text.find("evo_region_lease_remaining_milliseconds 1234\n") != std::string::npos); CHECK(text.find("evo_region_dropped_total{reason=\"fenced\"} 5\n") != std::string::npos); // Series count is fixed: 6 phases + 4 reasons + 23 unlabelled families. CHECK(samples.size() == 6 + 4 + 23); // Worst-case values keep the size bounded and never underflow the lag. HostStatus big; big.phase = HostPhase::quarantined; constexpr auto most = std::numeric_limits::max(); big.epoch = big.committed_tick = big.committed_sequence = big.tick_debt = most; big.max_tick_debt = big.commits = big.store_failures = big.unknown_outcomes = most; big.fences = big.lease_remaining_ms = most; big.simulated_tick = 1; // Lag saturates at zero rather than wrapping. const auto bigc = worst_counters(); const auto large = render_metrics(big, bigc); CHECK(large.size() <= max_metrics_bytes); CHECK(large.find("evo_region_commit_lag_ticks 0\n") != std::string::npos); CHECK(large.find("evo_region_ready 0\n") != std::string::npos); CHECK(parse(large).size() == samples.size()); for (std::size_t i = 0; i < host_phase_count; ++i) CHECK(std::string(phase_name(static_cast(i))) != "unknown"); for (std::size_t i = 0; i < unacknowledged_count; ++i) CHECK(std::string(unacknowledged_name(static_cast(i))) != "unknown"); std::printf("ops_metrics_bytes typical=%zu worst=%zu\n", text.size(), large.size()); } // M10.9a (ADR-0097): the listener's governor families appear only when it is enabled. void m10_load_metrics() { HostStatus s; OpsCounters c; const auto without = parse(render_metrics(s, c)).size(); c.m10_enabled = true; c.m10_sessions = 21; c.m10_refused_logins = 3; c.m10_load = {true, 2, 17'000, 19'500, 23'000, 40, 900, 1'100}; const auto text = render_metrics(s, c); CHECK(parse(text).size() == without + 9); CHECK(text.find("evo_m10_session_work_p99_microseconds 1100\n") != std::string::npos); CHECK(text.find("evo_m10_sessions 21\n") != std::string::npos); CHECK(text.find("evo_m10_refusing 1\n") != std::string::npos); CHECK(text.find("evo_m10_shed_level 2\n") != std::string::npos); CHECK(text.find("evo_m10_tick_work_p99_microseconds 19500\n") != std::string::npos); CHECK(text.find("evo_m10_refused_logins_total 3\n") != std::string::npos); constexpr auto most = std::numeric_limits::max(); HostStatus big; big.epoch = big.committed_tick = big.committed_sequence = big.tick_debt = most; big.max_tick_debt = big.commits = big.store_failures = big.unknown_outcomes = most; big.fences = big.lease_remaining_ms = most; auto bigc = worst_counters(); bigc.m10_enabled = true; bigc.m10_sessions = bigc.m10_refused_logins = most; bigc.m10_load = {true, 3, most, most, most, 40, most, most}; CHECK(render_metrics(big, bigc).size() <= max_metrics_bytes); } // M10.9a (ADR-0097): refuse, then shed one level per second; relax one level per 5 s of // calm, then accept again. Samples are 50 ms ticks. void load_governor() { CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 15'000, 1'000'000, 5'000'000, 3})); CHECK(!LoadGovernor::create({25'000, 21'000, 20'000, 10'000, 1'000'000, 5'000'000, 3})); CHECK(!LoadGovernor::create({19'000, 15'000, 20'000, 10'000, 1'000'000, 5'000'000, 3})); CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 10'000, 0, 5'000'000, 3})); CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 10'000, 1'000'000, 5'000'000, 4})); // After a (re)start, work counts only once the window is full (seen on evolution-dev: // the first slow ticks alone tripped refusal before any session). Tick debt counts at once. { auto fresh = *LoadGovernor::create({}); std::uint64_t t = 0; for (int i = 0; i < 39; ++i) CHECK(!fresh.observe(30'000, 1'000, 0, t += 50'000) && !fresh.state().refusing); CHECK(fresh.state().p95_us == 30'000); CHECK(fresh.observe(30'000, 1'000, 0, t += 50'000) && fresh.state().refusing && fresh.state().shed == 1); auto debt = *LoadGovernor::create({}); CHECK(debt.observe(1'000, 100, 2, 50'000) && debt.state().refusing); } auto made = LoadGovernor::create({}); CHECK(made.has_value()); auto& g = *made; std::uint64_t now = 0; const auto run = [&](std::size_t ticks, std::uint64_t work, std::uint64_t debt = 0) { for (std::size_t i = 0; i < ticks; ++i) { now += 50'000; g.observe(work, work / 4, debt, now); } return g.state(); }; // Quiet: nothing changes; the window reports its percentiles. CHECK(run(60, 5'000) == LoadState{false, 0, 5'000, 5'000, 5'000, 40, 1'250, 1'250}); // A single slow tick is not the p95 of 40. CHECK(run(1, 30'000).max_us == 30'000 && !g.state().refusing && g.state().p95_us == 5'000); // Sustained 16 ms: refuse, no shedding. auto st = run(40, 16'000); CHECK(st.refusing && st.shed == 0 && st.p95_us == 16'000); // 22 ms: one shed level per second, never past 3. // The p95 (nearest rank, 38th of 40) reaches the threshold with the third slow tick. CHECK(run(2, 22'000).shed == 0 && run(1, 22'000).shed == 1); CHECK(run(18, 22'000).shed == 1 && run(2, 22'000).shed == 2); CHECK(run(100, 22'000).shed == 3); // Calm only once the window's p95 is below 10 ms; then one level per 5 s. st = run(37, 4'000); CHECK(st.shed == 3 && st.p95_us == 22'000); CHECK(run(1, 4'000).p95_us == 4'000); // Calm starts now. CHECK(run(99, 4'000).shed == 3 && run(1, 4'000).shed == 2); // A non-calm window (p95 >= 10 ms, still under the refuse threshold) restarts the timer. CHECK(run(50, 4'000).shed == 2 && run(3, 12'000).p95_us == 12'000); CHECK(run(37, 4'000).p95_us == 12'000 && run(1, 4'000).p95_us == 4'000); CHECK(run(99, 4'000).shed == 2 && run(1, 4'000).shed == 1); // Down to 0 after another 5 s, then logins after 5 s more. CHECK(run(100, 4'000).shed == 0 && g.state().refusing); CHECK(run(99, 4'000).refusing && !run(1, 4'000).refusing); // Tick debt alone, even with light work: one tick is jitter, 2 refuse, 4 shed. CHECK(!run(1, 1'000, 1).refusing); CHECK(run(1, 1'000, 2).refusing && g.state().shed == 0); CHECK(run(1, 1'000, 4).shed == 1); CHECK(!g.observe(1'000, 250, 4, now + 10)); // Within the step interval: no change. CHECK(g.state().session_p95_us == 1'000); // The window is mostly 4 ms ticks. // Debt gone and calm (from the next sample): recovers. CHECK(run(100, 1'000).shed == 1 && run(1, 1'000).shed == 0); CHECK(run(99, 1'000).refusing && !run(1, 1'000).refusing); } } // namespace int main() { try { exposition_is_bounded_and_well_formed(); m10_load_metrics(); load_governor(); std::puts("ops contracts passed"); return 0; } catch (const std::exception& e) { std::fprintf(stderr, "%s\n", e.what()); return 1; } }