Files
aevolution/tests/ops_tests.cpp
T
dingo c625a4cb68
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m24s
Foundation / checks (network) (push) Successful in 20m44s
Foundation / checks (client) (push) Successful in 22m3s
Foundation / checks (tsan) (push) Successful in 23m1s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 24m7s
Foundation / checks (postgres) (push) Canceled after 24m3s
M12.4: retain migration source freezes and explicit aborts
2026-10-18 09:08:50 -04:00

233 lines
10 KiB
C++

#include "evo/server/ops.hpp"
#include <algorithm>
#include <cstdio>
#include <limits>
#include <set>
#include <source_location>
#include <sstream>
#include <stdexcept>
#include <string>
#include <vector>
namespace {
using namespace evo::server;
void check(bool ok, std::source_location at = std::source_location::current()) {
if (!ok)
throw std::runtime_error("ops line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
struct Sample {
std::string name, labels, value;
};
// Parses exposition lines; every family must be introduced by HELP and TYPE.
std::vector<Sample> parse(const std::string& text) {
std::vector<Sample> out;
std::set<std::string> typed, helped;
std::istringstream in(text);
for (std::string line; std::getline(in, line);) {
CHECK(!line.empty());
if (line.rfind("# HELP ", 0) == 0) {
helped.insert(line.substr(7, line.find(' ', 7) - 7));
continue;
}
if (line.rfind("# TYPE ", 0) == 0) {
const auto name = line.substr(7, line.find(' ', 7) - 7);
const auto type = line.substr(line.rfind(' ') + 1);
CHECK(type == "gauge" || type == "counter");
CHECK(helped.contains(name) && !typed.contains(name)); // One family, documented.
typed.insert(name);
continue;
}
const auto space = line.rfind(' ');
const auto brace = line.find('{');
Sample s{line.substr(0, std::min(space, brace)), "", line.substr(space + 1)};
if (brace < space)
s.labels = line.substr(brace, space - brace);
CHECK(typed.contains(s.name));
CHECK(!s.value.empty() && s.value.find_first_not_of("0123456789") == std::string::npos);
out.push_back(s);
}
return out;
}
// Every host counter at its largest value (the M10 section stays disabled).
OpsCounters worst_counters() {
constexpr auto most = std::numeric_limits<std::uint64_t>::max();
OpsCounters c;
c.queue_capacity_inputs = c.queue_capacity_bytes = most;
c.checkpoint_age_ms = c.acks = c.outcomes = most;
c.dropped = {most, most, most, most};
c.objects_available_bytes = c.objects_reserve_bytes = most;
return c;
}
void exposition_is_bounded_and_well_formed() {
HostStatus s;
s.phase = HostPhase::running;
s.ready = true;
s.epoch = 3;
s.simulated_tick = 50;
s.committed_tick = 41;
s.lease_remaining_ms = 1234;
s.ecological_pending_ticks = 64;
s.migration_frozen = true;
OpsCounters c;
c.queue_capacity_inputs = 64;
c.queue_capacity_bytes = 65536;
c.dropped[2] = 5;
const auto text = render_metrics(s, c);
const auto samples = parse(text);
const std::set<std::string> label_names{"", "phase", "reason"};
std::size_t phases_on = 0;
for (const auto& x : samples) {
CHECK(x.name.rfind("evo_region_", 0) == 0);
const auto key =
x.labels.empty() ? std::string{} : x.labels.substr(1, x.labels.find('=') - 1);
CHECK(label_names.contains(key)); // No world/region/entity/request labels.
if (x.name == "evo_region_phase")
phases_on += x.value == "1";
}
CHECK(phases_on == 1);
CHECK(text.find("evo_region_phase{phase=\"running\"} 1\n") != std::string::npos);
CHECK(text.find("evo_region_ready 1\n") != std::string::npos);
CHECK(text.find("evo_region_commit_lag_ticks 9\n") != std::string::npos);
CHECK(text.find("evo_region_ecological_pending_ticks 64\n") != std::string::npos);
CHECK(text.find("evo_region_migration_frozen 1\n") != std::string::npos);
CHECK(text.find("evo_region_lease_remaining_milliseconds 1234\n") != std::string::npos);
CHECK(text.find("evo_region_dropped_total{reason=\"fenced\"} 5\n") != std::string::npos);
// Series count is fixed: 6 phases + 4 reasons + 24 unlabelled families.
CHECK(samples.size() == 6 + 4 + 24);
// Worst-case values keep the size bounded and never underflow the lag.
HostStatus big;
big.phase = HostPhase::quarantined;
constexpr auto most = std::numeric_limits<std::uint64_t>::max();
big.epoch = big.committed_tick = big.committed_sequence = big.tick_debt = most;
big.max_tick_debt = big.commits = big.store_failures = big.unknown_outcomes = most;
big.fences = big.lease_remaining_ms = most;
big.simulated_tick = 1; // Lag saturates at zero rather than wrapping.
const auto bigc = worst_counters();
const auto large = render_metrics(big, bigc);
CHECK(large.size() <= max_metrics_bytes);
CHECK(large.find("evo_region_commit_lag_ticks 0\n") != std::string::npos);
CHECK(large.find("evo_region_ready 0\n") != std::string::npos);
CHECK(parse(large).size() == samples.size());
for (std::size_t i = 0; i < host_phase_count; ++i)
CHECK(std::string(phase_name(static_cast<HostPhase>(i))) != "unknown");
for (std::size_t i = 0; i < unacknowledged_count; ++i)
CHECK(std::string(unacknowledged_name(static_cast<Unacknowledged>(i))) != "unknown");
std::printf("ops_metrics_bytes typical=%zu worst=%zu\n", text.size(), large.size());
}
// M10.9a (ADR-0097): the listener's governor families appear only when it is enabled.
void m10_load_metrics() {
HostStatus s;
OpsCounters c;
const auto without = parse(render_metrics(s, c)).size();
c.m10_enabled = true;
c.m10_sessions = 21;
c.m10_refused_logins = 3;
c.m10_load = {true, 2, 17'000, 19'500, 23'000, 40, 900, 1'100};
const auto text = render_metrics(s, c);
CHECK(parse(text).size() == without + 9);
CHECK(text.find("evo_m10_session_work_p99_microseconds 1100\n") != std::string::npos);
CHECK(text.find("evo_m10_sessions 21\n") != std::string::npos);
CHECK(text.find("evo_m10_refusing 1\n") != std::string::npos);
CHECK(text.find("evo_m10_shed_level 2\n") != std::string::npos);
CHECK(text.find("evo_m10_tick_work_p99_microseconds 19500\n") != std::string::npos);
CHECK(text.find("evo_m10_refused_logins_total 3\n") != std::string::npos);
constexpr auto most = std::numeric_limits<std::uint64_t>::max();
HostStatus big;
big.epoch = big.committed_tick = big.committed_sequence = big.tick_debt = most;
big.max_tick_debt = big.commits = big.store_failures = big.unknown_outcomes = most;
big.fences = big.lease_remaining_ms = most;
auto bigc = worst_counters();
bigc.m10_enabled = true;
bigc.m10_sessions = bigc.m10_refused_logins = most;
bigc.m10_load = {true, 3, most, most, most, 40, most, most};
CHECK(render_metrics(big, bigc).size() <= max_metrics_bytes);
}
// M10.9a (ADR-0097): refuse, then shed one level per second; relax one level per 5 s of
// calm, then accept again. Samples are 50 ms ticks.
void load_governor() {
CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 15'000, 1'000'000, 5'000'000, 3}));
CHECK(!LoadGovernor::create({25'000, 21'000, 20'000, 10'000, 1'000'000, 5'000'000, 3}));
CHECK(!LoadGovernor::create({19'000, 15'000, 20'000, 10'000, 1'000'000, 5'000'000, 3}));
CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 10'000, 0, 5'000'000, 3}));
CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 10'000, 1'000'000, 5'000'000, 4}));
// After a (re)start, work counts only once the window is full (seen on evolution-dev:
// the first slow ticks alone tripped refusal before any session). Tick debt counts at once.
{
auto fresh = *LoadGovernor::create({});
std::uint64_t t = 0;
for (int i = 0; i < 39; ++i)
CHECK(!fresh.observe(30'000, 1'000, 0, t += 50'000) && !fresh.state().refusing);
CHECK(fresh.state().p95_us == 30'000);
CHECK(fresh.observe(30'000, 1'000, 0, t += 50'000) && fresh.state().refusing &&
fresh.state().shed == 1);
auto debt = *LoadGovernor::create({});
CHECK(debt.observe(1'000, 100, 2, 50'000) && debt.state().refusing);
}
auto made = LoadGovernor::create({});
CHECK(made.has_value());
auto& g = *made;
std::uint64_t now = 0;
const auto run = [&](std::size_t ticks, std::uint64_t work, std::uint64_t debt = 0) {
for (std::size_t i = 0; i < ticks; ++i) {
now += 50'000;
g.observe(work, work / 4, debt, now);
}
return g.state();
};
// Quiet: nothing changes; the window reports its percentiles.
CHECK(run(60, 5'000) == LoadState{false, 0, 5'000, 5'000, 5'000, 40, 1'250, 1'250});
// A single slow tick is not the p95 of 40.
CHECK(run(1, 30'000).max_us == 30'000 && !g.state().refusing && g.state().p95_us == 5'000);
// Sustained 16 ms: refuse, no shedding.
auto st = run(40, 16'000);
CHECK(st.refusing && st.shed == 0 && st.p95_us == 16'000);
// 22 ms: one shed level per second, never past 3.
// The p95 (nearest rank, 38th of 40) reaches the threshold with the third slow tick.
CHECK(run(2, 22'000).shed == 0 && run(1, 22'000).shed == 1);
CHECK(run(18, 22'000).shed == 1 && run(2, 22'000).shed == 2);
CHECK(run(100, 22'000).shed == 3);
// Calm only once the window's p95 is below 10 ms; then one level per 5 s.
st = run(37, 4'000);
CHECK(st.shed == 3 && st.p95_us == 22'000);
CHECK(run(1, 4'000).p95_us == 4'000); // Calm starts now.
CHECK(run(99, 4'000).shed == 3 && run(1, 4'000).shed == 2);
// A non-calm window (p95 >= 10 ms, still under the refuse threshold) restarts the timer.
CHECK(run(50, 4'000).shed == 2 && run(3, 12'000).p95_us == 12'000);
CHECK(run(37, 4'000).p95_us == 12'000 && run(1, 4'000).p95_us == 4'000);
CHECK(run(99, 4'000).shed == 2 && run(1, 4'000).shed == 1);
// Down to 0 after another 5 s, then logins after 5 s more.
CHECK(run(100, 4'000).shed == 0 && g.state().refusing);
CHECK(run(99, 4'000).refusing && !run(1, 4'000).refusing);
// Tick debt alone, even with light work: one tick is jitter, 2 refuse, 4 shed.
CHECK(!run(1, 1'000, 1).refusing);
CHECK(run(1, 1'000, 2).refusing && g.state().shed == 0);
CHECK(run(1, 1'000, 4).shed == 1);
CHECK(!g.observe(1'000, 250, 4, now + 10)); // Within the step interval: no change.
CHECK(g.state().session_p95_us == 1'000); // The window is mostly 4 ms ticks.
// Debt gone and calm (from the next sample): recovers.
CHECK(run(100, 1'000).shed == 1 && run(1, 1'000).shed == 0);
CHECK(run(99, 1'000).refusing && !run(1, 1'000).refusing);
}
} // namespace
int main() {
try {
exposition_is_bounded_and_well_formed();
m10_load_metrics();
load_governor();
std::puts("ops contracts passed");
return 0;
} catch (const std::exception& e) {
std::fprintf(stderr, "%s\n", e.what());
return 1;
}
}