Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m28s
Foundation / checks (tsan) (push) Failing after 18m22s
Foundation / checks (network) (push) Successful in 20m38s
Foundation / checks (client) (push) Failing after 21m6s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (postgres) (push) Canceled after 46m54s
Foundation / checks (checks) (push) Canceled after 47m3s
231 lines
10 KiB
C++
231 lines
10 KiB
C++
#include "evo/server/ops.hpp"
|
|
|
|
#include <algorithm>
|
|
#include <cstdio>
|
|
#include <limits>
|
|
#include <set>
|
|
#include <source_location>
|
|
#include <sstream>
|
|
#include <stdexcept>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
namespace {
|
|
using namespace evo::server;
|
|
void check(bool ok, std::source_location at = std::source_location::current()) {
|
|
if (!ok)
|
|
throw std::runtime_error("ops line " + std::to_string(at.line()));
|
|
}
|
|
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
|
|
|
struct Sample {
|
|
std::string name, labels, value;
|
|
};
|
|
// Parses exposition lines; every family must be introduced by HELP and TYPE.
|
|
std::vector<Sample> parse(const std::string& text) {
|
|
std::vector<Sample> out;
|
|
std::set<std::string> typed, helped;
|
|
std::istringstream in(text);
|
|
for (std::string line; std::getline(in, line);) {
|
|
CHECK(!line.empty());
|
|
if (line.rfind("# HELP ", 0) == 0) {
|
|
helped.insert(line.substr(7, line.find(' ', 7) - 7));
|
|
continue;
|
|
}
|
|
if (line.rfind("# TYPE ", 0) == 0) {
|
|
const auto name = line.substr(7, line.find(' ', 7) - 7);
|
|
const auto type = line.substr(line.rfind(' ') + 1);
|
|
CHECK(type == "gauge" || type == "counter");
|
|
CHECK(helped.contains(name) && !typed.contains(name)); // One family, documented.
|
|
typed.insert(name);
|
|
continue;
|
|
}
|
|
const auto space = line.rfind(' ');
|
|
const auto brace = line.find('{');
|
|
Sample s{line.substr(0, std::min(space, brace)), "", line.substr(space + 1)};
|
|
if (brace < space)
|
|
s.labels = line.substr(brace, space - brace);
|
|
CHECK(typed.contains(s.name));
|
|
CHECK(!s.value.empty() && s.value.find_first_not_of("0123456789") == std::string::npos);
|
|
out.push_back(s);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// Every host counter at its largest value (the M10 section stays disabled).
|
|
OpsCounters worst_counters() {
|
|
constexpr auto most = std::numeric_limits<std::uint64_t>::max();
|
|
OpsCounters c;
|
|
c.queue_capacity_inputs = c.queue_capacity_bytes = most;
|
|
c.checkpoint_age_ms = c.acks = c.outcomes = most;
|
|
c.dropped = {most, most, most, most};
|
|
c.objects_available_bytes = c.objects_reserve_bytes = most;
|
|
return c;
|
|
}
|
|
|
|
void exposition_is_bounded_and_well_formed() {
|
|
HostStatus s;
|
|
s.phase = HostPhase::running;
|
|
s.ready = true;
|
|
s.epoch = 3;
|
|
s.simulated_tick = 50;
|
|
s.committed_tick = 41;
|
|
s.lease_remaining_ms = 1234;
|
|
s.ecological_pending_ticks = 64;
|
|
OpsCounters c;
|
|
c.queue_capacity_inputs = 64;
|
|
c.queue_capacity_bytes = 65536;
|
|
c.dropped[2] = 5;
|
|
const auto text = render_metrics(s, c);
|
|
const auto samples = parse(text);
|
|
const std::set<std::string> label_names{"", "phase", "reason"};
|
|
std::size_t phases_on = 0;
|
|
for (const auto& x : samples) {
|
|
CHECK(x.name.rfind("evo_region_", 0) == 0);
|
|
const auto key =
|
|
x.labels.empty() ? std::string{} : x.labels.substr(1, x.labels.find('=') - 1);
|
|
CHECK(label_names.contains(key)); // No world/region/entity/request labels.
|
|
if (x.name == "evo_region_phase")
|
|
phases_on += x.value == "1";
|
|
}
|
|
CHECK(phases_on == 1);
|
|
CHECK(text.find("evo_region_phase{phase=\"running\"} 1\n") != std::string::npos);
|
|
CHECK(text.find("evo_region_ready 1\n") != std::string::npos);
|
|
CHECK(text.find("evo_region_commit_lag_ticks 9\n") != std::string::npos);
|
|
CHECK(text.find("evo_region_ecological_pending_ticks 64\n") != std::string::npos);
|
|
CHECK(text.find("evo_region_lease_remaining_milliseconds 1234\n") != std::string::npos);
|
|
CHECK(text.find("evo_region_dropped_total{reason=\"fenced\"} 5\n") != std::string::npos);
|
|
// Series count is fixed: 6 phases + 4 reasons + 23 unlabelled families.
|
|
CHECK(samples.size() == 6 + 4 + 23);
|
|
|
|
// Worst-case values keep the size bounded and never underflow the lag.
|
|
HostStatus big;
|
|
big.phase = HostPhase::quarantined;
|
|
constexpr auto most = std::numeric_limits<std::uint64_t>::max();
|
|
big.epoch = big.committed_tick = big.committed_sequence = big.tick_debt = most;
|
|
big.max_tick_debt = big.commits = big.store_failures = big.unknown_outcomes = most;
|
|
big.fences = big.lease_remaining_ms = most;
|
|
big.simulated_tick = 1; // Lag saturates at zero rather than wrapping.
|
|
const auto bigc = worst_counters();
|
|
const auto large = render_metrics(big, bigc);
|
|
CHECK(large.size() <= max_metrics_bytes);
|
|
CHECK(large.find("evo_region_commit_lag_ticks 0\n") != std::string::npos);
|
|
CHECK(large.find("evo_region_ready 0\n") != std::string::npos);
|
|
CHECK(parse(large).size() == samples.size());
|
|
for (std::size_t i = 0; i < host_phase_count; ++i)
|
|
CHECK(std::string(phase_name(static_cast<HostPhase>(i))) != "unknown");
|
|
for (std::size_t i = 0; i < unacknowledged_count; ++i)
|
|
CHECK(std::string(unacknowledged_name(static_cast<Unacknowledged>(i))) != "unknown");
|
|
std::printf("ops_metrics_bytes typical=%zu worst=%zu\n", text.size(), large.size());
|
|
}
|
|
|
|
// M10.9a (ADR-0097): the listener's governor families appear only when it is enabled.
|
|
void m10_load_metrics() {
|
|
HostStatus s;
|
|
OpsCounters c;
|
|
const auto without = parse(render_metrics(s, c)).size();
|
|
c.m10_enabled = true;
|
|
c.m10_sessions = 21;
|
|
c.m10_refused_logins = 3;
|
|
c.m10_load = {true, 2, 17'000, 19'500, 23'000, 40, 900, 1'100};
|
|
const auto text = render_metrics(s, c);
|
|
CHECK(parse(text).size() == without + 9);
|
|
CHECK(text.find("evo_m10_session_work_p99_microseconds 1100\n") != std::string::npos);
|
|
CHECK(text.find("evo_m10_sessions 21\n") != std::string::npos);
|
|
CHECK(text.find("evo_m10_refusing 1\n") != std::string::npos);
|
|
CHECK(text.find("evo_m10_shed_level 2\n") != std::string::npos);
|
|
CHECK(text.find("evo_m10_tick_work_p99_microseconds 19500\n") != std::string::npos);
|
|
CHECK(text.find("evo_m10_refused_logins_total 3\n") != std::string::npos);
|
|
constexpr auto most = std::numeric_limits<std::uint64_t>::max();
|
|
HostStatus big;
|
|
big.epoch = big.committed_tick = big.committed_sequence = big.tick_debt = most;
|
|
big.max_tick_debt = big.commits = big.store_failures = big.unknown_outcomes = most;
|
|
big.fences = big.lease_remaining_ms = most;
|
|
auto bigc = worst_counters();
|
|
bigc.m10_enabled = true;
|
|
bigc.m10_sessions = bigc.m10_refused_logins = most;
|
|
bigc.m10_load = {true, 3, most, most, most, 40, most, most};
|
|
CHECK(render_metrics(big, bigc).size() <= max_metrics_bytes);
|
|
}
|
|
|
|
// M10.9a (ADR-0097): refuse, then shed one level per second; relax one level per 5 s of
|
|
// calm, then accept again. Samples are 50 ms ticks.
|
|
void load_governor() {
|
|
CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 15'000, 1'000'000, 5'000'000, 3}));
|
|
CHECK(!LoadGovernor::create({25'000, 21'000, 20'000, 10'000, 1'000'000, 5'000'000, 3}));
|
|
CHECK(!LoadGovernor::create({19'000, 15'000, 20'000, 10'000, 1'000'000, 5'000'000, 3}));
|
|
CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 10'000, 0, 5'000'000, 3}));
|
|
CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 10'000, 1'000'000, 5'000'000, 4}));
|
|
// After a (re)start, work counts only once the window is full (seen on evolution-dev:
|
|
// the first slow ticks alone tripped refusal before any session). Tick debt counts at once.
|
|
{
|
|
auto fresh = *LoadGovernor::create({});
|
|
std::uint64_t t = 0;
|
|
for (int i = 0; i < 39; ++i)
|
|
CHECK(!fresh.observe(30'000, 1'000, 0, t += 50'000) && !fresh.state().refusing);
|
|
CHECK(fresh.state().p95_us == 30'000);
|
|
CHECK(fresh.observe(30'000, 1'000, 0, t += 50'000) && fresh.state().refusing &&
|
|
fresh.state().shed == 1);
|
|
auto debt = *LoadGovernor::create({});
|
|
CHECK(debt.observe(1'000, 100, 2, 50'000) && debt.state().refusing);
|
|
}
|
|
auto made = LoadGovernor::create({});
|
|
CHECK(made.has_value());
|
|
auto& g = *made;
|
|
std::uint64_t now = 0;
|
|
const auto run = [&](std::size_t ticks, std::uint64_t work, std::uint64_t debt = 0) {
|
|
for (std::size_t i = 0; i < ticks; ++i) {
|
|
now += 50'000;
|
|
g.observe(work, work / 4, debt, now);
|
|
}
|
|
return g.state();
|
|
};
|
|
// Quiet: nothing changes; the window reports its percentiles.
|
|
CHECK(run(60, 5'000) == LoadState{false, 0, 5'000, 5'000, 5'000, 40, 1'250, 1'250});
|
|
// A single slow tick is not the p95 of 40.
|
|
CHECK(run(1, 30'000).max_us == 30'000 && !g.state().refusing && g.state().p95_us == 5'000);
|
|
// Sustained 16 ms: refuse, no shedding.
|
|
auto st = run(40, 16'000);
|
|
CHECK(st.refusing && st.shed == 0 && st.p95_us == 16'000);
|
|
// 22 ms: one shed level per second, never past 3.
|
|
// The p95 (nearest rank, 38th of 40) reaches the threshold with the third slow tick.
|
|
CHECK(run(2, 22'000).shed == 0 && run(1, 22'000).shed == 1);
|
|
CHECK(run(18, 22'000).shed == 1 && run(2, 22'000).shed == 2);
|
|
CHECK(run(100, 22'000).shed == 3);
|
|
// Calm only once the window's p95 is below 10 ms; then one level per 5 s.
|
|
st = run(37, 4'000);
|
|
CHECK(st.shed == 3 && st.p95_us == 22'000);
|
|
CHECK(run(1, 4'000).p95_us == 4'000); // Calm starts now.
|
|
CHECK(run(99, 4'000).shed == 3 && run(1, 4'000).shed == 2);
|
|
// A non-calm window (p95 >= 10 ms, still under the refuse threshold) restarts the timer.
|
|
CHECK(run(50, 4'000).shed == 2 && run(3, 12'000).p95_us == 12'000);
|
|
CHECK(run(37, 4'000).p95_us == 12'000 && run(1, 4'000).p95_us == 4'000);
|
|
CHECK(run(99, 4'000).shed == 2 && run(1, 4'000).shed == 1);
|
|
// Down to 0 after another 5 s, then logins after 5 s more.
|
|
CHECK(run(100, 4'000).shed == 0 && g.state().refusing);
|
|
CHECK(run(99, 4'000).refusing && !run(1, 4'000).refusing);
|
|
// Tick debt alone, even with light work: one tick is jitter, 2 refuse, 4 shed.
|
|
CHECK(!run(1, 1'000, 1).refusing);
|
|
CHECK(run(1, 1'000, 2).refusing && g.state().shed == 0);
|
|
CHECK(run(1, 1'000, 4).shed == 1);
|
|
CHECK(!g.observe(1'000, 250, 4, now + 10)); // Within the step interval: no change.
|
|
CHECK(g.state().session_p95_us == 1'000); // The window is mostly 4 ms ticks.
|
|
// Debt gone and calm (from the next sample): recovers.
|
|
CHECK(run(100, 1'000).shed == 1 && run(1, 1'000).shed == 0);
|
|
CHECK(run(99, 1'000).refusing && !run(1, 1'000).refusing);
|
|
}
|
|
} // namespace
|
|
|
|
int main() {
|
|
try {
|
|
exposition_is_bounded_and_well_formed();
|
|
m10_load_metrics();
|
|
load_governor();
|
|
std::puts("ops contracts passed");
|
|
return 0;
|
|
} catch (const std::exception& e) {
|
|
std::fprintf(stderr, "%s\n", e.what());
|
|
return 1;
|
|
}
|
|
}
|