Fix CI: run M10 process tests with the load governor off
Foundation / checks (deploy) (push) Successful in 3m15s
Foundation / checks (ecs) (push) Successful in 1m57s
Foundation / checks (client) (push) Successful in 12m33s
Foundation / checks (network) (push) Successful in 13m19s
Foundation / checks (tsan) (push) Successful in 8m5s
Foundation / checks (checks) (push) Successful in 32m42s
Foundation / checks (postgres) (push) Successful in 43m45s
Foundation / postgres-tools (push) Successful in 20s
Foundation / server-image (push) Successful in 24s
Foundation / image (push) Successful in 2m31s

checks (postgres) failed on both M10.9 pushes: on the shared CI runner one
session measured 22 ms p95 per tick, so the governor correctly refused
and the test's assumption about host speed failed (server-image was
skipped as a result). evo-server gains --m10-load-governor on|off (dev
only): off keeps measuring and exposing the work but never refuses or
sheds. The M10 session, load-smoke and two-process handoff tests use it;
the governor stays covered by its unit tests. Refreshes the dev session
window.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-07 11:58:37 -04:00
co-authored by Claude Opus 5.5
parent 1ba6748288
commit 66efe87c90
6 changed files with 77 additions and 26 deletions
+15 -2
View File
@@ -55,6 +55,9 @@ struct M10Options {
std::string listen, certificate, key, issuer, environment, session_window;
// M10.5c: dev only, 60..3600 s; 0 keeps the default (300 s, ADR-0091).
std::uint64_t max_grant_lifetime{};
// M10.9 (ADR-0097, dev only): `--m10-load-governor off` keeps measuring and exposing the
// work but never refuses or sheds (process tests whose host speed is unknown).
bool load_governor{true};
// M10.8c (ADR-0096, dev only): this region's marker area in cells "X0,Y0,X1,Y1"
// (inclusive), its handoff key (32-byte P-256 scalar; key id = region id) and up to 8
// neighbours "REGION=X0,Y0,X1,Y1=HOST:PORT=SERVER_NAME=PUBLIC_KEY_FILE".
@@ -63,7 +66,7 @@ struct M10Options {
[[nodiscard]] bool any() const noexcept {
return !listen.empty() || !certificate.empty() || !key.empty() || !issuer.empty() ||
!environment.empty() || !session_window.empty() || max_grant_lifetime != 0 ||
!area.empty() || !handoff_key.empty() || !neighbours.empty();
!area.empty() || !handoff_key.empty() || !neighbours.empty() || !load_governor;
}
[[nodiscard]] bool complete() const noexcept {
return !listen.empty() && !certificate.empty() && !key.empty() && !issuer.empty() &&
@@ -115,6 +118,8 @@ template <typename Host> class M10Bridge {
return fail("error m10-grant-lifetime");
config.control.max_grant_lifetime = o.max_grant_lifetime;
}
if (!o.load_governor && config.control.environment != network::DeploymentEnvironment::dev)
return fail("error m10-load-governor");
std::optional<std::array<std::int64_t, 4>> area_cells;
if (!o.area.empty() && !(area_cells = cells_of(o.area)))
return fail("error m10-area");
@@ -139,6 +144,9 @@ template <typename Host> class M10Bridge {
config.ledger = bridge.control_.get();
config.actions = bridge.actions_.get();
bridge.area_cells_ = area_cells;
bridge.governed_ = o.load_governor;
if (!o.load_governor)
std::puts("m10 load governor off");
if (!o.handoff_key.empty()) {
const auto scalar = read(o.handoff_key);
auto signer = network::HandoffSigningKey::import(
@@ -250,6 +258,10 @@ template <typename Host> class M10Bridge {
c.m10_sessions = sessions_->sessions();
c.m10_refused_logins = sessions_->refused_for_load();
c.m10_load = governor_.state();
if (!governed_) { // Measured only: nothing is refused or shed.
c.m10_load.refusing = false;
c.m10_load.shed = 0;
}
}
void shutdown() {
if (sessions_)
@@ -284,7 +296,7 @@ template <typename Host> class M10Bridge {
bool changed = false;
for (std::uint64_t i = 0; i < ticks; ++i)
changed = governor_.observe(per_tick, session_per_tick, s.tick_debt, now_us) || changed;
if (!changed)
if (!changed || !governed_)
return;
const auto& load = governor_.state();
sessions_->set_load(load.refusing, load.shed);
@@ -435,6 +447,7 @@ template <typename Host> class M10Bridge {
std::unique_ptr<server::PostgresControlLedger> control_;
std::unique_ptr<server::PostgresActionLedger> actions_;
server::LoadGovernor governor_{*server::LoadGovernor::create({})};
bool governed_{true};
std::chrono::nanoseconds busy_{}, session_busy_{};
std::uint64_t measured_tick_{};
std::unique_ptr<server::PostgresProbeLedger> probes_; // Handoff mode only.
+4 -1
View File
@@ -237,6 +237,8 @@ std::optional<Options> parse(int argc, char** argv) {
o.m10.handoff_key = value;
else if (flag == "--m10-neighbour" && o.m10.neighbours.size() < 8)
o.m10.neighbours.emplace_back(value);
else if (flag == "--m10-load-governor" && (value == "on" || value == "off"))
o.m10.load_governor = value == "on";
else if (flag == "--crash-at") {
const auto colon = value.find(':');
const auto ordinal =
@@ -607,7 +609,8 @@ int main(int argc, char** argv) {
"--m10-environment dev|stage|world0 --session-window ACTIVATION:UTC_EXPIRY "
"[--m10-max-grant-lifetime 60..3600] [--m10-area X0,Y0,X1,Y1] "
"[--m10-handoff-key F [--m10-neighbour "
"REGION=X0,Y0,X1,Y1=HOST:PORT=NAME=PUBFILE]...]]\n",
"REGION=X0,Y0,X1,Y1=HOST:PORT=NAME=PUBFILE]...] "
"[--m10-load-governor on|off]]\n",
stderr);
return 2;
}
@@ -12,7 +12,7 @@ spec:
env:
- {name: REGION_SESSION, value: "dev"}
- {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"}
- {name: REGION_SESSION_WINDOW, value: "dev-20261007T125653-8fc4:1791379013"}
- {name: REGION_SESSION_WINDOW, value: "dev-20261007T155240-b76a:1791389560"}
ports:
# QUIC (ADR-0090): UDP only; nothing listens on TCP 7400.
- {name: session, containerPort: 7400, protocol: UDP}
+20
View File
@@ -4988,3 +4988,23 @@ Executed:
Not done:
- the reference-worker run (10.9d);
- the abuse suite (10.9c).
## 2026-10-07 — CI correction for M10.9a/b (ADR-0097)
With agent access to Gitea Actions it became visible that `checks (postgres)` had failed
on both M10.9 pushes (`b8cb9db`, `1ba6748`). The `server-image` and `image` jobs were
therefore skipped, and the last published server image is still from `97a3756`. The local
runs recorded above passed, but CI is the authority and was red.
Cause: `region_m10_session_tests` asserted that a lightly loaded region is not refusing.
On the shared CI runner (Release build) one session measured work p95 22 ms and p99 37 ms
(session layer p99 22 ms), so the load governor correctly refused, and the test's
assumption about host speed failed.
Fix: evo-server gains `--m10-load-governor on|off` (dev only; default on). With `off`,
work is still measured and exposed, but nothing is refused or shed, and the gauges report
refusing 0 and shed 0. The M10 session, load-smoke and two-process handoff tests use it;
the governor itself remains covered by its unit tests.
Executed: local `postgres-check.sh` with QUIC, all suites. The CI result for this commit
is checked via the Gitea API after the push.
+2
View File
@@ -191,6 +191,8 @@ struct Pair {
(files / "issuer.pub").string(),
"--m10-environment",
"dev",
"--m10-load-governor",
"off",
"--session-window",
window,
"--m10-area",
+35 -22
View File
@@ -126,23 +126,36 @@ struct Setup {
}
[[nodiscard]] std::vector<std::string> args() const {
return server_args(
region,
{"--synthetic-inputs", "development-v1", "--ops-listen", "127.0.0.1:0", "--m10-listen",
"127.0.0.1:0", "--m10-cert", (region.root / "server.pem").string(), "--m10-key",
(region.root / "server.key").string(), "--m10-issuer",
(region.root / "issuer.pub").string(), "--m10-environment", "dev",
"--m10-max-grant-lifetime", "900", "--session-window",
"test-" +
std::to_string(static_cast<std::uint64_t>(
std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count())) +
":" +
std::to_string(static_cast<std::uint64_t>(
std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count()) +
1200)});
region, {"--synthetic-inputs",
"development-v1",
"--m10-load-governor",
"off",
"--ops-listen",
"127.0.0.1:0",
"--m10-listen",
"127.0.0.1:0",
"--m10-cert",
(region.root / "server.pem").string(),
"--m10-key",
(region.root / "server.key").string(),
"--m10-issuer",
(region.root / "issuer.pub").string(),
"--m10-environment",
"dev",
"--m10-max-grant-lifetime",
"900",
"--session-window",
"test-" +
std::to_string(static_cast<std::uint64_t>(
std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count())) +
":" +
std::to_string(static_cast<std::uint64_t>(
std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count()) +
1200)});
}
[[nodiscard]] WireGrant grant(const AccountId& who, std::uint64_t serial) const {
GrantClaims c;
@@ -273,8 +286,9 @@ void end_to_end() {
CHECK(std::ranges::all_of(shown->cells(),
[&](const auto& c) { return c.voxel.z == shown->slice_z(); }));
CHECK(until(*server, *a, [&] { return a->server_tick() >= first_tick + 40; }));
// M10.9a (ADR-0097): the listener's load is exposed; a lightly loaded region measures
// per-tick work, accepts logins and sheds nothing.
// M10.9a (ADR-0097): the listener's load is exposed. These servers run with
// --m10-load-governor off: their host's speed is unknown (a shared CI runner measured
// 22 ms p95 for one session), so work is measured but nothing is refused or shed.
const auto sessions = metric(server->ops, "evo_m10_sessions");
const auto p95 = metric(server->ops, "evo_m10_tick_work_p95_microseconds");
std::printf("m10 load report sessions=%llu p95_us=%llu p99_us=%llu session_p99_us=%llu\n",
@@ -356,9 +370,8 @@ void end_to_end() {
const auto o3 = outcome_of(*server, *c, 3, true);
CHECK(o3 && must(o3).result == ActionResult::committed_no_effect && !record(alice, 3).empty());
// The headless probe: login, intentions, actions and a query, then a clean exit. After the
// restarts the load governor (ADR-0097) may still refuse logins while the host catches up;
// the probe does not retry, so wait until it accepts.
// The headless probe: login, intentions, actions and a query, then a clean exit. The
// probe does not retry, so make sure the region accepts logins (governor off: at once).
CHECK(until(*server, *c, [&] { return metric(server->ops, "evo_m10_refusing") == 0u; }));
const auto probe = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-session-probe";
Child p{{probe.string(), "run",