Fix CI: run M10 process tests with the load governor off
Foundation / checks (deploy) (push) Successful in 3m15s
Foundation / checks (ecs) (push) Successful in 1m57s
Foundation / checks (client) (push) Successful in 12m33s
Foundation / checks (network) (push) Successful in 13m19s
Foundation / checks (tsan) (push) Successful in 8m5s
Foundation / checks (checks) (push) Successful in 32m42s
Foundation / checks (postgres) (push) Successful in 43m45s
Foundation / postgres-tools (push) Successful in 20s
Foundation / server-image (push) Successful in 24s
Foundation / image (push) Successful in 2m31s
Foundation / checks (deploy) (push) Successful in 3m15s
Foundation / checks (ecs) (push) Successful in 1m57s
Foundation / checks (client) (push) Successful in 12m33s
Foundation / checks (network) (push) Successful in 13m19s
Foundation / checks (tsan) (push) Successful in 8m5s
Foundation / checks (checks) (push) Successful in 32m42s
Foundation / checks (postgres) (push) Successful in 43m45s
Foundation / postgres-tools (push) Successful in 20s
Foundation / server-image (push) Successful in 24s
Foundation / image (push) Successful in 2m31s
checks (postgres) failed on both M10.9 pushes: on the shared CI runner one session measured 22 ms p95 per tick, so the governor correctly refused and the test's assumption about host speed failed (server-image was skipped as a result). evo-server gains --m10-load-governor on|off (dev only): off keeps measuring and exposing the work but never refuses or sheds. The M10 session, load-smoke and two-process handoff tests use it; the governor stays covered by its unit tests. Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -55,6 +55,9 @@ struct M10Options {
|
||||
std::string listen, certificate, key, issuer, environment, session_window;
|
||||
// M10.5c: dev only, 60..3600 s; 0 keeps the default (300 s, ADR-0091).
|
||||
std::uint64_t max_grant_lifetime{};
|
||||
// M10.9 (ADR-0097, dev only): `--m10-load-governor off` keeps measuring and exposing the
|
||||
// work but never refuses or sheds (process tests whose host speed is unknown).
|
||||
bool load_governor{true};
|
||||
// M10.8c (ADR-0096, dev only): this region's marker area in cells "X0,Y0,X1,Y1"
|
||||
// (inclusive), its handoff key (32-byte P-256 scalar; key id = region id) and up to 8
|
||||
// neighbours "REGION=X0,Y0,X1,Y1=HOST:PORT=SERVER_NAME=PUBLIC_KEY_FILE".
|
||||
@@ -63,7 +66,7 @@ struct M10Options {
|
||||
[[nodiscard]] bool any() const noexcept {
|
||||
return !listen.empty() || !certificate.empty() || !key.empty() || !issuer.empty() ||
|
||||
!environment.empty() || !session_window.empty() || max_grant_lifetime != 0 ||
|
||||
!area.empty() || !handoff_key.empty() || !neighbours.empty();
|
||||
!area.empty() || !handoff_key.empty() || !neighbours.empty() || !load_governor;
|
||||
}
|
||||
[[nodiscard]] bool complete() const noexcept {
|
||||
return !listen.empty() && !certificate.empty() && !key.empty() && !issuer.empty() &&
|
||||
@@ -115,6 +118,8 @@ template <typename Host> class M10Bridge {
|
||||
return fail("error m10-grant-lifetime");
|
||||
config.control.max_grant_lifetime = o.max_grant_lifetime;
|
||||
}
|
||||
if (!o.load_governor && config.control.environment != network::DeploymentEnvironment::dev)
|
||||
return fail("error m10-load-governor");
|
||||
std::optional<std::array<std::int64_t, 4>> area_cells;
|
||||
if (!o.area.empty() && !(area_cells = cells_of(o.area)))
|
||||
return fail("error m10-area");
|
||||
@@ -139,6 +144,9 @@ template <typename Host> class M10Bridge {
|
||||
config.ledger = bridge.control_.get();
|
||||
config.actions = bridge.actions_.get();
|
||||
bridge.area_cells_ = area_cells;
|
||||
bridge.governed_ = o.load_governor;
|
||||
if (!o.load_governor)
|
||||
std::puts("m10 load governor off");
|
||||
if (!o.handoff_key.empty()) {
|
||||
const auto scalar = read(o.handoff_key);
|
||||
auto signer = network::HandoffSigningKey::import(
|
||||
@@ -250,6 +258,10 @@ template <typename Host> class M10Bridge {
|
||||
c.m10_sessions = sessions_->sessions();
|
||||
c.m10_refused_logins = sessions_->refused_for_load();
|
||||
c.m10_load = governor_.state();
|
||||
if (!governed_) { // Measured only: nothing is refused or shed.
|
||||
c.m10_load.refusing = false;
|
||||
c.m10_load.shed = 0;
|
||||
}
|
||||
}
|
||||
void shutdown() {
|
||||
if (sessions_)
|
||||
@@ -284,7 +296,7 @@ template <typename Host> class M10Bridge {
|
||||
bool changed = false;
|
||||
for (std::uint64_t i = 0; i < ticks; ++i)
|
||||
changed = governor_.observe(per_tick, session_per_tick, s.tick_debt, now_us) || changed;
|
||||
if (!changed)
|
||||
if (!changed || !governed_)
|
||||
return;
|
||||
const auto& load = governor_.state();
|
||||
sessions_->set_load(load.refusing, load.shed);
|
||||
@@ -435,6 +447,7 @@ template <typename Host> class M10Bridge {
|
||||
std::unique_ptr<server::PostgresControlLedger> control_;
|
||||
std::unique_ptr<server::PostgresActionLedger> actions_;
|
||||
server::LoadGovernor governor_{*server::LoadGovernor::create({})};
|
||||
bool governed_{true};
|
||||
std::chrono::nanoseconds busy_{}, session_busy_{};
|
||||
std::uint64_t measured_tick_{};
|
||||
std::unique_ptr<server::PostgresProbeLedger> probes_; // Handoff mode only.
|
||||
|
||||
@@ -237,6 +237,8 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
o.m10.handoff_key = value;
|
||||
else if (flag == "--m10-neighbour" && o.m10.neighbours.size() < 8)
|
||||
o.m10.neighbours.emplace_back(value);
|
||||
else if (flag == "--m10-load-governor" && (value == "on" || value == "off"))
|
||||
o.m10.load_governor = value == "on";
|
||||
else if (flag == "--crash-at") {
|
||||
const auto colon = value.find(':');
|
||||
const auto ordinal =
|
||||
@@ -607,7 +609,8 @@ int main(int argc, char** argv) {
|
||||
"--m10-environment dev|stage|world0 --session-window ACTIVATION:UTC_EXPIRY "
|
||||
"[--m10-max-grant-lifetime 60..3600] [--m10-area X0,Y0,X1,Y1] "
|
||||
"[--m10-handoff-key F [--m10-neighbour "
|
||||
"REGION=X0,Y0,X1,Y1=HOST:PORT=NAME=PUBFILE]...]]\n",
|
||||
"REGION=X0,Y0,X1,Y1=HOST:PORT=NAME=PUBFILE]...] "
|
||||
"[--m10-load-governor on|off]]\n",
|
||||
stderr);
|
||||
return 2;
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ spec:
|
||||
env:
|
||||
- {name: REGION_SESSION, value: "dev"}
|
||||
- {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"}
|
||||
- {name: REGION_SESSION_WINDOW, value: "dev-20261007T125653-8fc4:1791379013"}
|
||||
- {name: REGION_SESSION_WINDOW, value: "dev-20261007T155240-b76a:1791389560"}
|
||||
ports:
|
||||
# QUIC (ADR-0090): UDP only; nothing listens on TCP 7400.
|
||||
- {name: session, containerPort: 7400, protocol: UDP}
|
||||
|
||||
@@ -4988,3 +4988,23 @@ Executed:
|
||||
Not done:
|
||||
- the reference-worker run (10.9d);
|
||||
- the abuse suite (10.9c).
|
||||
|
||||
## 2026-10-07 — CI correction for M10.9a/b (ADR-0097)
|
||||
|
||||
With agent access to Gitea Actions it became visible that `checks (postgres)` had failed
|
||||
on both M10.9 pushes (`b8cb9db`, `1ba6748`). The `server-image` and `image` jobs were
|
||||
therefore skipped, and the last published server image is still from `97a3756`. The local
|
||||
runs recorded above passed, but CI is the authority and was red.
|
||||
|
||||
Cause: `region_m10_session_tests` asserted that a lightly loaded region is not refusing.
|
||||
On the shared CI runner (Release build) one session measured work p95 22 ms and p99 37 ms
|
||||
(session layer p99 22 ms), so the load governor correctly refused, and the test's
|
||||
assumption about host speed failed.
|
||||
|
||||
Fix: evo-server gains `--m10-load-governor on|off` (dev only; default on). With `off`,
|
||||
work is still measured and exposed, but nothing is refused or shed, and the gauges report
|
||||
refusing 0 and shed 0. The M10 session, load-smoke and two-process handoff tests use it;
|
||||
the governor itself remains covered by its unit tests.
|
||||
|
||||
Executed: local `postgres-check.sh` with QUIC, all suites. The CI result for this commit
|
||||
is checked via the Gitea API after the push.
|
||||
|
||||
@@ -191,6 +191,8 @@ struct Pair {
|
||||
(files / "issuer.pub").string(),
|
||||
"--m10-environment",
|
||||
"dev",
|
||||
"--m10-load-governor",
|
||||
"off",
|
||||
"--session-window",
|
||||
window,
|
||||
"--m10-area",
|
||||
|
||||
@@ -126,23 +126,36 @@ struct Setup {
|
||||
}
|
||||
[[nodiscard]] std::vector<std::string> args() const {
|
||||
return server_args(
|
||||
region,
|
||||
{"--synthetic-inputs", "development-v1", "--ops-listen", "127.0.0.1:0", "--m10-listen",
|
||||
"127.0.0.1:0", "--m10-cert", (region.root / "server.pem").string(), "--m10-key",
|
||||
(region.root / "server.key").string(), "--m10-issuer",
|
||||
(region.root / "issuer.pub").string(), "--m10-environment", "dev",
|
||||
"--m10-max-grant-lifetime", "900", "--session-window",
|
||||
"test-" +
|
||||
std::to_string(static_cast<std::uint64_t>(
|
||||
std::chrono::duration_cast<std::chrono::seconds>(
|
||||
std::chrono::system_clock::now().time_since_epoch())
|
||||
.count())) +
|
||||
":" +
|
||||
std::to_string(static_cast<std::uint64_t>(
|
||||
std::chrono::duration_cast<std::chrono::seconds>(
|
||||
std::chrono::system_clock::now().time_since_epoch())
|
||||
.count()) +
|
||||
1200)});
|
||||
region, {"--synthetic-inputs",
|
||||
"development-v1",
|
||||
"--m10-load-governor",
|
||||
"off",
|
||||
"--ops-listen",
|
||||
"127.0.0.1:0",
|
||||
"--m10-listen",
|
||||
"127.0.0.1:0",
|
||||
"--m10-cert",
|
||||
(region.root / "server.pem").string(),
|
||||
"--m10-key",
|
||||
(region.root / "server.key").string(),
|
||||
"--m10-issuer",
|
||||
(region.root / "issuer.pub").string(),
|
||||
"--m10-environment",
|
||||
"dev",
|
||||
"--m10-max-grant-lifetime",
|
||||
"900",
|
||||
"--session-window",
|
||||
"test-" +
|
||||
std::to_string(static_cast<std::uint64_t>(
|
||||
std::chrono::duration_cast<std::chrono::seconds>(
|
||||
std::chrono::system_clock::now().time_since_epoch())
|
||||
.count())) +
|
||||
":" +
|
||||
std::to_string(static_cast<std::uint64_t>(
|
||||
std::chrono::duration_cast<std::chrono::seconds>(
|
||||
std::chrono::system_clock::now().time_since_epoch())
|
||||
.count()) +
|
||||
1200)});
|
||||
}
|
||||
[[nodiscard]] WireGrant grant(const AccountId& who, std::uint64_t serial) const {
|
||||
GrantClaims c;
|
||||
@@ -273,8 +286,9 @@ void end_to_end() {
|
||||
CHECK(std::ranges::all_of(shown->cells(),
|
||||
[&](const auto& c) { return c.voxel.z == shown->slice_z(); }));
|
||||
CHECK(until(*server, *a, [&] { return a->server_tick() >= first_tick + 40; }));
|
||||
// M10.9a (ADR-0097): the listener's load is exposed; a lightly loaded region measures
|
||||
// per-tick work, accepts logins and sheds nothing.
|
||||
// M10.9a (ADR-0097): the listener's load is exposed. These servers run with
|
||||
// --m10-load-governor off: their host's speed is unknown (a shared CI runner measured
|
||||
// 22 ms p95 for one session), so work is measured but nothing is refused or shed.
|
||||
const auto sessions = metric(server->ops, "evo_m10_sessions");
|
||||
const auto p95 = metric(server->ops, "evo_m10_tick_work_p95_microseconds");
|
||||
std::printf("m10 load report sessions=%llu p95_us=%llu p99_us=%llu session_p99_us=%llu\n",
|
||||
@@ -356,9 +370,8 @@ void end_to_end() {
|
||||
const auto o3 = outcome_of(*server, *c, 3, true);
|
||||
CHECK(o3 && must(o3).result == ActionResult::committed_no_effect && !record(alice, 3).empty());
|
||||
|
||||
// The headless probe: login, intentions, actions and a query, then a clean exit. After the
|
||||
// restarts the load governor (ADR-0097) may still refuse logins while the host catches up;
|
||||
// the probe does not retry, so wait until it accepts.
|
||||
// The headless probe: login, intentions, actions and a query, then a clean exit. The
|
||||
// probe does not retry, so make sure the region accepts logins (governor off: at once).
|
||||
CHECK(until(*server, *c, [&] { return metric(server->ops, "evo_m10_refusing") == 0u; }));
|
||||
const auto probe = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-session-probe";
|
||||
Child p{{probe.string(), "run",
|
||||
|
||||
Reference in New Issue
Block a user