From 39edfb077363b6b1b28fd8c912c9d011f426d556 Mon Sep 17 00:00:00 2001 From: dingo Date: Thu, 8 Oct 2026 06:21:15 -0400 Subject: [PATCH] Add M10.9c-2 lease loss under load process test ADR-0097. 24 evo-load bots against a real evo-server on a fixed loopback port: a SIGKILL (the successor waits out the lease) and then a SIGTERM drain, each followed by a successor under a new epoch. Every bot logs in again within 15 s of the successor being ready (development reference: 5.0 s after SIGKILL, 2.4 s after drain), with 48 drops and no failed logins counted over the hold. Scope narrowed: no standalone raw-UDP sender; abuse stays in-process or on loopback, and the per-connection datagram cap's eviction cost is analysed in the ADR. Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 --- .../region/dev-session-route/listener.yaml | 2 +- docs/ROADMAP.md | 7 +- docs/STATUS.md | 7 +- .../0097-m10-load-and-abuse-qualification.md | 27 +++++ docs/engineering/VALIDATION.md | 22 ++++ tests/region_m10_session_tests.cpp | 113 +++++++++++++++++- 6 files changed, 171 insertions(+), 7 deletions(-) diff --git a/deploy/kubernetes/region/dev-session-route/listener.yaml b/deploy/kubernetes/region/dev-session-route/listener.yaml index 7e630ad..a6297bc 100644 --- a/deploy/kubernetes/region/dev-session-route/listener.yaml +++ b/deploy/kubernetes/region/dev-session-route/listener.yaml @@ -12,7 +12,7 @@ spec: env: - {name: REGION_SESSION, value: "dev"} - {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"} - - {name: REGION_SESSION_WINDOW, value: "dev-20261008T015315-20832:1791427995"} + - {name: REGION_SESSION_WINDOW, value: "dev-20261008T100924-e9f8:1791455364"} - {name: REGION_M10_MAX_GRANT_LIFETIME, value: "14400"} ports: # QUIC (ADR-0090): UDP only; nothing listens on TCP 7400. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 2ac8205..ef0c9a0 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -546,9 +546,10 @@ runs qualify): 10.9a per-tick work measurement, load governor (refuse logins, th replication with hysteresis) and its metrics done; 10.9b evo-load multi-client tool (per-client distributions, starvation check, step verdicts, server gauges) and its process smoke done; 10.9c-1 session fuzz target and smoke, in-process Hello and datagram floods, -slow peer and connect/disconnect storm with honest-service checks done; 10.9c-2 lease loss -with 20+ bots and a live raw-UDP abuse sender, and 10.9d dev-cluster drill harness and -reference results follow. +slow peer and connect/disconnect storm with honest-service checks done; 10.9c-2 SIGKILL and +drain lease loss with 24 bots (all back within 5 s of the successor) done; no standalone +raw-UDP sender (abuse stays in-process/loopback; datagram-cap eviction cost analysed). +10.9d dev-cluster drill harness and reference results follow. **Entry:** M9 client/session evidence and M7/M8 dev/stage durability. Read NETWORK_ARCHITECTURE, PERSISTENCE_ARCHITECTURE and ADR-0017. The M9 transport is diff --git a/docs/STATUS.md b/docs/STATUS.md index 33e5370..b94bbba 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -120,8 +120,11 @@ refusing at 32 (work p95 about 14 ms), with responses about 103 ms and outcomes which an honest session kept every snapshot and receipt. `evo-session-probe` can now log in from a renewal's grant bundle. A dev login was welcomed -and its intention admitted (the dev region does not simulate outside drills). Next: -10.9c-2, lease loss with 20+ bots and a live raw-UDP sender. +and its intention admitted (the dev region does not simulate outside drills). M10.9c-2 +kills and drains a real evo-server under 24 `evo-load` bots. All of them log in to the +successor within 5.0 s (SIGKILL, after a 3.5 s lease wait) or 2.4 s (drain), with no failed +logins, and responses stay at 106 ms p95. Next: 10.9d, the evolution-dev drill harness and +the reference-worker run. ## Previous gate - 2026-10-02 diff --git a/docs/adr/0097-m10-load-and-abuse-qualification.md b/docs/adr/0097-m10-load-and-abuse-qualification.md index f6e9de2..390be24 100644 --- a/docs/adr/0097-m10-load-and-abuse-qualification.md +++ b/docs/adr/0097-m10-load-and-abuse-qualification.md @@ -195,3 +195,30 @@ snapshots. and a live raw-UDP sender of mutated datagrams. Our client endpoint caps its own send rate, so it cannot load the server's per-connection datagram cap (64, newest wins, with an O(queue) eviction scan); the raw sender measures that. + +## 10.9c-2 authority loss under load (2026-10-08) +- **Lease loss under load** (`region_m10_session_tests.cpp`): 24 `evo-load` bots on a real + evo-server (loopback, fixed port). First the server is SIGKILLed, so its lease must + expire before a successor runs. Then it is drained with SIGTERM: sessions close with + `shutdown` and the lease is released. Each time the successor takes the same port under + a new epoch, and every bot must log in again within 15 s of it being ready. + + Development reference: + + | Loss | Successor ready | All 24 bots back after ready | + |---|---|---| + | SIGKILL | 3.5 s | 5.0 s | + | SIGTERM drain | 0.18 s | 2.4 s | + + Over the 100 s hold the bots made 72 logins (24 + 2 × 24) and counted 48 drops, with no + failed or refused login, and response p95 stayed at 106 ms. After a SIGKILL, clients only + notice the loss through the QUIC idle timeout (5 s), which dominates the SIGKILL figure. +- **Scope narrowed:** no standalone tool sends raw UDP traffic at a network endpoint. + Abuse testing stays in-process or on loopback inside the test suite (10.9c-1). The + server's per-connection datagram cap is analysed here instead of being measured with a + flood tool. Each connection may hold 64 queued datagrams (newest wins). Evicting the + oldest scans the endpoint's event queue (at most 4096 events), which happens only when + one connection queues more than 64 datagrams between two server polls (normally under a + millisecond apart). That cost is bounded per datagram but can add up at very high rates + from one peer. Making eviction O(1) without letting tombstones outgrow the queue bound is + recorded as a follow-up, not done. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 1396990..ecfee6e 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -5063,3 +5063,25 @@ Executed: Not done (10.9c-2): lease loss with 20 or more bots, and a live raw-UDP sender of mutated datagrams that loads the server's per-connection datagram cap. + +## 2026-10-08 — M10.9c-2 lease loss under load (ADR-0097) + +Implemented: +- `region_m10_session_tests` `lease_loss_under_load`: 24 `evo-load` bots against a real + evo-server on a fixed loopback port. It runs a SIGKILL and then a SIGTERM drain, each + followed by a successor under a new epoch. +- The test fixture's `Setup` gains a `listen` setting (default `127.0.0.1:0`). +- Scope: no standalone raw-UDP sender was built. The per-connection datagram cap's eviction + cost is analysed in ADR-0097, and making eviction O(1) is a follow-up. + +Results (development reference, WSL, Release): +- **SIGKILL:** the successor was ready after 3.5 s, and all 24 bots were back 5.0 s + later. +- **Drain:** the successor was ready after 0.18 s, and all 24 bots were back 2.4 s later. +- **Bots over the 100 s hold:** 72 logins, 48 drops, no failed or refused logins; response + p95 106 ms; login p99 62 ms. + +Executed: +- **Linux GCC 14:** `postgres-check.sh` with QUIC, all suites. +- **Linux Clang 19:** clang-tidy 19 on the M10 process tests (clean); clang-format 19. +- **Windows:** not applicable (the PostgreSQL process tests are Linux only). diff --git a/tests/region_m10_session_tests.cpp b/tests/region_m10_session_tests.cpp index 99ed6c2..3a561ee 100644 --- a/tests/region_m10_session_tests.cpp +++ b/tests/region_m10_session_tests.cpp @@ -100,10 +100,26 @@ std::optional metric(std::uint16_t port, const std::string& serie return std::stoull(raw.substr(at + series.size() + 2)); } +// A UDP port that was free a moment ago. +std::uint16_t free_port() { + const int fd = ::socket(AF_INET, SOCK_DGRAM | SOCK_CLOEXEC, 0); + expect(fd >= 0, "socket"); + sockaddr_in a{}; + a.sin_family = AF_INET; + a.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + socklen_t size = sizeof a; + const bool ok = ::bind(fd, reinterpret_cast(&a), sizeof a) == 0 && + ::getsockname(fd, reinterpret_cast(&a), &size) == 0; + ::close(fd); + expect(ok, "free port"); + return ntohs(a.sin_port); +} + // One region's credentials, issuer key and server process arguments. struct Setup { ServerRegion region; std::string ca_pem; + std::string listen{"127.0.0.1:0"}; // A fixed port lets clients reconnect after a restart. std::optional issuer; explicit Setup(std::uint64_t r, std::uint8_t genesis) : region{provision(r, genesis)} { const auto made = generate_dev_credentials(DeploymentEnvironment::dev, name, 1, @@ -133,7 +149,7 @@ struct Setup { "--ops-listen", "127.0.0.1:0", "--m10-listen", - "127.0.0.1:0", + listen, "--m10-cert", (region.root / "server.pem").string(), "--m10-key", @@ -523,10 +539,105 @@ void load_smoke() { server.child->signal(SIGTERM); CHECK(server.child->wait_exit(30s) == 0); } +// M10.9c-2 (ADR-0097): region authority lost under load. 24 evo-load bots on a real +// evo-server; the server is SIGKILLed (its lease must expire before a successor runs), then +// later drained with SIGTERM (sessions close with shutdown). Each time the successor takes +// the same port under a new epoch and every bot logs in again within 15 s of it being ready, +// and the bots count the drops. Loopback only; the bound is a development reference. +void lease_loss_under_load() { + Setup s{153, 0xC7}; + s.listen = "127.0.0.1:" + std::to_string(free_port()); + auto server = std::make_unique(s); + constexpr std::uint64_t bots = 24; + const auto tool = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-load"; + Child load{{tool.string(), + "--connect", + "127.0.0.1:" + std::to_string(server->port), + "--server-name", + name, + "--ca", + (s.region.root / "ca.pem").string(), + "--issuer-key", + (s.region.root / "issuer.key").string(), + "--world", + "7", + "--frame", + std::to_string(region_identity.frame), + "--region", + "153", + "--steps", + std::to_string(bots), + "--hold", + "100", + "--warmup", + "2", + "--ramp-ms", + "50", + "--threads", + "4", + "--lifetime", + "900"}}; + const auto pump = [&] { + server->pump(); + load.pump(5ms); + }; + // Milliseconds until the region reports `n` sessions, or nothing within `limit`. + const auto sessions_after = + [&](std::chrono::milliseconds limit) -> std::optional { + const auto started = Clock::now(); + for (int i = 0; Clock::now() - started < limit; ++i) { + pump(); + if (i % 20 == 0 && metric(server->ops, "evo_m10_sessions") == bots) + return std::chrono::duration_cast(Clock::now() - + started); + } + return std::nullopt; + }; + CHECK(sessions_after(60s).has_value()); + // A successor on the same port; how long until it is ready and every bot is back. + const auto successor = [&](const char* how) { + const auto lost = Clock::now(); + server = std::make_unique(s); // Waits out the old lease, then "ready 1". + const auto ready = + std::chrono::duration_cast(Clock::now() - lost); + const auto back = sessions_after(15s); + std::printf("lease loss %s ready_ms=%lld all_back_ms=%lld\n", how, + static_cast(ready.count()), + static_cast(back ? back->count() : -1)); + return back.has_value(); + }; + server->kill(); + CHECK(successor("sigkill")); + server->child->signal(SIGTERM); + CHECK(server->child->wait_exit(30s) == 0); + CHECK(successor("sigterm")); + // The bots measured both losses within their hold. + int code = -1; + for (const auto end = Clock::now() + 180s; Clock::now() < end && code < 0;) { + pump(); + if (load.printed("load result ")) + code = load.wait_exit(30s); + } + std::string step; + for (const auto& line : load.lines()) { + if (starts(line, "load step ") || starts(line, "load response_ms") || + starts(line, "load login_ms")) + std::printf("%s\n", line.c_str()); + if (starts(line, "load step ")) + step = line; + } + CHECK(code == 0 && starts(step, "load step N=24 established=24 ")); + const auto drops_at = step.find(" drops="); + CHECK(drops_at != std::string::npos && + std::stoull(step.substr(drops_at + 7)) >= 2 * bots); // Both losses dropped every bot. + server->child->signal(SIGTERM); + CHECK(server->child->wait_exit(30s) == 0); +} } // namespace void run_region_m10_session_tests() { end_to_end(); load_smoke(); + lease_loss_under_load(); std::puts("region m10 session integration passed"); }