From 1ba6748288008376517107893a84685718743fd4 Mon Sep 17 00:00:00 2001 From: dingo Date: Wed, 7 Oct 2026 09:02:57 -0400 Subject: [PATCH] Add M10.9b evo-load multi-client load generator and process smoke ADR-0097. evo-load runs bots on worker threads, each a RegionSessionClient with its own account and an in-process dev grant: one intention per tick at its input lead with marker prediction, actions every 2 s, interest moves every 10 s, and re-login after any close. Stepped ramps report pooled and per-client distributions (response, receipt, outcome, login, prediction error), snapshot rate, gaps and bandwidth, drops, failed logins and refusals, generator CPU, a starvation check among connected bots, a budget verdict and the region's evo_m10_* gauges, ending with the sustainable bot count. A Postgres process smoke runs 4 then 8 bots against a real evo-server. Fixes found while ramping: the process fixture's short pumps never read (a chatty child blocked on its stdout pipe), and the load governor refused on a single tick of debt (now 2 refuse, 4 shed). Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 9 +- CMakePresets.json | 2 +- .../region/dev-session-route/listener.yaml | 2 +- docs/ROADMAP.md | 7 +- docs/STATUS.md | 8 +- .../0097-m10-load-and-abuse-qualification.md | 54 +- docs/engineering/VALIDATION.md | 40 + include/evo/server/load_governor.hpp | 3 + schemas/network/m10-budgets-v1.md | 2 +- src/server/load_governor.cpp | 4 +- tests/fixtures/server_process.hpp | 5 +- tests/ops_tests.cpp | 9 +- tests/region_m10_session_tests.cpp | 96 ++- tools/network/load.cpp | 764 ++++++++++++++++++ 14 files changed, 983 insertions(+), 22 deletions(-) create mode 100644 tools/network/load.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 5aa3eac..ca75d73 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -160,6 +160,13 @@ if(EVO_BUILD_QUIC) target_link_libraries(evo-session-probe PRIVATE evo_session_client evo_network_tls) evo_owned_target(evo-session-probe) evo_msquic_runtime(evo-session-probe) + # M10.9b multi-client load generator (development and qualification; ADR-0097). + find_package(Threads REQUIRED) + add_executable(evo-load tools/network/load.cpp) + target_link_libraries(evo-load PRIVATE evo_session_client evo_network_tls evo_client_prediction + Threads::Threads) + evo_owned_target(evo-load) + evo_msquic_runtime(evo-load) if(BUILD_TESTING) # Owned impairment relay (tests and measurements only). add_library(evo_network_impair STATIC tools/network/impair/udp_relay.cpp) @@ -269,7 +276,7 @@ if(EVO_BUILD_POSTGRES) tests/region_handoff_process_tests.cpp) target_link_libraries(evo-postgres-tests PRIVATE evo_session_client) target_compile_definitions(evo-postgres-tests PRIVATE EVO_POSTGRES_M10=1) - add_dependencies(evo-postgres-tests evo-session-probe) + add_dependencies(evo-postgres-tests evo-session-probe evo-load) endif() add_dependencies(evo-postgres-tests evo-server evo-world-clone) # Spawned by process tests. evo_owned_target(evo-postgres-tests) diff --git a/CMakePresets.json b/CMakePresets.json index 6296102..1f3aac4 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -24,7 +24,7 @@ {"name": "client", "configurePreset": "client", "targets": ["evo-client-state-tests", "evo-client", "evo-client-lifecycle-tests", "evo-client-environment-tests", "evo-client-organism-tests", "evo-client-ui-model-tests", "evo-client-panel-tests", "evo-client-bindings-tests"]}, {"name": "client-probe", "configurePreset": "client-probe", "targets": ["evo-client-probe"]}, {"name": "network", "configurePreset": "network", "targets": ["evo-client-state-tests", "evo-network-tests", "evo-dev-session"]}, - {"name": "quic", "configurePreset": "quic", "targets": ["evo-quic-smoke-tests", "evo-quic-transport-tests", "evo-quic-session-tests", "evo-quic-hol-measure", "evo-session-probe", "evo-network-tests"]}, + {"name": "quic", "configurePreset": "quic", "targets": ["evo-quic-smoke-tests", "evo-quic-transport-tests", "evo-quic-session-tests", "evo-quic-hol-measure", "evo-session-probe", "evo-load", "evo-network-tests"]}, {"name": "client-network", "configurePreset": "client-network", "targets": ["evo-client-state-tests", "evo-client", "evo-client-lifecycle-tests", "evo-client-environment-tests", "evo-client-organism-tests", "evo-client-ui-model-tests", "evo-client-panel-tests", "evo-client-bindings-tests", "evo-network-tests", "evo-dev-session", "evo-quic-session-tests", "evo-client-remote-tests", "evo-motion-trace", "evo-session-probe"]}, {"name": "client-package", "configurePreset": "client-package", "targets": ["evo-client-state-tests", "evo-client", "evo-client-lifecycle-tests", "evo-client-environment-tests", "evo-client-organism-tests", "evo-client-ui-model-tests", "evo-client-panel-tests", "evo-client-bindings-tests", "evo-network-tests", "evo-dev-session", "evo-quic-session-tests", "evo-client-remote-tests", "evo-motion-trace"]} ], diff --git a/deploy/kubernetes/region/dev-session-route/listener.yaml b/deploy/kubernetes/region/dev-session-route/listener.yaml index c5c8d37..369f692 100644 --- a/deploy/kubernetes/region/dev-session-route/listener.yaml +++ b/deploy/kubernetes/region/dev-session-route/listener.yaml @@ -12,7 +12,7 @@ spec: env: - {name: REGION_SESSION, value: "dev"} - {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"} - - {name: REGION_SESSION_WINDOW, value: "dev-20261007T102834-1d97:1791370114"} + - {name: REGION_SESSION_WINDOW, value: "dev-20261007T125653-8fc4:1791379013"} ports: # QUIC (ADR-0090): UDP only; nothing listens on TCP 7400. - {name: session, containerPort: 7400, protocol: UDP} diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 50af1b4..e72648f 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -543,9 +543,10 @@ configuration done; 10.8d two evo-server processes on PostgreSQL with SIGKILL ar durable step, duplicate/stale retries and a failed destination done (schema v14 recovery sweep, replay guard). M10.8 complete. 10.9 split (ADR-0097; only evolution-dev reference-worker runs qualify): 10.9a per-tick work measurement, load governor (refuse logins, then shed -replication with hysteresis) and its metrics done; 10.9b evo-load multi-client tool, 10.9c -abuse suite (fuzz, floods, slow peers, storms, lease loss) and 10.9d dev-cluster harness and -reference results follow. +replication with hysteresis) and its metrics done; 10.9b evo-load multi-client tool +(per-client distributions, starvation check, step verdicts, server gauges) and its process +smoke done; 10.9c abuse suite (fuzz, floods, slow peers, storms, lease loss) and 10.9d +dev-cluster harness and reference results follow. **Entry:** M9 client/session evidence and M7/M8 dev/stage durability. Read NETWORK_ARCHITECTURE, PERSISTENCE_ARCHITECTURE and ADR-0017. The M9 transport is diff --git a/docs/STATUS.md b/docs/STATUS.md index 1826547..f4e715d 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -107,7 +107,13 @@ layer) and feeds a load governor: - both relax after 5 s of calm. `/metrics` exposes the work percentiles (total and session layer) and the governor's -state. Next: 10.9b, the `evo-load` multi-client tool. +state. M10.9b adds `evo-load`, a multi-client load generator. It runs bots on worker +threads that send intentions every tick, actions, interest moves and reconnects. It reports +per step pooled and per-client distributions, a starvation check, a budget verdict and the +region's gauges, ending with the sustainable bot count. In a local development reference +(not evidence), one region stayed within budget up to 24 bots and the governor began +refusing at 32 (work p95 about 14 ms), with responses about 103 ms and outcomes about +665 ms throughout. Next: 10.9c, the abuse suite. ## Previous gate - 2026-10-02 diff --git a/docs/adr/0097-m10-load-and-abuse-qualification.md b/docs/adr/0097-m10-load-and-abuse-qualification.md index 70799ff..f7edf41 100644 --- a/docs/adr/0097-m10-load-and-abuse-qualification.md +++ b/docs/adr/0097-m10-load-and-abuse-qualification.md @@ -44,10 +44,14 @@ snapshots. - **Load governor** (`include/evo/server/load_governor.hpp`, portable, no clock). It keeps a window of the last 40 samples (2 s). It reports the window's p95, p99 and maximum, plus two outputs: - - **refusing** turns on when p95 ≥ 15 ms or tick debt > 0; - - **shed level** (0-3) rises by one when p95 ≥ 20 ms or tick debt ≥ 2, at most once per + - **refusing** turns on when p95 ≥ 15 ms or tick debt ≥ 2; + - **shed level** (0-3) rises by one when p95 ≥ 20 ms or tick debt ≥ 4, at most once per second. + One tick of debt is jitter (a commit flush or a scheduler hiccup) and only interrupts calm. + As first written, any debt refused logins for at least 5 s, which made a login just after + a restart's catch-up fail intermittently in a process test (amended in 10.9b). + Calm means p95 < 10 ms and no debt. After 5 s of calm the level drops by one. At level 0, 5 s more of calm turns refusing off. Any non-calm sample restarts the calm timer. Thresholds are configurable and validated (`relax < refuse ≤ shed ≤ tick budget`). @@ -105,3 +109,49 @@ snapshots. 7 ms and p99 about 9 ms per tick. Of that, the session layer's p99 was about 4.5 ms. That is far above 0.5 ms per session, so 10.9b examines it with evo-load. Likely contributors are the per-tick snapshot source query and synchronous control renewals. + +## 10.9b load tool (2026-10-07) +- **`evo-load`** (`tools/network/load.cpp`, quic preset; usage in its header). Bots run on + worker threads, each a `RegionSessionClient` with its own account and a dev grant signed + in-process. A bot: + - sends one intention per server tick at its `InputLead` (a random walk turning every + 2 s) and predicts its marker with `MotionPredictor`; + - sends a probe action every 2 s and moves its interest every 10 s (a fresh baseline); + - logs in again 2 s after any close. + + Steps raise the bot count (`--ramp-ms` apart), warm up, then hold. Each step reports: + - pooled p50, p95, p99 and maximum, plus the per-client p95 spread, for responsiveness + (input to the first own-marker snapshot applying it), receipt RTT, final-outcome + latency, login time and prediction error; + - snapshot rate, gaps and bandwidth per client; + - drops, failed logins and refusals; + - the generator's own CPU; + - with `--metrics`, the region's `evo_m10_*` gauges. + + Starvation is judged among the bots connected during the hold: a snapshot rate under + 80 % of the median, or a response p95 over both twice the median and the budget. A + step's verdict is within budget only with every bot established, no refusals or drops, + no starvation, response p95 within budget (150 ms by default), outcome p95 ≤ 1.5 s, + server work p99 ≤ 25 ms, no shedding or refusing, and no tick debt. The run ends with + `load result sustainable=N saturated_at=M`. +- **Smoke** (Postgres `region_m10_session_tests`): 4 then 8 bots against a real evo-server + with `--metrics`. Every bot logs in once with no drops or refusals, both steps measure + responses, outcomes and snapshots, and the report is complete. +- **Found while ramping:** + - The process-test fixture's `Child::pump(1ms)` rounded its wait down to 0 ms and never + read. A chatty evo-server then blocked on its full stdout pipe, which looked like a + server stall at 32 bots. The wait is now rounded up. + - The governor's any-debt refusal was too twitchy (see the amended decision above). +- **Local development reference** (WSL, Release, one region; not evidence): + + | Bots | Verdict | Work p95 / max | Session p95 (per session) | Response p95 | Outcome p95 | Snapshots | + |---|---|---|---|---|---|---| + | 8 | within | 9.0 / 9.6 ms | 1.7 ms (0.22) | 106 ms | 659 ms | 20 Hz, no gaps | + | 16 | within | 11.0 / 11.2 ms | 3.5 ms (0.22) | 102 ms | 661 ms | 20 Hz | + | 24 | within | 13.1 / 13.3 ms | 5.8 ms (0.24) | 103 ms | 664 ms | 20 Hz | + | 32 | refusing | 14.3 / 17.0 ms | 8.4 ms (0.26) | 103 ms | 665 ms | 20 Hz | + + At 48 and 64 bots, the bots beyond the 32-session cap fail at the transport connection + limit, and the 32 established sessions are unaffected. Prediction errors were 0 (no + loss on loopback), and no client starved at any step. The host step dominates the work; + the session layer stays near 0.25 ms per session (budget 0.5). diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 57ea2e7..d3b428c 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -4948,3 +4948,43 @@ Not done: - qualification on the evolution-dev reference worker (10.9d; needs v4 and v12-v14 deployed); - load generation (10.9b) and the abuse suite (10.9c). + +## 2026-10-07 — M10.9b evo-load (ADR-0097) + +Implemented: +- `evo-load` (`tools/network/load.cpp`, quic preset): bots on worker threads (intentions + every tick at the input lead with prediction, actions every 2 s, interest moves every + 10 s, re-login after any close), stepped ramps. +- Per step, it reports pooled and per-client distributions (response, receipt, outcome, + login, prediction error), snapshot rate, gaps and bandwidth, drops, failed logins and + refusals, the generator's CPU, a starvation check and a budget verdict. With + `--metrics`, it adds the region's gauges. The run ends with `sustainable` and + `saturated_at`. +- A Postgres process smoke (`region_m10_session_tests` `load_smoke`): 4 then 8 bots against + a real evo-server. +- Fixes found while ramping: + - The process fixture's `Child::pump` rounded short waits down to 0 ms and never read, + so a chatty child blocked on its stdout pipe. + - The load governor refused on a single tick of debt. It now refuses at 2 and sheds at + 4; one tick only interrupts calm (ADR-0097 amended). +- The M10 end-to-end test waits until the server is not refusing before the headless + probe logs in, and dumps the probe and server lines on failure. One run had failed + there intermittently before the debt change. + +Executed: +- **Linux GCC 14:** + - `postgres-check.sh` with QUIC, all suites, 3 runs in a row (2 after the final change); + - the smoke passes with every bot logged in once and both steps within budget; + - presets `quic` (ops, network, QUIC sessions) 3/3. +- **Local development reference ramp** (temporary steps 8-64, not committed, not + evidence): within budget up to 24 bots; the governor refused from 32 (work p95 about + 14 ms, maximum 17 ms); session work stayed near 0.25 ms per session; responses p95 about + 103 ms, outcomes about 665 ms, 20 Hz snapshots, no starvation. Beyond the 32-session cap, + extra logins failed at the transport limit and established sessions were unaffected. +- **Linux Clang 19:** clang-tidy 19 on `load.cpp`, the governor, ops and the M10 process + tests (clean); clang-format 19. +- **Windows MSVC 19.44:** presets `quic` 4/4 (`evo-load` builds) and `debug` 60/60. + +Not done: +- the reference-worker run (10.9d); +- the abuse suite (10.9c). diff --git a/include/evo/server/load_governor.hpp b/include/evo/server/load_governor.hpp index 30d40ef..9e40092 100644 --- a/include/evo/server/load_governor.hpp +++ b/include/evo/server/load_governor.hpp @@ -19,6 +19,9 @@ struct LoadPolicy { std::uint8_t max_shed{3}; }; inline constexpr std::size_t load_window_ticks = 40; // 2 s of 50 ms ticks. +// Tick debt (due minus simulated) that refuses logins and that sheds. One tick is jitter (a +// commit flush, a scheduler hiccup): it only interrupts calm. +inline constexpr std::uint64_t refuse_debt_ticks = 2, shed_debt_ticks = 4; struct LoadState { bool refusing{}; std::uint8_t shed{}; diff --git a/schemas/network/m10-budgets-v1.md b/schemas/network/m10-budgets-v1.md index 73a82e9..98a79e3 100644 --- a/schemas/network/m10-budgets-v1.md +++ b/schemas/network/m10-budgets-v1.md @@ -35,7 +35,7 @@ in real time; changing it requires a versioned rules migration (ADR-0089). | Per-tick region work | p99 <= 25 ms on the reference worker | budget | 10.9 (measured: `evo_m10_tick_work_*`, ADR-0097) | | Session handling | <= 0.5 ms per session per tick | budget | 10.9 (measured: `evo_m10_session_work_*`, ADR-0097) | | Host input queue | 64 inputs, 64 KiB | budget | region host | -| Overload response | shed interest, refuse new sessions before tick debt | budget | 10.9a load governor: refuse at p95 >= 15 ms or debt, shed at p95 >= 20 ms or debt >= 2, relax below 10 ms after 5 s (ADR-0097; provisional until 10.9d) | +| Overload response | shed interest, refuse new sessions before tick debt | budget | 10.9a load governor: refuse at p95 >= 15 ms or debt >= 2, shed at p95 >= 20 ms or debt >= 4, relax below 10 ms after 5 s (ADR-0097; provisional until 10.9d) | Determinism: authoritative state is exact given the checkpoint and committed per-tick inputs; timing, delivery, prediction, interpolation and presentation are tolerant diff --git a/src/server/load_governor.cpp b/src/server/load_governor.cpp index 377ad32..45b33e0 100644 --- a/src/server/load_governor.cpp +++ b/src/server/load_governor.cpp @@ -42,8 +42,8 @@ bool LoadGovernor::observe(std::uint64_t work_us, std::uint64_t session_us, std: state_.session_p99_us = rank(sessions, n, 99); const auto before = std::pair{state_.refusing, state_.shed}; - const bool hot = state_.p95_us >= policy_.refuse_us || tick_debt > 0; - const bool overloaded = state_.p95_us >= policy_.shed_us || tick_debt >= 2; + const bool hot = state_.p95_us >= policy_.refuse_us || tick_debt >= refuse_debt_ticks; + const bool overloaded = state_.p95_us >= policy_.shed_us || tick_debt >= shed_debt_ticks; if (hot) state_.refusing = true; if (overloaded && state_.shed < policy_.max_shed && diff --git a/tests/fixtures/server_process.hpp b/tests/fixtures/server_process.hpp index a46542a..fe4cb7a 100644 --- a/tests/fixtures/server_process.hpp +++ b/tests/fixtures/server_process.hpp @@ -119,8 +119,9 @@ class Child { for (; seen < lines_.size(); ++seen) if (want(lines_[seen])) return true; - const auto left = - std::chrono::duration_cast(deadline - Clock::now()); + // Rounded up: a short pump (1 ms) must still read what is waiting, or a chatty + // child blocks on its full stdout pipe. + const auto left = std::chrono::ceil(deadline - Clock::now()); if (left.count() <= 0 || !read_some(static_cast(left.count()))) return false; } diff --git a/tests/ops_tests.cpp b/tests/ops_tests.cpp index 3869f7d..d2baa4d 100644 --- a/tests/ops_tests.cpp +++ b/tests/ops_tests.cpp @@ -189,10 +189,11 @@ void load_governor() { // Down to 0 after another 5 s, then logins after 5 s more. CHECK(run(100, 4'000).shed == 0 && g.state().refusing); CHECK(run(99, 4'000).refusing && !run(1, 4'000).refusing); - // Tick debt alone: refuse at 1, shed at 2, even with light work. - CHECK(run(1, 1'000, 1).refusing && g.state().shed == 0); - CHECK(run(1, 1'000, 2).shed == 1); - CHECK(!g.observe(1'000, 250, 2, now + 10)); // Within the step interval: no change. + // Tick debt alone, even with light work: one tick is jitter, 2 refuse, 4 shed. + CHECK(!run(1, 1'000, 1).refusing); + CHECK(run(1, 1'000, 2).refusing && g.state().shed == 0); + CHECK(run(1, 1'000, 4).shed == 1); + CHECK(!g.observe(1'000, 250, 4, now + 10)); // Within the step interval: no change. CHECK(g.state().session_p95_us == 1'000); // The window is mostly 4 ms ticks. // Debt gone and calm (from the next sample): recovers. CHECK(run(100, 1'000).shed == 1 && run(1, 1'000).shed == 0); diff --git a/tests/region_m10_session_tests.cpp b/tests/region_m10_session_tests.cpp index 3a38570..e86de18 100644 --- a/tests/region_m10_session_tests.cpp +++ b/tests/region_m10_session_tests.cpp @@ -356,7 +356,10 @@ void end_to_end() { const auto o3 = outcome_of(*server, *c, 3, true); CHECK(o3 && must(o3).result == ActionResult::committed_no_effect && !record(alice, 3).empty()); - // The headless probe: login, intentions, actions and a query, then a clean exit. + // The headless probe: login, intentions, actions and a query, then a clean exit. After the + // restarts the load governor (ADR-0097) may still refuse logins while the host catches up; + // the probe does not retry, so wait until it accepts. + CHECK(until(*server, *c, [&] { return metric(server->ops, "evo_m10_refusing") == 0u; })); const auto probe = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-session-probe"; Child p{{probe.string(), "run", "--connect", "127.0.0.1:" + std::to_string(server->port), @@ -381,9 +384,18 @@ void end_to_end() { if (p.printed("done") || p.printed("timeout") || p.printed("closed")) code = p.wait_exit(10s); } - CHECK(code == 0 && p.printed("welcome ") && p.printed("intention 1 0") && - p.printed("outcome 10 1 ") && p.printed("outcome 11 1 ") && p.printed("snapshot ") && - p.printed("snapshot-bytes ")); + const bool probed = code == 0 && p.printed("welcome ") && p.printed("intention 1 0") && + p.printed("outcome 10 1 ") && p.printed("outcome 11 1 ") && + p.printed("snapshot ") && p.printed("snapshot-bytes "); + if (!probed) { + for (const auto& line : p.lines()) + if (!starts(line, "snapshot ") && !starts(line, "marker ")) + std::printf("probe: %s\n", line.c_str()); + for (const auto& line : server->child->lines()) + if (starts(line, "m10 load") || starts(line, "m10 refused") || starts(line, "phase")) + std::printf("server: %s\n", line.c_str()); + } + CHECK(probed); CHECK(!record(bob, 10).empty() && !record(bob, 11).empty()); // The probe's marker (ADR-0094): its intentions (100, 0) were applied and moved it +x. std::vector> marks; // tick, x, y, applied @@ -402,9 +414,85 @@ void end_to_end() { CHECK(server->child->printed("m10 opened ") && server->child->printed("m10 committed ") && server->child->printed("m10 motion area ")); } +// M10.9b (ADR-0097): evo-load against a real evo-server, two steps with the region's +// metrics. A smoke test of the tool and the server under concurrent sessions, not +// qualification (only evolution-dev runs count): it checks that every bot logs in, that +// every step measures responses, outcomes and snapshots for its bots, and that the report +// is complete. The lines are kept in the log as a development reference. +void load_smoke() { + Setup s{152, 0xC6}; + Server server{s}; + const auto tool = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-load"; + Child load{{tool.string(), + "--connect", + "127.0.0.1:" + std::to_string(server.port), + "--server-name", + name, + "--ca", + (s.region.root / "ca.pem").string(), + "--issuer-key", + (s.region.root / "issuer.key").string(), + "--world", + "7", + "--frame", + std::to_string(region_identity.frame), + "--region", + "152", + "--steps", + "4,8", + "--hold", + "8", + "--warmup", + "3", + "--ramp-ms", + "50", + "--threads", + "2", + "--lifetime", + "900", + "--metrics", + "127.0.0.1:" + std::to_string(server.ops), + "--per-client"}}; + int code = -1; + for (const auto end = Clock::now() + 120s; Clock::now() < end && code < 0;) { + server.pump(); + load.pump(5ms); + if (load.printed("load result ")) + code = load.wait_exit(30s); + } + for (const auto& line : load.lines()) + if (starts(line, "load ")) + std::printf("%s\n", line.c_str()); + CHECK(code == 0); + std::size_t steps = 0; + for (const auto& line : load.lines()) { + if (!starts(line, "load step ")) + continue; + ++steps; + // Every bot logged in once: 4 in the first step, 4 more in the second. + CHECK((starts(line, "load step N=4 established=4 logins=4 ") || + starts(line, "load step N=8 established=8 logins=4 ")) && + line.find(" drops=0 login_failures=0 refused=0 ") != std::string::npos); + } + const auto has = [&](std::string_view prefix) { + return std::ranges::count_if(load.lines(), [&](const std::string& l) { + return starts(l, std::string{prefix}); + }) == 2; + }; + CHECK(steps == 2 && has("load response_ms ") && has("load outcome_ms ") && + has("load snapshots ") && has("load server ") && has("load verdict ")); + CHECK(std::ranges::none_of(load.lines(), [](const std::string& l) { + return starts(l, "load response_ms") && l.find(" n=0 ") != std::string::npos; + })); + CHECK(std::ranges::count_if( + load.lines(), [](const std::string& l) { return starts(l, "load client "); }) == 12); + server.child->signal(SIGTERM); + CHECK(server.child->wait_exit(30s) == 0); +} } // namespace void run_region_m10_session_tests() { end_to_end(); + load_smoke(); std::puts("region m10 session integration passed"); } diff --git a/tools/network/load.cpp b/tools/network/load.cpp new file mode 100644 index 0000000..d0f752a --- /dev/null +++ b/tools/network/load.cpp @@ -0,0 +1,764 @@ +// M10.9b multi-client load generator (ADR-0097). Development and qualification tool: only +// runs against the evolution-dev reference worker count as evidence (10.9d); local runs are +// smoke tests and development references. +// +// evo-load --connect HOST:PORT --server-name NAME --ca FILE --issuer-key FILE +// --world N --frame N --region N [--steps N[,N]...] [--hold S] [--warmup S] +// [--ramp-ms MS] [--threads T] [--seed X] [--lifetime S] [--metrics HOST:PORT] +// [--response-budget-ms MS] [--per-client] [--expect-established] +// +// Each simulated client (bot) is a RegionSessionClient with its own account (from --seed and +// its index) logging in with a dev grant signed here (the issuer key never leaves this +// process). Bots run on T worker threads (each owns a slice). Every server tick a bot sends +// one intention for tick + its input lead (a random walk turning every 2 s) and predicts its +// probe marker; every 2 s it sends a probe action; every 10 s it moves its interest by a +// cell (a fresh baseline). A closed or refused bot logs in again after 2 s with the next +// serial. +// +// Steps: the bot count rises to each step's N (one login every --ramp-ms), waits --warmup +// seconds, then measures for --hold seconds. Bots stay connected into the next step. Per +// step it prints: +// load step N=.. established=.. logins=.. drops=.. login_failures=.. refused=.. seconds=.. +// (drops: established sessions closed; login failures: closed before Welcome, including +// transport refusals at the connection limit; refused: closed with capacity or +// rate_limited, either way. Logins, drops and failures include the step's ramp and warmup; +// everything else is the hold) +// load response_ms p50/p95/p99/max=.. (input -> first own-marker snapshot applying it) +// client_p95 min/med/max=.. +// load receipt_ms ... (intention -> receipt) +// load outcome_ms ... (action -> final outcome) +// load login_ms ... (connect -> Welcome) +// load correction_mm ... (prediction error per authoritative marker) +// load snapshots hz min/med/max=.. gap_ticks p99/max=.. kib_s med/max=.. +// intents=.. refused_intents=.. +// load client_cpu_pct=.. (this process, all threads; Linux) +// load server ... (with --metrics: the region's evo_m10_* gauges at step end) +// load starved N [indices] (among bots connected during the hold: snapshot rate < 80 % of +// the median, or a response p95 over both twice the median and the budget) +// load verdict N=.. within_budget=0|1 [reasons] +// With --per-client each bot's own line follows its step. --expect-established exits 1 +// unless every step ended with all its bots established and every step saw snapshots and +// final outcomes (the smoke test). +#include "evo/client/motion_prediction.hpp" +#include "evo/client/region_session.hpp" +#include "evo/network/session_grant.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifndef _WIN32 +#include +#include +#include +#include +#include +#endif + +namespace { +using namespace evo; +using namespace evo::network; +using Clock = std::chrono::steady_clock; +using std::chrono::milliseconds; + +std::optional number(std::string_view v) { + std::uint64_t n = 0; + const auto [end, ec] = std::from_chars(v.data(), v.data() + v.size(), n); + if (ec != std::errc{} || end != v.data() + v.size()) + return std::nullopt; + return n; +} +std::string read(const std::filesystem::path& path) { + std::ifstream file{path, std::ios::binary}; + return {std::istreambuf_iterator{file}, std::istreambuf_iterator{}}; +} +std::uint64_t now_seconds() { + return static_cast(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count()); +} +std::uint64_t micros(Clock::duration d) { + return static_cast(std::max( + 0, std::chrono::duration_cast(d).count())); +} +// Nearest rank over a copy. +std::uint64_t percentile(std::vector v, double p) { + if (v.empty()) + return 0; + std::ranges::sort(v); + const auto rank = static_cast(std::ceil(p * static_cast(v.size()))); + return v.at(std::clamp(rank, 1, v.size()) - 1); +} +double ms(std::uint64_t us) { + return static_cast(us) / 1000.0; +} + +struct Options { + std::string host, server_name, ca, issuer_key, metrics_host; + std::uint16_t port{}, metrics_port{}; + std::uint64_t world{}, frame{}, region{}, seed{1}, lifetime{900}; + std::vector steps{20}; + std::uint64_t hold_s{30}, warmup_s{5}, ramp_ms{100}, threads{4}, response_budget_ms{150}; + bool per_client{}, expect_established{}; +}; + +// What one bot measured since the last take. +struct Samples { + std::vector response_us, receipt_us, outcome_us, login_us, correction_um, + gap_ticks; + std::uint64_t snapshots{}, bytes{}, intents{}, refused_intents{}, logins{}, drops{}, + login_failures{}, refused{}; + std::uint64_t connected_polls{}; // Polls while established (starvation is judged on these). +}; +void append(std::vector& to, const std::vector& from) { + to.insert(to.end(), from.begin(), from.end()); +} + +struct Bot { + std::size_t index{}; + AccountId account; + std::uint64_t serial{}; + std::unique_ptr c; + std::atomic active{false}, established{false}; + Clock::time_point connect_at, retry_at, next_action, next_interest, next_turn; + client::MotionPredictor predictor; + client::InputLead lead; + std::uint64_t last_tick{}, last_view{}, last_bytes{}, next_action_id{1}; + std::uint64_t applied{}; // Newest applied intention sequence seen in an own marker. + std::map receipt_pending, response_pending, outcome_pending; + std::map intent_tick; + std::int16_t vx{}, vy{}; + std::optional home; + int interest_step{}; + std::mt19937_64 rng; + Samples s; +}; + +class Load { + public: + Load(Options o, GrantSigningKey key, std::string address) + : o_{std::move(o)}, key_{std::move(key)}, address_{std::move(address)}, + authority_{read(o_.ca)} { + const auto most = *std::ranges::max_element(o_.steps); + bots_.reserve(most); + for (std::size_t i = 0; i < most; ++i) { + auto b = std::make_unique(); + b->index = i; + for (std::size_t k = 0; k < 8; ++k) { + b->account.bytes.at(k) = static_cast((o_.seed >> (8U * k)) & 0xFFU); + b->account.bytes.at(8 + k) = + static_cast(((i + 1) >> (8U * k)) & 0xFFU); + } + b->serial = now_seconds() * 64; // Later runs supersede earlier ones. + b->rng.seed(o_.seed * 1'000'003 + i); + bots_.push_back(std::move(b)); + } + const auto threads = std::clamp(o_.threads, 1, 64); + locks_ = std::vector(threads); + for (std::size_t t = 0; t < threads; ++t) + workers_.emplace_back([this, t, threads] { work(t, threads); }); + } + Load(const Load&) = delete; + Load& operator=(const Load&) = delete; + ~Load() { + stop_ = true; + for (auto& w : workers_) + w.join(); + } + void activate(std::size_t n) { + for (std::size_t i = 0; i < n && i < bots_.size(); ++i) { + if (bots_[i]->active.exchange(true)) + continue; + std::this_thread::sleep_for(milliseconds{o_.ramp_ms}); + } + } + // Every active bot's samples since the last take (cleared). + std::vector take(std::size_t n) { + std::vector out(n); + for (std::size_t t = 0; t < locks_.size(); ++t) { + const std::scoped_lock lock{locks_[t]}; + for (std::size_t i = t; i < n; i += locks_.size()) + out[i] = std::exchange(bots_[i]->s, Samples{}); + } + return out; + } + [[nodiscard]] std::size_t established(std::size_t n) const { + return static_cast( + std::count_if(bots_.begin(), bots_.begin() + static_cast(n), + [](const auto& b) { return b->established.load(); })); + } + + private: + void work(std::size_t t, std::size_t threads) { + while (!stop_) { + { + const std::scoped_lock lock{locks_[t]}; + const auto now = Clock::now(); + for (std::size_t i = t; i < bots_.size(); i += threads) + if (bots_[i]->active) + step(*bots_[i], now); + } + std::this_thread::sleep_for(milliseconds{2}); + } + const std::scoped_lock lock{locks_[t]}; + for (std::size_t i = t; i < bots_.size(); i += threads) + if (auto& c = bots_[i]->c; c && c->state() == client::RegionSessionState::established) + c->close(); + for (int k = 0; k < 50; ++k) { // Let the closes flush. + for (std::size_t i = t; i < bots_.size(); i += threads) + if (bots_[i]->c) + bots_[i]->c->poll(Clock::now()); + std::this_thread::sleep_for(milliseconds{2}); + } + } + void login(Bot& b, Clock::time_point now) { + GrantClaims claims; + claims.account = b.account; + claims.serial = ++b.serial; + claims.world = {o_.world, o_.frame}; + claims.region = o_.region; + claims.issued_at = now_seconds(); + claims.expires_at = claims.issued_at + o_.lifetime; + claims.grant_id = (b.serial << 16U) ^ (b.index + 1); + const auto grant = key_.sign(claims); + client::RegionSessionClientConfig config; + config.transport.server_address = address_; + config.transport.port = o_.port; + config.transport.server_name = o_.server_name; + config.transport.authority_pem = authority_; + config.transport.authority_pem_file = o_.ca; + if (!grant) { + b.retry_at = now + std::chrono::seconds{2}; + return; + } + config.grant = *grant; + auto made = client::RegionSessionClient::connect(std::move(config), now); + if (!made) { + b.retry_at = now + std::chrono::seconds{2}; + return; + } + b.c = std::move(*made); + b.connect_at = now; + b.last_tick = b.last_view = b.last_bytes = b.applied = 0; + b.receipt_pending.clear(); + b.response_pending.clear(); + b.outcome_pending.clear(); + b.intent_tick.clear(); + b.predictor.reset(); + b.lead.reset(); + b.home.reset(); + } + void step(Bot& b, Clock::time_point now) { + if (!b.c) { + if (now >= b.retry_at) + login(b, now); + return; + } + auto& c = *b.c; + c.poll(now); + if (c.state() == client::RegionSessionState::closed) { + if (b.established.exchange(false)) + ++b.s.drops; + else + ++b.s.login_failures; + if (c.close_reason() == SessionCloseReason::capacity || + c.close_reason() == SessionCloseReason::rate_limited) + ++b.s.refused; + b.c.reset(); + b.retry_at = now + std::chrono::seconds{2} + + milliseconds{std::uniform_int_distribution{0, 500}(b.rng)}; + return; + } + if (c.state() != client::RegionSessionState::established) + return; + ++b.s.connected_polls; + if (!b.established.exchange(true)) { + ++b.s.logins; + b.s.login_us.push_back(micros(now - b.connect_at)); + b.next_action = now + milliseconds{std::uniform_int_distribution{0, 2000}(b.rng)}; + b.next_interest = now + std::chrono::seconds{10}; + } + // Receipts: RTT, lead and refusals. + for (const auto& r : c.take_intention_receipts()) { + if (const auto it = b.receipt_pending.find(r.sequence); it != b.receipt_pending.end()) { + b.s.receipt_us.push_back(micros(now - it->second)); + b.receipt_pending.erase(it); + } + const auto tick = b.intent_tick.find(r.sequence); + if (r.status == AdmitStatus::admitted) { + if (tick != b.intent_tick.end()) + b.lead.admitted(tick->second, r.server_tick); + } else { + ++b.s.refused_intents; + b.predictor.rejected(r.sequence); + b.response_pending.erase(r.sequence); + if (r.status == AdmitStatus::outside_window) + b.lead.too_late(); + } + if (tick != b.intent_tick.end()) + b.intent_tick.erase(tick); + } + // Snapshots: cadence, own marker (responsiveness, prediction errors), bytes. + if (const auto* v = c.view(); v && v->tick != b.last_view) { + if (b.last_view != 0 && v->tick > b.last_view) + b.s.gap_ticks.push_back(v->tick - b.last_view); + b.last_view = v->tick; + ++b.s.snapshots; + if (!b.home && !v->environment.empty()) { + std::int64_t x0 = v->environment.front().voxel.x, x1 = x0; + std::int64_t y0 = v->environment.front().voxel.y, y1 = y0; + for (const auto& cell : v->environment) { + x0 = std::min(x0, cell.voxel.x); + x1 = std::max(x1, cell.voxel.x); + y0 = std::min(y0, cell.voxel.y); + y1 = std::max(y1, cell.voxel.y); + } + b.home = SessionInterest{(x0 + x1) / 2, (y0 + y1) / 2, v->slice_z, 7}; + } + for (const auto& m : v->markers) { + if (!m.own) + continue; + b.predictor.authoritative({v->tick, m.applied_sequence, m.state, m.area}); + if (m.applied_sequence > b.applied) { + b.applied = m.applied_sequence; + while (!b.response_pending.empty() && + b.response_pending.begin()->first <= b.applied) { + b.s.response_us.push_back(micros(now - b.response_pending.begin()->second)); + b.response_pending.erase(b.response_pending.begin()); + } + } + } + } + for (const auto e : b.predictor.take_errors()) + b.s.correction_um.push_back(e); + const auto bytes = c.snapshot_bytes(); + b.s.bytes += bytes - std::min(bytes, b.last_bytes); + b.last_bytes = bytes; + // Outcomes: final ones only. + for (const auto& out : c.take_action_outcomes()) + if (out.result == ActionResult::committed_no_effect || + out.result == ActionResult::not_committed) + if (const auto it = b.outcome_pending.find(out.action_id); + it != b.outcome_pending.end()) { + b.s.outcome_us.push_back(micros(now - it->second)); + b.outcome_pending.erase(it); + } + (void)c.take_action_receipts(); + // Once per server tick: an intention, a prediction, sometimes an action or interest. + const auto tick = c.server_tick(); + if (tick == 0 || tick == b.last_tick) + return; + b.last_tick = tick; + if (now >= b.next_turn) { + const double angle = std::uniform_real_distribution{0, 6.283185307}(b.rng); + b.vx = static_cast(std::lround(700.0 * std::cos(angle))); + b.vy = static_cast(std::lround(700.0 * std::sin(angle))); + b.next_turn = now + std::chrono::seconds{2}; + } + const auto target = tick + b.lead.lead(); + if (!b.predictor.full()) + if (const auto seq = c.send_intention(target, b.vx, b.vy)) { + ++b.s.intents; + (void)b.predictor.input({*seq, target, {b.vx, b.vy}}); + b.receipt_pending[*seq] = now; + b.response_pending[*seq] = now; + b.intent_tick[*seq] = target; + for (auto* pending : {&b.receipt_pending, &b.response_pending}) + while (pending->size() > 256) + pending->erase(pending->begin()); + while (b.intent_tick.size() > 256) + b.intent_tick.erase(b.intent_tick.begin()); + } + (void)b.predictor.predict(target); + (void)b.predictor.take_correction(); + if (now >= b.next_action) { + const auto id = b.next_action_id++; + if (c.send_action(id, ActionKind::probe, tick + 3)) { + b.outcome_pending[id] = now; + while (b.outcome_pending.size() > 64) + b.outcome_pending.erase(b.outcome_pending.begin()); + } + b.next_action = now + std::chrono::seconds{2}; + } + if (now >= b.next_interest && b.home) { + auto in = *b.home; + in.x += (b.interest_step++ % 2 == 0) ? 1 : 0; // Alternate: a fresh baseline each. + (void)c.set_interest(in); + b.next_interest = now + std::chrono::seconds{10}; + } + } + + Options o_; + GrantSigningKey key_; + std::string address_, authority_; + std::vector> bots_; + std::vector locks_; + std::vector workers_; + std::atomic stop_{false}; +}; + +// The region's /metrics (evo_m10_* and evo_region_tick_debt), unlabelled series only. +std::map scrape(const Options& o) { + std::map out; +#ifndef _WIN32 + if (o.metrics_port == 0) + return out; + const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); + if (fd < 0) + return out; + sockaddr_in a{}; + a.sin_family = AF_INET; + a.sin_port = htons(o.metrics_port); + timeval timeout{3, 0}; + ::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout); + std::string raw; + if (::inet_pton(AF_INET, o.metrics_host.c_str(), &a.sin_addr) == 1 && + ::connect(fd, reinterpret_cast(&a), sizeof a) == 0) { + const std::string request = "GET /metrics HTTP/1.1\r\nHost: load\r\n\r\n"; + if (::send(fd, request.data(), request.size(), MSG_NOSIGNAL) == + static_cast(request.size())) { + std::array chunk{}; + for (ssize_t n = 0; (n = ::recv(fd, chunk.data(), chunk.size(), 0)) > 0;) + raw.append(chunk.data(), static_cast(n)); + } + } + ::close(fd); + std::size_t at = 0; + while (at < raw.size()) { + const auto end = std::min(raw.find('\n', at), raw.size()); + const std::string_view line{raw.data() + at, end - at}; + at = end + 1; + if (!line.starts_with("evo_m10_") && !line.starts_with("evo_region_tick_debt")) + continue; + const auto space = line.find(' '); + if (space == std::string_view::npos || line.find('{') != std::string_view::npos) + continue; + if (const auto v = number(line.substr(space + 1))) + out[std::string{line.substr(0, space)}] = *v; + } +#else + (void)o; +#endif + return out; +} +double cpu_seconds() { +#ifndef _WIN32 + rusage u{}; + ::getrusage(RUSAGE_SELF, &u); + return static_cast(u.ru_utime.tv_sec + u.ru_stime.tv_sec) + + (static_cast(u.ru_utime.tv_usec + u.ru_stime.tv_usec) / 1e6); +#else + return 0; +#endif +} + +void line(const char* name, const std::vector& pooled, + const std::vector& client_p95, double scale, const char* unit) { + const auto f = [&](std::uint64_t v) { return static_cast(v) / scale; }; + std::printf("load %s_%s p50/p95/p99/max=%.1f/%.1f/%.1f/%.1f n=%zu client_p95 " + "min/med/max=%.1f/%.1f/%.1f\n", + name, unit, f(percentile(pooled, 0.50)), f(percentile(pooled, 0.95)), + f(percentile(pooled, 0.99)), f(percentile(pooled, 1.0)), pooled.size(), + f(percentile(client_p95, 0.0)), f(percentile(client_p95, 0.5)), + f(percentile(client_p95, 1.0))); +} + +// One step's report; true when within budget. +bool report(const Options& o, std::size_t n, std::size_t established, + const std::vector& all, double seconds, double cpu, + const std::map& server) { + Samples pooled; + std::vector response95, receipt95, outcome95, login95, correction95, rates, kib; + for (const auto& s : all) { + append(pooled.response_us, s.response_us); + append(pooled.receipt_us, s.receipt_us); + append(pooled.outcome_us, s.outcome_us); + append(pooled.login_us, s.login_us); + append(pooled.correction_um, s.correction_um); + append(pooled.gap_ticks, s.gap_ticks); + pooled.snapshots += s.snapshots; + pooled.intents += s.intents; + pooled.refused_intents += s.refused_intents; + pooled.logins += s.logins; + pooled.drops += s.drops; + pooled.login_failures += s.login_failures; + pooled.refused += s.refused; + response95.push_back(percentile(s.response_us, 0.95)); + receipt95.push_back(percentile(s.receipt_us, 0.95)); + outcome95.push_back(percentile(s.outcome_us, 0.95)); + if (!s.login_us.empty()) + login95.push_back(percentile(s.login_us, 0.95)); + correction95.push_back(percentile(s.correction_um, 0.95)); + rates.push_back(static_cast( + std::lround(1000.0 * static_cast(s.snapshots) / std::max(seconds, 0.001)))); + kib.push_back(static_cast( + std::lround(static_cast(s.bytes) / 1024.0 / std::max(seconds, 0.001)))); + } + std::printf("load step N=%zu established=%zu logins=%llu drops=%llu login_failures=%llu " + "refused=%llu seconds=%.1f\n", + n, established, static_cast(pooled.logins), + static_cast(pooled.drops), + static_cast(pooled.login_failures), + static_cast(pooled.refused), seconds); + line("response", pooled.response_us, response95, 1000.0, "ms"); + line("receipt", pooled.receipt_us, receipt95, 1000.0, "ms"); + line("outcome", pooled.outcome_us, outcome95, 1000.0, "ms"); + line("login", pooled.login_us, login95, 1000.0, "ms"); + line("correction", pooled.correction_um, correction95, 1000.0, "mm"); + std::printf("load snapshots hz min/med/max=%.2f/%.2f/%.2f gap_ticks p99/max=%llu/%llu " + "kib_s med/max=%llu/%llu intents=%llu refused_intents=%llu\n", + static_cast(percentile(rates, 0.0)) / 1000.0, + static_cast(percentile(rates, 0.5)) / 1000.0, + static_cast(percentile(rates, 1.0)) / 1000.0, + static_cast(percentile(pooled.gap_ticks, 0.99)), + static_cast(percentile(pooled.gap_ticks, 1.0)), + static_cast(percentile(kib, 0.5)), + static_cast(percentile(kib, 1.0)), + static_cast(pooled.intents), + static_cast(pooled.refused_intents)); + std::printf("load client_cpu_pct=%.0f\n", 100.0 * cpu / std::max(seconds, 0.001)); + if (!server.empty()) { + std::printf("load server"); + for (const auto& [k, v] : server) + std::printf(" %s=%llu", k.c_str(), static_cast(v)); + std::printf("\n"); + } + // Starvation: throughput must not hide a client left behind. Judged among the bots that + // were connected during the hold (the others are reported as not established). + std::vector connected_rates, connected_response; + for (std::size_t i = 0; i < all.size(); ++i) + if (all[i].connected_polls > 0) { + connected_rates.push_back(rates[i]); + connected_response.push_back(response95[i]); + } + const auto median_rate = percentile(connected_rates, 0.5); + const auto median_response = percentile(connected_response, 0.5); + const auto budget_us = o.response_budget_ms * 1000; + std::vector starved; + for (std::size_t i = 0; i < all.size(); ++i) + if (all[i].connected_polls > 0 && + (rates[i] * 5 < median_rate * 4 || + (response95[i] > 2 * median_response && response95[i] > budget_us))) + starved.push_back(i); + std::printf("load starved %zu", starved.size()); + for (const auto i : starved) + std::printf(" %zu", i); + std::printf("\n"); + std::vector why; + if (established < n) + why.emplace_back("not-established"); + if (pooled.refused > 0 || pooled.login_failures > 0) + why.emplace_back("refused"); + if (pooled.drops > 0) + why.emplace_back("drops"); + if (!starved.empty()) + why.emplace_back("starved"); + if (pooled.response_us.empty() || percentile(pooled.response_us, 0.95) > budget_us) + why.emplace_back("response"); + if (pooled.outcome_us.empty() || percentile(pooled.outcome_us, 0.95) > 1'500'000) + why.emplace_back("outcome"); + const auto get = [&](const char* k) { + const auto it = server.find(k); + return it == server.end() ? 0 : it->second; + }; + if (get("evo_m10_tick_work_p99_microseconds") > 25'000) + why.emplace_back("tick-work"); + if (get("evo_m10_refusing") != 0 || get("evo_m10_shed_level") != 0) + why.emplace_back("governor"); + if (get("evo_region_tick_debt") != 0) + why.emplace_back("tick-debt"); + std::printf("load verdict N=%zu within_budget=%d", n, why.empty() ? 1 : 0); + for (const auto& w : why) + std::printf(" %s", w.c_str()); + std::printf("\n"); + if (o.per_client) + for (std::size_t i = 0; i < all.size(); ++i) + std::printf("load client %zu snapshots=%llu response_p95_ms=%.1f receipt_p95_ms=%.1f " + "outcome_p95_ms=%.1f correction_p95_mm=%.1f drops=%llu " + "login_failures=%llu\n", + i, static_cast(all[i].snapshots), ms(response95[i]), + ms(receipt95[i]), ms(outcome95[i]), + static_cast(correction95[i]) / 1000.0, + static_cast(all[i].drops), + static_cast(all[i].login_failures)); + std::fflush(stdout); + return why.empty(); +} + +std::optional> endpoint(std::string_view v) { + const auto colon = v.rfind(':'); + if (colon == std::string_view::npos || colon == 0) + return std::nullopt; + const auto port = number(v.substr(colon + 1)); + if (!port || *port == 0 || *port > 65535) + return std::nullopt; + return std::pair{std::string{v.substr(0, colon)}, static_cast(*port)}; +} +std::optional> steps_of(std::string_view v) { + std::vector out; + while (!v.empty()) { + const auto comma = v.find(','); + const auto n = number(v.substr(0, comma)); + if (!n || *n == 0 || *n > 256 || (!out.empty() && *n < out.back())) + return std::nullopt; + out.push_back(*n); + v.remove_prefix(comma == std::string_view::npos ? v.size() : comma + 1); + } + if (out.empty()) + return std::nullopt; + return out; +} +} // namespace + +int main(int argc, char** argv) { + const std::vector args(argv + 1, argv + argc); + Options o; + bool ok = true; + for (std::size_t i = 0; i < args.size() && ok; ++i) { + const auto flag = args[i]; + if (flag == "--per-client") { + o.per_client = true; + continue; + } + if (flag == "--expect-established") { + o.expect_established = true; + continue; + } + if (i + 1 >= args.size()) { + ok = false; + break; + } + const auto value = args[++i]; + const auto n = number(value); + if (flag == "--connect") { + const auto e = endpoint(value); + ok = e.has_value(); + if (e) + std::tie(o.host, o.port) = *e; + } else if (flag == "--metrics") { + const auto e = endpoint(value); + ok = e.has_value(); + if (e) + std::tie(o.metrics_host, o.metrics_port) = *e; + } else if (flag == "--server-name") + o.server_name = value; + else if (flag == "--ca") + o.ca = value; + else if (flag == "--issuer-key") + o.issuer_key = value; + else if (flag == "--steps") { + const auto s = steps_of(value); + ok = s.has_value(); + if (s) + o.steps = *s; + } else if (n && flag == "--world") + o.world = *n; + else if (n && flag == "--frame") + o.frame = *n; + else if (n && flag == "--region") + o.region = *n; + else if (n && flag == "--seed" && *n != 0) + o.seed = *n; + else if (n && flag == "--lifetime" && *n >= 60 && *n <= 3600) + o.lifetime = *n; + else if (n && flag == "--hold" && *n >= 1 && *n <= 3600) + o.hold_s = *n; + else if (n && flag == "--warmup" && *n <= 600) + o.warmup_s = *n; + else if (n && flag == "--ramp-ms" && *n <= 10'000) + o.ramp_ms = *n; + else if (n && flag == "--threads" && *n >= 1 && *n <= 64) + o.threads = *n; + else if (n && flag == "--response-budget-ms" && *n >= 1) + o.response_budget_ms = *n; + else + ok = false; + } + if (!ok || o.host.empty() || o.server_name.empty() || o.ca.empty() || o.issuer_key.empty() || + o.world == 0 || o.frame == 0 || o.region == 0) { + std::fputs("usage: evo-load --connect HOST:PORT --server-name NAME --ca FILE " + "--issuer-key FILE --world N --frame N --region N [--steps N[,N]...] " + "[--hold S] [--warmup S] [--ramp-ms MS] [--threads T] [--seed X] " + "[--lifetime S] [--metrics HOST:PORT] [--response-budget-ms MS] " + "[--per-client] [--expect-established]\n", + stderr); + return 2; + } + const auto scalar = read(o.issuer_key); + auto key = GrantSigningKey::import( + DeploymentEnvironment::dev, 1, + std::span{reinterpret_cast(scalar.data()), scalar.size()}); + if (!key) { + std::puts("error issuer-key"); + return 2; + } + const auto address = quic::resolve_ipv4(o.host); + if (!address) { + std::puts("error resolve"); + return 2; + } + std::printf("load start steps=%zu hold=%llu warmup=%llu threads=%llu budget_ms=%llu\n", + o.steps.size(), static_cast(o.hold_s), + static_cast(o.warmup_s), + static_cast(o.threads), + static_cast(o.response_budget_ms)); + std::fflush(stdout); + bool smoke_ok = true; + std::optional sustainable, saturated; + { + Load load{o, std::move(*key), *address}; + for (const auto n : o.steps) { + load.activate(n); + std::this_thread::sleep_for(std::chrono::seconds{o.warmup_s}); + // The ramp's and warmup's logins (and closes) belong to this step; the rest of + // the warmup is discarded. + const auto warm = load.take(n); + const auto started = Clock::now(); + const auto cpu0 = cpu_seconds(); + std::this_thread::sleep_for(std::chrono::seconds{o.hold_s}); + auto samples = load.take(n); + for (std::size_t i = 0; i < n; ++i) { + append(samples[i].login_us, warm[i].login_us); + samples[i].logins += warm[i].logins; + samples[i].drops += warm[i].drops; + samples[i].login_failures += warm[i].login_failures; + samples[i].refused += warm[i].refused; + } + const auto seconds = std::chrono::duration(Clock::now() - started).count(); + const auto established = load.established(n); + const bool within = + report(o, n, established, samples, seconds, cpu_seconds() - cpu0, scrape(o)); + if (within && !saturated) + sustainable = n; + if (!within && !saturated) + saturated = n; + std::uint64_t snapshots = 0, outcomes = 0; + for (const auto& s : samples) { + snapshots += s.snapshots; + outcomes += s.outcome_us.size(); + } + smoke_ok = smoke_ok && established == n && snapshots > 0 && outcomes > 0; + } + } + std::printf("load result sustainable=%zu saturated_at=%zu\n", sustainable.value_or(0), + saturated.value_or(0)); + std::fflush(stdout); + return o.expect_established && !smoke_ok ? 1 : 0; +}