From ada77fd761e3fb3e1bef937bca35ea22940f00a3 Mon Sep 17 00:00:00 2001 From: dingo Date: Thu, 8 Oct 2026 08:38:13 -0400 Subject: [PATCH] Add M10.9d load drill harness and serialize Mbed TLS use ADR-0097. tools/ops/load-drill.sh (runbook docs/networking/M10_LOAD_DRILL.md) runs the evolution-dev reference qualification: synthetic inputs on for the run (the ADR-0073 drill-time override, restored on exit), bundle-mode evo-load bots through the public route, and each step's hold matched with the server's reports. evo-server logs "m10 load-report" every 10 s, since the agent cannot reach the ops port. evo-load gains --grants FILE --grants-per-bot C, exhausted=, hold_utc= and close_reasons=; evo-session-probe grants gains --accounts N. Finding: Mbed TLS/PSA are built without MBEDTLS_THREADING_C, and bots on different threads verifying their first connections at once were refused as untrusted (two per smoke run). One process-wide recursive crypto lock now guards every network-library entry point that touches them; the smoke went from 2/2 failing to 3/3 clean. quic-transport adds a concurrent-connect test. Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 --- apps/evo-server/m10_bridge.hpp | 19 +++ apps/evo-session-probe/main.cpp | 72 ++++---- .../region/dev-session-route/listener.yaml | 2 +- docs/ROADMAP.md | 3 +- docs/STATUS.md | 9 +- .../0097-m10-load-and-abuse-qualification.md | 33 ++++ docs/engineering/VALIDATION.md | 40 +++++ docs/networking/M10_LOAD_DRILL.md | 74 ++++++++ include/evo/network/crypto_lock.hpp | 18 ++ src/network/tls/handoff_ticket.cpp | 11 ++ src/network/tls/session_grant.cpp | 12 ++ src/network/transport/quic_endpoint.cpp | 2 + tests/quic_transport.cpp | 62 +++++++ tests/region_m10_session_tests.cpp | 21 ++- tools/network/load.cpp | 140 +++++++++++---- tools/ops/load-drill.sh | 161 ++++++++++++++++++ 16 files changed, 609 insertions(+), 70 deletions(-) create mode 100644 docs/networking/M10_LOAD_DRILL.md create mode 100644 include/evo/network/crypto_lock.hpp create mode 100644 tools/ops/load-drill.sh diff --git a/apps/evo-server/m10_bridge.hpp b/apps/evo-server/m10_bridge.hpp index 80f2dba..5ab4cde 100644 --- a/apps/evo-server/m10_bridge.hpp +++ b/apps/evo-server/m10_bridge.hpp @@ -296,6 +296,7 @@ template class M10Bridge { bool changed = false; for (std::uint64_t i = 0; i < ticks; ++i) changed = governor_.observe(per_tick, session_per_tick, s.tick_debt, now_us) || changed; + report(s, now_us); if (!changed || !governed_) return; const auto& load = governor_.state(); @@ -303,6 +304,23 @@ template class M10Bridge { std::printf("m10 load refusing=%d shed=%u p95_us=%llu\n", load.refusing ? 1 : 0, static_cast(load.shed), static_cast(load.p95_us)); } + // M10.9d (ADR-0097): every 10 s, the load as /metrics would show it, in the log (drills + // read it with `kubectl logs --timestamps` where the ops port is not reachable). + void report(const server::HostStatus& s, std::uint64_t now_us) { + if (reported_us_ != 0 && now_us - reported_us_ < 10'000'000) + return; + reported_us_ = now_us; + const auto& load = governor_.state(); + std::printf("m10 load-report sessions=%zu p95_us=%llu p99_us=%llu max_us=%llu " + "session_p95_us=%llu session_p99_us=%llu refusing=%d shed=%u tick_debt=%llu\n", + sessions_->sessions(), static_cast(load.p95_us), + static_cast(load.p99_us), + static_cast(load.max_us), + static_cast(load.session_p95_us), + static_cast(load.session_p99_us), + governed_ && load.refusing ? 1 : 0, governed_ ? load.shed : 0U, + static_cast(s.tick_debt)); + } static const processes::EnvironmentalCommunity& community(const processes::EnvironmentalCommunity& c) { return c; @@ -448,6 +466,7 @@ template class M10Bridge { std::unique_ptr actions_; server::LoadGovernor governor_{*server::LoadGovernor::create({})}; bool governed_{true}; + std::uint64_t reported_us_{}; std::chrono::nanoseconds busy_{}, session_busy_{}; std::uint64_t measured_tick_{}; std::unique_ptr probes_; // Handoff mode only. diff --git a/apps/evo-session-probe/main.cpp b/apps/evo-session-probe/main.cpp index 4f8fd7e..5fe8f81 100644 --- a/apps/evo-session-probe/main.cpp +++ b/apps/evo-session-probe/main.cpp @@ -15,6 +15,9 @@ // interleaved in serial order, so a client crossing between regions always has a newer // serial for the next one (serials must rise world-wide). A handoff key is a key pair // from `keys` (issuer.key for --m10-handoff-key, issuer.pub for --m10-neighbour). +// M10.9d (ADR-0097): --accounts N (1..256) writes the same for N accounts, account by +// account: account i is --account with its last two bytes replaced by i + 1 (big +// endian). This is the load bundle `evo-load --grants FILE --grants-per-bot COUNT` uses. // evo-session-probe run --connect HOST:PORT --server-name NAME --ca FILE --issuer-key FILE // --world N --frame N --region N --account HEX32 --serial N [--lifetime S] // [--intents N] [--actions N --action-base ID] [--query ID]... [--timeout-ms N] @@ -206,14 +209,14 @@ bool write_new(const std::filesystem::path& path, std::span return true; } -int grants(const RunOptions& o, const std::string& out, std::uint64_t count) { +int grants(const RunOptions& o, const std::string& out, std::uint64_t count, + std::uint64_t accounts) { const auto key = issuer(o.issuer_key); if (!key || !o.account) { std::puts("error issuer-key"); return 2; } GrantClaims claims; - claims.account = *o.account; claims.world = {o.world, o.frame}; claims.issued_at = now_seconds(); claims.expires_at = claims.issued_at + o.lifetime; @@ -221,31 +224,39 @@ int grants(const RunOptions& o, const std::string& out, std::uint64_t count) { std::vector regions{o.region}; regions.insert(regions.end(), o.more_regions.begin(), o.more_regions.end()); if (count * regions.size() > 64) { - std::puts("error count (at most 64 grants in all)"); + std::puts("error count (at most 64 grants per account)"); return 2; } std::vector bundle; - std::uint64_t serial = first; - for (std::uint64_t i = 0; i < count; ++i) - for (const auto region : regions) { - claims.region = region; - claims.serial = serial++; - const auto grant = sign(*key, claims); - if (!grant) { - std::puts("error grant"); - return 2; - } - bundle.insert(bundle.end(), grant->begin(), grant->end()); + for (std::uint64_t a = 0; a < accounts; ++a) { + claims.account = *o.account; + if (accounts > 1) { // Account a: the last two bytes are a + 1. + claims.account.bytes.at(14) = static_cast(((a + 1) >> 8U) & 0xFFU); + claims.account.bytes.at(15) = static_cast((a + 1) & 0xFFU); } + std::uint64_t serial = first; + for (std::uint64_t i = 0; i < count; ++i) + for (const auto region : regions) { + claims.region = region; + claims.serial = serial++; + const auto grant = sign(*key, claims); + if (!grant) { + std::puts("error grant"); + return 2; + } + bundle.insert(bundle.end(), grant->begin(), grant->end()); + } + } count *= regions.size(); if (!write_new(out, bundle)) { std::puts("error out (exists or unwritable)"); return 1; } - std::printf("wrote %llu grants serial %llu..%llu valid %llu s\n", - static_cast(count), static_cast(first), - static_cast(first + count - 1), - static_cast(o.lifetime)); + std::printf( + "wrote %llu grants serial %llu..%llu valid %llu s accounts %llu\n", + static_cast(count) * accounts, static_cast(first), + static_cast(first + count - 1), + static_cast(o.lifetime), static_cast(accounts)); return 0; } @@ -430,7 +441,7 @@ int main(int argc, char** argv) { RunOptions o; const bool bundle = !args.empty() && args[0] == "grants"; std::string out; - std::uint64_t count = 1; + std::uint64_t count = 1, accounts = 1; if (bundle) o.lifetime = 300; // The region's default maximum. bool ok = !args.empty() && (args[0] == "run" || bundle) && args.size() % 2 == 1; @@ -489,26 +500,29 @@ int main(int argc, char** argv) { out = value; else if (bundle && n && flag == "--count" && *n >= 1 && *n <= 64) count = *n; + else if (bundle && n && flag == "--accounts" && *n >= 1 && *n <= 256) + accounts = *n; else ok = false; } if (bundle && ok && !o.issuer_key.empty() && o.account && o.world != 0 && o.frame != 0 && o.region != 0 && !out.empty() && o.lifetime >= 60 && o.host.empty()) - return grants(o, out, count); + return grants(o, out, count, accounts); const bool signs = o.grants_file.empty(); if (bundle || !ok || o.host.empty() || o.server_name.empty() || o.ca.empty() || (signs && (o.issuer_key.empty() || !o.account || o.world == 0 || o.frame == 0 || o.region == 0 || o.serial == 0)) || (!signs && !o.issuer_key.empty())) { - std::fputs("usage: evo-session-probe keys --out DIR | grants --issuer-key FILE " - "--world N --frame N --region N --account HEX32 --out FILE [--serial N] " - "[--count 1..64] [--lifetime 60..14400] | run --connect HOST:PORT " - "--server-name NAME --ca FILE --issuer-key FILE --world N --frame N " - "--region N --account HEX32 --serial N [--lifetime S] [--intents N] " - "[--actions N --action-base ID] [--query ID]... [--timeout-ms N] " - "[--interest X,Y,Z,R] [--snapshots N] | run --connect HOST:PORT " - "--server-name NAME --ca FILE --grants FILE [--grant N] [run options]\n", - stderr); + std::fputs( + "usage: evo-session-probe keys --out DIR | grants --issuer-key FILE " + "--world N --frame N --region N --account HEX32 --out FILE [--serial N] " + "[--count 1..64] [--accounts 1..256] [--lifetime 60..14400] | run --connect HOST:PORT " + "--server-name NAME --ca FILE --issuer-key FILE --world N --frame N " + "--region N --account HEX32 --serial N [--lifetime S] [--intents N] " + "[--actions N --action-base ID] [--query ID]... [--timeout-ms N] " + "[--interest X,Y,Z,R] [--snapshots N] | run --connect HOST:PORT " + "--server-name NAME --ca FILE --grants FILE [--grant N] [run options]\n", + stderr); return 2; } return run(o); diff --git a/deploy/kubernetes/region/dev-session-route/listener.yaml b/deploy/kubernetes/region/dev-session-route/listener.yaml index eee9882..083ae7c 100644 --- a/deploy/kubernetes/region/dev-session-route/listener.yaml +++ b/deploy/kubernetes/region/dev-session-route/listener.yaml @@ -12,7 +12,7 @@ spec: env: - {name: REGION_SESSION, value: "dev"} - {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"} - - {name: REGION_SESSION_WINDOW, value: "dev-20261008T112240-4ebe:1791472360"} + - {name: REGION_SESSION_WINDOW, value: "dev-20261008T123155-7759:1791476515"} - {name: REGION_M10_MAX_GRANT_LIFETIME, value: "14400"} ports: # QUIC (ADR-0090): UDP only; nothing listens on TCP 7400. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index ef0c9a0..8dedec1 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -549,7 +549,8 @@ smoke done; 10.9c-1 session fuzz target and smoke, in-process Hello and datagram slow peer and connect/disconnect storm with honest-service checks done; 10.9c-2 SIGKILL and drain lease loss with 24 bots (all back within 5 s of the successor) done; no standalone raw-UDP sender (abuse stays in-process/loopback; datagram-cap eviction cost analysed). -10.9d dev-cluster drill harness and reference results follow. +10.9d drill harness (load-drill.sh, server load-report, bundle-mode evo-load, Mbed TLS +crypto lock) done; the evolution-dev reference run follows. **Entry:** M9 client/session evidence and M7/M8 dev/stage durability. Read NETWORK_ARCHITECTURE, PERSISTENCE_ARCHITECTURE and ADR-0017. The M9 transport is diff --git a/docs/STATUS.md b/docs/STATUS.md index b94bbba..8222e17 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -123,8 +123,13 @@ refusing at 32 (work p95 about 14 ms), with responses about 103 ms and outcomes and its intention admitted (the dev region does not simulate outside drills). M10.9c-2 kills and drains a real evo-server under 24 `evo-load` bots. All of them log in to the successor within 5.0 s (SIGKILL, after a 3.5 s lease wait) or 2.4 s (drain), with no failed -logins, and responses stay at 106 ms p95. Next: 10.9d, the evolution-dev drill harness and -the reference-worker run. +logins, and responses stay at 106 ms p95. M10.9d adds the evolution-dev load drill +(`tools/ops/load-drill.sh`). It turns synthetic inputs on for the run, drives bundle-mode +`evo-load` bots through the public route, and matches each step with the server's 10 s +`m10 load-report` log lines. Building it found that Mbed TLS (built without its threading +layer) raced when bots verified connections on several threads; one process-wide crypto +lock fixed it. Next: the reference run, which needs a promoted image, a load bundle and +the maintainer's go-ahead. ## Previous gate - 2026-10-02 diff --git a/docs/adr/0097-m10-load-and-abuse-qualification.md b/docs/adr/0097-m10-load-and-abuse-qualification.md index 390be24..cb00172 100644 --- a/docs/adr/0097-m10-load-and-abuse-qualification.md +++ b/docs/adr/0097-m10-load-and-abuse-qualification.md @@ -222,3 +222,36 @@ snapshots. millisecond apart). That cost is bounded per datagram but can add up at very high rates from one peer. Making eviction O(1) without letting tombstones outgrow the queue bound is recorded as a follow-up, not done. + +## 10.9d drill harness (2026-10-08) +- **`tools/ops/load-drill.sh`** (runbook: `docs/networking/M10_LOAD_DRILL.md`): + 1. turns synthetic inputs on in evolution-dev for the run, the drill-time override of + ADR-0073 (never in Git); + 2. runs `evo-load` through the public route; + 3. matches each step's hold with the server's reports in it; + 4. restores `REGION_INPUTS` on exit, including on failure. +- **`evo-server`** logs `m10 load-report …` every 10 s: sessions, work and session-work + percentiles, governor state and tick debt. The agent cannot reach the ops port (no + port-forward, and the API-server proxy is refused), but it can read logs. +- **Bundle mode:** + - `evo-load --grants FILE --grants-per-bot C` logs bots in from a pre-issued bundle, so + the drill machine never holds the issuer key; + - `evo-session-probe grants --accounts N` writes such a bundle for N accounts; + - each step line now carries `exhausted=` and `hold_utc=`. +- **Evidence:** the M10 load smoke runs in bundle mode (8 accounts × 2 grants), and the + M10 process test checks the `load-report` line. The drill's summariser was tested on + synthetic logs; that test found a window-overlap bug, now fixed. +- **Finding: Mbed TLS is not thread-safe here.** Bundle mode made logins fast enough + that two bots on different threads verified their first connections at once. Both were + refused as `untrusted` (QUIC cause 3), which surfaced as `close_reasons=259:2`. Mbed TLS + and PSA are built without `MBEDTLS_THREADING_C`. Every network-library entry point that + touches them now holds one process-wide recursive lock + (`include/evo/network/crypto_lock.hpp`): the QUIC client's pinned check and the + session-grant and handoff-ticket keys. The smoke failed 2 of 2 runs before and passed 3 + of 3 after. `quic-transport` gains a concurrent-connect test (8 threads, 5 rounds); it + does not reliably reproduce the race, because PSA is already initialised there. +- `evo-load` step lines carry `close_reasons=REASON:COUNT,…`: session close reasons, or + 256 plus the QUIC cause when there was no session Close. +- **Not yet:** the reference run. It needs an image containing this change promoted to + dev, a load bundle made where the issuer key lives, and the maintainer's go-ahead, + because the drill restarts the dev region twice. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 529d38f..b22abff 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -5118,3 +5118,43 @@ Prevention: - DEPLOYMENT.md documents the Deployment-only sync symptom and the hook re-run. Executed: deploy policy check (0 failures) and selftest (46 cases, 0 failures). + +## 2026-10-08 — M10.9d load drill harness (ADR-0097) + +Implemented: +- `tools/ops/load-drill.sh` and its runbook `docs/networking/M10_LOAD_DRILL.md`: + synthetic inputs on for the run (restored on exit), `evo-load` through the public route, + and server reports matched to each hold. +- evo-server `m10 load-report` every 10 s. The agent cannot reach the ops port: no + port-forward, and the API-server proxy is refused. +- `evo-load --grants FILE --grants-per-bot C` with `exhausted=`, `hold_utc=` and + `close_reasons=`; `evo-session-probe grants --accounts N`. +- `include/evo/network/crypto_lock.hpp`: one recursive lock at every Mbed TLS/PSA entry + point of the network library (`MBEDTLS_THREADING_C` is off). + +Findings: +- Concurrent first verifications on different threads were refused as `untrusted`, two + per smoke run in bundle mode (2 of 2 runs). With the lock, 3 of 3 runs had no failures. +- The drill summariser, tested on synthetic logs, counted reports from after the hold + (the next step's ramp). Fixed: only reports wholly inside the hold count. + +Tests: +- M10 load smoke in bundle mode (8 accounts × 2 grants) with `exhausted=0`. +- The M10 process test checks `m10 load-report`. +- `quic-transport` `concurrent_clients`: 8 threads × 5 rounds all connect. It does not + reliably reproduce the race; the smoke does. +- The lease-loss test's close reasons read 24 × `shutdown` (drain) and 24 × idle + (SIGKILL). + +Executed: +- **Linux GCC 14:** presets `quic` and `network` 5/5; `postgres-check.sh` with QUIC, all 18 + suites. +- **Linux Clang 19:** clang-tidy 19 on `load.cpp`, the probe, `session_grant.cpp`, + `handoff_ticket.cpp`, `quic_endpoint.cpp`, `quic_transport.cpp`, evo-server `main.cpp` + with the bridge, and the M10 process tests (3 findings fixed); clang-format 19. +- **Windows MSVC 19.44:** presets `quic` 5/5 and `client-network` 15/15. +- **Drill script:** `bash -n`, plus the summariser on synthetic logs; shellcheck is not + installed here. + +Not done: the reference run on evolution-dev. It needs this image promoted, a load bundle +and the maintainer's go-ahead. diff --git a/docs/networking/M10_LOAD_DRILL.md b/docs/networking/M10_LOAD_DRILL.md new file mode 100644 index 0000000..a78e5dd --- /dev/null +++ b/docs/networking/M10_LOAD_DRILL.md @@ -0,0 +1,74 @@ +# M10 load drill (evolution-dev reference worker) + +ADR-0097 qualifies M10 load only on the evolution-dev reference worker. +`tools/ops/load-drill.sh` performs that run. It is a drill: it changes the live dev region +for the run's duration and restores it on exit, including on failure. It is never part of +CI. + +## What it changes + +- `REGION_INPUTS=development-v1` on `Deployment/evo-region` while it runs, the drill-time + override of ADR-0073. Synthetic inputs stay off in Git (deploy policy). On exit the + value goes back to empty, and the region restarts twice: once to enable, once to + restore. +- Nothing else: no Secrets, migrations, Jobs, SQL or Argo operations. While it runs, Argo + shows `Deployment/evo-region` OutOfSync, and Synced again after the restore. + +## Inputs + +In a tokens directory: +- `dev-ca.pem`: the current dev session CA, from the renewal. +- `load-grants.bin`: one bundle with grants for every bot, made where the dev issuer key + lives (the key never leaves that machine). For the default drill (up to 32 bots, 4 + logins each): + + ```bash + evo-session-probe grants --issuer-key issuer.key --world 100 --frame 9 --region 1 \ + --account --accounts 32 --count 4 --lifetime 14400 \ + --out load-grants.bin + ``` + + Account i is the base account with its last two bytes set to i + 1. Each bot uses its + own 4 grants in order, one per login, and a bot that has used all 4 stays out + (`exhausted=` in the report). Grants are valid for at most 4 hours. + +The session window (`REGION_SESSION_WINDOW`) must stay open until the drill ends; the +script refuses to start otherwise. The agent's kubectl access is enough: patch +`Deployment/evo-region`, read pods and logs. + +## Running + +```bash +EVO_LOAD=/path/to/evo-load tools/ops/load-drill.sh \ + [--steps 20,24,28,32] [--hold 60] [--warmup 10] [--grants-per-bot 4] +``` + +Defaults: steps 20, 24, 28 and 32 bots; 60 s holds; 10 s warmups; 4 grants per bot; +`void.aeternalisgames.com:7400`. The drill takes about 6 minutes plus two region +restarts. + +## Output + +- **evo-load's per-step lines** (see `tools/network/load.cpp`): client distributions + (response, receipt, outcome, login), snapshot cadence, starvation, `close_reasons` + (why sessions or logins ended) and a client-side verdict, then + `load result sustainable=N saturated_at=M`. +- **One `drill step` line per step**, from the server's `m10 load-report` log lines + (every 10 s, read with `kubectl logs --timestamps` because the ops port is not + reachable for the agent) that fall inside the step's hold: + - the worst work p99 and maximum; + - the session-layer p99 and its per-session share; + - the governor's refusing and shed state; + - tick debt; + - `server_within_budget`: work p99 ≤ 25 ms, session p99 ≤ 0.5 ms × sessions, no debt. + +The work directory (kept and printed at the end) holds `load.log` and `server.log`. + +## Interpreting + +A step qualifies when both evo-load's verdict and `server_within_budget` are 1. The +largest such step is the region's sustainable session count on the reference worker. +That number, with the first failing step and its reasons, goes into ADR-0097 and +`schemas/network/m10-budgets-v1.md`. If the governor engages (refusing or shed above 0), +that step is the overload response in action, and the thresholds are revisited +against the measured work. diff --git a/include/evo/network/crypto_lock.hpp b/include/evo/network/crypto_lock.hpp new file mode 100644 index 0000000..cd1c126 --- /dev/null +++ b/include/evo/network/crypto_lock.hpp @@ -0,0 +1,18 @@ +#pragma once +#include + +// M10.9d (ADR-0097): Mbed TLS and PSA are built without MBEDTLS_THREADING_C, so their +// global state (the PSA core, key slots, the random generator) is not safe to use from +// several threads at once. Every network-library entry point that touches them takes this +// one process-wide lock: the QUIC client's pinned certificate check and the session grant +// and handoff ticket keys. Recursive, since entry points may call each other. Calls are rare +// (per connection, per grant), so the lock costs nothing measurable. Found by evo-load: two +// bots on different threads verifying their first connections at once were refused as +// `untrusted`. +namespace evo::network { +inline std::recursive_mutex& crypto_mutex() noexcept { + static std::recursive_mutex mutex; + return mutex; +} +using CryptoLock = std::scoped_lock; +} // namespace evo::network diff --git a/src/network/tls/handoff_ticket.cpp b/src/network/tls/handoff_ticket.cpp index d804663..1172c3c 100644 --- a/src/network/tls/handoff_ticket.cpp +++ b/src/network/tls/handoff_ticket.cpp @@ -1,4 +1,5 @@ #include "evo/network/handoff_ticket.hpp" +#include "evo/network/crypto_lock.hpp" #include #include #include @@ -57,6 +58,7 @@ bool looks_like_handoff_ticket(std::span bytes) noexcept { std::expected HandoffSigningKey::generate(DeploymentEnvironment environment, std::uint32_t key_id) { + const CryptoLock crypto{crypto_mutex()}; if (key_id == 0) return std::unexpected(HandoffTicketError::invalid_key); if (psa_crypto_init() != PSA_SUCCESS) @@ -73,6 +75,7 @@ HandoffSigningKey::generate(DeploymentEnvironment environment, std::uint32_t key std::expected HandoffSigningKey::import(DeploymentEnvironment environment, std::uint32_t key_id, std::span private_key) { + const CryptoLock crypto{crypto_mutex()}; if (key_id == 0 || private_key.size() != grant_private_key_bytes) return std::unexpected(HandoffTicketError::invalid_key); if (psa_crypto_init() != PSA_SUCCESS) @@ -90,6 +93,7 @@ HandoffSigningKey::HandoffSigningKey(HandoffSigningKey&& other) noexcept key_{std::exchange(other.key_, PSA_KEY_ID_NULL)} {} HandoffSigningKey& HandoffSigningKey::operator=(HandoffSigningKey&& other) noexcept { + const CryptoLock crypto{crypto_mutex()}; if (this != &other) { if (key_ != PSA_KEY_ID_NULL) (void)psa_destroy_key(key_); @@ -101,12 +105,14 @@ HandoffSigningKey& HandoffSigningKey::operator=(HandoffSigningKey&& other) noexc } HandoffSigningKey::~HandoffSigningKey() { + const CryptoLock crypto{crypto_mutex()}; if (key_ != PSA_KEY_ID_NULL) (void)psa_destroy_key(key_); } std::expected HandoffSigningKey::sign(const TicketClaims& claims) const { + const CryptoLock crypto{crypto_mutex()}; if (key_ == PSA_KEY_ID_NULL) return std::unexpected(HandoffTicketError::invalid_key); if (!structurally_valid(claims)) @@ -134,6 +140,7 @@ HandoffSigningKey::sign(const TicketClaims& claims) const { } std::expected HandoffSigningKey::public_key() const { + const CryptoLock crypto{crypto_mutex()}; if (key_ == PSA_KEY_ID_NULL) return std::unexpected(HandoffTicketError::invalid_key); GrantPublicKey out{}; @@ -147,6 +154,7 @@ std::expected HandoffSigningKey::public_key( std::expected HandoffVerifier::create(DeploymentEnvironment environment, world::WorldIdentity world, std::uint64_t region) { + const CryptoLock crypto{crypto_mutex()}; if (world.world == 0 || world.frame == 0 || region == 0) return std::unexpected(HandoffTicketError::invalid_claims); if (psa_crypto_init() != PSA_SUCCESS) @@ -174,6 +182,7 @@ HandoffVerifier::~HandoffVerifier() { } void HandoffVerifier::destroy() noexcept { + const CryptoLock crypto{crypto_mutex()}; for (const auto& k : keys_) (void)psa_destroy_key(k.key); keys_.clear(); @@ -182,6 +191,7 @@ void HandoffVerifier::destroy() noexcept { std::expected HandoffVerifier::add_key(std::uint64_t source_region, std::uint32_t key_id, std::span public_key) { + const CryptoLock crypto{crypto_mutex()}; if (source_region == 0 || source_region == region_ || key_id == 0 || public_key.size() != grant_public_key_bytes) return std::unexpected(HandoffTicketError::invalid_key); @@ -204,6 +214,7 @@ HandoffVerifier::add_key(std::uint64_t source_region, std::uint32_t key_id, std::expected HandoffVerifier::verify(std::span ticket, std::uint64_t now_seconds) const { + const CryptoLock crypto{crypto_mutex()}; if (ticket.size() != handoff_ticket_bytes) return std::unexpected(HandoffTicketError::malformed); // The key id only selects which key must verify; it is itself signed. diff --git a/src/network/tls/session_grant.cpp b/src/network/tls/session_grant.cpp index 91927c0..1d2edb5 100644 --- a/src/network/tls/session_grant.cpp +++ b/src/network/tls/session_grant.cpp @@ -1,4 +1,5 @@ #include "evo/network/session_grant.hpp" +#include "evo/network/crypto_lock.hpp" #include #include #include @@ -51,6 +52,7 @@ psa_key_attributes_t pair_attributes() { } // namespace std::expected random_bytes(std::span out) { + const CryptoLock crypto{crypto_mutex()}; if (psa_crypto_init() != PSA_SUCCESS || psa_generate_random(out.data(), out.size()) != PSA_SUCCESS) return std::unexpected(SessionGrantError::crypto); @@ -68,6 +70,7 @@ std::expected generate_account_id() { std::expected GrantSigningKey::generate(DeploymentEnvironment environment, std::uint32_t key_id) { + const CryptoLock crypto{crypto_mutex()}; if (key_id == 0) return std::unexpected(SessionGrantError::invalid_key); if (psa_crypto_init() != PSA_SUCCESS) @@ -84,6 +87,7 @@ GrantSigningKey::generate(DeploymentEnvironment environment, std::uint32_t key_i std::expected GrantSigningKey::import(DeploymentEnvironment environment, std::uint32_t key_id, std::span private_key) { + const CryptoLock crypto{crypto_mutex()}; if (key_id == 0 || private_key.size() != grant_private_key_bytes) return std::unexpected(SessionGrantError::invalid_key); if (psa_crypto_init() != PSA_SUCCESS) @@ -101,6 +105,7 @@ GrantSigningKey::GrantSigningKey(GrantSigningKey&& other) noexcept key_{std::exchange(other.key_, PSA_KEY_ID_NULL)} {} GrantSigningKey& GrantSigningKey::operator=(GrantSigningKey&& other) noexcept { + const CryptoLock crypto{crypto_mutex()}; if (this != &other) { if (key_ != PSA_KEY_ID_NULL) (void)psa_destroy_key(key_); @@ -112,12 +117,14 @@ GrantSigningKey& GrantSigningKey::operator=(GrantSigningKey&& other) noexcept { } GrantSigningKey::~GrantSigningKey() { + const CryptoLock crypto{crypto_mutex()}; if (key_ != PSA_KEY_ID_NULL) (void)psa_destroy_key(key_); } std::expected GrantSigningKey::sign(const GrantClaims& claims) const { + const CryptoLock crypto{crypto_mutex()}; if (key_ == PSA_KEY_ID_NULL) return std::unexpected(SessionGrantError::invalid_key); if (!structurally_valid(claims)) @@ -145,6 +152,7 @@ GrantSigningKey::sign(const GrantClaims& claims) const { } std::expected GrantSigningKey::public_key() const { + const CryptoLock crypto{crypto_mutex()}; if (key_ == PSA_KEY_ID_NULL) return std::unexpected(SessionGrantError::invalid_key); GrantPublicKey out{}; @@ -157,6 +165,7 @@ std::expected GrantSigningKey::public_key() c std::expected GrantVerifier::create(DeploymentEnvironment environment) { + const CryptoLock crypto{crypto_mutex()}; if (psa_crypto_init() != PSA_SUCCESS) return std::unexpected(SessionGrantError::crypto); return GrantVerifier{environment}; @@ -179,6 +188,7 @@ GrantVerifier::~GrantVerifier() { } void GrantVerifier::destroy() noexcept { + const CryptoLock crypto{crypto_mutex()}; for (const auto& k : keys_) (void)psa_destroy_key(k.key); keys_.clear(); @@ -186,6 +196,7 @@ void GrantVerifier::destroy() noexcept { std::expected GrantVerifier::add_key(std::uint32_t key_id, std::span public_key) { + const CryptoLock crypto{crypto_mutex()}; if (key_id == 0 || public_key.size() != grant_public_key_bytes) return std::unexpected(SessionGrantError::invalid_key); if (std::ranges::any_of(keys_, [&](const Key& k) { return k.key_id == key_id; })) @@ -207,6 +218,7 @@ GrantVerifier::add_key(std::uint32_t key_id, std::span publi std::expected GrantVerifier::verify(std::span grant) const { + const CryptoLock crypto{crypto_mutex()}; if (grant.size() != session_grant_bytes) return std::unexpected(SessionGrantError::malformed); // The key id only selects which key must verify; it is itself signed. diff --git a/src/network/transport/quic_endpoint.cpp b/src/network/transport/quic_endpoint.cpp index 81e2d65..4d9ec3d 100644 --- a/src/network/transport/quic_endpoint.cpp +++ b/src/network/transport/quic_endpoint.cpp @@ -11,6 +11,7 @@ #endif #endif +#include "evo/network/crypto_lock.hpp" #include "evo/network/quic_transport.hpp" #include #include @@ -43,6 +44,7 @@ bool tls_or_certificate_failure(QUIC_STATUS status) { // Leaf (DER) verified against exactly one CA (PEM) for one name; nothing else trusted. bool pinned(std::span leaf, const std::string& authority_pem, const std::string& name) { + const evo::network::CryptoLock crypto{evo::network::crypto_mutex()}; if (psa_crypto_init() != PSA_SUCCESS || leaf.empty()) return false; mbedtls_x509_crt ca; diff --git a/tests/quic_transport.cpp b/tests/quic_transport.cpp index fde99b8..0e04340 100644 --- a/tests/quic_transport.cpp +++ b/tests/quic_transport.cpp @@ -5,12 +5,15 @@ #include "evo/network/dev_credentials.hpp" #include "udp_relay.hpp" #include +#include +#include #include #include #include #include #include #include +#include #include #include #include @@ -558,8 +561,67 @@ void garbage_and_close() { } } // namespace +// M10.9d (ADR-0097): client endpoints on several threads connecting at the same moment all +// connect and pass the owned pin check. Mbed TLS runs without its threading layer; evo-load +// saw two concurrent first verifications refused as `untrusted` before the shared crypto +// lock. This test covers concurrent connects; it does not reliably reproduce that race (PSA +// is already initialised here by the credential generation), which evo-load's smoke does. +void concurrent_clients() { + const auto f = credentials("concurrent"); + Limits limits; + limits.max_connections = 64; // 5 rounds x 8, whatever the server has not yet reaped. + auto server = serve(f, limits); + constexpr int threads = 8; + for (int round = 0; round < 5; ++round) { + std::atomic connected{0}, finished{0}; + std::array, 8> causes{}; + std::latch start{threads}; + std::vector clients; + clients.reserve(threads); + for (int t = 0; t < threads; ++t) + clients.emplace_back([&] { + start.arrive_and_wait(); + auto client = dial(f, server->port()); + const auto end = std::chrono::steady_clock::now() + std::chrono::seconds{10}; + bool done = false; + while (!done && std::chrono::steady_clock::now() < end) { + for (const auto& e : client->poll()) { + if (e.kind == EventKind::connected) { + ++connected; + done = true; + } else if (e.kind == EventKind::closed) { + causes[static_cast(e.cause)] += 1; + done = true; + } + } + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + } + ++finished; + }); + while (finished < threads) { + (void)server->poll(); + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + } + for (auto& c : clients) + c.join(); + if (connected != threads) { + std::printf("concurrent round %d connected=%d", round, connected.load()); + for (std::size_t i = 0; i < causes.size(); ++i) + std::printf(" cause%zu=%d", i, causes.at(i).load()); + std::printf("\n"); + } + CHECK(connected == threads); + for (int i = 0; i < 200; ++i) { // Let the server see the closes. + (void)server->poll(); + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + } + } + std::filesystem::remove_all(f.dir); +} + int main() try { framing(); + concurrent_clients(); channels(); pinning(); impaired(); diff --git a/tests/region_m10_session_tests.cpp b/tests/region_m10_session_tests.cpp index 3a561ee..c48d22f 100644 --- a/tests/region_m10_session_tests.cpp +++ b/tests/region_m10_session_tests.cpp @@ -462,7 +462,8 @@ void end_to_end() { server->child->signal(SIGTERM); CHECK(server->child->wait_exit(30s) == 0); CHECK(server->child->printed("m10 opened ") && server->child->printed("m10 committed ") && - server->child->printed("m10 motion area ")); + server->child->printed("m10 motion area ") && + server->child->printed("m10 load-report sessions=")); } // M10.9b (ADR-0097): evo-load against a real evo-server, two steps with the region's // metrics. A smoke test of the tool and the server under concurrent sessions, not @@ -473,6 +474,15 @@ void load_smoke() { Setup s{152, 0xC6}; Server server{s}; const auto tool = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-load"; + // M10.9d: the bots log in from a bundle (as the dev drill does): 8 accounts x 2 grants. + const auto probe = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-session-probe"; + const auto bundle = s.region.root / "load-grants.bin"; + Child g{{probe.string(), "grants", "--issuer-key", (s.region.root / "issuer.key").string(), + "--world", "7", "--frame", std::to_string(region_identity.frame), + "--region", "152", "--account", "0000000000000000000000000000b000", + "--accounts", "8", "--count", "2", + "--lifetime", "900", "--out", bundle.string()}}; + CHECK(g.wait_exit(30s) == 0 && g.printed("wrote 16 grants ")); Child load{{tool.string(), "--connect", "127.0.0.1:" + std::to_string(server.port), @@ -480,8 +490,10 @@ void load_smoke() { name, "--ca", (s.region.root / "ca.pem").string(), - "--issuer-key", - (s.region.root / "issuer.key").string(), + "--grants", + bundle.string(), + "--grants-per-bot", + "2", "--world", "7", "--frame", @@ -522,7 +534,8 @@ void load_smoke() { // Every bot logged in once: 4 in the first step, 4 more in the second. CHECK((starts(line, "load step N=4 established=4 logins=4 ") || starts(line, "load step N=8 established=8 logins=4 ")) && - line.find(" drops=0 login_failures=0 refused=0 ") != std::string::npos); + line.find(" drops=0 login_failures=0 refused=0 ") != std::string::npos && + line.find(" exhausted=0 hold_utc=") != std::string::npos); } const auto has = [&](std::string_view prefix) { return std::ranges::count_if(load.lines(), [&](const std::string& l) { diff --git a/tools/network/load.cpp b/tools/network/load.cpp index d0f752a..b2c7c08 100644 --- a/tools/network/load.cpp +++ b/tools/network/load.cpp @@ -2,7 +2,8 @@ // runs against the evolution-dev reference worker count as evidence (10.9d); local runs are // smoke tests and development references. // -// evo-load --connect HOST:PORT --server-name NAME --ca FILE --issuer-key FILE +// evo-load --connect HOST:PORT --server-name NAME --ca FILE +// (--issuer-key FILE | --grants FILE --grants-per-bot C) // --world N --frame N --region N [--steps N[,N]...] [--hold S] [--warmup S] // [--ramp-ms MS] [--threads T] [--seed X] [--lifetime S] [--metrics HOST:PORT] // [--response-budget-ms MS] [--per-client] [--expect-established] @@ -14,11 +15,17 @@ // probe marker; every 2 s it sends a probe action; every 10 s it moves its interest by a // cell (a fresh baseline). A closed or refused bot logs in again after 2 s with the next // serial. +// M10.9d: with --grants (a bundle from `evo-session-probe grants --accounts N --count C`, +// e.g. made where the dev issuer key lives), bot i logs in with grants i*C .. i*C+C-1 in +// order instead of signing; a bot that used all C stays out (counted as exhausted). // // Steps: the bot count rises to each step's N (one login every --ramp-ms), waits --warmup // seconds, then measures for --hold seconds. Bots stay connected into the next step. Per // step it prints: // load step N=.. established=.. logins=.. drops=.. login_failures=.. refused=.. seconds=.. +// exhausted=.. hold_utc=START..END (unix seconds of the hold, to match server samples) +// close_reasons=REASON:COUNT,... (session-v1 close reasons; without a session Close, +// 256 + the QUIC transport cause, or 255 when unknown) // (drops: established sessions closed; login failures: closed before Welcome, including // transport refusals at the connection limit; refused: closed with capacity or // rate_limited, either way. Logins, drops and failures include the step's ramp and warmup; @@ -111,7 +118,8 @@ double ms(std::uint64_t us) { } struct Options { - std::string host, server_name, ca, issuer_key, metrics_host; + std::string host, server_name, ca, issuer_key, grants_file, metrics_host; + std::uint64_t grants_per_bot{}; std::uint16_t port{}, metrics_port{}; std::uint64_t world{}, frame{}, region{}, seed{1}, lifetime{900}; std::vector steps{20}; @@ -125,6 +133,7 @@ struct Samples { gap_ticks; std::uint64_t snapshots{}, bytes{}, intents{}, refused_intents{}, logins{}, drops{}, login_failures{}, refused{}; + std::map closes; // By session close reason (255: no Close). std::uint64_t connected_polls{}; // Polls while established (starvation is judged on these). }; void append(std::vector& to, const std::vector& from) { @@ -142,6 +151,8 @@ struct Bot { client::InputLead lead; std::uint64_t last_tick{}, last_view{}, last_bytes{}, next_action_id{1}; std::uint64_t applied{}; // Newest applied intention sequence seen in an own marker. + std::uint64_t used{}; // Bundle grants used (bundle mode). + std::atomic exhausted{false}; std::map receipt_pending, response_pending, outcome_pending; std::map intent_tick; std::int16_t vx{}, vy{}; @@ -153,9 +164,11 @@ struct Bot { class Load { public: - Load(Options o, GrantSigningKey key, std::string address) - : o_{std::move(o)}, key_{std::move(key)}, address_{std::move(address)}, - authority_{read(o_.ca)} { + // Signs with `key`, or logs in from `bundle` (concatenated grants, grants_per_bot each). + Load(Options o, std::optional key, std::vector bundle, + std::string address) + : o_{std::move(o)}, key_{std::move(key)}, bundle_{std::move(bundle)}, + address_{std::move(address)}, authority_{read(o_.ca)} { const auto most = *std::ranges::max_element(o_.steps); bots_.reserve(most); for (std::size_t i = 0; i < most; ++i) { @@ -199,6 +212,11 @@ class Load { } return out; } + [[nodiscard]] std::size_t exhausted(std::size_t n) const { + return static_cast( + std::count_if(bots_.begin(), bots_.begin() + static_cast(n), + [](const auto& b) { return b->exhausted.load(); })); + } [[nodiscard]] std::size_t established(std::size_t n) const { return static_cast( std::count_if(bots_.begin(), bots_.begin() + static_cast(n), @@ -229,15 +247,26 @@ class Load { } } void login(Bot& b, Clock::time_point now) { - GrantClaims claims; - claims.account = b.account; - claims.serial = ++b.serial; - claims.world = {o_.world, o_.frame}; - claims.region = o_.region; - claims.issued_at = now_seconds(); - claims.expires_at = claims.issued_at + o_.lifetime; - claims.grant_id = (b.serial << 16U) ^ (b.index + 1); - const auto grant = key_.sign(claims); + std::optional grant; + if (key_) { + GrantClaims claims; + claims.account = b.account; + claims.serial = ++b.serial; + claims.world = {o_.world, o_.frame}; + claims.region = o_.region; + claims.issued_at = now_seconds(); + claims.expires_at = claims.issued_at + o_.lifetime; + claims.grant_id = (b.serial << 16U) ^ (b.index + 1); + if (auto signed_grant = key_->sign(claims)) + grant = *signed_grant; + } else { + if (b.used >= o_.grants_per_bot) { + b.exhausted = true; // Out of grants: stays out. + b.retry_at = Clock::time_point::max(); + return; + } + grant = bundle_.at((b.index * o_.grants_per_bot) + b.used++); + } client::RegionSessionClientConfig config; config.transport.server_address = address_; config.transport.port = o_.port; @@ -278,6 +307,13 @@ class Load { ++b.s.drops; else ++b.s.login_failures; + if (const auto reason = c.close_reason(); + reason && *reason != SessionCloseReason::connection_lost) + ++b.s.closes[static_cast(*reason)]; + else if (const auto cause = c.transport_cause()) // No session Close: 256 + cause. + ++b.s.closes[256U + static_cast(*cause)]; + else + ++b.s.closes[255U]; if (c.close_reason() == SessionCloseReason::capacity || c.close_reason() == SessionCloseReason::rate_limited) ++b.s.refused; @@ -406,7 +442,8 @@ class Load { } Options o_; - GrantSigningKey key_; + std::optional key_; + std::vector bundle_; std::string address_, authority_; std::vector> bots_; std::vector locks_; @@ -481,9 +518,9 @@ void line(const char* name, const std::vector& pooled, } // One step's report; true when within budget. -bool report(const Options& o, std::size_t n, std::size_t established, - const std::vector& all, double seconds, double cpu, - const std::map& server) { +bool report(const Options& o, std::size_t n, std::size_t established, std::size_t exhausted, + std::pair hold_utc, const std::vector& all, + double seconds, double cpu, const std::map& server) { Samples pooled; std::vector response95, receipt95, outcome95, login95, correction95, rates, kib; for (const auto& s : all) { @@ -499,6 +536,8 @@ bool report(const Options& o, std::size_t n, std::size_t established, pooled.logins += s.logins; pooled.drops += s.drops; pooled.login_failures += s.login_failures; + for (const auto& [why, count] : s.closes) + pooled.closes[why] += count; pooled.refused += s.refused; response95.push_back(percentile(s.response_us, 0.95)); receipt95.push_back(percentile(s.receipt_us, 0.95)); @@ -511,12 +550,18 @@ bool report(const Options& o, std::size_t n, std::size_t established, kib.push_back(static_cast( std::lround(static_cast(s.bytes) / 1024.0 / std::max(seconds, 0.001)))); } + std::string reasons; + for (const auto& [why, count] : pooled.closes) + reasons += (reasons.empty() ? "" : ",") + std::to_string(why) + ":" + std::to_string(count); std::printf("load step N=%zu established=%zu logins=%llu drops=%llu login_failures=%llu " - "refused=%llu seconds=%.1f\n", + "refused=%llu seconds=%.1f exhausted=%zu hold_utc=%llu..%llu close_reasons=%s\n", n, established, static_cast(pooled.logins), static_cast(pooled.drops), static_cast(pooled.login_failures), - static_cast(pooled.refused), seconds); + static_cast(pooled.refused), seconds, exhausted, + static_cast(hold_utc.first), + static_cast(hold_utc.second), + reasons.empty() ? "-" : reasons.c_str()); line("response", pooled.response_us, response95, 1000.0, "ms"); line("receipt", pooled.receipt_us, receipt95, 1000.0, "ms"); line("outcome", pooled.outcome_us, outcome95, 1000.0, "ms"); @@ -663,6 +708,10 @@ int main(int argc, char** argv) { o.ca = value; else if (flag == "--issuer-key") o.issuer_key = value; + else if (flag == "--grants") + o.grants_file = value; + else if (n && flag == "--grants-per-bot" && *n >= 1 && *n <= 64) + o.grants_per_bot = *n; else if (flag == "--steps") { const auto s = steps_of(value); ok = s.has_value(); @@ -676,7 +725,7 @@ int main(int argc, char** argv) { o.region = *n; else if (n && flag == "--seed" && *n != 0) o.seed = *n; - else if (n && flag == "--lifetime" && *n >= 60 && *n <= 3600) + else if (n && flag == "--lifetime" && *n >= 60 && *n <= 14400) o.lifetime = *n; else if (n && flag == "--hold" && *n >= 1 && *n <= 3600) o.hold_s = *n; @@ -691,23 +740,44 @@ int main(int argc, char** argv) { else ok = false; } - if (!ok || o.host.empty() || o.server_name.empty() || o.ca.empty() || o.issuer_key.empty() || - o.world == 0 || o.frame == 0 || o.region == 0) { + const bool signs = !o.issuer_key.empty(); + if (!ok || o.host.empty() || o.server_name.empty() || o.ca.empty() || + signs == !o.grants_file.empty() || (!signs && o.grants_per_bot == 0) || o.world == 0 || + o.frame == 0 || o.region == 0) { std::fputs("usage: evo-load --connect HOST:PORT --server-name NAME --ca FILE " - "--issuer-key FILE --world N --frame N --region N [--steps N[,N]...] " + "(--issuer-key FILE | --grants FILE --grants-per-bot C) --world N " + "--frame N --region N [--steps N[,N]...] " "[--hold S] [--warmup S] [--ramp-ms MS] [--threads T] [--seed X] " "[--lifetime S] [--metrics HOST:PORT] [--response-budget-ms MS] " "[--per-client] [--expect-established]\n", stderr); return 2; } - const auto scalar = read(o.issuer_key); - auto key = GrantSigningKey::import( - DeploymentEnvironment::dev, 1, - std::span{reinterpret_cast(scalar.data()), scalar.size()}); - if (!key) { - std::puts("error issuer-key"); - return 2; + std::optional key; + std::vector bundle; + if (signs) { + const auto scalar = read(o.issuer_key); + auto imported = GrantSigningKey::import( + DeploymentEnvironment::dev, 1, + std::span{reinterpret_cast(scalar.data()), scalar.size()}); + if (!imported) { + std::puts("error issuer-key"); + return 2; + } + key.emplace(std::move(*imported)); + } else { + const auto bytes = read(o.grants_file); + const auto needed = *std::ranges::max_element(o.steps) * o.grants_per_bot; + if (bytes.size() % session_grant_bytes != 0 || + bytes.size() / session_grant_bytes < needed) { + std::printf("error grants (need %llu grants: bots x grants-per-bot)\n", + static_cast(needed)); + return 2; + } + bundle.resize(bytes.size() / session_grant_bytes); + for (std::size_t i = 0; i < bundle.size(); ++i) + std::copy_n(bytes.begin() + static_cast(i * session_grant_bytes), + session_grant_bytes, reinterpret_cast(bundle[i].data())); } const auto address = quic::resolve_ipv4(o.host); if (!address) { @@ -723,7 +793,7 @@ int main(int argc, char** argv) { bool smoke_ok = true; std::optional sustainable, saturated; { - Load load{o, std::move(*key), *address}; + Load load{o, std::move(key), std::move(bundle), *address}; for (const auto n : o.steps) { load.activate(n); std::this_thread::sleep_for(std::chrono::seconds{o.warmup_s}); @@ -731,6 +801,7 @@ int main(int argc, char** argv) { // the warmup is discarded. const auto warm = load.take(n); const auto started = Clock::now(); + const auto hold_start = now_seconds(); const auto cpu0 = cpu_seconds(); std::this_thread::sleep_for(std::chrono::seconds{o.hold_s}); auto samples = load.take(n); @@ -739,12 +810,15 @@ int main(int argc, char** argv) { samples[i].logins += warm[i].logins; samples[i].drops += warm[i].drops; samples[i].login_failures += warm[i].login_failures; + for (const auto& [why, count] : warm[i].closes) + samples[i].closes[why] += count; samples[i].refused += warm[i].refused; } const auto seconds = std::chrono::duration(Clock::now() - started).count(); const auto established = load.established(n); const bool within = - report(o, n, established, samples, seconds, cpu_seconds() - cpu0, scrape(o)); + report(o, n, established, load.exhausted(n), {hold_start, now_seconds()}, samples, + seconds, cpu_seconds() - cpu0, scrape(o)); if (within && !saturated) sustainable = n; if (!within && !saturated) diff --git a/tools/ops/load-drill.sh b/tools/ops/load-drill.sh new file mode 100644 index 0000000..d03ec09 --- /dev/null +++ b/tools/ops/load-drill.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# M10.9d load drill (ADR-0097) for evolution-dev only: the reference-worker qualification run. +# +# tools/ops/load-drill.sh [--context ] [--steps N[,N]...] [--hold S] +# [--warmup S] [--grants-per-bot C] [--connect HOST:PORT] [--server-name NAME] +# +# holds dev-ca.pem (the current dev session CA) and load-grants.bin, a bundle +# made where the dev issuer key lives: +# evo-session-probe grants --issuer-key issuer.key --world W --frame F --region R \ +# --account --accounts --count --lifetime 14400 \ +# --out load-grants.bin +# The issuer key never reaches the machine running the drill. Requires kubectl access to +# evolution-dev (patch Deployment/evo-region, read pods and logs), `evo-load` (EVO_LOAD, +# default: on PATH) and python3. +# +# Steps: refuse unless the session window outlasts the drill; enable synthetic inputs on +# evo-region (the drill-time override of ADR-0073, never committed) and wait for the new +# pod to be Ready; follow its log with timestamps (the server prints `m10 load-report` +# every 10 s); run evo-load through the public route; then restore REGION_INPUTS to the Git +# value (also on failure) so Argo returns to Synced. The report matches each step's hold +# (evo-load's hold_utc) with the server's reports in it and prints, per step, the client +# verdict and the server's worst tick work, session work, governor state and tick debt. +# Stage and World0 are never touched: the namespace is fixed. +set -euo pipefail +usage() { + echo "usage: $0 [--context ] [--steps N[,N]...] [--hold S] [--warmup S]" \ + "[--grants-per-bot C] [--connect HOST:PORT] [--server-name NAME]" >&2 + exit 2 +} +[ $# -ge 1 ] || usage +tokens="$1"; shift +k=(kubectl --request-timeout=30s) +steps=20,24,28,32 hold=60 warmup=10 per_bot=4 +connect=void.aeternalisgames.com:7400 server_name=void.aeternalisgames.com +while [ $# -gt 0 ]; do + case "$1" in + --context) [ -n "${2:-}" ] || usage; k+=(--context "$2"); shift 2 ;; + --steps) [ -n "${2:-}" ] || usage; steps="$2"; shift 2 ;; + --hold) [ -n "${2:-}" ] || usage; hold="$2"; shift 2 ;; + --warmup) [ -n "${2:-}" ] || usage; warmup="$2"; shift 2 ;; + --grants-per-bot) [ -n "${2:-}" ] || usage; per_bot="$2"; shift 2 ;; + --connect) [ -n "${2:-}" ] || usage; connect="$2"; shift 2 ;; + --server-name) [ -n "${2:-}" ] || usage; server_name="$2"; shift 2 ;; + *) usage ;; + esac +done +ns=evolution-dev # Fixed: qualification runs on the dev reference worker only. +tool="${EVO_LOAD:-evo-load}" +py="$(command -v python3 || command -v python || true)" +for f in dev-ca.pem load-grants.bin; do + [ -f "$tokens/$f" ] || { echo "missing $tokens/$f" >&2; exit 2; } +done +command -v "$tool" > /dev/null || [ -x "$tool" ] || { echo "evo-load not found (set EVO_LOAD)" >&2; exit 2; } +[ -n "$py" ] || { echo "python3 is required" >&2; exit 2; } +selector=app.kubernetes.io/name=evo-region +work="$(mktemp -d)" +chmod 700 "$work" +follow_pid="" +say() { echo "$(date -u +%H:%M:%S) $*"; } +fail() { say "FAIL $*"; exit 1; } +env_of() { + "${k[@]}" get deployment -n "$ns" evo-region \ + -o jsonpath="{.spec.template.spec.containers[0].env[?(@.name==\"$1\")].value}" +} +ready_pod() { # Waits up to $2 seconds for a Ready pod other than $1. + local old="$1" limit="${2:-240}" + for _ in $(seq 1 $((limit * 5))); do + for p in $("${k[@]}" get pods -n "$ns" -l "$selector" \ + -o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}'); do + [ "$p" = "$old" ] && continue + [ "$("${k[@]}" get pod -n "$ns" "$p" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2> /dev/null)" = True ] && + { echo "$p"; return 0; } + done + sleep 0.2 + done + return 1 +} +restore() { + set +e + [ -n "$follow_pid" ] && kill "$follow_pid" 2> /dev/null + say "restoring the Git configuration of evo-region (REGION_INPUTS empty)" + local current + current="$("${k[@]}" get pods -n "$ns" -l "$selector" -o jsonpath='{.items[0].metadata.name}' 2> /dev/null)" + "${k[@]}" set env -n "$ns" deployment/evo-region REGION_INPUTS= > /dev/null + ready_pod "$current" 240 > /dev/null && say "restored: evo-region Ready" || say "WARN evo-region not Ready after restore" + say "drill files kept in $work" +} + +world="$(env_of REGION_WORLD)" region="$(env_of REGION_ID)" frame="$(env_of REGION_FRAME)" +window="$(env_of REGION_SESSION_WINDOW)" +[ -n "$world" ] && [ -n "$region" ] && [ -n "$frame" ] || fail "cannot read the region identity" +[ -z "$(env_of REGION_INPUTS)" ] || fail "REGION_INPUTS is already set: another drill may be running" +# The drill must end before the session window closes. +count=$(echo "$steps" | tr ',' '\n' | wc -l) +most=$(echo "$steps" | tr ',' '\n' | sort -n | tail -1) +needed=$(( count * (hold + warmup + most / 10 + 5) + 600 )) +expiry="${window##*:}" +[ -n "$expiry" ] && [ "$expiry" -gt $(( $(date -u +%s) + needed )) ] 2> /dev/null || + fail "the session window ($window) closes within the drill's ${needed} s: renew it first" +size=$(wc -c < "$tokens/load-grants.bin") +[ $(( size / 160 )) -ge $(( most * per_bot )) ] || + fail "load-grants.bin holds $(( size / 160 )) grants; $most bots x $per_bot need $(( most * per_bot ))" +image="$("${k[@]}" get deployment -n "$ns" evo-region -o jsonpath='{.spec.template.spec.containers[0].image}')" +say "drill: world $world region $region image $image steps $steps hold ${hold}s" + +trap restore EXIT +before="$("${k[@]}" get pods -n "$ns" -l "$selector" -o jsonpath='{.items[0].metadata.name}')" +say "enabling synthetic inputs in $ns (drill-time override, restored on exit)" +"${k[@]}" set env -n "$ns" deployment/evo-region REGION_INPUTS=development-v1 > /dev/null +pod="$(ready_pod "$before" 240)" || fail "no Ready evo-region pod with synthetic inputs" +"${k[@]}" logs -n "$ns" -f --timestamps "$pod" > "$work/server.log" 2> /dev/null & +follow_pid=$! +sleep 5 +grep -q "ready 1" "$work/server.log" || say "WARN no 'ready 1' in the server log yet" +say "running evo-load against $connect" +set +e +"$tool" --connect "$connect" --server-name "$server_name" --ca "$tokens/dev-ca.pem" \ + --grants "$tokens/load-grants.bin" --grants-per-bot "$per_bot" --world "$world" \ + --frame "$frame" --region "$region" --steps "$steps" --hold "$hold" --warmup "$warmup" \ + --threads 8 --per-client > "$work/load.log" 2>&1 +load_exit=$? +set -e +sleep 12 # One more server report after the last hold. +grep -E "^load (step|response_ms|outcome_ms|snapshots|starved|verdict|result)" "$work/load.log" || true +[ "$load_exit" = 0 ] || say "WARN evo-load exited $load_exit (see $work/load.log)" +"$py" - "$work/load.log" "$work/server.log" <<'PY' +import re, sys +from datetime import datetime +load, server = sys.argv[1], sys.argv[2] +reports = [] +for line in open(server, encoding='utf-8', errors='replace'): + m = re.match(r'(\S+) m10 load-report (.*)', line.strip()) + if not m: + continue + stamp = m.group(1)[:19] + t = datetime.strptime(stamp, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=None) + epoch = int((t - datetime(1970, 1, 1)).total_seconds()) + fields = dict(kv.split('=') for kv in m.group(2).split()) + reports.append((epoch, {k: int(v) for k, v in fields.items()})) +print(f"drill server reports={len(reports)}") +for line in open(load, encoding='utf-8', errors='replace'): + m = re.match(r'load step N=(\d+) .*hold_utc=(\d+)\.\.(\d+)', line) + if not m: + continue + n, t0, t1 = int(m.group(1)), int(m.group(2)), int(m.group(3)) + # A report covers the 2 s before it: only reports wholly inside the hold count. + inside = [r for t, r in reports if t0 + 2 <= t <= t1] + if not inside: + print(f"drill step N={n} server_reports=0") + continue + worst = lambda key: max(r[key] for r in inside) + sessions = max(1, worst('sessions')) + print(f"drill step N={n} server_reports={len(inside)} sessions_max={worst('sessions')} " + f"work_p99_max_ms={worst('p99_us') / 1000:.1f} work_max_ms={worst('max_us') / 1000:.1f} " + f"session_p99_max_ms={worst('session_p99_us') / 1000:.1f} " + f"per_session_ms={worst('session_p99_us') / 1000 / sessions:.3f} " + f"refusing_max={worst('refusing')} shed_max={worst('shed')} " + f"tick_debt_max={worst('tick_debt')} " + f"server_within_budget={int(worst('p99_us') <= 25000 and worst('session_p99_us') <= 500 * sessions and worst('tick_debt') == 0)}") +PY +say "drill complete (evo-load exit $load_exit); restoring"