Add M10.9d load drill harness and serialize Mbed TLS use
Foundation / checks (deploy) (push) Successful in 3m24s
Foundation / checks (ecs) (push) Successful in 1m56s
Foundation / checks (client) (push) Successful in 12m18s
Foundation / checks (network) (push) Successful in 13m30s
Foundation / checks (tsan) (push) Successful in 8m0s
Foundation / checks (checks) (push) Successful in 32m39s
Foundation / checks (postgres) (push) Successful in 45m47s
Foundation / postgres-tools (push) Successful in 17s
Foundation / image (push) Successful in 2m30s
Foundation / server-image (push) Successful in 4m8s

ADR-0097. tools/ops/load-drill.sh (runbook docs/networking/M10_LOAD_DRILL.md)
runs the evolution-dev reference qualification: synthetic inputs on for
the run (the ADR-0073 drill-time override, restored on exit), bundle-mode
evo-load bots through the public route, and each step's hold matched with
the server's reports. evo-server logs "m10 load-report" every 10 s, since
the agent cannot reach the ops port. evo-load gains --grants FILE
--grants-per-bot C, exhausted=, hold_utc= and close_reasons=;
evo-session-probe grants gains --accounts N.

Finding: Mbed TLS/PSA are built without MBEDTLS_THREADING_C, and bots on
different threads verifying their first connections at once were refused
as untrusted (two per smoke run). One process-wide recursive crypto lock
now guards every network-library entry point that touches them; the
smoke went from 2/2 failing to 3/3 clean. quic-transport adds a
concurrent-connect test. Refreshes the dev session window.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-08 08:38:13 -04:00
co-authored by Claude Opus 5.5
parent 47b84bce74
commit ada77fd761
16 changed files with 609 additions and 70 deletions
+19
View File
@@ -296,6 +296,7 @@ template <typename Host> class M10Bridge {
bool changed = false;
for (std::uint64_t i = 0; i < ticks; ++i)
changed = governor_.observe(per_tick, session_per_tick, s.tick_debt, now_us) || changed;
report(s, now_us);
if (!changed || !governed_)
return;
const auto& load = governor_.state();
@@ -303,6 +304,23 @@ template <typename Host> class M10Bridge {
std::printf("m10 load refusing=%d shed=%u p95_us=%llu\n", load.refusing ? 1 : 0,
static_cast<unsigned>(load.shed), static_cast<unsigned long long>(load.p95_us));
}
// M10.9d (ADR-0097): every 10 s, the load as /metrics would show it, in the log (drills
// read it with `kubectl logs --timestamps` where the ops port is not reachable).
void report(const server::HostStatus& s, std::uint64_t now_us) {
if (reported_us_ != 0 && now_us - reported_us_ < 10'000'000)
return;
reported_us_ = now_us;
const auto& load = governor_.state();
std::printf("m10 load-report sessions=%zu p95_us=%llu p99_us=%llu max_us=%llu "
"session_p95_us=%llu session_p99_us=%llu refusing=%d shed=%u tick_debt=%llu\n",
sessions_->sessions(), static_cast<unsigned long long>(load.p95_us),
static_cast<unsigned long long>(load.p99_us),
static_cast<unsigned long long>(load.max_us),
static_cast<unsigned long long>(load.session_p95_us),
static_cast<unsigned long long>(load.session_p99_us),
governed_ && load.refusing ? 1 : 0, governed_ ? load.shed : 0U,
static_cast<unsigned long long>(s.tick_debt));
}
static const processes::EnvironmentalCommunity&
community(const processes::EnvironmentalCommunity& c) {
return c;
@@ -448,6 +466,7 @@ template <typename Host> class M10Bridge {
std::unique_ptr<server::PostgresActionLedger> actions_;
server::LoadGovernor governor_{*server::LoadGovernor::create({})};
bool governed_{true};
std::uint64_t reported_us_{};
std::chrono::nanoseconds busy_{}, session_busy_{};
std::uint64_t measured_tick_{};
std::unique_ptr<server::PostgresProbeLedger> probes_; // Handoff mode only.
+43 -29
View File
@@ -15,6 +15,9 @@
// interleaved in serial order, so a client crossing between regions always has a newer
// serial for the next one (serials must rise world-wide). A handoff key is a key pair
// from `keys` (issuer.key for --m10-handoff-key, issuer.pub for --m10-neighbour).
// M10.9d (ADR-0097): --accounts N (1..256) writes the same for N accounts, account by
// account: account i is --account with its last two bytes replaced by i + 1 (big
// endian). This is the load bundle `evo-load --grants FILE --grants-per-bot COUNT` uses.
// evo-session-probe run --connect HOST:PORT --server-name NAME --ca FILE --issuer-key FILE
// --world N --frame N --region N --account HEX32 --serial N [--lifetime S]
// [--intents N] [--actions N --action-base ID] [--query ID]... [--timeout-ms N]
@@ -206,14 +209,14 @@ bool write_new(const std::filesystem::path& path, std::span<const std::uint8_t>
return true;
}
int grants(const RunOptions& o, const std::string& out, std::uint64_t count) {
int grants(const RunOptions& o, const std::string& out, std::uint64_t count,
std::uint64_t accounts) {
const auto key = issuer(o.issuer_key);
if (!key || !o.account) {
std::puts("error issuer-key");
return 2;
}
GrantClaims claims;
claims.account = *o.account;
claims.world = {o.world, o.frame};
claims.issued_at = now_seconds();
claims.expires_at = claims.issued_at + o.lifetime;
@@ -221,31 +224,39 @@ int grants(const RunOptions& o, const std::string& out, std::uint64_t count) {
std::vector<std::uint64_t> regions{o.region};
regions.insert(regions.end(), o.more_regions.begin(), o.more_regions.end());
if (count * regions.size() > 64) {
std::puts("error count (at most 64 grants in all)");
std::puts("error count (at most 64 grants per account)");
return 2;
}
std::vector<std::uint8_t> bundle;
std::uint64_t serial = first;
for (std::uint64_t i = 0; i < count; ++i)
for (const auto region : regions) {
claims.region = region;
claims.serial = serial++;
const auto grant = sign(*key, claims);
if (!grant) {
std::puts("error grant");
return 2;
}
bundle.insert(bundle.end(), grant->begin(), grant->end());
for (std::uint64_t a = 0; a < accounts; ++a) {
claims.account = *o.account;
if (accounts > 1) { // Account a: the last two bytes are a + 1.
claims.account.bytes.at(14) = static_cast<std::uint8_t>(((a + 1) >> 8U) & 0xFFU);
claims.account.bytes.at(15) = static_cast<std::uint8_t>((a + 1) & 0xFFU);
}
std::uint64_t serial = first;
for (std::uint64_t i = 0; i < count; ++i)
for (const auto region : regions) {
claims.region = region;
claims.serial = serial++;
const auto grant = sign(*key, claims);
if (!grant) {
std::puts("error grant");
return 2;
}
bundle.insert(bundle.end(), grant->begin(), grant->end());
}
}
count *= regions.size();
if (!write_new(out, bundle)) {
std::puts("error out (exists or unwritable)");
return 1;
}
std::printf("wrote %llu grants serial %llu..%llu valid %llu s\n",
static_cast<unsigned long long>(count), static_cast<unsigned long long>(first),
static_cast<unsigned long long>(first + count - 1),
static_cast<unsigned long long>(o.lifetime));
std::printf(
"wrote %llu grants serial %llu..%llu valid %llu s accounts %llu\n",
static_cast<unsigned long long>(count) * accounts, static_cast<unsigned long long>(first),
static_cast<unsigned long long>(first + count - 1),
static_cast<unsigned long long>(o.lifetime), static_cast<unsigned long long>(accounts));
return 0;
}
@@ -430,7 +441,7 @@ int main(int argc, char** argv) {
RunOptions o;
const bool bundle = !args.empty() && args[0] == "grants";
std::string out;
std::uint64_t count = 1;
std::uint64_t count = 1, accounts = 1;
if (bundle)
o.lifetime = 300; // The region's default maximum.
bool ok = !args.empty() && (args[0] == "run" || bundle) && args.size() % 2 == 1;
@@ -489,26 +500,29 @@ int main(int argc, char** argv) {
out = value;
else if (bundle && n && flag == "--count" && *n >= 1 && *n <= 64)
count = *n;
else if (bundle && n && flag == "--accounts" && *n >= 1 && *n <= 256)
accounts = *n;
else
ok = false;
}
if (bundle && ok && !o.issuer_key.empty() && o.account && o.world != 0 && o.frame != 0 &&
o.region != 0 && !out.empty() && o.lifetime >= 60 && o.host.empty())
return grants(o, out, count);
return grants(o, out, count, accounts);
const bool signs = o.grants_file.empty();
if (bundle || !ok || o.host.empty() || o.server_name.empty() || o.ca.empty() ||
(signs && (o.issuer_key.empty() || !o.account || o.world == 0 || o.frame == 0 ||
o.region == 0 || o.serial == 0)) ||
(!signs && !o.issuer_key.empty())) {
std::fputs("usage: evo-session-probe keys --out DIR | grants --issuer-key FILE "
"--world N --frame N --region N --account HEX32 --out FILE [--serial N] "
"[--count 1..64] [--lifetime 60..14400] | run --connect HOST:PORT "
"--server-name NAME --ca FILE --issuer-key FILE --world N --frame N "
"--region N --account HEX32 --serial N [--lifetime S] [--intents N] "
"[--actions N --action-base ID] [--query ID]... [--timeout-ms N] "
"[--interest X,Y,Z,R] [--snapshots N] | run --connect HOST:PORT "
"--server-name NAME --ca FILE --grants FILE [--grant N] [run options]\n",
stderr);
std::fputs(
"usage: evo-session-probe keys --out DIR | grants --issuer-key FILE "
"--world N --frame N --region N --account HEX32 --out FILE [--serial N] "
"[--count 1..64] [--accounts 1..256] [--lifetime 60..14400] | run --connect HOST:PORT "
"--server-name NAME --ca FILE --issuer-key FILE --world N --frame N "
"--region N --account HEX32 --serial N [--lifetime S] [--intents N] "
"[--actions N --action-base ID] [--query ID]... [--timeout-ms N] "
"[--interest X,Y,Z,R] [--snapshots N] | run --connect HOST:PORT "
"--server-name NAME --ca FILE --grants FILE [--grant N] [run options]\n",
stderr);
return 2;
}
return run(o);
@@ -12,7 +12,7 @@ spec:
env:
- {name: REGION_SESSION, value: "dev"}
- {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"}
- {name: REGION_SESSION_WINDOW, value: "dev-20261008T112240-4ebe:1791472360"}
- {name: REGION_SESSION_WINDOW, value: "dev-20261008T123155-7759:1791476515"}
- {name: REGION_M10_MAX_GRANT_LIFETIME, value: "14400"}
ports:
# QUIC (ADR-0090): UDP only; nothing listens on TCP 7400.
+2 -1
View File
@@ -549,7 +549,8 @@ smoke done; 10.9c-1 session fuzz target and smoke, in-process Hello and datagram
slow peer and connect/disconnect storm with honest-service checks done; 10.9c-2 SIGKILL and
drain lease loss with 24 bots (all back within 5 s of the successor) done; no standalone
raw-UDP sender (abuse stays in-process/loopback; datagram-cap eviction cost analysed).
10.9d dev-cluster drill harness and reference results follow.
10.9d drill harness (load-drill.sh, server load-report, bundle-mode evo-load, Mbed TLS
crypto lock) done; the evolution-dev reference run follows.
**Entry:** M9 client/session evidence and M7/M8 dev/stage durability. Read
NETWORK_ARCHITECTURE, PERSISTENCE_ARCHITECTURE and ADR-0017. The M9 transport is
+7 -2
View File
@@ -123,8 +123,13 @@ refusing at 32 (work p95 about 14 ms), with responses about 103 ms and outcomes
and its intention admitted (the dev region does not simulate outside drills). M10.9c-2
kills and drains a real evo-server under 24 `evo-load` bots. All of them log in to the
successor within 5.0 s (SIGKILL, after a 3.5 s lease wait) or 2.4 s (drain), with no failed
logins, and responses stay at 106 ms p95. Next: 10.9d, the evolution-dev drill harness and
the reference-worker run.
logins, and responses stay at 106 ms p95. M10.9d adds the evolution-dev load drill
(`tools/ops/load-drill.sh`). It turns synthetic inputs on for the run, drives bundle-mode
`evo-load` bots through the public route, and matches each step with the server's 10 s
`m10 load-report` log lines. Building it found that Mbed TLS (built without its threading
layer) raced when bots verified connections on several threads; one process-wide crypto
lock fixed it. Next: the reference run, which needs a promoted image, a load bundle and
the maintainer's go-ahead.
## Previous gate - 2026-10-02
@@ -222,3 +222,36 @@ snapshots.
millisecond apart). That cost is bounded per datagram but can add up at very high rates
from one peer. Making eviction O(1) without letting tombstones outgrow the queue bound is
recorded as a follow-up, not done.
## 10.9d drill harness (2026-10-08)
- **`tools/ops/load-drill.sh`** (runbook: `docs/networking/M10_LOAD_DRILL.md`):
1. turns synthetic inputs on in evolution-dev for the run, the drill-time override of
ADR-0073 (never in Git);
2. runs `evo-load` through the public route;
3. matches each step's hold with the server's reports in it;
4. restores `REGION_INPUTS` on exit, including on failure.
- **`evo-server`** logs `m10 load-report …` every 10 s: sessions, work and session-work
percentiles, governor state and tick debt. The agent cannot reach the ops port (no
port-forward, and the API-server proxy is refused), but it can read logs.
- **Bundle mode:**
- `evo-load --grants FILE --grants-per-bot C` logs bots in from a pre-issued bundle, so
the drill machine never holds the issuer key;
- `evo-session-probe grants --accounts N` writes such a bundle for N accounts;
- each step line now carries `exhausted=` and `hold_utc=`.
- **Evidence:** the M10 load smoke runs in bundle mode (8 accounts × 2 grants), and the
M10 process test checks the `load-report` line. The drill's summariser was tested on
synthetic logs; that test found a window-overlap bug, now fixed.
- **Finding: Mbed TLS is not thread-safe here.** Bundle mode made logins fast enough
that two bots on different threads verified their first connections at once. Both were
refused as `untrusted` (QUIC cause 3), which surfaced as `close_reasons=259:2`. Mbed TLS
and PSA are built without `MBEDTLS_THREADING_C`. Every network-library entry point that
touches them now holds one process-wide recursive lock
(`include/evo/network/crypto_lock.hpp`): the QUIC client's pinned check and the
session-grant and handoff-ticket keys. The smoke failed 2 of 2 runs before and passed 3
of 3 after. `quic-transport` gains a concurrent-connect test (8 threads, 5 rounds); it
does not reliably reproduce the race, because PSA is already initialised there.
- `evo-load` step lines carry `close_reasons=REASON:COUNT,…`: session close reasons, or
256 plus the QUIC cause when there was no session Close.
- **Not yet:** the reference run. It needs an image containing this change promoted to
dev, a load bundle made where the issuer key lives, and the maintainer's go-ahead,
because the drill restarts the dev region twice.
+40
View File
@@ -5118,3 +5118,43 @@ Prevention:
- DEPLOYMENT.md documents the Deployment-only sync symptom and the hook re-run.
Executed: deploy policy check (0 failures) and selftest (46 cases, 0 failures).
## 2026-10-08 — M10.9d load drill harness (ADR-0097)
Implemented:
- `tools/ops/load-drill.sh` and its runbook `docs/networking/M10_LOAD_DRILL.md`:
synthetic inputs on for the run (restored on exit), `evo-load` through the public route,
and server reports matched to each hold.
- evo-server `m10 load-report` every 10 s. The agent cannot reach the ops port: no
port-forward, and the API-server proxy is refused.
- `evo-load --grants FILE --grants-per-bot C` with `exhausted=`, `hold_utc=` and
`close_reasons=`; `evo-session-probe grants --accounts N`.
- `include/evo/network/crypto_lock.hpp`: one recursive lock at every Mbed TLS/PSA entry
point of the network library (`MBEDTLS_THREADING_C` is off).
Findings:
- Concurrent first verifications on different threads were refused as `untrusted`, two
per smoke run in bundle mode (2 of 2 runs). With the lock, 3 of 3 runs had no failures.
- The drill summariser, tested on synthetic logs, counted reports from after the hold
(the next step's ramp). Fixed: only reports wholly inside the hold count.
Tests:
- M10 load smoke in bundle mode (8 accounts × 2 grants) with `exhausted=0`.
- The M10 process test checks `m10 load-report`.
- `quic-transport` `concurrent_clients`: 8 threads × 5 rounds all connect. It does not
reliably reproduce the race; the smoke does.
- The lease-loss test's close reasons read 24 × `shutdown` (drain) and 24 × idle
(SIGKILL).
Executed:
- **Linux GCC 14:** presets `quic` and `network` 5/5; `postgres-check.sh` with QUIC, all 18
suites.
- **Linux Clang 19:** clang-tidy 19 on `load.cpp`, the probe, `session_grant.cpp`,
`handoff_ticket.cpp`, `quic_endpoint.cpp`, `quic_transport.cpp`, evo-server `main.cpp`
with the bridge, and the M10 process tests (3 findings fixed); clang-format 19.
- **Windows MSVC 19.44:** presets `quic` 5/5 and `client-network` 15/15.
- **Drill script:** `bash -n`, plus the summariser on synthetic logs; shellcheck is not
installed here.
Not done: the reference run on evolution-dev. It needs this image promoted, a load bundle
and the maintainer's go-ahead.
+74
View File
@@ -0,0 +1,74 @@
# M10 load drill (evolution-dev reference worker)
ADR-0097 qualifies M10 load only on the evolution-dev reference worker.
`tools/ops/load-drill.sh` performs that run. It is a drill: it changes the live dev region
for the run's duration and restores it on exit, including on failure. It is never part of
CI.
## What it changes
- `REGION_INPUTS=development-v1` on `Deployment/evo-region` while it runs, the drill-time
override of ADR-0073. Synthetic inputs stay off in Git (deploy policy). On exit the
value goes back to empty, and the region restarts twice: once to enable, once to
restore.
- Nothing else: no Secrets, migrations, Jobs, SQL or Argo operations. While it runs, Argo
shows `Deployment/evo-region` OutOfSync, and Synced again after the restore.
## Inputs
In a tokens directory:
- `dev-ca.pem`: the current dev session CA, from the renewal.
- `load-grants.bin`: one bundle with grants for every bot, made where the dev issuer key
lives (the key never leaves that machine). For the default drill (up to 32 bots, 4
logins each):
```bash
evo-session-probe grants --issuer-key issuer.key --world 100 --frame 9 --region 1 \
--account <HEX32 base account> --accounts 32 --count 4 --lifetime 14400 \
--out load-grants.bin
```
Account i is the base account with its last two bytes set to i + 1. Each bot uses its
own 4 grants in order, one per login, and a bot that has used all 4 stays out
(`exhausted=` in the report). Grants are valid for at most 4 hours.
The session window (`REGION_SESSION_WINDOW`) must stay open until the drill ends; the
script refuses to start otherwise. The agent's kubectl access is enough: patch
`Deployment/evo-region`, read pods and logs.
## Running
```bash
EVO_LOAD=/path/to/evo-load tools/ops/load-drill.sh <tokens-dir> \
[--steps 20,24,28,32] [--hold 60] [--warmup 10] [--grants-per-bot 4]
```
Defaults: steps 20, 24, 28 and 32 bots; 60 s holds; 10 s warmups; 4 grants per bot;
`void.aeternalisgames.com:7400`. The drill takes about 6 minutes plus two region
restarts.
## Output
- **evo-load's per-step lines** (see `tools/network/load.cpp`): client distributions
(response, receipt, outcome, login), snapshot cadence, starvation, `close_reasons`
(why sessions or logins ended) and a client-side verdict, then
`load result sustainable=N saturated_at=M`.
- **One `drill step` line per step**, from the server's `m10 load-report` log lines
(every 10 s, read with `kubectl logs --timestamps` because the ops port is not
reachable for the agent) that fall inside the step's hold:
- the worst work p99 and maximum;
- the session-layer p99 and its per-session share;
- the governor's refusing and shed state;
- tick debt;
- `server_within_budget`: work p99 ≤ 25 ms, session p99 ≤ 0.5 ms × sessions, no debt.
The work directory (kept and printed at the end) holds `load.log` and `server.log`.
## Interpreting
A step qualifies when both evo-load's verdict and `server_within_budget` are 1. The
largest such step is the region's sustainable session count on the reference worker.
That number, with the first failing step and its reasons, goes into ADR-0097 and
`schemas/network/m10-budgets-v1.md`. If the governor engages (refusing or shed above 0),
that step is the overload response in action, and the thresholds are revisited
against the measured work.
+18
View File
@@ -0,0 +1,18 @@
#pragma once
#include <mutex>
// M10.9d (ADR-0097): Mbed TLS and PSA are built without MBEDTLS_THREADING_C, so their
// global state (the PSA core, key slots, the random generator) is not safe to use from
// several threads at once. Every network-library entry point that touches them takes this
// one process-wide lock: the QUIC client's pinned certificate check and the session grant
// and handoff ticket keys. Recursive, since entry points may call each other. Calls are rare
// (per connection, per grant), so the lock costs nothing measurable. Found by evo-load: two
// bots on different threads verifying their first connections at once were refused as
// `untrusted`.
namespace evo::network {
inline std::recursive_mutex& crypto_mutex() noexcept {
static std::recursive_mutex mutex;
return mutex;
}
using CryptoLock = std::scoped_lock<std::recursive_mutex>;
} // namespace evo::network
+11
View File
@@ -1,4 +1,5 @@
#include "evo/network/handoff_ticket.hpp"
#include "evo/network/crypto_lock.hpp"
#include <algorithm>
#include <psa/crypto.h>
#include <utility>
@@ -57,6 +58,7 @@ bool looks_like_handoff_ticket(std::span<const std::uint8_t> bytes) noexcept {
std::expected<HandoffSigningKey, HandoffTicketError>
HandoffSigningKey::generate(DeploymentEnvironment environment, std::uint32_t key_id) {
const CryptoLock crypto{crypto_mutex()};
if (key_id == 0)
return std::unexpected(HandoffTicketError::invalid_key);
if (psa_crypto_init() != PSA_SUCCESS)
@@ -73,6 +75,7 @@ HandoffSigningKey::generate(DeploymentEnvironment environment, std::uint32_t key
std::expected<HandoffSigningKey, HandoffTicketError>
HandoffSigningKey::import(DeploymentEnvironment environment, std::uint32_t key_id,
std::span<const std::uint8_t> private_key) {
const CryptoLock crypto{crypto_mutex()};
if (key_id == 0 || private_key.size() != grant_private_key_bytes)
return std::unexpected(HandoffTicketError::invalid_key);
if (psa_crypto_init() != PSA_SUCCESS)
@@ -90,6 +93,7 @@ HandoffSigningKey::HandoffSigningKey(HandoffSigningKey&& other) noexcept
key_{std::exchange(other.key_, PSA_KEY_ID_NULL)} {}
HandoffSigningKey& HandoffSigningKey::operator=(HandoffSigningKey&& other) noexcept {
const CryptoLock crypto{crypto_mutex()};
if (this != &other) {
if (key_ != PSA_KEY_ID_NULL)
(void)psa_destroy_key(key_);
@@ -101,12 +105,14 @@ HandoffSigningKey& HandoffSigningKey::operator=(HandoffSigningKey&& other) noexc
}
HandoffSigningKey::~HandoffSigningKey() {
const CryptoLock crypto{crypto_mutex()};
if (key_ != PSA_KEY_ID_NULL)
(void)psa_destroy_key(key_);
}
std::expected<HandoffTicket, HandoffTicketError>
HandoffSigningKey::sign(const TicketClaims& claims) const {
const CryptoLock crypto{crypto_mutex()};
if (key_ == PSA_KEY_ID_NULL)
return std::unexpected(HandoffTicketError::invalid_key);
if (!structurally_valid(claims))
@@ -134,6 +140,7 @@ HandoffSigningKey::sign(const TicketClaims& claims) const {
}
std::expected<GrantPublicKey, HandoffTicketError> HandoffSigningKey::public_key() const {
const CryptoLock crypto{crypto_mutex()};
if (key_ == PSA_KEY_ID_NULL)
return std::unexpected(HandoffTicketError::invalid_key);
GrantPublicKey out{};
@@ -147,6 +154,7 @@ std::expected<GrantPublicKey, HandoffTicketError> HandoffSigningKey::public_key(
std::expected<HandoffVerifier, HandoffTicketError>
HandoffVerifier::create(DeploymentEnvironment environment, world::WorldIdentity world,
std::uint64_t region) {
const CryptoLock crypto{crypto_mutex()};
if (world.world == 0 || world.frame == 0 || region == 0)
return std::unexpected(HandoffTicketError::invalid_claims);
if (psa_crypto_init() != PSA_SUCCESS)
@@ -174,6 +182,7 @@ HandoffVerifier::~HandoffVerifier() {
}
void HandoffVerifier::destroy() noexcept {
const CryptoLock crypto{crypto_mutex()};
for (const auto& k : keys_)
(void)psa_destroy_key(k.key);
keys_.clear();
@@ -182,6 +191,7 @@ void HandoffVerifier::destroy() noexcept {
std::expected<void, HandoffTicketError>
HandoffVerifier::add_key(std::uint64_t source_region, std::uint32_t key_id,
std::span<const std::uint8_t> public_key) {
const CryptoLock crypto{crypto_mutex()};
if (source_region == 0 || source_region == region_ || key_id == 0 ||
public_key.size() != grant_public_key_bytes)
return std::unexpected(HandoffTicketError::invalid_key);
@@ -204,6 +214,7 @@ HandoffVerifier::add_key(std::uint64_t source_region, std::uint32_t key_id,
std::expected<VerifiedHandoffTicket, HandoffTicketError>
HandoffVerifier::verify(std::span<const std::uint8_t> ticket, std::uint64_t now_seconds) const {
const CryptoLock crypto{crypto_mutex()};
if (ticket.size() != handoff_ticket_bytes)
return std::unexpected(HandoffTicketError::malformed);
// The key id only selects which key must verify; it is itself signed.
+12
View File
@@ -1,4 +1,5 @@
#include "evo/network/session_grant.hpp"
#include "evo/network/crypto_lock.hpp"
#include <algorithm>
#include <psa/crypto.h>
#include <type_traits>
@@ -51,6 +52,7 @@ psa_key_attributes_t pair_attributes() {
} // namespace
std::expected<void, SessionGrantError> random_bytes(std::span<std::uint8_t> out) {
const CryptoLock crypto{crypto_mutex()};
if (psa_crypto_init() != PSA_SUCCESS ||
psa_generate_random(out.data(), out.size()) != PSA_SUCCESS)
return std::unexpected(SessionGrantError::crypto);
@@ -68,6 +70,7 @@ std::expected<AccountId, SessionGrantError> generate_account_id() {
std::expected<GrantSigningKey, SessionGrantError>
GrantSigningKey::generate(DeploymentEnvironment environment, std::uint32_t key_id) {
const CryptoLock crypto{crypto_mutex()};
if (key_id == 0)
return std::unexpected(SessionGrantError::invalid_key);
if (psa_crypto_init() != PSA_SUCCESS)
@@ -84,6 +87,7 @@ GrantSigningKey::generate(DeploymentEnvironment environment, std::uint32_t key_i
std::expected<GrantSigningKey, SessionGrantError>
GrantSigningKey::import(DeploymentEnvironment environment, std::uint32_t key_id,
std::span<const std::uint8_t> private_key) {
const CryptoLock crypto{crypto_mutex()};
if (key_id == 0 || private_key.size() != grant_private_key_bytes)
return std::unexpected(SessionGrantError::invalid_key);
if (psa_crypto_init() != PSA_SUCCESS)
@@ -101,6 +105,7 @@ GrantSigningKey::GrantSigningKey(GrantSigningKey&& other) noexcept
key_{std::exchange(other.key_, PSA_KEY_ID_NULL)} {}
GrantSigningKey& GrantSigningKey::operator=(GrantSigningKey&& other) noexcept {
const CryptoLock crypto{crypto_mutex()};
if (this != &other) {
if (key_ != PSA_KEY_ID_NULL)
(void)psa_destroy_key(key_);
@@ -112,12 +117,14 @@ GrantSigningKey& GrantSigningKey::operator=(GrantSigningKey&& other) noexcept {
}
GrantSigningKey::~GrantSigningKey() {
const CryptoLock crypto{crypto_mutex()};
if (key_ != PSA_KEY_ID_NULL)
(void)psa_destroy_key(key_);
}
std::expected<SessionGrant, SessionGrantError>
GrantSigningKey::sign(const GrantClaims& claims) const {
const CryptoLock crypto{crypto_mutex()};
if (key_ == PSA_KEY_ID_NULL)
return std::unexpected(SessionGrantError::invalid_key);
if (!structurally_valid(claims))
@@ -145,6 +152,7 @@ GrantSigningKey::sign(const GrantClaims& claims) const {
}
std::expected<GrantPublicKey, SessionGrantError> GrantSigningKey::public_key() const {
const CryptoLock crypto{crypto_mutex()};
if (key_ == PSA_KEY_ID_NULL)
return std::unexpected(SessionGrantError::invalid_key);
GrantPublicKey out{};
@@ -157,6 +165,7 @@ std::expected<GrantPublicKey, SessionGrantError> GrantSigningKey::public_key() c
std::expected<GrantVerifier, SessionGrantError>
GrantVerifier::create(DeploymentEnvironment environment) {
const CryptoLock crypto{crypto_mutex()};
if (psa_crypto_init() != PSA_SUCCESS)
return std::unexpected(SessionGrantError::crypto);
return GrantVerifier{environment};
@@ -179,6 +188,7 @@ GrantVerifier::~GrantVerifier() {
}
void GrantVerifier::destroy() noexcept {
const CryptoLock crypto{crypto_mutex()};
for (const auto& k : keys_)
(void)psa_destroy_key(k.key);
keys_.clear();
@@ -186,6 +196,7 @@ void GrantVerifier::destroy() noexcept {
std::expected<void, SessionGrantError>
GrantVerifier::add_key(std::uint32_t key_id, std::span<const std::uint8_t> public_key) {
const CryptoLock crypto{crypto_mutex()};
if (key_id == 0 || public_key.size() != grant_public_key_bytes)
return std::unexpected(SessionGrantError::invalid_key);
if (std::ranges::any_of(keys_, [&](const Key& k) { return k.key_id == key_id; }))
@@ -207,6 +218,7 @@ GrantVerifier::add_key(std::uint32_t key_id, std::span<const std::uint8_t> publi
std::expected<VerifiedSessionGrant, SessionGrantError>
GrantVerifier::verify(std::span<const std::uint8_t> grant) const {
const CryptoLock crypto{crypto_mutex()};
if (grant.size() != session_grant_bytes)
return std::unexpected(SessionGrantError::malformed);
// The key id only selects which key must verify; it is itself signed.
+2
View File
@@ -11,6 +11,7 @@
#endif
#endif
#include "evo/network/crypto_lock.hpp"
#include "evo/network/quic_transport.hpp"
#include <algorithm>
#include <array>
@@ -43,6 +44,7 @@ bool tls_or_certificate_failure(QUIC_STATUS status) {
// Leaf (DER) verified against exactly one CA (PEM) for one name; nothing else trusted.
bool pinned(std::span<const std::uint8_t> leaf, const std::string& authority_pem,
const std::string& name) {
const evo::network::CryptoLock crypto{evo::network::crypto_mutex()};
if (psa_crypto_init() != PSA_SUCCESS || leaf.empty())
return false;
mbedtls_x509_crt ca;
+62
View File
@@ -5,12 +5,15 @@
#include "evo/network/dev_credentials.hpp"
#include "udp_relay.hpp"
#include <algorithm>
#include <array>
#include <atomic>
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <functional>
#include <latch>
#include <set>
#include <source_location>
#include <stdexcept>
@@ -558,8 +561,67 @@ void garbage_and_close() {
}
} // namespace
// M10.9d (ADR-0097): client endpoints on several threads connecting at the same moment all
// connect and pass the owned pin check. Mbed TLS runs without its threading layer; evo-load
// saw two concurrent first verifications refused as `untrusted` before the shared crypto
// lock. This test covers concurrent connects; it does not reliably reproduce that race (PSA
// is already initialised here by the credential generation), which evo-load's smoke does.
void concurrent_clients() {
const auto f = credentials("concurrent");
Limits limits;
limits.max_connections = 64; // 5 rounds x 8, whatever the server has not yet reaped.
auto server = serve(f, limits);
constexpr int threads = 8;
for (int round = 0; round < 5; ++round) {
std::atomic<int> connected{0}, finished{0};
std::array<std::atomic<int>, 8> causes{};
std::latch start{threads};
std::vector<std::thread> clients;
clients.reserve(threads);
for (int t = 0; t < threads; ++t)
clients.emplace_back([&] {
start.arrive_and_wait();
auto client = dial(f, server->port());
const auto end = std::chrono::steady_clock::now() + std::chrono::seconds{10};
bool done = false;
while (!done && std::chrono::steady_clock::now() < end) {
for (const auto& e : client->poll()) {
if (e.kind == EventKind::connected) {
++connected;
done = true;
} else if (e.kind == EventKind::closed) {
causes[static_cast<std::size_t>(e.cause)] += 1;
done = true;
}
}
std::this_thread::sleep_for(std::chrono::milliseconds{1});
}
++finished;
});
while (finished < threads) {
(void)server->poll();
std::this_thread::sleep_for(std::chrono::milliseconds{1});
}
for (auto& c : clients)
c.join();
if (connected != threads) {
std::printf("concurrent round %d connected=%d", round, connected.load());
for (std::size_t i = 0; i < causes.size(); ++i)
std::printf(" cause%zu=%d", i, causes.at(i).load());
std::printf("\n");
}
CHECK(connected == threads);
for (int i = 0; i < 200; ++i) { // Let the server see the closes.
(void)server->poll();
std::this_thread::sleep_for(std::chrono::milliseconds{1});
}
}
std::filesystem::remove_all(f.dir);
}
int main() try {
framing();
concurrent_clients();
channels();
pinning();
impaired();
+17 -4
View File
@@ -462,7 +462,8 @@ void end_to_end() {
server->child->signal(SIGTERM);
CHECK(server->child->wait_exit(30s) == 0);
CHECK(server->child->printed("m10 opened ") && server->child->printed("m10 committed ") &&
server->child->printed("m10 motion area "));
server->child->printed("m10 motion area ") &&
server->child->printed("m10 load-report sessions="));
}
// M10.9b (ADR-0097): evo-load against a real evo-server, two steps with the region's
// metrics. A smoke test of the tool and the server under concurrent sessions, not
@@ -473,6 +474,15 @@ void load_smoke() {
Setup s{152, 0xC6};
Server server{s};
const auto tool = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-load";
// M10.9d: the bots log in from a bundle (as the dev drill does): 8 accounts x 2 grants.
const auto probe = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-session-probe";
const auto bundle = s.region.root / "load-grants.bin";
Child g{{probe.string(), "grants", "--issuer-key", (s.region.root / "issuer.key").string(),
"--world", "7", "--frame", std::to_string(region_identity.frame),
"--region", "152", "--account", "0000000000000000000000000000b000",
"--accounts", "8", "--count", "2",
"--lifetime", "900", "--out", bundle.string()}};
CHECK(g.wait_exit(30s) == 0 && g.printed("wrote 16 grants "));
Child load{{tool.string(),
"--connect",
"127.0.0.1:" + std::to_string(server.port),
@@ -480,8 +490,10 @@ void load_smoke() {
name,
"--ca",
(s.region.root / "ca.pem").string(),
"--issuer-key",
(s.region.root / "issuer.key").string(),
"--grants",
bundle.string(),
"--grants-per-bot",
"2",
"--world",
"7",
"--frame",
@@ -522,7 +534,8 @@ void load_smoke() {
// Every bot logged in once: 4 in the first step, 4 more in the second.
CHECK((starts(line, "load step N=4 established=4 logins=4 ") ||
starts(line, "load step N=8 established=8 logins=4 ")) &&
line.find(" drops=0 login_failures=0 refused=0 ") != std::string::npos);
line.find(" drops=0 login_failures=0 refused=0 ") != std::string::npos &&
line.find(" exhausted=0 hold_utc=") != std::string::npos);
}
const auto has = [&](std::string_view prefix) {
return std::ranges::count_if(load.lines(), [&](const std::string& l) {
+107 -33
View File
@@ -2,7 +2,8 @@
// runs against the evolution-dev reference worker count as evidence (10.9d); local runs are
// smoke tests and development references.
//
// evo-load --connect HOST:PORT --server-name NAME --ca FILE --issuer-key FILE
// evo-load --connect HOST:PORT --server-name NAME --ca FILE
// (--issuer-key FILE | --grants FILE --grants-per-bot C)
// --world N --frame N --region N [--steps N[,N]...] [--hold S] [--warmup S]
// [--ramp-ms MS] [--threads T] [--seed X] [--lifetime S] [--metrics HOST:PORT]
// [--response-budget-ms MS] [--per-client] [--expect-established]
@@ -14,11 +15,17 @@
// probe marker; every 2 s it sends a probe action; every 10 s it moves its interest by a
// cell (a fresh baseline). A closed or refused bot logs in again after 2 s with the next
// serial.
// M10.9d: with --grants (a bundle from `evo-session-probe grants --accounts N --count C`,
// e.g. made where the dev issuer key lives), bot i logs in with grants i*C .. i*C+C-1 in
// order instead of signing; a bot that used all C stays out (counted as exhausted).
//
// Steps: the bot count rises to each step's N (one login every --ramp-ms), waits --warmup
// seconds, then measures for --hold seconds. Bots stay connected into the next step. Per
// step it prints:
// load step N=.. established=.. logins=.. drops=.. login_failures=.. refused=.. seconds=..
// exhausted=.. hold_utc=START..END (unix seconds of the hold, to match server samples)
// close_reasons=REASON:COUNT,... (session-v1 close reasons; without a session Close,
// 256 + the QUIC transport cause, or 255 when unknown)
// (drops: established sessions closed; login failures: closed before Welcome, including
// transport refusals at the connection limit; refused: closed with capacity or
// rate_limited, either way. Logins, drops and failures include the step's ramp and warmup;
@@ -111,7 +118,8 @@ double ms(std::uint64_t us) {
}
struct Options {
std::string host, server_name, ca, issuer_key, metrics_host;
std::string host, server_name, ca, issuer_key, grants_file, metrics_host;
std::uint64_t grants_per_bot{};
std::uint16_t port{}, metrics_port{};
std::uint64_t world{}, frame{}, region{}, seed{1}, lifetime{900};
std::vector<std::size_t> steps{20};
@@ -125,6 +133,7 @@ struct Samples {
gap_ticks;
std::uint64_t snapshots{}, bytes{}, intents{}, refused_intents{}, logins{}, drops{},
login_failures{}, refused{};
std::map<unsigned, std::uint64_t> closes; // By session close reason (255: no Close).
std::uint64_t connected_polls{}; // Polls while established (starvation is judged on these).
};
void append(std::vector<std::uint64_t>& to, const std::vector<std::uint64_t>& from) {
@@ -142,6 +151,8 @@ struct Bot {
client::InputLead lead;
std::uint64_t last_tick{}, last_view{}, last_bytes{}, next_action_id{1};
std::uint64_t applied{}; // Newest applied intention sequence seen in an own marker.
std::uint64_t used{}; // Bundle grants used (bundle mode).
std::atomic<bool> exhausted{false};
std::map<std::uint64_t, Clock::time_point> receipt_pending, response_pending, outcome_pending;
std::map<std::uint64_t, std::uint64_t> intent_tick;
std::int16_t vx{}, vy{};
@@ -153,9 +164,11 @@ struct Bot {
class Load {
public:
Load(Options o, GrantSigningKey key, std::string address)
: o_{std::move(o)}, key_{std::move(key)}, address_{std::move(address)},
authority_{read(o_.ca)} {
// Signs with `key`, or logs in from `bundle` (concatenated grants, grants_per_bot each).
Load(Options o, std::optional<GrantSigningKey> key, std::vector<SessionGrant> bundle,
std::string address)
: o_{std::move(o)}, key_{std::move(key)}, bundle_{std::move(bundle)},
address_{std::move(address)}, authority_{read(o_.ca)} {
const auto most = *std::ranges::max_element(o_.steps);
bots_.reserve(most);
for (std::size_t i = 0; i < most; ++i) {
@@ -199,6 +212,11 @@ class Load {
}
return out;
}
[[nodiscard]] std::size_t exhausted(std::size_t n) const {
return static_cast<std::size_t>(
std::count_if(bots_.begin(), bots_.begin() + static_cast<std::ptrdiff_t>(n),
[](const auto& b) { return b->exhausted.load(); }));
}
[[nodiscard]] std::size_t established(std::size_t n) const {
return static_cast<std::size_t>(
std::count_if(bots_.begin(), bots_.begin() + static_cast<std::ptrdiff_t>(n),
@@ -229,15 +247,26 @@ class Load {
}
}
void login(Bot& b, Clock::time_point now) {
GrantClaims claims;
claims.account = b.account;
claims.serial = ++b.serial;
claims.world = {o_.world, o_.frame};
claims.region = o_.region;
claims.issued_at = now_seconds();
claims.expires_at = claims.issued_at + o_.lifetime;
claims.grant_id = (b.serial << 16U) ^ (b.index + 1);
const auto grant = key_.sign(claims);
std::optional<SessionGrant> grant;
if (key_) {
GrantClaims claims;
claims.account = b.account;
claims.serial = ++b.serial;
claims.world = {o_.world, o_.frame};
claims.region = o_.region;
claims.issued_at = now_seconds();
claims.expires_at = claims.issued_at + o_.lifetime;
claims.grant_id = (b.serial << 16U) ^ (b.index + 1);
if (auto signed_grant = key_->sign(claims))
grant = *signed_grant;
} else {
if (b.used >= o_.grants_per_bot) {
b.exhausted = true; // Out of grants: stays out.
b.retry_at = Clock::time_point::max();
return;
}
grant = bundle_.at((b.index * o_.grants_per_bot) + b.used++);
}
client::RegionSessionClientConfig config;
config.transport.server_address = address_;
config.transport.port = o_.port;
@@ -278,6 +307,13 @@ class Load {
++b.s.drops;
else
++b.s.login_failures;
if (const auto reason = c.close_reason();
reason && *reason != SessionCloseReason::connection_lost)
++b.s.closes[static_cast<unsigned>(*reason)];
else if (const auto cause = c.transport_cause()) // No session Close: 256 + cause.
++b.s.closes[256U + static_cast<unsigned>(*cause)];
else
++b.s.closes[255U];
if (c.close_reason() == SessionCloseReason::capacity ||
c.close_reason() == SessionCloseReason::rate_limited)
++b.s.refused;
@@ -406,7 +442,8 @@ class Load {
}
Options o_;
GrantSigningKey key_;
std::optional<GrantSigningKey> key_;
std::vector<SessionGrant> bundle_;
std::string address_, authority_;
std::vector<std::unique_ptr<Bot>> bots_;
std::vector<std::mutex> locks_;
@@ -481,9 +518,9 @@ void line(const char* name, const std::vector<std::uint64_t>& pooled,
}
// One step's report; true when within budget.
bool report(const Options& o, std::size_t n, std::size_t established,
const std::vector<Samples>& all, double seconds, double cpu,
const std::map<std::string, std::uint64_t>& server) {
bool report(const Options& o, std::size_t n, std::size_t established, std::size_t exhausted,
std::pair<std::uint64_t, std::uint64_t> hold_utc, const std::vector<Samples>& all,
double seconds, double cpu, const std::map<std::string, std::uint64_t>& server) {
Samples pooled;
std::vector<std::uint64_t> response95, receipt95, outcome95, login95, correction95, rates, kib;
for (const auto& s : all) {
@@ -499,6 +536,8 @@ bool report(const Options& o, std::size_t n, std::size_t established,
pooled.logins += s.logins;
pooled.drops += s.drops;
pooled.login_failures += s.login_failures;
for (const auto& [why, count] : s.closes)
pooled.closes[why] += count;
pooled.refused += s.refused;
response95.push_back(percentile(s.response_us, 0.95));
receipt95.push_back(percentile(s.receipt_us, 0.95));
@@ -511,12 +550,18 @@ bool report(const Options& o, std::size_t n, std::size_t established,
kib.push_back(static_cast<std::uint64_t>(
std::lround(static_cast<double>(s.bytes) / 1024.0 / std::max(seconds, 0.001))));
}
std::string reasons;
for (const auto& [why, count] : pooled.closes)
reasons += (reasons.empty() ? "" : ",") + std::to_string(why) + ":" + std::to_string(count);
std::printf("load step N=%zu established=%zu logins=%llu drops=%llu login_failures=%llu "
"refused=%llu seconds=%.1f\n",
"refused=%llu seconds=%.1f exhausted=%zu hold_utc=%llu..%llu close_reasons=%s\n",
n, established, static_cast<unsigned long long>(pooled.logins),
static_cast<unsigned long long>(pooled.drops),
static_cast<unsigned long long>(pooled.login_failures),
static_cast<unsigned long long>(pooled.refused), seconds);
static_cast<unsigned long long>(pooled.refused), seconds, exhausted,
static_cast<unsigned long long>(hold_utc.first),
static_cast<unsigned long long>(hold_utc.second),
reasons.empty() ? "-" : reasons.c_str());
line("response", pooled.response_us, response95, 1000.0, "ms");
line("receipt", pooled.receipt_us, receipt95, 1000.0, "ms");
line("outcome", pooled.outcome_us, outcome95, 1000.0, "ms");
@@ -663,6 +708,10 @@ int main(int argc, char** argv) {
o.ca = value;
else if (flag == "--issuer-key")
o.issuer_key = value;
else if (flag == "--grants")
o.grants_file = value;
else if (n && flag == "--grants-per-bot" && *n >= 1 && *n <= 64)
o.grants_per_bot = *n;
else if (flag == "--steps") {
const auto s = steps_of(value);
ok = s.has_value();
@@ -676,7 +725,7 @@ int main(int argc, char** argv) {
o.region = *n;
else if (n && flag == "--seed" && *n != 0)
o.seed = *n;
else if (n && flag == "--lifetime" && *n >= 60 && *n <= 3600)
else if (n && flag == "--lifetime" && *n >= 60 && *n <= 14400)
o.lifetime = *n;
else if (n && flag == "--hold" && *n >= 1 && *n <= 3600)
o.hold_s = *n;
@@ -691,23 +740,44 @@ int main(int argc, char** argv) {
else
ok = false;
}
if (!ok || o.host.empty() || o.server_name.empty() || o.ca.empty() || o.issuer_key.empty() ||
o.world == 0 || o.frame == 0 || o.region == 0) {
const bool signs = !o.issuer_key.empty();
if (!ok || o.host.empty() || o.server_name.empty() || o.ca.empty() ||
signs == !o.grants_file.empty() || (!signs && o.grants_per_bot == 0) || o.world == 0 ||
o.frame == 0 || o.region == 0) {
std::fputs("usage: evo-load --connect HOST:PORT --server-name NAME --ca FILE "
"--issuer-key FILE --world N --frame N --region N [--steps N[,N]...] "
"(--issuer-key FILE | --grants FILE --grants-per-bot C) --world N "
"--frame N --region N [--steps N[,N]...] "
"[--hold S] [--warmup S] [--ramp-ms MS] [--threads T] [--seed X] "
"[--lifetime S] [--metrics HOST:PORT] [--response-budget-ms MS] "
"[--per-client] [--expect-established]\n",
stderr);
return 2;
}
const auto scalar = read(o.issuer_key);
auto key = GrantSigningKey::import(
DeploymentEnvironment::dev, 1,
std::span{reinterpret_cast<const std::uint8_t*>(scalar.data()), scalar.size()});
if (!key) {
std::puts("error issuer-key");
return 2;
std::optional<GrantSigningKey> key;
std::vector<SessionGrant> bundle;
if (signs) {
const auto scalar = read(o.issuer_key);
auto imported = GrantSigningKey::import(
DeploymentEnvironment::dev, 1,
std::span{reinterpret_cast<const std::uint8_t*>(scalar.data()), scalar.size()});
if (!imported) {
std::puts("error issuer-key");
return 2;
}
key.emplace(std::move(*imported));
} else {
const auto bytes = read(o.grants_file);
const auto needed = *std::ranges::max_element(o.steps) * o.grants_per_bot;
if (bytes.size() % session_grant_bytes != 0 ||
bytes.size() / session_grant_bytes < needed) {
std::printf("error grants (need %llu grants: bots x grants-per-bot)\n",
static_cast<unsigned long long>(needed));
return 2;
}
bundle.resize(bytes.size() / session_grant_bytes);
for (std::size_t i = 0; i < bundle.size(); ++i)
std::copy_n(bytes.begin() + static_cast<std::ptrdiff_t>(i * session_grant_bytes),
session_grant_bytes, reinterpret_cast<char*>(bundle[i].data()));
}
const auto address = quic::resolve_ipv4(o.host);
if (!address) {
@@ -723,7 +793,7 @@ int main(int argc, char** argv) {
bool smoke_ok = true;
std::optional<std::size_t> sustainable, saturated;
{
Load load{o, std::move(*key), *address};
Load load{o, std::move(key), std::move(bundle), *address};
for (const auto n : o.steps) {
load.activate(n);
std::this_thread::sleep_for(std::chrono::seconds{o.warmup_s});
@@ -731,6 +801,7 @@ int main(int argc, char** argv) {
// the warmup is discarded.
const auto warm = load.take(n);
const auto started = Clock::now();
const auto hold_start = now_seconds();
const auto cpu0 = cpu_seconds();
std::this_thread::sleep_for(std::chrono::seconds{o.hold_s});
auto samples = load.take(n);
@@ -739,12 +810,15 @@ int main(int argc, char** argv) {
samples[i].logins += warm[i].logins;
samples[i].drops += warm[i].drops;
samples[i].login_failures += warm[i].login_failures;
for (const auto& [why, count] : warm[i].closes)
samples[i].closes[why] += count;
samples[i].refused += warm[i].refused;
}
const auto seconds = std::chrono::duration<double>(Clock::now() - started).count();
const auto established = load.established(n);
const bool within =
report(o, n, established, samples, seconds, cpu_seconds() - cpu0, scrape(o));
report(o, n, established, load.exhausted(n), {hold_start, now_seconds()}, samples,
seconds, cpu_seconds() - cpu0, scrape(o));
if (within && !saturated)
sustainable = n;
if (!within && !saturated)
+161
View File
@@ -0,0 +1,161 @@
#!/usr/bin/env bash
# M10.9d load drill (ADR-0097) for evolution-dev only: the reference-worker qualification run.
#
# tools/ops/load-drill.sh <tokens-dir> [--context <name>] [--steps N[,N]...] [--hold S]
# [--warmup S] [--grants-per-bot C] [--connect HOST:PORT] [--server-name NAME]
#
# <tokens-dir> holds dev-ca.pem (the current dev session CA) and load-grants.bin, a bundle
# made where the dev issuer key lives:
# evo-session-probe grants --issuer-key issuer.key --world W --frame F --region R \
# --account <HEX32> --accounts <most bots> --count <grants per bot> --lifetime 14400 \
# --out load-grants.bin
# The issuer key never reaches the machine running the drill. Requires kubectl access to
# evolution-dev (patch Deployment/evo-region, read pods and logs), `evo-load` (EVO_LOAD,
# default: on PATH) and python3.
#
# Steps: refuse unless the session window outlasts the drill; enable synthetic inputs on
# evo-region (the drill-time override of ADR-0073, never committed) and wait for the new
# pod to be Ready; follow its log with timestamps (the server prints `m10 load-report`
# every 10 s); run evo-load through the public route; then restore REGION_INPUTS to the Git
# value (also on failure) so Argo returns to Synced. The report matches each step's hold
# (evo-load's hold_utc) with the server's reports in it and prints, per step, the client
# verdict and the server's worst tick work, session work, governor state and tick debt.
# Stage and World0 are never touched: the namespace is fixed.
set -euo pipefail
usage() {
echo "usage: $0 <tokens-dir> [--context <name>] [--steps N[,N]...] [--hold S] [--warmup S]" \
"[--grants-per-bot C] [--connect HOST:PORT] [--server-name NAME]" >&2
exit 2
}
[ $# -ge 1 ] || usage
tokens="$1"; shift
k=(kubectl --request-timeout=30s)
steps=20,24,28,32 hold=60 warmup=10 per_bot=4
connect=void.aeternalisgames.com:7400 server_name=void.aeternalisgames.com
while [ $# -gt 0 ]; do
case "$1" in
--context) [ -n "${2:-}" ] || usage; k+=(--context "$2"); shift 2 ;;
--steps) [ -n "${2:-}" ] || usage; steps="$2"; shift 2 ;;
--hold) [ -n "${2:-}" ] || usage; hold="$2"; shift 2 ;;
--warmup) [ -n "${2:-}" ] || usage; warmup="$2"; shift 2 ;;
--grants-per-bot) [ -n "${2:-}" ] || usage; per_bot="$2"; shift 2 ;;
--connect) [ -n "${2:-}" ] || usage; connect="$2"; shift 2 ;;
--server-name) [ -n "${2:-}" ] || usage; server_name="$2"; shift 2 ;;
*) usage ;;
esac
done
ns=evolution-dev # Fixed: qualification runs on the dev reference worker only.
tool="${EVO_LOAD:-evo-load}"
py="$(command -v python3 || command -v python || true)"
for f in dev-ca.pem load-grants.bin; do
[ -f "$tokens/$f" ] || { echo "missing $tokens/$f" >&2; exit 2; }
done
command -v "$tool" > /dev/null || [ -x "$tool" ] || { echo "evo-load not found (set EVO_LOAD)" >&2; exit 2; }
[ -n "$py" ] || { echo "python3 is required" >&2; exit 2; }
selector=app.kubernetes.io/name=evo-region
work="$(mktemp -d)"
chmod 700 "$work"
follow_pid=""
say() { echo "$(date -u +%H:%M:%S) $*"; }
fail() { say "FAIL $*"; exit 1; }
env_of() {
"${k[@]}" get deployment -n "$ns" evo-region \
-o jsonpath="{.spec.template.spec.containers[0].env[?(@.name==\"$1\")].value}"
}
ready_pod() { # Waits up to $2 seconds for a Ready pod other than $1.
local old="$1" limit="${2:-240}"
for _ in $(seq 1 $((limit * 5))); do
for p in $("${k[@]}" get pods -n "$ns" -l "$selector" \
-o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}'); do
[ "$p" = "$old" ] && continue
[ "$("${k[@]}" get pod -n "$ns" "$p" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2> /dev/null)" = True ] &&
{ echo "$p"; return 0; }
done
sleep 0.2
done
return 1
}
restore() {
set +e
[ -n "$follow_pid" ] && kill "$follow_pid" 2> /dev/null
say "restoring the Git configuration of evo-region (REGION_INPUTS empty)"
local current
current="$("${k[@]}" get pods -n "$ns" -l "$selector" -o jsonpath='{.items[0].metadata.name}' 2> /dev/null)"
"${k[@]}" set env -n "$ns" deployment/evo-region REGION_INPUTS= > /dev/null
ready_pod "$current" 240 > /dev/null && say "restored: evo-region Ready" || say "WARN evo-region not Ready after restore"
say "drill files kept in $work"
}
world="$(env_of REGION_WORLD)" region="$(env_of REGION_ID)" frame="$(env_of REGION_FRAME)"
window="$(env_of REGION_SESSION_WINDOW)"
[ -n "$world" ] && [ -n "$region" ] && [ -n "$frame" ] || fail "cannot read the region identity"
[ -z "$(env_of REGION_INPUTS)" ] || fail "REGION_INPUTS is already set: another drill may be running"
# The drill must end before the session window closes.
count=$(echo "$steps" | tr ',' '\n' | wc -l)
most=$(echo "$steps" | tr ',' '\n' | sort -n | tail -1)
needed=$(( count * (hold + warmup + most / 10 + 5) + 600 ))
expiry="${window##*:}"
[ -n "$expiry" ] && [ "$expiry" -gt $(( $(date -u +%s) + needed )) ] 2> /dev/null ||
fail "the session window ($window) closes within the drill's ${needed} s: renew it first"
size=$(wc -c < "$tokens/load-grants.bin")
[ $(( size / 160 )) -ge $(( most * per_bot )) ] ||
fail "load-grants.bin holds $(( size / 160 )) grants; $most bots x $per_bot need $(( most * per_bot ))"
image="$("${k[@]}" get deployment -n "$ns" evo-region -o jsonpath='{.spec.template.spec.containers[0].image}')"
say "drill: world $world region $region image $image steps $steps hold ${hold}s"
trap restore EXIT
before="$("${k[@]}" get pods -n "$ns" -l "$selector" -o jsonpath='{.items[0].metadata.name}')"
say "enabling synthetic inputs in $ns (drill-time override, restored on exit)"
"${k[@]}" set env -n "$ns" deployment/evo-region REGION_INPUTS=development-v1 > /dev/null
pod="$(ready_pod "$before" 240)" || fail "no Ready evo-region pod with synthetic inputs"
"${k[@]}" logs -n "$ns" -f --timestamps "$pod" > "$work/server.log" 2> /dev/null &
follow_pid=$!
sleep 5
grep -q "ready 1" "$work/server.log" || say "WARN no 'ready 1' in the server log yet"
say "running evo-load against $connect"
set +e
"$tool" --connect "$connect" --server-name "$server_name" --ca "$tokens/dev-ca.pem" \
--grants "$tokens/load-grants.bin" --grants-per-bot "$per_bot" --world "$world" \
--frame "$frame" --region "$region" --steps "$steps" --hold "$hold" --warmup "$warmup" \
--threads 8 --per-client > "$work/load.log" 2>&1
load_exit=$?
set -e
sleep 12 # One more server report after the last hold.
grep -E "^load (step|response_ms|outcome_ms|snapshots|starved|verdict|result)" "$work/load.log" || true
[ "$load_exit" = 0 ] || say "WARN evo-load exited $load_exit (see $work/load.log)"
"$py" - "$work/load.log" "$work/server.log" <<'PY'
import re, sys
from datetime import datetime
load, server = sys.argv[1], sys.argv[2]
reports = []
for line in open(server, encoding='utf-8', errors='replace'):
m = re.match(r'(\S+) m10 load-report (.*)', line.strip())
if not m:
continue
stamp = m.group(1)[:19]
t = datetime.strptime(stamp, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=None)
epoch = int((t - datetime(1970, 1, 1)).total_seconds())
fields = dict(kv.split('=') for kv in m.group(2).split())
reports.append((epoch, {k: int(v) for k, v in fields.items()}))
print(f"drill server reports={len(reports)}")
for line in open(load, encoding='utf-8', errors='replace'):
m = re.match(r'load step N=(\d+) .*hold_utc=(\d+)\.\.(\d+)', line)
if not m:
continue
n, t0, t1 = int(m.group(1)), int(m.group(2)), int(m.group(3))
# A report covers the 2 s before it: only reports wholly inside the hold count.
inside = [r for t, r in reports if t0 + 2 <= t <= t1]
if not inside:
print(f"drill step N={n} server_reports=0")
continue
worst = lambda key: max(r[key] for r in inside)
sessions = max(1, worst('sessions'))
print(f"drill step N={n} server_reports={len(inside)} sessions_max={worst('sessions')} "
f"work_p99_max_ms={worst('p99_us') / 1000:.1f} work_max_ms={worst('max_us') / 1000:.1f} "
f"session_p99_max_ms={worst('session_p99_us') / 1000:.1f} "
f"per_session_ms={worst('session_p99_us') / 1000 / sessions:.3f} "
f"refusing_max={worst('refusing')} shed_max={worst('shed')} "
f"tick_debt_max={worst('tick_debt')} "
f"server_within_budget={int(worst('p99_us') <= 25000 and worst('session_p99_us') <= 500 * sessions and worst('tick_debt') == 0)}")
PY
say "drill complete (evo-load exit $load_exit); restoring"