Add M10.9d load drill harness and serialize Mbed TLS use
Foundation / checks (deploy) (push) Successful in 3m24s
Foundation / checks (ecs) (push) Successful in 1m56s
Foundation / checks (client) (push) Successful in 12m18s
Foundation / checks (network) (push) Successful in 13m30s
Foundation / checks (tsan) (push) Successful in 8m0s
Foundation / checks (checks) (push) Successful in 32m39s
Foundation / checks (postgres) (push) Successful in 45m47s
Foundation / postgres-tools (push) Successful in 17s
Foundation / image (push) Successful in 2m30s
Foundation / server-image (push) Successful in 4m8s
Foundation / checks (deploy) (push) Successful in 3m24s
Foundation / checks (ecs) (push) Successful in 1m56s
Foundation / checks (client) (push) Successful in 12m18s
Foundation / checks (network) (push) Successful in 13m30s
Foundation / checks (tsan) (push) Successful in 8m0s
Foundation / checks (checks) (push) Successful in 32m39s
Foundation / checks (postgres) (push) Successful in 45m47s
Foundation / postgres-tools (push) Successful in 17s
Foundation / image (push) Successful in 2m30s
Foundation / server-image (push) Successful in 4m8s
ADR-0097. tools/ops/load-drill.sh (runbook docs/networking/M10_LOAD_DRILL.md) runs the evolution-dev reference qualification: synthetic inputs on for the run (the ADR-0073 drill-time override, restored on exit), bundle-mode evo-load bots through the public route, and each step's hold matched with the server's reports. evo-server logs "m10 load-report" every 10 s, since the agent cannot reach the ops port. evo-load gains --grants FILE --grants-per-bot C, exhausted=, hold_utc= and close_reasons=; evo-session-probe grants gains --accounts N. Finding: Mbed TLS/PSA are built without MBEDTLS_THREADING_C, and bots on different threads verifying their first connections at once were refused as untrusted (two per smoke run). One process-wide recursive crypto lock now guards every network-library entry point that touches them; the smoke went from 2/2 failing to 3/3 clean. quic-transport adds a concurrent-connect test. Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -296,6 +296,7 @@ template <typename Host> class M10Bridge {
|
||||
bool changed = false;
|
||||
for (std::uint64_t i = 0; i < ticks; ++i)
|
||||
changed = governor_.observe(per_tick, session_per_tick, s.tick_debt, now_us) || changed;
|
||||
report(s, now_us);
|
||||
if (!changed || !governed_)
|
||||
return;
|
||||
const auto& load = governor_.state();
|
||||
@@ -303,6 +304,23 @@ template <typename Host> class M10Bridge {
|
||||
std::printf("m10 load refusing=%d shed=%u p95_us=%llu\n", load.refusing ? 1 : 0,
|
||||
static_cast<unsigned>(load.shed), static_cast<unsigned long long>(load.p95_us));
|
||||
}
|
||||
// M10.9d (ADR-0097): every 10 s, the load as /metrics would show it, in the log (drills
|
||||
// read it with `kubectl logs --timestamps` where the ops port is not reachable).
|
||||
void report(const server::HostStatus& s, std::uint64_t now_us) {
|
||||
if (reported_us_ != 0 && now_us - reported_us_ < 10'000'000)
|
||||
return;
|
||||
reported_us_ = now_us;
|
||||
const auto& load = governor_.state();
|
||||
std::printf("m10 load-report sessions=%zu p95_us=%llu p99_us=%llu max_us=%llu "
|
||||
"session_p95_us=%llu session_p99_us=%llu refusing=%d shed=%u tick_debt=%llu\n",
|
||||
sessions_->sessions(), static_cast<unsigned long long>(load.p95_us),
|
||||
static_cast<unsigned long long>(load.p99_us),
|
||||
static_cast<unsigned long long>(load.max_us),
|
||||
static_cast<unsigned long long>(load.session_p95_us),
|
||||
static_cast<unsigned long long>(load.session_p99_us),
|
||||
governed_ && load.refusing ? 1 : 0, governed_ ? load.shed : 0U,
|
||||
static_cast<unsigned long long>(s.tick_debt));
|
||||
}
|
||||
static const processes::EnvironmentalCommunity&
|
||||
community(const processes::EnvironmentalCommunity& c) {
|
||||
return c;
|
||||
@@ -448,6 +466,7 @@ template <typename Host> class M10Bridge {
|
||||
std::unique_ptr<server::PostgresActionLedger> actions_;
|
||||
server::LoadGovernor governor_{*server::LoadGovernor::create({})};
|
||||
bool governed_{true};
|
||||
std::uint64_t reported_us_{};
|
||||
std::chrono::nanoseconds busy_{}, session_busy_{};
|
||||
std::uint64_t measured_tick_{};
|
||||
std::unique_ptr<server::PostgresProbeLedger> probes_; // Handoff mode only.
|
||||
|
||||
@@ -15,6 +15,9 @@
|
||||
// interleaved in serial order, so a client crossing between regions always has a newer
|
||||
// serial for the next one (serials must rise world-wide). A handoff key is a key pair
|
||||
// from `keys` (issuer.key for --m10-handoff-key, issuer.pub for --m10-neighbour).
|
||||
// M10.9d (ADR-0097): --accounts N (1..256) writes the same for N accounts, account by
|
||||
// account: account i is --account with its last two bytes replaced by i + 1 (big
|
||||
// endian). This is the load bundle `evo-load --grants FILE --grants-per-bot COUNT` uses.
|
||||
// evo-session-probe run --connect HOST:PORT --server-name NAME --ca FILE --issuer-key FILE
|
||||
// --world N --frame N --region N --account HEX32 --serial N [--lifetime S]
|
||||
// [--intents N] [--actions N --action-base ID] [--query ID]... [--timeout-ms N]
|
||||
@@ -206,14 +209,14 @@ bool write_new(const std::filesystem::path& path, std::span<const std::uint8_t>
|
||||
return true;
|
||||
}
|
||||
|
||||
int grants(const RunOptions& o, const std::string& out, std::uint64_t count) {
|
||||
int grants(const RunOptions& o, const std::string& out, std::uint64_t count,
|
||||
std::uint64_t accounts) {
|
||||
const auto key = issuer(o.issuer_key);
|
||||
if (!key || !o.account) {
|
||||
std::puts("error issuer-key");
|
||||
return 2;
|
||||
}
|
||||
GrantClaims claims;
|
||||
claims.account = *o.account;
|
||||
claims.world = {o.world, o.frame};
|
||||
claims.issued_at = now_seconds();
|
||||
claims.expires_at = claims.issued_at + o.lifetime;
|
||||
@@ -221,31 +224,39 @@ int grants(const RunOptions& o, const std::string& out, std::uint64_t count) {
|
||||
std::vector<std::uint64_t> regions{o.region};
|
||||
regions.insert(regions.end(), o.more_regions.begin(), o.more_regions.end());
|
||||
if (count * regions.size() > 64) {
|
||||
std::puts("error count (at most 64 grants in all)");
|
||||
std::puts("error count (at most 64 grants per account)");
|
||||
return 2;
|
||||
}
|
||||
std::vector<std::uint8_t> bundle;
|
||||
std::uint64_t serial = first;
|
||||
for (std::uint64_t i = 0; i < count; ++i)
|
||||
for (const auto region : regions) {
|
||||
claims.region = region;
|
||||
claims.serial = serial++;
|
||||
const auto grant = sign(*key, claims);
|
||||
if (!grant) {
|
||||
std::puts("error grant");
|
||||
return 2;
|
||||
}
|
||||
bundle.insert(bundle.end(), grant->begin(), grant->end());
|
||||
for (std::uint64_t a = 0; a < accounts; ++a) {
|
||||
claims.account = *o.account;
|
||||
if (accounts > 1) { // Account a: the last two bytes are a + 1.
|
||||
claims.account.bytes.at(14) = static_cast<std::uint8_t>(((a + 1) >> 8U) & 0xFFU);
|
||||
claims.account.bytes.at(15) = static_cast<std::uint8_t>((a + 1) & 0xFFU);
|
||||
}
|
||||
std::uint64_t serial = first;
|
||||
for (std::uint64_t i = 0; i < count; ++i)
|
||||
for (const auto region : regions) {
|
||||
claims.region = region;
|
||||
claims.serial = serial++;
|
||||
const auto grant = sign(*key, claims);
|
||||
if (!grant) {
|
||||
std::puts("error grant");
|
||||
return 2;
|
||||
}
|
||||
bundle.insert(bundle.end(), grant->begin(), grant->end());
|
||||
}
|
||||
}
|
||||
count *= regions.size();
|
||||
if (!write_new(out, bundle)) {
|
||||
std::puts("error out (exists or unwritable)");
|
||||
return 1;
|
||||
}
|
||||
std::printf("wrote %llu grants serial %llu..%llu valid %llu s\n",
|
||||
static_cast<unsigned long long>(count), static_cast<unsigned long long>(first),
|
||||
static_cast<unsigned long long>(first + count - 1),
|
||||
static_cast<unsigned long long>(o.lifetime));
|
||||
std::printf(
|
||||
"wrote %llu grants serial %llu..%llu valid %llu s accounts %llu\n",
|
||||
static_cast<unsigned long long>(count) * accounts, static_cast<unsigned long long>(first),
|
||||
static_cast<unsigned long long>(first + count - 1),
|
||||
static_cast<unsigned long long>(o.lifetime), static_cast<unsigned long long>(accounts));
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -430,7 +441,7 @@ int main(int argc, char** argv) {
|
||||
RunOptions o;
|
||||
const bool bundle = !args.empty() && args[0] == "grants";
|
||||
std::string out;
|
||||
std::uint64_t count = 1;
|
||||
std::uint64_t count = 1, accounts = 1;
|
||||
if (bundle)
|
||||
o.lifetime = 300; // The region's default maximum.
|
||||
bool ok = !args.empty() && (args[0] == "run" || bundle) && args.size() % 2 == 1;
|
||||
@@ -489,26 +500,29 @@ int main(int argc, char** argv) {
|
||||
out = value;
|
||||
else if (bundle && n && flag == "--count" && *n >= 1 && *n <= 64)
|
||||
count = *n;
|
||||
else if (bundle && n && flag == "--accounts" && *n >= 1 && *n <= 256)
|
||||
accounts = *n;
|
||||
else
|
||||
ok = false;
|
||||
}
|
||||
if (bundle && ok && !o.issuer_key.empty() && o.account && o.world != 0 && o.frame != 0 &&
|
||||
o.region != 0 && !out.empty() && o.lifetime >= 60 && o.host.empty())
|
||||
return grants(o, out, count);
|
||||
return grants(o, out, count, accounts);
|
||||
const bool signs = o.grants_file.empty();
|
||||
if (bundle || !ok || o.host.empty() || o.server_name.empty() || o.ca.empty() ||
|
||||
(signs && (o.issuer_key.empty() || !o.account || o.world == 0 || o.frame == 0 ||
|
||||
o.region == 0 || o.serial == 0)) ||
|
||||
(!signs && !o.issuer_key.empty())) {
|
||||
std::fputs("usage: evo-session-probe keys --out DIR | grants --issuer-key FILE "
|
||||
"--world N --frame N --region N --account HEX32 --out FILE [--serial N] "
|
||||
"[--count 1..64] [--lifetime 60..14400] | run --connect HOST:PORT "
|
||||
"--server-name NAME --ca FILE --issuer-key FILE --world N --frame N "
|
||||
"--region N --account HEX32 --serial N [--lifetime S] [--intents N] "
|
||||
"[--actions N --action-base ID] [--query ID]... [--timeout-ms N] "
|
||||
"[--interest X,Y,Z,R] [--snapshots N] | run --connect HOST:PORT "
|
||||
"--server-name NAME --ca FILE --grants FILE [--grant N] [run options]\n",
|
||||
stderr);
|
||||
std::fputs(
|
||||
"usage: evo-session-probe keys --out DIR | grants --issuer-key FILE "
|
||||
"--world N --frame N --region N --account HEX32 --out FILE [--serial N] "
|
||||
"[--count 1..64] [--accounts 1..256] [--lifetime 60..14400] | run --connect HOST:PORT "
|
||||
"--server-name NAME --ca FILE --issuer-key FILE --world N --frame N "
|
||||
"--region N --account HEX32 --serial N [--lifetime S] [--intents N] "
|
||||
"[--actions N --action-base ID] [--query ID]... [--timeout-ms N] "
|
||||
"[--interest X,Y,Z,R] [--snapshots N] | run --connect HOST:PORT "
|
||||
"--server-name NAME --ca FILE --grants FILE [--grant N] [run options]\n",
|
||||
stderr);
|
||||
return 2;
|
||||
}
|
||||
return run(o);
|
||||
|
||||
@@ -12,7 +12,7 @@ spec:
|
||||
env:
|
||||
- {name: REGION_SESSION, value: "dev"}
|
||||
- {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"}
|
||||
- {name: REGION_SESSION_WINDOW, value: "dev-20261008T112240-4ebe:1791472360"}
|
||||
- {name: REGION_SESSION_WINDOW, value: "dev-20261008T123155-7759:1791476515"}
|
||||
- {name: REGION_M10_MAX_GRANT_LIFETIME, value: "14400"}
|
||||
ports:
|
||||
# QUIC (ADR-0090): UDP only; nothing listens on TCP 7400.
|
||||
|
||||
+2
-1
@@ -549,7 +549,8 @@ smoke done; 10.9c-1 session fuzz target and smoke, in-process Hello and datagram
|
||||
slow peer and connect/disconnect storm with honest-service checks done; 10.9c-2 SIGKILL and
|
||||
drain lease loss with 24 bots (all back within 5 s of the successor) done; no standalone
|
||||
raw-UDP sender (abuse stays in-process/loopback; datagram-cap eviction cost analysed).
|
||||
10.9d dev-cluster drill harness and reference results follow.
|
||||
10.9d drill harness (load-drill.sh, server load-report, bundle-mode evo-load, Mbed TLS
|
||||
crypto lock) done; the evolution-dev reference run follows.
|
||||
|
||||
**Entry:** M9 client/session evidence and M7/M8 dev/stage durability. Read
|
||||
NETWORK_ARCHITECTURE, PERSISTENCE_ARCHITECTURE and ADR-0017. The M9 transport is
|
||||
|
||||
+7
-2
@@ -123,8 +123,13 @@ refusing at 32 (work p95 about 14 ms), with responses about 103 ms and outcomes
|
||||
and its intention admitted (the dev region does not simulate outside drills). M10.9c-2
|
||||
kills and drains a real evo-server under 24 `evo-load` bots. All of them log in to the
|
||||
successor within 5.0 s (SIGKILL, after a 3.5 s lease wait) or 2.4 s (drain), with no failed
|
||||
logins, and responses stay at 106 ms p95. Next: 10.9d, the evolution-dev drill harness and
|
||||
the reference-worker run.
|
||||
logins, and responses stay at 106 ms p95. M10.9d adds the evolution-dev load drill
|
||||
(`tools/ops/load-drill.sh`). It turns synthetic inputs on for the run, drives bundle-mode
|
||||
`evo-load` bots through the public route, and matches each step with the server's 10 s
|
||||
`m10 load-report` log lines. Building it found that Mbed TLS (built without its threading
|
||||
layer) raced when bots verified connections on several threads; one process-wide crypto
|
||||
lock fixed it. Next: the reference run, which needs a promoted image, a load bundle and
|
||||
the maintainer's go-ahead.
|
||||
|
||||
## Previous gate - 2026-10-02
|
||||
|
||||
|
||||
@@ -222,3 +222,36 @@ snapshots.
|
||||
millisecond apart). That cost is bounded per datagram but can add up at very high rates
|
||||
from one peer. Making eviction O(1) without letting tombstones outgrow the queue bound is
|
||||
recorded as a follow-up, not done.
|
||||
|
||||
## 10.9d drill harness (2026-10-08)
|
||||
- **`tools/ops/load-drill.sh`** (runbook: `docs/networking/M10_LOAD_DRILL.md`):
|
||||
1. turns synthetic inputs on in evolution-dev for the run, the drill-time override of
|
||||
ADR-0073 (never in Git);
|
||||
2. runs `evo-load` through the public route;
|
||||
3. matches each step's hold with the server's reports in it;
|
||||
4. restores `REGION_INPUTS` on exit, including on failure.
|
||||
- **`evo-server`** logs `m10 load-report …` every 10 s: sessions, work and session-work
|
||||
percentiles, governor state and tick debt. The agent cannot reach the ops port (no
|
||||
port-forward, and the API-server proxy is refused), but it can read logs.
|
||||
- **Bundle mode:**
|
||||
- `evo-load --grants FILE --grants-per-bot C` logs bots in from a pre-issued bundle, so
|
||||
the drill machine never holds the issuer key;
|
||||
- `evo-session-probe grants --accounts N` writes such a bundle for N accounts;
|
||||
- each step line now carries `exhausted=` and `hold_utc=`.
|
||||
- **Evidence:** the M10 load smoke runs in bundle mode (8 accounts × 2 grants), and the
|
||||
M10 process test checks the `load-report` line. The drill's summariser was tested on
|
||||
synthetic logs; that test found a window-overlap bug, now fixed.
|
||||
- **Finding: Mbed TLS is not thread-safe here.** Bundle mode made logins fast enough
|
||||
that two bots on different threads verified their first connections at once. Both were
|
||||
refused as `untrusted` (QUIC cause 3), which surfaced as `close_reasons=259:2`. Mbed TLS
|
||||
and PSA are built without `MBEDTLS_THREADING_C`. Every network-library entry point that
|
||||
touches them now holds one process-wide recursive lock
|
||||
(`include/evo/network/crypto_lock.hpp`): the QUIC client's pinned check and the
|
||||
session-grant and handoff-ticket keys. The smoke failed 2 of 2 runs before and passed 3
|
||||
of 3 after. `quic-transport` gains a concurrent-connect test (8 threads, 5 rounds); it
|
||||
does not reliably reproduce the race, because PSA is already initialised there.
|
||||
- `evo-load` step lines carry `close_reasons=REASON:COUNT,…`: session close reasons, or
|
||||
256 plus the QUIC cause when there was no session Close.
|
||||
- **Not yet:** the reference run. It needs an image containing this change promoted to
|
||||
dev, a load bundle made where the issuer key lives, and the maintainer's go-ahead,
|
||||
because the drill restarts the dev region twice.
|
||||
|
||||
@@ -5118,3 +5118,43 @@ Prevention:
|
||||
- DEPLOYMENT.md documents the Deployment-only sync symptom and the hook re-run.
|
||||
|
||||
Executed: deploy policy check (0 failures) and selftest (46 cases, 0 failures).
|
||||
|
||||
## 2026-10-08 — M10.9d load drill harness (ADR-0097)
|
||||
|
||||
Implemented:
|
||||
- `tools/ops/load-drill.sh` and its runbook `docs/networking/M10_LOAD_DRILL.md`:
|
||||
synthetic inputs on for the run (restored on exit), `evo-load` through the public route,
|
||||
and server reports matched to each hold.
|
||||
- evo-server `m10 load-report` every 10 s. The agent cannot reach the ops port: no
|
||||
port-forward, and the API-server proxy is refused.
|
||||
- `evo-load --grants FILE --grants-per-bot C` with `exhausted=`, `hold_utc=` and
|
||||
`close_reasons=`; `evo-session-probe grants --accounts N`.
|
||||
- `include/evo/network/crypto_lock.hpp`: one recursive lock at every Mbed TLS/PSA entry
|
||||
point of the network library (`MBEDTLS_THREADING_C` is off).
|
||||
|
||||
Findings:
|
||||
- Concurrent first verifications on different threads were refused as `untrusted`, two
|
||||
per smoke run in bundle mode (2 of 2 runs). With the lock, 3 of 3 runs had no failures.
|
||||
- The drill summariser, tested on synthetic logs, counted reports from after the hold
|
||||
(the next step's ramp). Fixed: only reports wholly inside the hold count.
|
||||
|
||||
Tests:
|
||||
- M10 load smoke in bundle mode (8 accounts × 2 grants) with `exhausted=0`.
|
||||
- The M10 process test checks `m10 load-report`.
|
||||
- `quic-transport` `concurrent_clients`: 8 threads × 5 rounds all connect. It does not
|
||||
reliably reproduce the race; the smoke does.
|
||||
- The lease-loss test's close reasons read 24 × `shutdown` (drain) and 24 × idle
|
||||
(SIGKILL).
|
||||
|
||||
Executed:
|
||||
- **Linux GCC 14:** presets `quic` and `network` 5/5; `postgres-check.sh` with QUIC, all 18
|
||||
suites.
|
||||
- **Linux Clang 19:** clang-tidy 19 on `load.cpp`, the probe, `session_grant.cpp`,
|
||||
`handoff_ticket.cpp`, `quic_endpoint.cpp`, `quic_transport.cpp`, evo-server `main.cpp`
|
||||
with the bridge, and the M10 process tests (3 findings fixed); clang-format 19.
|
||||
- **Windows MSVC 19.44:** presets `quic` 5/5 and `client-network` 15/15.
|
||||
- **Drill script:** `bash -n`, plus the summariser on synthetic logs; shellcheck is not
|
||||
installed here.
|
||||
|
||||
Not done: the reference run on evolution-dev. It needs this image promoted, a load bundle
|
||||
and the maintainer's go-ahead.
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
# M10 load drill (evolution-dev reference worker)
|
||||
|
||||
ADR-0097 qualifies M10 load only on the evolution-dev reference worker.
|
||||
`tools/ops/load-drill.sh` performs that run. It is a drill: it changes the live dev region
|
||||
for the run's duration and restores it on exit, including on failure. It is never part of
|
||||
CI.
|
||||
|
||||
## What it changes
|
||||
|
||||
- `REGION_INPUTS=development-v1` on `Deployment/evo-region` while it runs, the drill-time
|
||||
override of ADR-0073. Synthetic inputs stay off in Git (deploy policy). On exit the
|
||||
value goes back to empty, and the region restarts twice: once to enable, once to
|
||||
restore.
|
||||
- Nothing else: no Secrets, migrations, Jobs, SQL or Argo operations. While it runs, Argo
|
||||
shows `Deployment/evo-region` OutOfSync, and Synced again after the restore.
|
||||
|
||||
## Inputs
|
||||
|
||||
In a tokens directory:
|
||||
- `dev-ca.pem`: the current dev session CA, from the renewal.
|
||||
- `load-grants.bin`: one bundle with grants for every bot, made where the dev issuer key
|
||||
lives (the key never leaves that machine). For the default drill (up to 32 bots, 4
|
||||
logins each):
|
||||
|
||||
```bash
|
||||
evo-session-probe grants --issuer-key issuer.key --world 100 --frame 9 --region 1 \
|
||||
--account <HEX32 base account> --accounts 32 --count 4 --lifetime 14400 \
|
||||
--out load-grants.bin
|
||||
```
|
||||
|
||||
Account i is the base account with its last two bytes set to i + 1. Each bot uses its
|
||||
own 4 grants in order, one per login, and a bot that has used all 4 stays out
|
||||
(`exhausted=` in the report). Grants are valid for at most 4 hours.
|
||||
|
||||
The session window (`REGION_SESSION_WINDOW`) must stay open until the drill ends; the
|
||||
script refuses to start otherwise. The agent's kubectl access is enough: patch
|
||||
`Deployment/evo-region`, read pods and logs.
|
||||
|
||||
## Running
|
||||
|
||||
```bash
|
||||
EVO_LOAD=/path/to/evo-load tools/ops/load-drill.sh <tokens-dir> \
|
||||
[--steps 20,24,28,32] [--hold 60] [--warmup 10] [--grants-per-bot 4]
|
||||
```
|
||||
|
||||
Defaults: steps 20, 24, 28 and 32 bots; 60 s holds; 10 s warmups; 4 grants per bot;
|
||||
`void.aeternalisgames.com:7400`. The drill takes about 6 minutes plus two region
|
||||
restarts.
|
||||
|
||||
## Output
|
||||
|
||||
- **evo-load's per-step lines** (see `tools/network/load.cpp`): client distributions
|
||||
(response, receipt, outcome, login), snapshot cadence, starvation, `close_reasons`
|
||||
(why sessions or logins ended) and a client-side verdict, then
|
||||
`load result sustainable=N saturated_at=M`.
|
||||
- **One `drill step` line per step**, from the server's `m10 load-report` log lines
|
||||
(every 10 s, read with `kubectl logs --timestamps` because the ops port is not
|
||||
reachable for the agent) that fall inside the step's hold:
|
||||
- the worst work p99 and maximum;
|
||||
- the session-layer p99 and its per-session share;
|
||||
- the governor's refusing and shed state;
|
||||
- tick debt;
|
||||
- `server_within_budget`: work p99 ≤ 25 ms, session p99 ≤ 0.5 ms × sessions, no debt.
|
||||
|
||||
The work directory (kept and printed at the end) holds `load.log` and `server.log`.
|
||||
|
||||
## Interpreting
|
||||
|
||||
A step qualifies when both evo-load's verdict and `server_within_budget` are 1. The
|
||||
largest such step is the region's sustainable session count on the reference worker.
|
||||
That number, with the first failing step and its reasons, goes into ADR-0097 and
|
||||
`schemas/network/m10-budgets-v1.md`. If the governor engages (refusing or shed above 0),
|
||||
that step is the overload response in action, and the thresholds are revisited
|
||||
against the measured work.
|
||||
@@ -0,0 +1,18 @@
|
||||
#pragma once
|
||||
#include <mutex>
|
||||
|
||||
// M10.9d (ADR-0097): Mbed TLS and PSA are built without MBEDTLS_THREADING_C, so their
|
||||
// global state (the PSA core, key slots, the random generator) is not safe to use from
|
||||
// several threads at once. Every network-library entry point that touches them takes this
|
||||
// one process-wide lock: the QUIC client's pinned certificate check and the session grant
|
||||
// and handoff ticket keys. Recursive, since entry points may call each other. Calls are rare
|
||||
// (per connection, per grant), so the lock costs nothing measurable. Found by evo-load: two
|
||||
// bots on different threads verifying their first connections at once were refused as
|
||||
// `untrusted`.
|
||||
namespace evo::network {
|
||||
inline std::recursive_mutex& crypto_mutex() noexcept {
|
||||
static std::recursive_mutex mutex;
|
||||
return mutex;
|
||||
}
|
||||
using CryptoLock = std::scoped_lock<std::recursive_mutex>;
|
||||
} // namespace evo::network
|
||||
@@ -1,4 +1,5 @@
|
||||
#include "evo/network/handoff_ticket.hpp"
|
||||
#include "evo/network/crypto_lock.hpp"
|
||||
#include <algorithm>
|
||||
#include <psa/crypto.h>
|
||||
#include <utility>
|
||||
@@ -57,6 +58,7 @@ bool looks_like_handoff_ticket(std::span<const std::uint8_t> bytes) noexcept {
|
||||
|
||||
std::expected<HandoffSigningKey, HandoffTicketError>
|
||||
HandoffSigningKey::generate(DeploymentEnvironment environment, std::uint32_t key_id) {
|
||||
const CryptoLock crypto{crypto_mutex()};
|
||||
if (key_id == 0)
|
||||
return std::unexpected(HandoffTicketError::invalid_key);
|
||||
if (psa_crypto_init() != PSA_SUCCESS)
|
||||
@@ -73,6 +75,7 @@ HandoffSigningKey::generate(DeploymentEnvironment environment, std::uint32_t key
|
||||
std::expected<HandoffSigningKey, HandoffTicketError>
|
||||
HandoffSigningKey::import(DeploymentEnvironment environment, std::uint32_t key_id,
|
||||
std::span<const std::uint8_t> private_key) {
|
||||
const CryptoLock crypto{crypto_mutex()};
|
||||
if (key_id == 0 || private_key.size() != grant_private_key_bytes)
|
||||
return std::unexpected(HandoffTicketError::invalid_key);
|
||||
if (psa_crypto_init() != PSA_SUCCESS)
|
||||
@@ -90,6 +93,7 @@ HandoffSigningKey::HandoffSigningKey(HandoffSigningKey&& other) noexcept
|
||||
key_{std::exchange(other.key_, PSA_KEY_ID_NULL)} {}
|
||||
|
||||
HandoffSigningKey& HandoffSigningKey::operator=(HandoffSigningKey&& other) noexcept {
|
||||
const CryptoLock crypto{crypto_mutex()};
|
||||
if (this != &other) {
|
||||
if (key_ != PSA_KEY_ID_NULL)
|
||||
(void)psa_destroy_key(key_);
|
||||
@@ -101,12 +105,14 @@ HandoffSigningKey& HandoffSigningKey::operator=(HandoffSigningKey&& other) noexc
|
||||
}
|
||||
|
||||
HandoffSigningKey::~HandoffSigningKey() {
|
||||
const CryptoLock crypto{crypto_mutex()};
|
||||
if (key_ != PSA_KEY_ID_NULL)
|
||||
(void)psa_destroy_key(key_);
|
||||
}
|
||||
|
||||
std::expected<HandoffTicket, HandoffTicketError>
|
||||
HandoffSigningKey::sign(const TicketClaims& claims) const {
|
||||
const CryptoLock crypto{crypto_mutex()};
|
||||
if (key_ == PSA_KEY_ID_NULL)
|
||||
return std::unexpected(HandoffTicketError::invalid_key);
|
||||
if (!structurally_valid(claims))
|
||||
@@ -134,6 +140,7 @@ HandoffSigningKey::sign(const TicketClaims& claims) const {
|
||||
}
|
||||
|
||||
std::expected<GrantPublicKey, HandoffTicketError> HandoffSigningKey::public_key() const {
|
||||
const CryptoLock crypto{crypto_mutex()};
|
||||
if (key_ == PSA_KEY_ID_NULL)
|
||||
return std::unexpected(HandoffTicketError::invalid_key);
|
||||
GrantPublicKey out{};
|
||||
@@ -147,6 +154,7 @@ std::expected<GrantPublicKey, HandoffTicketError> HandoffSigningKey::public_key(
|
||||
std::expected<HandoffVerifier, HandoffTicketError>
|
||||
HandoffVerifier::create(DeploymentEnvironment environment, world::WorldIdentity world,
|
||||
std::uint64_t region) {
|
||||
const CryptoLock crypto{crypto_mutex()};
|
||||
if (world.world == 0 || world.frame == 0 || region == 0)
|
||||
return std::unexpected(HandoffTicketError::invalid_claims);
|
||||
if (psa_crypto_init() != PSA_SUCCESS)
|
||||
@@ -174,6 +182,7 @@ HandoffVerifier::~HandoffVerifier() {
|
||||
}
|
||||
|
||||
void HandoffVerifier::destroy() noexcept {
|
||||
const CryptoLock crypto{crypto_mutex()};
|
||||
for (const auto& k : keys_)
|
||||
(void)psa_destroy_key(k.key);
|
||||
keys_.clear();
|
||||
@@ -182,6 +191,7 @@ void HandoffVerifier::destroy() noexcept {
|
||||
std::expected<void, HandoffTicketError>
|
||||
HandoffVerifier::add_key(std::uint64_t source_region, std::uint32_t key_id,
|
||||
std::span<const std::uint8_t> public_key) {
|
||||
const CryptoLock crypto{crypto_mutex()};
|
||||
if (source_region == 0 || source_region == region_ || key_id == 0 ||
|
||||
public_key.size() != grant_public_key_bytes)
|
||||
return std::unexpected(HandoffTicketError::invalid_key);
|
||||
@@ -204,6 +214,7 @@ HandoffVerifier::add_key(std::uint64_t source_region, std::uint32_t key_id,
|
||||
|
||||
std::expected<VerifiedHandoffTicket, HandoffTicketError>
|
||||
HandoffVerifier::verify(std::span<const std::uint8_t> ticket, std::uint64_t now_seconds) const {
|
||||
const CryptoLock crypto{crypto_mutex()};
|
||||
if (ticket.size() != handoff_ticket_bytes)
|
||||
return std::unexpected(HandoffTicketError::malformed);
|
||||
// The key id only selects which key must verify; it is itself signed.
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#include "evo/network/session_grant.hpp"
|
||||
#include "evo/network/crypto_lock.hpp"
|
||||
#include <algorithm>
|
||||
#include <psa/crypto.h>
|
||||
#include <type_traits>
|
||||
@@ -51,6 +52,7 @@ psa_key_attributes_t pair_attributes() {
|
||||
} // namespace
|
||||
|
||||
std::expected<void, SessionGrantError> random_bytes(std::span<std::uint8_t> out) {
|
||||
const CryptoLock crypto{crypto_mutex()};
|
||||
if (psa_crypto_init() != PSA_SUCCESS ||
|
||||
psa_generate_random(out.data(), out.size()) != PSA_SUCCESS)
|
||||
return std::unexpected(SessionGrantError::crypto);
|
||||
@@ -68,6 +70,7 @@ std::expected<AccountId, SessionGrantError> generate_account_id() {
|
||||
|
||||
std::expected<GrantSigningKey, SessionGrantError>
|
||||
GrantSigningKey::generate(DeploymentEnvironment environment, std::uint32_t key_id) {
|
||||
const CryptoLock crypto{crypto_mutex()};
|
||||
if (key_id == 0)
|
||||
return std::unexpected(SessionGrantError::invalid_key);
|
||||
if (psa_crypto_init() != PSA_SUCCESS)
|
||||
@@ -84,6 +87,7 @@ GrantSigningKey::generate(DeploymentEnvironment environment, std::uint32_t key_i
|
||||
std::expected<GrantSigningKey, SessionGrantError>
|
||||
GrantSigningKey::import(DeploymentEnvironment environment, std::uint32_t key_id,
|
||||
std::span<const std::uint8_t> private_key) {
|
||||
const CryptoLock crypto{crypto_mutex()};
|
||||
if (key_id == 0 || private_key.size() != grant_private_key_bytes)
|
||||
return std::unexpected(SessionGrantError::invalid_key);
|
||||
if (psa_crypto_init() != PSA_SUCCESS)
|
||||
@@ -101,6 +105,7 @@ GrantSigningKey::GrantSigningKey(GrantSigningKey&& other) noexcept
|
||||
key_{std::exchange(other.key_, PSA_KEY_ID_NULL)} {}
|
||||
|
||||
GrantSigningKey& GrantSigningKey::operator=(GrantSigningKey&& other) noexcept {
|
||||
const CryptoLock crypto{crypto_mutex()};
|
||||
if (this != &other) {
|
||||
if (key_ != PSA_KEY_ID_NULL)
|
||||
(void)psa_destroy_key(key_);
|
||||
@@ -112,12 +117,14 @@ GrantSigningKey& GrantSigningKey::operator=(GrantSigningKey&& other) noexcept {
|
||||
}
|
||||
|
||||
GrantSigningKey::~GrantSigningKey() {
|
||||
const CryptoLock crypto{crypto_mutex()};
|
||||
if (key_ != PSA_KEY_ID_NULL)
|
||||
(void)psa_destroy_key(key_);
|
||||
}
|
||||
|
||||
std::expected<SessionGrant, SessionGrantError>
|
||||
GrantSigningKey::sign(const GrantClaims& claims) const {
|
||||
const CryptoLock crypto{crypto_mutex()};
|
||||
if (key_ == PSA_KEY_ID_NULL)
|
||||
return std::unexpected(SessionGrantError::invalid_key);
|
||||
if (!structurally_valid(claims))
|
||||
@@ -145,6 +152,7 @@ GrantSigningKey::sign(const GrantClaims& claims) const {
|
||||
}
|
||||
|
||||
std::expected<GrantPublicKey, SessionGrantError> GrantSigningKey::public_key() const {
|
||||
const CryptoLock crypto{crypto_mutex()};
|
||||
if (key_ == PSA_KEY_ID_NULL)
|
||||
return std::unexpected(SessionGrantError::invalid_key);
|
||||
GrantPublicKey out{};
|
||||
@@ -157,6 +165,7 @@ std::expected<GrantPublicKey, SessionGrantError> GrantSigningKey::public_key() c
|
||||
|
||||
std::expected<GrantVerifier, SessionGrantError>
|
||||
GrantVerifier::create(DeploymentEnvironment environment) {
|
||||
const CryptoLock crypto{crypto_mutex()};
|
||||
if (psa_crypto_init() != PSA_SUCCESS)
|
||||
return std::unexpected(SessionGrantError::crypto);
|
||||
return GrantVerifier{environment};
|
||||
@@ -179,6 +188,7 @@ GrantVerifier::~GrantVerifier() {
|
||||
}
|
||||
|
||||
void GrantVerifier::destroy() noexcept {
|
||||
const CryptoLock crypto{crypto_mutex()};
|
||||
for (const auto& k : keys_)
|
||||
(void)psa_destroy_key(k.key);
|
||||
keys_.clear();
|
||||
@@ -186,6 +196,7 @@ void GrantVerifier::destroy() noexcept {
|
||||
|
||||
std::expected<void, SessionGrantError>
|
||||
GrantVerifier::add_key(std::uint32_t key_id, std::span<const std::uint8_t> public_key) {
|
||||
const CryptoLock crypto{crypto_mutex()};
|
||||
if (key_id == 0 || public_key.size() != grant_public_key_bytes)
|
||||
return std::unexpected(SessionGrantError::invalid_key);
|
||||
if (std::ranges::any_of(keys_, [&](const Key& k) { return k.key_id == key_id; }))
|
||||
@@ -207,6 +218,7 @@ GrantVerifier::add_key(std::uint32_t key_id, std::span<const std::uint8_t> publi
|
||||
|
||||
std::expected<VerifiedSessionGrant, SessionGrantError>
|
||||
GrantVerifier::verify(std::span<const std::uint8_t> grant) const {
|
||||
const CryptoLock crypto{crypto_mutex()};
|
||||
if (grant.size() != session_grant_bytes)
|
||||
return std::unexpected(SessionGrantError::malformed);
|
||||
// The key id only selects which key must verify; it is itself signed.
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#include "evo/network/crypto_lock.hpp"
|
||||
#include "evo/network/quic_transport.hpp"
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
@@ -43,6 +44,7 @@ bool tls_or_certificate_failure(QUIC_STATUS status) {
|
||||
// Leaf (DER) verified against exactly one CA (PEM) for one name; nothing else trusted.
|
||||
bool pinned(std::span<const std::uint8_t> leaf, const std::string& authority_pem,
|
||||
const std::string& name) {
|
||||
const evo::network::CryptoLock crypto{evo::network::crypto_mutex()};
|
||||
if (psa_crypto_init() != PSA_SUCCESS || leaf.empty())
|
||||
return false;
|
||||
mbedtls_x509_crt ca;
|
||||
|
||||
@@ -5,12 +5,15 @@
|
||||
#include "evo/network/dev_credentials.hpp"
|
||||
#include "udp_relay.hpp"
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <functional>
|
||||
#include <latch>
|
||||
#include <set>
|
||||
#include <source_location>
|
||||
#include <stdexcept>
|
||||
@@ -558,8 +561,67 @@ void garbage_and_close() {
|
||||
}
|
||||
} // namespace
|
||||
|
||||
// M10.9d (ADR-0097): client endpoints on several threads connecting at the same moment all
|
||||
// connect and pass the owned pin check. Mbed TLS runs without its threading layer; evo-load
|
||||
// saw two concurrent first verifications refused as `untrusted` before the shared crypto
|
||||
// lock. This test covers concurrent connects; it does not reliably reproduce that race (PSA
|
||||
// is already initialised here by the credential generation), which evo-load's smoke does.
|
||||
void concurrent_clients() {
|
||||
const auto f = credentials("concurrent");
|
||||
Limits limits;
|
||||
limits.max_connections = 64; // 5 rounds x 8, whatever the server has not yet reaped.
|
||||
auto server = serve(f, limits);
|
||||
constexpr int threads = 8;
|
||||
for (int round = 0; round < 5; ++round) {
|
||||
std::atomic<int> connected{0}, finished{0};
|
||||
std::array<std::atomic<int>, 8> causes{};
|
||||
std::latch start{threads};
|
||||
std::vector<std::thread> clients;
|
||||
clients.reserve(threads);
|
||||
for (int t = 0; t < threads; ++t)
|
||||
clients.emplace_back([&] {
|
||||
start.arrive_and_wait();
|
||||
auto client = dial(f, server->port());
|
||||
const auto end = std::chrono::steady_clock::now() + std::chrono::seconds{10};
|
||||
bool done = false;
|
||||
while (!done && std::chrono::steady_clock::now() < end) {
|
||||
for (const auto& e : client->poll()) {
|
||||
if (e.kind == EventKind::connected) {
|
||||
++connected;
|
||||
done = true;
|
||||
} else if (e.kind == EventKind::closed) {
|
||||
causes[static_cast<std::size_t>(e.cause)] += 1;
|
||||
done = true;
|
||||
}
|
||||
}
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds{1});
|
||||
}
|
||||
++finished;
|
||||
});
|
||||
while (finished < threads) {
|
||||
(void)server->poll();
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds{1});
|
||||
}
|
||||
for (auto& c : clients)
|
||||
c.join();
|
||||
if (connected != threads) {
|
||||
std::printf("concurrent round %d connected=%d", round, connected.load());
|
||||
for (std::size_t i = 0; i < causes.size(); ++i)
|
||||
std::printf(" cause%zu=%d", i, causes.at(i).load());
|
||||
std::printf("\n");
|
||||
}
|
||||
CHECK(connected == threads);
|
||||
for (int i = 0; i < 200; ++i) { // Let the server see the closes.
|
||||
(void)server->poll();
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds{1});
|
||||
}
|
||||
}
|
||||
std::filesystem::remove_all(f.dir);
|
||||
}
|
||||
|
||||
int main() try {
|
||||
framing();
|
||||
concurrent_clients();
|
||||
channels();
|
||||
pinning();
|
||||
impaired();
|
||||
|
||||
@@ -462,7 +462,8 @@ void end_to_end() {
|
||||
server->child->signal(SIGTERM);
|
||||
CHECK(server->child->wait_exit(30s) == 0);
|
||||
CHECK(server->child->printed("m10 opened ") && server->child->printed("m10 committed ") &&
|
||||
server->child->printed("m10 motion area "));
|
||||
server->child->printed("m10 motion area ") &&
|
||||
server->child->printed("m10 load-report sessions="));
|
||||
}
|
||||
// M10.9b (ADR-0097): evo-load against a real evo-server, two steps with the region's
|
||||
// metrics. A smoke test of the tool and the server under concurrent sessions, not
|
||||
@@ -473,6 +474,15 @@ void load_smoke() {
|
||||
Setup s{152, 0xC6};
|
||||
Server server{s};
|
||||
const auto tool = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-load";
|
||||
// M10.9d: the bots log in from a bundle (as the dev drill does): 8 accounts x 2 grants.
|
||||
const auto probe = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-session-probe";
|
||||
const auto bundle = s.region.root / "load-grants.bin";
|
||||
Child g{{probe.string(), "grants", "--issuer-key", (s.region.root / "issuer.key").string(),
|
||||
"--world", "7", "--frame", std::to_string(region_identity.frame),
|
||||
"--region", "152", "--account", "0000000000000000000000000000b000",
|
||||
"--accounts", "8", "--count", "2",
|
||||
"--lifetime", "900", "--out", bundle.string()}};
|
||||
CHECK(g.wait_exit(30s) == 0 && g.printed("wrote 16 grants "));
|
||||
Child load{{tool.string(),
|
||||
"--connect",
|
||||
"127.0.0.1:" + std::to_string(server.port),
|
||||
@@ -480,8 +490,10 @@ void load_smoke() {
|
||||
name,
|
||||
"--ca",
|
||||
(s.region.root / "ca.pem").string(),
|
||||
"--issuer-key",
|
||||
(s.region.root / "issuer.key").string(),
|
||||
"--grants",
|
||||
bundle.string(),
|
||||
"--grants-per-bot",
|
||||
"2",
|
||||
"--world",
|
||||
"7",
|
||||
"--frame",
|
||||
@@ -522,7 +534,8 @@ void load_smoke() {
|
||||
// Every bot logged in once: 4 in the first step, 4 more in the second.
|
||||
CHECK((starts(line, "load step N=4 established=4 logins=4 ") ||
|
||||
starts(line, "load step N=8 established=8 logins=4 ")) &&
|
||||
line.find(" drops=0 login_failures=0 refused=0 ") != std::string::npos);
|
||||
line.find(" drops=0 login_failures=0 refused=0 ") != std::string::npos &&
|
||||
line.find(" exhausted=0 hold_utc=") != std::string::npos);
|
||||
}
|
||||
const auto has = [&](std::string_view prefix) {
|
||||
return std::ranges::count_if(load.lines(), [&](const std::string& l) {
|
||||
|
||||
+107
-33
@@ -2,7 +2,8 @@
|
||||
// runs against the evolution-dev reference worker count as evidence (10.9d); local runs are
|
||||
// smoke tests and development references.
|
||||
//
|
||||
// evo-load --connect HOST:PORT --server-name NAME --ca FILE --issuer-key FILE
|
||||
// evo-load --connect HOST:PORT --server-name NAME --ca FILE
|
||||
// (--issuer-key FILE | --grants FILE --grants-per-bot C)
|
||||
// --world N --frame N --region N [--steps N[,N]...] [--hold S] [--warmup S]
|
||||
// [--ramp-ms MS] [--threads T] [--seed X] [--lifetime S] [--metrics HOST:PORT]
|
||||
// [--response-budget-ms MS] [--per-client] [--expect-established]
|
||||
@@ -14,11 +15,17 @@
|
||||
// probe marker; every 2 s it sends a probe action; every 10 s it moves its interest by a
|
||||
// cell (a fresh baseline). A closed or refused bot logs in again after 2 s with the next
|
||||
// serial.
|
||||
// M10.9d: with --grants (a bundle from `evo-session-probe grants --accounts N --count C`,
|
||||
// e.g. made where the dev issuer key lives), bot i logs in with grants i*C .. i*C+C-1 in
|
||||
// order instead of signing; a bot that used all C stays out (counted as exhausted).
|
||||
//
|
||||
// Steps: the bot count rises to each step's N (one login every --ramp-ms), waits --warmup
|
||||
// seconds, then measures for --hold seconds. Bots stay connected into the next step. Per
|
||||
// step it prints:
|
||||
// load step N=.. established=.. logins=.. drops=.. login_failures=.. refused=.. seconds=..
|
||||
// exhausted=.. hold_utc=START..END (unix seconds of the hold, to match server samples)
|
||||
// close_reasons=REASON:COUNT,... (session-v1 close reasons; without a session Close,
|
||||
// 256 + the QUIC transport cause, or 255 when unknown)
|
||||
// (drops: established sessions closed; login failures: closed before Welcome, including
|
||||
// transport refusals at the connection limit; refused: closed with capacity or
|
||||
// rate_limited, either way. Logins, drops and failures include the step's ramp and warmup;
|
||||
@@ -111,7 +118,8 @@ double ms(std::uint64_t us) {
|
||||
}
|
||||
|
||||
struct Options {
|
||||
std::string host, server_name, ca, issuer_key, metrics_host;
|
||||
std::string host, server_name, ca, issuer_key, grants_file, metrics_host;
|
||||
std::uint64_t grants_per_bot{};
|
||||
std::uint16_t port{}, metrics_port{};
|
||||
std::uint64_t world{}, frame{}, region{}, seed{1}, lifetime{900};
|
||||
std::vector<std::size_t> steps{20};
|
||||
@@ -125,6 +133,7 @@ struct Samples {
|
||||
gap_ticks;
|
||||
std::uint64_t snapshots{}, bytes{}, intents{}, refused_intents{}, logins{}, drops{},
|
||||
login_failures{}, refused{};
|
||||
std::map<unsigned, std::uint64_t> closes; // By session close reason (255: no Close).
|
||||
std::uint64_t connected_polls{}; // Polls while established (starvation is judged on these).
|
||||
};
|
||||
void append(std::vector<std::uint64_t>& to, const std::vector<std::uint64_t>& from) {
|
||||
@@ -142,6 +151,8 @@ struct Bot {
|
||||
client::InputLead lead;
|
||||
std::uint64_t last_tick{}, last_view{}, last_bytes{}, next_action_id{1};
|
||||
std::uint64_t applied{}; // Newest applied intention sequence seen in an own marker.
|
||||
std::uint64_t used{}; // Bundle grants used (bundle mode).
|
||||
std::atomic<bool> exhausted{false};
|
||||
std::map<std::uint64_t, Clock::time_point> receipt_pending, response_pending, outcome_pending;
|
||||
std::map<std::uint64_t, std::uint64_t> intent_tick;
|
||||
std::int16_t vx{}, vy{};
|
||||
@@ -153,9 +164,11 @@ struct Bot {
|
||||
|
||||
class Load {
|
||||
public:
|
||||
Load(Options o, GrantSigningKey key, std::string address)
|
||||
: o_{std::move(o)}, key_{std::move(key)}, address_{std::move(address)},
|
||||
authority_{read(o_.ca)} {
|
||||
// Signs with `key`, or logs in from `bundle` (concatenated grants, grants_per_bot each).
|
||||
Load(Options o, std::optional<GrantSigningKey> key, std::vector<SessionGrant> bundle,
|
||||
std::string address)
|
||||
: o_{std::move(o)}, key_{std::move(key)}, bundle_{std::move(bundle)},
|
||||
address_{std::move(address)}, authority_{read(o_.ca)} {
|
||||
const auto most = *std::ranges::max_element(o_.steps);
|
||||
bots_.reserve(most);
|
||||
for (std::size_t i = 0; i < most; ++i) {
|
||||
@@ -199,6 +212,11 @@ class Load {
|
||||
}
|
||||
return out;
|
||||
}
|
||||
[[nodiscard]] std::size_t exhausted(std::size_t n) const {
|
||||
return static_cast<std::size_t>(
|
||||
std::count_if(bots_.begin(), bots_.begin() + static_cast<std::ptrdiff_t>(n),
|
||||
[](const auto& b) { return b->exhausted.load(); }));
|
||||
}
|
||||
[[nodiscard]] std::size_t established(std::size_t n) const {
|
||||
return static_cast<std::size_t>(
|
||||
std::count_if(bots_.begin(), bots_.begin() + static_cast<std::ptrdiff_t>(n),
|
||||
@@ -229,15 +247,26 @@ class Load {
|
||||
}
|
||||
}
|
||||
void login(Bot& b, Clock::time_point now) {
|
||||
GrantClaims claims;
|
||||
claims.account = b.account;
|
||||
claims.serial = ++b.serial;
|
||||
claims.world = {o_.world, o_.frame};
|
||||
claims.region = o_.region;
|
||||
claims.issued_at = now_seconds();
|
||||
claims.expires_at = claims.issued_at + o_.lifetime;
|
||||
claims.grant_id = (b.serial << 16U) ^ (b.index + 1);
|
||||
const auto grant = key_.sign(claims);
|
||||
std::optional<SessionGrant> grant;
|
||||
if (key_) {
|
||||
GrantClaims claims;
|
||||
claims.account = b.account;
|
||||
claims.serial = ++b.serial;
|
||||
claims.world = {o_.world, o_.frame};
|
||||
claims.region = o_.region;
|
||||
claims.issued_at = now_seconds();
|
||||
claims.expires_at = claims.issued_at + o_.lifetime;
|
||||
claims.grant_id = (b.serial << 16U) ^ (b.index + 1);
|
||||
if (auto signed_grant = key_->sign(claims))
|
||||
grant = *signed_grant;
|
||||
} else {
|
||||
if (b.used >= o_.grants_per_bot) {
|
||||
b.exhausted = true; // Out of grants: stays out.
|
||||
b.retry_at = Clock::time_point::max();
|
||||
return;
|
||||
}
|
||||
grant = bundle_.at((b.index * o_.grants_per_bot) + b.used++);
|
||||
}
|
||||
client::RegionSessionClientConfig config;
|
||||
config.transport.server_address = address_;
|
||||
config.transport.port = o_.port;
|
||||
@@ -278,6 +307,13 @@ class Load {
|
||||
++b.s.drops;
|
||||
else
|
||||
++b.s.login_failures;
|
||||
if (const auto reason = c.close_reason();
|
||||
reason && *reason != SessionCloseReason::connection_lost)
|
||||
++b.s.closes[static_cast<unsigned>(*reason)];
|
||||
else if (const auto cause = c.transport_cause()) // No session Close: 256 + cause.
|
||||
++b.s.closes[256U + static_cast<unsigned>(*cause)];
|
||||
else
|
||||
++b.s.closes[255U];
|
||||
if (c.close_reason() == SessionCloseReason::capacity ||
|
||||
c.close_reason() == SessionCloseReason::rate_limited)
|
||||
++b.s.refused;
|
||||
@@ -406,7 +442,8 @@ class Load {
|
||||
}
|
||||
|
||||
Options o_;
|
||||
GrantSigningKey key_;
|
||||
std::optional<GrantSigningKey> key_;
|
||||
std::vector<SessionGrant> bundle_;
|
||||
std::string address_, authority_;
|
||||
std::vector<std::unique_ptr<Bot>> bots_;
|
||||
std::vector<std::mutex> locks_;
|
||||
@@ -481,9 +518,9 @@ void line(const char* name, const std::vector<std::uint64_t>& pooled,
|
||||
}
|
||||
|
||||
// One step's report; true when within budget.
|
||||
bool report(const Options& o, std::size_t n, std::size_t established,
|
||||
const std::vector<Samples>& all, double seconds, double cpu,
|
||||
const std::map<std::string, std::uint64_t>& server) {
|
||||
bool report(const Options& o, std::size_t n, std::size_t established, std::size_t exhausted,
|
||||
std::pair<std::uint64_t, std::uint64_t> hold_utc, const std::vector<Samples>& all,
|
||||
double seconds, double cpu, const std::map<std::string, std::uint64_t>& server) {
|
||||
Samples pooled;
|
||||
std::vector<std::uint64_t> response95, receipt95, outcome95, login95, correction95, rates, kib;
|
||||
for (const auto& s : all) {
|
||||
@@ -499,6 +536,8 @@ bool report(const Options& o, std::size_t n, std::size_t established,
|
||||
pooled.logins += s.logins;
|
||||
pooled.drops += s.drops;
|
||||
pooled.login_failures += s.login_failures;
|
||||
for (const auto& [why, count] : s.closes)
|
||||
pooled.closes[why] += count;
|
||||
pooled.refused += s.refused;
|
||||
response95.push_back(percentile(s.response_us, 0.95));
|
||||
receipt95.push_back(percentile(s.receipt_us, 0.95));
|
||||
@@ -511,12 +550,18 @@ bool report(const Options& o, std::size_t n, std::size_t established,
|
||||
kib.push_back(static_cast<std::uint64_t>(
|
||||
std::lround(static_cast<double>(s.bytes) / 1024.0 / std::max(seconds, 0.001))));
|
||||
}
|
||||
std::string reasons;
|
||||
for (const auto& [why, count] : pooled.closes)
|
||||
reasons += (reasons.empty() ? "" : ",") + std::to_string(why) + ":" + std::to_string(count);
|
||||
std::printf("load step N=%zu established=%zu logins=%llu drops=%llu login_failures=%llu "
|
||||
"refused=%llu seconds=%.1f\n",
|
||||
"refused=%llu seconds=%.1f exhausted=%zu hold_utc=%llu..%llu close_reasons=%s\n",
|
||||
n, established, static_cast<unsigned long long>(pooled.logins),
|
||||
static_cast<unsigned long long>(pooled.drops),
|
||||
static_cast<unsigned long long>(pooled.login_failures),
|
||||
static_cast<unsigned long long>(pooled.refused), seconds);
|
||||
static_cast<unsigned long long>(pooled.refused), seconds, exhausted,
|
||||
static_cast<unsigned long long>(hold_utc.first),
|
||||
static_cast<unsigned long long>(hold_utc.second),
|
||||
reasons.empty() ? "-" : reasons.c_str());
|
||||
line("response", pooled.response_us, response95, 1000.0, "ms");
|
||||
line("receipt", pooled.receipt_us, receipt95, 1000.0, "ms");
|
||||
line("outcome", pooled.outcome_us, outcome95, 1000.0, "ms");
|
||||
@@ -663,6 +708,10 @@ int main(int argc, char** argv) {
|
||||
o.ca = value;
|
||||
else if (flag == "--issuer-key")
|
||||
o.issuer_key = value;
|
||||
else if (flag == "--grants")
|
||||
o.grants_file = value;
|
||||
else if (n && flag == "--grants-per-bot" && *n >= 1 && *n <= 64)
|
||||
o.grants_per_bot = *n;
|
||||
else if (flag == "--steps") {
|
||||
const auto s = steps_of(value);
|
||||
ok = s.has_value();
|
||||
@@ -676,7 +725,7 @@ int main(int argc, char** argv) {
|
||||
o.region = *n;
|
||||
else if (n && flag == "--seed" && *n != 0)
|
||||
o.seed = *n;
|
||||
else if (n && flag == "--lifetime" && *n >= 60 && *n <= 3600)
|
||||
else if (n && flag == "--lifetime" && *n >= 60 && *n <= 14400)
|
||||
o.lifetime = *n;
|
||||
else if (n && flag == "--hold" && *n >= 1 && *n <= 3600)
|
||||
o.hold_s = *n;
|
||||
@@ -691,23 +740,44 @@ int main(int argc, char** argv) {
|
||||
else
|
||||
ok = false;
|
||||
}
|
||||
if (!ok || o.host.empty() || o.server_name.empty() || o.ca.empty() || o.issuer_key.empty() ||
|
||||
o.world == 0 || o.frame == 0 || o.region == 0) {
|
||||
const bool signs = !o.issuer_key.empty();
|
||||
if (!ok || o.host.empty() || o.server_name.empty() || o.ca.empty() ||
|
||||
signs == !o.grants_file.empty() || (!signs && o.grants_per_bot == 0) || o.world == 0 ||
|
||||
o.frame == 0 || o.region == 0) {
|
||||
std::fputs("usage: evo-load --connect HOST:PORT --server-name NAME --ca FILE "
|
||||
"--issuer-key FILE --world N --frame N --region N [--steps N[,N]...] "
|
||||
"(--issuer-key FILE | --grants FILE --grants-per-bot C) --world N "
|
||||
"--frame N --region N [--steps N[,N]...] "
|
||||
"[--hold S] [--warmup S] [--ramp-ms MS] [--threads T] [--seed X] "
|
||||
"[--lifetime S] [--metrics HOST:PORT] [--response-budget-ms MS] "
|
||||
"[--per-client] [--expect-established]\n",
|
||||
stderr);
|
||||
return 2;
|
||||
}
|
||||
const auto scalar = read(o.issuer_key);
|
||||
auto key = GrantSigningKey::import(
|
||||
DeploymentEnvironment::dev, 1,
|
||||
std::span{reinterpret_cast<const std::uint8_t*>(scalar.data()), scalar.size()});
|
||||
if (!key) {
|
||||
std::puts("error issuer-key");
|
||||
return 2;
|
||||
std::optional<GrantSigningKey> key;
|
||||
std::vector<SessionGrant> bundle;
|
||||
if (signs) {
|
||||
const auto scalar = read(o.issuer_key);
|
||||
auto imported = GrantSigningKey::import(
|
||||
DeploymentEnvironment::dev, 1,
|
||||
std::span{reinterpret_cast<const std::uint8_t*>(scalar.data()), scalar.size()});
|
||||
if (!imported) {
|
||||
std::puts("error issuer-key");
|
||||
return 2;
|
||||
}
|
||||
key.emplace(std::move(*imported));
|
||||
} else {
|
||||
const auto bytes = read(o.grants_file);
|
||||
const auto needed = *std::ranges::max_element(o.steps) * o.grants_per_bot;
|
||||
if (bytes.size() % session_grant_bytes != 0 ||
|
||||
bytes.size() / session_grant_bytes < needed) {
|
||||
std::printf("error grants (need %llu grants: bots x grants-per-bot)\n",
|
||||
static_cast<unsigned long long>(needed));
|
||||
return 2;
|
||||
}
|
||||
bundle.resize(bytes.size() / session_grant_bytes);
|
||||
for (std::size_t i = 0; i < bundle.size(); ++i)
|
||||
std::copy_n(bytes.begin() + static_cast<std::ptrdiff_t>(i * session_grant_bytes),
|
||||
session_grant_bytes, reinterpret_cast<char*>(bundle[i].data()));
|
||||
}
|
||||
const auto address = quic::resolve_ipv4(o.host);
|
||||
if (!address) {
|
||||
@@ -723,7 +793,7 @@ int main(int argc, char** argv) {
|
||||
bool smoke_ok = true;
|
||||
std::optional<std::size_t> sustainable, saturated;
|
||||
{
|
||||
Load load{o, std::move(*key), *address};
|
||||
Load load{o, std::move(key), std::move(bundle), *address};
|
||||
for (const auto n : o.steps) {
|
||||
load.activate(n);
|
||||
std::this_thread::sleep_for(std::chrono::seconds{o.warmup_s});
|
||||
@@ -731,6 +801,7 @@ int main(int argc, char** argv) {
|
||||
// the warmup is discarded.
|
||||
const auto warm = load.take(n);
|
||||
const auto started = Clock::now();
|
||||
const auto hold_start = now_seconds();
|
||||
const auto cpu0 = cpu_seconds();
|
||||
std::this_thread::sleep_for(std::chrono::seconds{o.hold_s});
|
||||
auto samples = load.take(n);
|
||||
@@ -739,12 +810,15 @@ int main(int argc, char** argv) {
|
||||
samples[i].logins += warm[i].logins;
|
||||
samples[i].drops += warm[i].drops;
|
||||
samples[i].login_failures += warm[i].login_failures;
|
||||
for (const auto& [why, count] : warm[i].closes)
|
||||
samples[i].closes[why] += count;
|
||||
samples[i].refused += warm[i].refused;
|
||||
}
|
||||
const auto seconds = std::chrono::duration<double>(Clock::now() - started).count();
|
||||
const auto established = load.established(n);
|
||||
const bool within =
|
||||
report(o, n, established, samples, seconds, cpu_seconds() - cpu0, scrape(o));
|
||||
report(o, n, established, load.exhausted(n), {hold_start, now_seconds()}, samples,
|
||||
seconds, cpu_seconds() - cpu0, scrape(o));
|
||||
if (within && !saturated)
|
||||
sustainable = n;
|
||||
if (!within && !saturated)
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env bash
|
||||
# M10.9d load drill (ADR-0097) for evolution-dev only: the reference-worker qualification run.
|
||||
#
|
||||
# tools/ops/load-drill.sh <tokens-dir> [--context <name>] [--steps N[,N]...] [--hold S]
|
||||
# [--warmup S] [--grants-per-bot C] [--connect HOST:PORT] [--server-name NAME]
|
||||
#
|
||||
# <tokens-dir> holds dev-ca.pem (the current dev session CA) and load-grants.bin, a bundle
|
||||
# made where the dev issuer key lives:
|
||||
# evo-session-probe grants --issuer-key issuer.key --world W --frame F --region R \
|
||||
# --account <HEX32> --accounts <most bots> --count <grants per bot> --lifetime 14400 \
|
||||
# --out load-grants.bin
|
||||
# The issuer key never reaches the machine running the drill. Requires kubectl access to
|
||||
# evolution-dev (patch Deployment/evo-region, read pods and logs), `evo-load` (EVO_LOAD,
|
||||
# default: on PATH) and python3.
|
||||
#
|
||||
# Steps: refuse unless the session window outlasts the drill; enable synthetic inputs on
|
||||
# evo-region (the drill-time override of ADR-0073, never committed) and wait for the new
|
||||
# pod to be Ready; follow its log with timestamps (the server prints `m10 load-report`
|
||||
# every 10 s); run evo-load through the public route; then restore REGION_INPUTS to the Git
|
||||
# value (also on failure) so Argo returns to Synced. The report matches each step's hold
|
||||
# (evo-load's hold_utc) with the server's reports in it and prints, per step, the client
|
||||
# verdict and the server's worst tick work, session work, governor state and tick debt.
|
||||
# Stage and World0 are never touched: the namespace is fixed.
|
||||
set -euo pipefail
|
||||
usage() {
|
||||
echo "usage: $0 <tokens-dir> [--context <name>] [--steps N[,N]...] [--hold S] [--warmup S]" \
|
||||
"[--grants-per-bot C] [--connect HOST:PORT] [--server-name NAME]" >&2
|
||||
exit 2
|
||||
}
|
||||
[ $# -ge 1 ] || usage
|
||||
tokens="$1"; shift
|
||||
k=(kubectl --request-timeout=30s)
|
||||
steps=20,24,28,32 hold=60 warmup=10 per_bot=4
|
||||
connect=void.aeternalisgames.com:7400 server_name=void.aeternalisgames.com
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--context) [ -n "${2:-}" ] || usage; k+=(--context "$2"); shift 2 ;;
|
||||
--steps) [ -n "${2:-}" ] || usage; steps="$2"; shift 2 ;;
|
||||
--hold) [ -n "${2:-}" ] || usage; hold="$2"; shift 2 ;;
|
||||
--warmup) [ -n "${2:-}" ] || usage; warmup="$2"; shift 2 ;;
|
||||
--grants-per-bot) [ -n "${2:-}" ] || usage; per_bot="$2"; shift 2 ;;
|
||||
--connect) [ -n "${2:-}" ] || usage; connect="$2"; shift 2 ;;
|
||||
--server-name) [ -n "${2:-}" ] || usage; server_name="$2"; shift 2 ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
done
|
||||
ns=evolution-dev # Fixed: qualification runs on the dev reference worker only.
|
||||
tool="${EVO_LOAD:-evo-load}"
|
||||
py="$(command -v python3 || command -v python || true)"
|
||||
for f in dev-ca.pem load-grants.bin; do
|
||||
[ -f "$tokens/$f" ] || { echo "missing $tokens/$f" >&2; exit 2; }
|
||||
done
|
||||
command -v "$tool" > /dev/null || [ -x "$tool" ] || { echo "evo-load not found (set EVO_LOAD)" >&2; exit 2; }
|
||||
[ -n "$py" ] || { echo "python3 is required" >&2; exit 2; }
|
||||
selector=app.kubernetes.io/name=evo-region
|
||||
work="$(mktemp -d)"
|
||||
chmod 700 "$work"
|
||||
follow_pid=""
|
||||
say() { echo "$(date -u +%H:%M:%S) $*"; }
|
||||
fail() { say "FAIL $*"; exit 1; }
|
||||
env_of() {
|
||||
"${k[@]}" get deployment -n "$ns" evo-region \
|
||||
-o jsonpath="{.spec.template.spec.containers[0].env[?(@.name==\"$1\")].value}"
|
||||
}
|
||||
ready_pod() { # Waits up to $2 seconds for a Ready pod other than $1.
|
||||
local old="$1" limit="${2:-240}"
|
||||
for _ in $(seq 1 $((limit * 5))); do
|
||||
for p in $("${k[@]}" get pods -n "$ns" -l "$selector" \
|
||||
-o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}'); do
|
||||
[ "$p" = "$old" ] && continue
|
||||
[ "$("${k[@]}" get pod -n "$ns" "$p" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2> /dev/null)" = True ] &&
|
||||
{ echo "$p"; return 0; }
|
||||
done
|
||||
sleep 0.2
|
||||
done
|
||||
return 1
|
||||
}
|
||||
restore() {
|
||||
set +e
|
||||
[ -n "$follow_pid" ] && kill "$follow_pid" 2> /dev/null
|
||||
say "restoring the Git configuration of evo-region (REGION_INPUTS empty)"
|
||||
local current
|
||||
current="$("${k[@]}" get pods -n "$ns" -l "$selector" -o jsonpath='{.items[0].metadata.name}' 2> /dev/null)"
|
||||
"${k[@]}" set env -n "$ns" deployment/evo-region REGION_INPUTS= > /dev/null
|
||||
ready_pod "$current" 240 > /dev/null && say "restored: evo-region Ready" || say "WARN evo-region not Ready after restore"
|
||||
say "drill files kept in $work"
|
||||
}
|
||||
|
||||
world="$(env_of REGION_WORLD)" region="$(env_of REGION_ID)" frame="$(env_of REGION_FRAME)"
|
||||
window="$(env_of REGION_SESSION_WINDOW)"
|
||||
[ -n "$world" ] && [ -n "$region" ] && [ -n "$frame" ] || fail "cannot read the region identity"
|
||||
[ -z "$(env_of REGION_INPUTS)" ] || fail "REGION_INPUTS is already set: another drill may be running"
|
||||
# The drill must end before the session window closes.
|
||||
count=$(echo "$steps" | tr ',' '\n' | wc -l)
|
||||
most=$(echo "$steps" | tr ',' '\n' | sort -n | tail -1)
|
||||
needed=$(( count * (hold + warmup + most / 10 + 5) + 600 ))
|
||||
expiry="${window##*:}"
|
||||
[ -n "$expiry" ] && [ "$expiry" -gt $(( $(date -u +%s) + needed )) ] 2> /dev/null ||
|
||||
fail "the session window ($window) closes within the drill's ${needed} s: renew it first"
|
||||
size=$(wc -c < "$tokens/load-grants.bin")
|
||||
[ $(( size / 160 )) -ge $(( most * per_bot )) ] ||
|
||||
fail "load-grants.bin holds $(( size / 160 )) grants; $most bots x $per_bot need $(( most * per_bot ))"
|
||||
image="$("${k[@]}" get deployment -n "$ns" evo-region -o jsonpath='{.spec.template.spec.containers[0].image}')"
|
||||
say "drill: world $world region $region image $image steps $steps hold ${hold}s"
|
||||
|
||||
trap restore EXIT
|
||||
before="$("${k[@]}" get pods -n "$ns" -l "$selector" -o jsonpath='{.items[0].metadata.name}')"
|
||||
say "enabling synthetic inputs in $ns (drill-time override, restored on exit)"
|
||||
"${k[@]}" set env -n "$ns" deployment/evo-region REGION_INPUTS=development-v1 > /dev/null
|
||||
pod="$(ready_pod "$before" 240)" || fail "no Ready evo-region pod with synthetic inputs"
|
||||
"${k[@]}" logs -n "$ns" -f --timestamps "$pod" > "$work/server.log" 2> /dev/null &
|
||||
follow_pid=$!
|
||||
sleep 5
|
||||
grep -q "ready 1" "$work/server.log" || say "WARN no 'ready 1' in the server log yet"
|
||||
say "running evo-load against $connect"
|
||||
set +e
|
||||
"$tool" --connect "$connect" --server-name "$server_name" --ca "$tokens/dev-ca.pem" \
|
||||
--grants "$tokens/load-grants.bin" --grants-per-bot "$per_bot" --world "$world" \
|
||||
--frame "$frame" --region "$region" --steps "$steps" --hold "$hold" --warmup "$warmup" \
|
||||
--threads 8 --per-client > "$work/load.log" 2>&1
|
||||
load_exit=$?
|
||||
set -e
|
||||
sleep 12 # One more server report after the last hold.
|
||||
grep -E "^load (step|response_ms|outcome_ms|snapshots|starved|verdict|result)" "$work/load.log" || true
|
||||
[ "$load_exit" = 0 ] || say "WARN evo-load exited $load_exit (see $work/load.log)"
|
||||
"$py" - "$work/load.log" "$work/server.log" <<'PY'
|
||||
import re, sys
|
||||
from datetime import datetime
|
||||
load, server = sys.argv[1], sys.argv[2]
|
||||
reports = []
|
||||
for line in open(server, encoding='utf-8', errors='replace'):
|
||||
m = re.match(r'(\S+) m10 load-report (.*)', line.strip())
|
||||
if not m:
|
||||
continue
|
||||
stamp = m.group(1)[:19]
|
||||
t = datetime.strptime(stamp, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=None)
|
||||
epoch = int((t - datetime(1970, 1, 1)).total_seconds())
|
||||
fields = dict(kv.split('=') for kv in m.group(2).split())
|
||||
reports.append((epoch, {k: int(v) for k, v in fields.items()}))
|
||||
print(f"drill server reports={len(reports)}")
|
||||
for line in open(load, encoding='utf-8', errors='replace'):
|
||||
m = re.match(r'load step N=(\d+) .*hold_utc=(\d+)\.\.(\d+)', line)
|
||||
if not m:
|
||||
continue
|
||||
n, t0, t1 = int(m.group(1)), int(m.group(2)), int(m.group(3))
|
||||
# A report covers the 2 s before it: only reports wholly inside the hold count.
|
||||
inside = [r for t, r in reports if t0 + 2 <= t <= t1]
|
||||
if not inside:
|
||||
print(f"drill step N={n} server_reports=0")
|
||||
continue
|
||||
worst = lambda key: max(r[key] for r in inside)
|
||||
sessions = max(1, worst('sessions'))
|
||||
print(f"drill step N={n} server_reports={len(inside)} sessions_max={worst('sessions')} "
|
||||
f"work_p99_max_ms={worst('p99_us') / 1000:.1f} work_max_ms={worst('max_us') / 1000:.1f} "
|
||||
f"session_p99_max_ms={worst('session_p99_us') / 1000:.1f} "
|
||||
f"per_session_ms={worst('session_p99_us') / 1000 / sessions:.3f} "
|
||||
f"refusing_max={worst('refusing')} shed_max={worst('shed')} "
|
||||
f"tick_debt_max={worst('tick_debt')} "
|
||||
f"server_within_budget={int(worst('p99_us') <= 25000 and worst('session_p99_us') <= 500 * sessions and worst('tick_debt') == 0)}")
|
||||
PY
|
||||
say "drill complete (evo-load exit $load_exit); restoring"
|
||||
Reference in New Issue
Block a user