Add M10.9d load drill harness and serialize Mbed TLS use
Foundation / checks (deploy) (push) Successful in 3m24s
Foundation / checks (ecs) (push) Successful in 1m56s
Foundation / checks (client) (push) Successful in 12m18s
Foundation / checks (network) (push) Successful in 13m30s
Foundation / checks (tsan) (push) Successful in 8m0s
Foundation / checks (checks) (push) Successful in 32m39s
Foundation / checks (postgres) (push) Successful in 45m47s
Foundation / postgres-tools (push) Successful in 17s
Foundation / image (push) Successful in 2m30s
Foundation / server-image (push) Successful in 4m8s

ADR-0097. tools/ops/load-drill.sh (runbook docs/networking/M10_LOAD_DRILL.md)
runs the evolution-dev reference qualification: synthetic inputs on for
the run (the ADR-0073 drill-time override, restored on exit), bundle-mode
evo-load bots through the public route, and each step's hold matched with
the server's reports. evo-server logs "m10 load-report" every 10 s, since
the agent cannot reach the ops port. evo-load gains --grants FILE
--grants-per-bot C, exhausted=, hold_utc= and close_reasons=;
evo-session-probe grants gains --accounts N.

Finding: Mbed TLS/PSA are built without MBEDTLS_THREADING_C, and bots on
different threads verifying their first connections at once were refused
as untrusted (two per smoke run). One process-wide recursive crypto lock
now guards every network-library entry point that touches them; the
smoke went from 2/2 failing to 3/3 clean. quic-transport adds a
concurrent-connect test. Refreshes the dev session window.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-08 08:38:13 -04:00
co-authored by Claude Opus 5.5
parent 47b84bce74
commit ada77fd761
16 changed files with 609 additions and 70 deletions
+107 -33
View File
@@ -2,7 +2,8 @@
// runs against the evolution-dev reference worker count as evidence (10.9d); local runs are
// smoke tests and development references.
//
// evo-load --connect HOST:PORT --server-name NAME --ca FILE --issuer-key FILE
// evo-load --connect HOST:PORT --server-name NAME --ca FILE
// (--issuer-key FILE | --grants FILE --grants-per-bot C)
// --world N --frame N --region N [--steps N[,N]...] [--hold S] [--warmup S]
// [--ramp-ms MS] [--threads T] [--seed X] [--lifetime S] [--metrics HOST:PORT]
// [--response-budget-ms MS] [--per-client] [--expect-established]
@@ -14,11 +15,17 @@
// probe marker; every 2 s it sends a probe action; every 10 s it moves its interest by a
// cell (a fresh baseline). A closed or refused bot logs in again after 2 s with the next
// serial.
// M10.9d: with --grants (a bundle from `evo-session-probe grants --accounts N --count C`,
// e.g. made where the dev issuer key lives), bot i logs in with grants i*C .. i*C+C-1 in
// order instead of signing; a bot that used all C stays out (counted as exhausted).
//
// Steps: the bot count rises to each step's N (one login every --ramp-ms), waits --warmup
// seconds, then measures for --hold seconds. Bots stay connected into the next step. Per
// step it prints:
// load step N=.. established=.. logins=.. drops=.. login_failures=.. refused=.. seconds=..
// exhausted=.. hold_utc=START..END (unix seconds of the hold, to match server samples)
// close_reasons=REASON:COUNT,... (session-v1 close reasons; without a session Close,
// 256 + the QUIC transport cause, or 255 when unknown)
// (drops: established sessions closed; login failures: closed before Welcome, including
// transport refusals at the connection limit; refused: closed with capacity or
// rate_limited, either way. Logins, drops and failures include the step's ramp and warmup;
@@ -111,7 +118,8 @@ double ms(std::uint64_t us) {
}
struct Options {
std::string host, server_name, ca, issuer_key, metrics_host;
std::string host, server_name, ca, issuer_key, grants_file, metrics_host;
std::uint64_t grants_per_bot{};
std::uint16_t port{}, metrics_port{};
std::uint64_t world{}, frame{}, region{}, seed{1}, lifetime{900};
std::vector<std::size_t> steps{20};
@@ -125,6 +133,7 @@ struct Samples {
gap_ticks;
std::uint64_t snapshots{}, bytes{}, intents{}, refused_intents{}, logins{}, drops{},
login_failures{}, refused{};
std::map<unsigned, std::uint64_t> closes; // By session close reason (255: no Close).
std::uint64_t connected_polls{}; // Polls while established (starvation is judged on these).
};
void append(std::vector<std::uint64_t>& to, const std::vector<std::uint64_t>& from) {
@@ -142,6 +151,8 @@ struct Bot {
client::InputLead lead;
std::uint64_t last_tick{}, last_view{}, last_bytes{}, next_action_id{1};
std::uint64_t applied{}; // Newest applied intention sequence seen in an own marker.
std::uint64_t used{}; // Bundle grants used (bundle mode).
std::atomic<bool> exhausted{false};
std::map<std::uint64_t, Clock::time_point> receipt_pending, response_pending, outcome_pending;
std::map<std::uint64_t, std::uint64_t> intent_tick;
std::int16_t vx{}, vy{};
@@ -153,9 +164,11 @@ struct Bot {
class Load {
public:
Load(Options o, GrantSigningKey key, std::string address)
: o_{std::move(o)}, key_{std::move(key)}, address_{std::move(address)},
authority_{read(o_.ca)} {
// Signs with `key`, or logs in from `bundle` (concatenated grants, grants_per_bot each).
Load(Options o, std::optional<GrantSigningKey> key, std::vector<SessionGrant> bundle,
std::string address)
: o_{std::move(o)}, key_{std::move(key)}, bundle_{std::move(bundle)},
address_{std::move(address)}, authority_{read(o_.ca)} {
const auto most = *std::ranges::max_element(o_.steps);
bots_.reserve(most);
for (std::size_t i = 0; i < most; ++i) {
@@ -199,6 +212,11 @@ class Load {
}
return out;
}
[[nodiscard]] std::size_t exhausted(std::size_t n) const {
return static_cast<std::size_t>(
std::count_if(bots_.begin(), bots_.begin() + static_cast<std::ptrdiff_t>(n),
[](const auto& b) { return b->exhausted.load(); }));
}
[[nodiscard]] std::size_t established(std::size_t n) const {
return static_cast<std::size_t>(
std::count_if(bots_.begin(), bots_.begin() + static_cast<std::ptrdiff_t>(n),
@@ -229,15 +247,26 @@ class Load {
}
}
void login(Bot& b, Clock::time_point now) {
GrantClaims claims;
claims.account = b.account;
claims.serial = ++b.serial;
claims.world = {o_.world, o_.frame};
claims.region = o_.region;
claims.issued_at = now_seconds();
claims.expires_at = claims.issued_at + o_.lifetime;
claims.grant_id = (b.serial << 16U) ^ (b.index + 1);
const auto grant = key_.sign(claims);
std::optional<SessionGrant> grant;
if (key_) {
GrantClaims claims;
claims.account = b.account;
claims.serial = ++b.serial;
claims.world = {o_.world, o_.frame};
claims.region = o_.region;
claims.issued_at = now_seconds();
claims.expires_at = claims.issued_at + o_.lifetime;
claims.grant_id = (b.serial << 16U) ^ (b.index + 1);
if (auto signed_grant = key_->sign(claims))
grant = *signed_grant;
} else {
if (b.used >= o_.grants_per_bot) {
b.exhausted = true; // Out of grants: stays out.
b.retry_at = Clock::time_point::max();
return;
}
grant = bundle_.at((b.index * o_.grants_per_bot) + b.used++);
}
client::RegionSessionClientConfig config;
config.transport.server_address = address_;
config.transport.port = o_.port;
@@ -278,6 +307,13 @@ class Load {
++b.s.drops;
else
++b.s.login_failures;
if (const auto reason = c.close_reason();
reason && *reason != SessionCloseReason::connection_lost)
++b.s.closes[static_cast<unsigned>(*reason)];
else if (const auto cause = c.transport_cause()) // No session Close: 256 + cause.
++b.s.closes[256U + static_cast<unsigned>(*cause)];
else
++b.s.closes[255U];
if (c.close_reason() == SessionCloseReason::capacity ||
c.close_reason() == SessionCloseReason::rate_limited)
++b.s.refused;
@@ -406,7 +442,8 @@ class Load {
}
Options o_;
GrantSigningKey key_;
std::optional<GrantSigningKey> key_;
std::vector<SessionGrant> bundle_;
std::string address_, authority_;
std::vector<std::unique_ptr<Bot>> bots_;
std::vector<std::mutex> locks_;
@@ -481,9 +518,9 @@ void line(const char* name, const std::vector<std::uint64_t>& pooled,
}
// One step's report; true when within budget.
bool report(const Options& o, std::size_t n, std::size_t established,
const std::vector<Samples>& all, double seconds, double cpu,
const std::map<std::string, std::uint64_t>& server) {
bool report(const Options& o, std::size_t n, std::size_t established, std::size_t exhausted,
std::pair<std::uint64_t, std::uint64_t> hold_utc, const std::vector<Samples>& all,
double seconds, double cpu, const std::map<std::string, std::uint64_t>& server) {
Samples pooled;
std::vector<std::uint64_t> response95, receipt95, outcome95, login95, correction95, rates, kib;
for (const auto& s : all) {
@@ -499,6 +536,8 @@ bool report(const Options& o, std::size_t n, std::size_t established,
pooled.logins += s.logins;
pooled.drops += s.drops;
pooled.login_failures += s.login_failures;
for (const auto& [why, count] : s.closes)
pooled.closes[why] += count;
pooled.refused += s.refused;
response95.push_back(percentile(s.response_us, 0.95));
receipt95.push_back(percentile(s.receipt_us, 0.95));
@@ -511,12 +550,18 @@ bool report(const Options& o, std::size_t n, std::size_t established,
kib.push_back(static_cast<std::uint64_t>(
std::lround(static_cast<double>(s.bytes) / 1024.0 / std::max(seconds, 0.001))));
}
std::string reasons;
for (const auto& [why, count] : pooled.closes)
reasons += (reasons.empty() ? "" : ",") + std::to_string(why) + ":" + std::to_string(count);
std::printf("load step N=%zu established=%zu logins=%llu drops=%llu login_failures=%llu "
"refused=%llu seconds=%.1f\n",
"refused=%llu seconds=%.1f exhausted=%zu hold_utc=%llu..%llu close_reasons=%s\n",
n, established, static_cast<unsigned long long>(pooled.logins),
static_cast<unsigned long long>(pooled.drops),
static_cast<unsigned long long>(pooled.login_failures),
static_cast<unsigned long long>(pooled.refused), seconds);
static_cast<unsigned long long>(pooled.refused), seconds, exhausted,
static_cast<unsigned long long>(hold_utc.first),
static_cast<unsigned long long>(hold_utc.second),
reasons.empty() ? "-" : reasons.c_str());
line("response", pooled.response_us, response95, 1000.0, "ms");
line("receipt", pooled.receipt_us, receipt95, 1000.0, "ms");
line("outcome", pooled.outcome_us, outcome95, 1000.0, "ms");
@@ -663,6 +708,10 @@ int main(int argc, char** argv) {
o.ca = value;
else if (flag == "--issuer-key")
o.issuer_key = value;
else if (flag == "--grants")
o.grants_file = value;
else if (n && flag == "--grants-per-bot" && *n >= 1 && *n <= 64)
o.grants_per_bot = *n;
else if (flag == "--steps") {
const auto s = steps_of(value);
ok = s.has_value();
@@ -676,7 +725,7 @@ int main(int argc, char** argv) {
o.region = *n;
else if (n && flag == "--seed" && *n != 0)
o.seed = *n;
else if (n && flag == "--lifetime" && *n >= 60 && *n <= 3600)
else if (n && flag == "--lifetime" && *n >= 60 && *n <= 14400)
o.lifetime = *n;
else if (n && flag == "--hold" && *n >= 1 && *n <= 3600)
o.hold_s = *n;
@@ -691,23 +740,44 @@ int main(int argc, char** argv) {
else
ok = false;
}
if (!ok || o.host.empty() || o.server_name.empty() || o.ca.empty() || o.issuer_key.empty() ||
o.world == 0 || o.frame == 0 || o.region == 0) {
const bool signs = !o.issuer_key.empty();
if (!ok || o.host.empty() || o.server_name.empty() || o.ca.empty() ||
signs == !o.grants_file.empty() || (!signs && o.grants_per_bot == 0) || o.world == 0 ||
o.frame == 0 || o.region == 0) {
std::fputs("usage: evo-load --connect HOST:PORT --server-name NAME --ca FILE "
"--issuer-key FILE --world N --frame N --region N [--steps N[,N]...] "
"(--issuer-key FILE | --grants FILE --grants-per-bot C) --world N "
"--frame N --region N [--steps N[,N]...] "
"[--hold S] [--warmup S] [--ramp-ms MS] [--threads T] [--seed X] "
"[--lifetime S] [--metrics HOST:PORT] [--response-budget-ms MS] "
"[--per-client] [--expect-established]\n",
stderr);
return 2;
}
const auto scalar = read(o.issuer_key);
auto key = GrantSigningKey::import(
DeploymentEnvironment::dev, 1,
std::span{reinterpret_cast<const std::uint8_t*>(scalar.data()), scalar.size()});
if (!key) {
std::puts("error issuer-key");
return 2;
std::optional<GrantSigningKey> key;
std::vector<SessionGrant> bundle;
if (signs) {
const auto scalar = read(o.issuer_key);
auto imported = GrantSigningKey::import(
DeploymentEnvironment::dev, 1,
std::span{reinterpret_cast<const std::uint8_t*>(scalar.data()), scalar.size()});
if (!imported) {
std::puts("error issuer-key");
return 2;
}
key.emplace(std::move(*imported));
} else {
const auto bytes = read(o.grants_file);
const auto needed = *std::ranges::max_element(o.steps) * o.grants_per_bot;
if (bytes.size() % session_grant_bytes != 0 ||
bytes.size() / session_grant_bytes < needed) {
std::printf("error grants (need %llu grants: bots x grants-per-bot)\n",
static_cast<unsigned long long>(needed));
return 2;
}
bundle.resize(bytes.size() / session_grant_bytes);
for (std::size_t i = 0; i < bundle.size(); ++i)
std::copy_n(bytes.begin() + static_cast<std::ptrdiff_t>(i * session_grant_bytes),
session_grant_bytes, reinterpret_cast<char*>(bundle[i].data()));
}
const auto address = quic::resolve_ipv4(o.host);
if (!address) {
@@ -723,7 +793,7 @@ int main(int argc, char** argv) {
bool smoke_ok = true;
std::optional<std::size_t> sustainable, saturated;
{
Load load{o, std::move(*key), *address};
Load load{o, std::move(key), std::move(bundle), *address};
for (const auto n : o.steps) {
load.activate(n);
std::this_thread::sleep_for(std::chrono::seconds{o.warmup_s});
@@ -731,6 +801,7 @@ int main(int argc, char** argv) {
// the warmup is discarded.
const auto warm = load.take(n);
const auto started = Clock::now();
const auto hold_start = now_seconds();
const auto cpu0 = cpu_seconds();
std::this_thread::sleep_for(std::chrono::seconds{o.hold_s});
auto samples = load.take(n);
@@ -739,12 +810,15 @@ int main(int argc, char** argv) {
samples[i].logins += warm[i].logins;
samples[i].drops += warm[i].drops;
samples[i].login_failures += warm[i].login_failures;
for (const auto& [why, count] : warm[i].closes)
samples[i].closes[why] += count;
samples[i].refused += warm[i].refused;
}
const auto seconds = std::chrono::duration<double>(Clock::now() - started).count();
const auto established = load.established(n);
const bool within =
report(o, n, established, samples, seconds, cpu_seconds() - cpu0, scrape(o));
report(o, n, established, load.exhausted(n), {hold_start, now_seconds()}, samples,
seconds, cpu_seconds() - cpu0, scrape(o));
if (within && !saturated)
sustainable = n;
if (!within && !saturated)
+161
View File
@@ -0,0 +1,161 @@
#!/usr/bin/env bash
# M10.9d load drill (ADR-0097) for evolution-dev only: the reference-worker qualification run.
#
# tools/ops/load-drill.sh <tokens-dir> [--context <name>] [--steps N[,N]...] [--hold S]
# [--warmup S] [--grants-per-bot C] [--connect HOST:PORT] [--server-name NAME]
#
# <tokens-dir> holds dev-ca.pem (the current dev session CA) and load-grants.bin, a bundle
# made where the dev issuer key lives:
# evo-session-probe grants --issuer-key issuer.key --world W --frame F --region R \
# --account <HEX32> --accounts <most bots> --count <grants per bot> --lifetime 14400 \
# --out load-grants.bin
# The issuer key never reaches the machine running the drill. Requires kubectl access to
# evolution-dev (patch Deployment/evo-region, read pods and logs), `evo-load` (EVO_LOAD,
# default: on PATH) and python3.
#
# Steps: refuse unless the session window outlasts the drill; enable synthetic inputs on
# evo-region (the drill-time override of ADR-0073, never committed) and wait for the new
# pod to be Ready; follow its log with timestamps (the server prints `m10 load-report`
# every 10 s); run evo-load through the public route; then restore REGION_INPUTS to the Git
# value (also on failure) so Argo returns to Synced. The report matches each step's hold
# (evo-load's hold_utc) with the server's reports in it and prints, per step, the client
# verdict and the server's worst tick work, session work, governor state and tick debt.
# Stage and World0 are never touched: the namespace is fixed.
set -euo pipefail
usage() {
echo "usage: $0 <tokens-dir> [--context <name>] [--steps N[,N]...] [--hold S] [--warmup S]" \
"[--grants-per-bot C] [--connect HOST:PORT] [--server-name NAME]" >&2
exit 2
}
[ $# -ge 1 ] || usage
tokens="$1"; shift
k=(kubectl --request-timeout=30s)
steps=20,24,28,32 hold=60 warmup=10 per_bot=4
connect=void.aeternalisgames.com:7400 server_name=void.aeternalisgames.com
while [ $# -gt 0 ]; do
case "$1" in
--context) [ -n "${2:-}" ] || usage; k+=(--context "$2"); shift 2 ;;
--steps) [ -n "${2:-}" ] || usage; steps="$2"; shift 2 ;;
--hold) [ -n "${2:-}" ] || usage; hold="$2"; shift 2 ;;
--warmup) [ -n "${2:-}" ] || usage; warmup="$2"; shift 2 ;;
--grants-per-bot) [ -n "${2:-}" ] || usage; per_bot="$2"; shift 2 ;;
--connect) [ -n "${2:-}" ] || usage; connect="$2"; shift 2 ;;
--server-name) [ -n "${2:-}" ] || usage; server_name="$2"; shift 2 ;;
*) usage ;;
esac
done
ns=evolution-dev # Fixed: qualification runs on the dev reference worker only.
tool="${EVO_LOAD:-evo-load}"
py="$(command -v python3 || command -v python || true)"
for f in dev-ca.pem load-grants.bin; do
[ -f "$tokens/$f" ] || { echo "missing $tokens/$f" >&2; exit 2; }
done
command -v "$tool" > /dev/null || [ -x "$tool" ] || { echo "evo-load not found (set EVO_LOAD)" >&2; exit 2; }
[ -n "$py" ] || { echo "python3 is required" >&2; exit 2; }
selector=app.kubernetes.io/name=evo-region
work="$(mktemp -d)"
chmod 700 "$work"
follow_pid=""
say() { echo "$(date -u +%H:%M:%S) $*"; }
fail() { say "FAIL $*"; exit 1; }
env_of() {
"${k[@]}" get deployment -n "$ns" evo-region \
-o jsonpath="{.spec.template.spec.containers[0].env[?(@.name==\"$1\")].value}"
}
ready_pod() { # Waits up to $2 seconds for a Ready pod other than $1.
local old="$1" limit="${2:-240}"
for _ in $(seq 1 $((limit * 5))); do
for p in $("${k[@]}" get pods -n "$ns" -l "$selector" \
-o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}'); do
[ "$p" = "$old" ] && continue
[ "$("${k[@]}" get pod -n "$ns" "$p" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2> /dev/null)" = True ] &&
{ echo "$p"; return 0; }
done
sleep 0.2
done
return 1
}
restore() {
set +e
[ -n "$follow_pid" ] && kill "$follow_pid" 2> /dev/null
say "restoring the Git configuration of evo-region (REGION_INPUTS empty)"
local current
current="$("${k[@]}" get pods -n "$ns" -l "$selector" -o jsonpath='{.items[0].metadata.name}' 2> /dev/null)"
"${k[@]}" set env -n "$ns" deployment/evo-region REGION_INPUTS= > /dev/null
ready_pod "$current" 240 > /dev/null && say "restored: evo-region Ready" || say "WARN evo-region not Ready after restore"
say "drill files kept in $work"
}
world="$(env_of REGION_WORLD)" region="$(env_of REGION_ID)" frame="$(env_of REGION_FRAME)"
window="$(env_of REGION_SESSION_WINDOW)"
[ -n "$world" ] && [ -n "$region" ] && [ -n "$frame" ] || fail "cannot read the region identity"
[ -z "$(env_of REGION_INPUTS)" ] || fail "REGION_INPUTS is already set: another drill may be running"
# The drill must end before the session window closes.
count=$(echo "$steps" | tr ',' '\n' | wc -l)
most=$(echo "$steps" | tr ',' '\n' | sort -n | tail -1)
needed=$(( count * (hold + warmup + most / 10 + 5) + 600 ))
expiry="${window##*:}"
[ -n "$expiry" ] && [ "$expiry" -gt $(( $(date -u +%s) + needed )) ] 2> /dev/null ||
fail "the session window ($window) closes within the drill's ${needed} s: renew it first"
size=$(wc -c < "$tokens/load-grants.bin")
[ $(( size / 160 )) -ge $(( most * per_bot )) ] ||
fail "load-grants.bin holds $(( size / 160 )) grants; $most bots x $per_bot need $(( most * per_bot ))"
image="$("${k[@]}" get deployment -n "$ns" evo-region -o jsonpath='{.spec.template.spec.containers[0].image}')"
say "drill: world $world region $region image $image steps $steps hold ${hold}s"
trap restore EXIT
before="$("${k[@]}" get pods -n "$ns" -l "$selector" -o jsonpath='{.items[0].metadata.name}')"
say "enabling synthetic inputs in $ns (drill-time override, restored on exit)"
"${k[@]}" set env -n "$ns" deployment/evo-region REGION_INPUTS=development-v1 > /dev/null
pod="$(ready_pod "$before" 240)" || fail "no Ready evo-region pod with synthetic inputs"
"${k[@]}" logs -n "$ns" -f --timestamps "$pod" > "$work/server.log" 2> /dev/null &
follow_pid=$!
sleep 5
grep -q "ready 1" "$work/server.log" || say "WARN no 'ready 1' in the server log yet"
say "running evo-load against $connect"
set +e
"$tool" --connect "$connect" --server-name "$server_name" --ca "$tokens/dev-ca.pem" \
--grants "$tokens/load-grants.bin" --grants-per-bot "$per_bot" --world "$world" \
--frame "$frame" --region "$region" --steps "$steps" --hold "$hold" --warmup "$warmup" \
--threads 8 --per-client > "$work/load.log" 2>&1
load_exit=$?
set -e
sleep 12 # One more server report after the last hold.
grep -E "^load (step|response_ms|outcome_ms|snapshots|starved|verdict|result)" "$work/load.log" || true
[ "$load_exit" = 0 ] || say "WARN evo-load exited $load_exit (see $work/load.log)"
"$py" - "$work/load.log" "$work/server.log" <<'PY'
import re, sys
from datetime import datetime
load, server = sys.argv[1], sys.argv[2]
reports = []
for line in open(server, encoding='utf-8', errors='replace'):
m = re.match(r'(\S+) m10 load-report (.*)', line.strip())
if not m:
continue
stamp = m.group(1)[:19]
t = datetime.strptime(stamp, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=None)
epoch = int((t - datetime(1970, 1, 1)).total_seconds())
fields = dict(kv.split('=') for kv in m.group(2).split())
reports.append((epoch, {k: int(v) for k, v in fields.items()}))
print(f"drill server reports={len(reports)}")
for line in open(load, encoding='utf-8', errors='replace'):
m = re.match(r'load step N=(\d+) .*hold_utc=(\d+)\.\.(\d+)', line)
if not m:
continue
n, t0, t1 = int(m.group(1)), int(m.group(2)), int(m.group(3))
# A report covers the 2 s before it: only reports wholly inside the hold count.
inside = [r for t, r in reports if t0 + 2 <= t <= t1]
if not inside:
print(f"drill step N={n} server_reports=0")
continue
worst = lambda key: max(r[key] for r in inside)
sessions = max(1, worst('sessions'))
print(f"drill step N={n} server_reports={len(inside)} sessions_max={worst('sessions')} "
f"work_p99_max_ms={worst('p99_us') / 1000:.1f} work_max_ms={worst('max_us') / 1000:.1f} "
f"session_p99_max_ms={worst('session_p99_us') / 1000:.1f} "
f"per_session_ms={worst('session_p99_us') / 1000 / sessions:.3f} "
f"refusing_max={worst('refusing')} shed_max={worst('shed')} "
f"tick_debt_max={worst('tick_debt')} "
f"server_within_budget={int(worst('p99_us') <= 25000 and worst('session_p99_us') <= 500 * sessions and worst('tick_debt') == 0)}")
PY
say "drill complete (evo-load exit $load_exit); restoring"