diff --git a/deploy/kubernetes/region/dev-session-route/listener.yaml b/deploy/kubernetes/region/dev-session-route/listener.yaml index 36667e2..bf5f1a1 100644 --- a/deploy/kubernetes/region/dev-session-route/listener.yaml +++ b/deploy/kubernetes/region/dev-session-route/listener.yaml @@ -12,7 +12,7 @@ spec: env: - {name: REGION_SESSION, value: "dev"} - {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"} - - {name: REGION_SESSION_WINDOW, value: "dev-20261008T135748-32746:1791482268"} + - {name: REGION_SESSION_WINDOW, value: "dev-20261008T145441-2860:1791485081"} - {name: REGION_M10_MAX_GRANT_LIFETIME, value: "14400"} ports: # QUIC (ADR-0090): UDP only; nothing listens on TCP 7400. diff --git a/docs/adr/0097-m10-load-and-abuse-qualification.md b/docs/adr/0097-m10-load-and-abuse-qualification.md index cb00172..17043c6 100644 --- a/docs/adr/0097-m10-load-and-abuse-qualification.md +++ b/docs/adr/0097-m10-load-and-abuse-qualification.md @@ -255,3 +255,38 @@ snapshots. - **Not yet:** the reference run. It needs an image containing this change promoted to dev, a load bundle made where the issuer key lives, and the maintainer's go-ahead, because the drill restarts the dev region twice. + +## 10.9d first reference run (2026-10-08, evolution-dev, image `6c96629e…`) +Drill with steps 20, 24, 28 and 32 bots, 60 s holds and 4 grants per bot, run from the +maintainer's workstation over the LAN route. Synthetic inputs were on from 14:33:46 to +14:38:51 UTC; the region was restored (`REGION_INPUTS` empty, epoch 60, Synced and +Healthy), and committed history advanced from 12,084 to 18,142 ticks. + +It did not measure capacity: +- **The governor tripped six seconds after the restart, with no session.** It went + `refusing=1 shed=1` at p95 24.4 ms over a window holding only the restart's first + ticks (cold caches, first commits). +- **The bots burned their grants.** They retried every 2–2.5 s, so 19 of 20 used all 4 + grants within about 10 s (76 refusals: 63 `capacity`, 13 `rate_limited`). Only one + session was measured, and the later steps added nobody. +- **The drill's live log follow ended at 14:34:19,** so it holds 4 server reports. + +**Measured with 1 session**, the first reference numbers: +- tick work: p95 4.0 ms, p99 and maximum 33 ms (isolated spikes); +- session layer: p95 0.68 ms, p99 3.0 ms; +- client side: response p95 111 ms, outcome p95 1.13 s, snapshots 20 Hz. + +The spikes exceed the 25 ms p99 budget and need to be attributed (commit flush or +snapshot rebuild) in the next run. + +Fixes before the next run: +- **Governor:** work counts only over a full 40-tick window; tick debt still counts at + once (unit test added). +- **evo-load:** a login refused for load backs off 10 s, and in bundle mode reuses its + grant, which the region refused before using it. +- **Drill:** + - waits for a calm `load-report` before the bots start; + - takes the drill pod's whole log before restoring; + - waits after restoring for a Ready pod other than the drill pod (the old check could + accept a pod still running during the rollout); + - checks that `REGION_INPUTS` is empty again. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index b22abff..497b396 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -5158,3 +5158,29 @@ Executed: Not done: the reference run on evolution-dev. It needs this image promoted, a load bundle and the maintainer's go-ahead. + +## 2026-10-08 — M10.9d first evolution-dev load drill (ADR-0097) + +Ran `tools/ops/load-drill.sh` on evolution-dev, with the image `6c96629e…` promoted and the +provision Job run with it. Steps 20/24/28/32, 60 s holds, 4 grants per bot, from the +maintainer workstation. Restore verified: `REGION_INPUTS` empty, epoch 60, `ready 1`, Argo +Synced and Healthy. + +Result: not a capacity qualification. +- The governor refused from six seconds after the restart, with no session (p95 24.4 ms + over a partial window of startup ticks). +- 19 of 20 bots spent all 4 grants on refusals within about 10 s. +- The live log follow ended early, so only 4 server reports were captured. + +Reference numbers with 1 session (`m10 load-report` at 14:33:58): +- tick work p95 4.0 ms, p99 and maximum 33 ms; +- session layer p95 0.68 ms, p99 3.0 ms; +- client response p95 111 ms, outcome p95 1.13 s, snapshots at 20 Hz. + +Fixes (in the following commit): +- the governor acts on work only over a full window; +- evo-load backs off 10 s after a load refusal and reuses the unused bundle grant; +- the drill waits for a calm report, takes the pod log before restoring, and has a + stricter restore check. + +The next run needs a new image and a new bundle: some of this bundle's grants are spent. diff --git a/src/server/load_governor.cpp b/src/server/load_governor.cpp index 45b33e0..d44d2cb 100644 --- a/src/server/load_governor.cpp +++ b/src/server/load_governor.cpp @@ -42,8 +42,13 @@ bool LoadGovernor::observe(std::uint64_t work_us, std::uint64_t session_us, std: state_.session_p99_us = rank(sessions, n, 99); const auto before = std::pair{state_.refusing, state_.shed}; - const bool hot = state_.p95_us >= policy_.refuse_us || tick_debt >= refuse_debt_ticks; - const bool overloaded = state_.p95_us >= policy_.shed_us || tick_debt >= shed_debt_ticks; + // Work counts only over a full window: after a (re)start the first ticks (cold caches, the + // first commits) would otherwise be the p95 alone (seen on evolution-dev, 10.9d). Tick + // debt is acted on at once. + const bool full = n == window_.size(); + const bool hot = (full && state_.p95_us >= policy_.refuse_us) || tick_debt >= refuse_debt_ticks; + const bool overloaded = + (full && state_.p95_us >= policy_.shed_us) || tick_debt >= shed_debt_ticks; if (hot) state_.refusing = true; if (overloaded && state_.shed < policy_.max_shed && diff --git a/tests/ops_tests.cpp b/tests/ops_tests.cpp index d2baa4d..3e89f75 100644 --- a/tests/ops_tests.cpp +++ b/tests/ops_tests.cpp @@ -154,6 +154,19 @@ void load_governor() { CHECK(!LoadGovernor::create({19'000, 15'000, 20'000, 10'000, 1'000'000, 5'000'000, 3})); CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 10'000, 0, 5'000'000, 3})); CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 10'000, 1'000'000, 5'000'000, 4})); + // After a (re)start, work counts only once the window is full (seen on evolution-dev: + // the first slow ticks alone tripped refusal before any session). Tick debt counts at once. + { + auto fresh = *LoadGovernor::create({}); + std::uint64_t t = 0; + for (int i = 0; i < 39; ++i) + CHECK(!fresh.observe(30'000, 1'000, 0, t += 50'000) && !fresh.state().refusing); + CHECK(fresh.state().p95_us == 30'000); + CHECK(fresh.observe(30'000, 1'000, 0, t += 50'000) && fresh.state().refusing && + fresh.state().shed == 1); + auto debt = *LoadGovernor::create({}); + CHECK(debt.observe(1'000, 100, 2, 50'000) && debt.state().refusing); + } auto made = LoadGovernor::create({}); CHECK(made.has_value()); auto& g = *made; diff --git a/tools/network/load.cpp b/tools/network/load.cpp index b2c7c08..776d3c5 100644 --- a/tools/network/load.cpp +++ b/tools/network/load.cpp @@ -13,8 +13,9 @@ // process). Bots run on T worker threads (each owns a slice). Every server tick a bot sends // one intention for tick + its input lead (a random walk turning every 2 s) and predicts its // probe marker; every 2 s it sends a probe action; every 10 s it moves its interest by a -// cell (a fresh baseline). A closed or refused bot logs in again after 2 s with the next -// serial. +// cell (a fresh baseline). A closed bot logs in again after 2 s with the next serial; one +// refused for load (capacity, rate_limited) waits 10 s and, in bundle mode, reuses the same +// grant (the region refused it before using it). // M10.9d: with --grants (a bundle from `evo-session-probe grants --accounts N --count C`, // e.g. made where the dev issuer key lives), bot i logs in with grants i*C .. i*C+C-1 in // order instead of signing; a bot that used all C stays out (counted as exhausted). @@ -314,12 +315,20 @@ class Load { ++b.s.closes[256U + static_cast(*cause)]; else ++b.s.closes[255U]; - if (c.close_reason() == SessionCloseReason::capacity || - c.close_reason() == SessionCloseReason::rate_limited) + const bool refused = c.close_reason() == SessionCloseReason::capacity || + c.close_reason() == SessionCloseReason::rate_limited; + auto backoff = std::chrono::seconds{2}; + if (refused) { ++b.s.refused; + // Refused for load before the grant was used: the region is shedding, so back + // off longer, and a bundle grant stays usable for the next try (10.9d). + backoff = std::chrono::seconds{10}; + if (!key_ && b.used > 0) + --b.used; + } b.c.reset(); - b.retry_at = now + std::chrono::seconds{2} + - milliseconds{std::uniform_int_distribution{0, 500}(b.rng)}; + b.retry_at = + now + backoff + milliseconds{std::uniform_int_distribution{0, 500}(b.rng)}; return; } if (c.state() != client::RegionSessionState::established) diff --git a/tools/ops/load-drill.sh b/tools/ops/load-drill.sh index d03ec09..57d521c 100644 --- a/tools/ops/load-drill.sh +++ b/tools/ops/load-drill.sh @@ -15,9 +15,10 @@ # # Steps: refuse unless the session window outlasts the drill; enable synthetic inputs on # evo-region (the drill-time override of ADR-0073, never committed) and wait for the new -# pod to be Ready; follow its log with timestamps (the server prints `m10 load-report` -# every 10 s); run evo-load through the public route; then restore REGION_INPUTS to the Git -# value (also on failure) so Argo returns to Synced. The report matches each step's hold +# pod to be Ready and for a calm `m10 load-report` (the server prints one every 10 s; the +# governor acts on a full 40-tick window); run evo-load through the public route; take the +# drill pod's whole log with timestamps; then restore REGION_INPUTS to the Git value (also +# on failure) and wait for a Ready pod other than the drill pod, so Argo returns to Synced. The report matches each step's hold # (evo-load's hold_utc) with the server's reports in it and prints, per step, the client # verdict and the server's worst tick work, session work, governor state and tick debt. # Stage and World0 are never touched: the namespace is fixed. @@ -55,7 +56,7 @@ command -v "$tool" > /dev/null || [ -x "$tool" ] || { echo "evo-load not found ( selector=app.kubernetes.io/name=evo-region work="$(mktemp -d)" chmod 700 "$work" -follow_pid="" +drill_pod="" say() { echo "$(date -u +%H:%M:%S) $*"; } fail() { say "FAIL $*"; exit 1; } env_of() { @@ -77,12 +78,14 @@ ready_pod() { # Waits up to $2 seconds for a Ready pod other than $1. } restore() { set +e - [ -n "$follow_pid" ] && kill "$follow_pid" 2> /dev/null say "restoring the Git configuration of evo-region (REGION_INPUTS empty)" - local current - current="$("${k[@]}" get pods -n "$ns" -l "$selector" -o jsonpath='{.items[0].metadata.name}' 2> /dev/null)" "${k[@]}" set env -n "$ns" deployment/evo-region REGION_INPUTS= > /dev/null - ready_pod "$current" 240 > /dev/null && say "restored: evo-region Ready" || say "WARN evo-region not Ready after restore" + # Ready means a pod other than the drill's, i.e. the restored configuration. + ready_pod "${drill_pod:-none}" 240 > /dev/null && say "restored: evo-region Ready" || + say "WARN evo-region not Ready after restore" + [ "$("${k[@]}" get deployment -n "$ns" evo-region \ + -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="REGION_INPUTS")].value}')" = "" ] || + say "WARN REGION_INPUTS is not empty after restore" say "drill files kept in $work" } @@ -108,10 +111,16 @@ before="$("${k[@]}" get pods -n "$ns" -l "$selector" -o jsonpath='{.items[0].met say "enabling synthetic inputs in $ns (drill-time override, restored on exit)" "${k[@]}" set env -n "$ns" deployment/evo-region REGION_INPUTS=development-v1 > /dev/null pod="$(ready_pod "$before" 240)" || fail "no Ready evo-region pod with synthetic inputs" -"${k[@]}" logs -n "$ns" -f --timestamps "$pod" > "$work/server.log" 2> /dev/null & -follow_pid=$! -sleep 5 -grep -q "ready 1" "$work/server.log" || say "WARN no 'ready 1' in the server log yet" +drill_pod="$pod" +# Warm up: wait for a full governor window and a calm report (refusing=0 shed=0). +calm="" +for _ in $(seq 1 60); do + sleep 3 + last="$("${k[@]}" logs -n "$ns" --since=15s "$pod" 2> /dev/null | grep "m10 load-report" | tail -1)" + case "$last" in *"refusing=0 shed=0"*) calm=1; break ;; esac +done +[ -n "$calm" ] || fail "the region did not report calm within 3 minutes of starting: $last" +say "region calm: $last" say "running evo-load against $connect" set +e "$tool" --connect "$connect" --server-name "$server_name" --ca "$tokens/dev-ca.pem" \ @@ -121,6 +130,8 @@ set +e load_exit=$? set -e sleep 12 # One more server report after the last hold. +"${k[@]}" logs -n "$ns" --timestamps "$pod" > "$work/server.log" 2> /dev/null || + say "WARN could not read the drill pod's log" grep -E "^load (step|response_ms|outcome_ms|snapshots|starved|verdict|result)" "$work/load.log" || true [ "$load_exit" = 0 ] || say "WARN evo-load exited $load_exit (see $work/load.log)" "$py" - "$work/load.log" "$work/server.log" <<'PY'