Files
aevolution/tools/ops/load-drill.sh
T
dingoandClaude Opus 5.5 46b6358606
Foundation / checks (deploy) (push) Successful in 3m26s
Foundation / checks (ecs) (push) Successful in 1m54s
Foundation / checks (client) (push) Successful in 12m36s
Foundation / checks (network) (push) Successful in 13m35s
Foundation / checks (tsan) (push) Successful in 7m48s
Foundation / checks (checks) (push) Successful in 32m51s
Foundation / checks (postgres) (push) Successful in 46m20s
Foundation / postgres-tools (push) Successful in 1m35s
Foundation / image (push) Successful in 2m32s
Foundation / server-image (push) Successful in 4m14s
Share replication encoding per interest; fix dev retained builds
ADR-0097. The second evolution-dev load drill (image 7952185d) held 20
sessions with client budgets met (response p95 111 ms, outcome p95 about
1.2 s, 20 Hz, no starvation) but the server over its own: tick work p99
48-70 ms (25 ms budget), session layer 1.2-2.0 ms per session at p99
(0.5 ms budget), governor refusing and shedding to level 3 to little
effect. ReplicationServer::tick re-encoded every environment cell and
organism for each session. Sessions with the same requested interest
(no perception predicate) now share one encoding per tick; markers stay
per session. Locally 21 sessions: about 970 us -> 360 us per tick.
The drill now waits for two consecutive calm reports before the bots.

The dev overlay's REGION_RETAINED listed the deployed build itself
(61df6a1, f88bee2), which the deploy policy refuses (CI deploy failed on
those commits); it now lists only the other builds, keeping 6c96629e,
the build of the region head. Refreshes the dev session window.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 15:03:45 -04:00

176 lines
9.1 KiB
Bash

#!/usr/bin/env bash
# M10.9d load drill (ADR-0097) for evolution-dev only: the reference-worker qualification run.
#
# tools/ops/load-drill.sh <tokens-dir> [--context <name>] [--steps N[,N]...] [--hold S]
# [--warmup S] [--grants-per-bot C] [--connect HOST:PORT] [--server-name NAME]
#
# <tokens-dir> holds dev-ca.pem (the current dev session CA) and load-grants.bin, a bundle
# made where the dev issuer key lives:
# evo-session-probe grants --issuer-key issuer.key --world W --frame F --region R \
# --account <HEX32> --accounts <most bots> --count <grants per bot> --lifetime 14400 \
# --out load-grants.bin
# The issuer key never reaches the machine running the drill. Requires kubectl access to
# evolution-dev (patch Deployment/evo-region, read pods and logs), `evo-load` (EVO_LOAD,
# default: on PATH) and python3.
#
# Steps: refuse unless the session window outlasts the drill; enable synthetic inputs on
# evo-region (the drill-time override of ADR-0073, never committed) and wait for the new
# pod to be Ready and for a calm `m10 load-report` (the server prints one every 10 s; the
# governor acts on a full 40-tick window); run evo-load through the public route; take the
# drill pod's whole log with timestamps; then restore REGION_INPUTS to the Git value (also
# on failure) and wait for a Ready pod other than the drill pod, so Argo returns to Synced. The report matches each step's hold
# (evo-load's hold_utc) with the server's reports in it and prints, per step, the client
# verdict and the server's worst tick work, session work, governor state and tick debt.
# Stage and World0 are never touched: the namespace is fixed.
set -euo pipefail
usage() {
echo "usage: $0 <tokens-dir> [--context <name>] [--steps N[,N]...] [--hold S] [--warmup S]" \
"[--grants-per-bot C] [--connect HOST:PORT] [--server-name NAME]" >&2
exit 2
}
[ $# -ge 1 ] || usage
tokens="$1"; shift
k=(kubectl --request-timeout=30s)
steps=20,24,28,32 hold=60 warmup=10 per_bot=4
connect=void.aeternalisgames.com:7400 server_name=void.aeternalisgames.com
while [ $# -gt 0 ]; do
case "$1" in
--context) [ -n "${2:-}" ] || usage; k+=(--context "$2"); shift 2 ;;
--steps) [ -n "${2:-}" ] || usage; steps="$2"; shift 2 ;;
--hold) [ -n "${2:-}" ] || usage; hold="$2"; shift 2 ;;
--warmup) [ -n "${2:-}" ] || usage; warmup="$2"; shift 2 ;;
--grants-per-bot) [ -n "${2:-}" ] || usage; per_bot="$2"; shift 2 ;;
--connect) [ -n "${2:-}" ] || usage; connect="$2"; shift 2 ;;
--server-name) [ -n "${2:-}" ] || usage; server_name="$2"; shift 2 ;;
*) usage ;;
esac
done
ns=evolution-dev # Fixed: qualification runs on the dev reference worker only.
tool="${EVO_LOAD:-evo-load}"
py="$(command -v python3 || command -v python || true)"
for f in dev-ca.pem load-grants.bin; do
[ -f "$tokens/$f" ] || { echo "missing $tokens/$f" >&2; exit 2; }
done
command -v "$tool" > /dev/null || [ -x "$tool" ] || { echo "evo-load not found (set EVO_LOAD)" >&2; exit 2; }
[ -n "$py" ] || { echo "python3 is required" >&2; exit 2; }
selector=app.kubernetes.io/name=evo-region
work="$(mktemp -d)"
chmod 700 "$work"
drill_pod=""
say() { echo "$(date -u +%H:%M:%S) $*"; }
fail() { say "FAIL $*"; exit 1; }
env_of() {
"${k[@]}" get deployment -n "$ns" evo-region \
-o jsonpath="{.spec.template.spec.containers[0].env[?(@.name==\"$1\")].value}"
}
ready_pod() { # Waits up to $2 seconds for a Ready pod other than $1.
local old="$1" limit="${2:-240}"
for _ in $(seq 1 $((limit * 5))); do
for p in $("${k[@]}" get pods -n "$ns" -l "$selector" \
-o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}'); do
[ "$p" = "$old" ] && continue
[ "$("${k[@]}" get pod -n "$ns" "$p" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2> /dev/null)" = True ] &&
{ echo "$p"; return 0; }
done
sleep 0.2
done
return 1
}
restore() {
set +e
say "restoring the Git configuration of evo-region (REGION_INPUTS empty)"
"${k[@]}" set env -n "$ns" deployment/evo-region REGION_INPUTS= > /dev/null
# Ready means a pod other than the drill's, i.e. the restored configuration.
ready_pod "${drill_pod:-none}" 240 > /dev/null && say "restored: evo-region Ready" ||
say "WARN evo-region not Ready after restore"
[ "$("${k[@]}" get deployment -n "$ns" evo-region \
-o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="REGION_INPUTS")].value}')" = "" ] ||
say "WARN REGION_INPUTS is not empty after restore"
say "drill files kept in $work"
}
world="$(env_of REGION_WORLD)" region="$(env_of REGION_ID)" frame="$(env_of REGION_FRAME)"
window="$(env_of REGION_SESSION_WINDOW)"
[ -n "$world" ] && [ -n "$region" ] && [ -n "$frame" ] || fail "cannot read the region identity"
[ -z "$(env_of REGION_INPUTS)" ] || fail "REGION_INPUTS is already set: another drill may be running"
# The drill must end before the session window closes.
count=$(echo "$steps" | tr ',' '\n' | wc -l)
most=$(echo "$steps" | tr ',' '\n' | sort -n | tail -1)
needed=$(( count * (hold + warmup + most / 10 + 5) + 600 ))
expiry="${window##*:}"
[ -n "$expiry" ] && [ "$expiry" -gt $(( $(date -u +%s) + needed )) ] 2> /dev/null ||
fail "the session window ($window) closes within the drill's ${needed} s: renew it first"
size=$(wc -c < "$tokens/load-grants.bin")
[ $(( size / 160 )) -ge $(( most * per_bot )) ] ||
fail "load-grants.bin holds $(( size / 160 )) grants; $most bots x $per_bot need $(( most * per_bot ))"
image="$("${k[@]}" get deployment -n "$ns" evo-region -o jsonpath='{.spec.template.spec.containers[0].image}')"
say "drill: world $world region $region image $image steps $steps hold ${hold}s"
trap restore EXIT
before="$("${k[@]}" get pods -n "$ns" -l "$selector" -o jsonpath='{.items[0].metadata.name}')"
say "enabling synthetic inputs in $ns (drill-time override, restored on exit)"
"${k[@]}" set env -n "$ns" deployment/evo-region REGION_INPUTS=development-v1 > /dev/null
pod="$(ready_pod "$before" 240)" || fail "no Ready evo-region pod with synthetic inputs"
drill_pod="$pod"
# Warm up: two consecutive calm reports (refusing=0 shed=0, about 10 s apart), so the
# governor's window no longer holds the restart's first ticks when the bots arrive (the
# second reference run met one calm report, then a refusal at the first login).
calm=""
for _ in $(seq 1 60); do
sleep 3
last2="$("${k[@]}" logs -n "$ns" --since=25s "$pod" 2> /dev/null | grep "m10 load-report" | tail -2)"
last="$(echo "$last2" | tail -1)"
[ "$(echo "$last2" | grep -c "refusing=0 shed=0")" = 2 ] && { calm=1; break; }
done
[ -n "$calm" ] || fail "the region did not report calm twice within 3 minutes of starting: $last"
say "region calm: $last"
say "running evo-load against $connect"
set +e
"$tool" --connect "$connect" --server-name "$server_name" --ca "$tokens/dev-ca.pem" \
--grants "$tokens/load-grants.bin" --grants-per-bot "$per_bot" --world "$world" \
--frame "$frame" --region "$region" --steps "$steps" --hold "$hold" --warmup "$warmup" \
--threads 8 --per-client > "$work/load.log" 2>&1
load_exit=$?
set -e
sleep 12 # One more server report after the last hold.
"${k[@]}" logs -n "$ns" --timestamps "$pod" > "$work/server.log" 2> /dev/null ||
say "WARN could not read the drill pod's log"
grep -E "^load (step|response_ms|outcome_ms|snapshots|starved|verdict|result)" "$work/load.log" || true
[ "$load_exit" = 0 ] || say "WARN evo-load exited $load_exit (see $work/load.log)"
"$py" - "$work/load.log" "$work/server.log" <<'PY'
import re, sys
from datetime import datetime
load, server = sys.argv[1], sys.argv[2]
reports = []
for line in open(server, encoding='utf-8', errors='replace'):
m = re.match(r'(\S+) m10 load-report (.*)', line.strip())
if not m:
continue
stamp = m.group(1)[:19]
t = datetime.strptime(stamp, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=None)
epoch = int((t - datetime(1970, 1, 1)).total_seconds())
fields = dict(kv.split('=') for kv in m.group(2).split())
reports.append((epoch, {k: int(v) for k, v in fields.items()}))
print(f"drill server reports={len(reports)}")
for line in open(load, encoding='utf-8', errors='replace'):
m = re.match(r'load step N=(\d+) .*hold_utc=(\d+)\.\.(\d+)', line)
if not m:
continue
n, t0, t1 = int(m.group(1)), int(m.group(2)), int(m.group(3))
# A report covers the 2 s before it: only reports wholly inside the hold count.
inside = [r for t, r in reports if t0 + 2 <= t <= t1]
if not inside:
print(f"drill step N={n} server_reports=0")
continue
worst = lambda key: max(r[key] for r in inside)
sessions = max(1, worst('sessions'))
print(f"drill step N={n} server_reports={len(inside)} sessions_max={worst('sessions')} "
f"work_p99_max_ms={worst('p99_us') / 1000:.1f} work_max_ms={worst('max_us') / 1000:.1f} "
f"session_p99_max_ms={worst('session_p99_us') / 1000:.1f} "
f"per_session_ms={worst('session_p99_us') / 1000 / sessions:.3f} "
f"refusing_max={worst('refusing')} shed_max={worst('shed')} "
f"tick_debt_max={worst('tick_debt')} "
f"server_within_budget={int(worst('p99_us') <= 25000 and worst('session_p99_us') <= 500 * sessions and worst('tick_debt') == 0)}")
PY
say "drill complete (evo-load exit $load_exit); restoring"