#!/usr/bin/env bash # M10.9d load drill (ADR-0097) for evolution-dev only: the reference-worker qualification run. # # tools/ops/load-drill.sh [--context ] [--steps N[,N]...] [--hold S] # [--warmup S] [--grants-per-bot C] [--connect HOST:PORT] [--server-name NAME] # # holds dev-ca.pem (the current dev session CA) and load-grants.bin, a bundle # made where the dev issuer key lives: # evo-session-probe grants --issuer-key issuer.key --world W --frame F --region R \ # --account --accounts --count --lifetime 14400 \ # --out load-grants.bin # The issuer key never reaches the machine running the drill. Requires kubectl access to # evolution-dev (patch Deployment/evo-region, read pods and logs), `evo-load` (EVO_LOAD, # default: on PATH) and python3. # # Steps: refuse unless the session window outlasts the drill; enable synthetic inputs on # evo-region (the drill-time override of ADR-0073, never committed) and wait for the new # pod to be Ready and for a calm `m10 load-report` (the server prints one every 10 s; the # governor acts on a full 40-tick window); run evo-load through the public route; take the # drill pod's whole log with timestamps; then restore REGION_INPUTS to the Git value (also # on failure) and wait for a Ready pod other than the drill pod, so Argo returns to Synced. The report matches each step's hold # (evo-load's hold_utc) with the server's reports in it and prints, per step, the client # verdict and the server's worst tick work, session work, governor state and tick debt. # Stage and World0 are never touched: the namespace is fixed. set -euo pipefail usage() { echo "usage: $0 [--context ] [--steps N[,N]...] [--hold S] [--warmup S]" \ "[--grants-per-bot C] [--connect HOST:PORT] [--server-name NAME]" >&2 exit 2 } [ $# -ge 1 ] || usage tokens="$1"; shift k=(kubectl --request-timeout=30s) steps=20,24,28,32 hold=60 warmup=10 per_bot=4 connect=void.aeternalisgames.com:7400 server_name=void.aeternalisgames.com while [ $# -gt 0 ]; do case "$1" in --context) [ -n "${2:-}" ] || usage; k+=(--context "$2"); shift 2 ;; --steps) [ -n "${2:-}" ] || usage; steps="$2"; shift 2 ;; --hold) [ -n "${2:-}" ] || usage; hold="$2"; shift 2 ;; --warmup) [ -n "${2:-}" ] || usage; warmup="$2"; shift 2 ;; --grants-per-bot) [ -n "${2:-}" ] || usage; per_bot="$2"; shift 2 ;; --connect) [ -n "${2:-}" ] || usage; connect="$2"; shift 2 ;; --server-name) [ -n "${2:-}" ] || usage; server_name="$2"; shift 2 ;; *) usage ;; esac done ns=evolution-dev # Fixed: qualification runs on the dev reference worker only. tool="${EVO_LOAD:-evo-load}" py="$(command -v python3 || command -v python || true)" for f in dev-ca.pem load-grants.bin; do [ -f "$tokens/$f" ] || { echo "missing $tokens/$f" >&2; exit 2; } done command -v "$tool" > /dev/null || [ -x "$tool" ] || { echo "evo-load not found (set EVO_LOAD)" >&2; exit 2; } [ -n "$py" ] || { echo "python3 is required" >&2; exit 2; } selector=app.kubernetes.io/name=evo-region work="$(mktemp -d)" chmod 700 "$work" drill_pod="" say() { echo "$(date -u +%H:%M:%S) $*"; } fail() { say "FAIL $*"; exit 1; } env_of() { "${k[@]}" get deployment -n "$ns" evo-region \ -o jsonpath="{.spec.template.spec.containers[0].env[?(@.name==\"$1\")].value}" } ready_pod() { # Waits up to $2 seconds for a Ready pod other than $1. local old="$1" limit="${2:-240}" for _ in $(seq 1 $((limit * 5))); do for p in $("${k[@]}" get pods -n "$ns" -l "$selector" \ -o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}'); do [ "$p" = "$old" ] && continue [ "$("${k[@]}" get pod -n "$ns" "$p" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2> /dev/null)" = True ] && { echo "$p"; return 0; } done sleep 0.2 done return 1 } restore() { set +e say "restoring the Git configuration of evo-region (REGION_INPUTS empty)" "${k[@]}" set env -n "$ns" deployment/evo-region REGION_INPUTS= > /dev/null # Ready means a pod other than the drill's, i.e. the restored configuration. ready_pod "${drill_pod:-none}" 240 > /dev/null && say "restored: evo-region Ready" || say "WARN evo-region not Ready after restore" [ "$("${k[@]}" get deployment -n "$ns" evo-region \ -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="REGION_INPUTS")].value}')" = "" ] || say "WARN REGION_INPUTS is not empty after restore" say "drill files kept in $work" } world="$(env_of REGION_WORLD)" region="$(env_of REGION_ID)" frame="$(env_of REGION_FRAME)" [ -n "$world" ] && [ -n "$region" ] && [ -n "$frame" ] || fail "cannot read the region identity" [ -z "$(env_of REGION_INPUTS)" ] || fail "REGION_INPUTS is already set: another drill may be running" # The drill must end before the session window closes: the window is in SQL (ADR-0087 # amendment), so open it in the admin portal for at least the drill's duration first. count=$(echo "$steps" | tr ',' '\n' | wc -l) most=$(echo "$steps" | tr ',' '\n' | sort -n | tail -1) needed=$(( count * (hold + warmup + most / 10 + 5) + 600 )) say "the drill needs about ${needed} s of open session window (admin portal)" size=$(wc -c < "$tokens/load-grants.bin") [ $(( size / 160 )) -ge $(( most * per_bot )) ] || fail "load-grants.bin holds $(( size / 160 )) grants; $most bots x $per_bot need $(( most * per_bot ))" image="$("${k[@]}" get deployment -n "$ns" evo-region -o jsonpath='{.spec.template.spec.containers[0].image}')" say "drill: world $world region $region image $image steps $steps hold ${hold}s" trap restore EXIT before="$("${k[@]}" get pods -n "$ns" -l "$selector" -o jsonpath='{.items[0].metadata.name}')" say "enabling synthetic inputs in $ns (drill-time override, restored on exit)" "${k[@]}" set env -n "$ns" deployment/evo-region REGION_INPUTS=development-v1 > /dev/null pod="$(ready_pod "$before" 240)" || fail "no Ready evo-region pod with synthetic inputs" drill_pod="$pod" # Warm up: two consecutive calm reports (refusing=0 shed=0, about 10 s apart), so the # governor's window no longer holds the restart's first ticks when the bots arrive (the # second reference run met one calm report, then a refusal at the first login). calm="" for _ in $(seq 1 60); do sleep 3 last2="$("${k[@]}" logs -n "$ns" --since=25s "$pod" 2> /dev/null | grep "m10 load-report" | tail -2)" last="$(echo "$last2" | tail -1)" [ "$(echo "$last2" | grep -c "refusing=0 shed=0")" = 2 ] && { calm=1; break; } done [ -n "$calm" ] || fail "the region did not report calm twice within 3 minutes of starting: $last" say "region calm: $last" say "running evo-load against $connect" set +e "$tool" --connect "$connect" --server-name "$server_name" --ca "$tokens/dev-ca.pem" \ --grants "$tokens/load-grants.bin" --grants-per-bot "$per_bot" --world "$world" \ --frame "$frame" --region "$region" --steps "$steps" --hold "$hold" --warmup "$warmup" \ --threads 8 --ramp-ms 500 --per-client > "$work/load.log" 2>&1 load_exit=$? set -e sleep 12 # One more server report after the last hold. "${k[@]}" logs -n "$ns" --timestamps "$pod" > "$work/server.log" 2> /dev/null || say "WARN could not read the drill pod's log" grep -E "^load (step|response_ms|outcome_ms|snapshots|starved|verdict|result)" "$work/load.log" || true [ "$load_exit" = 0 ] || say "WARN evo-load exited $load_exit (see $work/load.log)" "$py" - "$work/load.log" "$work/server.log" <<'PY' import re, sys from datetime import datetime load, server = sys.argv[1], sys.argv[2] reports = [] for line in open(server, encoding='utf-8', errors='replace'): m = re.match(r'(\S+) m10 load-report (.*)', line.strip()) if not m: continue stamp = m.group(1)[:19] t = datetime.strptime(stamp, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=None) epoch = int((t - datetime(1970, 1, 1)).total_seconds()) fields = dict(kv.split('=') for kv in m.group(2).split()) reports.append((epoch, {k: int(v) for k, v in fields.items()})) print(f"drill server reports={len(reports)}") for line in open(load, encoding='utf-8', errors='replace'): m = re.match(r'load step N=(\d+) .*hold_utc=(\d+)\.\.(\d+)', line) if not m: continue n, t0, t1 = int(m.group(1)), int(m.group(2)), int(m.group(3)) # A report covers the 2 s before it: only reports wholly inside the hold count. inside = [r for t, r in reports if t0 + 2 <= t <= t1] if not inside: print(f"drill step N={n} server_reports=0") continue worst = lambda key: max(r.get(key, 0) for r in inside) mean = lambda key: sum(r.get(key, 0) for r in inside) / len(inside) sessions = max(1, worst('sessions')) # Budgets v2 (schemas/network/m10-budgets-v2.md): tick work p95 <= 25 ms over all ticks, # ticks that commit a batch <= 50 ms; session layer p99 <= 0.5 ms per session; no debt. within = (worst('p95_us') <= 25000 and worst('commit_max_us') <= 50000 and worst('session_p99_us') <= 500 * sessions and worst('tick_debt') == 0) print(f"drill step N={n} server_reports={len(inside)} sessions_max={worst('sessions')} " f"work_p95_max_ms={worst('p95_us') / 1000:.1f} work_p99_max_ms={worst('p99_us') / 1000:.1f} " f"commit_max_ms={worst('commit_max_us') / 1000:.1f} other_max_ms={worst('other_max_us') / 1000:.1f} " f"session_p95_max_ms={worst('session_p95_us') / 1000:.1f} " f"session_p99_max_ms={worst('session_p99_us') / 1000:.1f} " f"per_session_p99_ms={worst('session_p99_us') / 1000 / sessions:.3f} " f"poll_avg_ms={mean('poll_avg_us') / 1000:.2f} replicate_avg_ms={mean('replicate_avg_us') / 1000:.2f} " f"refusing_max={worst('refusing')} shed_max={worst('shed')} " f"tick_debt_max={worst('tick_debt')} server_within_budget={int(within)}") PY say "drill complete (evo-load exit $load_exit); restoring"