Foundation / checks (deploy) (push) Successful in 3m24s
Foundation / checks (ecs) (push) Successful in 2m0s
Foundation / checks (client) (push) Successful in 12m24s
Foundation / checks (network) (push) Successful in 13m35s
Foundation / checks (tsan) (push) Successful in 7m58s
Foundation / checks (checks) (push) Successful in 32m32s
Foundation / checks (postgres) (push) Successful in 45m25s
Foundation / postgres-tools (push) Successful in 17s
Foundation / image (push) Successful in 2m32s
Foundation / server-image (push) Successful in 4m15s
ADR-0097. The first evolution-dev load drill (image 6c96629e) did not measure capacity: the governor refused six seconds after the restart with no session (p95 over a window of only startup ticks), and the bots spent their grants retrying every 2 s. Reference numbers with one session: tick work p95 4.0 ms with 33 ms spikes, session layer p95 0.68 ms. - The governor acts on work only over a full 40-tick window; tick debt still counts at once. - evo-load backs off 10 s after a load refusal and, in bundle mode, reuses the grant the region refused before using it. - The drill waits for a calm load-report before the bots start, takes the drill pod's whole log before restoring (the live follow died), and confirms the restore against a pod other than the drill pod. Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
173 lines
8.9 KiB
Bash
173 lines
8.9 KiB
Bash
#!/usr/bin/env bash
|
|
# M10.9d load drill (ADR-0097) for evolution-dev only: the reference-worker qualification run.
|
|
#
|
|
# tools/ops/load-drill.sh <tokens-dir> [--context <name>] [--steps N[,N]...] [--hold S]
|
|
# [--warmup S] [--grants-per-bot C] [--connect HOST:PORT] [--server-name NAME]
|
|
#
|
|
# <tokens-dir> holds dev-ca.pem (the current dev session CA) and load-grants.bin, a bundle
|
|
# made where the dev issuer key lives:
|
|
# evo-session-probe grants --issuer-key issuer.key --world W --frame F --region R \
|
|
# --account <HEX32> --accounts <most bots> --count <grants per bot> --lifetime 14400 \
|
|
# --out load-grants.bin
|
|
# The issuer key never reaches the machine running the drill. Requires kubectl access to
|
|
# evolution-dev (patch Deployment/evo-region, read pods and logs), `evo-load` (EVO_LOAD,
|
|
# default: on PATH) and python3.
|
|
#
|
|
# Steps: refuse unless the session window outlasts the drill; enable synthetic inputs on
|
|
# evo-region (the drill-time override of ADR-0073, never committed) and wait for the new
|
|
# pod to be Ready and for a calm `m10 load-report` (the server prints one every 10 s; the
|
|
# governor acts on a full 40-tick window); run evo-load through the public route; take the
|
|
# drill pod's whole log with timestamps; then restore REGION_INPUTS to the Git value (also
|
|
# on failure) and wait for a Ready pod other than the drill pod, so Argo returns to Synced. The report matches each step's hold
|
|
# (evo-load's hold_utc) with the server's reports in it and prints, per step, the client
|
|
# verdict and the server's worst tick work, session work, governor state and tick debt.
|
|
# Stage and World0 are never touched: the namespace is fixed.
|
|
set -euo pipefail
|
|
usage() {
|
|
echo "usage: $0 <tokens-dir> [--context <name>] [--steps N[,N]...] [--hold S] [--warmup S]" \
|
|
"[--grants-per-bot C] [--connect HOST:PORT] [--server-name NAME]" >&2
|
|
exit 2
|
|
}
|
|
[ $# -ge 1 ] || usage
|
|
tokens="$1"; shift
|
|
k=(kubectl --request-timeout=30s)
|
|
steps=20,24,28,32 hold=60 warmup=10 per_bot=4
|
|
connect=void.aeternalisgames.com:7400 server_name=void.aeternalisgames.com
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--context) [ -n "${2:-}" ] || usage; k+=(--context "$2"); shift 2 ;;
|
|
--steps) [ -n "${2:-}" ] || usage; steps="$2"; shift 2 ;;
|
|
--hold) [ -n "${2:-}" ] || usage; hold="$2"; shift 2 ;;
|
|
--warmup) [ -n "${2:-}" ] || usage; warmup="$2"; shift 2 ;;
|
|
--grants-per-bot) [ -n "${2:-}" ] || usage; per_bot="$2"; shift 2 ;;
|
|
--connect) [ -n "${2:-}" ] || usage; connect="$2"; shift 2 ;;
|
|
--server-name) [ -n "${2:-}" ] || usage; server_name="$2"; shift 2 ;;
|
|
*) usage ;;
|
|
esac
|
|
done
|
|
ns=evolution-dev # Fixed: qualification runs on the dev reference worker only.
|
|
tool="${EVO_LOAD:-evo-load}"
|
|
py="$(command -v python3 || command -v python || true)"
|
|
for f in dev-ca.pem load-grants.bin; do
|
|
[ -f "$tokens/$f" ] || { echo "missing $tokens/$f" >&2; exit 2; }
|
|
done
|
|
command -v "$tool" > /dev/null || [ -x "$tool" ] || { echo "evo-load not found (set EVO_LOAD)" >&2; exit 2; }
|
|
[ -n "$py" ] || { echo "python3 is required" >&2; exit 2; }
|
|
selector=app.kubernetes.io/name=evo-region
|
|
work="$(mktemp -d)"
|
|
chmod 700 "$work"
|
|
drill_pod=""
|
|
say() { echo "$(date -u +%H:%M:%S) $*"; }
|
|
fail() { say "FAIL $*"; exit 1; }
|
|
env_of() {
|
|
"${k[@]}" get deployment -n "$ns" evo-region \
|
|
-o jsonpath="{.spec.template.spec.containers[0].env[?(@.name==\"$1\")].value}"
|
|
}
|
|
ready_pod() { # Waits up to $2 seconds for a Ready pod other than $1.
|
|
local old="$1" limit="${2:-240}"
|
|
for _ in $(seq 1 $((limit * 5))); do
|
|
for p in $("${k[@]}" get pods -n "$ns" -l "$selector" \
|
|
-o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}'); do
|
|
[ "$p" = "$old" ] && continue
|
|
[ "$("${k[@]}" get pod -n "$ns" "$p" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2> /dev/null)" = True ] &&
|
|
{ echo "$p"; return 0; }
|
|
done
|
|
sleep 0.2
|
|
done
|
|
return 1
|
|
}
|
|
restore() {
|
|
set +e
|
|
say "restoring the Git configuration of evo-region (REGION_INPUTS empty)"
|
|
"${k[@]}" set env -n "$ns" deployment/evo-region REGION_INPUTS= > /dev/null
|
|
# Ready means a pod other than the drill's, i.e. the restored configuration.
|
|
ready_pod "${drill_pod:-none}" 240 > /dev/null && say "restored: evo-region Ready" ||
|
|
say "WARN evo-region not Ready after restore"
|
|
[ "$("${k[@]}" get deployment -n "$ns" evo-region \
|
|
-o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="REGION_INPUTS")].value}')" = "" ] ||
|
|
say "WARN REGION_INPUTS is not empty after restore"
|
|
say "drill files kept in $work"
|
|
}
|
|
|
|
world="$(env_of REGION_WORLD)" region="$(env_of REGION_ID)" frame="$(env_of REGION_FRAME)"
|
|
window="$(env_of REGION_SESSION_WINDOW)"
|
|
[ -n "$world" ] && [ -n "$region" ] && [ -n "$frame" ] || fail "cannot read the region identity"
|
|
[ -z "$(env_of REGION_INPUTS)" ] || fail "REGION_INPUTS is already set: another drill may be running"
|
|
# The drill must end before the session window closes.
|
|
count=$(echo "$steps" | tr ',' '\n' | wc -l)
|
|
most=$(echo "$steps" | tr ',' '\n' | sort -n | tail -1)
|
|
needed=$(( count * (hold + warmup + most / 10 + 5) + 600 ))
|
|
expiry="${window##*:}"
|
|
[ -n "$expiry" ] && [ "$expiry" -gt $(( $(date -u +%s) + needed )) ] 2> /dev/null ||
|
|
fail "the session window ($window) closes within the drill's ${needed} s: renew it first"
|
|
size=$(wc -c < "$tokens/load-grants.bin")
|
|
[ $(( size / 160 )) -ge $(( most * per_bot )) ] ||
|
|
fail "load-grants.bin holds $(( size / 160 )) grants; $most bots x $per_bot need $(( most * per_bot ))"
|
|
image="$("${k[@]}" get deployment -n "$ns" evo-region -o jsonpath='{.spec.template.spec.containers[0].image}')"
|
|
say "drill: world $world region $region image $image steps $steps hold ${hold}s"
|
|
|
|
trap restore EXIT
|
|
before="$("${k[@]}" get pods -n "$ns" -l "$selector" -o jsonpath='{.items[0].metadata.name}')"
|
|
say "enabling synthetic inputs in $ns (drill-time override, restored on exit)"
|
|
"${k[@]}" set env -n "$ns" deployment/evo-region REGION_INPUTS=development-v1 > /dev/null
|
|
pod="$(ready_pod "$before" 240)" || fail "no Ready evo-region pod with synthetic inputs"
|
|
drill_pod="$pod"
|
|
# Warm up: wait for a full governor window and a calm report (refusing=0 shed=0).
|
|
calm=""
|
|
for _ in $(seq 1 60); do
|
|
sleep 3
|
|
last="$("${k[@]}" logs -n "$ns" --since=15s "$pod" 2> /dev/null | grep "m10 load-report" | tail -1)"
|
|
case "$last" in *"refusing=0 shed=0"*) calm=1; break ;; esac
|
|
done
|
|
[ -n "$calm" ] || fail "the region did not report calm within 3 minutes of starting: $last"
|
|
say "region calm: $last"
|
|
say "running evo-load against $connect"
|
|
set +e
|
|
"$tool" --connect "$connect" --server-name "$server_name" --ca "$tokens/dev-ca.pem" \
|
|
--grants "$tokens/load-grants.bin" --grants-per-bot "$per_bot" --world "$world" \
|
|
--frame "$frame" --region "$region" --steps "$steps" --hold "$hold" --warmup "$warmup" \
|
|
--threads 8 --per-client > "$work/load.log" 2>&1
|
|
load_exit=$?
|
|
set -e
|
|
sleep 12 # One more server report after the last hold.
|
|
"${k[@]}" logs -n "$ns" --timestamps "$pod" > "$work/server.log" 2> /dev/null ||
|
|
say "WARN could not read the drill pod's log"
|
|
grep -E "^load (step|response_ms|outcome_ms|snapshots|starved|verdict|result)" "$work/load.log" || true
|
|
[ "$load_exit" = 0 ] || say "WARN evo-load exited $load_exit (see $work/load.log)"
|
|
"$py" - "$work/load.log" "$work/server.log" <<'PY'
|
|
import re, sys
|
|
from datetime import datetime
|
|
load, server = sys.argv[1], sys.argv[2]
|
|
reports = []
|
|
for line in open(server, encoding='utf-8', errors='replace'):
|
|
m = re.match(r'(\S+) m10 load-report (.*)', line.strip())
|
|
if not m:
|
|
continue
|
|
stamp = m.group(1)[:19]
|
|
t = datetime.strptime(stamp, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=None)
|
|
epoch = int((t - datetime(1970, 1, 1)).total_seconds())
|
|
fields = dict(kv.split('=') for kv in m.group(2).split())
|
|
reports.append((epoch, {k: int(v) for k, v in fields.items()}))
|
|
print(f"drill server reports={len(reports)}")
|
|
for line in open(load, encoding='utf-8', errors='replace'):
|
|
m = re.match(r'load step N=(\d+) .*hold_utc=(\d+)\.\.(\d+)', line)
|
|
if not m:
|
|
continue
|
|
n, t0, t1 = int(m.group(1)), int(m.group(2)), int(m.group(3))
|
|
# A report covers the 2 s before it: only reports wholly inside the hold count.
|
|
inside = [r for t, r in reports if t0 + 2 <= t <= t1]
|
|
if not inside:
|
|
print(f"drill step N={n} server_reports=0")
|
|
continue
|
|
worst = lambda key: max(r[key] for r in inside)
|
|
sessions = max(1, worst('sessions'))
|
|
print(f"drill step N={n} server_reports={len(inside)} sessions_max={worst('sessions')} "
|
|
f"work_p99_max_ms={worst('p99_us') / 1000:.1f} work_max_ms={worst('max_us') / 1000:.1f} "
|
|
f"session_p99_max_ms={worst('session_p99_us') / 1000:.1f} "
|
|
f"per_session_ms={worst('session_p99_us') / 1000 / sessions:.3f} "
|
|
f"refusing_max={worst('refusing')} shed_max={worst('shed')} "
|
|
f"tick_debt_max={worst('tick_debt')} "
|
|
f"server_within_budget={int(worst('p99_us') <= 25000 and worst('session_p99_us') <= 500 * sessions and worst('tick_debt') == 0)}")
|
|
PY
|
|
say "drill complete (evo-load exit $load_exit); restoring"
|