Files
aevolution/tools/ops/region-drill.sh
T
dingoandClaude Opus 5.5 8ef69fab1d Add M8.4 region host deployment, provisioning hook and drill
Deploy evo-region per environment (ADR-0071): one replica with the
Recreate strategy (a rolling update would deadlock on the lease),
/readyz readiness, /livez liveness, no CPU limit and a drain inside the
grace period, with the image digest doubling as the build identity. A
new idempotent evo-world-clone provision subcommand creates genesis from
development_opening (byte-identical to the qualification fixture) as a
Sync hook after the migration. Dev and stage paths include their region;
a dev AppProject allows Deployment and Service; region-drill.sh times
rollout and forced-deletion recovery; the policy check enforces the
host's invariants.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-29 21:32:56 -04:00

80 lines
3.7 KiB
Bash

#!/usr/bin/env bash
# M8.4 region host drill (ADR-0071) for a disposable environment only.
#
# tools/ops/region-drill.sh <dev|stage> [--context <name>]
#
# 1. Reads the ready region's authority from /metrics (epoch, committed tick).
# 2. Graceful rollout: `kubectl rollout restart`. With the Recreate strategy the old
# pod drains (commits, releases the lease) before the new one starts; measures the
# time until the new pod is Ready and checks it resumed at the committed tick with
# a newer epoch.
# 3. Forced worker deletion: deletes the pod with --grace-period=0 --force; the
# replacement must wait out the dead holder's lease (up to 20 s) and recover; the
# time to Ready is measured the same way.
# Refuses anything but dev or stage before touching the cluster; World0 is never
# drilled from here. Prints one line per measurement; exits nonzero on any failure.
set -euo pipefail
usage() { echo "usage: $0 <dev|stage> [--context <name>]" >&2; exit 2; }
[ $# -ge 1 ] || usage
env="$1"; shift
case "$env" in
dev|stage) ;;
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
exit 2 ;;
esac
k=(kubectl --request-timeout=30s)
[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2")
ns="evolution-$env"
selector=app.kubernetes.io/name=evo-region
pod_name() { "${k[@]}" get pods -n "$ns" -l "$selector" \
-o jsonpath='{range .items[?(@.metadata.deletionTimestamp == "")]}{.metadata.name}{"\n"}{end}' |
head -1; }
ready_pod() { # Waits (up to 120 s) for a Ready pod other than $1; prints its name.
local old="$1"
for _ in $(seq 1 600); do
for p in $("${k[@]}" get pods -n "$ns" -l "$selector" -o name 2>/dev/null); do
p="${p#pod/}"
[ "$p" = "$old" ] && continue
ready="$("${k[@]}" get pod -n "$ns" "$p" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)"
[ "$ready" = True ] && { echo "$p"; return 0; }
done
sleep 0.2
done
return 1
}
metric() { # metric <pod> <series>: one value from the pod's own /metrics (bash /dev/tcp).
"${k[@]}" exec -n "$ns" "$1" -- /bin/bash -c \
'exec 3<>/dev/tcp/127.0.0.1/9100; printf "GET /metrics HTTP/1.1\r\nHost: drill\r\n\r\n" >&3; cat <&3' |
awk -v s="$2" '$1 == s {print $2}'
}
now_ms() { date +%s%3N; }
first="$(pod_name)"
[ -n "$first" ] || { echo "FAIL no evo-region pod in $ns" >&2; exit 1; }
first="$(ready_pod "")" || { echo "FAIL no Ready evo-region pod in $ns" >&2; exit 1; }
epoch0="$(metric "$first" evo_region_lease_epoch)"
tick0="$(metric "$first" evo_region_committed_tick)"
echo "start pod=$first epoch=$epoch0 committed_tick=$tick0"
start="$(now_ms)"
"${k[@]}" rollout restart -n "$ns" deployment/evo-region > /dev/null
second="$(ready_pod "$first")" || { echo "FAIL rollout: no Ready replacement" >&2; exit 1; }
rollout_ms=$(( $(now_ms) - start ))
epoch1="$(metric "$second" evo_region_lease_epoch)"
tick1="$(metric "$second" evo_region_committed_tick)"
echo "rollout ready_ms=$rollout_ms pod=$second epoch=$epoch1 committed_tick=$tick1"
[ "$epoch1" -gt "$epoch0" ] && [ "$tick1" -ge "$tick0" ] ||
{ echo "FAIL rollout did not resume with a newer epoch at the committed tick" >&2; exit 1; }
start="$(now_ms)"
"${k[@]}" delete pod -n "$ns" "$second" --grace-period=0 --force > /dev/null 2>&1
third="$(ready_pod "$second")" || { echo "FAIL forced deletion: no Ready replacement" >&2; exit 1; }
forced_ms=$(( $(now_ms) - start ))
epoch2="$(metric "$third" evo_region_lease_epoch)"
tick2="$(metric "$third" evo_region_committed_tick)"
echo "forced ready_ms=$forced_ms pod=$third epoch=$epoch2 committed_tick=$tick2"
[ "$epoch2" -gt "$epoch1" ] && [ "$tick2" -ge "$tick1" ] ||
{ echo "FAIL forced deletion did not resume with a newer epoch at the committed tick" >&2; exit 1; }
echo "PASS region drill in $ns"