Deploy evo-region per environment (ADR-0071): one replica with the Recreate strategy (a rolling update would deadlock on the lease), /readyz readiness, /livez liveness, no CPU limit and a drain inside the grace period, with the image digest doubling as the build identity. A new idempotent evo-world-clone provision subcommand creates genesis from development_opening (byte-identical to the qualification fixture) as a Sync hook after the migration. Dev and stage paths include their region; a dev AppProject allows Deployment and Service; region-drill.sh times rollout and forced-deletion recovery; the policy check enforces the host's invariants. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
80 lines
3.7 KiB
Bash
80 lines
3.7 KiB
Bash
#!/usr/bin/env bash
|
|
# M8.4 region host drill (ADR-0071) for a disposable environment only.
|
|
#
|
|
# tools/ops/region-drill.sh <dev|stage> [--context <name>]
|
|
#
|
|
# 1. Reads the ready region's authority from /metrics (epoch, committed tick).
|
|
# 2. Graceful rollout: `kubectl rollout restart`. With the Recreate strategy the old
|
|
# pod drains (commits, releases the lease) before the new one starts; measures the
|
|
# time until the new pod is Ready and checks it resumed at the committed tick with
|
|
# a newer epoch.
|
|
# 3. Forced worker deletion: deletes the pod with --grace-period=0 --force; the
|
|
# replacement must wait out the dead holder's lease (up to 20 s) and recover; the
|
|
# time to Ready is measured the same way.
|
|
# Refuses anything but dev or stage before touching the cluster; World0 is never
|
|
# drilled from here. Prints one line per measurement; exits nonzero on any failure.
|
|
set -euo pipefail
|
|
usage() { echo "usage: $0 <dev|stage> [--context <name>]" >&2; exit 2; }
|
|
[ $# -ge 1 ] || usage
|
|
env="$1"; shift
|
|
case "$env" in
|
|
dev|stage) ;;
|
|
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
|
|
exit 2 ;;
|
|
esac
|
|
k=(kubectl --request-timeout=30s)
|
|
[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2")
|
|
ns="evolution-$env"
|
|
selector=app.kubernetes.io/name=evo-region
|
|
|
|
pod_name() { "${k[@]}" get pods -n "$ns" -l "$selector" \
|
|
-o jsonpath='{range .items[?(@.metadata.deletionTimestamp == "")]}{.metadata.name}{"\n"}{end}' |
|
|
head -1; }
|
|
ready_pod() { # Waits (up to 120 s) for a Ready pod other than $1; prints its name.
|
|
local old="$1"
|
|
for _ in $(seq 1 600); do
|
|
for p in $("${k[@]}" get pods -n "$ns" -l "$selector" -o name 2>/dev/null); do
|
|
p="${p#pod/}"
|
|
[ "$p" = "$old" ] && continue
|
|
ready="$("${k[@]}" get pod -n "$ns" "$p" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)"
|
|
[ "$ready" = True ] && { echo "$p"; return 0; }
|
|
done
|
|
sleep 0.2
|
|
done
|
|
return 1
|
|
}
|
|
metric() { # metric <pod> <series>: one value from the pod's own /metrics (bash /dev/tcp).
|
|
"${k[@]}" exec -n "$ns" "$1" -- /bin/bash -c \
|
|
'exec 3<>/dev/tcp/127.0.0.1/9100; printf "GET /metrics HTTP/1.1\r\nHost: drill\r\n\r\n" >&3; cat <&3' |
|
|
awk -v s="$2" '$1 == s {print $2}'
|
|
}
|
|
now_ms() { date +%s%3N; }
|
|
|
|
first="$(pod_name)"
|
|
[ -n "$first" ] || { echo "FAIL no evo-region pod in $ns" >&2; exit 1; }
|
|
first="$(ready_pod "")" || { echo "FAIL no Ready evo-region pod in $ns" >&2; exit 1; }
|
|
epoch0="$(metric "$first" evo_region_lease_epoch)"
|
|
tick0="$(metric "$first" evo_region_committed_tick)"
|
|
echo "start pod=$first epoch=$epoch0 committed_tick=$tick0"
|
|
|
|
start="$(now_ms)"
|
|
"${k[@]}" rollout restart -n "$ns" deployment/evo-region > /dev/null
|
|
second="$(ready_pod "$first")" || { echo "FAIL rollout: no Ready replacement" >&2; exit 1; }
|
|
rollout_ms=$(( $(now_ms) - start ))
|
|
epoch1="$(metric "$second" evo_region_lease_epoch)"
|
|
tick1="$(metric "$second" evo_region_committed_tick)"
|
|
echo "rollout ready_ms=$rollout_ms pod=$second epoch=$epoch1 committed_tick=$tick1"
|
|
[ "$epoch1" -gt "$epoch0" ] && [ "$tick1" -ge "$tick0" ] ||
|
|
{ echo "FAIL rollout did not resume with a newer epoch at the committed tick" >&2; exit 1; }
|
|
|
|
start="$(now_ms)"
|
|
"${k[@]}" delete pod -n "$ns" "$second" --grace-period=0 --force > /dev/null 2>&1
|
|
third="$(ready_pod "$second")" || { echo "FAIL forced deletion: no Ready replacement" >&2; exit 1; }
|
|
forced_ms=$(( $(now_ms) - start ))
|
|
epoch2="$(metric "$third" evo_region_lease_epoch)"
|
|
tick2="$(metric "$third" evo_region_committed_tick)"
|
|
echo "forced ready_ms=$forced_ms pod=$third epoch=$epoch2 committed_tick=$tick2"
|
|
[ "$epoch2" -gt "$epoch1" ] && [ "$tick2" -ge "$tick1" ] ||
|
|
{ echo "FAIL forced deletion did not resume with a newer epoch at the committed tick" >&2; exit 1; }
|
|
echo "PASS region drill in $ns"
|