#!/usr/bin/env bash # M8.4 region host drill (ADR-0071) for a disposable environment only. # # tools/ops/region-drill.sh [--context ] # # 1. Reads the ready region's authority from /metrics (epoch, committed tick). # 2. Graceful rollout: `kubectl rollout restart`. With the Recreate strategy the old # pod drains (commits, releases the lease) before the new one starts; measures the # time until the new pod is Ready and checks it resumed at the committed tick with # a newer epoch. # 3. Forced worker deletion: deletes the pod with --grace-period=0 --force; the # replacement must wait out the dead holder's lease (up to 20 s) and recover; the # time to Ready is measured the same way. # Refuses anything but dev or stage before touching the cluster; World0 is never # drilled from here. Prints one line per measurement; exits nonzero on any failure. set -euo pipefail usage() { echo "usage: $0 [--context ]" >&2; exit 2; } [ $# -ge 1 ] || usage env="$1"; shift case "$env" in dev|stage) ;; *) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2 exit 2 ;; esac k=(kubectl --request-timeout=30s) [ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2") ns="evolution-$env" selector=app.kubernetes.io/name=evo-region pod_name() { "${k[@]}" get pods -n "$ns" -l "$selector" \ -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | head -1; } ready_pod() { # Waits (up to 120 s) for a Ready pod other than $1; prints its name. local old="$1" for _ in $(seq 1 600); do for p in $("${k[@]}" get pods -n "$ns" -l "$selector" -o name 2>/dev/null); do p="${p#pod/}" [ "$p" = "$old" ] && continue ready="$("${k[@]}" get pod -n "$ns" "$p" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" [ "$ready" = True ] && { echo "$p"; return 0; } done sleep 0.2 done return 1 } metric() { # metric : one value from the pod's own /metrics (bash /dev/tcp). "${k[@]}" exec -n "$ns" "$1" -- /bin/bash -c \ 'exec 3<>/dev/tcp/127.0.0.1/9100; printf "GET /metrics HTTP/1.1\r\nHost: drill\r\n\r\n" >&3; cat <&3' | awk -v s="$2" '$1 == s {print $2}' } now_ms() { date +%s%3N; } first="$(pod_name)" [ -n "$first" ] || { echo "FAIL no evo-region pod in $ns" >&2; exit 1; } first="$(ready_pod "")" || { echo "FAIL no Ready evo-region pod in $ns" >&2; exit 1; } epoch0="$(metric "$first" evo_region_lease_epoch)" tick0="$(metric "$first" evo_region_committed_tick)" echo "start pod=$first epoch=$epoch0 committed_tick=$tick0" start="$(now_ms)" "${k[@]}" rollout restart -n "$ns" deployment/evo-region > /dev/null second="$(ready_pod "$first")" || { echo "FAIL rollout: no Ready replacement" >&2; exit 1; } rollout_ms=$(( $(now_ms) - start )) epoch1="$(metric "$second" evo_region_lease_epoch)" tick1="$(metric "$second" evo_region_committed_tick)" echo "rollout ready_ms=$rollout_ms pod=$second epoch=$epoch1 committed_tick=$tick1" [ "$epoch1" -gt "$epoch0" ] && [ "$tick1" -ge "$tick0" ] || { echo "FAIL rollout did not resume with a newer epoch at the committed tick" >&2; exit 1; } start="$(now_ms)" "${k[@]}" delete pod -n "$ns" "$second" --grace-period=0 --force > /dev/null 2>&1 third="$(ready_pod "$second")" || { echo "FAIL forced deletion: no Ready replacement" >&2; exit 1; } forced_ms=$(( $(now_ms) - start )) epoch2="$(metric "$third" evo_region_lease_epoch)" tick2="$(metric "$third" evo_region_committed_tick)" echo "forced ready_ms=$forced_ms pod=$third epoch=$epoch2 committed_tick=$tick2" [ "$epoch2" -gt "$epoch1" ] && [ "$tick2" -ge "$tick1" ] || { echo "FAIL forced deletion did not resume with a newer epoch at the committed tick" >&2; exit 1; } echo "PASS region drill in $ns"