Files
aevolution/tools/ops/fault-drill.sh
T
dingoandClaude Opus 5.5 6b5da397e8 fix(ops): list live region pods with a go-template in fault-drill
kubectl jsonpath has no negation, so the deletionTimestamp filter was a
parse error on every call. Use a go-template that skips pods already
being deleted; verified against sample pods with Go's text/template,
which kubectl's go-template output uses.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 14:37:17 -04:00

175 lines
9.0 KiB
Bash

#!/usr/bin/env bash
# M8.6 fault drill (ADR-0073) for a disposable environment only.
#
# tools/ops/fault-drill.sh <dev|stage> [kills|full|empty|db-outage|all] [--context <name>]
#
# Puts the region under synthetic load (REGION_INPUTS=development-v1), then runs:
# kills three forced pod deletions under load: time to Ready (RTO) and zero
# lost acknowledgements (every tick a dying pod acknowledged is at or
# below the committed head its successor recovers);
# full the object volume "fills" (REGION_RESERVE_MIB beyond any disk): the
# host keeps simulating but acknowledges nothing; restoring the reserve
# resumes from the committed head with nothing lost;
# empty the object volume is replaced by an empty one: the host must
# quarantine, never become ready and never start a fresh world;
# restoring the volume recovers the same committed head;
# db-outage interactive: the operator blocks the database for longer than the
# lease (e.g. drop port 5432 from the store node on the database VM);
# the host must stop being ready and acknowledge nothing, then after
# the block is lifted re-acquire under a newer epoch with nothing lost.
# `all` runs kills, full and empty (db-outage needs an operator). Every change is
# made with `kubectl set env` / `kubectl patch` on the evo-region Deployment and
# reverted on exit, including on failure, back to the Git values (so Argo returns to
# Synced); World0 is refused before anything touches the cluster.
set -euo pipefail
usage() { echo "usage: $0 <dev|stage> [kills|full|empty|db-outage|all] [--context <name>]" >&2; exit 2; }
[ $# -ge 1 ] || usage
env="$1"; shift
case "$env" in
dev|stage) ;;
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
exit 2 ;;
esac
scenario="${1:-all}"
case "$scenario" in kills|full|empty|db-outage|all) shift || true ;; --context) scenario=all ;; *) usage ;; esac
k=(kubectl --request-timeout=30s)
[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2")
ns="evolution-$env"
selector=app.kubernetes.io/name=evo-region
work="$(mktemp -d)"
log_pids=()
now_ms() { date +%s%3N; }
say() { echo "$(date -u +%H:%M:%S) $*"; }
fail() { say "FAIL $*"; exit 1; }
current_pods() { "${k[@]}" get pods -n "$ns" -l "$selector" \
-o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}'; }
# (kubectl jsonpath has no negation; the go-template skips pods already being deleted.)
ready_pod() { # Waits up to $2 seconds (default 120) for a Ready pod other than $1.
local old="$1" limit="${2:-120}"
for _ in $(seq 1 $((limit * 5))); do
for p in $(current_pods); do
[ "$p" = "$old" ] && continue
[ "$("${k[@]}" get pod -n "$ns" "$p" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null)" = True ] &&
{ echo "$p"; return 0; }
done
sleep 0.2
done
return 1
}
metric() { # metric <pod> <series>: from the pod's own /metrics (bash /dev/tcp).
"${k[@]}" exec -n "$ns" "$1" -- /bin/bash -c \
'exec 3<>/dev/tcp/127.0.0.1/9100; printf "GET /metrics HTTP/1.1\r\nHost: drill\r\n\r\n" >&3; cat <&3' 2>/dev/null |
awk -v s="$2" '$1 == s {print $2}'
}
capture() { # Streams a pod's log to $work/<pod>.log for acknowledgement accounting.
"${k[@]}" logs -n "$ns" -f "$1" > "$work/$1.log" 2>/dev/null &
log_pids+=($!)
}
max_ack() { awk '$1 == "ack" {if ($3 > m) m = $3} END {print m + 0}' "$work/$1.log"; }
recovered_at() { # committed tick from the pod's "phase running epoch=E committed=C" line.
sed -n 's/^phase running epoch=\([0-9]*\) committed=\([0-9]*\) .*/\1 \2/p' "$work/$1.log" | tail -1
}
wait_for_line() { # wait_for_line <pod> <regex> <seconds>
for _ in $(seq 1 $(($3 * 5))); do grep -qE "$2" "$work/$1.log" && return 0; sleep 0.2; done
return 1
}
set_env() { "${k[@]}" set env -n "$ns" deployment/evo-region "$@" > /dev/null; }
restore() {
set +e
say "restoring the Git configuration of evo-region"
set_env REGION_INPUTS= REGION_RESERVE_MIB=1024
"${k[@]}" get deployment -n "$ns" evo-region -o jsonpath='{.spec.template.spec.volumes[0].emptyDir}' |
grep -q . && "${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \
'[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","persistentVolumeClaim":{"claimName":"aevolution-objects"}}}]' > /dev/null
for pid in "${log_pids[@]}"; do kill "$pid" 2>/dev/null; done
ready_pod "" 120 > /dev/null && say "restored: evo-region Ready" || say "WARN evo-region not Ready after restore"
say "logs kept in $work"
}
trap restore EXIT
POD=""
loaded_pod() { # Sets POD to a Ready pod other than $1 once acknowledgements flow.
POD="$(ready_pod "${1:-}" 120)" || fail "no Ready evo-region pod"
capture "$POD"
wait_for_line "$POD" '^ack ' 60 || fail "$POD acknowledges nothing under load"
}
say "enabling synthetic load in $ns"
before="$(current_pods | head -1)"
set_env REGION_INPUTS=development-v1
loaded_pod "$before"; pod="$POD"
say "loaded pod=$pod ($(recovered_at "$pod"))"
if [ "$scenario" = kills ] || [ "$scenario" = all ]; then
for round in 1 2 3; do
sleep 10
acked="$(max_ack "$pod")"
start="$(now_ms)"
"${k[@]}" delete pod -n "$ns" "$pod" --grace-period=0 --force > /dev/null 2>&1
loaded_pod "$pod"; next="$POD"
rto=$(( $(now_ms) - start ))
read -r epoch committed <<< "$(recovered_at "$next")"
say "kill round=$round rto_ms=$rto dead_acked_max=$acked recovered_committed=$committed epoch=$epoch pod=$next"
[ "$acked" -le "$committed" ] || fail "round $round lost acknowledgements ($acked > $committed)"
pod="$next"
done
fi
if [ "$scenario" = full ] || [ "$scenario" = all ]; then
sleep 5
acked="$(max_ack "$pod")"
set_env REGION_RESERVE_MIB=1048576
full_pod="$(ready_pod "$pod" 120)" || fail "full: host did not recover (recovery needs no space)"
capture "$full_pod"
sleep 30
new_acks="$(awk '$1 == "ack"' "$work/$full_pod.log" | wc -l)"
available="$(metric "$full_pod" evo_region_objects_available_bytes)"
reserve="$(metric "$full_pod" evo_region_objects_reserve_bytes)"
failures="$(metric "$full_pod" evo_region_store_failures_total)"
say "full acks_while_full=$new_acks available=$available reserve=$reserve store_failures=$failures"
[ "$new_acks" -eq 0 ] || fail "full: acknowledged while the volume was full"
set_env REGION_RESERVE_MIB=1024
loaded_pod "$full_pod"; pod="$POD"
read -r epoch committed <<< "$(recovered_at "$pod")"
say "full restored committed=$committed (acked before=$acked) epoch=$epoch pod=$pod"
[ "$acked" -le "$committed" ] || fail "full: lost acknowledgements"
fi
if [ "$scenario" = empty ] || [ "$scenario" = all ]; then
sleep 5
acked="$(max_ack "$pod")"
"${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \
'[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","emptyDir":{"sizeLimit":"64Mi"}}}]' > /dev/null
sleep 45
if ready_pod "$pod" 1 > /dev/null; then fail "empty: a host became Ready on an empty volume"; fi
empty_pod="$(current_pods | grep -v "^$pod$" | head -1)"
quarantined="$("${k[@]}" logs -n "$ns" "$empty_pod" --previous 2>/dev/null | grep -c '^phase quarantined' || true)"
say "empty pod=$empty_pod ready=never quarantined_lines=$quarantined"
"${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \
'[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","persistentVolumeClaim":{"claimName":"aevolution-objects"}}}]' > /dev/null
loaded_pod "$empty_pod"; pod="$POD"
read -r epoch committed <<< "$(recovered_at "$pod")"
say "empty restored committed=$committed (acked before=$acked) epoch=$epoch pod=$pod"
[ "$acked" -le "$committed" ] || fail "empty: the world was reset or acknowledgements lost"
fi
if [ "$scenario" = db-outage ]; then
sleep 5
epoch0="$(metric "$pod" evo_region_lease_epoch)"
say "block the database from the store node for at least 30 s now (lease 20 s), then lift it."
say "waiting for the host to lose readiness..."
for _ in $(seq 1 600); do [ "$(metric "$pod" evo_region_ready)" = 0 ] && break; sleep 0.5; done
[ "$(metric "$pod" evo_region_ready)" = 0 ] || fail "db-outage: host stayed ready"
lost="$(now_ms)"
acked="$(max_ack "$pod")"
say "host not ready; acknowledgements stopped at tick $acked. Waiting for recovery after the block is lifted..."
for _ in $(seq 1 1200); do [ "$(metric "$pod" evo_region_ready)" = 1 ] && break; sleep 0.5; done
[ "$(metric "$pod" evo_region_ready)" = 1 ] || fail "db-outage: host did not recover within 10 minutes"
epoch1="$(metric "$pod" evo_region_lease_epoch)"
committed="$(metric "$pod" evo_region_committed_tick)"
say "db-outage unavailable_ms=$(( $(now_ms) - lost )) epoch=$epoch0->$epoch1 committed=$committed acked_before=$acked fences=$(metric "$pod" evo_region_fences_total)"
[ "$epoch1" -gt "$epoch0" ] && [ "$acked" -le "$committed" ] || fail "db-outage: stale authority resumed or acknowledgements lost"
fi
say "PASS fault drill ($scenario) in $ns"