Files
dingoandClaude Opus 5.5 3a34ae39bc fix: M8 review findings (restore ownership, barrier tool, World0 claimRef, CI selection, drill verdicts)
P1: restore-world.sh claims its objects path (mkdir) and the database (a held
session with a restore try-lock and the exclusive upgrade lock), and removes only
what it created, so a losing restore never drops a winner's schema.
P2: barrier Ready-pod selection via shared jsonpath helper with fixtures; barrier
pod fsGroup; script-generated pods inspected by the policy check; World0 PV claim
references protected; ci-paths.sh uses the event's real base and runs on
uncertainty; fault-drill asserts RTO and needs positive quarantine evidence.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 22:53:11 -04:00

216 lines
12 KiB
Bash

#!/usr/bin/env bash
# M8.6 fault drill (ADR-0073) for a disposable environment only.
#
# tools/ops/fault-drill.sh <dev|stage> [kills|full|empty|db-outage|all] [--context <name>]
#
# Puts the region under synthetic load (REGION_INPUTS=development-v1), then runs:
# kills three forced pod deletions under load: time to Ready (RTO) and zero
# lost acknowledgements (every tick a dying pod acknowledged is at or
# below the committed head its successor recovers);
# full the object volume "fills" (REGION_RESERVE_MIB beyond any disk): the
# host keeps simulating but acknowledges nothing; restoring the reserve
# resumes from the committed head with nothing lost;
# empty the object volume is replaced by an empty one: the host must
# quarantine, never become ready and never start a fresh world;
# restoring the volume recovers the same committed head;
# db-outage interactive: the operator blocks the database for longer than the
# lease (e.g. drop port 5432 from the store node on the database VM);
# the host must stop being ready and acknowledge nothing, then after
# the block is lifted re-acquire under a newer epoch with nothing lost.
# `all` runs kills, full and empty (db-outage needs an operator). Every change is
# made with `kubectl set env` / `kubectl patch` on the evo-region Deployment and
# reverted on exit, including on failure, back to the Git values (so Argo returns to
# Synced); World0 is refused before anything touches the cluster.
#
# Verdicts (tools/ops/lib/drill.sh): each kill must recover within RTO_BUDGET_MS
# (default 60000, ADR-0075); the empty scenario needs positive quarantine evidence.
# A missing observation fails the drill. Limits of log-based evidence: an owner's
# acknowledgements are read from its streamed log, finalized after it is gone; a line
# written but never streamed before a forced kill is not observed, so the
# zero-lost-acknowledgement check covers every observed acknowledgement only.
set -euo pipefail
usage() { echo "usage: $0 <dev|stage> [kills|full|empty|db-outage|all] [--context <name>]" >&2; exit 2; }
[ $# -ge 1 ] || usage
env="$1"; shift
case "$env" in
dev|stage) ;;
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
exit 2 ;;
esac
scenario="${1:-all}"
case "$scenario" in kills|full|empty|db-outage|all) shift || true ;; --context) scenario=all ;; *) usage ;; esac
k=(kubectl --request-timeout=30s)
[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2")
ns="evolution-$env"
rto_budget="${RTO_BUDGET_MS:-60000}"
# shellcheck source=lib/drill.sh
. "$(cd "$(dirname "$0")" && pwd)/lib/drill.sh"
selector=app.kubernetes.io/name=evo-region
work="$(mktemp -d)"
log_pids=()
declare -A capture_pid=()
now_ms() { date +%s%3N; }
say() { echo "$(date -u +%H:%M:%S) $*"; }
fail() { say "FAIL $*"; exit 1; }
current_pods() { "${k[@]}" get pods -n "$ns" -l "$selector" \
-o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}'; }
# (kubectl jsonpath has no negation; the go-template skips pods already being deleted.)
ready_pod() { # Waits up to $2 seconds (default 120) for a Ready pod other than $1.
local old="$1" limit="${2:-120}"
for _ in $(seq 1 $((limit * 5))); do
for p in $(current_pods); do
[ "$p" = "$old" ] && continue
[ "$("${k[@]}" get pod -n "$ns" "$p" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null)" = True ] &&
{ echo "$p"; return 0; }
done
sleep 0.2
done
return 1
}
metric() { # metric <pod> <series>: from the pod's own /metrics (bash /dev/tcp).
"${k[@]}" exec -n "$ns" "$1" -- /bin/bash -c \
'exec 3<>/dev/tcp/127.0.0.1/9100; printf "GET /metrics HTTP/1.1\r\nHost: drill\r\n\r\n" >&3; cat <&3' 2>/dev/null |
awk -v s="$2" '$1 == s {print $2}'
}
capture() { # Streams a pod's log to $work/<pod>.log for acknowledgement accounting.
"${k[@]}" logs -n "$ns" -f "$1" > "$work/$1.log" 2>/dev/null &
log_pids+=($!)
capture_pid[$1]=$!
}
finalize() { # Waits (up to 15 s) for a gone pod's log stream to end, so its log is complete.
local pid="${capture_pid[$1]:-}"
[ -n "$pid" ] || return 0
for _ in $(seq 1 75); do kill -0 "$pid" 2> /dev/null || return 0; sleep 0.2; done
kill "$pid" 2> /dev/null || true
}
max_ack() { awk '$1 == "ack" {if ($3 > m) m = $3} END {print m + 0}' "$work/$1.log"; }
recovered_at() { # committed tick from the pod's "phase running epoch=E committed=C" line.
sed -n 's/^phase running epoch=\([0-9]*\) committed=\([0-9]*\) .*/\1 \2/p' "$work/$1.log" | tail -1
}
wait_for_line() { # wait_for_line <pod> <regex> <seconds>
for _ in $(seq 1 $(($3 * 5))); do grep -qE "$2" "$work/$1.log" && return 0; sleep 0.2; done
return 1
}
set_env() { "${k[@]}" set env -n "$ns" deployment/evo-region "$@" > /dev/null; }
restore() {
set +e
say "restoring the Git configuration of evo-region"
set_env REGION_INPUTS= REGION_RESERVE_MIB=1024
"${k[@]}" get deployment -n "$ns" evo-region -o jsonpath='{.spec.template.spec.volumes[0].emptyDir}' |
grep -q . && "${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \
'[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","persistentVolumeClaim":{"claimName":"aevolution-objects"}}}]' > /dev/null
for pid in "${log_pids[@]}"; do kill "$pid" 2>/dev/null; done
ready_pod "" 120 > /dev/null && say "restored: evo-region Ready" || say "WARN evo-region not Ready after restore"
say "logs kept in $work"
}
trap restore EXIT
POD=""
loaded_pod() { # Sets POD to a Ready pod other than $1 once acknowledgements flow.
POD="$(ready_pod "${1:-}" 120)" || fail "no Ready evo-region pod"
capture "$POD"
wait_for_line "$POD" '^ack ' 60 || fail "$POD acknowledges nothing under load"
}
say "enabling synthetic load in $ns"
before="$(current_pods | head -1)"
set_env REGION_INPUTS=development-v1
loaded_pod "$before"; pod="$POD"
say "loaded pod=$pod ($(recovered_at "$pod"))"
if [ "$scenario" = kills ] || [ "$scenario" = all ]; then
for round in 1 2 3; do
sleep 10
start="$(now_ms)"
"${k[@]}" delete pod -n "$ns" "$pod" --grace-period=0 --force > /dev/null 2>&1
POD="$(ready_pod "$pod" 120)" || fail "round $round: no Ready successor within 120 s"
rto=$(( $(now_ms) - start ))
next="$POD"
capture "$next"
wait_for_line "$next" '^ack ' 60 || fail "$next acknowledges nothing under load"
finalize "$pod"
acked="$(max_ack "$pod")" # Everything the dead owner was observed to acknowledge.
read -r epoch committed <<< "$(recovered_at "$next")"
say "kill round=$round rto_ms=$rto budget_ms=$rto_budget dead_acked_max=$acked recovered_committed=$committed epoch=$epoch pod=$next"
verdict="$(rto_verdict "$rto" "$rto_budget")"
[ "$verdict" = PASS ] || fail "round $round: $verdict"
verdict="$(acks_verdict "$acked" "$committed")"
[ "$verdict" = PASS ] || fail "round $round lost acknowledgements: $verdict"
pod="$next"
done
fi
if [ "$scenario" = full ] || [ "$scenario" = all ]; then
sleep 5
set_env REGION_RESERVE_MIB=1048576
full_pod="$(ready_pod "$pod" 120)" || fail "full: host did not recover (recovery needs no space)"
capture "$full_pod"
finalize "$pod"; acked="$(max_ack "$pod")" # The drained owner's complete log.
sleep 30
new_acks="$(awk '$1 == "ack"' "$work/$full_pod.log" | wc -l)"
available="$(metric "$full_pod" evo_region_objects_available_bytes)"
reserve="$(metric "$full_pod" evo_region_objects_reserve_bytes)"
failures="$(metric "$full_pod" evo_region_store_failures_total)"
say "full acks_while_full=$new_acks available=$available reserve=$reserve store_failures=$failures"
[ "$new_acks" -eq 0 ] || fail "full: acknowledged while the volume was full"
set_env REGION_RESERVE_MIB=1024
loaded_pod "$full_pod"; pod="$POD"
read -r epoch committed <<< "$(recovered_at "$pod")"
say "full restored committed=$committed (acked before=$acked) epoch=$epoch pod=$pod"
[ "$acked" -le "$committed" ] || fail "full: lost acknowledgements"
fi
if [ "$scenario" = empty ] || [ "$scenario" = all ]; then
sleep 5
"${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \
'[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","emptyDir":{"sizeLimit":"64Mi"}}}]' > /dev/null
sleep 45
finalize "$pod"; acked="$(max_ack "$pod")" # The drained owner's complete log.
if ready_pod "$pod" 1 > /dev/null; then fail "empty: a host became Ready on an empty volume"; fi
empty_pod="$(current_pods | grep -v "^$pod$" | head -1)"
[ -n "$empty_pod" ] || fail "empty: no replacement pod was created"
# Evidence: the container's exits (4 = quarantined) and both of its logs.
"${k[@]}" logs -n "$ns" "$empty_pod" > "$work/$empty_pod.current.log" 2>&1 || true
"${k[@]}" logs -n "$ns" "$empty_pod" --previous > "$work/$empty_pod.previous.log" 2>&1 || true
restarts="$("${k[@]}" get pod -n "$ns" "$empty_pod" -o jsonpath='{.status.containerStatuses[0].restartCount}')"
exit_code="$("${k[@]}" get pod -n "$ns" "$empty_pod" -o jsonpath='{.status.containerStatuses[0].lastState.terminated.exitCode}')"
quarantined="$(cat "$work/$empty_pod.current.log" "$work/$empty_pod.previous.log" | grep -c '^phase quarantined' || true)"
say "empty pod=$empty_pod ready=never restarts=${restarts:-0} last_exit=${exit_code:-none} quarantined_lines=$quarantined"
say "empty last log lines:"; tail -n 5 "$work/$empty_pod.previous.log" "$work/$empty_pod.current.log" | sed 's/^/ /'
lines="$(cat "$work/$empty_pod.current.log" "$work/$empty_pod.previous.log" | grep -c '^phase ' || true)"
verdict="$(empty_verdict "${restarts:-0}" "${exit_code:-}" "$quarantined" "$lines")"
[ "$verdict" = PASS ] || fail "empty: $verdict"
"${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \
'[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","persistentVolumeClaim":{"claimName":"aevolution-objects"}}}]' > /dev/null
loaded_pod "$empty_pod"; pod="$POD"
read -r epoch committed <<< "$(recovered_at "$pod")"
say "empty restored committed=$committed (acked before=$acked) epoch=$epoch pod=$pod"
[ "$acked" -le "$committed" ] || fail "empty: the world was reset or acknowledgements lost"
fi
if [ "$scenario" = db-outage ]; then
sleep 5
epoch0="$(metric "$pod" evo_region_lease_epoch)"
node="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.spec.nodeName}')"
node_ip="$("${k[@]}" get node "$node" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}')"
# The pod reaches the database from its node's address (egress is SNATed off-cluster).
say "pod $pod runs on $node ($node_ip). On the database VM, block it for at least 30 s (lease 20 s):"
say " sudo iptables -I INPUT -s $node_ip -p tcp --dport 5432 -j DROP"
say " then lift it with:"
say " sudo iptables -D INPUT -s $node_ip -p tcp --dport 5432 -j DROP"
say "waiting for the host to lose readiness..."
for _ in $(seq 1 600); do [ "$(metric "$pod" evo_region_ready)" = 0 ] && break; sleep 0.5; done
[ "$(metric "$pod" evo_region_ready)" = 0 ] || fail "db-outage: host stayed ready"
lost="$(now_ms)"
acked="$(max_ack "$pod")"
say "host not ready; acknowledgements stopped at tick $acked. Waiting for recovery after the block is lifted..."
for _ in $(seq 1 1200); do [ "$(metric "$pod" evo_region_ready)" = 1 ] && break; sleep 0.5; done
[ "$(metric "$pod" evo_region_ready)" = 1 ] || fail "db-outage: host did not recover within 10 minutes"
epoch1="$(metric "$pod" evo_region_lease_epoch)"
committed="$(metric "$pod" evo_region_committed_tick)"
say "db-outage unavailable_ms=$(( $(now_ms) - lost )) epoch=$epoch0->$epoch1 committed=$committed acked_before=$acked fences=$(metric "$pod" evo_region_fences_total)"
[ "$epoch1" -gt "$epoch0" ] && [ "$acked" -le "$committed" ] || fail "db-outage: stale authority resumed or acknowledgements lost"
fi
say "PASS fault drill ($scenario) in $ns"