#!/usr/bin/env bash # M8.6 fault drill (ADR-0073) for a disposable environment only. # # tools/ops/fault-drill.sh [kills|full|empty|db-outage|all] [--context ] # # Puts the region under synthetic load (REGION_INPUTS=development-v1), then runs: # kills three forced pod deletions under load: time to Ready (RTO) and zero # lost acknowledgements (every tick a dying pod acknowledged is at or # below the committed head its successor recovers); # full the object volume "fills" (REGION_RESERVE_MIB beyond any disk): the # host keeps simulating but acknowledges nothing; restoring the reserve # resumes from the committed head with nothing lost; # empty the object volume is replaced by an empty one: the host must # quarantine, never become ready and never start a fresh world; # restoring the volume recovers the same committed head; # db-outage interactive: the operator blocks the database for longer than the # lease (e.g. drop port 5432 from the store node on the database VM); # the host must stop being ready and acknowledge nothing, then after # the block is lifted re-acquire under a newer epoch with nothing lost. # `all` runs kills, full and empty (db-outage needs an operator). Every change is # made with `kubectl set env` / `kubectl patch` on the evo-region Deployment and # reverted on exit, including on failure, back to the Git values (so Argo returns to # Synced); World0 is refused before anything touches the cluster. # # Verdicts (tools/ops/lib/drill.sh): each kill must recover within RTO_BUDGET_MS # (default 60000, ADR-0075); the empty scenario needs positive quarantine evidence. # A missing observation fails the drill. Limits of log-based evidence: an owner's # acknowledgements are read from its streamed log, finalized after it is gone; a line # written but never streamed before a forced kill is not observed, so the # zero-lost-acknowledgement check covers every observed acknowledgement only. set -euo pipefail usage() { echo "usage: $0 [kills|full|empty|db-outage|all] [--context ]" >&2; exit 2; } [ $# -ge 1 ] || usage env="$1"; shift case "$env" in dev|stage) ;; *) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2 exit 2 ;; esac scenario="${1:-all}" case "$scenario" in kills|full|empty|db-outage|all) shift || true ;; --context) scenario=all ;; *) usage ;; esac k=(kubectl --request-timeout=30s) [ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2") ns="evolution-$env" rto_budget="${RTO_BUDGET_MS:-60000}" # shellcheck source=lib/drill.sh . "$(cd "$(dirname "$0")" && pwd)/lib/drill.sh" selector=app.kubernetes.io/name=evo-region work="$(mktemp -d)" log_pids=() declare -A capture_pid=() now_ms() { date +%s%3N; } say() { echo "$(date -u +%H:%M:%S) $*"; } fail() { say "FAIL $*"; exit 1; } current_pods() { "${k[@]}" get pods -n "$ns" -l "$selector" \ -o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}'; } # (kubectl jsonpath has no negation; the go-template skips pods already being deleted.) ready_pod() { # Waits up to $2 seconds (default 120) for a Ready pod other than $1. local old="$1" limit="${2:-120}" for _ in $(seq 1 $((limit * 5))); do for p in $(current_pods); do [ "$p" = "$old" ] && continue [ "$("${k[@]}" get pod -n "$ns" "$p" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null)" = True ] && { echo "$p"; return 0; } done sleep 0.2 done return 1 } metric() { # metric : from the pod's own /metrics (bash /dev/tcp). "${k[@]}" exec -n "$ns" "$1" -- /bin/bash -c \ 'exec 3<>/dev/tcp/127.0.0.1/9100; printf "GET /metrics HTTP/1.1\r\nHost: drill\r\n\r\n" >&3; cat <&3' 2>/dev/null | awk -v s="$2" '$1 == s {print $2}' } capture() { # Streams a pod's log to $work/.log for acknowledgement accounting. "${k[@]}" logs -n "$ns" -f "$1" > "$work/$1.log" 2>/dev/null & log_pids+=($!) capture_pid[$1]=$! } finalize() { # Waits (up to 15 s) for a gone pod's log stream to end, so its log is complete. local pid="${capture_pid[$1]:-}" [ -n "$pid" ] || return 0 for _ in $(seq 1 75); do kill -0 "$pid" 2> /dev/null || return 0; sleep 0.2; done kill "$pid" 2> /dev/null || true } max_ack() { awk '$1 == "ack" {if ($3 > m) m = $3} END {print m + 0}' "$work/$1.log"; } recovered_at() { # committed tick from the pod's "phase running epoch=E committed=C" line. sed -n 's/^phase running epoch=\([0-9]*\) committed=\([0-9]*\) .*/\1 \2/p' "$work/$1.log" | tail -1 } wait_for_line() { # wait_for_line for _ in $(seq 1 $(($3 * 5))); do grep -qE "$2" "$work/$1.log" && return 0; sleep 0.2; done return 1 } set_env() { "${k[@]}" set env -n "$ns" deployment/evo-region "$@" > /dev/null; } restore() { set +e say "restoring the Git configuration of evo-region" set_env REGION_INPUTS= REGION_RESERVE_MIB=1024 "${k[@]}" get deployment -n "$ns" evo-region -o jsonpath='{.spec.template.spec.volumes[0].emptyDir}' | grep -q . && "${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \ '[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","persistentVolumeClaim":{"claimName":"aevolution-objects"}}}]' > /dev/null for pid in "${log_pids[@]}"; do kill "$pid" 2>/dev/null; done ready_pod "" 120 > /dev/null && say "restored: evo-region Ready" || say "WARN evo-region not Ready after restore" say "logs kept in $work" } trap restore EXIT POD="" loaded_pod() { # Sets POD to a Ready pod other than $1 once acknowledgements flow. POD="$(ready_pod "${1:-}" 120)" || fail "no Ready evo-region pod" capture "$POD" wait_for_line "$POD" '^ack ' 60 || fail "$POD acknowledges nothing under load" } say "enabling synthetic load in $ns" before="$(current_pods | head -1)" set_env REGION_INPUTS=development-v1 loaded_pod "$before"; pod="$POD" say "loaded pod=$pod ($(recovered_at "$pod"))" if [ "$scenario" = kills ] || [ "$scenario" = all ]; then for round in 1 2 3; do sleep 10 start="$(now_ms)" "${k[@]}" delete pod -n "$ns" "$pod" --grace-period=0 --force > /dev/null 2>&1 POD="$(ready_pod "$pod" 120)" || fail "round $round: no Ready successor within 120 s" rto=$(( $(now_ms) - start )) next="$POD" capture "$next" wait_for_line "$next" '^ack ' 60 || fail "$next acknowledges nothing under load" finalize "$pod" acked="$(max_ack "$pod")" # Everything the dead owner was observed to acknowledge. read -r epoch committed <<< "$(recovered_at "$next")" say "kill round=$round rto_ms=$rto budget_ms=$rto_budget dead_acked_max=$acked recovered_committed=$committed epoch=$epoch pod=$next" verdict="$(rto_verdict "$rto" "$rto_budget")" [ "$verdict" = PASS ] || fail "round $round: $verdict" verdict="$(acks_verdict "$acked" "$committed")" [ "$verdict" = PASS ] || fail "round $round lost acknowledgements: $verdict" pod="$next" done fi if [ "$scenario" = full ] || [ "$scenario" = all ]; then sleep 5 set_env REGION_RESERVE_MIB=1048576 full_pod="$(ready_pod "$pod" 120)" || fail "full: host did not recover (recovery needs no space)" capture "$full_pod" finalize "$pod"; acked="$(max_ack "$pod")" # The drained owner's complete log. sleep 30 new_acks="$(awk '$1 == "ack"' "$work/$full_pod.log" | wc -l)" available="$(metric "$full_pod" evo_region_objects_available_bytes)" reserve="$(metric "$full_pod" evo_region_objects_reserve_bytes)" failures="$(metric "$full_pod" evo_region_store_failures_total)" say "full acks_while_full=$new_acks available=$available reserve=$reserve store_failures=$failures" [ "$new_acks" -eq 0 ] || fail "full: acknowledged while the volume was full" set_env REGION_RESERVE_MIB=1024 loaded_pod "$full_pod"; pod="$POD" read -r epoch committed <<< "$(recovered_at "$pod")" say "full restored committed=$committed (acked before=$acked) epoch=$epoch pod=$pod" [ "$acked" -le "$committed" ] || fail "full: lost acknowledgements" fi if [ "$scenario" = empty ] || [ "$scenario" = all ]; then sleep 5 "${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \ '[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","emptyDir":{"sizeLimit":"64Mi"}}}]' > /dev/null sleep 45 finalize "$pod"; acked="$(max_ack "$pod")" # The drained owner's complete log. if ready_pod "$pod" 1 > /dev/null; then fail "empty: a host became Ready on an empty volume"; fi empty_pod="$(current_pods | grep -v "^$pod$" | head -1)" [ -n "$empty_pod" ] || fail "empty: no replacement pod was created" # Evidence: the container's exits (4 = quarantined) and both of its logs. "${k[@]}" logs -n "$ns" "$empty_pod" > "$work/$empty_pod.current.log" 2>&1 || true "${k[@]}" logs -n "$ns" "$empty_pod" --previous > "$work/$empty_pod.previous.log" 2>&1 || true restarts="$("${k[@]}" get pod -n "$ns" "$empty_pod" -o jsonpath='{.status.containerStatuses[0].restartCount}')" exit_code="$("${k[@]}" get pod -n "$ns" "$empty_pod" -o jsonpath='{.status.containerStatuses[0].lastState.terminated.exitCode}')" quarantined="$(cat "$work/$empty_pod.current.log" "$work/$empty_pod.previous.log" | grep -c '^phase quarantined' || true)" say "empty pod=$empty_pod ready=never restarts=${restarts:-0} last_exit=${exit_code:-none} quarantined_lines=$quarantined" say "empty last log lines:"; tail -n 5 "$work/$empty_pod.previous.log" "$work/$empty_pod.current.log" | sed 's/^/ /' lines="$(cat "$work/$empty_pod.current.log" "$work/$empty_pod.previous.log" | grep -c '^phase ' || true)" verdict="$(empty_verdict "${restarts:-0}" "${exit_code:-}" "$quarantined" "$lines")" [ "$verdict" = PASS ] || fail "empty: $verdict" "${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \ '[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","persistentVolumeClaim":{"claimName":"aevolution-objects"}}}]' > /dev/null loaded_pod "$empty_pod"; pod="$POD" read -r epoch committed <<< "$(recovered_at "$pod")" say "empty restored committed=$committed (acked before=$acked) epoch=$epoch pod=$pod" [ "$acked" -le "$committed" ] || fail "empty: the world was reset or acknowledgements lost" fi if [ "$scenario" = db-outage ]; then sleep 5 epoch0="$(metric "$pod" evo_region_lease_epoch)" node="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.spec.nodeName}')" node_ip="$("${k[@]}" get node "$node" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}')" # The pod reaches the database from its node's address (egress is SNATed off-cluster). say "pod $pod runs on $node ($node_ip). On the database VM, block it for at least 30 s (lease 20 s):" say " sudo iptables -I INPUT -s $node_ip -p tcp --dport 5432 -j DROP" say " then lift it with:" say " sudo iptables -D INPUT -s $node_ip -p tcp --dport 5432 -j DROP" say "waiting for the host to lose readiness..." for _ in $(seq 1 600); do [ "$(metric "$pod" evo_region_ready)" = 0 ] && break; sleep 0.5; done [ "$(metric "$pod" evo_region_ready)" = 0 ] || fail "db-outage: host stayed ready" lost="$(now_ms)" acked="$(max_ack "$pod")" say "host not ready; acknowledgements stopped at tick $acked. Waiting for recovery after the block is lifted..." for _ in $(seq 1 1200); do [ "$(metric "$pod" evo_region_ready)" = 1 ] && break; sleep 0.5; done [ "$(metric "$pod" evo_region_ready)" = 1 ] || fail "db-outage: host did not recover within 10 minutes" epoch1="$(metric "$pod" evo_region_lease_epoch)" committed="$(metric "$pod" evo_region_committed_tick)" say "db-outage unavailable_ms=$(( $(now_ms) - lost )) epoch=$epoch0->$epoch1 committed=$committed acked_before=$acked fences=$(metric "$pod" evo_region_fences_total)" [ "$epoch1" -gt "$epoch0" ] && [ "$acked" -le "$committed" ] || fail "db-outage: stale authority resumed or acknowledgements lost" fi say "PASS fault drill ($scenario) in $ns"