# Fault-drill verdicts (sourced by tools/ops/fault-drill.sh; tested offline with # fixtures by tools/ci/ops-lib-check.sh). Each prints PASS, or FAIL/INCONCLUSIVE with # a reason; the drill passes a scenario only on PASS (review M8 P2: a missing # observation is never a pass). # rto_verdict : the ratified objective (ADR-0075: 60 s for dev/stage). rto_verdict() { if [[ "${1:-}" =~ ^[0-9]+$ ]] && [ "$1" -le "$2" ]; then echo PASS; else echo "FAIL rto ${1:-unknown} ms > ${2} ms"; fi } # empty_verdict # Positive evidence only: the replacement host ran and logged "phase quarantined" # (on an empty volume the database and builds are unchanged, so unavailable committed # state is the only cause), and if its container ended, it ended with exit 4. # Another exit (image, permission, crash) is a FAIL; no evidence is INCONCLUSIVE. empty_verdict() { local restarts="${1:-0}" code="${2:-}" quarantined="${3:-0}" lines="${4:-0}" if [ -n "$code" ] && [ "$code" != 4 ]; then echo "FAIL the host exited $code, not 4 (quarantined)" elif [ "$quarantined" -gt 0 ]; then echo PASS elif [ "$restarts" = 0 ] && [ "$lines" = 0 ]; then echo "INCONCLUSIVE the replacement host never ran (Pending, image pull or start failure?)" else echo "INCONCLUSIVE the host ran but logged no quarantine" fi } # acks_verdict : every acknowledgement the dead # owner was observed to make is at or below the head its successor recovered. acks_verdict() { if [[ "${1:-}" =~ ^[0-9]+$ && "${2:-}" =~ ^[0-9]+$ ]] && [ "$1" -le "$2" ]; then echo PASS else echo "FAIL acknowledged ${1:-?} > recovered ${2:-?}"; fi }