Files
aevolution/tools/ci/dr-check.sh
T
dingoandClaude Opus 5.5 518a960864 feat(ops): M8.8 World0 acknowledgment gate and host-loss recovery (ADR-0075)
Ratifies dev/stage RPO/RTO from measurements and keeps World0 launch blocked
(policy-enforced) until acknowledged outcomes survive host loss in an
independent failure domain. Adds in-place restore with an explicit rewind,
barrier-based fencing of a superseded database, and promotion of stage-pinned
digests only. dr-check.sh rehearses host loss in every PostgreSQL build.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 17:07:07 -04:00

101 lines
6.6 KiB
Bash

#!/usr/bin/env bash
# M8.8 host-loss rehearsal (ADR-0075). Run by postgres-check.sh inside its throwaway
# cluster (PG* set; EVO_SERVER_BIN, EVO_WORLD_CLONE_BIN, runtime roles exported).
#
# The old environment keeps serving world 400 past an online backup, so it has
# acknowledged ticks the backup does not hold. The world is restored in place, under
# the same identity, into a replacement database, and a new host serves it. The old
# database is then fenced while its host is still running. Expected:
# - the new owner resumes exactly at the backup head, under a newer epoch;
# - the tick-20 death stays in history exactly once and is never repeated;
# - the full history audit (deaths, offspring, inventory, ancestry) passes;
# - the old host quarantines and acknowledges nothing after the fence;
# - a host started later against the old database never becomes ready;
# - the rewind (acknowledged by the old host but absent from the restore) is
# measured and reported. It is why World0 launch stays blocked.
set -euo pipefail
root="$(cd "$(dirname "$0")/../.." && pwd)"
work="$(mktemp -d /tmp/evo-dr-check.XXXXXXXX)"
chmod 700 "$work"
pids=()
trap 'for p in "${pids[@]}"; do kill -9 "$p" 2> /dev/null || true; done; rm -rf -- "$work"' EXIT
fail() { echo "host-loss rehearsal FAILED: $*" >&2; exit 1; }
build=b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1
db() { local d="$1"; shift; PGDATABASE="$d" psql -XAt -v ON_ERROR_STOP=1 "$@"; }
serve() { # serve <database> <objects> <log>: evo-server under synthetic load.
"$EVO_SERVER_BIN" --profile lineage --world 400 --region 1 --frame 9 \
--pg-host "$PGHOST" --pg-port "$PGPORT" --pg-database "$1" --pg-user "$EVO_PG_RUNTIME_USER" \
--objects "$2" --build "$build" --flush-ms 200 --lease-profile fast \
--synthetic-inputs development-v1 < /dev/null > "$3" 2>&1 &
pids+=($!)
server_pid=$!
}
acked_through() { awk -v t="$2" '$1 == "ack" && $3 >= t {f = 1} END {exit !f}' "$1"; }
max_ack() { awk '$1 == "ack" && $3 > m {m = $3} END {print m + 0}' "$1"; }
wait_ack() { for _ in $(seq 1 300); do acked_through "$1" "$2" && return 0; sleep 0.1; done; return 1; }
# The old environment: provisioned, serving, backed up online, then serving on.
createdb evo_dr_old
PGDATABASE=evo_dr_old bash "$root/tools/migrate-postgres.sh" > /dev/null
mkdir -m 700 "$work/old-objects"
"$EVO_WORLD_CLONE_BIN" provision --world 400 --region 1 --opening development-v1 \
--pg-host "$PGHOST" --pg-port "$PGPORT" --pg-database evo_dr_old --pg-user "$PGUSER" \
--objects "$work/old-objects" --build "$build" --retain-build true > /dev/null
serve evo_dr_old "$work/old-objects" "$work/old.log"; old_pid=$server_pid
wait_ack "$work/old.log" 40 || fail "the old host acknowledged nothing"
PGDATABASE=evo_dr_old bash "$root/tools/backup/backup-world.sh" --objects "$work/old-objects" \
--out "$work/bundle" --label rehearsal > /dev/null
read -r _ _ _ _ _ cut _ < <(grep '^head 400 1 ' "$work/bundle/MANIFEST")
[ "$cut" -ge 20 ] || fail "the backup precedes the death ($cut)"
wait_ack "$work/old.log" $((cut + 30)) || fail "the old host stopped after the backup"
# Host loss: the world is restored in place into a replacement database, while the
# old host still runs (a partition, the worst case for competing authority).
createdb evo_dr_new
PGDATABASE=evo_dr_new bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" \
--objects "$work/new-objects" --in-place --runtime-role "$EVO_PG_RUNTIME_ROLE" --accept-rewind \
> "$work/restore.log" 2>&1 || { cat "$work/restore.log" >&2; fail "in-place restore"; }
grep -q "^REWIND region 400/1 restored at tick $cut " "$work/restore.log" &&
grep -q '^RESTORE IN PLACE VERIFIED regions=1 ' "$work/restore.log" || fail "restore evidence"
[ "$(db evo_dr_new -c 'SELECT count(*) FROM evolution.region_leases WHERE holder IS NOT NULL')" = 0 ] ||
fail "a restored lease is still held"
serve evo_dr_new "$work/new-objects" "$work/new.log"; new_pid=$server_pid
wait_ack "$work/new.log" $((cut + 30)) || { tail "$work/new.log" >&2; fail "the new host did not serve"; }
grep -q "^phase running epoch=2 committed=$cut " "$work/new.log" ||
fail "the new owner did not resume at the backup head under a newer epoch"
first_new="$(awk '$1 == "ack" {print $3; exit}' "$work/new.log")"
[ "$first_new" = $((cut + 1)) ] || fail "the new owner resumed at $first_new, not $((cut + 1))"
# Fence the old database while its host is still running.
PGDATABASE=evo_dr_old bash "$root/tools/backup/fence-database.sh" --superseded-by evo_dr_new \
--confirm > "$work/fence.log" || fail "fence"
grep -qx 'current 0' "$work/fence.log" && grep -q "^retired $build " "$work/fence.log" || fail "fence evidence"
final_old="$(db evo_dr_old -c 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 400')"
fenced_acks="$(max_ack "$work/old.log")"
code=0
for _ in $(seq 1 300); do kill -0 "$old_pid" 2> /dev/null || break; sleep 0.1; done
wait "$old_pid" || code=$?
[ "$code" = 4 ] || { tail "$work/old.log" >&2; fail "the fenced old host exited $code, not 4 (quarantined)"; }
[ "$(max_ack "$work/old.log")" -le "$final_old" ] || fail "the old host acknowledged past its final head"
[ "$(db evo_dr_old -c 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 400')" = "$final_old" ] ||
fail "the old database advanced after the fence"
# A host started later against the old database never serves.
serve evo_dr_old "$work/old-objects" "$work/late.log"; late_pid=$server_pid
code=0; wait "$late_pid" || code=$?
[ "$code" = 4 ] && ! grep -q '^ready 1' "$work/late.log" && ! grep -q '^ack ' "$work/late.log" ||
fail "a late host on the fenced database exited $code or served"
kill -TERM "$new_pid"; code=0; wait "$new_pid" || code=$?
[ "$code" = 0 ] || fail "the new host exited $code"
new_head="$(db evo_dr_new -c 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 400')"
audit="$("$EVO_WORLD_CLONE_BIN" audit --world 400 --region 1 --pg-host "$PGHOST" --pg-port "$PGPORT" \
--pg-database evo_dr_new --pg-user "$PGUSER" --objects "$work/new-objects" --retained "$build")" ||
fail "history audit of the restored world"
[[ "$audit" == *" tick=$new_head "* ]] || fail "audit: $audit"
# The death at tick 20 is durable once, before the cut, and never repeated.
[ "$(db evo_dr_new -c 'SELECT count(*) FROM evolution.region_decisions WHERE world_id = 400')" = 1 ] ||
fail "the restored world lost or repeated the death"
[ "$(grep -c '^outcome 20 ' "$work/new.log" || true)" = 0 ] || fail "the new owner repeated the death"
echo "host-loss rehearsal passed: cut=$cut old_acknowledged=$fenced_acks old_final=$final_old" \
"rewound=$((final_old - cut)) ticks new_head=$new_head ($audit)"