#!/usr/bin/env bash # M8.8 host-loss rehearsal (ADR-0075). Run by postgres-check.sh inside its throwaway # cluster (PG* set; EVO_SERVER_BIN, EVO_WORLD_CLONE_BIN, runtime roles exported). # # The old environment keeps serving world 400 past an online backup, so it has # acknowledged ticks the backup does not hold. The world is restored in place, under # the same identity, into a replacement database, and a new host serves it. The old # database is then fenced while its host is still running. Expected: # - the new owner resumes exactly at the backup head, under a newer epoch; # - the tick-20 death stays in history exactly once and is never repeated; # - the full history audit (deaths, offspring, inventory, ancestry) passes; # - the old host quarantines and acknowledges nothing after the fence; # - a host started later against the old database never becomes ready; # - the rewind (acknowledged by the old host but absent from the restore) is # measured and reported. It is why World0 launch stays blocked. set -euo pipefail root="$(cd "$(dirname "$0")/../.." && pwd)" work="$(mktemp -d /tmp/evo-dr-check.XXXXXXXX)" chmod 700 "$work" pids=() trap 'for p in "${pids[@]}"; do kill -9 "$p" 2> /dev/null || true; done; rm -rf -- "$work"' EXIT fail() { echo "host-loss rehearsal FAILED: $*" >&2; exit 1; } build=b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1 db() { local d="$1"; shift; PGDATABASE="$d" psql -XAt -v ON_ERROR_STOP=1 "$@"; } serve() { # serve : evo-server under synthetic load. "$EVO_SERVER_BIN" --profile lineage --world 400 --region 1 --frame 9 \ --pg-host "$PGHOST" --pg-port "$PGPORT" --pg-database "$1" --pg-user "$EVO_PG_RUNTIME_USER" \ --objects "$2" --build "$build" --flush-ms 200 --lease-profile fast \ --synthetic-inputs development-v1 < /dev/null > "$3" 2>&1 & pids+=($!) server_pid=$! } acked_through() { awk -v t="$2" '$1 == "ack" && $3 >= t {f = 1} END {exit !f}' "$1"; } max_ack() { awk '$1 == "ack" && $3 > m {m = $3} END {print m + 0}' "$1"; } wait_ack() { for _ in $(seq 1 300); do acked_through "$1" "$2" && return 0; sleep 0.1; done; return 1; } # The old environment: provisioned, serving, backed up online, then serving on. createdb evo_dr_old PGDATABASE=evo_dr_old bash "$root/tools/migrate-postgres.sh" > /dev/null mkdir -m 700 "$work/old-objects" "$EVO_WORLD_CLONE_BIN" provision --world 400 --region 1 --opening development-v1 \ --pg-host "$PGHOST" --pg-port "$PGPORT" --pg-database evo_dr_old --pg-user "$PGUSER" \ --objects "$work/old-objects" --build "$build" --retain-build true > /dev/null serve evo_dr_old "$work/old-objects" "$work/old.log"; old_pid=$server_pid wait_ack "$work/old.log" 40 || fail "the old host acknowledged nothing" PGDATABASE=evo_dr_old bash "$root/tools/backup/backup-world.sh" --objects "$work/old-objects" \ --out "$work/bundle" --label rehearsal > /dev/null read -r _ _ _ _ _ cut _ < <(grep '^head 400 1 ' "$work/bundle/MANIFEST") [ "$cut" -ge 20 ] || fail "the backup precedes the death ($cut)" wait_ack "$work/old.log" $((cut + 30)) || fail "the old host stopped after the backup" # Host loss: the world is restored in place into a replacement database, while the # old host still runs (a partition, the worst case for competing authority). createdb evo_dr_new PGDATABASE=evo_dr_new bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" \ --objects "$work/new-objects" --in-place --runtime-role "$EVO_PG_RUNTIME_ROLE" --accept-rewind \ > "$work/restore.log" 2>&1 || { cat "$work/restore.log" >&2; fail "in-place restore"; } grep -q "^REWIND region 400/1 restored at tick $cut " "$work/restore.log" && grep -q '^RESTORE IN PLACE VERIFIED regions=1 ' "$work/restore.log" || fail "restore evidence" [ "$(db evo_dr_new -c 'SELECT count(*) FROM evolution.region_leases WHERE holder IS NOT NULL')" = 0 ] || fail "a restored lease is still held" old_domain="$(db evo_dr_old -c 'SELECT evolution.load_history_domain_v1()')" new_domain="$(db evo_dr_new -c 'SELECT evolution.load_history_domain_v1()')" [ -n "$new_domain" ] && [ "$new_domain" != "$old_domain" ] || fail 'rewind reused source history domain' [ "$(db evo_dr_new -c "SELECT parent FROM evolution.history_domains WHERE domain='$new_domain'")" = "$old_domain" ] || fail 'rewind lost source domain' [ "$(db evo_dr_new -c "SELECT end_tick FROM evolution.history_domain_cuts WHERE domain='$new_domain' AND world_id=400 AND region_id=1")" = "$cut" ] || fail 'rewind cut provenance' serve evo_dr_new "$work/new-objects" "$work/new.log"; new_pid=$server_pid wait_ack "$work/new.log" $((cut + 30)) || { tail "$work/new.log" >&2; fail "the new host did not serve"; } grep -q "^phase running epoch=2 committed=$cut " "$work/new.log" || fail "the new owner did not resume at the backup head under a newer epoch" first_new="$(awk '$1 == "ack" {print $3; exit}' "$work/new.log")" [ "$first_new" = $((cut + 1)) ] || fail "the new owner resumed at $first_new, not $((cut + 1))" # Fence the old database while its host is still running. PGDATABASE=evo_dr_old bash "$root/tools/backup/fence-database.sh" --superseded-by evo_dr_new \ --confirm > "$work/fence.log" || fail "fence" grep -qx 'current 0' "$work/fence.log" && grep -q "^retired $build " "$work/fence.log" || fail "fence evidence" final_old="$(db evo_dr_old -c 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 400')" fenced_acks="$(max_ack "$work/old.log")" code=0 for _ in $(seq 1 300); do kill -0 "$old_pid" 2> /dev/null || break; sleep 0.1; done wait "$old_pid" || code=$? [ "$code" = 4 ] || { tail "$work/old.log" >&2; fail "the fenced old host exited $code, not 4 (quarantined)"; } [ "$(max_ack "$work/old.log")" -le "$final_old" ] || fail "the old host acknowledged past its final head" [ "$(db evo_dr_old -c 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 400')" = "$final_old" ] || fail "the old database advanced after the fence" # A host started later against the old database never serves. serve evo_dr_old "$work/old-objects" "$work/late.log"; late_pid=$server_pid code=0; wait "$late_pid" || code=$? [ "$code" = 4 ] && ! grep -q '^ready 1' "$work/late.log" && ! grep -q '^ack ' "$work/late.log" || fail "a late host on the fenced database exited $code or served" kill -TERM "$new_pid"; code=0; wait "$new_pid" || code=$? [ "$code" = 0 ] || fail "the new host exited $code" new_head="$(db evo_dr_new -c 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 400')" audit="$("$EVO_WORLD_CLONE_BIN" audit --world 400 --region 1 --pg-host "$PGHOST" --pg-port "$PGPORT" \ --pg-database evo_dr_new --pg-user "$PGUSER" --objects "$work/new-objects" --retained "$build")" || fail "history audit of the restored world" [[ "$audit" == *" tick=$new_head "* ]] || fail "audit: $audit" # The death at tick 20 is durable once, before the cut, and never repeated. [ "$(db evo_dr_new -c 'SELECT count(*) FROM evolution.region_decisions WHERE world_id = 400')" = 1 ] || fail "the restored world lost or repeated the death" [ "$(grep -c '^outcome 20 ' "$work/new.log" || true)" = 0 ] || fail "the new owner repeated the death" echo "host-loss rehearsal passed: cut=$cut old_acknowledged=$fenced_acks old_final=$final_old" \ "rewound=$((final_old - cut)) ticks new_head=$new_head ($audit)"