The restore no longer relies on an auxiliary lock session. One install transaction holds the dump, a nonce claim row, fenced leases and retired builds, so the world is unservable until a nonce-checked publish after verification, regardless of connection loss or restarts. A failed install is resolved from the claim (committed, rolled back, or unknown with the target kept unservable and --abandon NONCE to resolve); cleanup only removes what the nonce owns. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
182 lines
12 KiB
Bash
182 lines
12 KiB
Bash
#!/usr/bin/env bash
|
|
# M8.7 backup and restore qualification (ADR-0074). Run by postgres-check.sh inside
|
|
# its throwaway cluster (PG* already set; EVO_SERVER_BIN, EVO_WORLD_CLONE_BIN and the
|
|
# runtime roles exported). A real evo-server commits under synthetic load while the
|
|
# backup is taken; the bundle is then restored on a separate cluster and rebuilt as a
|
|
# runnable isolated world, and damaged bundles are refused.
|
|
set -euo pipefail
|
|
root="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
work="$(mktemp -d /tmp/evo-backup-check.XXXXXXXX)"
|
|
chmod 700 "$work"
|
|
server_pid=""
|
|
trap '[ -n "$server_pid" ] && kill -9 "$server_pid" 2> /dev/null; rm -rf -- "$work"' EXIT
|
|
fail() { echo "backup check FAILED: $*" >&2; exit 1; }
|
|
build=b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1
|
|
createdb evo_backup_src
|
|
export PGDATABASE=evo_backup_src
|
|
bash "$root/tools/migrate-postgres.sh" > /dev/null
|
|
mkdir -m 700 "$work/objects"
|
|
pg=(--pg-host "$PGHOST" --pg-port "$PGPORT" --pg-database evo_backup_src)
|
|
"$EVO_WORLD_CLONE_BIN" provision --world 300 --region 1 --opening development-v1 \
|
|
"${pg[@]}" --pg-user "$PGUSER" --objects "$work/objects" --build "$build" --retain-build true > /dev/null
|
|
"$EVO_SERVER_BIN" --profile lineage --world 300 --region 1 --frame 9 "${pg[@]}" \
|
|
--pg-user "$EVO_PG_RUNTIME_USER" --objects "$work/objects" --build "$build" \
|
|
--flush-ms 200 --lease-profile fast --synthetic-inputs development-v1 \
|
|
< /dev/null > "$work/server.log" 2>&1 &
|
|
server_pid=$!
|
|
acked_through() { awk -v t="$1" '$1 == "ack" && $3 >= t {f = 1} END {exit !f}' "$work/server.log"; }
|
|
max_ack() { awk '$1 == "ack" && $3 > m {m = $3} END {print m + 0}' "$work/server.log"; }
|
|
for _ in $(seq 1 300); do acked_through 60 && break; sleep 0.1; done
|
|
acked_through 60 || fail "the source server acknowledged nothing"
|
|
|
|
# Online backup while the server keeps committing.
|
|
acked_before="$(max_ack)"
|
|
bash "$root/tools/backup/backup-world.sh" --objects "$work/objects" --out "$work/bundle" --label check
|
|
read -r _ _ _ _ _ backup_tick _ < <(grep '^head 300 1 ' "$work/bundle/MANIFEST")
|
|
[ "$backup_tick" -ge "$acked_before" ] || fail "backup head $backup_tick misses acknowledged tick $acked_before"
|
|
for _ in $(seq 1 300); do acked_through $((backup_tick + 40)) && break; sleep 0.1; done
|
|
kill -TERM "$server_pid"; code=0; wait "$server_pid" || code=$?; server_pid=""
|
|
[ "$code" = 0 ] || fail "the source server exited $code"
|
|
source_tick="$(psql -XAtc 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 300')"
|
|
[ "$source_tick" -gt "$backup_tick" ] || fail "the source did not move past the backup ($source_tick)"
|
|
grep -q '^build b1b1' "$work/bundle/MANIFEST" && grep -q '^schema 9 ' "$work/bundle/MANIFEST" ||
|
|
fail "manifest lacks builds or schema versions"
|
|
if grep -q -i -E 'password|pgpass' "$work/bundle/MANIFEST"; then fail "manifest names credentials"; fi
|
|
|
|
# Full restore on an independent cluster, rebuilt and served as isolated world 301.
|
|
EVO_VERIFY_PORT=$((PGPORT + 1)) bash "$root/tools/backup/verify-backup.sh" "$work/bundle" \
|
|
--isolate 301 > "$work/verify.log" 2>&1 || { cat "$work/verify.log" >&2; fail "verify-backup"; }
|
|
grep -q "^RESTORE VERIFIED regions=1 " "$work/verify.log" &&
|
|
grep -q "PASS restore verified and world 300 rebuilt as isolated world 301" "$work/verify.log" ||
|
|
{ cat "$work/verify.log" >&2; fail "verify-backup evidence"; }
|
|
|
|
# Damaged bundles, restored straight into scratch databases of this cluster.
|
|
case_n=0
|
|
expect_refusal() { # expect_refusal <exit code> <bundle> <what>
|
|
case_n=$((case_n + 1))
|
|
createdb "evo_restore_case_$case_n"
|
|
local code=0
|
|
PGDATABASE="evo_restore_case_$case_n" bash "$root/tools/backup/restore-world.sh" \
|
|
--bundle "$2" --objects "$work/restored-$case_n" > "$work/case-$case_n.log" 2>&1 || code=$?
|
|
[ "$code" = "$1" ] || { cat "$work/case-$case_n.log" >&2; fail "$3: exit $code, expected $1"; }
|
|
[ ! -e "$work/restored-$case_n" ] || fail "$3: restored objects were left behind"
|
|
[ "$(PGDATABASE="evo_restore_case_$case_n" psql -XAtc "SELECT count(*) FROM pg_namespace WHERE nspname IN ('evolution', 'evolution_restore')")" = 0 ] ||
|
|
fail "$3: a restored schema was left behind"
|
|
}
|
|
damaged() { rm -rf "$work/damaged"; cp -a "$work/bundle" "$work/damaged"; }
|
|
resum() { (cd "$work/damaged" && find . -type f ! -name SHA256SUMS -printf '%P\0' | sort -z | xargs -0 sha256sum -- > SHA256SUMS); }
|
|
genesis="$(psql -XAtc "SELECT encode(payload_sha256, 'hex') FROM evolution.region_commits WHERE world_id = 300 AND commit_sequence = 0")"
|
|
genesis_object="$work/damaged/objects/w300/r1/$genesis"
|
|
|
|
damaged; printf 'corrupt' | dd of="$genesis_object" bs=1 conv=notrunc status=none
|
|
expect_refusal 4 "$work/damaged" "a corrupt object"
|
|
damaged; echo extra > "$work/damaged/objects/w300/r1/notes"
|
|
expect_refusal 4 "$work/damaged" "an unlisted file"
|
|
damaged; printf 'x' >> "$work/damaged/database.dump"
|
|
expect_refusal 4 "$work/damaged" "a damaged dump"
|
|
# Consistently re-summed damage passes the checksums and must fail verification.
|
|
damaged; rm "$genesis_object"; resum
|
|
expect_refusal 3 "$work/damaged" "a missing payload object"
|
|
grep -q 'audit failed for 300/1' "$work/case-$case_n.log" || fail "missing object: no audit failure"
|
|
damaged; sed -i -E "s/^(head 300 1 [0-9]+ [0-9]+) $backup_tick /\1 $((backup_tick - 1)) /" "$work/damaged/MANIFEST"; resum
|
|
expect_refusal 3 "$work/damaged" "a manifest head that disagrees with the database"
|
|
# A target that already holds a world is refused before anything is written.
|
|
createdb evo_restore_twice
|
|
PGDATABASE=evo_restore_twice bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" \
|
|
--objects "$work/twice-a" > /dev/null
|
|
code=0
|
|
PGDATABASE=evo_restore_twice bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" \
|
|
--objects "$work/twice-b" > "$work/twice.log" 2>&1 || code=$?
|
|
[ "$code" = 4 ] && grep -q 'already has an evolution schema' "$work/twice.log" || fail "a second restore into one database"
|
|
[ "$(PGDATABASE=evo_restore_twice psql -XAtc 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 300')" = "$backup_tick" ] ||
|
|
fail "the refused second restore changed the first"
|
|
# Ownership and failure paths (M8 review P1; follow-up P1/P2). Each case names the
|
|
# boundary it interrupts; only this invocation's state may ever be removed.
|
|
restore() { # restore <database> <objects> [env...]: runs restore-world.sh, sets rc.
|
|
local db="$1" dir="$2"; shift 2
|
|
rc=0
|
|
env PGDATABASE="$db" "$@" bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" \
|
|
--objects "$dir" > "$work/$(basename "$dir").log" 2>&1 || rc=$?
|
|
}
|
|
schemas() { PGDATABASE="$1" psql -XAtc "SELECT count(*) FROM pg_namespace WHERE nspname IN ('evolution', 'evolution_restore')"; }
|
|
acquire_refused() { # A lease acquisition in <database> is refused (the world is unservable).
|
|
! PGDATABASE="$1" psql -XAt -v ON_ERROR_STOP=1 \
|
|
-c "SELECT * FROM evolution.acquire_region_lease(300, 1, 20000, decode(repeat('b1', 32), 'hex'))" \
|
|
> "$work/acquire-$1.log" 2>&1 && grep -q 'build not current' "$work/acquire-$1.log"
|
|
}
|
|
log_of() { cat "$work/$(basename "$1").log" >&2; }
|
|
|
|
# A competing actor creates the schema after this restore's checks: its install
|
|
# transaction rolls back (confirmed), and the competitor's schema survives.
|
|
createdb evo_restore_race
|
|
restore evo_restore_race "$work/race-a" EVO_RESTORE_TEST_HOOK='[ "$1" = before-install ] && psql -XAtq -c "CREATE SCHEMA evolution" -c "CREATE TABLE evolution.winner(x int)" || true'
|
|
[ "$rc" = 4 ] && grep -q 'rolled back; the database was not changed (confirmed)' "$work/race-a.log" ||
|
|
{ log_of race-a; fail "a pre-empted restore: exit $rc"; }
|
|
[ "$(PGDATABASE=evo_restore_race psql -XAtc "SELECT count(*) FROM pg_tables WHERE schemaname = 'evolution' AND tablename = 'winner'")" = 1 ] ||
|
|
fail "a pre-empted restore deleted the competing schema"
|
|
[ ! -e "$work/race-a" ] || fail "a pre-empted restore left its own objects directory"
|
|
|
|
# Two restores of one database: whichever installs first wins; the other rolls back
|
|
# and leaves the winner published and intact.
|
|
createdb evo_restore_pair
|
|
restore evo_restore_pair "$work/pair-a" EVO_RESTORE_TEST_HOOK='[ "$1" = before-install ] && sleep 4 || true' &
|
|
pair_pid=$!
|
|
sleep 1
|
|
restore evo_restore_pair "$work/pair-b"
|
|
[ "$rc" = 0 ] && grep -q '^RESTORE VERIFIED regions=1 ' "$work/pair-b.log" || { log_of pair-b; fail "the first installer: exit $rc"; }
|
|
code=0; wait "$pair_pid" || code=$?
|
|
grep -q 'rolled back; the database was not changed (confirmed)' "$work/pair-a.log" && [ ! -e "$work/pair-a" ] ||
|
|
{ log_of pair-a; fail "the second installer: exit $code"; }
|
|
[ "$(PGDATABASE=evo_restore_pair psql -XAtc "SELECT state FROM evolution_restore.claims")" = published ] &&
|
|
[ "$(PGDATABASE=evo_restore_pair psql -XAtc 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 300')" = "$backup_tick" ] ||
|
|
fail "the losing restore disturbed the winner"
|
|
|
|
# After install and before publication the world is unservable, durably: across a
|
|
# database restart (every connection and lock lost) acquisition is still refused, and
|
|
# the restore then verifies and publishes; afterwards the build is current again.
|
|
createdb evo_restore_gate
|
|
restore evo_restore_gate "$work/gate" EVO_RESTORE_TEST_HOOK='
|
|
if [ "$1" = after-install ]; then
|
|
psql -XAt -c "SELECT * FROM evolution.acquire_region_lease(300, 1, 20000, decode(repeat('"'"'b1'"'"', 32), '"'"'hex'"'"'))" > "'"$work"'/gate-before.log" 2>&1 || true
|
|
pg_ctl -D "$PGHOST/data" -m fast -w restart > /dev/null
|
|
psql -XAt -c "SELECT * FROM evolution.acquire_region_lease(300, 1, 20000, decode(repeat('"'"'b1'"'"', 32), '"'"'hex'"'"'))" > "'"$work"'/gate-after.log" 2>&1 || true
|
|
fi'
|
|
[ "$rc" = 0 ] && grep -q '^RESTORE VERIFIED regions=1 ' "$work/gate.log" || { log_of gate; fail "a restore across a restart: exit $rc"; }
|
|
grep -q 'build not current' "$work/gate-before.log" && grep -q 'build not current' "$work/gate-after.log" ||
|
|
{ cat "$work/gate-before.log" "$work/gate-after.log" >&2; fail "authority was available before publication"; }
|
|
[ "$(PGDATABASE=evo_restore_gate psql -XAtc 'SELECT count(*) FROM evolution.retained_builds WHERE retired_at IS NULL')" = 1 ] ||
|
|
fail "publication did not restore the current build"
|
|
|
|
# A committed install whose completion is lost is resolved from the claim and
|
|
# verified, not reported as unchanged.
|
|
createdb evo_restore_lost
|
|
restore evo_restore_lost "$work/lost" EVO_RESTORE_TEST_INSTALL_ACK=lost
|
|
[ "$rc" = 0 ] && grep -q '^install outcome resolved: committed' "$work/lost.log" &&
|
|
grep -q '^RESTORE VERIFIED regions=1 ' "$work/lost.log" || { log_of lost; fail "a lost install acknowledgement: exit $rc"; }
|
|
|
|
# An install whose outcome cannot be resolved keeps everything, stays unservable,
|
|
# and is removed only by --abandon with its claim.
|
|
createdb evo_restore_unknown
|
|
restore evo_restore_unknown "$work/unknown" EVO_RESTORE_TEST_INSTALL_ACK=unreachable
|
|
nonce="$(sed -n 's/^restore claim \([0-9a-f]\{32\}\)$/\1/p' "$work/unknown.log")"
|
|
[ "$rc" = 5 ] && grep -q '^RESTORE OUTCOME UNKNOWN' "$work/unknown.log" && [ -d "$work/unknown" ] && [ -n "$nonce" ] ||
|
|
{ log_of unknown; fail "an unresolvable install: exit $rc"; }
|
|
[ "$(schemas evo_restore_unknown)" = 2 ] && acquire_refused evo_restore_unknown || fail "an unresolved restore is servable"
|
|
code=0
|
|
PGDATABASE=evo_restore_unknown bash "$root/tools/backup/restore-world.sh" --abandon 00000000000000000000000000000000 \
|
|
--objects "$work/unknown" > "$work/abandon-wrong.log" 2>&1 || code=$?
|
|
[ "$(schemas evo_restore_unknown)" = 2 ] && [ -d "$work/unknown" ] || fail "--abandon with another claim removed state"
|
|
PGDATABASE=evo_restore_unknown bash "$root/tools/backup/restore-world.sh" --abandon "$nonce" \
|
|
--objects "$work/unknown" > "$work/abandon.log" 2>&1 || { cat "$work/abandon.log" >&2; fail "--abandon"; }
|
|
[ "$(schemas evo_restore_unknown)" = 0 ] && [ ! -e "$work/unknown" ] || fail "--abandon left state behind"
|
|
restore evo_restore_unknown "$work/unknown-retry"
|
|
[ "$rc" = 0 ] || { log_of unknown-retry; fail "a restore after --abandon: exit $rc"; }
|
|
|
|
# An existing objects path is never claimed or removed.
|
|
mkdir "$work/occupied"; echo keep > "$work/occupied/mine"
|
|
createdb evo_restore_occupied
|
|
restore evo_restore_occupied "$work/occupied"
|
|
[ "$rc" = 4 ] && [ "$(cat "$work/occupied/mine")" = keep ] && [ "$(schemas evo_restore_occupied)" = 0 ] ||
|
|
fail "an occupied objects path: exit $rc"
|
|
echo "backup and restore qualification passed (backup tick $backup_tick, acknowledged $acked_before, source $source_tick)"
|