#!/usr/bin/env bash # M8.7 backup and restore qualification (ADR-0074). Run by postgres-check.sh inside # its throwaway cluster (PG* already set; EVO_SERVER_BIN, EVO_WORLD_CLONE_BIN and the # runtime roles exported). A real evo-server commits under synthetic load while the # backup is taken; the bundle is then restored on a separate cluster and rebuilt as a # runnable isolated world, and damaged bundles are refused. set -euo pipefail root="$(cd "$(dirname "$0")/../.." && pwd)" work="$(mktemp -d /tmp/evo-backup-check.XXXXXXXX)" chmod 700 "$work" server_pid="" trap '[ -n "$server_pid" ] && kill -9 "$server_pid" 2> /dev/null; rm -rf -- "$work"' EXIT fail() { echo "backup check FAILED: $*" >&2; exit 1; } build=b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1 createdb evo_backup_src export PGDATABASE=evo_backup_src bash "$root/tools/migrate-postgres.sh" > /dev/null mkdir -m 700 "$work/objects" pg=(--pg-host "$PGHOST" --pg-port "$PGPORT" --pg-database evo_backup_src) "$EVO_WORLD_CLONE_BIN" provision --world 300 --region 1 --opening development-v1 \ "${pg[@]}" --pg-user "$PGUSER" --objects "$work/objects" --build "$build" --retain-build true > /dev/null "$EVO_SERVER_BIN" --profile lineage --world 300 --region 1 --frame 9 "${pg[@]}" \ --pg-user "$EVO_PG_RUNTIME_USER" --objects "$work/objects" --build "$build" \ --flush-ms 200 --lease-profile fast --synthetic-inputs development-v1 \ < /dev/null > "$work/server.log" 2>&1 & server_pid=$! acked_through() { awk -v t="$1" '$1 == "ack" && $3 >= t {f = 1} END {exit !f}' "$work/server.log"; } max_ack() { awk '$1 == "ack" && $3 > m {m = $3} END {print m + 0}' "$work/server.log"; } for _ in $(seq 1 300); do acked_through 60 && break; sleep 0.1; done acked_through 60 || fail "the source server acknowledged nothing" # Online backup while the server keeps committing. acked_before="$(max_ack)" bash "$root/tools/backup/backup-world.sh" --objects "$work/objects" --out "$work/bundle" --label check read -r _ _ _ _ _ backup_tick _ < <(grep '^head 300 1 ' "$work/bundle/MANIFEST") [ "$backup_tick" -ge "$acked_before" ] || fail "backup head $backup_tick misses acknowledged tick $acked_before" for _ in $(seq 1 300); do acked_through $((backup_tick + 40)) && break; sleep 0.1; done kill -TERM "$server_pid"; code=0; wait "$server_pid" || code=$?; server_pid="" [ "$code" = 0 ] || fail "the source server exited $code" source_tick="$(psql -XAtc 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 300')" [ "$source_tick" -gt "$backup_tick" ] || fail "the source did not move past the backup ($source_tick)" grep -q '^build b1b1' "$work/bundle/MANIFEST" && grep -q '^schema 9 ' "$work/bundle/MANIFEST" || fail "manifest lacks builds or schema versions" if grep -q -i -E 'password|pgpass' "$work/bundle/MANIFEST"; then fail "manifest names credentials"; fi # Full restore on an independent cluster, rebuilt and served as isolated world 301. EVO_VERIFY_PORT=$((PGPORT + 1)) bash "$root/tools/backup/verify-backup.sh" "$work/bundle" \ --isolate 301 > "$work/verify.log" 2>&1 || { cat "$work/verify.log" >&2; fail "verify-backup"; } grep -q "^RESTORE VERIFIED regions=1 " "$work/verify.log" && grep -q "PASS restore verified and world 300 rebuilt as isolated world 301" "$work/verify.log" || { cat "$work/verify.log" >&2; fail "verify-backup evidence"; } # Damaged bundles, restored straight into scratch databases of this cluster. case_n=0 expect_refusal() { # expect_refusal case_n=$((case_n + 1)) createdb "evo_restore_case_$case_n" local code=0 PGDATABASE="evo_restore_case_$case_n" bash "$root/tools/backup/restore-world.sh" \ --bundle "$2" --objects "$work/restored-$case_n" > "$work/case-$case_n.log" 2>&1 || code=$? [ "$code" = "$1" ] || { cat "$work/case-$case_n.log" >&2; fail "$3: exit $code, expected $1"; } [ ! -e "$work/restored-$case_n" ] || fail "$3: restored objects were left behind" [ "$(PGDATABASE="evo_restore_case_$case_n" psql -XAtc "SELECT count(*) FROM pg_namespace WHERE nspname IN ('evolution', 'evolution_restore')")" = 0 ] || fail "$3: a restored schema was left behind" } damaged() { rm -rf "$work/damaged"; cp -a "$work/bundle" "$work/damaged"; } resum() { (cd "$work/damaged" && find . -type f ! -name SHA256SUMS -printf '%P\0' | sort -z | xargs -0 sha256sum -- > SHA256SUMS); } genesis="$(psql -XAtc "SELECT encode(payload_sha256, 'hex') FROM evolution.region_commits WHERE world_id = 300 AND commit_sequence = 0")" genesis_object="$work/damaged/objects/w300/r1/$genesis" damaged; printf 'corrupt' | dd of="$genesis_object" bs=1 conv=notrunc status=none expect_refusal 4 "$work/damaged" "a corrupt object" damaged; echo extra > "$work/damaged/objects/w300/r1/notes" expect_refusal 4 "$work/damaged" "an unlisted file" damaged; printf 'x' >> "$work/damaged/database.dump" expect_refusal 4 "$work/damaged" "a damaged dump" # Consistently re-summed damage passes the checksums and must fail verification. damaged; rm "$genesis_object"; resum expect_refusal 3 "$work/damaged" "a missing payload object" grep -q 'audit failed for 300/1' "$work/case-$case_n.log" || fail "missing object: no audit failure" damaged; sed -i -E "s/^(head 300 1 [0-9]+ [0-9]+) $backup_tick /\1 $((backup_tick - 1)) /" "$work/damaged/MANIFEST"; resum expect_refusal 3 "$work/damaged" "a manifest head that disagrees with the database" # A target that already holds a world is refused before anything is written. createdb evo_restore_twice PGDATABASE=evo_restore_twice bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" \ --objects "$work/twice-a" > /dev/null code=0 PGDATABASE=evo_restore_twice bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" \ --objects "$work/twice-b" > "$work/twice.log" 2>&1 || code=$? [ "$code" = 4 ] && grep -q 'already has an evolution schema' "$work/twice.log" || fail "a second restore into one database" [ "$(PGDATABASE=evo_restore_twice psql -XAtc 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 300')" = "$backup_tick" ] || fail "the refused second restore changed the first" # Ownership and failure paths (M8 review P1; follow-up P1/P2). Each case names the # boundary it interrupts; only this invocation's state may ever be removed. restore() { # restore [env...]: runs restore-world.sh, sets rc. local db="$1" dir="$2"; shift 2 rc=0 env PGDATABASE="$db" "$@" bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" \ --objects "$dir" > "$work/$(basename "$dir").log" 2>&1 || rc=$? } schemas() { PGDATABASE="$1" psql -XAtc "SELECT count(*) FROM pg_namespace WHERE nspname IN ('evolution', 'evolution_restore')"; } acquire_refused() { # A lease acquisition in is refused (the world is unservable). ! PGDATABASE="$1" psql -XAt -v ON_ERROR_STOP=1 \ -c "SELECT * FROM evolution.acquire_region_lease(300, 1, 20000, decode(repeat('b1', 32), 'hex'))" \ > "$work/acquire-$1.log" 2>&1 && grep -q 'build not current' "$work/acquire-$1.log" } log_of() { cat "$work/$(basename "$1").log" >&2; } # A competing actor creates the schema after this restore's checks: its install # transaction rolls back (confirmed), and the competitor's schema survives. createdb evo_restore_race restore evo_restore_race "$work/race-a" EVO_RESTORE_TEST_HOOK='[ "$1" = before-install ] && psql -XAtq -c "CREATE SCHEMA evolution" -c "CREATE TABLE evolution.winner(x int)" || true' [ "$rc" = 4 ] && grep -q 'rolled back; the database was not changed (confirmed)' "$work/race-a.log" || { log_of race-a; fail "a pre-empted restore: exit $rc"; } [ "$(PGDATABASE=evo_restore_race psql -XAtc "SELECT count(*) FROM pg_tables WHERE schemaname = 'evolution' AND tablename = 'winner'")" = 1 ] || fail "a pre-empted restore deleted the competing schema" [ ! -e "$work/race-a" ] || fail "a pre-empted restore left its own objects directory" # Two restores of one database: whichever installs first wins; the other rolls back # and leaves the winner published and intact. createdb evo_restore_pair restore evo_restore_pair "$work/pair-a" EVO_RESTORE_TEST_HOOK='[ "$1" = before-install ] && sleep 4 || true' & pair_pid=$! sleep 1 restore evo_restore_pair "$work/pair-b" [ "$rc" = 0 ] && grep -q '^RESTORE VERIFIED regions=1 ' "$work/pair-b.log" || { log_of pair-b; fail "the first installer: exit $rc"; } code=0; wait "$pair_pid" || code=$? grep -q 'rolled back; the database was not changed (confirmed)' "$work/pair-a.log" && [ ! -e "$work/pair-a" ] || { log_of pair-a; fail "the second installer: exit $code"; } [ "$(PGDATABASE=evo_restore_pair psql -XAtc "SELECT state FROM evolution_restore.claims")" = published ] && [ "$(PGDATABASE=evo_restore_pair psql -XAtc 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 300')" = "$backup_tick" ] || fail "the losing restore disturbed the winner" # After install and before publication the world is unservable, durably: across a # database restart (every connection and lock lost) acquisition is still refused, and # the restore then verifies and publishes; afterwards the build is current again. createdb evo_restore_gate restore evo_restore_gate "$work/gate" EVO_RESTORE_TEST_HOOK=' if [ "$1" = after-install ]; then psql -XAt -c "SELECT * FROM evolution.acquire_region_lease(300, 1, 20000, decode(repeat('"'"'b1'"'"', 32), '"'"'hex'"'"'))" > "'"$work"'/gate-before.log" 2>&1 || true pg_ctl -D "$PGHOST/data" -m fast -w restart > /dev/null psql -XAt -c "SELECT * FROM evolution.acquire_region_lease(300, 1, 20000, decode(repeat('"'"'b1'"'"', 32), '"'"'hex'"'"'))" > "'"$work"'/gate-after.log" 2>&1 || true fi' [ "$rc" = 0 ] && grep -q '^RESTORE VERIFIED regions=1 ' "$work/gate.log" || { log_of gate; fail "a restore across a restart: exit $rc"; } grep -q 'build not current' "$work/gate-before.log" && grep -q 'build not current' "$work/gate-after.log" || { cat "$work/gate-before.log" "$work/gate-after.log" >&2; fail "authority was available before publication"; } [ "$(PGDATABASE=evo_restore_gate psql -XAtc 'SELECT count(*) FROM evolution.retained_builds WHERE retired_at IS NULL')" = 1 ] || fail "publication did not restore the current build" # A committed install whose completion is lost is resolved from the claim and # verified, not reported as unchanged. createdb evo_restore_lost restore evo_restore_lost "$work/lost" EVO_RESTORE_TEST_INSTALL_ACK=lost [ "$rc" = 0 ] && grep -q '^install outcome resolved: committed' "$work/lost.log" && grep -q '^RESTORE VERIFIED regions=1 ' "$work/lost.log" || { log_of lost; fail "a lost install acknowledgement: exit $rc"; } # An install whose outcome cannot be resolved keeps everything, stays unservable, # and is removed only by --abandon with its claim. createdb evo_restore_unknown restore evo_restore_unknown "$work/unknown" EVO_RESTORE_TEST_INSTALL_ACK=unreachable nonce="$(sed -n 's/^restore claim \([0-9a-f]\{32\}\)$/\1/p' "$work/unknown.log")" [ "$rc" = 5 ] && grep -q '^RESTORE OUTCOME UNKNOWN' "$work/unknown.log" && [ -d "$work/unknown" ] && [ -n "$nonce" ] || { log_of unknown; fail "an unresolvable install: exit $rc"; } [ "$(schemas evo_restore_unknown)" = 2 ] && acquire_refused evo_restore_unknown || fail "an unresolved restore is servable" code=0 PGDATABASE=evo_restore_unknown bash "$root/tools/backup/restore-world.sh" --abandon 00000000000000000000000000000000 \ --objects "$work/unknown" > "$work/abandon-wrong.log" 2>&1 || code=$? [ "$(schemas evo_restore_unknown)" = 2 ] && [ -d "$work/unknown" ] || fail "--abandon with another claim removed state" PGDATABASE=evo_restore_unknown bash "$root/tools/backup/restore-world.sh" --abandon "$nonce" \ --objects "$work/unknown" > "$work/abandon.log" 2>&1 || { cat "$work/abandon.log" >&2; fail "--abandon"; } [ "$(schemas evo_restore_unknown)" = 0 ] && [ ! -e "$work/unknown" ] || fail "--abandon left state behind" restore evo_restore_unknown "$work/unknown-retry" [ "$rc" = 0 ] || { log_of unknown-retry; fail "a restore after --abandon: exit $rc"; } # An existing objects path is never claimed or removed. mkdir "$work/occupied"; echo keep > "$work/occupied/mine" createdb evo_restore_occupied restore evo_restore_occupied "$work/occupied" [ "$rc" = 4 ] && [ "$(cat "$work/occupied/mine")" = keep ] && [ "$(schemas evo_restore_occupied)" = 0 ] || fail "an occupied objects path: exit $rc" echo "backup and restore qualification passed (backup tick $backup_tick, acknowledged $acked_before, source $source_tick)"