Files
aevolution/tools/ci/backup-check.sh
T
dingoandClaude Opus 5.5 5512fdec54 feat(backup): M8.7 world backup bundles and verified restore (ADR-0074)
Online, snapshot-consistent bundles (pg_dump at an exported snapshot, immutable
objects verified by digest, MANIFEST of versions/builds/heads, SHA256SUMS); a
fail-closed restore that audits every region's full history on independent
storage; an isolated rebuild through clones that serves under load; an
operator-run in-cluster backup streamed off the cluster. backup-check.sh runs in
every PostgreSQL build, with corruption, missing-object, tampered-manifest and
non-empty-target refusals. Destination, encryption, schedule and WAL archiving
remain on the backlog.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 15:51:04 -04:00

94 lines
6.1 KiB
Bash

#!/usr/bin/env bash
# M8.7 backup and restore qualification (ADR-0074). Run by postgres-check.sh inside
# its throwaway cluster (PG* already set; EVO_SERVER_BIN, EVO_WORLD_CLONE_BIN and the
# runtime roles exported). A real evo-server commits under synthetic load while the
# backup is taken; the bundle is then restored on a separate cluster and rebuilt as a
# runnable isolated world, and damaged bundles are refused.
set -euo pipefail
root="$(cd "$(dirname "$0")/../.." && pwd)"
work="$(mktemp -d /tmp/evo-backup-check.XXXXXXXX)"
chmod 700 "$work"
server_pid=""
trap '[ -n "$server_pid" ] && kill -9 "$server_pid" 2> /dev/null; rm -rf -- "$work"' EXIT
fail() { echo "backup check FAILED: $*" >&2; exit 1; }
build=b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1
createdb evo_backup_src
export PGDATABASE=evo_backup_src
bash "$root/tools/migrate-postgres.sh" > /dev/null
mkdir -m 700 "$work/objects"
pg=(--pg-host "$PGHOST" --pg-port "$PGPORT" --pg-database evo_backup_src)
"$EVO_WORLD_CLONE_BIN" provision --world 300 --region 1 --opening development-v1 \
"${pg[@]}" --pg-user "$PGUSER" --objects "$work/objects" --build "$build" --retain-build true > /dev/null
"$EVO_SERVER_BIN" --profile lineage --world 300 --region 1 --frame 9 "${pg[@]}" \
--pg-user "$EVO_PG_RUNTIME_USER" --objects "$work/objects" --build "$build" \
--flush-ms 200 --lease-profile fast --synthetic-inputs development-v1 \
< /dev/null > "$work/server.log" 2>&1 &
server_pid=$!
acked_through() { awk -v t="$1" '$1 == "ack" && $3 >= t {f = 1} END {exit !f}' "$work/server.log"; }
max_ack() { awk '$1 == "ack" && $3 > m {m = $3} END {print m + 0}' "$work/server.log"; }
for _ in $(seq 1 300); do acked_through 60 && break; sleep 0.1; done
acked_through 60 || fail "the source server acknowledged nothing"
# Online backup while the server keeps committing.
acked_before="$(max_ack)"
bash "$root/tools/backup/backup-world.sh" --objects "$work/objects" --out "$work/bundle" --label check
read -r _ _ _ _ _ backup_tick _ < <(grep '^head 300 1 ' "$work/bundle/MANIFEST")
[ "$backup_tick" -ge "$acked_before" ] || fail "backup head $backup_tick misses acknowledged tick $acked_before"
for _ in $(seq 1 300); do acked_through $((backup_tick + 40)) && break; sleep 0.1; done
kill -TERM "$server_pid"; code=0; wait "$server_pid" || code=$?; server_pid=""
[ "$code" = 0 ] || fail "the source server exited $code"
source_tick="$(psql -XAtc 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 300')"
[ "$source_tick" -gt "$backup_tick" ] || fail "the source did not move past the backup ($source_tick)"
grep -q '^build b1b1' "$work/bundle/MANIFEST" && grep -q '^schema 9 ' "$work/bundle/MANIFEST" ||
fail "manifest lacks builds or schema versions"
if grep -q -i -E 'password|pgpass' "$work/bundle/MANIFEST"; then fail "manifest names credentials"; fi
# Full restore on an independent cluster, rebuilt and served as isolated world 301.
EVO_VERIFY_PORT=$((PGPORT + 1)) bash "$root/tools/backup/verify-backup.sh" "$work/bundle" \
--isolate 301 > "$work/verify.log" 2>&1 || { cat "$work/verify.log" >&2; fail "verify-backup"; }
grep -q "^RESTORE VERIFIED regions=1 " "$work/verify.log" &&
grep -q "PASS restore verified and world 300 rebuilt as isolated world 301" "$work/verify.log" ||
{ cat "$work/verify.log" >&2; fail "verify-backup evidence"; }
# Damaged bundles, restored straight into scratch databases of this cluster.
case_n=0
expect_refusal() { # expect_refusal <exit code> <bundle> <what>
case_n=$((case_n + 1))
createdb "evo_restore_case_$case_n"
local code=0
PGDATABASE="evo_restore_case_$case_n" bash "$root/tools/backup/restore-world.sh" \
--bundle "$2" --objects "$work/restored-$case_n" > "$work/case-$case_n.log" 2>&1 || code=$?
[ "$code" = "$1" ] || { cat "$work/case-$case_n.log" >&2; fail "$3: exit $code, expected $1"; }
[ ! -e "$work/restored-$case_n" ] || fail "$3: restored objects were left behind"
[ "$(PGDATABASE="evo_restore_case_$case_n" psql -XAtc "SELECT count(*) FROM pg_namespace WHERE nspname = 'evolution'")" = 0 ] ||
fail "$3: a restored schema was left behind"
}
damaged() { rm -rf "$work/damaged"; cp -a "$work/bundle" "$work/damaged"; }
resum() { (cd "$work/damaged" && find . -type f ! -name SHA256SUMS -printf '%P\0' | sort -z | xargs -0 sha256sum -- > SHA256SUMS); }
genesis="$(psql -XAtc "SELECT encode(payload_sha256, 'hex') FROM evolution.region_commits WHERE world_id = 300 AND commit_sequence = 0")"
genesis_object="$work/damaged/objects/w300/r1/$genesis"
damaged; printf 'corrupt' | dd of="$genesis_object" bs=1 conv=notrunc status=none
expect_refusal 4 "$work/damaged" "a corrupt object"
damaged; echo extra > "$work/damaged/objects/w300/r1/notes"
expect_refusal 4 "$work/damaged" "an unlisted file"
damaged; printf 'x' >> "$work/damaged/database.dump"
expect_refusal 4 "$work/damaged" "a damaged dump"
# Consistently re-summed damage passes the checksums and must fail verification.
damaged; rm "$genesis_object"; resum
expect_refusal 3 "$work/damaged" "a missing payload object"
grep -q 'audit failed for 300/1' "$work/case-$case_n.log" || fail "missing object: no audit failure"
damaged; sed -i -E "s/^(head 300 1 [0-9]+ [0-9]+) $backup_tick /\1 $((backup_tick - 1)) /" "$work/damaged/MANIFEST"; resum
expect_refusal 3 "$work/damaged" "a manifest head that disagrees with the database"
# A target that already holds a world is refused before anything is written.
createdb evo_restore_twice
PGDATABASE=evo_restore_twice bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" \
--objects "$work/twice-a" > /dev/null
code=0
PGDATABASE=evo_restore_twice bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" \
--objects "$work/twice-b" > "$work/twice.log" 2>&1 || code=$?
[ "$code" = 4 ] && grep -q 'already has an evolution schema' "$work/twice.log" || fail "a second restore into one database"
[ "$(PGDATABASE=evo_restore_twice psql -XAtc 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 300')" = "$backup_tick" ] ||
fail "the refused second restore changed the first"
echo "backup and restore qualification passed (backup tick $backup_tick, acknowledged $acked_before, source $source_tick)"