Online, snapshot-consistent bundles (pg_dump at an exported snapshot, immutable objects verified by digest, MANIFEST of versions/builds/heads, SHA256SUMS); a fail-closed restore that audits every region's full history on independent storage; an isolated rebuild through clones that serves under load; an operator-run in-cluster backup streamed off the cluster. backup-check.sh runs in every PostgreSQL build, with corruption, missing-object, tampered-manifest and non-empty-target refusals. Destination, encryption, schedule and WAL archiving remain on the backlog. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
94 lines
6.1 KiB
Bash
94 lines
6.1 KiB
Bash
#!/usr/bin/env bash
|
|
# M8.7 backup and restore qualification (ADR-0074). Run by postgres-check.sh inside
|
|
# its throwaway cluster (PG* already set; EVO_SERVER_BIN, EVO_WORLD_CLONE_BIN and the
|
|
# runtime roles exported). A real evo-server commits under synthetic load while the
|
|
# backup is taken; the bundle is then restored on a separate cluster and rebuilt as a
|
|
# runnable isolated world, and damaged bundles are refused.
|
|
set -euo pipefail
|
|
root="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
work="$(mktemp -d /tmp/evo-backup-check.XXXXXXXX)"
|
|
chmod 700 "$work"
|
|
server_pid=""
|
|
trap '[ -n "$server_pid" ] && kill -9 "$server_pid" 2> /dev/null; rm -rf -- "$work"' EXIT
|
|
fail() { echo "backup check FAILED: $*" >&2; exit 1; }
|
|
build=b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1
|
|
createdb evo_backup_src
|
|
export PGDATABASE=evo_backup_src
|
|
bash "$root/tools/migrate-postgres.sh" > /dev/null
|
|
mkdir -m 700 "$work/objects"
|
|
pg=(--pg-host "$PGHOST" --pg-port "$PGPORT" --pg-database evo_backup_src)
|
|
"$EVO_WORLD_CLONE_BIN" provision --world 300 --region 1 --opening development-v1 \
|
|
"${pg[@]}" --pg-user "$PGUSER" --objects "$work/objects" --build "$build" --retain-build true > /dev/null
|
|
"$EVO_SERVER_BIN" --profile lineage --world 300 --region 1 --frame 9 "${pg[@]}" \
|
|
--pg-user "$EVO_PG_RUNTIME_USER" --objects "$work/objects" --build "$build" \
|
|
--flush-ms 200 --lease-profile fast --synthetic-inputs development-v1 \
|
|
< /dev/null > "$work/server.log" 2>&1 &
|
|
server_pid=$!
|
|
acked_through() { awk -v t="$1" '$1 == "ack" && $3 >= t {f = 1} END {exit !f}' "$work/server.log"; }
|
|
max_ack() { awk '$1 == "ack" && $3 > m {m = $3} END {print m + 0}' "$work/server.log"; }
|
|
for _ in $(seq 1 300); do acked_through 60 && break; sleep 0.1; done
|
|
acked_through 60 || fail "the source server acknowledged nothing"
|
|
|
|
# Online backup while the server keeps committing.
|
|
acked_before="$(max_ack)"
|
|
bash "$root/tools/backup/backup-world.sh" --objects "$work/objects" --out "$work/bundle" --label check
|
|
read -r _ _ _ _ _ backup_tick _ < <(grep '^head 300 1 ' "$work/bundle/MANIFEST")
|
|
[ "$backup_tick" -ge "$acked_before" ] || fail "backup head $backup_tick misses acknowledged tick $acked_before"
|
|
for _ in $(seq 1 300); do acked_through $((backup_tick + 40)) && break; sleep 0.1; done
|
|
kill -TERM "$server_pid"; code=0; wait "$server_pid" || code=$?; server_pid=""
|
|
[ "$code" = 0 ] || fail "the source server exited $code"
|
|
source_tick="$(psql -XAtc 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 300')"
|
|
[ "$source_tick" -gt "$backup_tick" ] || fail "the source did not move past the backup ($source_tick)"
|
|
grep -q '^build b1b1' "$work/bundle/MANIFEST" && grep -q '^schema 9 ' "$work/bundle/MANIFEST" ||
|
|
fail "manifest lacks builds or schema versions"
|
|
if grep -q -i -E 'password|pgpass' "$work/bundle/MANIFEST"; then fail "manifest names credentials"; fi
|
|
|
|
# Full restore on an independent cluster, rebuilt and served as isolated world 301.
|
|
EVO_VERIFY_PORT=$((PGPORT + 1)) bash "$root/tools/backup/verify-backup.sh" "$work/bundle" \
|
|
--isolate 301 > "$work/verify.log" 2>&1 || { cat "$work/verify.log" >&2; fail "verify-backup"; }
|
|
grep -q "^RESTORE VERIFIED regions=1 " "$work/verify.log" &&
|
|
grep -q "PASS restore verified and world 300 rebuilt as isolated world 301" "$work/verify.log" ||
|
|
{ cat "$work/verify.log" >&2; fail "verify-backup evidence"; }
|
|
|
|
# Damaged bundles, restored straight into scratch databases of this cluster.
|
|
case_n=0
|
|
expect_refusal() { # expect_refusal <exit code> <bundle> <what>
|
|
case_n=$((case_n + 1))
|
|
createdb "evo_restore_case_$case_n"
|
|
local code=0
|
|
PGDATABASE="evo_restore_case_$case_n" bash "$root/tools/backup/restore-world.sh" \
|
|
--bundle "$2" --objects "$work/restored-$case_n" > "$work/case-$case_n.log" 2>&1 || code=$?
|
|
[ "$code" = "$1" ] || { cat "$work/case-$case_n.log" >&2; fail "$3: exit $code, expected $1"; }
|
|
[ ! -e "$work/restored-$case_n" ] || fail "$3: restored objects were left behind"
|
|
[ "$(PGDATABASE="evo_restore_case_$case_n" psql -XAtc "SELECT count(*) FROM pg_namespace WHERE nspname = 'evolution'")" = 0 ] ||
|
|
fail "$3: a restored schema was left behind"
|
|
}
|
|
damaged() { rm -rf "$work/damaged"; cp -a "$work/bundle" "$work/damaged"; }
|
|
resum() { (cd "$work/damaged" && find . -type f ! -name SHA256SUMS -printf '%P\0' | sort -z | xargs -0 sha256sum -- > SHA256SUMS); }
|
|
genesis="$(psql -XAtc "SELECT encode(payload_sha256, 'hex') FROM evolution.region_commits WHERE world_id = 300 AND commit_sequence = 0")"
|
|
genesis_object="$work/damaged/objects/w300/r1/$genesis"
|
|
|
|
damaged; printf 'corrupt' | dd of="$genesis_object" bs=1 conv=notrunc status=none
|
|
expect_refusal 4 "$work/damaged" "a corrupt object"
|
|
damaged; echo extra > "$work/damaged/objects/w300/r1/notes"
|
|
expect_refusal 4 "$work/damaged" "an unlisted file"
|
|
damaged; printf 'x' >> "$work/damaged/database.dump"
|
|
expect_refusal 4 "$work/damaged" "a damaged dump"
|
|
# Consistently re-summed damage passes the checksums and must fail verification.
|
|
damaged; rm "$genesis_object"; resum
|
|
expect_refusal 3 "$work/damaged" "a missing payload object"
|
|
grep -q 'audit failed for 300/1' "$work/case-$case_n.log" || fail "missing object: no audit failure"
|
|
damaged; sed -i -E "s/^(head 300 1 [0-9]+ [0-9]+) $backup_tick /\1 $((backup_tick - 1)) /" "$work/damaged/MANIFEST"; resum
|
|
expect_refusal 3 "$work/damaged" "a manifest head that disagrees with the database"
|
|
# A target that already holds a world is refused before anything is written.
|
|
createdb evo_restore_twice
|
|
PGDATABASE=evo_restore_twice bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" \
|
|
--objects "$work/twice-a" > /dev/null
|
|
code=0
|
|
PGDATABASE=evo_restore_twice bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" \
|
|
--objects "$work/twice-b" > "$work/twice.log" 2>&1 || code=$?
|
|
[ "$code" = 4 ] && grep -q 'already has an evolution schema' "$work/twice.log" || fail "a second restore into one database"
|
|
[ "$(PGDATABASE=evo_restore_twice psql -XAtc 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 300')" = "$backup_tick" ] ||
|
|
fail "the refused second restore changed the first"
|
|
echo "backup and restore qualification passed (backup tick $backup_tick, acknowledged $acked_before, source $source_tick)"
|