Files
aevolution/tools/backup/verify-backup.sh
T
dingoandClaude Opus 5.5 5512fdec54 feat(backup): M8.7 world backup bundles and verified restore (ADR-0074)
Online, snapshot-consistent bundles (pg_dump at an exported snapshot, immutable
objects verified by digest, MANIFEST of versions/builds/heads, SHA256SUMS); a
fail-closed restore that audits every region's full history on independent
storage; an isolated rebuild through clones that serves under load; an
operator-run in-cluster backup streamed off the cluster. backup-check.sh runs in
every PostgreSQL build, with corruption, missing-object, tampered-manifest and
non-empty-target refusals. Destination, encryption, schedule and WAL archiving
remain on the backlog.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 15:51:04 -04:00

121 lines
6.2 KiB
Bash

#!/usr/bin/env bash
# M8.7 restore test on independent storage (ADR-0074).
#
# tools/backup/verify-backup.sh BUNDLE [--isolate WORLD [--source-world W]] [--keep]
#
# Starts a private, throwaway PostgreSQL cluster (initdb/pg_ctl/psql/pg_restore of
# the pinned server version on PATH) on local disk, with no connection to the
# source database or cluster, then:
# 1. restores BUNDLE with tools/backup/restore-world.sh as a fresh login that owns
# only the restore database (no source role, credential or key is used);
# 2. with --isolate WORLD, rebuilds a runnable isolated world: a separate database
# with its own owner, runtime group and login (migrated by migrate-postgres.sh),
# each region of the source world cloned into WORLD from the restored copy
# (evo-world-clone clone, source history audited first), and evo-server run on
# each cloned region under synthetic load until it acknowledges 20 ticks past
# the clone, then drained (exit 0).
# The cluster and its files are removed on exit unless --keep. Binaries come from
# EVO_WORLD_CLONE_BIN / EVO_SERVER_BIN or PATH. Port: EVO_VERIFY_PORT (55499).
set -euo pipefail
usage() { echo "usage: $0 BUNDLE [--isolate WORLD [--source-world W]] [--keep]" >&2; exit 2; }
[ $# -ge 1 ] || usage
bundle="$(realpath -- "$1")"; shift
isolate="" source_world="" keep=0
while [ $# -gt 0 ]; do
case "$1" in
--isolate) isolate="${2:-}"; shift 2 ;;
--source-world) source_world="${2:-}"; shift 2 ;;
--keep) keep=1; shift ;;
*) usage ;;
esac
done
[[ -z "$isolate" || "$isolate" =~ ^[1-9][0-9]*$ ]] || usage
root="$(cd "$(dirname "$0")/../.." && pwd)"
clone_bin="${EVO_WORLD_CLONE_BIN:-evo-world-clone}"
server_bin="${EVO_SERVER_BIN:-evo-server}"
port="${EVO_VERIFY_PORT:-55499}"
say() { echo "$(date -u +%H:%M:%S) $*"; }
cluster="$(mktemp -d /tmp/evo-verify.XXXXXXXX)"
chmod 700 "$cluster"
server_pid=""
finish() {
[ -n "$server_pid" ] && kill -9 "$server_pid" 2> /dev/null
pg_ctl -D "$cluster/data" -m immediate stop > /dev/null 2>&1 || true
if [ "$keep" = 1 ]; then say "kept $cluster"; else rm -rf -- "$cluster"; fi
}
trap finish EXIT
unset PGSERVICE PGSERVICEFILE PGPASSFILE PGHOSTADDR PGPASSWORD EVO_PGPASS_SOURCE
export PGHOST="$cluster" PGPORT="$port"
initdb -D "$cluster/data" -U evo_verify_admin --no-locale --encoding=UTF8 \
--auth-local=trust --auth-host=reject > "$cluster/initdb.log"
pg_ctl -D "$cluster/data" -l "$cluster/server.log" -w \
-o "-k $cluster -p $port -c listen_addresses=''" start > /dev/null
admin() { PGUSER=evo_verify_admin PGDATABASE=postgres psql -X -q -v ON_ERROR_STOP=1 "$@"; }
admin -c 'CREATE ROLE evo_restore_owner LOGIN' \
-c 'CREATE DATABASE evo_restored OWNER evo_restore_owner'
say "restoring $bundle on independent storage ($cluster)"
PGUSER=evo_restore_owner PGDATABASE=evo_restored \
bash "$root/tools/backup/restore-world.sh" --bundle "$bundle" --objects "$cluster/restored"
[ -n "$isolate" ] || { say "PASS restore verified"; exit 0; }
# The source world to rebuild: the only one in the bundle, or --source-world.
worlds="$(grep '^head ' "$bundle/MANIFEST" | cut -d' ' -f2 | sort -u)"
if [ -z "$source_world" ]; then
[ "$(wc -l <<< "$worlds")" = 1 ] || { echo "bundle holds several worlds; pass --source-world" >&2; exit 2; }
source_world="$worlds"
fi
grep -qx "$source_world" <<< "$worlds" || { echo "world $source_world is not in the bundle" >&2; exit 2; }
[ "$source_world" != "$isolate" ] || { echo "the isolated world must differ from the source world" >&2; exit 2; }
admin -c 'CREATE ROLE evo_isolated_owner LOGIN' -c 'CREATE ROLE evo_isolated_runtime NOLOGIN' \
-c 'CREATE ROLE evo_isolated_login LOGIN IN ROLE evo_isolated_runtime' \
-c 'CREATE DATABASE evo_isolated OWNER evo_isolated_owner'
PGUSER=evo_isolated_owner PGDATABASE=evo_isolated EVO_PG_RUNTIME_ROLE=evo_isolated_runtime \
bash "$root/tools/migrate-postgres.sh" > /dev/null
retained=()
while read -r _ build _; do
retained+=(--retained "$build")
PGUSER=evo_isolated_owner PGDATABASE=evo_isolated psql -X -q -v ON_ERROR_STOP=1 -c \
"INSERT INTO evolution.retained_builds(build_identity) VALUES (decode('$build', 'hex'))"
done < <(grep '^build ' "$bundle/MANIFEST")
mkdir -m 700 "$cluster/isolated"
local_pg() { # local_pg <prefix> <database> <user>
echo "$1host $cluster $1port $port $1database $2 $1user $3"
}
while read -r _ world region frame _ tick _ _ build; do
[ "$world" = "$source_world" ] || continue
# shellcheck disable=SC2046 # word splitting of the fixed connection flags is intended.
cloned="$("$clone_bin" clone --source-world "$world" --source-region "$region" \
--target-world "$isolate" --target-region "$region" \
$(local_pg --source-pg- evo_restored evo_restore_owner) \
$(local_pg --target-pg- evo_isolated evo_isolated_owner) \
--source-objects "$cluster/restored" --target-objects "$cluster/isolated" \
--audit-max 1048576 "${retained[@]}")"
say "$cloned"
log="$cluster/server-$region.log"
"$server_bin" --profile lineage --world "$isolate" --region "$region" --frame "$frame" \
$(local_pg --pg- evo_isolated evo_isolated_login) --objects "$cluster/isolated" \
--build "$build" "${retained[@]}" --flush-ms 500 --drain-ms 20000 \
--synthetic-inputs development-v1 < /dev/null > "$log" 2>&1 &
server_pid=$!
target=$((tick + 20)) served=0
for _ in $(seq 1 600); do
if awk -v t="$target" '$1 == "ack" && $3 >= t {found = 1} END {exit !found}' "$log"; then served=1; break; fi
kill -0 "$server_pid" 2> /dev/null || break
sleep 0.1
done
running="$(grep -m1 '^phase running' "$log" || true)"
kill -TERM "$server_pid" 2> /dev/null || true
code=0; wait "$server_pid" || code=$?
server_pid=""
[ "$served" = 1 ] && [ "$code" = 0 ] ||
{ tail -n 20 "$log" >&2; echo "FAIL isolated $isolate/$region did not serve (exit $code)" >&2; exit 3; }
[[ "$running" == "phase running epoch=1 committed=$tick "* ]] ||
{ echo "FAIL isolated $isolate/$region resumed at '$running', not the backup head $tick" >&2; exit 3; }
say "isolated $isolate/$region served from tick $tick to $(awk '$1 == "ack" {t = $3} END {print t}' "$log") and drained"
done < <(grep '^head ' "$bundle/MANIFEST")
say "PASS restore verified and world $source_world rebuilt as isolated world $isolate"