Online, snapshot-consistent bundles (pg_dump at an exported snapshot, immutable objects verified by digest, MANIFEST of versions/builds/heads, SHA256SUMS); a fail-closed restore that audits every region's full history on independent storage; an isolated rebuild through clones that serves under load; an operator-run in-cluster backup streamed off the cluster. backup-check.sh runs in every PostgreSQL build, with corruption, missing-object, tampered-manifest and non-empty-target refusals. Destination, encryption, schedule and WAL archiving remain on the backlog. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
121 lines
6.2 KiB
Bash
121 lines
6.2 KiB
Bash
#!/usr/bin/env bash
|
|
# M8.7 restore test on independent storage (ADR-0074).
|
|
#
|
|
# tools/backup/verify-backup.sh BUNDLE [--isolate WORLD [--source-world W]] [--keep]
|
|
#
|
|
# Starts a private, throwaway PostgreSQL cluster (initdb/pg_ctl/psql/pg_restore of
|
|
# the pinned server version on PATH) on local disk, with no connection to the
|
|
# source database or cluster, then:
|
|
# 1. restores BUNDLE with tools/backup/restore-world.sh as a fresh login that owns
|
|
# only the restore database (no source role, credential or key is used);
|
|
# 2. with --isolate WORLD, rebuilds a runnable isolated world: a separate database
|
|
# with its own owner, runtime group and login (migrated by migrate-postgres.sh),
|
|
# each region of the source world cloned into WORLD from the restored copy
|
|
# (evo-world-clone clone, source history audited first), and evo-server run on
|
|
# each cloned region under synthetic load until it acknowledges 20 ticks past
|
|
# the clone, then drained (exit 0).
|
|
# The cluster and its files are removed on exit unless --keep. Binaries come from
|
|
# EVO_WORLD_CLONE_BIN / EVO_SERVER_BIN or PATH. Port: EVO_VERIFY_PORT (55499).
|
|
set -euo pipefail
|
|
usage() { echo "usage: $0 BUNDLE [--isolate WORLD [--source-world W]] [--keep]" >&2; exit 2; }
|
|
[ $# -ge 1 ] || usage
|
|
bundle="$(realpath -- "$1")"; shift
|
|
isolate="" source_world="" keep=0
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--isolate) isolate="${2:-}"; shift 2 ;;
|
|
--source-world) source_world="${2:-}"; shift 2 ;;
|
|
--keep) keep=1; shift ;;
|
|
*) usage ;;
|
|
esac
|
|
done
|
|
[[ -z "$isolate" || "$isolate" =~ ^[1-9][0-9]*$ ]] || usage
|
|
root="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
clone_bin="${EVO_WORLD_CLONE_BIN:-evo-world-clone}"
|
|
server_bin="${EVO_SERVER_BIN:-evo-server}"
|
|
port="${EVO_VERIFY_PORT:-55499}"
|
|
say() { echo "$(date -u +%H:%M:%S) $*"; }
|
|
|
|
cluster="$(mktemp -d /tmp/evo-verify.XXXXXXXX)"
|
|
chmod 700 "$cluster"
|
|
server_pid=""
|
|
finish() {
|
|
[ -n "$server_pid" ] && kill -9 "$server_pid" 2> /dev/null
|
|
pg_ctl -D "$cluster/data" -m immediate stop > /dev/null 2>&1 || true
|
|
if [ "$keep" = 1 ]; then say "kept $cluster"; else rm -rf -- "$cluster"; fi
|
|
}
|
|
trap finish EXIT
|
|
unset PGSERVICE PGSERVICEFILE PGPASSFILE PGHOSTADDR PGPASSWORD EVO_PGPASS_SOURCE
|
|
export PGHOST="$cluster" PGPORT="$port"
|
|
initdb -D "$cluster/data" -U evo_verify_admin --no-locale --encoding=UTF8 \
|
|
--auth-local=trust --auth-host=reject > "$cluster/initdb.log"
|
|
pg_ctl -D "$cluster/data" -l "$cluster/server.log" -w \
|
|
-o "-k $cluster -p $port -c listen_addresses=''" start > /dev/null
|
|
admin() { PGUSER=evo_verify_admin PGDATABASE=postgres psql -X -q -v ON_ERROR_STOP=1 "$@"; }
|
|
admin -c 'CREATE ROLE evo_restore_owner LOGIN' \
|
|
-c 'CREATE DATABASE evo_restored OWNER evo_restore_owner'
|
|
|
|
say "restoring $bundle on independent storage ($cluster)"
|
|
PGUSER=evo_restore_owner PGDATABASE=evo_restored \
|
|
bash "$root/tools/backup/restore-world.sh" --bundle "$bundle" --objects "$cluster/restored"
|
|
[ -n "$isolate" ] || { say "PASS restore verified"; exit 0; }
|
|
|
|
# The source world to rebuild: the only one in the bundle, or --source-world.
|
|
worlds="$(grep '^head ' "$bundle/MANIFEST" | cut -d' ' -f2 | sort -u)"
|
|
if [ -z "$source_world" ]; then
|
|
[ "$(wc -l <<< "$worlds")" = 1 ] || { echo "bundle holds several worlds; pass --source-world" >&2; exit 2; }
|
|
source_world="$worlds"
|
|
fi
|
|
grep -qx "$source_world" <<< "$worlds" || { echo "world $source_world is not in the bundle" >&2; exit 2; }
|
|
[ "$source_world" != "$isolate" ] || { echo "the isolated world must differ from the source world" >&2; exit 2; }
|
|
|
|
admin -c 'CREATE ROLE evo_isolated_owner LOGIN' -c 'CREATE ROLE evo_isolated_runtime NOLOGIN' \
|
|
-c 'CREATE ROLE evo_isolated_login LOGIN IN ROLE evo_isolated_runtime' \
|
|
-c 'CREATE DATABASE evo_isolated OWNER evo_isolated_owner'
|
|
PGUSER=evo_isolated_owner PGDATABASE=evo_isolated EVO_PG_RUNTIME_ROLE=evo_isolated_runtime \
|
|
bash "$root/tools/migrate-postgres.sh" > /dev/null
|
|
retained=()
|
|
while read -r _ build _; do
|
|
retained+=(--retained "$build")
|
|
PGUSER=evo_isolated_owner PGDATABASE=evo_isolated psql -X -q -v ON_ERROR_STOP=1 -c \
|
|
"INSERT INTO evolution.retained_builds(build_identity) VALUES (decode('$build', 'hex'))"
|
|
done < <(grep '^build ' "$bundle/MANIFEST")
|
|
mkdir -m 700 "$cluster/isolated"
|
|
local_pg() { # local_pg <prefix> <database> <user>
|
|
echo "$1host $cluster $1port $port $1database $2 $1user $3"
|
|
}
|
|
|
|
while read -r _ world region frame _ tick _ _ build; do
|
|
[ "$world" = "$source_world" ] || continue
|
|
# shellcheck disable=SC2046 # word splitting of the fixed connection flags is intended.
|
|
cloned="$("$clone_bin" clone --source-world "$world" --source-region "$region" \
|
|
--target-world "$isolate" --target-region "$region" \
|
|
$(local_pg --source-pg- evo_restored evo_restore_owner) \
|
|
$(local_pg --target-pg- evo_isolated evo_isolated_owner) \
|
|
--source-objects "$cluster/restored" --target-objects "$cluster/isolated" \
|
|
--audit-max 1048576 "${retained[@]}")"
|
|
say "$cloned"
|
|
log="$cluster/server-$region.log"
|
|
"$server_bin" --profile lineage --world "$isolate" --region "$region" --frame "$frame" \
|
|
$(local_pg --pg- evo_isolated evo_isolated_login) --objects "$cluster/isolated" \
|
|
--build "$build" "${retained[@]}" --flush-ms 500 --drain-ms 20000 \
|
|
--synthetic-inputs development-v1 < /dev/null > "$log" 2>&1 &
|
|
server_pid=$!
|
|
target=$((tick + 20)) served=0
|
|
for _ in $(seq 1 600); do
|
|
if awk -v t="$target" '$1 == "ack" && $3 >= t {found = 1} END {exit !found}' "$log"; then served=1; break; fi
|
|
kill -0 "$server_pid" 2> /dev/null || break
|
|
sleep 0.1
|
|
done
|
|
running="$(grep -m1 '^phase running' "$log" || true)"
|
|
kill -TERM "$server_pid" 2> /dev/null || true
|
|
code=0; wait "$server_pid" || code=$?
|
|
server_pid=""
|
|
[ "$served" = 1 ] && [ "$code" = 0 ] ||
|
|
{ tail -n 20 "$log" >&2; echo "FAIL isolated $isolate/$region did not serve (exit $code)" >&2; exit 3; }
|
|
[[ "$running" == "phase running epoch=1 committed=$tick "* ]] ||
|
|
{ echo "FAIL isolated $isolate/$region resumed at '$running', not the backup head $tick" >&2; exit 3; }
|
|
say "isolated $isolate/$region served from tick $tick to $(awk '$1 == "ack" {t = $3} END {print t}' "$log") and drained"
|
|
done < <(grep '^head ' "$bundle/MANIFEST")
|
|
say "PASS restore verified and world $source_world rebuilt as isolated world $isolate"
|