#!/usr/bin/env bash # M8.7 restore test on independent storage (ADR-0074). # # tools/backup/verify-backup.sh BUNDLE [--isolate WORLD [--source-world W]] [--keep] # # Starts a private, throwaway PostgreSQL cluster (initdb/pg_ctl/psql/pg_restore of # the pinned server version on PATH) on local disk, with no connection to the # source database or cluster, then: # 1. restores BUNDLE with tools/backup/restore-world.sh as a fresh login that owns # only the restore database (no source role, credential or key is used); # 2. with --isolate WORLD, rebuilds a runnable isolated world: a separate database # with its own owner, runtime group and login (migrated by migrate-postgres.sh), # each region of the source world cloned into WORLD from the restored copy # (evo-world-clone clone, source history audited first), and evo-server run on # each cloned region under synthetic load until it acknowledges 20 ticks past # the clone, then drained (exit 0). # The cluster and its files are removed on exit unless --keep. Binaries come from # EVO_WORLD_CLONE_BIN / EVO_SERVER_BIN or PATH. Port: EVO_VERIFY_PORT (55499). set -euo pipefail usage() { echo "usage: $0 BUNDLE [--isolate WORLD [--source-world W]] [--keep]" >&2; exit 2; } [ $# -ge 1 ] || usage bundle="$(realpath -- "$1")"; shift isolate="" source_world="" keep=0 while [ $# -gt 0 ]; do case "$1" in --isolate) isolate="${2:-}"; shift 2 ;; --source-world) source_world="${2:-}"; shift 2 ;; --keep) keep=1; shift ;; *) usage ;; esac done [[ -z "$isolate" || "$isolate" =~ ^[1-9][0-9]*$ ]] || usage root="$(cd "$(dirname "$0")/../.." && pwd)" clone_bin="${EVO_WORLD_CLONE_BIN:-evo-world-clone}" server_bin="${EVO_SERVER_BIN:-evo-server}" port="${EVO_VERIFY_PORT:-55499}" say() { echo "$(date -u +%H:%M:%S) $*"; } cluster="$(mktemp -d /tmp/evo-verify.XXXXXXXX)" chmod 700 "$cluster" server_pid="" finish() { [ -n "$server_pid" ] && kill -9 "$server_pid" 2> /dev/null pg_ctl -D "$cluster/data" -m immediate stop > /dev/null 2>&1 || true if [ "$keep" = 1 ]; then say "kept $cluster"; else rm -rf -- "$cluster"; fi } trap finish EXIT unset PGSERVICE PGSERVICEFILE PGPASSFILE PGHOSTADDR PGPASSWORD EVO_PGPASS_SOURCE export PGHOST="$cluster" PGPORT="$port" initdb -D "$cluster/data" -U evo_verify_admin --no-locale --encoding=UTF8 \ --auth-local=trust --auth-host=reject > "$cluster/initdb.log" pg_ctl -D "$cluster/data" -l "$cluster/server.log" -w \ -o "-k $cluster -p $port -c listen_addresses=''" start > /dev/null admin() { PGUSER=evo_verify_admin PGDATABASE=postgres psql -X -q -v ON_ERROR_STOP=1 "$@"; } admin -c 'CREATE ROLE evo_restore_owner LOGIN' \ -c 'CREATE DATABASE evo_restored OWNER evo_restore_owner' say "restoring $bundle on independent storage ($cluster)" PGUSER=evo_restore_owner PGDATABASE=evo_restored \ bash "$root/tools/backup/restore-world.sh" --bundle "$bundle" --objects "$cluster/restored" [ -n "$isolate" ] || { say "PASS restore verified"; exit 0; } # The source world to rebuild: the only one in the bundle, or --source-world. worlds="$(grep '^head ' "$bundle/MANIFEST" | cut -d' ' -f2 | sort -u)" if [ -z "$source_world" ]; then [ "$(wc -l <<< "$worlds")" = 1 ] || { echo "bundle holds several worlds; pass --source-world" >&2; exit 2; } source_world="$worlds" fi grep -qx "$source_world" <<< "$worlds" || { echo "world $source_world is not in the bundle" >&2; exit 2; } [ "$source_world" != "$isolate" ] || { echo "the isolated world must differ from the source world" >&2; exit 2; } admin -c 'CREATE ROLE evo_isolated_owner LOGIN' -c 'CREATE ROLE evo_isolated_runtime NOLOGIN' \ -c 'CREATE ROLE evo_isolated_login LOGIN IN ROLE evo_isolated_runtime' \ -c 'CREATE DATABASE evo_isolated OWNER evo_isolated_owner' PGUSER=evo_isolated_owner PGDATABASE=evo_isolated EVO_PG_RUNTIME_ROLE=evo_isolated_runtime \ bash "$root/tools/migrate-postgres.sh" > /dev/null retained=() while read -r _ build _; do retained+=(--retained "$build") PGUSER=evo_isolated_owner PGDATABASE=evo_isolated psql -X -q -v ON_ERROR_STOP=1 -c \ "INSERT INTO evolution.retained_builds(build_identity) VALUES (decode('$build', 'hex'))" done < <(grep '^build ' "$bundle/MANIFEST") mkdir -m 700 "$cluster/isolated" local_pg() { # local_pg echo "$1host $cluster $1port $port $1database $2 $1user $3" } while read -r _ world region frame _ tick _ _ build; do [ "$world" = "$source_world" ] || continue # shellcheck disable=SC2046 # word splitting of the fixed connection flags is intended. cloned="$("$clone_bin" clone --source-world "$world" --source-region "$region" \ --target-world "$isolate" --target-region "$region" \ $(local_pg --source-pg- evo_restored evo_restore_owner) \ $(local_pg --target-pg- evo_isolated evo_isolated_owner) \ --source-objects "$cluster/restored" --target-objects "$cluster/isolated" \ --audit-max 1048576 "${retained[@]}")" say "$cloned" log="$cluster/server-$region.log" "$server_bin" --profile lineage --world "$isolate" --region "$region" --frame "$frame" \ $(local_pg --pg- evo_isolated evo_isolated_login) --objects "$cluster/isolated" \ --build "$build" "${retained[@]}" --flush-ms 500 --drain-ms 20000 \ --synthetic-inputs development-v1 < /dev/null > "$log" 2>&1 & server_pid=$! target=$((tick + 20)) served=0 for _ in $(seq 1 600); do if awk -v t="$target" '$1 == "ack" && $3 >= t {found = 1} END {exit !found}' "$log"; then served=1; break; fi kill -0 "$server_pid" 2> /dev/null || break sleep 0.1 done running="$(grep -m1 '^phase running' "$log" || true)" kill -TERM "$server_pid" 2> /dev/null || true code=0; wait "$server_pid" || code=$? server_pid="" [ "$served" = 1 ] && [ "$code" = 0 ] || { tail -n 20 "$log" >&2; echo "FAIL isolated $isolate/$region did not serve (exit $code)" >&2; exit 3; } [[ "$running" == "phase running epoch=1 committed=$tick "* ]] || { echo "FAIL isolated $isolate/$region resumed at '$running', not the backup head $tick" >&2; exit 3; } say "isolated $isolate/$region served from tick $tick to $(awk '$1 == "ack" {t = $3} END {print t}' "$log") and drained" done < <(grep '^head ' "$bundle/MANIFEST") say "PASS restore verified and world $source_world rebuilt as isolated world $isolate"