Files
aevolution/tools/backup/verify-backup.sh
T
dingoandClaude Opus 5.5 46c106409e
Foundation / checks (checks) (push) Successful in 5s
Foundation / checks (client) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 4s
Foundation / checks (network) (push) Successful in 4s
Foundation / checks (postgres) (push) Successful in 4s
Foundation / checks (tsan) (push) Successful in 5s
Foundation / checks (deploy) (push) Successful in 4m33s
Foundation / postgres-tools (push) Successful in 2m14s
Foundation / server-image (push) Successful in 5m21s
Foundation / image (push) Successful in 6m18s
feat(admin): session windows in SQL, opened from the admin portal (schema 62)
The dev session window moves from Git to SQL (ADR-0087 amendment): the admin
portal opens, extends (30 min to 4 h from now) and closes it, and the cellular
region re-reads it every 5 s with --session-window-source db. No push, sync or
restart per window; a closed window keeps the region running instead of
crash-looping.

- Schema 62: append-only session_windows (4 h cap, no rewrites),
  load_session_window_v1 for the runtime group, admin_set_session_window_v1
  and admin_session_windows_v1 for a dedicated admin group role, granted and
  audited each migration run when EVO_PG_ADMIN_ROLE is set (aevolution_admin in
  DEV). Schema 61's views are revoked from the runtime group.
- RegionSessions admission switch: while closed every Hello is refused before
  signature work and closing closes open sessions (new sessions were still
  admitted after expiry before).
- Admin API: GET/POST /api/regions/<n>/session-window, window state in the
  regions list; the portal's session-window card with history.
- Git: REGION_SESSION_WINDOW and the pre-push refresh rule are removed; the
  policy check requires the database source and refuses a committed window.

Needs the new image and the aevolution_admin role (with the portal's login in
aevolution-admin-pg) before a sync.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-27 15:13:02 -04:00

123 lines
6.3 KiB
Bash

#!/usr/bin/env bash
# M8.7 restore test on independent storage (ADR-0074).
#
# tools/backup/verify-backup.sh BUNDLE [--isolate WORLD [--source-world W]] [--keep]
#
# Starts a private, throwaway PostgreSQL cluster (initdb/pg_ctl/psql/pg_restore of
# the pinned server version on PATH) on local disk, with no connection to the
# source database or cluster, then:
# 1. restores BUNDLE with tools/backup/restore-world.sh as a fresh login that owns
# only the restore database (no source role, credential or key is used);
# 2. with --isolate WORLD, rebuilds a runnable isolated world: a separate database
# with its own owner, runtime group and login (migrated by migrate-postgres.sh),
# each region of the source world cloned into WORLD from the restored copy
# (evo-world-clone clone, source history audited first), and evo-server run on
# each cloned region under synthetic load until it acknowledges 20 ticks past
# the clone, then drained (exit 0).
# The cluster and its files are removed on exit unless --keep. Binaries come from
# EVO_WORLD_CLONE_BIN / EVO_SERVER_BIN or PATH. Port: EVO_VERIFY_PORT (55499).
set -euo pipefail
# The private cluster has only its own roles: no admin role grants (ADR-0182) apply here.
unset EVO_PG_ADMIN_ROLE
usage() { echo "usage: $0 BUNDLE [--isolate WORLD [--source-world W]] [--keep]" >&2; exit 2; }
[ $# -ge 1 ] || usage
bundle="$(realpath -- "$1")"; shift
isolate="" source_world="" keep=0
while [ $# -gt 0 ]; do
case "$1" in
--isolate) isolate="${2:-}"; shift 2 ;;
--source-world) source_world="${2:-}"; shift 2 ;;
--keep) keep=1; shift ;;
*) usage ;;
esac
done
[[ -z "$isolate" || "$isolate" =~ ^[1-9][0-9]*$ ]] || usage
root="$(cd "$(dirname "$0")/../.." && pwd)"
clone_bin="${EVO_WORLD_CLONE_BIN:-evo-world-clone}"
server_bin="${EVO_SERVER_BIN:-evo-server}"
port="${EVO_VERIFY_PORT:-55499}"
say() { echo "$(date -u +%H:%M:%S) $*"; }
cluster="$(mktemp -d /tmp/evo-verify.XXXXXXXX)"
chmod 700 "$cluster"
server_pid=""
finish() {
[ -n "$server_pid" ] && kill -9 "$server_pid" 2> /dev/null
pg_ctl -D "$cluster/data" -m immediate stop > /dev/null 2>&1 || true
if [ "$keep" = 1 ]; then say "kept $cluster"; else rm -rf -- "$cluster"; fi
}
trap finish EXIT
unset PGSERVICE PGSERVICEFILE PGPASSFILE PGHOSTADDR PGPASSWORD EVO_PGPASS_SOURCE
export PGHOST="$cluster" PGPORT="$port"
initdb -D "$cluster/data" -U evo_verify_admin --no-locale --encoding=UTF8 \
--auth-local=trust --auth-host=reject > "$cluster/initdb.log"
pg_ctl -D "$cluster/data" -l "$cluster/server.log" -w \
-o "-k $cluster -p $port -c listen_addresses=''" start > /dev/null
admin() { PGUSER=evo_verify_admin PGDATABASE=postgres psql -X -q -v ON_ERROR_STOP=1 "$@"; }
admin -c 'CREATE ROLE evo_restore_owner LOGIN' \
-c 'CREATE DATABASE evo_restored OWNER evo_restore_owner'
say "restoring $bundle on independent storage ($cluster)"
PGUSER=evo_restore_owner PGDATABASE=evo_restored \
bash "$root/tools/backup/restore-world.sh" --bundle "$bundle" --objects "$cluster/restored"
[ -n "$isolate" ] || { say "PASS restore verified"; exit 0; }
# The source world to rebuild: the only one in the bundle, or --source-world.
worlds="$(grep '^head ' "$bundle/MANIFEST" | cut -d' ' -f2 | sort -u)"
if [ -z "$source_world" ]; then
[ "$(wc -l <<< "$worlds")" = 1 ] || { echo "bundle holds several worlds; pass --source-world" >&2; exit 2; }
source_world="$worlds"
fi
grep -qx "$source_world" <<< "$worlds" || { echo "world $source_world is not in the bundle" >&2; exit 2; }
[ "$source_world" != "$isolate" ] || { echo "the isolated world must differ from the source world" >&2; exit 2; }
admin -c 'CREATE ROLE evo_isolated_owner LOGIN' -c 'CREATE ROLE evo_isolated_runtime NOLOGIN' \
-c 'CREATE ROLE evo_isolated_login LOGIN IN ROLE evo_isolated_runtime' \
-c 'CREATE DATABASE evo_isolated OWNER evo_isolated_owner'
PGUSER=evo_isolated_owner PGDATABASE=evo_isolated EVO_PG_RUNTIME_ROLE=evo_isolated_runtime \
bash "$root/tools/migrate-postgres.sh" > /dev/null
retained=()
while read -r _ build _; do
retained+=(--retained "$build")
PGUSER=evo_isolated_owner PGDATABASE=evo_isolated psql -X -q -v ON_ERROR_STOP=1 -c \
"INSERT INTO evolution.retained_builds(build_identity) VALUES (decode('$build', 'hex'))"
done < <(grep '^build ' "$bundle/MANIFEST")
mkdir -m 700 "$cluster/isolated"
local_pg() { # local_pg <prefix> <database> <user>
echo "$1host $cluster $1port $port $1database $2 $1user $3"
}
while read -r _ world region frame _ tick _ _ build; do
[ "$world" = "$source_world" ] || continue
# shellcheck disable=SC2046 # word splitting of the fixed connection flags is intended.
cloned="$("$clone_bin" clone --source-world "$world" --source-region "$region" \
--target-world "$isolate" --target-region "$region" \
$(local_pg --source-pg- evo_restored evo_restore_owner) \
$(local_pg --target-pg- evo_isolated evo_isolated_owner) \
--source-objects "$cluster/restored" --target-objects "$cluster/isolated" \
--audit-max 1048576 "${retained[@]}")"
say "$cloned"
log="$cluster/server-$region.log"
"$server_bin" --profile lineage --world "$isolate" --region "$region" --frame "$frame" \
$(local_pg --pg- evo_isolated evo_isolated_login) --objects "$cluster/isolated" \
--build "$build" "${retained[@]}" --flush-ms 500 --drain-ms 20000 \
--synthetic-inputs development-v1 < /dev/null > "$log" 2>&1 &
server_pid=$!
target=$((tick + 20)) served=0
for _ in $(seq 1 600); do
if awk -v t="$target" '$1 == "ack" && $3 >= t {found = 1} END {exit !found}' "$log"; then served=1; break; fi
kill -0 "$server_pid" 2> /dev/null || break
sleep 0.1
done
running="$(grep -m1 '^phase running' "$log" || true)"
kill -TERM "$server_pid" 2> /dev/null || true
code=0; wait "$server_pid" || code=$?
server_pid=""
[ "$served" = 1 ] && [ "$code" = 0 ] ||
{ tail -n 20 "$log" >&2; echo "FAIL isolated $isolate/$region did not serve (exit $code)" >&2; exit 3; }
[[ "$running" == "phase running epoch=1 committed=$tick "* ]] ||
{ echo "FAIL isolated $isolate/$region resumed at '$running', not the backup head $tick" >&2; exit 3; }
say "isolated $isolate/$region served from tick $tick to $(awk '$1 == "ack" {t = $3} END {print t}' "$log") and drained"
done < <(grep '^head ' "$bundle/MANIFEST")
say "PASS restore verified and world $source_world rebuilt as isolated world $isolate"