#!/usr/bin/env bash # M8.7 verified restore of a backup bundle (ADR-0074) onto independent storage. # # tools/backup/restore-world.sh --bundle DIR --objects DIR # [--in-place --runtime-role ROLE --accept-rewind] # tools/backup/restore-world.sh --abandon NONCE --objects DIR # # Target connection: PG* environment naming an EMPTY database that the login owns # (PGHOST/PGPORT/PGDATABASE/PGUSER, or PGSERVICE with PGSERVICEFILE/PGPASSFILE). # The restore needs no source role, credential or key: owners are not restored, and # grants only with --in-place, so the target login owns everything restored. # evo-world-clone comes from EVO_WORLD_CLONE_BIN or PATH. # # Order, failing closed at each step: # 1. every bundle file matches SHA256SUMS, no file is unlisted, and every object's # name is its digest; nothing is touched before this passes; # 2. --objects is claimed by creating it (mkdir, never -p: an existing path is # refused and never touched) and marked with this invocation's random NONCE; the # target must have neither `evolution` nor `evolution_restore` and must run the # manifest's server version; # 3. INSTALL, one transaction: the dump, a claim row evolution_restore.claims(NONCE, # unpublished), every lease fenced, every current build retired, and a fresh # history domain with parent/exact-cut provenance for schema-51 bundles. From that # commit on the world is unservable: acquisition, renewal and commits refuse # until publication, whatever happens to any connection or the server (review # M8 follow-up P1). Then the objects are copied; # 4. the restored heads equal the MANIFEST heads exactly; # 5. every region's full history is audited (evo-world-clone audit), and with # --in-place migrate-postgres.sh audits the runtime role's effective rights; # 6. PUBLISH, one transaction that first locks this NONCE's unpublished claim: # exactly the builds the install retired become current again and the claim # becomes published. A restore that lost its claim cannot publish. # A failed or interrupted install is resolved, not assumed (review M8 follow-up P2): # the claim row says whether this invocation's transaction committed. Committed -> # verification continues; absent -> rolled back (the database was not changed); # unreachable -> OUTCOME UNKNOWN (exit 5): the target stays unservable and both the # schema and the objects are kept; resolve it later with --abandon NONCE. # Cleanup only ever removes what this NONCE owns: the schemas only through a # transaction that locks this NONCE's unpublished claim, the objects directory only # while it carries this NONCE's marker. A competing restore's or provisioner's state # is never touched. The default result retains the source's biological world # identities/bytes, with a fresh history domain when supported, no runtime grants # and never served; to run it, clone it # into an isolated world (tools/backup/verify-backup.sh --isolate). # # --in-place (M8.8 host-loss recovery for dev/stage, ADR-0075) restores the world # under its own identity so its hosts can serve it again after publication: the # dump's grants are restored (the environment's roles must already exist in the # target cluster). Every acknowledgement after the bundle's cut is undone: deaths may # reverse and resources may reappear. --accept-rewind states that this is # acceptable, which it is for dev and stage only; World0 launch stays blocked until # acknowledgements survive host loss (ADR-0075). Fence the old database with # fence-database.sh before any of its hosts can return. # # Exit: 0 verified and published, 2 usage, 3 verification failed (this restore's # state removed), 4 refused or rolled back (nothing of this restore left), # 5 outcome unknown (target unservable; evidence kept). # Tests only: EVO_RESTORE_TEST_HOOK is run by bash with $1 = before-install, # after-install or before-publish; EVO_RESTORE_TEST_INSTALL_ACK=lost treats a committed # install as failed (an ambiguous completion), =unreachable also makes its resolution # unreachable. set -euo pipefail usage() { echo "usage: $0 --bundle DIR --objects DIR [--in-place --runtime-role ROLE --accept-rewind]" >&2 echo " $0 --abandon NONCE --objects DIR" >&2 exit 2 } bundle="" objects="" in_place=0 runtime_role="" accept_rewind=0 abandon="" while [ $# -gt 0 ]; do case "$1" in --bundle) bundle="${2:-}"; shift 2 ;; --objects) objects="${2:-}"; shift 2 ;; --in-place) in_place=1; shift ;; --runtime-role) runtime_role="${2:-}"; shift 2 ;; --accept-rewind) accept_rewind=1; shift ;; --abandon) abandon="${2:-}"; shift 2 ;; *) usage ;; esac done [ -n "$objects" ] || usage if [ -n "$abandon" ]; then [[ "$abandon" =~ ^[0-9a-f]{32}$ ]] && [ -z "$bundle" ] && [ "$in_place" = 0 ] || usage else [ -n "$bundle" ] || usage fi if [ "$in_place" = 1 ]; then [ -n "$runtime_role" ] || { echo "--in-place needs --runtime-role" >&2; exit 2; } [ "$accept_rewind" = 1 ] || { echo "--in-place undoes every acknowledgement after the backup; pass --accept-rewind (dev/stage only)" >&2; exit 2; } elif [ -n "$runtime_role" ] || [ "$accept_rewind" = 1 ]; then usage fi root="$(cd "$(dirname "$0")/../.." && pwd)" objects="$(realpath -m -- "$objects")" clone_bin="${EVO_WORLD_CLONE_BIN:-evo-world-clone}" umask 077 begin_ms="$(date +%s%3N)" q() { psql -X --no-password -q -A -t -v ON_ERROR_STOP=1 "$@"; } # The claim row of NONCE: none, unpublished, published, or unknown (unreachable). claim_state() { local nonce="$1" exists state exists="$(q -c "SELECT to_regclass('evolution_restore.claims') IS NOT NULL" 2> /dev/null)" || { echo unknown; return; } [ "$exists" = t ] || { echo none; return; } state="$(q -c "SELECT state FROM evolution_restore.claims WHERE nonce = '$nonce'" 2> /dev/null)" || { echo unknown; return; } echo "${state:-none}" } # Removes NONCE's unpublished restore: both schemas, in one transaction that first # locks that claim. Anything else (another claim, a published one) is refused. drop_claimed() { q -c "DO \$\$ BEGIN IF to_regclass('evolution_restore.claims') IS NULL THEN RAISE EXCEPTION 'no restore claim'; END IF; PERFORM 1 FROM evolution_restore.claims WHERE nonce = '$1' AND state = 'unpublished' FOR UPDATE; IF NOT FOUND THEN RAISE EXCEPTION 'restore claim % is not unpublished here', '$1'; END IF; DROP SCHEMA evolution CASCADE; DROP SCHEMA evolution_restore CASCADE; END \$\$" > /dev/null 2>&1 } # Removes the objects directory only while it carries NONCE's marker. drop_objects() { [ -f "$objects/.restore-claim" ] && [ "$(cat "$objects/.restore-claim")" = "$1" ] && rm -rf -- "$objects" } if [ -n "$abandon" ]; then state="$(claim_state "$abandon")" case "$state" in unpublished) drop_claimed "$abandon" || { echo "ABANDON FAILED: could not remove claim $abandon" >&2; exit 3; } ;; none) ;; published) echo "ABANDON REFUSED: claim $abandon is published (a verified world)" >&2; exit 4 ;; *) echo "ABANDON FAILED: the target database is unreachable" >&2; exit 5 ;; esac drop_objects "$abandon" || true echo "ABANDONED claim $abandon (database state was $state; objects $([ -e "$objects" ] && echo kept || echo removed))" exit 0 fi bundle="$(realpath -- "$bundle")" refuse() { echo "RESTORE REFUSED: $*" >&2; exit 4; } # 1. Bundle integrity, before anything else is touched. [ -f "$bundle/MANIFEST" ] && [ -f "$bundle/SHA256SUMS" ] && [ -f "$bundle/database.dump" ] || refuse "incomplete bundle $bundle" (cd "$bundle" && sha256sum --check --strict --quiet SHA256SUMS) || refuse "bundle checksum mismatch" listed="$(sed -E 's/^[0-9a-f]{64} //' "$bundle/SHA256SUMS" | sort)" present="$(cd "$bundle" && find . -type f ! -name SHA256SUMS -printf '%P\n' | sort)" [ "$listed" = "$present" ] || refuse "bundle files differ from SHA256SUMS" while IFS= read -r rel; do case "$rel" in objects/*) [[ "$rel" =~ ^objects/w[0-9]+/r[0-9]+/([0-9a-f]{64})$ ]] || refuse "unexpected object path $rel" grep -qE "^${BASH_REMATCH[1]} $rel\$" "$bundle/SHA256SUMS" || refuse "object $rel is not its digest" ;; esac done <<< "$present" grep -qx 'format aevolution-backup-v1' "$bundle/MANIFEST" || refuse "unknown bundle format" expected_heads="$(grep '^head ' "$bundle/MANIFEST" | cut -d' ' -f2- | sort)" [ -n "$expected_heads" ] || refuse "manifest names no region" privileges=(--no-privileges) mode=isolated if [ "$in_place" = 1 ]; then grep -qx 'dump privileges' "$bundle/MANIFEST" || refuse "bundle predates in-place restore (no grants in its dump)" privileges=() mode=in-place fi manifest_sha="$(sha256sum "$bundle/MANIFEST")"; manifest_sha="${manifest_sha%% *}" label="$(sed -n 's/^label \([A-Za-z0-9._:-]*\)$/\1/p' "$bundle/MANIFEST" | head -1)" hook() { [ -z "${EVO_RESTORE_TEST_HOOK:-}" ] || bash -c "$EVO_RESTORE_TEST_HOOK" hook "$1"; } # 2. Claim the objects path, then check the target. nonce="$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')" mkdir -m 700 -- "$objects" 2> /dev/null || refuse "objects target already exists or cannot be created: $objects" echo "$nonce" > "$objects/.restore-claim" work="$(mktemp -d /tmp/evo-restore.XXXXXXXX)" installed=unknown_before published=0 cleanup() { local code=$? if [ "$published" != 1 ]; then case "$installed" in no) drop_objects "$nonce" || true ;; yes) if drop_claimed "$nonce"; then drop_objects "$nonce" || true echo "RESTORE FAILED: removed this restore's schema and objects (claim $nonce)" >&2 else echo "RESTORE FAILED: claim $nonce could not be removed; the target stays unservable." \ "Resolve with: $0 --abandon $nonce --objects $objects" >&2 code=5 fi ;; unknown_before) drop_objects "$nonce" || true ;; *) echo "RESTORE OUTCOME UNKNOWN: claim $nonce. If installed, the target is unservable" \ "(every build retired). Resolve with: $0 --abandon $nonce --objects $objects" >&2 code=5 ;; esac fi rm -rf -- "$work" exit "$code" } trap cleanup EXIT echo "restore claim $nonce" version="$(q -c "SELECT current_setting('server_version')")" grep -qx "database $version" "$bundle/MANIFEST" || refuse "target server $version differs from the manifest" existing="$(q -c "SELECT count(*) FROM pg_namespace WHERE nspname IN ('evolution', 'evolution_restore')")" [ "$existing" = 0 ] || refuse "target database already has an evolution schema" # 3. INSTALL: dump, claim and the serving gate commit together or not at all. The # COMMIT is only sent after the complete script: a truncated stream rolls back. pg_restore --no-owner "${privileges[@]}" --file="$work/restore.sql" "$bundle/database.dump" || refuse "the dump could not be read" { cat < "$work/install.sql" hook before-install status=0 psql -X --no-password -q -f "$work/install.sql" > "$work/install.log" 2>&1 || status=$? case "${EVO_RESTORE_TEST_INSTALL_ACK:-}" in lost) status=1 ;; unreachable) status=1; export PGPORT=1 PGHOST=/nonexistent ;; esac if [ "$status" = 0 ]; then installed=yes else state="$(claim_state "$nonce")" case "$state" in unpublished) installed=yes; echo "install outcome resolved: committed (claim $nonce)" ;; none) installed=no sed -n 's/^psql:[^:]*:[0-9]*: //p' "$work/install.log" | head -3 >&2 refuse "the restore transaction rolled back; the database was not changed (confirmed)" ;; *) installed=unknown; exit 5 ;; esac fi (cd "$bundle/objects" && find . -mindepth 2 -maxdepth 2 -type d -printf '%P\0' | xargs -0 -r -I{} mkdir -p -m 700 -- "$objects/{}") (cd "$bundle/objects" && find . -type f -printf '%P\0' | xargs -0 -r -I{} cp -- {} "$objects/{}") hook after-install # 4. Heads exactly as recorded. actual_heads="$(q -F ' ' -c " SELECT h.world_id, h.region_id, l.frame_id, h.commit_sequence, h.end_tick, h.commit_id, encode(h.digest, 'hex'), encode(c.build_identity, 'hex') FROM evolution.region_heads h JOIN evolution.region_leases l USING (world_id, region_id) JOIN evolution.region_commits c ON c.world_id = h.world_id AND c.region_id = h.region_id AND c.commit_sequence = h.commit_sequence" | sort)" [ "$actual_heads" = "$expected_heads" ] || { echo "restored heads differ from the manifest" >&2; exit 3; } # 5. Full history audit of every region (the CLI's retained list, not the database's # temporarily retired state), then the runtime rights audit for --in-place. pg=() if [ -n "${PGSERVICEFILE:-}" ]; then pg=(--pg-service "$PGSERVICEFILE") [ -n "${PGPASSFILE:-}" ] && pg+=(--pg-passfile "$PGPASSFILE") else pg=(--pg-host "${PGHOST:?}" --pg-port "${PGPORT:-5432}" --pg-database "${PGDATABASE:?}" --pg-user "${PGUSER:?}") [ -n "${PGPASSFILE:-}" ] && pg+=(--pg-passfile "$PGPASSFILE") fi retained=() while read -r _ build _; do retained+=(--retained "$build"); done < <(grep '^build ' "$bundle/MANIFEST") commits=0 while read -r world region _ _ tick _; do line="$("$clone_bin" audit --world "$world" --region "$region" --objects "$objects" \ "${pg[@]}" "${retained[@]}")" || { echo "audit failed for $world/$region" >&2; exit 3; } [[ "$line" =~ ^audited\ commits=([0-9]+)\ tick=([0-9]+)\ ]] || { echo "audit output: $line" >&2; exit 3; } [ "${BASH_REMATCH[2]}" = "$tick" ] || { echo "audit of $world/$region ended at ${BASH_REMATCH[2]}, not $tick" >&2; exit 3; } commits=$((commits + BASH_REMATCH[1])) echo "region $world/$region $line" done <<< "$expected_heads" if [ "$in_place" = 1 ]; then EVO_PG_RUNTIME_ROLE="$runtime_role" bash "$root/tools/migrate-postgres.sh" > /dev/null || { echo "runtime role $runtime_role failed the rights audit" >&2; exit 3; } fi # 6. PUBLISH under this NONCE's claim; an ambiguous completion is resolved likewise. hook before-publish status=0 q -c "DO \$\$ DECLARE held bytea[]; BEGIN SELECT c.held_builds INTO held FROM evolution_restore.claims c WHERE c.nonce = '$nonce' AND c.state = 'unpublished' FOR UPDATE; IF NOT FOUND THEN RAISE EXCEPTION 'restore claim % lost before publication', '$nonce'; END IF; UPDATE evolution.retained_builds SET retired_at = NULL WHERE build_identity = ANY (held); UPDATE evolution_restore.claims SET state = 'published', published_at = clock_timestamp() WHERE nonce = '$nonce'; END \$\$" > /dev/null 2>&1 || status=$? if [ "$status" != 0 ]; then case "$(claim_state "$nonce")" in published) ;; unpublished) echo "publication failed" >&2; exit 3 ;; *) installed=unknown; exit 5 ;; esac fi published=1 rm -f -- "$objects/.restore-claim" if [ "$in_place" = 1 ]; then created="$(sed -n 's/^created //p' "$bundle/MANIFEST")" while read -r world region _ _ tick _; do echo "REWIND region $world/$region restored at tick $tick (backup $created): every acknowledgement after this tick is undone" done <<< "$expected_heads" fi what="RESTORE VERIFIED" [ "$in_place" = 1 ] && what="RESTORE IN PLACE VERIFIED" echo "$what regions=$(wc -l <<< "$expected_heads") commits=$commits claim=$nonce duration_ms=$(( $(date +%s%3N) - begin_ms ))"