The restore no longer relies on an auxiliary lock session. One install transaction holds the dump, a nonce claim row, fenced leases and retired builds, so the world is unservable until a nonce-checked publish after verification, regardless of connection loss or restarts. A failed install is resolved from the claim (committed, rolled back, or unknown with the target kept unservable and --abandon NONCE to resolve); cleanup only removes what the nonce owns. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
333 lines
16 KiB
Bash
333 lines
16 KiB
Bash
#!/usr/bin/env bash
|
|
# M8.7 verified restore of a backup bundle (ADR-0074) onto independent storage.
|
|
#
|
|
# tools/backup/restore-world.sh --bundle DIR --objects DIR
|
|
# [--in-place --runtime-role ROLE --accept-rewind]
|
|
# tools/backup/restore-world.sh --abandon NONCE --objects DIR
|
|
#
|
|
# Target connection: PG* environment naming an EMPTY database that the login owns
|
|
# (PGHOST/PGPORT/PGDATABASE/PGUSER, or PGSERVICE with PGSERVICEFILE/PGPASSFILE).
|
|
# The restore needs no source role, credential or key: owners are not restored, and
|
|
# grants only with --in-place, so the target login owns everything restored.
|
|
# evo-world-clone comes from EVO_WORLD_CLONE_BIN or PATH.
|
|
#
|
|
# Order, failing closed at each step:
|
|
# 1. every bundle file matches SHA256SUMS, no file is unlisted, and every object's
|
|
# name is its digest; nothing is touched before this passes;
|
|
# 2. --objects is claimed by creating it (mkdir, never -p: an existing path is
|
|
# refused and never touched) and marked with this invocation's random NONCE; the
|
|
# target must have neither `evolution` nor `evolution_restore` and must run the
|
|
# manifest's server version;
|
|
# 3. INSTALL, one transaction: the dump, a claim row evolution_restore.claims(NONCE,
|
|
# unpublished), every lease fenced and every current build retired. From that
|
|
# commit on the world is unservable: acquisition, renewal and commits refuse
|
|
# until publication, whatever happens to any connection or the server (review
|
|
# M8 follow-up P1). Then the objects are copied;
|
|
# 4. the restored heads equal the MANIFEST heads exactly;
|
|
# 5. every region's full history is audited (evo-world-clone audit), and with
|
|
# --in-place migrate-postgres.sh audits the runtime role's effective rights;
|
|
# 6. PUBLISH, one transaction that first locks this NONCE's unpublished claim:
|
|
# exactly the builds the install retired become current again and the claim
|
|
# becomes published. A restore that lost its claim cannot publish.
|
|
# A failed or interrupted install is resolved, not assumed (review M8 follow-up P2):
|
|
# the claim row says whether this invocation's transaction committed. Committed ->
|
|
# verification continues; absent -> rolled back (the database was not changed);
|
|
# unreachable -> OUTCOME UNKNOWN (exit 5): the target stays unservable and both the
|
|
# schema and the objects are kept; resolve it later with --abandon NONCE.
|
|
# Cleanup only ever removes what this NONCE owns: the schemas only through a
|
|
# transaction that locks this NONCE's unpublished claim, the objects directory only
|
|
# while it carries this NONCE's marker. A competing restore's or provisioner's state
|
|
# is never touched. The default result is a verbatim, owner-only copy with the
|
|
# source's world identities, no runtime grants and never served; to run it, clone it
|
|
# into an isolated world (tools/backup/verify-backup.sh --isolate).
|
|
#
|
|
# --in-place (M8.8 host-loss recovery for dev/stage, ADR-0075) restores the world
|
|
# under its own identity so its hosts can serve it again after publication: the
|
|
# dump's grants are restored (the environment's roles must already exist in the
|
|
# target cluster). Every acknowledgement after the bundle's cut is undone: deaths may
|
|
# reverse and resources may reappear. --accept-rewind states that this is
|
|
# acceptable, which it is for dev and stage only; World0 launch stays blocked until
|
|
# acknowledgements survive host loss (ADR-0075). Fence the old database with
|
|
# fence-database.sh before any of its hosts can return.
|
|
#
|
|
# Exit: 0 verified and published, 2 usage, 3 verification failed (this restore's
|
|
# state removed), 4 refused or rolled back (nothing of this restore left),
|
|
# 5 outcome unknown (target unservable; evidence kept).
|
|
# Tests only: EVO_RESTORE_TEST_HOOK is run by bash with $1 = before-install,
|
|
# after-install or before-publish; EVO_RESTORE_TEST_INSTALL_ACK=lost treats a committed
|
|
# install as failed (an ambiguous completion), =unreachable also makes its resolution
|
|
# unreachable.
|
|
set -euo pipefail
|
|
usage() {
|
|
echo "usage: $0 --bundle DIR --objects DIR [--in-place --runtime-role ROLE --accept-rewind]" >&2
|
|
echo " $0 --abandon NONCE --objects DIR" >&2
|
|
exit 2
|
|
}
|
|
bundle="" objects="" in_place=0 runtime_role="" accept_rewind=0 abandon=""
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--bundle) bundle="${2:-}"; shift 2 ;;
|
|
--objects) objects="${2:-}"; shift 2 ;;
|
|
--in-place) in_place=1; shift ;;
|
|
--runtime-role) runtime_role="${2:-}"; shift 2 ;;
|
|
--accept-rewind) accept_rewind=1; shift ;;
|
|
--abandon) abandon="${2:-}"; shift 2 ;;
|
|
*) usage ;;
|
|
esac
|
|
done
|
|
[ -n "$objects" ] || usage
|
|
if [ -n "$abandon" ]; then
|
|
[[ "$abandon" =~ ^[0-9a-f]{32}$ ]] && [ -z "$bundle" ] && [ "$in_place" = 0 ] || usage
|
|
else
|
|
[ -n "$bundle" ] || usage
|
|
fi
|
|
if [ "$in_place" = 1 ]; then
|
|
[ -n "$runtime_role" ] || { echo "--in-place needs --runtime-role" >&2; exit 2; }
|
|
[ "$accept_rewind" = 1 ] ||
|
|
{ echo "--in-place undoes every acknowledgement after the backup; pass --accept-rewind (dev/stage only)" >&2; exit 2; }
|
|
elif [ -n "$runtime_role" ] || [ "$accept_rewind" = 1 ]; then
|
|
usage
|
|
fi
|
|
root="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
objects="$(realpath -m -- "$objects")"
|
|
clone_bin="${EVO_WORLD_CLONE_BIN:-evo-world-clone}"
|
|
umask 077
|
|
begin_ms="$(date +%s%3N)"
|
|
q() { psql -X --no-password -q -A -t -v ON_ERROR_STOP=1 "$@"; }
|
|
|
|
# The claim row of NONCE: none, unpublished, published, or unknown (unreachable).
|
|
claim_state() {
|
|
local nonce="$1" exists state
|
|
exists="$(q -c "SELECT to_regclass('evolution_restore.claims') IS NOT NULL" 2> /dev/null)" || { echo unknown; return; }
|
|
[ "$exists" = t ] || { echo none; return; }
|
|
state="$(q -c "SELECT state FROM evolution_restore.claims WHERE nonce = '$nonce'" 2> /dev/null)" ||
|
|
{ echo unknown; return; }
|
|
echo "${state:-none}"
|
|
}
|
|
# Removes NONCE's unpublished restore: both schemas, in one transaction that first
|
|
# locks that claim. Anything else (another claim, a published one) is refused.
|
|
drop_claimed() {
|
|
q -c "DO \$\$ BEGIN
|
|
IF to_regclass('evolution_restore.claims') IS NULL THEN
|
|
RAISE EXCEPTION 'no restore claim';
|
|
END IF;
|
|
PERFORM 1 FROM evolution_restore.claims WHERE nonce = '$1' AND state = 'unpublished' FOR UPDATE;
|
|
IF NOT FOUND THEN
|
|
RAISE EXCEPTION 'restore claim % is not unpublished here', '$1';
|
|
END IF;
|
|
DROP SCHEMA evolution CASCADE;
|
|
DROP SCHEMA evolution_restore CASCADE;
|
|
END \$\$" > /dev/null 2>&1
|
|
}
|
|
# Removes the objects directory only while it carries NONCE's marker.
|
|
drop_objects() {
|
|
[ -f "$objects/.restore-claim" ] && [ "$(cat "$objects/.restore-claim")" = "$1" ] && rm -rf -- "$objects"
|
|
}
|
|
|
|
if [ -n "$abandon" ]; then
|
|
state="$(claim_state "$abandon")"
|
|
case "$state" in
|
|
unpublished) drop_claimed "$abandon" || { echo "ABANDON FAILED: could not remove claim $abandon" >&2; exit 3; } ;;
|
|
none) ;;
|
|
published) echo "ABANDON REFUSED: claim $abandon is published (a verified world)" >&2; exit 4 ;;
|
|
*) echo "ABANDON FAILED: the target database is unreachable" >&2; exit 5 ;;
|
|
esac
|
|
drop_objects "$abandon" || true
|
|
echo "ABANDONED claim $abandon (database state was $state; objects $([ -e "$objects" ] && echo kept || echo removed))"
|
|
exit 0
|
|
fi
|
|
|
|
bundle="$(realpath -- "$bundle")"
|
|
refuse() { echo "RESTORE REFUSED: $*" >&2; exit 4; }
|
|
|
|
# 1. Bundle integrity, before anything else is touched.
|
|
[ -f "$bundle/MANIFEST" ] && [ -f "$bundle/SHA256SUMS" ] && [ -f "$bundle/database.dump" ] ||
|
|
refuse "incomplete bundle $bundle"
|
|
(cd "$bundle" && sha256sum --check --strict --quiet SHA256SUMS) || refuse "bundle checksum mismatch"
|
|
listed="$(sed -E 's/^[0-9a-f]{64} //' "$bundle/SHA256SUMS" | sort)"
|
|
present="$(cd "$bundle" && find . -type f ! -name SHA256SUMS -printf '%P\n' | sort)"
|
|
[ "$listed" = "$present" ] || refuse "bundle files differ from SHA256SUMS"
|
|
while IFS= read -r rel; do
|
|
case "$rel" in
|
|
objects/*)
|
|
[[ "$rel" =~ ^objects/w[0-9]+/r[0-9]+/([0-9a-f]{64})$ ]] || refuse "unexpected object path $rel"
|
|
grep -qE "^${BASH_REMATCH[1]} $rel\$" "$bundle/SHA256SUMS" || refuse "object $rel is not its digest"
|
|
;;
|
|
esac
|
|
done <<< "$present"
|
|
grep -qx 'format aevolution-backup-v1' "$bundle/MANIFEST" || refuse "unknown bundle format"
|
|
expected_heads="$(grep '^head ' "$bundle/MANIFEST" | cut -d' ' -f2- | sort)"
|
|
[ -n "$expected_heads" ] || refuse "manifest names no region"
|
|
privileges=(--no-privileges) mode=isolated
|
|
if [ "$in_place" = 1 ]; then
|
|
grep -qx 'dump privileges' "$bundle/MANIFEST" || refuse "bundle predates in-place restore (no grants in its dump)"
|
|
privileges=() mode=in-place
|
|
fi
|
|
manifest_sha="$(sha256sum "$bundle/MANIFEST")"; manifest_sha="${manifest_sha%% *}"
|
|
label="$(sed -n 's/^label \([A-Za-z0-9._:-]*\)$/\1/p' "$bundle/MANIFEST" | head -1)"
|
|
hook() { [ -z "${EVO_RESTORE_TEST_HOOK:-}" ] || bash -c "$EVO_RESTORE_TEST_HOOK" hook "$1"; }
|
|
|
|
# 2. Claim the objects path, then check the target.
|
|
nonce="$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')"
|
|
mkdir -m 700 -- "$objects" 2> /dev/null || refuse "objects target already exists or cannot be created: $objects"
|
|
echo "$nonce" > "$objects/.restore-claim"
|
|
work="$(mktemp -d /tmp/evo-restore.XXXXXXXX)"
|
|
installed=unknown_before published=0
|
|
cleanup() {
|
|
local code=$?
|
|
if [ "$published" != 1 ]; then
|
|
case "$installed" in
|
|
no) drop_objects "$nonce" || true ;;
|
|
yes)
|
|
if drop_claimed "$nonce"; then
|
|
drop_objects "$nonce" || true
|
|
echo "RESTORE FAILED: removed this restore's schema and objects (claim $nonce)" >&2
|
|
else
|
|
echo "RESTORE FAILED: claim $nonce could not be removed; the target stays unservable." \
|
|
"Resolve with: $0 --abandon $nonce --objects $objects" >&2
|
|
code=5
|
|
fi
|
|
;;
|
|
unknown_before) drop_objects "$nonce" || true ;;
|
|
*)
|
|
echo "RESTORE OUTCOME UNKNOWN: claim $nonce. If installed, the target is unservable" \
|
|
"(every build retired). Resolve with: $0 --abandon $nonce --objects $objects" >&2
|
|
code=5
|
|
;;
|
|
esac
|
|
fi
|
|
rm -rf -- "$work"
|
|
exit "$code"
|
|
}
|
|
trap cleanup EXIT
|
|
echo "restore claim $nonce"
|
|
version="$(q -c "SELECT current_setting('server_version')")"
|
|
grep -qx "database $version" "$bundle/MANIFEST" || refuse "target server $version differs from the manifest"
|
|
existing="$(q -c "SELECT count(*) FROM pg_namespace WHERE nspname IN ('evolution', 'evolution_restore')")"
|
|
[ "$existing" = 0 ] || refuse "target database already has an evolution schema"
|
|
|
|
# 3. INSTALL: dump, claim and the serving gate commit together or not at all. The
|
|
# COMMIT is only sent after the complete script: a truncated stream rolls back.
|
|
pg_restore --no-owner "${privileges[@]}" --file="$work/restore.sql" "$bundle/database.dump" ||
|
|
refuse "the dump could not be read"
|
|
{
|
|
cat <<SQL
|
|
\\set ON_ERROR_STOP on
|
|
BEGIN;
|
|
CREATE SCHEMA evolution_restore;
|
|
REVOKE ALL ON SCHEMA evolution_restore FROM PUBLIC;
|
|
CREATE TABLE evolution_restore.claims (
|
|
nonce text PRIMARY KEY CHECK (nonce ~ '^[0-9a-f]{32}\$'),
|
|
state text NOT NULL CHECK (state IN ('unpublished', 'published')),
|
|
mode text NOT NULL, manifest_sha256 text NOT NULL, label text,
|
|
held_builds bytea[] NOT NULL,
|
|
installed_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
|
published_at timestamptz);
|
|
REVOKE ALL ON evolution_restore.claims FROM PUBLIC;
|
|
SQL
|
|
cat "$work/restore.sql"
|
|
cat <<SQL
|
|
INSERT INTO evolution_restore.claims(nonce, state, mode, manifest_sha256, label, held_builds)
|
|
SELECT '$nonce', 'unpublished', '$mode', '$manifest_sha', '$label',
|
|
coalesce(pg_catalog.array_agg(b.build_identity ORDER BY b.build_identity), '{}')
|
|
FROM evolution.retained_builds b WHERE b.retired_at IS NULL;
|
|
UPDATE evolution.retained_builds SET retired_at = pg_catalog.clock_timestamp() WHERE retired_at IS NULL;
|
|
UPDATE evolution.region_leases SET holder = NULL, expires_at = NULL WHERE holder IS NOT NULL;
|
|
COMMIT;
|
|
SQL
|
|
} > "$work/install.sql"
|
|
hook before-install
|
|
status=0
|
|
psql -X --no-password -q -f "$work/install.sql" > "$work/install.log" 2>&1 || status=$?
|
|
case "${EVO_RESTORE_TEST_INSTALL_ACK:-}" in
|
|
lost) status=1 ;;
|
|
unreachable) status=1; export PGPORT=1 PGHOST=/nonexistent ;;
|
|
esac
|
|
if [ "$status" = 0 ]; then
|
|
installed=yes
|
|
else
|
|
state="$(claim_state "$nonce")"
|
|
case "$state" in
|
|
unpublished) installed=yes; echo "install outcome resolved: committed (claim $nonce)" ;;
|
|
none)
|
|
installed=no
|
|
sed -n 's/^psql:[^:]*:[0-9]*: //p' "$work/install.log" | head -3 >&2
|
|
refuse "the restore transaction rolled back; the database was not changed (confirmed)"
|
|
;;
|
|
*) installed=unknown; exit 5 ;;
|
|
esac
|
|
fi
|
|
(cd "$bundle/objects" && find . -mindepth 2 -maxdepth 2 -type d -printf '%P\0' |
|
|
xargs -0 -r -I{} mkdir -p -m 700 -- "$objects/{}")
|
|
(cd "$bundle/objects" && find . -type f -printf '%P\0' | xargs -0 -r -I{} cp -- {} "$objects/{}")
|
|
hook after-install
|
|
|
|
# 4. Heads exactly as recorded.
|
|
actual_heads="$(q -F ' ' -c "
|
|
SELECT h.world_id, h.region_id, l.frame_id, h.commit_sequence, h.end_tick, h.commit_id,
|
|
encode(h.digest, 'hex'), encode(c.build_identity, 'hex')
|
|
FROM evolution.region_heads h
|
|
JOIN evolution.region_leases l USING (world_id, region_id)
|
|
JOIN evolution.region_commits c ON c.world_id = h.world_id AND c.region_id = h.region_id
|
|
AND c.commit_sequence = h.commit_sequence" | sort)"
|
|
[ "$actual_heads" = "$expected_heads" ] || { echo "restored heads differ from the manifest" >&2; exit 3; }
|
|
|
|
# 5. Full history audit of every region (the CLI's retained list, not the database's
|
|
# temporarily retired state), then the runtime rights audit for --in-place.
|
|
pg=()
|
|
if [ -n "${PGSERVICEFILE:-}" ]; then
|
|
pg=(--pg-service "$PGSERVICEFILE")
|
|
[ -n "${PGPASSFILE:-}" ] && pg+=(--pg-passfile "$PGPASSFILE")
|
|
else
|
|
pg=(--pg-host "${PGHOST:?}" --pg-port "${PGPORT:-5432}" --pg-database "${PGDATABASE:?}"
|
|
--pg-user "${PGUSER:?}")
|
|
[ -n "${PGPASSFILE:-}" ] && pg+=(--pg-passfile "$PGPASSFILE")
|
|
fi
|
|
retained=()
|
|
while read -r _ build _; do retained+=(--retained "$build"); done < <(grep '^build ' "$bundle/MANIFEST")
|
|
commits=0
|
|
while read -r world region _ _ tick _; do
|
|
line="$("$clone_bin" audit --world "$world" --region "$region" --objects "$objects" \
|
|
"${pg[@]}" "${retained[@]}")" || { echo "audit failed for $world/$region" >&2; exit 3; }
|
|
[[ "$line" =~ ^audited\ commits=([0-9]+)\ tick=([0-9]+)\ ]] || { echo "audit output: $line" >&2; exit 3; }
|
|
[ "${BASH_REMATCH[2]}" = "$tick" ] || { echo "audit of $world/$region ended at ${BASH_REMATCH[2]}, not $tick" >&2; exit 3; }
|
|
commits=$((commits + BASH_REMATCH[1]))
|
|
echo "region $world/$region $line"
|
|
done <<< "$expected_heads"
|
|
if [ "$in_place" = 1 ]; then
|
|
EVO_PG_RUNTIME_ROLE="$runtime_role" bash "$root/tools/migrate-postgres.sh" > /dev/null ||
|
|
{ echo "runtime role $runtime_role failed the rights audit" >&2; exit 3; }
|
|
fi
|
|
|
|
# 6. PUBLISH under this NONCE's claim; an ambiguous completion is resolved likewise.
|
|
hook before-publish
|
|
status=0
|
|
q -c "DO \$\$ DECLARE held bytea[]; BEGIN
|
|
SELECT c.held_builds INTO held FROM evolution_restore.claims c
|
|
WHERE c.nonce = '$nonce' AND c.state = 'unpublished' FOR UPDATE;
|
|
IF NOT FOUND THEN
|
|
RAISE EXCEPTION 'restore claim % lost before publication', '$nonce';
|
|
END IF;
|
|
UPDATE evolution.retained_builds SET retired_at = NULL WHERE build_identity = ANY (held);
|
|
UPDATE evolution_restore.claims SET state = 'published', published_at = clock_timestamp()
|
|
WHERE nonce = '$nonce';
|
|
END \$\$" > /dev/null 2>&1 || status=$?
|
|
if [ "$status" != 0 ]; then
|
|
case "$(claim_state "$nonce")" in
|
|
published) ;;
|
|
unpublished) echo "publication failed" >&2; exit 3 ;;
|
|
*) installed=unknown; exit 5 ;;
|
|
esac
|
|
fi
|
|
published=1
|
|
rm -f -- "$objects/.restore-claim"
|
|
if [ "$in_place" = 1 ]; then
|
|
created="$(sed -n 's/^created //p' "$bundle/MANIFEST")"
|
|
while read -r world region _ _ tick _; do
|
|
echo "REWIND region $world/$region restored at tick $tick (backup $created): every acknowledgement after this tick is undone"
|
|
done <<< "$expected_heads"
|
|
fi
|
|
what="RESTORE VERIFIED"
|
|
[ "$in_place" = 1 ] && what="RESTORE IN PLACE VERIFIED"
|
|
echo "$what regions=$(wc -l <<< "$expected_heads") commits=$commits claim=$nonce duration_ms=$(( $(date +%s%3N) - begin_ms ))"
|