Ratifies dev/stage RPO/RTO from measurements and keeps World0 launch blocked (policy-enforced) until acknowledged outcomes survive host loss in an independent failure domain. Adds in-place restore with an explicit rewind, barrier-based fencing of a superseded database, and promotion of stage-pinned digests only. dr-check.sh rehearses host loss in every PostgreSQL build. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
165 lines
8.5 KiB
Bash
165 lines
8.5 KiB
Bash
#!/usr/bin/env bash
|
|
# M8.7 verified restore of a backup bundle (ADR-0074) onto independent storage.
|
|
#
|
|
# tools/backup/restore-world.sh --bundle DIR --objects DIR
|
|
# [--in-place --runtime-role ROLE --accept-rewind]
|
|
#
|
|
# Target connection: PG* environment naming an EMPTY database that the login owns
|
|
# (PGHOST/PGPORT/PGDATABASE/PGUSER, or PGSERVICE with PGSERVICEFILE/PGPASSFILE).
|
|
# The restore needs no source role, credential or key: owners and grants are not
|
|
# restored, so the target login owns everything restored. --objects must not
|
|
# exist yet (it is created 0700). evo-world-clone comes from EVO_WORLD_CLONE_BIN or
|
|
# PATH.
|
|
#
|
|
# Order, failing closed at each step:
|
|
# 1. every bundle file matches SHA256SUMS, no file is unlisted, and every object's
|
|
# name is its digest; nothing is touched before this passes;
|
|
# 2. the target has no `evolution` schema and runs the manifest's server version;
|
|
# 3. pg_restore in one transaction, then objects are copied;
|
|
# 4. the restored heads equal the MANIFEST heads exactly;
|
|
# 5. every region's full history is audited (evo-world-clone audit: each link's
|
|
# manifest, payload, trace replay and outcomes, from genesis to the head).
|
|
# On a failure after step 2 the restored schema and the copied objects are removed,
|
|
# so an unverified restore can never be mistaken for a world. The result is a
|
|
# verbatim, owner-only copy with the source's world identities: it has no runtime
|
|
# grants and is never served. To run it, clone it into an isolated world
|
|
# (tools/backup/verify-backup.sh --isolate).
|
|
#
|
|
# --in-place (M8.8 host-loss recovery for dev/stage, ADR-0075) restores the world
|
|
# under its own identity so its hosts can serve it again: the dump's grants are
|
|
# restored (the environment's roles must already exist in the target cluster), every
|
|
# restored lease is fenced so a new owner acquires under a newer epoch, and
|
|
# migrate-postgres.sh audits the runtime role's effective rights. Every acknowledgement
|
|
# after the bundle's cut is undone: deaths may reverse and resources may reappear.
|
|
# --accept-rewind states that this is acceptable, which it is for dev and stage only;
|
|
# World0 launch stays blocked until acknowledgements survive host loss (ADR-0075).
|
|
# Fence the old database with fence-database.sh before any of its hosts can return.
|
|
set -euo pipefail
|
|
usage() {
|
|
echo "usage: $0 --bundle DIR --objects DIR [--in-place --runtime-role ROLE --accept-rewind]" >&2
|
|
exit 2
|
|
}
|
|
bundle="" objects="" in_place=0 runtime_role="" accept_rewind=0
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--bundle) bundle="${2:-}"; shift 2 ;;
|
|
--objects) objects="${2:-}"; shift 2 ;;
|
|
--in-place) in_place=1; shift ;;
|
|
--runtime-role) runtime_role="${2:-}"; shift 2 ;;
|
|
--accept-rewind) accept_rewind=1; shift ;;
|
|
*) usage ;;
|
|
esac
|
|
done
|
|
[ -n "$bundle" ] && [ -n "$objects" ] || usage
|
|
if [ "$in_place" = 1 ]; then
|
|
[ -n "$runtime_role" ] || { echo "--in-place needs --runtime-role" >&2; exit 2; }
|
|
[ "$accept_rewind" = 1 ] ||
|
|
{ echo "--in-place undoes every acknowledgement after the backup; pass --accept-rewind (dev/stage only)" >&2; exit 2; }
|
|
elif [ -n "$runtime_role" ] || [ "$accept_rewind" = 1 ]; then
|
|
usage
|
|
fi
|
|
root="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
bundle="$(realpath -- "$bundle")" objects="$(realpath -m -- "$objects")"
|
|
clone_bin="${EVO_WORLD_CLONE_BIN:-evo-world-clone}"
|
|
umask 077
|
|
begin_ms="$(date +%s%3N)"
|
|
refuse() { echo "RESTORE REFUSED: $*" >&2; exit 4; }
|
|
|
|
# 1. Bundle integrity, before anything else is touched.
|
|
[ -f "$bundle/MANIFEST" ] && [ -f "$bundle/SHA256SUMS" ] && [ -f "$bundle/database.dump" ] ||
|
|
refuse "incomplete bundle $bundle"
|
|
(cd "$bundle" && sha256sum --check --strict --quiet SHA256SUMS) || refuse "bundle checksum mismatch"
|
|
listed="$(sed -E 's/^[0-9a-f]{64} //' "$bundle/SHA256SUMS" | sort)"
|
|
present="$(cd "$bundle" && find . -type f ! -name SHA256SUMS -printf '%P\n' | sort)"
|
|
[ "$listed" = "$present" ] || refuse "bundle files differ from SHA256SUMS"
|
|
while IFS= read -r rel; do
|
|
case "$rel" in
|
|
objects/*)
|
|
[[ "$rel" =~ ^objects/w[0-9]+/r[0-9]+/([0-9a-f]{64})$ ]] || refuse "unexpected object path $rel"
|
|
grep -qE "^${BASH_REMATCH[1]} $rel\$" "$bundle/SHA256SUMS" || refuse "object $rel is not its digest"
|
|
;;
|
|
esac
|
|
done <<< "$present"
|
|
grep -qx 'format aevolution-backup-v1' "$bundle/MANIFEST" || refuse "unknown bundle format"
|
|
expected_heads="$(grep '^head ' "$bundle/MANIFEST" | cut -d' ' -f2- | sort)"
|
|
[ -n "$expected_heads" ] || refuse "manifest names no region"
|
|
privileges=(--no-privileges)
|
|
if [ "$in_place" = 1 ]; then
|
|
grep -qx 'dump privileges' "$bundle/MANIFEST" || refuse "bundle predates in-place restore (no grants in its dump)"
|
|
privileges=()
|
|
fi
|
|
|
|
# 2. Target: empty, same server version.
|
|
[ ! -e "$objects" ] || refuse "objects target already exists: $objects"
|
|
version="$(psql -X --no-password -A -t -v ON_ERROR_STOP=1 -c "SELECT current_setting('server_version')")"
|
|
grep -qx "database $version" "$bundle/MANIFEST" || refuse "target server $version differs from the manifest"
|
|
existing="$(psql -X --no-password -A -t -v ON_ERROR_STOP=1 -c "SELECT count(*) FROM pg_namespace WHERE nspname = 'evolution'")"
|
|
[ "$existing" = 0 ] || refuse "target database already has an evolution schema"
|
|
|
|
restored=0
|
|
cleanup() {
|
|
[ "$restored" = 1 ] && return
|
|
psql -X --no-password -q -v ON_ERROR_STOP=1 -c 'DROP SCHEMA IF EXISTS evolution CASCADE' > /dev/null 2>&1 || true
|
|
rm -rf -- "$objects"
|
|
echo "RESTORE FAILED: the restored schema and objects were removed" >&2
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
# 3. Database in one transaction, then objects.
|
|
target="${PGDATABASE:-}"
|
|
[ -n "${PGSERVICE:-}" ] && target="service=$PGSERVICE"
|
|
[ -n "$target" ] || { echo "set PGDATABASE or PGSERVICE for the target" >&2; exit 2; }
|
|
pg_restore --no-password --exit-on-error --single-transaction --no-owner "${privileges[@]}" \
|
|
--dbname="$target" "$bundle/database.dump"
|
|
mkdir -m 700 -- "$objects"
|
|
(cd "$bundle/objects" && find . -mindepth 2 -maxdepth 2 -type d -printf '%P\0' |
|
|
xargs -0 -r -I{} mkdir -p -m 700 -- "$objects/{}")
|
|
(cd "$bundle/objects" && find . -type f -printf '%P\0' | xargs -0 -r -I{} cp -- {} "$objects/{}")
|
|
|
|
# 4. Heads exactly as recorded.
|
|
actual_heads="$(psql -X --no-password -A -t -F ' ' -v ON_ERROR_STOP=1 -c "
|
|
SELECT h.world_id, h.region_id, l.frame_id, h.commit_sequence, h.end_tick, h.commit_id,
|
|
encode(h.digest, 'hex'), encode(c.build_identity, 'hex')
|
|
FROM evolution.region_heads h
|
|
JOIN evolution.region_leases l USING (world_id, region_id)
|
|
JOIN evolution.region_commits c ON c.world_id = h.world_id AND c.region_id = h.region_id
|
|
AND c.commit_sequence = h.commit_sequence" | sort)"
|
|
[ "$actual_heads" = "$expected_heads" ] || { echo "restored heads differ from the manifest" >&2; exit 3; }
|
|
|
|
# 5. Full history audit of every region.
|
|
pg=()
|
|
if [ -n "${PGSERVICEFILE:-}" ]; then
|
|
pg=(--pg-service "$PGSERVICEFILE")
|
|
[ -n "${PGPASSFILE:-}" ] && pg+=(--pg-passfile "$PGPASSFILE")
|
|
else
|
|
pg=(--pg-host "${PGHOST:?}" --pg-port "${PGPORT:-5432}" --pg-database "${PGDATABASE:?}"
|
|
--pg-user "${PGUSER:?}")
|
|
[ -n "${PGPASSFILE:-}" ] && pg+=(--pg-passfile "$PGPASSFILE")
|
|
fi
|
|
retained=()
|
|
while read -r _ build _; do retained+=(--retained "$build"); done < <(grep '^build ' "$bundle/MANIFEST")
|
|
commits=0
|
|
while read -r world region _ _ tick _; do
|
|
line="$("$clone_bin" audit --world "$world" --region "$region" --objects "$objects" \
|
|
"${pg[@]}" "${retained[@]}")" || { echo "audit failed for $world/$region" >&2; exit 3; }
|
|
[[ "$line" =~ ^audited\ commits=([0-9]+)\ tick=([0-9]+)\ ]] || { echo "audit output: $line" >&2; exit 3; }
|
|
[ "${BASH_REMATCH[2]}" = "$tick" ] || { echo "audit of $world/$region ended at ${BASH_REMATCH[2]}, not $tick" >&2; exit 3; }
|
|
commits=$((commits + BASH_REMATCH[1]))
|
|
echo "region $world/$region $line"
|
|
done <<< "$expected_heads"
|
|
if [ "$in_place" = 1 ]; then
|
|
# No lease from before the loss survives: the next owner acquires a newer epoch.
|
|
psql -X --no-password -q -v ON_ERROR_STOP=1 -c \
|
|
'UPDATE evolution.region_leases SET holder = NULL, expires_at = NULL WHERE holder IS NOT NULL' > /dev/null
|
|
EVO_PG_RUNTIME_ROLE="$runtime_role" bash "$root/tools/migrate-postgres.sh" > /dev/null ||
|
|
{ echo "runtime role $runtime_role failed the rights audit" >&2; exit 3; }
|
|
created="$(sed -n 's/^created //p' "$bundle/MANIFEST")"
|
|
while read -r world region _ _ tick _; do
|
|
echo "REWIND region $world/$region restored at tick $tick (backup $created): every acknowledgement after this tick is undone"
|
|
done <<< "$expected_heads"
|
|
fi
|
|
restored=1
|
|
what="RESTORE VERIFIED"
|
|
[ "$in_place" = 1 ] && what="RESTORE IN PLACE VERIFIED"
|
|
echo "$what regions=$(wc -l <<< "$expected_heads") commits=$commits duration_ms=$(( $(date +%s%3N) - begin_ms ))"
|