Ratifies dev/stage RPO/RTO from measurements and keeps World0 launch blocked (policy-enforced) until acknowledged outcomes survive host loss in an independent failure domain. Adds in-place restore with an explicit rewind, barrier-based fencing of a superseded database, and promotion of stage-pinned digests only. dr-check.sh rehearses host loss in every PostgreSQL build. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
108 lines
5.3 KiB
Bash
108 lines
5.3 KiB
Bash
#!/usr/bin/env bash
|
|
# M8.7 world backup (ADR-0074): one consistent, self-verifying bundle of an
|
|
# AEvolution database and its object store.
|
|
#
|
|
# tools/backup/backup-world.sh --objects DIR --out DIR [--label TEXT]
|
|
#
|
|
# Connection: PG* environment (PGSERVICE/PGSERVICEFILE, or PGHOST/PGDATABASE/...),
|
|
# optionally EVO_PGPASS_SOURCE (copied to a private file, as migrate does). The
|
|
# login must read the whole `evolution` schema (the migrator/owner). Never print
|
|
# the connection or enable shell tracing here.
|
|
#
|
|
# Cut: one exported REPEATABLE READ snapshot names the region heads recorded in the
|
|
# MANIFEST and is the snapshot pg_dump reads. Objects are copied afterwards: every
|
|
# object a committed row references was verified durable before its commit, and
|
|
# objects are immutable and never deleted (no GC), so each one the snapshot
|
|
# references is present and final when copied. Objects written after the snapshot
|
|
# are copied too and are harmless orphans in the bundle. Nothing is written to the
|
|
# source database or the object volume, and no admission barrier is needed.
|
|
#
|
|
# Bundle (DIR): MANIFEST, database.dump (pg_dump custom format, schema `evolution`,
|
|
# no owners; grants are kept for an in-place restore, ADR-0075, and skipped by every
|
|
# other restore, which therefore never needs the source's roles or credentials),
|
|
# objects/w<world>/r<region>/<sha256>, SHA256SUMS over every other file. A source
|
|
# object whose bytes differ from its name fails the backup.
|
|
set -euo pipefail
|
|
usage() { echo "usage: $0 --objects DIR --out DIR [--label TEXT]" >&2; exit 2; }
|
|
objects="" out="" label=""
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--objects) objects="${2:-}"; shift 2 ;;
|
|
--out) out="${2:-}"; shift 2 ;;
|
|
--label) label="${2:-}"; shift 2 ;;
|
|
*) usage ;;
|
|
esac
|
|
done
|
|
[ -n "$objects" ] && [ -n "$out" ] || usage
|
|
objects="${objects%/}"
|
|
[[ "$label" =~ ^[A-Za-z0-9._:-]*$ ]] || { echo "label: letters, digits and ._:- only" >&2; exit 2; }
|
|
[ -d "$objects" ] && [ ! -L "$objects" ] || { echo "objects: not a real directory: $objects" >&2; exit 2; }
|
|
if [ -e "$out" ] && [ -n "$(ls -A "$out")" ]; then echo "out: not empty: $out" >&2; exit 2; fi
|
|
umask 077
|
|
if [ -n "${EVO_PGPASS_SOURCE:-}" ]; then
|
|
export PGPASSFILE
|
|
PGPASSFILE="$(mktemp /tmp/evo-pgpass.XXXXXXXX)"
|
|
trap 'rm -f -- "$PGPASSFILE"' EXIT
|
|
cat "$EVO_PGPASS_SOURCE" > "$PGPASSFILE"
|
|
fi
|
|
mkdir -p "$out"
|
|
started="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
begin_ms="$(date +%s%3N)"
|
|
|
|
# The snapshot is held by this psql session while pg_dump reads it (\!).
|
|
export EVO_BACKUP_DUMP="$out/database.dump"
|
|
psql -X --no-password -q -A -t -v ON_ERROR_STOP=1 > "$out/.cut" <<'SQL'
|
|
\pset fieldsep ' '
|
|
BEGIN ISOLATION LEVEL REPEATABLE READ, READ ONLY;
|
|
SELECT pg_export_snapshot() AS snapshot \gset
|
|
SELECT 'database', current_setting('server_version');
|
|
SELECT 'schema', version, sha256 FROM evolution.schema_migrations ORDER BY version;
|
|
SELECT 'build', encode(build_identity, 'hex'),
|
|
CASE WHEN retired_at IS NULL THEN 'retained' ELSE 'retired' END
|
|
FROM evolution.retained_builds ORDER BY 2;
|
|
SELECT 'head', h.world_id, h.region_id, l.frame_id, h.commit_sequence, h.end_tick,
|
|
h.commit_id, encode(h.digest, 'hex'), encode(c.build_identity, 'hex')
|
|
FROM evolution.region_heads h
|
|
JOIN evolution.region_leases l USING (world_id, region_id)
|
|
JOIN evolution.region_commits c ON c.world_id = h.world_id AND c.region_id = h.region_id
|
|
AND c.commit_sequence = h.commit_sequence
|
|
ORDER BY h.world_id, h.region_id;
|
|
SELECT 'rows', (SELECT count(*) FROM evolution.region_commits),
|
|
(SELECT count(*) FROM evolution.region_decisions),
|
|
(SELECT count(*) FROM evolution.region_outbox),
|
|
(SELECT count(*) FROM evolution.region_origins);
|
|
\setenv EVO_BACKUP_SNAPSHOT :snapshot
|
|
\! pg_dump --no-password --format=custom --schema=evolution --no-owner --snapshot="$EVO_BACKUP_SNAPSHOT" --file="$EVO_BACKUP_DUMP" && echo 'dumped' || echo 'dump-failed'
|
|
COMMIT;
|
|
SQL
|
|
grep -qx 'dumped' "$out/.cut" || { echo "backup failed: pg_dump did not complete" >&2; exit 3; }
|
|
pg_restore --list "$out/database.dump" > /dev/null || { echo "backup failed: unreadable dump" >&2; exit 3; }
|
|
|
|
# Objects, verified by name while copying; private temporaries are skipped.
|
|
count=0 bytes=0
|
|
while IFS= read -r -d '' file; do
|
|
rel="${file#"$objects"/}"
|
|
name="${rel##*/}"
|
|
[[ "$rel" =~ ^w[0-9]+/r[0-9]+/[0-9a-f]{64}$ ]] || continue
|
|
mkdir -p "$out/objects/${rel%/*}"
|
|
cp -- "$file" "$out/objects/$rel"
|
|
sum="$(sha256sum -- "$out/objects/$rel")"
|
|
[ "${sum%% *}" = "$name" ] || { echo "backup failed: source object $rel is corrupt" >&2; exit 3; }
|
|
count=$((count + 1))
|
|
bytes=$((bytes + $(stat -c %s -- "$out/objects/$rel")))
|
|
done < <(find "$objects" -mindepth 3 -maxdepth 3 -type f ! -name '.tmp-*' -print0)
|
|
|
|
{
|
|
echo "format aevolution-backup-v1"
|
|
echo "created $started"
|
|
echo "label ${label:--}"
|
|
echo "dump privileges"
|
|
grep -E '^(database|schema|build|head|rows) ' "$out/.cut"
|
|
echo "objects $count $bytes"
|
|
} > "$out/MANIFEST"
|
|
rm -f "$out/.cut"
|
|
heads="$(grep -c '^head ' "$out/MANIFEST" || true)"
|
|
[ "$heads" -gt 0 ] || { echo "backup failed: no provisioned regions" >&2; exit 3; }
|
|
(cd "$out" && find . -type f ! -name SHA256SUMS -printf '%P\0' | sort -z | xargs -0 sha256sum -- > SHA256SUMS)
|
|
echo "backup complete regions=$heads objects=$count object_bytes=$bytes dump_bytes=$(stat -c %s "$out/database.dump") duration_ms=$(( $(date +%s%3N) - begin_ms )) out=$out"
|