Files
aevolution/tools/ci/backup-check.sh
T
dingo 709dfb9557
Foundation / checks (ecs) (push) Successful in 2m23s
Foundation / checks (deploy) (push) Successful in 4m10s
Foundation / checks (network) (push) Successful in 21m32s
Foundation / checks (client) (push) Successful in 22m14s
Foundation / checks (tsan) (push) Successful in 24m30s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 41m43s
Foundation / checks (postgres) (push) Canceled after 41m39s
M12.7: bind census evidence to database history domains
2026-10-18 20:33:16 -04:00

192 lines
14 KiB
Bash

#!/usr/bin/env bash
# M8.7 backup and restore qualification (ADR-0074). Run by postgres-check.sh inside
# its throwaway cluster (PG* already set; EVO_SERVER_BIN, EVO_WORLD_CLONE_BIN and the
# runtime roles exported). A real evo-server commits under synthetic load while the
# backup is taken; the bundle is then restored on a separate cluster and rebuilt as a
# runnable isolated world, and damaged bundles are refused.
set -euo pipefail
root="$(cd "$(dirname "$0")/../.." && pwd)"
work="$(mktemp -d /tmp/evo-backup-check.XXXXXXXX)"
chmod 700 "$work"
server_pid=""
trap '[ -n "$server_pid" ] && kill -9 "$server_pid" 2> /dev/null; rm -rf -- "$work"' EXIT
fail() { echo "backup check FAILED: $*" >&2; exit 1; }
build=b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1
createdb evo_backup_src
export PGDATABASE=evo_backup_src
bash "$root/tools/migrate-postgres.sh" > /dev/null
mkdir -m 700 "$work/objects"
pg=(--pg-host "$PGHOST" --pg-port "$PGPORT" --pg-database evo_backup_src)
"$EVO_WORLD_CLONE_BIN" provision --world 300 --region 1 --opening development-v1 \
"${pg[@]}" --pg-user "$PGUSER" --objects "$work/objects" --build "$build" --retain-build true > /dev/null
"$EVO_SERVER_BIN" --profile lineage --world 300 --region 1 --frame 9 "${pg[@]}" \
--pg-user "$EVO_PG_RUNTIME_USER" --objects "$work/objects" --build "$build" \
--flush-ms 200 --lease-profile fast --synthetic-inputs development-v1 \
< /dev/null > "$work/server.log" 2>&1 &
server_pid=$!
acked_through() { awk -v t="$1" '$1 == "ack" && $3 >= t {f = 1} END {exit !f}' "$work/server.log"; }
max_ack() { awk '$1 == "ack" && $3 > m {m = $3} END {print m + 0}' "$work/server.log"; }
for _ in $(seq 1 300); do acked_through 60 && break; sleep 0.1; done
acked_through 60 || fail "the source server acknowledged nothing"
# Online backup while the server keeps committing.
acked_before="$(max_ack)"
bash "$root/tools/backup/backup-world.sh" --objects "$work/objects" --out "$work/bundle" --label check
read -r _ _ _ _ _ backup_tick _ < <(grep '^head 300 1 ' "$work/bundle/MANIFEST")
[ "$backup_tick" -ge "$acked_before" ] || fail "backup head $backup_tick misses acknowledged tick $acked_before"
for _ in $(seq 1 300); do acked_through $((backup_tick + 40)) && break; sleep 0.1; done
kill -TERM "$server_pid"; code=0; wait "$server_pid" || code=$?; server_pid=""
[ "$code" = 0 ] || fail "the source server exited $code"
source_tick="$(psql -XAtc 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 300')"
[ "$source_tick" -gt "$backup_tick" ] || fail "the source did not move past the backup ($source_tick)"
grep -q '^build b1b1' "$work/bundle/MANIFEST" && grep -q '^schema 9 ' "$work/bundle/MANIFEST" ||
fail "manifest lacks builds or schema versions"
if grep -q -i -E 'password|pgpass' "$work/bundle/MANIFEST"; then fail "manifest names credentials"; fi
# Full restore on an independent cluster, rebuilt and served as isolated world 301.
EVO_VERIFY_PORT=$((PGPORT + 1)) bash "$root/tools/backup/verify-backup.sh" "$work/bundle" \
--isolate 301 > "$work/verify.log" 2>&1 || { cat "$work/verify.log" >&2; fail "verify-backup"; }
grep -q "^RESTORE VERIFIED regions=1 " "$work/verify.log" &&
grep -q "PASS restore verified and world 300 rebuilt as isolated world 301" "$work/verify.log" ||
{ cat "$work/verify.log" >&2; fail "verify-backup evidence"; }
# Damaged bundles, restored straight into scratch databases of this cluster.
case_n=0
expect_refusal() { # expect_refusal <exit code> <bundle> <what>
case_n=$((case_n + 1))
createdb "evo_restore_case_$case_n"
local code=0
PGDATABASE="evo_restore_case_$case_n" bash "$root/tools/backup/restore-world.sh" \
--bundle "$2" --objects "$work/restored-$case_n" > "$work/case-$case_n.log" 2>&1 || code=$?
[ "$code" = "$1" ] || { cat "$work/case-$case_n.log" >&2; fail "$3: exit $code, expected $1"; }
[ ! -e "$work/restored-$case_n" ] || fail "$3: restored objects were left behind"
[ "$(PGDATABASE="evo_restore_case_$case_n" psql -XAtc "SELECT count(*) FROM pg_namespace WHERE nspname IN ('evolution', 'evolution_restore')")" = 0 ] ||
fail "$3: a restored schema was left behind"
}
damaged() { rm -rf "$work/damaged"; cp -a "$work/bundle" "$work/damaged"; }
resum() { (cd "$work/damaged" && find . -type f ! -name SHA256SUMS -printf '%P\0' | sort -z | xargs -0 sha256sum -- > SHA256SUMS); }
genesis="$(psql -XAtc "SELECT encode(payload_sha256, 'hex') FROM evolution.region_commits WHERE world_id = 300 AND commit_sequence = 0")"
genesis_object="$work/damaged/objects/w300/r1/$genesis"
damaged; printf 'corrupt' | dd of="$genesis_object" bs=1 conv=notrunc status=none
expect_refusal 4 "$work/damaged" "a corrupt object"
damaged; echo extra > "$work/damaged/objects/w300/r1/notes"
expect_refusal 4 "$work/damaged" "an unlisted file"
damaged; printf 'x' >> "$work/damaged/database.dump"
expect_refusal 4 "$work/damaged" "a damaged dump"
# Consistently re-summed damage passes the checksums and must fail verification.
damaged; rm "$genesis_object"; resum
expect_refusal 3 "$work/damaged" "a missing payload object"
grep -q 'audit failed for 300/1' "$work/case-$case_n.log" || fail "missing object: no audit failure"
damaged; sed -i -E "s/^(head 300 1 [0-9]+ [0-9]+) $backup_tick /\1 $((backup_tick - 1)) /" "$work/damaged/MANIFEST"; resum
expect_refusal 3 "$work/damaged" "a manifest head that disagrees with the database"
# A target that already holds a world is refused before anything is written.
createdb evo_restore_twice
PGDATABASE=evo_restore_twice bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" \
--objects "$work/twice-a" > /dev/null
code=0
# A restored database keeps the biological world/cuts but gets a distinct
# evidence namespace before publication, including the isolated workflow.
source_domain="$(psql -XAtc 'SELECT evolution.load_history_domain_v1()')"
restored_domain="$(PGDATABASE=evo_restore_twice psql -XAtc 'SELECT evolution.load_history_domain_v1()')"
[ -n "$restored_domain" ] && [ "$restored_domain" != "$source_domain" ] || fail 'restore reused source history domain'
[ "$(PGDATABASE=evo_restore_twice psql -XAtc "SELECT parent FROM evolution.history_domains WHERE domain='$restored_domain'")" = "$source_domain" ] || fail 'restore lost parent domain'
[ "$(PGDATABASE=evo_restore_twice psql -XAtc 'SELECT count(*) FROM evolution.history_domains d JOIN evolution.current_history_domain h USING(domain) JOIN evolution_restore.claims c ON d.request=c.nonce::uuid')" = 1 ] || fail 'restore lost claim/domain binding'
[ "$(PGDATABASE=evo_restore_twice psql -XAtc "SELECT count(*) FROM evolution.history_domain_cuts d JOIN evolution.region_heads h USING(world_id,region_id) WHERE d.domain='$restored_domain' AND d.commit_sequence=h.commit_sequence AND d.commit_id=h.commit_id AND d.digest=h.digest AND d.end_tick=h.end_tick")" = 1 ] || fail 'restore lost exact cut provenance'
PGDATABASE=evo_restore_twice bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" \
--objects "$work/twice-b" > "$work/twice.log" 2>&1 || code=$?
[ "$code" = 4 ] && grep -q 'already has an evolution schema' "$work/twice.log" || fail "a second restore into one database"
[ "$(PGDATABASE=evo_restore_twice psql -XAtc 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 300')" = "$backup_tick" ] ||
fail "the refused second restore changed the first"
# Ownership and failure paths (M8 review P1; follow-up P1/P2). Each case names the
# boundary it interrupts; only this invocation's state may ever be removed.
restore() { # restore <database> <objects> [env...]: runs restore-world.sh, sets rc.
local db="$1" dir="$2"; shift 2
rc=0
env PGDATABASE="$db" "$@" bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" \
--objects "$dir" > "$work/$(basename "$dir").log" 2>&1 || rc=$?
}
schemas() { PGDATABASE="$1" psql -XAtc "SELECT count(*) FROM pg_namespace WHERE nspname IN ('evolution', 'evolution_restore')"; }
acquire_refused() { # A lease acquisition in <database> is refused (the world is unservable).
! PGDATABASE="$1" psql -XAt -v ON_ERROR_STOP=1 \
-c "SELECT * FROM evolution.acquire_region_lease(300, 1, 20000, decode(repeat('b1', 32), 'hex'))" \
> "$work/acquire-$1.log" 2>&1 && grep -q 'build not current' "$work/acquire-$1.log"
}
log_of() { cat "$work/$(basename "$1").log" >&2; }
# A competing actor creates the schema after this restore's checks: its install
# transaction rolls back (confirmed), and the competitor's schema survives.
createdb evo_restore_race
restore evo_restore_race "$work/race-a" EVO_RESTORE_TEST_HOOK='[ "$1" = before-install ] && psql -XAtq -c "CREATE SCHEMA evolution" -c "CREATE TABLE evolution.winner(x int)" || true'
[ "$rc" = 4 ] && grep -q 'rolled back; the database was not changed (confirmed)' "$work/race-a.log" ||
{ log_of race-a; fail "a pre-empted restore: exit $rc"; }
[ "$(PGDATABASE=evo_restore_race psql -XAtc "SELECT count(*) FROM pg_tables WHERE schemaname = 'evolution' AND tablename = 'winner'")" = 1 ] ||
fail "a pre-empted restore deleted the competing schema"
[ ! -e "$work/race-a" ] || fail "a pre-empted restore left its own objects directory"
# Two restores of one database: whichever installs first wins; the other rolls back
# and leaves the winner published and intact.
createdb evo_restore_pair
restore evo_restore_pair "$work/pair-a" EVO_RESTORE_TEST_HOOK='[ "$1" = before-install ] && sleep 4 || true' &
pair_pid=$!
sleep 1
restore evo_restore_pair "$work/pair-b"
[ "$rc" = 0 ] && grep -q '^RESTORE VERIFIED regions=1 ' "$work/pair-b.log" || { log_of pair-b; fail "the first installer: exit $rc"; }
code=0; wait "$pair_pid" || code=$?
grep -q 'rolled back; the database was not changed (confirmed)' "$work/pair-a.log" && [ ! -e "$work/pair-a" ] ||
{ log_of pair-a; fail "the second installer: exit $code"; }
[ "$(PGDATABASE=evo_restore_pair psql -XAtc "SELECT state FROM evolution_restore.claims")" = published ] &&
[ "$(PGDATABASE=evo_restore_pair psql -XAtc 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 300')" = "$backup_tick" ] ||
fail "the losing restore disturbed the winner"
# After install and before publication the world is unservable, durably: across a
# database restart (every connection and lock lost) acquisition is still refused, and
# the restore then verifies and publishes; afterwards the build is current again.
createdb evo_restore_gate
restore evo_restore_gate "$work/gate" EVO_RESTORE_TEST_HOOK='
if [ "$1" = after-install ]; then
psql -XAt -c "SELECT count(*) FROM evolution.history_domains d JOIN evolution.current_history_domain h USING(domain) JOIN evolution_restore.claims c ON d.request=c.nonce::uuid" > "'"$work"'/gate-domain.log"
psql -XAt -c "SELECT * FROM evolution.acquire_region_lease(300, 1, 20000, decode(repeat('"'"'b1'"'"', 32), '"'"'hex'"'"'))" > "'"$work"'/gate-before.log" 2>&1 || true
pg_ctl -D "$PGHOST/data" -m fast -w restart > /dev/null
psql -XAt -c "SELECT * FROM evolution.acquire_region_lease(300, 1, 20000, decode(repeat('"'"'b1'"'"', 32), '"'"'hex'"'"'))" > "'"$work"'/gate-after.log" 2>&1 || true
fi'
[ "$rc" = 0 ] && grep -q '^RESTORE VERIFIED regions=1 ' "$work/gate.log" || { log_of gate; fail "a restore across a restart: exit $rc"; }
grep -q 'build not current' "$work/gate-before.log" && grep -q 'build not current' "$work/gate-after.log" ||
{ cat "$work/gate-before.log" "$work/gate-after.log" >&2; fail "authority was available before publication"; }
[ "$(PGDATABASE=evo_restore_gate psql -XAtc 'SELECT count(*) FROM evolution.retained_builds WHERE retired_at IS NULL')" = 1 ] ||
fail "publication did not restore the current build"
[ "$(cat "$work/gate-domain.log")" = 1 ] || fail 'domain was not forked before verification/readmission'
# A committed install whose completion is lost is resolved from the claim and
# verified, not reported as unchanged.
createdb evo_restore_lost
restore evo_restore_lost "$work/lost" EVO_RESTORE_TEST_INSTALL_ACK=lost
[ "$rc" = 0 ] && grep -q '^install outcome resolved: committed' "$work/lost.log" &&
grep -q '^RESTORE VERIFIED regions=1 ' "$work/lost.log" || { log_of lost; fail "a lost install acknowledgement: exit $rc"; }
# An install whose outcome cannot be resolved keeps everything, stays unservable,
# and is removed only by --abandon with its claim.
createdb evo_restore_unknown
restore evo_restore_unknown "$work/unknown" EVO_RESTORE_TEST_INSTALL_ACK=unreachable
nonce="$(sed -n 's/^restore claim \([0-9a-f]\{32\}\)$/\1/p' "$work/unknown.log")"
[ "$rc" = 5 ] && grep -q '^RESTORE OUTCOME UNKNOWN' "$work/unknown.log" && [ -d "$work/unknown" ] && [ -n "$nonce" ] ||
{ log_of unknown; fail "an unresolvable install: exit $rc"; }
[ "$(schemas evo_restore_unknown)" = 2 ] && acquire_refused evo_restore_unknown || fail "an unresolved restore is servable"
code=0
PGDATABASE=evo_restore_unknown bash "$root/tools/backup/restore-world.sh" --abandon 00000000000000000000000000000000 \
--objects "$work/unknown" > "$work/abandon-wrong.log" 2>&1 || code=$?
[ "$(schemas evo_restore_unknown)" = 2 ] && [ -d "$work/unknown" ] || fail "--abandon with another claim removed state"
PGDATABASE=evo_restore_unknown bash "$root/tools/backup/restore-world.sh" --abandon "$nonce" \
--objects "$work/unknown" > "$work/abandon.log" 2>&1 || { cat "$work/abandon.log" >&2; fail "--abandon"; }
[ "$(schemas evo_restore_unknown)" = 0 ] && [ ! -e "$work/unknown" ] || fail "--abandon left state behind"
restore evo_restore_unknown "$work/unknown-retry"
[ "$rc" = 0 ] || { log_of unknown-retry; fail "a restore after --abandon: exit $rc"; }
# An existing objects path is never claimed or removed.
mkdir "$work/occupied"; echo keep > "$work/occupied/mine"
createdb evo_restore_occupied
restore evo_restore_occupied "$work/occupied"
[ "$rc" = 4 ] && [ "$(cat "$work/occupied/mine")" = keep ] && [ "$(schemas evo_restore_occupied)" = 0 ] ||
fail "an occupied objects path: exit $rc"
echo "backup and restore qualification passed (backup tick $backup_tick, acknowledged $acked_before, source $source_tick)"