Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 1m50s
Foundation / checks (network) (push) Successful in 9m8s
Foundation / checks (client) (push) Successful in 12m10s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (postgres) (push) Canceled after 7m33s
Foundation / checks (checks) (push) Canceled after 18m49s
Foundation / checks (tsan) (push) Canceled after 6m38s
Migration v10 keeps, per world and account, the highest accepted grant serial and the region session in control with a fencing generation and database-time expiry, reachable only through claim/renew/release functions under the caller's region lease. Serials survive restarts and apply across regions; a cross-region takeover returns a wait so control never overlaps. RegionSessions takes durable control via a ControlLedger before admitting a session, delays Welcome by the wait, renews, and closes sessions whose control is fenced or cannot be confirmed in time. PostgresControlLedger implements the ledger. ADR-0091. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
96 lines
5.7 KiB
Bash
96 lines
5.7 KiB
Bash
#!/usr/bin/env bash
|
|
# M8.9 version-barrier rehearsal (ADR-0076). Run by postgres-check.sh inside its
|
|
# throwaway cluster (PG* set; EVO_SERVER_BIN, EVO_WORLD_CLONE_BIN, runtime roles
|
|
# exported). Real evo-server processes on world 500:
|
|
# 1. Failed migration: a migration that takes the migration lock, alters a history
|
|
# table, holds its locks for 3 s and then fails runs while build b1 serves.
|
|
# Nothing it did remains (schema still v10, no new column); the host keeps its
|
|
# authority, and every acknowledgement stays durable.
|
|
# 2. Barrier (ADR-0068 operator order): build b2 is retained, and a b2 host is started
|
|
# while b1 still holds the lease (it waits, never ready). upgrade_barrier(b1) fences
|
|
# b1, which quarantines (exit 4); b2 resumes at the committed head under a newer
|
|
# epoch. A b1 host started afterwards quarantines at once, and the history across
|
|
# both builds passes the full audit.
|
|
set -euo pipefail
|
|
root="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
work="$(mktemp -d /tmp/evo-upgrade-check.XXXXXXXX)"
|
|
chmod 700 "$work"
|
|
pids=()
|
|
trap 'for p in "${pids[@]}"; do kill -9 "$p" 2> /dev/null || true; done; rm -rf -- "$work"' EXIT
|
|
fail() { echo "upgrade rehearsal FAILED: $*" >&2; exit 1; }
|
|
b1=b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1
|
|
b2=b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2
|
|
createdb evo_upgrade
|
|
export PGDATABASE=evo_upgrade
|
|
bash "$root/tools/migrate-postgres.sh" > /dev/null
|
|
q() { psql -XAt -v ON_ERROR_STOP=1 "$@"; }
|
|
mkdir -m 700 "$work/objects"
|
|
pg=(--pg-host "$PGHOST" --pg-port "$PGPORT" --pg-database evo_upgrade)
|
|
"$EVO_WORLD_CLONE_BIN" provision --world 500 --region 1 --opening development-v1 "${pg[@]}" \
|
|
--pg-user "$PGUSER" --objects "$work/objects" --build "$b1" --retain-build true > /dev/null
|
|
serve() { # serve <build> <log> [retained...]
|
|
local build="$1" log="$2"; shift 2
|
|
local retained=()
|
|
for r in "$@"; do retained+=(--retained "$r"); done
|
|
"$EVO_SERVER_BIN" --profile lineage --world 500 --region 1 --frame 9 "${pg[@]}" \
|
|
--pg-user "$EVO_PG_RUNTIME_USER" --objects "$work/objects" --build "$build" "${retained[@]}" \
|
|
--flush-ms 200 --lease-profile fast --synthetic-inputs development-v1 < /dev/null > "$log" 2>&1 &
|
|
pids+=($!)
|
|
server_pid=$!
|
|
}
|
|
acked_through() { awk -v t="$2" '$1 == "ack" && $3 >= t {f = 1} END {exit !f}' "$1"; }
|
|
max_ack() { awk '$1 == "ack" && $3 > m {m = $3} END {print m + 0}' "$1"; }
|
|
wait_ack() { for _ in $(seq 1 300); do acked_through "$1" "$2" && return 0; sleep 0.1; done; return 1; }
|
|
head_tick() { q -c 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 500'; }
|
|
|
|
serve "$b1" "$work/b1.log"; b1_pid=$server_pid
|
|
wait_ack "$work/b1.log" 30 || fail "b1 acknowledged nothing"
|
|
|
|
# 1. A failing migration while b1 serves.
|
|
before="$(max_ack "$work/b1.log")"
|
|
code=0
|
|
q -v ON_ERROR_STOP=1 > "$work/migration.log" 2>&1 <<'SQL' || code=$?
|
|
BEGIN;
|
|
SET LOCAL lock_timeout = '5s';
|
|
SELECT pg_catalog.pg_advisory_xact_lock(71420001);
|
|
ALTER TABLE evolution.region_commits ADD COLUMN rehearsal_column integer;
|
|
SELECT pg_catalog.pg_sleep(3);
|
|
SELECT 1 / 0; -- The migration fails after holding its locks.
|
|
COMMIT;
|
|
SQL
|
|
[ "$code" != 0 ] && grep -q 'division by zero' "$work/migration.log" || fail "the rehearsal migration did not fail"
|
|
[ "$(q -c "SELECT count(*) FROM information_schema.columns WHERE table_name = 'region_commits' AND column_name = 'rehearsal_column'")" = 0 ] ||
|
|
fail "the failed migration left a column"
|
|
[ "$(q -c 'SELECT max(version) FROM evolution.schema_migrations')" = 10 ] || fail "schema version moved"
|
|
wait_ack "$work/b1.log" $((before + 40)) || fail "b1 did not resume after the failed migration"
|
|
kill -0 "$b1_pid" 2> /dev/null || fail "b1 exited during the failed migration"
|
|
bash "$root/tools/migrate-postgres.sh" > /dev/null || fail "migrate after the failure"
|
|
|
|
# 2. The barrier: retain b2, start its host, then retire b1.
|
|
q -c "INSERT INTO evolution.retained_builds(build_identity) VALUES (decode('$b2', 'hex'))" > /dev/null
|
|
serve "$b2" "$work/b2.log" "$b1"; b2_pid=$server_pid
|
|
sleep 2
|
|
! grep -q '^ready 1' "$work/b2.log" || fail "b2 became ready while b1 held the lease"
|
|
[ "$(q -c "SELECT evolution.upgrade_barrier(decode('$b1', 'hex'))")" = 1 ] || fail "barrier fenced no lease"
|
|
code=0; wait "$b1_pid" || code=$?
|
|
[ "$code" = 4 ] || { tail "$work/b1.log" >&2; fail "b1 exited $code after the barrier, not 4"; }
|
|
b1_final="$(max_ack "$work/b1.log")"
|
|
wait_ack "$work/b2.log" $((b1_final + 30)) || { tail "$work/b2.log" >&2; fail "b2 did not take over"; }
|
|
resumed="$(sed -n 's/^phase running epoch=\([0-9]*\) committed=\([0-9]*\) .*/\1 \2/p' "$work/b2.log" | head -1)"
|
|
read -r epoch committed <<< "$resumed"
|
|
[ "$epoch" -ge 2 ] && [ "$committed" -ge "$b1_final" ] ||
|
|
fail "b2 resumed at epoch=$epoch committed=$committed; b1 acknowledged $b1_final"
|
|
[ "$(awk '$1 == "ack" {print $3; exit}' "$work/b2.log")" = $((committed + 1)) ] || fail "b2 skipped or repeated ticks"
|
|
serve "$b1" "$work/late.log"; late_pid=$server_pid
|
|
code=0; wait "$late_pid" || code=$?
|
|
[ "$code" = 4 ] && ! grep -q '^ready 1' "$work/late.log" || fail "a late b1 host exited $code or served"
|
|
kill -TERM "$b2_pid"; code=0; wait "$b2_pid" || code=$?
|
|
[ "$code" = 0 ] || fail "b2 exited $code"
|
|
audit="$("$EVO_WORLD_CLONE_BIN" audit --world 500 --region 1 "${pg[@]}" --pg-user "$PGUSER" \
|
|
--objects "$work/objects" --retained "$b1" --retained "$b2")" || fail "audit across builds"
|
|
[[ "$audit" == *" tick=$(head_tick) "* ]] || fail "audit: $audit"
|
|
builds="$(q -c 'SELECT count(DISTINCT build_identity) FROM evolution.region_commits WHERE world_id = 500')"
|
|
[ "$builds" = 2 ] || fail "history holds $builds builds, expected 2"
|
|
echo "upgrade rehearsal passed: failed migration held locks 3 s, b1 acknowledged $b1_final," \
|
|
"b2 resumed epoch=$epoch committed=$committed, head $(head_tick) ($audit)"
|