Files
aevolution/tools/ci/upgrade-check.sh
T
dingoandClaude Opus 5.5 bf5dcc7a6e
Foundation / checks (checks) (push) Successful in 5s
Foundation / checks (client) (push) Successful in 5s
Foundation / checks (network) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 5s
Foundation / checks (tsan) (push) Successful in 4s
Foundation / checks (deploy) (push) Successful in 3m28s
Foundation / checks (postgres) (push) Failing after 40m43s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
Check failed-migration schema version against its prior value
The upgrade rehearsal asserted the schema stayed at version 23 after a
deliberately failing migration, but migrations now reach 33, so it
reported "schema version moved" although nothing changed. Record the
version before the failing migration and compare against that. Refresh
the dev session window.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 16:28:57 -04:00

98 lines
5.8 KiB
Bash

#!/usr/bin/env bash
# M8.9 version-barrier rehearsal (ADR-0076). Run by postgres-check.sh inside its
# throwaway cluster (PG* set; EVO_SERVER_BIN, EVO_WORLD_CLONE_BIN, runtime roles
# exported). Real evo-server processes on world 500:
# 1. Failed migration: a migration that takes the migration lock, alters a history
# table, holds its locks for 3 s and then fails runs while build b1 serves.
# Nothing it did remains (schema still v16, no new column); the host keeps its
# authority, and every acknowledgement stays durable.
# 2. Barrier (ADR-0068 operator order): build b2 is retained, and a b2 host is started
# while b1 still holds the lease (it waits, never ready). upgrade_barrier(b1) fences
# b1, which quarantines (exit 4); b2 resumes at the committed head under a newer
# epoch. A b1 host started afterwards quarantines at once, and the history across
# both builds passes the full audit.
set -euo pipefail
root="$(cd "$(dirname "$0")/../.." && pwd)"
work="$(mktemp -d /tmp/evo-upgrade-check.XXXXXXXX)"
chmod 700 "$work"
pids=()
trap 'for p in "${pids[@]}"; do kill -9 "$p" 2> /dev/null || true; done; rm -rf -- "$work"' EXIT
fail() { echo "upgrade rehearsal FAILED: $*" >&2; exit 1; }
b1=b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1
b2=b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2
createdb evo_upgrade
export PGDATABASE=evo_upgrade
bash "$root/tools/migrate-postgres.sh" > /dev/null
q() { psql -XAt -v ON_ERROR_STOP=1 "$@"; }
mkdir -m 700 "$work/objects"
pg=(--pg-host "$PGHOST" --pg-port "$PGPORT" --pg-database evo_upgrade)
"$EVO_WORLD_CLONE_BIN" provision --world 500 --region 1 --opening development-v1 "${pg[@]}" \
--pg-user "$PGUSER" --objects "$work/objects" --build "$b1" --retain-build true > /dev/null
serve() { # serve <build> <log> [retained...]
local build="$1" log="$2"; shift 2
local retained=()
for r in "$@"; do retained+=(--retained "$r"); done
"$EVO_SERVER_BIN" --profile lineage --world 500 --region 1 --frame 9 "${pg[@]}" \
--pg-user "$EVO_PG_RUNTIME_USER" --objects "$work/objects" --build "$build" "${retained[@]}" \
--flush-ms 200 --lease-profile fast --synthetic-inputs development-v1 < /dev/null > "$log" 2>&1 &
pids+=($!)
server_pid=$!
}
acked_through() { awk -v t="$2" '$1 == "ack" && $3 >= t {f = 1} END {exit !f}' "$1"; }
max_ack() { awk '$1 == "ack" && $3 > m {m = $3} END {print m + 0}' "$1"; }
wait_ack() { for _ in $(seq 1 300); do acked_through "$1" "$2" && return 0; sleep 0.1; done; return 1; }
head_tick() { q -c 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 500'; }
serve "$b1" "$work/b1.log"; b1_pid=$server_pid
wait_ack "$work/b1.log" 30 || fail "b1 acknowledged nothing"
# 1. A failing migration while b1 serves.
before="$(max_ack "$work/b1.log")"
schema_before="$(q -c 'SELECT max(version) FROM evolution.schema_migrations')"
code=0
q -v ON_ERROR_STOP=1 > "$work/migration.log" 2>&1 <<'SQL' || code=$?
BEGIN;
SET LOCAL lock_timeout = '5s';
SELECT pg_catalog.pg_advisory_xact_lock(71420001);
ALTER TABLE evolution.region_commits ADD COLUMN rehearsal_column integer;
SELECT pg_catalog.pg_sleep(3);
SELECT 1 / 0; -- The migration fails after holding its locks.
COMMIT;
SQL
[ "$code" != 0 ] && grep -q 'division by zero' "$work/migration.log" || fail "the rehearsal migration did not fail"
[ "$(q -c "SELECT count(*) FROM information_schema.columns WHERE table_name = 'region_commits' AND column_name = 'rehearsal_column'")" = 0 ] ||
fail "the failed migration left a column"
[ "$(q -c 'SELECT max(version) FROM evolution.schema_migrations')" = "$schema_before" ] ||
fail "schema version moved"
wait_ack "$work/b1.log" $((before + 40)) || fail "b1 did not resume after the failed migration"
kill -0 "$b1_pid" 2> /dev/null || fail "b1 exited during the failed migration"
bash "$root/tools/migrate-postgres.sh" > /dev/null || fail "migrate after the failure"
# 2. The barrier: retain b2, start its host, then retire b1.
q -c "INSERT INTO evolution.retained_builds(build_identity) VALUES (decode('$b2', 'hex'))" > /dev/null
serve "$b2" "$work/b2.log" "$b1"; b2_pid=$server_pid
sleep 2
! grep -q '^ready 1' "$work/b2.log" || fail "b2 became ready while b1 held the lease"
[ "$(q -c "SELECT evolution.upgrade_barrier(decode('$b1', 'hex'))")" = 1 ] || fail "barrier fenced no lease"
code=0; wait "$b1_pid" || code=$?
[ "$code" = 4 ] || { tail "$work/b1.log" >&2; fail "b1 exited $code after the barrier, not 4"; }
b1_final="$(max_ack "$work/b1.log")"
wait_ack "$work/b2.log" $((b1_final + 30)) || { tail "$work/b2.log" >&2; fail "b2 did not take over"; }
resumed="$(sed -n 's/^phase running epoch=\([0-9]*\) committed=\([0-9]*\) .*/\1 \2/p' "$work/b2.log" | head -1)"
read -r epoch committed <<< "$resumed"
[ "$epoch" -ge 2 ] && [ "$committed" -ge "$b1_final" ] ||
fail "b2 resumed at epoch=$epoch committed=$committed; b1 acknowledged $b1_final"
[ "$(awk '$1 == "ack" {print $3; exit}' "$work/b2.log")" = $((committed + 1)) ] || fail "b2 skipped or repeated ticks"
serve "$b1" "$work/late.log"; late_pid=$server_pid
code=0; wait "$late_pid" || code=$?
[ "$code" = 4 ] && ! grep -q '^ready 1' "$work/late.log" || fail "a late b1 host exited $code or served"
kill -TERM "$b2_pid"; code=0; wait "$b2_pid" || code=$?
[ "$code" = 0 ] || fail "b2 exited $code"
audit="$("$EVO_WORLD_CLONE_BIN" audit --world 500 --region 1 "${pg[@]}" --pg-user "$PGUSER" \
--objects "$work/objects" --retained "$b1" --retained "$b2")" || fail "audit across builds"
[[ "$audit" == *" tick=$(head_tick) "* ]] || fail "audit: $audit"
builds="$(q -c 'SELECT count(DISTINCT build_identity) FROM evolution.region_commits WHERE world_id = 500')"
[ "$builds" = 2 ] || fail "history holds $builds builds, expected 2"
echo "upgrade rehearsal passed: failed migration held locks 3 s, b1 acknowledged $b1_final," \
"b2 resumed epoch=$epoch committed=$committed, head $(head_tick) ($audit)"