Files
aevolution/tools/ci/upgrade-check.sh
T
dingo bb6e00ebca
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (checks) (push) Failing after 43s
Foundation / checks (ecs) (push) Successful in 1m52s
Foundation / checks (postgres) (push) Failing after 52s
Foundation / checks (tsan) (push) Failing after 32s
Foundation / checks (network) (push) Failing after 3m4s
Foundation / checks (client) (push) Failing after 4m24s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
Publish cellular births with durable control
2026-10-11 00:44:57 -04:00

96 lines
5.7 KiB
Bash

#!/usr/bin/env bash
# M8.9 version-barrier rehearsal (ADR-0076). Run by postgres-check.sh inside its
# throwaway cluster (PG* set; EVO_SERVER_BIN, EVO_WORLD_CLONE_BIN, runtime roles
# exported). Real evo-server processes on world 500:
# 1. Failed migration: a migration that takes the migration lock, alters a history
# table, holds its locks for 3 s and then fails runs while build b1 serves.
# Nothing it did remains (schema still v16, no new column); the host keeps its
# authority, and every acknowledgement stays durable.
# 2. Barrier (ADR-0068 operator order): build b2 is retained, and a b2 host is started
# while b1 still holds the lease (it waits, never ready). upgrade_barrier(b1) fences
# b1, which quarantines (exit 4); b2 resumes at the committed head under a newer
# epoch. A b1 host started afterwards quarantines at once, and the history across
# both builds passes the full audit.
set -euo pipefail
root="$(cd "$(dirname "$0")/../.." && pwd)"
work="$(mktemp -d /tmp/evo-upgrade-check.XXXXXXXX)"
chmod 700 "$work"
pids=()
trap 'for p in "${pids[@]}"; do kill -9 "$p" 2> /dev/null || true; done; rm -rf -- "$work"' EXIT
fail() { echo "upgrade rehearsal FAILED: $*" >&2; exit 1; }
b1=b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1
b2=b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2
createdb evo_upgrade
export PGDATABASE=evo_upgrade
bash "$root/tools/migrate-postgres.sh" > /dev/null
q() { psql -XAt -v ON_ERROR_STOP=1 "$@"; }
mkdir -m 700 "$work/objects"
pg=(--pg-host "$PGHOST" --pg-port "$PGPORT" --pg-database evo_upgrade)
"$EVO_WORLD_CLONE_BIN" provision --world 500 --region 1 --opening development-v1 "${pg[@]}" \
--pg-user "$PGUSER" --objects "$work/objects" --build "$b1" --retain-build true > /dev/null
serve() { # serve <build> <log> [retained...]
local build="$1" log="$2"; shift 2
local retained=()
for r in "$@"; do retained+=(--retained "$r"); done
"$EVO_SERVER_BIN" --profile lineage --world 500 --region 1 --frame 9 "${pg[@]}" \
--pg-user "$EVO_PG_RUNTIME_USER" --objects "$work/objects" --build "$build" "${retained[@]}" \
--flush-ms 200 --lease-profile fast --synthetic-inputs development-v1 < /dev/null > "$log" 2>&1 &
pids+=($!)
server_pid=$!
}
acked_through() { awk -v t="$2" '$1 == "ack" && $3 >= t {f = 1} END {exit !f}' "$1"; }
max_ack() { awk '$1 == "ack" && $3 > m {m = $3} END {print m + 0}' "$1"; }
wait_ack() { for _ in $(seq 1 300); do acked_through "$1" "$2" && return 0; sleep 0.1; done; return 1; }
head_tick() { q -c 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 500'; }
serve "$b1" "$work/b1.log"; b1_pid=$server_pid
wait_ack "$work/b1.log" 30 || fail "b1 acknowledged nothing"
# 1. A failing migration while b1 serves.
before="$(max_ack "$work/b1.log")"
code=0
q -v ON_ERROR_STOP=1 > "$work/migration.log" 2>&1 <<'SQL' || code=$?
BEGIN;
SET LOCAL lock_timeout = '5s';
SELECT pg_catalog.pg_advisory_xact_lock(71420001);
ALTER TABLE evolution.region_commits ADD COLUMN rehearsal_column integer;
SELECT pg_catalog.pg_sleep(3);
SELECT 1 / 0; -- The migration fails after holding its locks.
COMMIT;
SQL
[ "$code" != 0 ] && grep -q 'division by zero' "$work/migration.log" || fail "the rehearsal migration did not fail"
[ "$(q -c "SELECT count(*) FROM information_schema.columns WHERE table_name = 'region_commits' AND column_name = 'rehearsal_column'")" = 0 ] ||
fail "the failed migration left a column"
[ "$(q -c 'SELECT max(version) FROM evolution.schema_migrations')" = 23 ] || fail "schema version moved"
wait_ack "$work/b1.log" $((before + 40)) || fail "b1 did not resume after the failed migration"
kill -0 "$b1_pid" 2> /dev/null || fail "b1 exited during the failed migration"
bash "$root/tools/migrate-postgres.sh" > /dev/null || fail "migrate after the failure"
# 2. The barrier: retain b2, start its host, then retire b1.
q -c "INSERT INTO evolution.retained_builds(build_identity) VALUES (decode('$b2', 'hex'))" > /dev/null
serve "$b2" "$work/b2.log" "$b1"; b2_pid=$server_pid
sleep 2
! grep -q '^ready 1' "$work/b2.log" || fail "b2 became ready while b1 held the lease"
[ "$(q -c "SELECT evolution.upgrade_barrier(decode('$b1', 'hex'))")" = 1 ] || fail "barrier fenced no lease"
code=0; wait "$b1_pid" || code=$?
[ "$code" = 4 ] || { tail "$work/b1.log" >&2; fail "b1 exited $code after the barrier, not 4"; }
b1_final="$(max_ack "$work/b1.log")"
wait_ack "$work/b2.log" $((b1_final + 30)) || { tail "$work/b2.log" >&2; fail "b2 did not take over"; }
resumed="$(sed -n 's/^phase running epoch=\([0-9]*\) committed=\([0-9]*\) .*/\1 \2/p' "$work/b2.log" | head -1)"
read -r epoch committed <<< "$resumed"
[ "$epoch" -ge 2 ] && [ "$committed" -ge "$b1_final" ] ||
fail "b2 resumed at epoch=$epoch committed=$committed; b1 acknowledged $b1_final"
[ "$(awk '$1 == "ack" {print $3; exit}' "$work/b2.log")" = $((committed + 1)) ] || fail "b2 skipped or repeated ticks"
serve "$b1" "$work/late.log"; late_pid=$server_pid
code=0; wait "$late_pid" || code=$?
[ "$code" = 4 ] && ! grep -q '^ready 1' "$work/late.log" || fail "a late b1 host exited $code or served"
kill -TERM "$b2_pid"; code=0; wait "$b2_pid" || code=$?
[ "$code" = 0 ] || fail "b2 exited $code"
audit="$("$EVO_WORLD_CLONE_BIN" audit --world 500 --region 1 "${pg[@]}" --pg-user "$PGUSER" \
--objects "$work/objects" --retained "$b1" --retained "$b2")" || fail "audit across builds"
[[ "$audit" == *" tick=$(head_tick) "* ]] || fail "audit: $audit"
builds="$(q -c 'SELECT count(DISTINCT build_identity) FROM evolution.region_commits WHERE world_id = 500')"
[ "$builds" = 2 ] || fail "history holds $builds builds, expected 2"
echo "upgrade rehearsal passed: failed migration held locks 3 s, b1 acknowledged $b1_final," \
"b2 resumed epoch=$epoch committed=$committed, head $(head_tick) ($audit)"