#!/usr/bin/env bash # M8.9 version-barrier rehearsal (ADR-0076). Run by postgres-check.sh inside its # throwaway cluster (PG* set; EVO_SERVER_BIN, EVO_WORLD_CLONE_BIN, runtime roles # exported). Real evo-server processes on world 500: # 1. Failed migration: a migration that takes the migration lock, alters a history # table, holds its locks for 3 s and then fails runs while build b1 serves. # Nothing it did remains (schema still v16, no new column); the host keeps its # authority, and every acknowledgement stays durable. # 2. Barrier (ADR-0068 operator order): build b2 is retained, and a b2 host is started # while b1 still holds the lease (it waits, never ready). upgrade_barrier(b1) fences # b1, which quarantines (exit 4); b2 resumes at the committed head under a newer # epoch. A b1 host started afterwards quarantines at once, and the history across # both builds passes the full audit. set -euo pipefail root="$(cd "$(dirname "$0")/../.." && pwd)" work="$(mktemp -d /tmp/evo-upgrade-check.XXXXXXXX)" chmod 700 "$work" pids=() trap 'for p in "${pids[@]}"; do kill -9 "$p" 2> /dev/null || true; done; rm -rf -- "$work"' EXIT fail() { echo "upgrade rehearsal FAILED: $*" >&2; exit 1; } b1=b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1 b2=b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2 createdb evo_upgrade export PGDATABASE=evo_upgrade bash "$root/tools/migrate-postgres.sh" > /dev/null q() { psql -XAt -v ON_ERROR_STOP=1 "$@"; } mkdir -m 700 "$work/objects" pg=(--pg-host "$PGHOST" --pg-port "$PGPORT" --pg-database evo_upgrade) "$EVO_WORLD_CLONE_BIN" provision --world 500 --region 1 --opening development-v1 "${pg[@]}" \ --pg-user "$PGUSER" --objects "$work/objects" --build "$b1" --retain-build true > /dev/null serve() { # serve [retained...] local build="$1" log="$2"; shift 2 local retained=() for r in "$@"; do retained+=(--retained "$r"); done "$EVO_SERVER_BIN" --profile lineage --world 500 --region 1 --frame 9 "${pg[@]}" \ --pg-user "$EVO_PG_RUNTIME_USER" --objects "$work/objects" --build "$build" "${retained[@]}" \ --flush-ms 200 --lease-profile fast --synthetic-inputs development-v1 < /dev/null > "$log" 2>&1 & pids+=($!) server_pid=$! } acked_through() { awk -v t="$2" '$1 == "ack" && $3 >= t {f = 1} END {exit !f}' "$1"; } max_ack() { awk '$1 == "ack" && $3 > m {m = $3} END {print m + 0}' "$1"; } wait_ack() { for _ in $(seq 1 300); do acked_through "$1" "$2" && return 0; sleep 0.1; done; return 1; } head_tick() { q -c 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 500'; } serve "$b1" "$work/b1.log"; b1_pid=$server_pid wait_ack "$work/b1.log" 30 || fail "b1 acknowledged nothing" # 1. A failing migration while b1 serves. before="$(max_ack "$work/b1.log")" schema_before="$(q -c 'SELECT max(version) FROM evolution.schema_migrations')" code=0 q -v ON_ERROR_STOP=1 > "$work/migration.log" 2>&1 <<'SQL' || code=$? BEGIN; SET LOCAL lock_timeout = '5s'; SELECT pg_catalog.pg_advisory_xact_lock(71420001); ALTER TABLE evolution.region_commits ADD COLUMN rehearsal_column integer; SELECT pg_catalog.pg_sleep(3); SELECT 1 / 0; -- The migration fails after holding its locks. COMMIT; SQL [ "$code" != 0 ] && grep -q 'division by zero' "$work/migration.log" || fail "the rehearsal migration did not fail" [ "$(q -c "SELECT count(*) FROM information_schema.columns WHERE table_name = 'region_commits' AND column_name = 'rehearsal_column'")" = 0 ] || fail "the failed migration left a column" [ "$(q -c 'SELECT max(version) FROM evolution.schema_migrations')" = "$schema_before" ] || fail "schema version moved" wait_ack "$work/b1.log" $((before + 40)) || fail "b1 did not resume after the failed migration" kill -0 "$b1_pid" 2> /dev/null || fail "b1 exited during the failed migration" bash "$root/tools/migrate-postgres.sh" > /dev/null || fail "migrate after the failure" # 2. The barrier: retain b2, start its host, then retire b1. q -c "INSERT INTO evolution.retained_builds(build_identity) VALUES (decode('$b2', 'hex'))" > /dev/null serve "$b2" "$work/b2.log" "$b1"; b2_pid=$server_pid sleep 2 ! grep -q '^ready 1' "$work/b2.log" || fail "b2 became ready while b1 held the lease" [ "$(q -c "SELECT evolution.upgrade_barrier(decode('$b1', 'hex'))")" = 1 ] || fail "barrier fenced no lease" code=0; wait "$b1_pid" || code=$? [ "$code" = 4 ] || { tail "$work/b1.log" >&2; fail "b1 exited $code after the barrier, not 4"; } b1_final="$(max_ack "$work/b1.log")" wait_ack "$work/b2.log" $((b1_final + 30)) || { tail "$work/b2.log" >&2; fail "b2 did not take over"; } resumed="$(sed -n 's/^phase running epoch=\([0-9]*\) committed=\([0-9]*\) .*/\1 \2/p' "$work/b2.log" | head -1)" read -r epoch committed <<< "$resumed" [ "$epoch" -ge 2 ] && [ "$committed" -ge "$b1_final" ] || fail "b2 resumed at epoch=$epoch committed=$committed; b1 acknowledged $b1_final" [ "$(awk '$1 == "ack" {print $3; exit}' "$work/b2.log")" = $((committed + 1)) ] || fail "b2 skipped or repeated ticks" serve "$b1" "$work/late.log"; late_pid=$server_pid code=0; wait "$late_pid" || code=$? [ "$code" = 4 ] && ! grep -q '^ready 1' "$work/late.log" || fail "a late b1 host exited $code or served" kill -TERM "$b2_pid"; code=0; wait "$b2_pid" || code=$? [ "$code" = 0 ] || fail "b2 exited $code" audit="$("$EVO_WORLD_CLONE_BIN" audit --world 500 --region 1 "${pg[@]}" --pg-user "$PGUSER" \ --objects "$work/objects" --retained "$b1" --retained "$b2")" || fail "audit across builds" [[ "$audit" == *" tick=$(head_tick) "* ]] || fail "audit: $audit" builds="$(q -c 'SELECT count(DISTINCT build_identity) FROM evolution.region_commits WHERE world_id = 500')" [ "$builds" = 2 ] || fail "history holds $builds builds, expected 2" echo "upgrade rehearsal passed: failed migration held locks 3 s, b1 acknowledged $b1_final," \ "b2 resumed epoch=$epoch committed=$committed, head $(head_tick) ($audit)"