Drill-only --synthetic-until/--accelerate and evo-world-clone compare (compare_region_histories) prove a candidate build runs the same world on clones of one cut before any barrier. upgrade-check.sh rehearses a failed migration under load and the version barrier with real hosts; upgrade-barrier.sh retires an old build in dev/stage only once the new digest is Ready. Nodes: aevolution.io/stores pins every objects-volume pod, aevolution.io/compute is preference-only (Alertmanager first), checked by policy and scheduling-check.sh. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
196 lines
15 KiB
Bash
196 lines
15 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
root="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
prefix="$root/.cache/postgres/install"
|
|
export PATH="$prefix/bin:$PATH"
|
|
export LD_LIBRARY_PATH="$prefix/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
|
|
test_dir="$(mktemp -d /tmp/evo-pg-test.XXXXXXXX)"
|
|
chmod 700 "$test_dir"
|
|
export PGHOST="$test_dir" PGPORT=55432 PGDATABASE=postgres PGUSER="$(id -un)"
|
|
unset PGHOSTADDR PGPASSWORD PGSERVICE PGSERVICEFILE
|
|
initdb -D "$test_dir/data" --no-locale --encoding=UTF8 --auth-local=trust --auth-host=reject > "$test_dir/init.log"
|
|
trap 'pg_ctl -D "$test_dir/data" -m immediate stop > /dev/null || true' EXIT
|
|
pg_ctl -D "$test_dir/data" -l "$test_dir/server.log" -o "-k $test_dir -p 55432 -c listen_addresses=''" -w start
|
|
# Runtime group role receives grants; a separate login inherits only that role.
|
|
psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_runtime_test NOLOGIN' \
|
|
-c 'CREATE ROLE evo_runtime_login LOGIN IN ROLE evo_runtime_test'
|
|
export EVO_PG_RUNTIME_ROLE=evo_runtime_test EVO_PG_RUNTIME_USER=evo_runtime_login
|
|
bash "$root/tools/migrate-postgres.sh"
|
|
bash "$root/tools/migrate-postgres.sh" # exact supported upgrade/restart is a no-op
|
|
# M7.9 clone target: a separate database with its own runtime group and login,
|
|
# so a clone shares neither the source's database nor its runtime credentials.
|
|
psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_clone_runtime NOLOGIN' -c 'CREATE ROLE evo_clone_login LOGIN IN ROLE evo_clone_runtime'
|
|
createdb evo_clone
|
|
(export PGDATABASE=evo_clone EVO_PG_RUNTIME_ROLE=evo_clone_runtime; bash "$root/tools/migrate-postgres.sh")
|
|
export EVO_PG_CLONE_DATABASE=evo_clone EVO_PG_CLONE_RUNTIME_USER=evo_clone_login
|
|
# The online migration test (M7.9b) runs the checked-in migrations from here.
|
|
export EVO_SOURCE_ROOT="$root"
|
|
# Process tests spawn the server and clone tool built beside the test binary.
|
|
export EVO_SERVER_BIN EVO_WORLD_CLONE_BIN
|
|
EVO_SERVER_BIN="$(dirname "${1:?path to evo-postgres-tests}")/evo-server"
|
|
EVO_WORLD_CLONE_BIN="$(dirname "$1")/evo-world-clone"
|
|
"$1"
|
|
# M8.7: online backup under load, restore on a separate cluster, isolated rebuild.
|
|
bash "$root/tools/ci/backup-check.sh"
|
|
# M8.8: host-loss rehearsal: in-place restore, fenced old environment, measured rewind.
|
|
bash "$root/tools/ci/dr-check.sh"
|
|
# M8.9: failed migration under load and the version barrier with real hosts.
|
|
bash "$root/tools/ci/upgrade-check.sh"
|
|
v1_sha="$(sha256sum "$root/migrations/0001_region_bootstrap.sql")"
|
|
v1_sha="${v1_sha%% *}"
|
|
# Migration failure must not publish a partial schema.
|
|
createdb migration_rollback
|
|
if (export PGDATABASE=migration_rollback; psql -X -v ON_ERROR_STOP=1 -1 \
|
|
-v migration_sha=invalid -f "$root/migrations/0001_region_bootstrap.sql"); then
|
|
echo 'Expected failed migration' >&2; exit 1
|
|
fi
|
|
test "$(PGDATABASE=migration_rollback psql -XAtc "SELECT count(*) FROM pg_namespace WHERE nspname='evolution'")" = 0
|
|
# A privileged or owner runtime role is refused and leaves no schema behind.
|
|
if (export PGDATABASE=migration_rollback EVO_PG_RUNTIME_ROLE="$PGUSER"; \
|
|
bash "$root/tools/migrate-postgres.sh"); then
|
|
echo 'Expected privileged runtime role rejection' >&2; exit 1
|
|
fi
|
|
if (export PGDATABASE=migration_rollback EVO_PG_RUNTIME_ROLE=no_such_evo_role; \
|
|
bash "$root/tools/migrate-postgres.sh"); then
|
|
echo 'Expected missing runtime role rejection' >&2; exit 1
|
|
fi
|
|
test "$(PGDATABASE=migration_rollback psql -XAtc "SELECT count(*) FROM pg_namespace WHERE nspname='evolution'")" = 0
|
|
# Supported upgrade path: an existing exact v1 database migrates to the latest.
|
|
createdb upgrade_path
|
|
PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" \
|
|
-f "$root/migrations/0001_region_bootstrap.sql" > /dev/null
|
|
test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1
|
|
PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh"
|
|
test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9
|
|
# v2 heads have no payload binding: v2->v3 fails closed and leaves history at v2.
|
|
v2_sha="$(sha256sum "$root/migrations/0002_region_fencing.sql")"
|
|
v2_sha="${v2_sha%% *}"
|
|
createdb v2_heads
|
|
PGDATABASE=v2_heads psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" -v "migration_sha_2=$v2_sha" -v runtime_role=evo_runtime_test -f "$root/migrations/0001_region_bootstrap.sql" -f "$root/migrations/0002_region_fencing.sql" > /dev/null
|
|
PGDATABASE=v2_heads psql -X -v ON_ERROR_STOP=1 -c "SELECT evolution.provision_region(1, 1, 1, gen_random_uuid(), decode(repeat('00', 32), 'hex'))" > /dev/null
|
|
if PGDATABASE=v2_heads bash "$root/tools/migrate-postgres.sh"; then
|
|
echo 'Expected v2 heads to block v3' >&2; exit 1
|
|
fi
|
|
v3_sha="$(sha256sum "$root/migrations/0003_region_checkpoints.sql")"
|
|
v3_sha="${v3_sha%% *}"
|
|
createdb v3_commits
|
|
PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" \
|
|
-v "migration_sha_2=$v2_sha" -v "migration_sha_3=$v3_sha" -v runtime_role=evo_runtime_test \
|
|
-f "$root/migrations/0001_region_bootstrap.sql" -f "$root/migrations/0002_region_fencing.sql" \
|
|
-f "$root/migrations/0003_region_checkpoints.sql" > /dev/null
|
|
PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 \
|
|
-c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \
|
|
-c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '01000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32), 'hex'))" > /dev/null
|
|
PGDATABASE=v3_commits bash "$root/tools/migrate-postgres.sh"
|
|
test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 9:1,2,3,4,5,6,7,8,9
|
|
test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) FROM evolution.region_commits")" = 1
|
|
# Review P1: a v4 database holding a published (possibly terminal) checkpoint has
|
|
# no durable decisions for it. The v5 upgrade must refuse before changing anything.
|
|
v4_sha="$(sha256sum "$root/migrations/0004_lineage_profile.sql")"
|
|
v4_sha="${v4_sha%% *}"
|
|
createdb v4_published
|
|
PGDATABASE=v4_published psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" \
|
|
-v "migration_sha_2=$v2_sha" -v "migration_sha_3=$v3_sha" -v "migration_sha_4=$v4_sha" \
|
|
-v runtime_role=evo_runtime_test \
|
|
-f "$root/migrations/0001_region_bootstrap.sql" -f "$root/migrations/0002_region_fencing.sql" \
|
|
-f "$root/migrations/0003_region_checkpoints.sql" -f "$root/migrations/0004_lineage_profile.sql" > /dev/null
|
|
PGDATABASE=v4_published psql -X -v ON_ERROR_STOP=1 > /dev/null <<'SQL'
|
|
INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'));
|
|
SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '01000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32), 'hex'));
|
|
SELECT lease_holder, lease_epoch FROM evolution.acquire_region_lease(1, 1, 20000) \gset
|
|
-- A real v4 publication of ticks 0..20 through the v4 fenced function.
|
|
SELECT p.* FROM evolution.region_commits c, LATERAL evolution.publish_region_checkpoint(
|
|
overlay(overlay(overlay(overlay(overlay(overlay(overlay(c.manifest
|
|
placing decode(repeat('22', 16), 'hex') from 41)
|
|
placing decode('0100000000000000', 'hex') from 57)
|
|
placing decode(repeat('11', 16), 'hex') from 65)
|
|
placing c.commit_digest from 81)
|
|
placing decode('1400000000000000', 'hex') from 121)
|
|
placing decode(replace(:'lease_holder', '-', ''), 'hex') from 129)
|
|
placing decode('0100000000000000', 'hex') from 145)) p
|
|
WHERE c.commit_sequence = 0;
|
|
SQL
|
|
test "$(PGDATABASE=v4_published psql -XAtc "SELECT count(*) FROM evolution.region_commits")" = 2
|
|
if PGDATABASE=v4_published bash "$root/tools/migrate-postgres.sh" 2> "$test_dir/v4_published.err"; then
|
|
echo 'Expected pre-v5 published checkpoints to block v5' >&2; exit 1
|
|
fi
|
|
grep -q 'pre-v5 published checkpoints lack durable decisions' "$test_dir/v4_published.err"
|
|
test "$(PGDATABASE=v4_published psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4
|
|
test "$(PGDATABASE=v4_published psql -XAtc "SELECT count(*) || ':' || max(end_tick) FROM evolution.region_commits")" = 2:20
|
|
test "$(PGDATABASE=v4_published psql -XAtc "SELECT count(*) FROM pg_tables WHERE schemaname = 'evolution' AND tablename = 'region_decisions'")" = 0
|
|
test "$(PGDATABASE=v2_heads psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2
|
|
# M7.9: a populated v6 database (genesis at sequence 0, tick 0) upgrades to v7 in
|
|
# place; the replaced head constraint still admits it and the v7 parser reads it.
|
|
v5_sha="$(sha256sum "$root/migrations/0005_batch_decisions.sql")"
|
|
v5_sha="${v5_sha%% *}"
|
|
v6_sha="$(sha256sum "$root/migrations/0006_commit_lookup.sql")"
|
|
v6_sha="${v6_sha%% *}"
|
|
createdb v6_populated
|
|
PGDATABASE=v6_populated psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" \
|
|
-v "migration_sha_2=$v2_sha" -v "migration_sha_3=$v3_sha" -v "migration_sha_4=$v4_sha" \
|
|
-v "migration_sha_5=$v5_sha" -v "migration_sha_6=$v6_sha" -v runtime_role=evo_runtime_test \
|
|
-f "$root/migrations/0001_region_bootstrap.sql" -f "$root/migrations/0002_region_fencing.sql" \
|
|
-f "$root/migrations/0003_region_checkpoints.sql" -f "$root/migrations/0004_lineage_profile.sql" \
|
|
-f "$root/migrations/0005_batch_decisions.sql" -f "$root/migrations/0006_commit_lookup.sql" > /dev/null
|
|
PGDATABASE=v6_populated psql -X -v ON_ERROR_STOP=1 \
|
|
-c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \
|
|
-c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '02000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32) || repeat('00', 80), 'hex'))" > /dev/null
|
|
PGDATABASE=v6_populated bash "$root/tools/migrate-postgres.sh"
|
|
test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9
|
|
test "$(PGDATABASE=v6_populated psql -XAtc "SELECT count(*) FROM evolution.region_commits c, LATERAL evolution.parse_checkpoint_manifest(c.manifest) m WHERE m.commit_sequence = 0 AND m.predecessor_commit IS NULL")" = 1
|
|
test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(conname, ',') FROM pg_constraint WHERE conrelid = 'evolution.region_heads'::regclass AND contype = 'c' AND pg_get_constraintdef(oid) LIKE '%end_tick%'")" = region_heads_published_tick
|
|
# Inherited privileges bypass fencing, so every migrate run audits effective rights.
|
|
psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_bad_runtime NOLOGIN IN ROLE pg_write_all_data'
|
|
if (export PGDATABASE=migration_rollback EVO_PG_RUNTIME_ROLE=evo_bad_runtime; \
|
|
bash "$root/tools/migrate-postgres.sh"); then
|
|
echo 'Expected inherited-privilege runtime rejection' >&2; exit 1
|
|
fi
|
|
test "$(PGDATABASE=migration_rollback psql -XAtc "SELECT count(*) FROM pg_namespace WHERE nspname='evolution'")" = 0
|
|
expect_audit_failure() {
|
|
psql -X -v ON_ERROR_STOP=1 -c "$1"
|
|
if bash "$root/tools/migrate-postgres.sh"; then
|
|
echo "Expected runtime audit failure after: $1" >&2; exit 1
|
|
fi
|
|
psql -X -v ON_ERROR_STOP=1 -c "$2"
|
|
}
|
|
# The reviewed scenario: the login inherits pg_write_all_data and can bypass the guard.
|
|
psql -X -v ON_ERROR_STOP=1 -c 'GRANT pg_write_all_data TO evo_runtime_login'
|
|
PGUSER=evo_runtime_login psql -X -v ON_ERROR_STOP=1 \
|
|
-c 'UPDATE evolution.region_leases SET epoch = 99 WHERE false' > /dev/null
|
|
psql -X -v ON_ERROR_STOP=1 -c 'REVOKE pg_write_all_data FROM evo_runtime_login'
|
|
expect_audit_failure 'GRANT pg_write_all_data TO evo_runtime_login' \
|
|
'REVOKE pg_write_all_data FROM evo_runtime_login'
|
|
expect_audit_failure 'GRANT pg_write_all_data TO evo_runtime_test WITH INHERIT FALSE' \
|
|
'REVOKE pg_write_all_data FROM evo_runtime_test'
|
|
expect_audit_failure 'GRANT UPDATE ON evolution.region_heads TO PUBLIC' \
|
|
'REVOKE UPDATE ON evolution.region_heads FROM PUBLIC'
|
|
expect_audit_failure 'GRANT EXECUTE ON FUNCTION evolution.check_region_lease(evolution.u64, evolution.u64, uuid, evolution.u64) TO evo_runtime_test' \
|
|
'REVOKE EXECUTE ON FUNCTION evolution.check_region_lease(evolution.u64, evolution.u64, uuid, evolution.u64) FROM evo_runtime_test'
|
|
expect_audit_failure 'ALTER ROLE evo_runtime_login CREATEDB' 'ALTER ROLE evo_runtime_login NOCREATEDB'
|
|
expect_audit_failure 'GRANT EXECUTE ON FUNCTION evolution.provision_region_checkpoint(bytea) TO evo_runtime_test' 'REVOKE EXECUTE ON FUNCTION evolution.provision_region_checkpoint(bytea) FROM evo_runtime_test'
|
|
expect_audit_failure 'GRANT SELECT ON evolution.region_commits TO evo_runtime_test' 'REVOKE SELECT ON evolution.region_commits FROM evo_runtime_test'
|
|
bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again.
|
|
# Unknown future versions and checksum drift must not be repaired silently.
|
|
psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=10 WHERE version=9'
|
|
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
|
|
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=9 WHERE version=10"
|
|
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=9"
|
|
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
|
|
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=8"
|
|
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
|
|
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=7"
|
|
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
|
|
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=6"
|
|
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
|
|
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=5"
|
|
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
|
|
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=4"
|
|
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
|
|
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=3"
|
|
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
|
|
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=2"
|
|
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
|
|
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64)"
|
|
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
|
|
echo 'postgres migrations, rollback, constraints, permissions, fencing, checkpoints and outcomes passed'
|