Add M10.8a probe handoff schema v12 and store
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 1m53s
Foundation / checks (client) (push) Successful in 12m32s
Foundation / checks (network) (push) Successful in 12m17s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (tsan) (push) Canceled after 2m2s
Foundation / checks (postgres) (push) Canceled after 3m48s
Foundation / checks (checks) (push) Canceled after 16m29s

ADR-0096, the persistence handoff ADR (EXPERIMENT probes, ADR-0094). Durable
probe records hold one owning region per account, so ownership is unique and
the test inventory is conserved by construction. A fenced transfer state
machine (prepared, staged, committed, aborted) under each caller's exact
region lease returns the recorded outcome on every retry; a commit requires
staging by the destination's current epoch; aborts are durable and only
before commit. handoff_account_control moves account control on a committed
transfer.

A PostgreSQL fault matrix covers restarts at each transition, stale epochs,
failed destinations, partitions and commit/abort races, checking one owner and
the conserved inventory after every scenario.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-06 18:15:49 -04:00
co-authored by Claude Opus 5.5
parent 5728e7682e
commit 6a645ef7e4
18 changed files with 1355 additions and 16 deletions
+10 -5
View File
@@ -61,7 +61,7 @@ PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha"
-f "$root/migrations/0001_region_bootstrap.sql" > /dev/null
test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1
PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh"
test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11
test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12
# v2 heads have no payload binding: v2->v3 fails closed and leaves history at v2.
v2_sha="$(sha256sum "$root/migrations/0002_region_fencing.sql")"
v2_sha="${v2_sha%% *}"
@@ -82,7 +82,7 @@ PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 \
-c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \
-c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '01000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32), 'hex'))" > /dev/null
PGDATABASE=v3_commits bash "$root/tools/migrate-postgres.sh"
test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 11:1,2,3,4,5,6,7,8,9,10,11
test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 12:1,2,3,4,5,6,7,8,9,10,11,12
test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) FROM evolution.region_commits")" = 1
# Review P1: a v4 database holding a published (possibly terminal) checkpoint has
# no durable decisions for it. The v5 upgrade must refuse before changing anything.
@@ -136,7 +136,7 @@ PGDATABASE=v6_populated psql -X -v ON_ERROR_STOP=1 \
-c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \
-c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '02000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32) || repeat('00', 80), 'hex'))" > /dev/null
PGDATABASE=v6_populated bash "$root/tools/migrate-postgres.sh"
test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11
test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12
test "$(PGDATABASE=v6_populated psql -XAtc "SELECT count(*) FROM evolution.region_commits c, LATERAL evolution.parse_checkpoint_manifest(c.manifest) m WHERE m.commit_sequence = 0 AND m.predecessor_commit IS NULL")" = 1
test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(conname, ',') FROM pg_constraint WHERE conrelid = 'evolution.region_heads'::regclass AND contype = 'c' AND pg_get_constraintdef(oid) LIKE '%end_tick%'")" = region_heads_published_tick
# Inherited privileges bypass fencing, so every migrate run audits effective rights.
@@ -169,6 +169,9 @@ expect_audit_failure 'GRANT EXECUTE ON FUNCTION evolution.check_region_lease(evo
expect_audit_failure 'ALTER ROLE evo_runtime_login CREATEDB' 'ALTER ROLE evo_runtime_login NOCREATEDB'
expect_audit_failure 'GRANT EXECUTE ON FUNCTION evolution.provision_region_checkpoint(bytea) TO evo_runtime_test' 'REVOKE EXECUTE ON FUNCTION evolution.provision_region_checkpoint(bytea) FROM evo_runtime_test'
expect_audit_failure 'GRANT SELECT ON evolution.region_commits TO evo_runtime_test' 'REVOKE SELECT ON evolution.region_commits FROM evo_runtime_test'
# Schema v12: probe records and transfers are reachable only through the fenced functions.
expect_audit_failure 'GRANT SELECT ON evolution.probe_records TO evo_runtime_test' 'REVOKE SELECT ON evolution.probe_records FROM evo_runtime_test'
expect_audit_failure 'GRANT UPDATE ON evolution.probe_transfers TO evo_runtime_test' 'REVOKE UPDATE ON evolution.probe_transfers FROM evo_runtime_test'
# Schema v11: action records are reachable only through the fenced functions.
expect_audit_failure 'GRANT SELECT ON evolution.region_actions TO evo_runtime_test' 'REVOKE SELECT ON evolution.region_actions FROM evo_runtime_test'
expect_audit_failure 'GRANT SELECT ON evolution.region_commit_actions TO evo_runtime_test' 'REVOKE SELECT ON evolution.region_commit_actions FROM evo_runtime_test'
@@ -178,9 +181,11 @@ expect_audit_failure 'GRANT SELECT ON evolution.account_controls TO evo_runtime_
expect_audit_failure 'REVOKE EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) FROM evo_runtime_test' 'GRANT EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) TO evo_runtime_test'
bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again.
# Unknown future versions and checksum drift must not be repaired silently.
psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=12 WHERE version=11'
psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=13 WHERE version=12'
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=12 WHERE version=13"
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=12"
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=11 WHERE version=12"
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=11"
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=10"
+2 -2
View File
@@ -4,7 +4,7 @@
# exported). Real evo-server processes on world 500:
# 1. Failed migration: a migration that takes the migration lock, alters a history
# table, holds its locks for 3 s and then fails runs while build b1 serves.
# Nothing it did remains (schema still v11, no new column); the host keeps its
# Nothing it did remains (schema still v12, no new column); the host keeps its
# authority, and every acknowledgement stays durable.
# 2. Barrier (ADR-0068 operator order): build b2 is retained, and a b2 host is started
# while b1 still holds the lease (it waits, never ready). upgrade_barrier(b1) fences
@@ -61,7 +61,7 @@ SQL
[ "$code" != 0 ] && grep -q 'division by zero' "$work/migration.log" || fail "the rehearsal migration did not fail"
[ "$(q -c "SELECT count(*) FROM information_schema.columns WHERE table_name = 'region_commits' AND column_name = 'rehearsal_column'")" = 0 ] ||
fail "the failed migration left a column"
[ "$(q -c 'SELECT max(version) FROM evolution.schema_migrations')" = 11 ] || fail "schema version moved"
[ "$(q -c 'SELECT max(version) FROM evolution.schema_migrations')" = 12 ] || fail "schema version moved"
wait_ack "$work/b1.log" $((before + 40)) || fail "b1 did not resume after the failed migration"
kill -0 "$b1_pid" 2> /dev/null || fail "b1 exited during the failed migration"
bash "$root/tools/migrate-postgres.sh" > /dev/null || fail "migrate after the failure"
+3 -1
View File
@@ -34,11 +34,13 @@ digest_10="$(sha256sum "$root/migrations/0010_account_control.sql")"
digest_10="${digest_10%% *}"
digest_11="$(sha256sum "$root/migrations/0011_region_actions.sql")"
digest_11="${digest_11%% *}"
digest_12="$(sha256sum "$root/migrations/0012_probe_handoff.sql")"
digest_12="${digest_12%% *}"
psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \
--set="migration_sha_2=$digest_2" --set="migration_sha_3=$digest_3" \
--set="migration_sha_4=$digest_4" --set="migration_sha_5=$digest_5" \
--set="migration_sha_6=$digest_6" --set="migration_sha_7=$digest_7" \
--set="migration_sha_8=$digest_8" --set="migration_sha_9=$digest_9" \
--set="migration_sha_10=$digest_10" --set="migration_sha_11=$digest_11" \
--set="migration_sha_10=$digest_10" --set="migration_sha_11=$digest_11" --set="migration_sha_12=$digest_12" \
--set="runtime_role=$runtime_role" \
--file="$root/migrations/apply.sql"