diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index a3fbfb6..1b7cba9 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -7142,3 +7142,13 @@ record. The server-image path selector did not include server image from this commit. The selector now includes that integration test; CI must confirm both the server-image build and Harbor publication. Harbor publishing also requires the `HARBOR_SERVER_REPOSITORY` repository variable. + +CI run #336 fixed the build selector and ran the PostgreSQL publication test, +which exposed an off-by-one in migration 20: `manifest_uint` reads from a +zero-based offset, but the publisher supplied 25 instead of 24 for each +canonical action tick. This made the durable tick disagree with the 24-byte +input, so PostgreSQL correctly rejected the action row. Migration 33 replaces +the publisher with the corrected offset while retaining migration 20's recorded +checksum. The migration runner's accepted latest history and upgrade fixtures +now extend through version 33. Local PostgreSQL is unavailable on this Windows +environment; migration, publication, and server-image checks are awaiting CI. diff --git a/migrations/0033_fix_cellular_action_tick_offset.sql b/migrations/0033_fix_cellular_action_tick_offset.sql new file mode 100644 index 0000000..fa8c783 --- /dev/null +++ b/migrations/0033_fix_cellular_action_tick_offset.sql @@ -0,0 +1,74 @@ +-- M11.3d: the action tick is the first eight bytes of canonical input. The +-- manifest_uint helper uses zero-based offsets, unlike bytea substring below. +-- Correct migration 20's off-by-one read without changing its recorded checksum. +CREATE OR REPLACE FUNCTION evolution.publish_cellular_checkpoint(p_manifest bytea, + p_decisions bytea, p_actions bytea) +RETURNS TABLE(region_frame evolution.u64, head_sequence evolution.u64, head_commit uuid, + head_digest text, head_tick evolution.u64) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + c evolution.checkpoint_manifest := evolution.parse_checkpoint_manifest(p_manifest); + n integer; + old_actions bytea; + r record; +BEGIN + IF c.profile <> 4 OR c.commit_sequence = 0 OR p_actions IS NULL THEN + RAISE EXCEPTION 'invalid cellular publication' USING ERRCODE = '22023'; + END IF; + n := evolution.check_cellular_action_block(p_actions, c.begin_tick, c.end_tick); + SELECT p.actions INTO old_actions FROM evolution.cellular_publications p + WHERE p.commit_id = c.commit_id; + IF FOUND THEN + IF old_actions <> p_actions THEN + RAISE EXCEPTION 'cellular commit identity reused' USING ERRCODE = 'EVL07'; + END IF; + RETURN QUERY SELECT * FROM evolution.publish_region_checkpoint_core(p_manifest, p_decisions); + RETURN; + END IF; + + PERFORM evolution.check_region_lease(c.world_id, c.region_id, c.holder, c.epoch); + -- Recheck after the lease lock: an uncertain first attempt may have completed. + SELECT p.actions INTO old_actions FROM evolution.cellular_publications p + WHERE p.commit_id = c.commit_id; + IF FOUND THEN + IF old_actions <> p_actions THEN + RAISE EXCEPTION 'cellular commit identity reused' USING ERRCODE = 'EVL07'; + END IF; + RETURN QUERY SELECT * FROM evolution.publish_region_checkpoint_core(p_manifest, p_decisions); + RETURN; + END IF; + + INSERT INTO evolution.cellular_publications(commit_id, actions) + VALUES (c.commit_id, p_actions); + FOR i IN 0 .. n - 1 LOOP + INSERT INTO evolution.region_actions(world_id, account_id, action_id, region_id, tick, + kind, commit_sequence, input_bytes, result) + VALUES (c.world_id, + encode(substring(p_actions FROM 48 * i + 1 FOR 16), 'hex')::uuid, + evolution.manifest_uint(p_actions, 48 * i + 16, 8), c.region_id, + evolution.manifest_uint(p_actions, 48 * i + 24, 8), 2, c.commit_sequence, + substring(p_actions FROM 48 * i + 25 FOR 24), 5) + ON CONFLICT (world_id, account_id, action_id) DO NOTHING; + IF NOT EXISTS (SELECT 1 FROM evolution.region_actions a + WHERE a.world_id = c.world_id + AND a.account_id = encode(substring(p_actions FROM 48 * i + 1 FOR 16), 'hex')::uuid + AND a.action_id = evolution.manifest_uint(p_actions, 48 * i + 16, 8) + AND a.kind = 2 AND a.result = 5 + AND a.input_bytes = substring(p_actions FROM 48 * i + 25 FOR 24)) THEN + RAISE EXCEPTION 'cellular action identity reused' USING ERRCODE = 'EVL07'; + END IF; + END LOOP; + -- The trigger on region_commits requires the private action-block row above. + -- A failure here rolls back the block and all outcome rows with the head advance. + SELECT * INTO r FROM evolution.publish_region_checkpoint_core(p_manifest, p_decisions); + IF NOT FOUND THEN + RAISE EXCEPTION 'publication returned no head' USING ERRCODE = 'XX000'; + END IF; + RETURN QUERY SELECT r.region_frame, r.head_sequence, r.head_commit, r.head_digest, + r.head_tick; +END $$; + +REVOKE ALL ON FUNCTION evolution.publish_cellular_checkpoint(bytea,bytea,bytea) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION evolution.publish_cellular_checkpoint(bytea,bytea,bytea) + TO :"runtime_role"; +INSERT INTO evolution.schema_migrations(version, sha256) VALUES (33, :'migration_sha_33'); diff --git a/migrations/apply.sql b/migrations/apply.sql index db4bc78..cf158eb 100644 --- a/migrations/apply.sql +++ b/migrations/apply.sql @@ -106,7 +106,9 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') SELECT right(:'history', length(',30:' || :'migration_sha_30')) = ',30:' || :'migration_sha_30' AS at_v30 \gset SELECT right(:'history', length(',31:' || :'migration_sha_31')) = ',31:' || :'migration_sha_31' AS at_v31 \gset SELECT right(:'history', length(',32:' || :'migration_sha_32')) = ',32:' || :'migration_sha_32' AS at_v32 \gset - \if :at_v32 + SELECT right(:'history', length(',33:' || :'migration_sha_33')) = ',33:' || :'migration_sha_33' AS at_v33 \gset + \if :at_v33 + \elif :at_v32 \elif :at_v31 \ir 0032_fix_cellular_v3_manifest_normalization.sql \elif :at_v30 @@ -464,7 +466,8 @@ SELECT coalesce(max(version), 0) = 24 AS at_v24, coalesce(max(version), 0) = 29 AS at_v29, coalesce(max(version), 0) = 30 AS at_v30, coalesce(max(version), 0) = 31 AS at_v31, - coalesce(max(version), 0) = 32 AS at_v32 + coalesce(max(version), 0) = 32 AS at_v32, + coalesce(max(version), 0) = 33 AS at_v33 FROM evolution.schema_migrations \gset \if :at_v24 \ir 0025_cellular_v2_bundle.sql @@ -475,6 +478,7 @@ SELECT coalesce(max(version), 0) = 24 AS at_v24, \ir 0030_cellular_birth_publication_binding.sql \ir 0031_fix_cellular_birth_publication_record.sql \ir 0032_fix_cellular_v3_manifest_normalization.sql + \ir 0033_fix_cellular_action_tick_offset.sql \elif :at_v25 \ir 0026_cellular_profile_migration.sql \ir 0027_cellular_v3_migration.sql @@ -483,6 +487,7 @@ SELECT coalesce(max(version), 0) = 24 AS at_v24, \ir 0030_cellular_birth_publication_binding.sql \ir 0031_fix_cellular_birth_publication_record.sql \ir 0032_fix_cellular_v3_manifest_normalization.sql + \ir 0033_fix_cellular_action_tick_offset.sql \elif :at_v26 \ir 0027_cellular_v3_migration.sql \ir 0028_cellular_v3_trace_bounds.sql @@ -490,29 +495,37 @@ SELECT coalesce(max(version), 0) = 24 AS at_v24, \ir 0030_cellular_birth_publication_binding.sql \ir 0031_fix_cellular_birth_publication_record.sql \ir 0032_fix_cellular_v3_manifest_normalization.sql + \ir 0033_fix_cellular_action_tick_offset.sql \elif :at_v27 \ir 0028_cellular_v3_trace_bounds.sql \ir 0029_cellular_mutation_reservations.sql \ir 0030_cellular_birth_publication_binding.sql \ir 0031_fix_cellular_birth_publication_record.sql \ir 0032_fix_cellular_v3_manifest_normalization.sql + \ir 0033_fix_cellular_action_tick_offset.sql \elif :at_v28 \ir 0029_cellular_mutation_reservations.sql \ir 0030_cellular_birth_publication_binding.sql \ir 0031_fix_cellular_birth_publication_record.sql \ir 0032_fix_cellular_v3_manifest_normalization.sql + \ir 0033_fix_cellular_action_tick_offset.sql \elif :at_v29 \ir 0030_cellular_birth_publication_binding.sql \ir 0031_fix_cellular_birth_publication_record.sql \ir 0032_fix_cellular_v3_manifest_normalization.sql + \ir 0033_fix_cellular_action_tick_offset.sql \elif :at_v30 \ir 0031_fix_cellular_birth_publication_record.sql \ir 0032_fix_cellular_v3_manifest_normalization.sql + \ir 0033_fix_cellular_action_tick_offset.sql \elif :at_v31 \ir 0032_fix_cellular_v3_manifest_normalization.sql + \ir 0033_fix_cellular_action_tick_offset.sql \elif :at_v32 + \ir 0033_fix_cellular_action_tick_offset.sql +\elif :at_v33 \else - \echo 'Unknown or changed migration history after v32' + \echo 'Unknown or changed migration history after v33' DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$; \endif -- Effective-privilege audit on every run, including no-ops (ADR-0060). Attributes diff --git a/tools/ci/postgres-check.sh b/tools/ci/postgres-check.sh index daf6825..43e4271 100644 --- a/tools/ci/postgres-check.sh +++ b/tools/ci/postgres-check.sh @@ -73,7 +73,7 @@ PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" -f "$root/migrations/0001_region_bootstrap.sql" > /dev/null test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1 PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32 +test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33 # v2 heads have no payload binding: v2->v3 fails closed and leaves history at v2. v2_sha="$(sha256sum "$root/migrations/0002_region_fencing.sql")" v2_sha="${v2_sha%% *}" @@ -94,7 +94,7 @@ PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '01000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32), 'hex'))" > /dev/null PGDATABASE=v3_commits bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 32:1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32 +test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 33:1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33 test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) FROM evolution.region_commits")" = 1 # Review P1: a v4 database holding a published (possibly terminal) checkpoint has # no durable decisions for it. The v5 upgrade must refuse before changing anything. @@ -148,7 +148,7 @@ PGDATABASE=v6_populated psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '02000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32) || repeat('00', 80), 'hex'))" > /dev/null PGDATABASE=v6_populated bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32 +test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT count(*) FROM evolution.region_commits c, LATERAL evolution.parse_checkpoint_manifest(c.manifest) m WHERE m.commit_sequence = 0 AND m.predecessor_commit IS NULL")" = 1 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(conname, ',') FROM pg_constraint WHERE conrelid = 'evolution.region_heads'::regclass AND contype = 'c' AND pg_get_constraintdef(oid) LIKE '%end_tick%'")" = region_heads_published_tick # Inherited privileges bypass fencing, so every migrate run audits effective rights. @@ -197,9 +197,9 @@ expect_audit_failure 'GRANT SELECT ON evolution.account_controls TO evo_runtime_ expect_audit_failure 'REVOKE EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) FROM evo_runtime_test' 'GRANT EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) TO evo_runtime_test' bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again. # Unknown future versions and checksum drift must not be repaired silently. -psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=33 WHERE version=32' +psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=34 WHERE version=33' if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi -psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=32 WHERE version=33" +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=33 WHERE version=34" psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=14" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=13" diff --git a/tools/migrate-postgres.sh b/tools/migrate-postgres.sh index 340ee89..8028015 100644 --- a/tools/migrate-postgres.sh +++ b/tools/migrate-postgres.sh @@ -76,6 +76,8 @@ digest_31="$(sha256sum "$root/migrations/0031_fix_cellular_birth_publication_rec digest_31="${digest_31%% *}" digest_32="$(sha256sum "$root/migrations/0032_fix_cellular_v3_manifest_normalization.sql")" digest_32="${digest_32%% *}" +digest_33="$(sha256sum "$root/migrations/0033_fix_cellular_action_tick_offset.sql")" +digest_33="${digest_33%% *}" psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_2=$digest_2" --set="migration_sha_3=$digest_3" \ --set="migration_sha_4=$digest_4" --set="migration_sha_5=$digest_5" \ @@ -101,5 +103,6 @@ psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_30=$digest_30" \ --set="migration_sha_31=$digest_31" \ --set="migration_sha_32=$digest_32" \ + --set="migration_sha_33=$digest_33" \ --set="runtime_role=$runtime_role" \ --file="$root/migrations/apply.sql"