From 65e007cc862241583169913fd201ef13f99d3407 Mon Sep 17 00:00:00 2001 From: dingo Date: Mon, 12 Oct 2026 02:24:21 -0400 Subject: [PATCH] Fix cellular v3 manifest normalization --- docs/engineering/VALIDATION.md | 8 +++++ ...fix_cellular_v3_manifest_normalization.sql | 33 +++++++++++++++++++ migrations/apply.sql | 19 +++++++++-- tools/ci/postgres-check.sh | 10 +++--- tools/migrate-postgres.sh | 3 ++ 5 files changed, 65 insertions(+), 8 deletions(-) create mode 100644 migrations/0032_fix_cellular_v3_manifest_normalization.sql diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 5b63f04..a3e2e13 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -7071,3 +7071,11 @@ unique across the database, and that fixture uses sixteen `0x50` bytes. The cellular migration genesis ID now has a distinct first byte while preserving the remaining test pattern. CI must rerun the PostgreSQL gate and downstream Harbor publish before a server image is available. + +CI run #325 reached the v2-to-v3 cellular profile migration and rejected the +normalized manifest as malformed. The v3 parser was replacing a four-byte +history-profile field with the eight-byte result of `manifest_u64le(3)`, which +increased the 360-byte manifest to 364 bytes. Migration 32 now intercepts v3 +manifests, writes the four-byte little-endian value directly, and delegates to +the preserved v27 parser. The PostgreSQL migration integration test covers the +v1-to-v2-to-v3 path. CI validation and Harbor image publication are pending. diff --git a/migrations/0032_fix_cellular_v3_manifest_normalization.sql b/migrations/0032_fix_cellular_v3_manifest_normalization.sql new file mode 100644 index 0000000..ea6e61b --- /dev/null +++ b/migrations/0032_fix_cellular_v3_manifest_normalization.sql @@ -0,0 +1,33 @@ +-- M11.4c, ADR-0122: normalize the v3 history field without changing manifest length. +-- The v27 parser used an eight-byte helper for this four-byte field, so overlay +-- enlarged the manifest and the v24 parser correctly rejected it as malformed. +-- Preserve v27's parser and intercept v3 manifests before its broken branch. +ALTER FUNCTION evolution.parse_checkpoint_manifest(bytea) + RENAME TO parse_checkpoint_manifest_v27; + +CREATE FUNCTION evolution.parse_checkpoint_manifest(m bytea) +RETURNS evolution.checkpoint_manifest LANGUAGE plpgsql IMMUTABLE +SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + r evolution.checkpoint_manifest; + normalized bytea; + v2_bundle constant bytea := '\x020000000200000001000000010000000100000002000000010000000100000001000000'; + v3_bundle constant bytea := '\x020000000300000001000000010000000100000002000000010000000200000001000000'; +BEGIN + IF m IS NOT NULL AND octet_length(m) = 360 + AND evolution.manifest_uint(m, 12, 4) = 4 + AND substring(m FROM 161 FOR 36) = v3_bundle + AND evolution.manifest_uint(m, 228, 4) = 0 + AND evolution.manifest_uint(m, 232, 4) = 4 + AND evolution.manifest_uint(m, 236, 4) = 1 THEN + normalized := overlay(m PLACING v2_bundle FROM 161 FOR 36); + normalized := overlay(normalized PLACING decode('03000000', 'hex') FROM 233 FOR 4); + r := evolution.parse_checkpoint_manifest_v27(normalized); + r.versions := v3_bundle; + RETURN r; + END IF; + RETURN evolution.parse_checkpoint_manifest_v27(m); +END $$; +REVOKE ALL ON FUNCTION evolution.parse_checkpoint_manifest(bytea) FROM PUBLIC; + +INSERT INTO evolution.schema_migrations(version, sha256) VALUES (32, :'migration_sha_32'); diff --git a/migrations/apply.sql b/migrations/apply.sql index 29b2e5d..db4bc78 100644 --- a/migrations/apply.sql +++ b/migrations/apply.sql @@ -105,7 +105,10 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') SELECT right(:'history', length(',29:' || :'migration_sha_29')) = ',29:' || :'migration_sha_29' AS at_v29 \gset SELECT right(:'history', length(',30:' || :'migration_sha_30')) = ',30:' || :'migration_sha_30' AS at_v30 \gset SELECT right(:'history', length(',31:' || :'migration_sha_31')) = ',31:' || :'migration_sha_31' AS at_v31 \gset - \if :at_v31 + SELECT right(:'history', length(',32:' || :'migration_sha_32')) = ',32:' || :'migration_sha_32' AS at_v32 \gset + \if :at_v32 + \elif :at_v31 + \ir 0032_fix_cellular_v3_manifest_normalization.sql \elif :at_v30 \ir 0031_fix_cellular_birth_publication_record.sql \elif :at_v29 @@ -460,7 +463,8 @@ SELECT coalesce(max(version), 0) = 24 AS at_v24, coalesce(max(version), 0) = 28 AS at_v28, coalesce(max(version), 0) = 29 AS at_v29, coalesce(max(version), 0) = 30 AS at_v30, - coalesce(max(version), 0) = 31 AS at_v31 + coalesce(max(version), 0) = 31 AS at_v31, + coalesce(max(version), 0) = 32 AS at_v32 FROM evolution.schema_migrations \gset \if :at_v24 \ir 0025_cellular_v2_bundle.sql @@ -470,6 +474,7 @@ SELECT coalesce(max(version), 0) = 24 AS at_v24, \ir 0029_cellular_mutation_reservations.sql \ir 0030_cellular_birth_publication_binding.sql \ir 0031_fix_cellular_birth_publication_record.sql + \ir 0032_fix_cellular_v3_manifest_normalization.sql \elif :at_v25 \ir 0026_cellular_profile_migration.sql \ir 0027_cellular_v3_migration.sql @@ -477,29 +482,37 @@ SELECT coalesce(max(version), 0) = 24 AS at_v24, \ir 0029_cellular_mutation_reservations.sql \ir 0030_cellular_birth_publication_binding.sql \ir 0031_fix_cellular_birth_publication_record.sql + \ir 0032_fix_cellular_v3_manifest_normalization.sql \elif :at_v26 \ir 0027_cellular_v3_migration.sql \ir 0028_cellular_v3_trace_bounds.sql \ir 0029_cellular_mutation_reservations.sql \ir 0030_cellular_birth_publication_binding.sql \ir 0031_fix_cellular_birth_publication_record.sql + \ir 0032_fix_cellular_v3_manifest_normalization.sql \elif :at_v27 \ir 0028_cellular_v3_trace_bounds.sql \ir 0029_cellular_mutation_reservations.sql \ir 0030_cellular_birth_publication_binding.sql \ir 0031_fix_cellular_birth_publication_record.sql + \ir 0032_fix_cellular_v3_manifest_normalization.sql \elif :at_v28 \ir 0029_cellular_mutation_reservations.sql \ir 0030_cellular_birth_publication_binding.sql \ir 0031_fix_cellular_birth_publication_record.sql + \ir 0032_fix_cellular_v3_manifest_normalization.sql \elif :at_v29 \ir 0030_cellular_birth_publication_binding.sql \ir 0031_fix_cellular_birth_publication_record.sql + \ir 0032_fix_cellular_v3_manifest_normalization.sql \elif :at_v30 \ir 0031_fix_cellular_birth_publication_record.sql + \ir 0032_fix_cellular_v3_manifest_normalization.sql \elif :at_v31 + \ir 0032_fix_cellular_v3_manifest_normalization.sql +\elif :at_v32 \else - \echo 'Unknown or changed migration history after v31' + \echo 'Unknown or changed migration history after v32' DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$; \endif -- Effective-privilege audit on every run, including no-ops (ADR-0060). Attributes diff --git a/tools/ci/postgres-check.sh b/tools/ci/postgres-check.sh index 585e3ed..daf6825 100644 --- a/tools/ci/postgres-check.sh +++ b/tools/ci/postgres-check.sh @@ -73,7 +73,7 @@ PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" -f "$root/migrations/0001_region_bootstrap.sql" > /dev/null test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1 PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 +test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32 # v2 heads have no payload binding: v2->v3 fails closed and leaves history at v2. v2_sha="$(sha256sum "$root/migrations/0002_region_fencing.sql")" v2_sha="${v2_sha%% *}" @@ -94,7 +94,7 @@ PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '01000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32), 'hex'))" > /dev/null PGDATABASE=v3_commits bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 31:1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 +test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 32:1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32 test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) FROM evolution.region_commits")" = 1 # Review P1: a v4 database holding a published (possibly terminal) checkpoint has # no durable decisions for it. The v5 upgrade must refuse before changing anything. @@ -148,7 +148,7 @@ PGDATABASE=v6_populated psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '02000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32) || repeat('00', 80), 'hex'))" > /dev/null PGDATABASE=v6_populated bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 +test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT count(*) FROM evolution.region_commits c, LATERAL evolution.parse_checkpoint_manifest(c.manifest) m WHERE m.commit_sequence = 0 AND m.predecessor_commit IS NULL")" = 1 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(conname, ',') FROM pg_constraint WHERE conrelid = 'evolution.region_heads'::regclass AND contype = 'c' AND pg_get_constraintdef(oid) LIKE '%end_tick%'")" = region_heads_published_tick # Inherited privileges bypass fencing, so every migrate run audits effective rights. @@ -197,9 +197,9 @@ expect_audit_failure 'GRANT SELECT ON evolution.account_controls TO evo_runtime_ expect_audit_failure 'REVOKE EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) FROM evo_runtime_test' 'GRANT EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) TO evo_runtime_test' bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again. # Unknown future versions and checksum drift must not be repaired silently. -psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=32 WHERE version=31' +psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=33 WHERE version=32' if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi -psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=31 WHERE version=32" +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=32 WHERE version=33" psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=14" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=13" diff --git a/tools/migrate-postgres.sh b/tools/migrate-postgres.sh index 9d1975d..340ee89 100644 --- a/tools/migrate-postgres.sh +++ b/tools/migrate-postgres.sh @@ -74,6 +74,8 @@ digest_30="$(sha256sum "$root/migrations/0030_cellular_birth_publication_binding digest_30="${digest_30%% *}" digest_31="$(sha256sum "$root/migrations/0031_fix_cellular_birth_publication_record.sql")" digest_31="${digest_31%% *}" +digest_32="$(sha256sum "$root/migrations/0032_fix_cellular_v3_manifest_normalization.sql")" +digest_32="${digest_32%% *}" psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_2=$digest_2" --set="migration_sha_3=$digest_3" \ --set="migration_sha_4=$digest_4" --set="migration_sha_5=$digest_5" \ @@ -98,5 +100,6 @@ psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_29=$digest_29" \ --set="migration_sha_30=$digest_30" \ --set="migration_sha_31=$digest_31" \ + --set="migration_sha_32=$digest_32" \ --set="runtime_role=$runtime_role" \ --file="$root/migrations/apply.sql"