Fix cellular v3 manifest normalization
Foundation / checks (checks) (push) Successful in 5s
Foundation / checks (client) (push) Successful in 5s
Foundation / checks (deploy) (push) Successful in 6s
Foundation / checks (ecs) (push) Successful in 4s
Foundation / checks (network) (push) Successful in 4s
Foundation / checks (tsan) (push) Successful in 5s
Foundation / checks (postgres) (push) Failing after 33m10s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped

This commit is contained in:
dingo
2026-10-12 02:24:21 -04:00
parent 7842703230
commit 65e007cc86
5 changed files with 65 additions and 8 deletions
+8
View File
@@ -7071,3 +7071,11 @@ unique across the database, and that fixture uses sixteen `0x50` bytes. The
cellular migration genesis ID now has a distinct first byte while preserving
the remaining test pattern. CI must rerun the PostgreSQL gate and downstream
Harbor publish before a server image is available.
CI run #325 reached the v2-to-v3 cellular profile migration and rejected the
normalized manifest as malformed. The v3 parser was replacing a four-byte
history-profile field with the eight-byte result of `manifest_u64le(3)`, which
increased the 360-byte manifest to 364 bytes. Migration 32 now intercepts v3
manifests, writes the four-byte little-endian value directly, and delegates to
the preserved v27 parser. The PostgreSQL migration integration test covers the
v1-to-v2-to-v3 path. CI validation and Harbor image publication are pending.
@@ -0,0 +1,33 @@
-- M11.4c, ADR-0122: normalize the v3 history field without changing manifest length.
-- The v27 parser used an eight-byte helper for this four-byte field, so overlay
-- enlarged the manifest and the v24 parser correctly rejected it as malformed.
-- Preserve v27's parser and intercept v3 manifests before its broken branch.
ALTER FUNCTION evolution.parse_checkpoint_manifest(bytea)
RENAME TO parse_checkpoint_manifest_v27;
CREATE FUNCTION evolution.parse_checkpoint_manifest(m bytea)
RETURNS evolution.checkpoint_manifest LANGUAGE plpgsql IMMUTABLE
SET search_path = pg_catalog, pg_temp AS $$
DECLARE
r evolution.checkpoint_manifest;
normalized bytea;
v2_bundle constant bytea := '\x020000000200000001000000010000000100000002000000010000000100000001000000';
v3_bundle constant bytea := '\x020000000300000001000000010000000100000002000000010000000200000001000000';
BEGIN
IF m IS NOT NULL AND octet_length(m) = 360
AND evolution.manifest_uint(m, 12, 4) = 4
AND substring(m FROM 161 FOR 36) = v3_bundle
AND evolution.manifest_uint(m, 228, 4) = 0
AND evolution.manifest_uint(m, 232, 4) = 4
AND evolution.manifest_uint(m, 236, 4) = 1 THEN
normalized := overlay(m PLACING v2_bundle FROM 161 FOR 36);
normalized := overlay(normalized PLACING decode('03000000', 'hex') FROM 233 FOR 4);
r := evolution.parse_checkpoint_manifest_v27(normalized);
r.versions := v3_bundle;
RETURN r;
END IF;
RETURN evolution.parse_checkpoint_manifest_v27(m);
END $$;
REVOKE ALL ON FUNCTION evolution.parse_checkpoint_manifest(bytea) FROM PUBLIC;
INSERT INTO evolution.schema_migrations(version, sha256) VALUES (32, :'migration_sha_32');
+16 -3
View File
@@ -105,7 +105,10 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution')
SELECT right(:'history', length(',29:' || :'migration_sha_29')) = ',29:' || :'migration_sha_29' AS at_v29 \gset
SELECT right(:'history', length(',30:' || :'migration_sha_30')) = ',30:' || :'migration_sha_30' AS at_v30 \gset
SELECT right(:'history', length(',31:' || :'migration_sha_31')) = ',31:' || :'migration_sha_31' AS at_v31 \gset
\if :at_v31
SELECT right(:'history', length(',32:' || :'migration_sha_32')) = ',32:' || :'migration_sha_32' AS at_v32 \gset
\if :at_v32
\elif :at_v31
\ir 0032_fix_cellular_v3_manifest_normalization.sql
\elif :at_v30
\ir 0031_fix_cellular_birth_publication_record.sql
\elif :at_v29
@@ -460,7 +463,8 @@ SELECT coalesce(max(version), 0) = 24 AS at_v24,
coalesce(max(version), 0) = 28 AS at_v28,
coalesce(max(version), 0) = 29 AS at_v29,
coalesce(max(version), 0) = 30 AS at_v30,
coalesce(max(version), 0) = 31 AS at_v31
coalesce(max(version), 0) = 31 AS at_v31,
coalesce(max(version), 0) = 32 AS at_v32
FROM evolution.schema_migrations \gset
\if :at_v24
\ir 0025_cellular_v2_bundle.sql
@@ -470,6 +474,7 @@ SELECT coalesce(max(version), 0) = 24 AS at_v24,
\ir 0029_cellular_mutation_reservations.sql
\ir 0030_cellular_birth_publication_binding.sql
\ir 0031_fix_cellular_birth_publication_record.sql
\ir 0032_fix_cellular_v3_manifest_normalization.sql
\elif :at_v25
\ir 0026_cellular_profile_migration.sql
\ir 0027_cellular_v3_migration.sql
@@ -477,29 +482,37 @@ SELECT coalesce(max(version), 0) = 24 AS at_v24,
\ir 0029_cellular_mutation_reservations.sql
\ir 0030_cellular_birth_publication_binding.sql
\ir 0031_fix_cellular_birth_publication_record.sql
\ir 0032_fix_cellular_v3_manifest_normalization.sql
\elif :at_v26
\ir 0027_cellular_v3_migration.sql
\ir 0028_cellular_v3_trace_bounds.sql
\ir 0029_cellular_mutation_reservations.sql
\ir 0030_cellular_birth_publication_binding.sql
\ir 0031_fix_cellular_birth_publication_record.sql
\ir 0032_fix_cellular_v3_manifest_normalization.sql
\elif :at_v27
\ir 0028_cellular_v3_trace_bounds.sql
\ir 0029_cellular_mutation_reservations.sql
\ir 0030_cellular_birth_publication_binding.sql
\ir 0031_fix_cellular_birth_publication_record.sql
\ir 0032_fix_cellular_v3_manifest_normalization.sql
\elif :at_v28
\ir 0029_cellular_mutation_reservations.sql
\ir 0030_cellular_birth_publication_binding.sql
\ir 0031_fix_cellular_birth_publication_record.sql
\ir 0032_fix_cellular_v3_manifest_normalization.sql
\elif :at_v29
\ir 0030_cellular_birth_publication_binding.sql
\ir 0031_fix_cellular_birth_publication_record.sql
\ir 0032_fix_cellular_v3_manifest_normalization.sql
\elif :at_v30
\ir 0031_fix_cellular_birth_publication_record.sql
\ir 0032_fix_cellular_v3_manifest_normalization.sql
\elif :at_v31
\ir 0032_fix_cellular_v3_manifest_normalization.sql
\elif :at_v32
\else
\echo 'Unknown or changed migration history after v31'
\echo 'Unknown or changed migration history after v32'
DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$;
\endif
-- Effective-privilege audit on every run, including no-ops (ADR-0060). Attributes
+5 -5
View File
@@ -73,7 +73,7 @@ PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha"
-f "$root/migrations/0001_region_bootstrap.sql" > /dev/null
test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1
PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh"
test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32
# v2 heads have no payload binding: v2->v3 fails closed and leaves history at v2.
v2_sha="$(sha256sum "$root/migrations/0002_region_fencing.sql")"
v2_sha="${v2_sha%% *}"
@@ -94,7 +94,7 @@ PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 \
-c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \
-c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '01000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32), 'hex'))" > /dev/null
PGDATABASE=v3_commits bash "$root/tools/migrate-postgres.sh"
test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 31:1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 32:1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32
test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) FROM evolution.region_commits")" = 1
# Review P1: a v4 database holding a published (possibly terminal) checkpoint has
# no durable decisions for it. The v5 upgrade must refuse before changing anything.
@@ -148,7 +148,7 @@ PGDATABASE=v6_populated psql -X -v ON_ERROR_STOP=1 \
-c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \
-c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '02000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32) || repeat('00', 80), 'hex'))" > /dev/null
PGDATABASE=v6_populated bash "$root/tools/migrate-postgres.sh"
test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32
test "$(PGDATABASE=v6_populated psql -XAtc "SELECT count(*) FROM evolution.region_commits c, LATERAL evolution.parse_checkpoint_manifest(c.manifest) m WHERE m.commit_sequence = 0 AND m.predecessor_commit IS NULL")" = 1
test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(conname, ',') FROM pg_constraint WHERE conrelid = 'evolution.region_heads'::regclass AND contype = 'c' AND pg_get_constraintdef(oid) LIKE '%end_tick%'")" = region_heads_published_tick
# Inherited privileges bypass fencing, so every migrate run audits effective rights.
@@ -197,9 +197,9 @@ expect_audit_failure 'GRANT SELECT ON evolution.account_controls TO evo_runtime_
expect_audit_failure 'REVOKE EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) FROM evo_runtime_test' 'GRANT EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) TO evo_runtime_test'
bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again.
# Unknown future versions and checksum drift must not be repaired silently.
psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=32 WHERE version=31'
psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=33 WHERE version=32'
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=31 WHERE version=32"
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=32 WHERE version=33"
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=14"
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=13"
+3
View File
@@ -74,6 +74,8 @@ digest_30="$(sha256sum "$root/migrations/0030_cellular_birth_publication_binding
digest_30="${digest_30%% *}"
digest_31="$(sha256sum "$root/migrations/0031_fix_cellular_birth_publication_record.sql")"
digest_31="${digest_31%% *}"
digest_32="$(sha256sum "$root/migrations/0032_fix_cellular_v3_manifest_normalization.sql")"
digest_32="${digest_32%% *}"
psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \
--set="migration_sha_2=$digest_2" --set="migration_sha_3=$digest_3" \
--set="migration_sha_4=$digest_4" --set="migration_sha_5=$digest_5" \
@@ -98,5 +100,6 @@ psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \
--set="migration_sha_29=$digest_29" \
--set="migration_sha_30=$digest_30" \
--set="migration_sha_31=$digest_31" \
--set="migration_sha_32=$digest_32" \
--set="runtime_role=$runtime_role" \
--file="$root/migrations/apply.sql"