diff --git a/include/evo/persistence/checkpoint_store.hpp b/include/evo/persistence/checkpoint_store.hpp index 50894e7..ffdc567 100644 --- a/include/evo/persistence/checkpoint_store.hpp +++ b/include/evo/persistence/checkpoint_store.hpp @@ -112,6 +112,17 @@ struct CellularPublicationFailure { ObjectError object{}; CellularPreparedCommit prepared; }; +struct CellularProfileMigrationPrepared { + RegionKey key{}; + Uuid commit{}; + std::vector manifest; +}; +struct CellularProfileMigrationFailure { + PublicationError kind{}; + LeaseStoreFailure store{}; + ObjectError object{}; + CellularProfileMigrationPrepared prepared; +}; struct PublishedCellularBatch { CommittedCheckpoint committed; }; @@ -220,6 +231,16 @@ resolve_cellular_prepared(Postgres& db, const CellularPreparedCommit& prepared, recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, const LeaseGrant& lease, std::span retained_builds, std::chrono::milliseconds timeout); +// Leased, no-tick representation migration. The v1 payload is restored and +// converted in memory before one immutable v1-to-v2 migration commit is written. +[[nodiscard]] std::expected +migrate_cellular_profile_to_v2(Postgres& db, const FilesystemObjectStore& objects, + const LeaseGrant& lease, const CommittedCheckpoint& predecessor, + Uuid commit, const Sha256Digest& build, + std::chrono::milliseconds timeout); +[[nodiscard]] std::expected +retry_cellular_profile_migration(Postgres& db, const CellularProfileMigrationPrepared& prepared, + std::chrono::milliseconds timeout); // After an `unknown` publication: resolve exactly the prepared bytes. A committed // result yields the final batch; absent may be retried with the same prepared bytes diff --git a/include/evo/persistence/region_lease_store.hpp b/include/evo/persistence/region_lease_store.hpp index 1ff4532..8dfaabf 100644 --- a/include/evo/persistence/region_lease_store.hpp +++ b/include/evo/persistence/region_lease_store.hpp @@ -36,6 +36,18 @@ struct CommittedCheckpoint { RegionHead head{}; std::vector manifest; }; +struct CellularProfileMigrationRecord { + std::uint64_t source_sequence{}; + Uuid source_commit{}; + Sha256Digest source_manifest_sha256{}, source_payload_sha256{}; + std::vector source_versions; + std::uint64_t target_sequence{}; + Uuid target_commit{}; + Sha256Digest target_manifest_sha256{}, target_payload_sha256{}; + std::vector target_versions; + friend bool operator==(const CellularProfileMigrationRecord&, + const CellularProfileMigrationRecord&) = default; +}; // Schema v8 (ADR-0068): authority is granted only to a retained, unretired build; // a retired or unknown build is unsupported and must not keep retrying. @@ -83,6 +95,12 @@ publish_cellular_checkpoint(Postgres& db, std::span manifest std::span decisions, std::span actions, std::span trace, std::chrono::milliseconds timeout); +[[nodiscard]] std::expected +publish_cellular_profile_migration(Postgres& db, std::span manifest, + std::chrono::milliseconds timeout); +[[nodiscard]] std::expected, LeaseStoreFailure> +load_cellular_profile_migration(Postgres& db, RegionKey key, const LeaseGrant& lease, + const Uuid& commit, std::chrono::milliseconds timeout); enum class Resolution : std::uint8_t { committed, // Exact bytes are durable; receipt holds the head. Grants no authority. diff --git a/migrations/0026_cellular_profile_migration.sql b/migrations/0026_cellular_profile_migration.sql new file mode 100644 index 0000000..ff23ddd --- /dev/null +++ b/migrations/0026_cellular_profile_migration.sql @@ -0,0 +1,229 @@ +-- M11.4b, ADR-0119: allow one leased, no-tick profile-4 v1-to-v2 link. +-- Ordinary publication continues to preserve its exact version bundle. + +ALTER FUNCTION evolution.parse_checkpoint_manifest(bytea) + RENAME TO parse_checkpoint_manifest_v25; + +CREATE FUNCTION evolution.manifest_u64le(n numeric) +RETURNS bytea LANGUAGE plpgsql IMMUTABLE STRICT +SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + result bytea := decode(repeat('00', 8), 'hex'); +BEGIN + IF n < 0 OR n > 18446744073709551615 OR trunc(n) <> n THEN + RAISE EXCEPTION 'u64 out of range' USING ERRCODE = '22023'; + END IF; + FOR i IN 0 .. 7 LOOP + result := set_byte(result, i, mod(n, 256)::integer); + n := trunc(n / 256); + END LOOP; + RETURN result; +END $$; +REVOKE ALL ON FUNCTION evolution.manifest_u64le(numeric) FROM PUBLIC; + +CREATE FUNCTION evolution.parse_checkpoint_manifest(m bytea) +RETURNS evolution.checkpoint_manifest LANGUAGE plpgsql IMMUTABLE +SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + r evolution.checkpoint_manifest; + normalized bytea; + zero32 constant bytea := decode(repeat('00', 32), 'hex'); + v1_bundle constant bytea := '\x010000000100000001000000010000000100000002000000010000000100000000000000'; + v2_bundle constant bytea := '\x020000000200000001000000010000000100000002000000010000000100000001000000'; + begin_tick numeric; + end_tick numeric; +BEGIN + IF m IS NOT NULL AND octet_length(m) = 360 + AND evolution.manifest_uint(m, 12, 4) = 4 + AND substring(m FROM 161 FOR 36) = v2_bundle + AND evolution.manifest_uint(m, 56, 8) > 0 + AND evolution.manifest_uint(m, 112, 8) = evolution.manifest_uint(m, 120, 8) + AND evolution.manifest_uint(m, 112, 8) > 0 + AND evolution.manifest_uint(m, 280, 8) = 0 + AND substring(m FROM 289 FOR 32) = zero32 + AND evolution.manifest_uint(m, 320, 4) = 0 + AND substring(m FROM 329 FOR 32) = zero32 THEN + begin_tick := evolution.manifest_uint(m, 112, 8); + end_tick := evolution.manifest_uint(m, 120, 8); + normalized := overlay(m PLACING v1_bundle FROM 161 FOR 36); + -- The v25 parser admits only advancing trace commits. Validate the same + -- immutable fields through it using a synthetic one-tick trace shape, then + -- restore the canonical no-tick evidence in the returned record. + normalized := overlay(normalized PLACING evolution.manifest_u64le(begin_tick - 1) + FROM 113 FOR 8); + normalized := overlay(normalized PLACING evolution.manifest_u64le(120) FROM 281 FOR 8); + normalized := overlay(normalized PLACING decode(repeat('01', 32), 'hex') FROM 289 FOR 32); + r := evolution.parse_checkpoint_manifest_v25(normalized); + r.versions := v2_bundle; + r.begin_tick := begin_tick; + r.end_tick := end_tick; + r.trace_bytes := 0; + r.trace_sha256 := zero32; + r.decision_count := 0; + r.decisions_sha256 := zero32; + RETURN r; + END IF; + RETURN evolution.parse_checkpoint_manifest_v25(m); +END $$; +REVOKE ALL ON FUNCTION evolution.parse_checkpoint_manifest(bytea) FROM PUBLIC; + +CREATE TABLE evolution.cellular_profile_migrations ( + target_commit uuid PRIMARY KEY REFERENCES evolution.region_commits(commit_id) + DEFERRABLE INITIALLY DEFERRED, + world_id evolution.u64 NOT NULL, + region_id evolution.u64 NOT NULL, + source_sequence evolution.u64 NOT NULL, + source_commit uuid NOT NULL, + source_manifest_sha256 bytea NOT NULL CHECK (octet_length(source_manifest_sha256) = 32), + source_payload_sha256 bytea NOT NULL CHECK (octet_length(source_payload_sha256) = 32), + source_versions bytea NOT NULL CHECK (octet_length(source_versions) = 36), + target_sequence evolution.u64 NOT NULL, + target_manifest_sha256 bytea NOT NULL CHECK (octet_length(target_manifest_sha256) = 32), + target_payload_sha256 bytea NOT NULL CHECK (octet_length(target_payload_sha256) = 32), + target_versions bytea NOT NULL CHECK (octet_length(target_versions) = 36), + CHECK (target_sequence = source_sequence + 1), + FOREIGN KEY (world_id, region_id) REFERENCES evolution.region_leases, + CHECK (source_commit <> target_commit) +); +REVOKE ALL ON evolution.cellular_profile_migrations FROM PUBLIC; +CREATE TRIGGER cellular_profile_migrations_immutable BEFORE UPDATE OR DELETE + ON evolution.cellular_profile_migrations FOR EACH ROW + EXECUTE FUNCTION evolution.refuse_history_rewrite(); +CREATE TRIGGER cellular_profile_migrations_no_truncate BEFORE TRUNCATE + ON evolution.cellular_profile_migrations FOR EACH STATEMENT + EXECUTE FUNCTION evolution.refuse_history_rewrite(); + +CREATE OR REPLACE FUNCTION evolution.refuse_unpublished_cellular() RETURNS trigger +LANGUAGE plpgsql SECURITY INVOKER SET search_path = pg_catalog, pg_temp AS $$ +BEGIN + IF evolution.manifest_uint(NEW.manifest, 12, 4) = 4 AND NEW.commit_sequence > 0 + AND NOT EXISTS (SELECT 1 FROM evolution.cellular_publications p + WHERE p.commit_id = NEW.commit_id) + AND NOT EXISTS (SELECT 1 FROM evolution.cellular_profile_migrations m + WHERE m.target_commit = NEW.commit_id) THEN + RAISE EXCEPTION 'cellular commits require their fenced publisher' USING ERRCODE = 'EVL06'; + END IF; + RETURN NEW; +END $$; + +CREATE FUNCTION evolution.publish_cellular_profile_migration(p_manifest bytea) +RETURNS TABLE(region_frame evolution.u64, head_sequence evolution.u64, head_commit uuid, + head_digest text, head_tick evolution.u64) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + c evolution.checkpoint_manifest := evolution.parse_checkpoint_manifest(p_manifest); + s evolution.checkpoint_manifest; + h evolution.region_heads%ROWTYPE; + p evolution.region_commits%ROWTYPE; + old evolution.region_commits%ROWTYPE; + frame evolution.u64; + d bytea := sha256(p_manifest); + v1_bundle constant bytea := '\x010000000100000001000000010000000100000002000000010000000100000000000000'; + v2_bundle constant bytea := '\x020000000200000001000000010000000100000002000000010000000100000001000000'; +BEGIN + IF c.profile <> 4 OR c.commit_sequence = 0 OR c.begin_tick <> c.end_tick + OR c.end_tick = 0 OR c.versions <> v2_bundle + OR evolution.manifest_uint(p_manifest, 228, 4) <> 0 + OR evolution.manifest_uint(p_manifest, 232, 4) <> 3 + OR evolution.manifest_uint(p_manifest, 236, 4) <> 1 + OR c.trace_bytes <> 0 OR c.trace_sha256 <> decode(repeat('00', 32), 'hex') + OR c.decision_count <> 0 OR c.decisions_sha256 <> decode(repeat('00', 32), 'hex') THEN + RAISE EXCEPTION 'invalid cellular profile migration' USING ERRCODE = '22023'; + END IF; + PERFORM evolution.check_region_lease(c.world_id, c.region_id, c.holder, c.epoch); + SELECT * INTO old FROM evolution.region_commits r WHERE r.commit_id = c.commit_id; + IF FOUND THEN + IF old.manifest <> p_manifest THEN + RAISE EXCEPTION 'cellular profile migration identity reused' USING ERRCODE = 'EVL07'; + END IF; + SELECT * INTO p FROM evolution.region_commits r + WHERE r.world_id = old.world_id AND r.region_id = old.region_id + AND r.commit_sequence = old.commit_sequence - 1; + IF NOT FOUND OR NOT EXISTS ( + SELECT 1 FROM evolution.cellular_profile_migrations m + WHERE m.target_commit = c.commit_id AND m.world_id = c.world_id + AND m.region_id = c.region_id AND m.source_sequence = p.commit_sequence + AND m.source_commit = p.commit_id AND m.source_manifest_sha256 = p.commit_digest + AND m.source_payload_sha256 = p.payload_sha256 AND m.source_versions = p.versions + AND m.target_sequence = old.commit_sequence AND m.target_manifest_sha256 = d + AND m.target_payload_sha256 = old.payload_sha256 AND m.target_versions = old.versions + AND m.source_commit = c.predecessor_commit + AND m.source_manifest_sha256 = c.predecessor_digest) THEN + RAISE EXCEPTION 'cellular profile migration identity reused' USING ERRCODE = 'EVL07'; + END IF; + SELECT l.frame_id INTO frame FROM evolution.region_leases l + WHERE l.world_id = c.world_id AND l.region_id = c.region_id; + RETURN QUERY SELECT frame, old.commit_sequence, old.commit_id, + encode(old.commit_digest, 'hex'), old.end_tick; + RETURN; + END IF; + SELECT * INTO h FROM evolution.region_heads r + WHERE r.world_id = c.world_id AND r.region_id = c.region_id FOR UPDATE; + IF NOT FOUND OR h.commit_sequence = 18446744073709551615 + OR c.commit_sequence <> h.commit_sequence + 1 OR c.predecessor_commit <> h.commit_id + OR c.predecessor_digest <> h.digest OR c.begin_tick <> h.end_tick THEN + RAISE EXCEPTION 'region head moved' USING ERRCODE = 'EVL03'; + END IF; + SELECT * INTO p FROM evolution.region_commits r + WHERE r.world_id = c.world_id AND r.region_id = c.region_id + AND r.commit_sequence = h.commit_sequence; + IF NOT FOUND THEN + RAISE EXCEPTION 'missing migration predecessor' USING ERRCODE = 'XX000'; + END IF; + s := evolution.parse_checkpoint_manifest(p.manifest); + IF s.profile <> 4 OR s.versions <> v1_bundle OR p.versions <> v1_bundle + OR c.frame_id <> s.frame_id OR c.tick_microseconds <> s.tick_microseconds + OR s.commit_id <> c.predecessor_commit OR s.end_tick <> c.end_tick + OR evolution.manifest_uint(p.manifest, 232, 4) <> 3 + OR NOT EXISTS (SELECT 1 FROM evolution.retained_builds b + WHERE b.build_identity = c.build_identity) THEN + RAISE EXCEPTION 'unsupported cellular profile migration' USING ERRCODE = 'EVL06'; + END IF; + SELECT l.frame_id INTO frame FROM evolution.region_leases l + WHERE l.world_id = c.world_id AND l.region_id = c.region_id; + IF c.frame_id <> frame THEN + RAISE EXCEPTION 'foreign frame' USING ERRCODE = '22023'; + END IF; + INSERT INTO evolution.cellular_profile_migrations(target_commit, world_id, region_id, + source_sequence, source_commit, source_manifest_sha256, source_payload_sha256, + source_versions, target_sequence, target_manifest_sha256, target_payload_sha256, + target_versions) + VALUES (c.commit_id, c.world_id, c.region_id, p.commit_sequence, p.commit_id, + p.commit_digest, p.payload_sha256, p.versions, c.commit_sequence, d, + c.payload_sha256, c.versions); + INSERT INTO evolution.region_commits(world_id, region_id, commit_sequence, commit_id, + commit_digest, manifest, begin_tick, end_tick, published_epoch, tick_microseconds, + versions, build_identity, payload_bytes, payload_sha256) + VALUES (c.world_id, c.region_id, c.commit_sequence, c.commit_id, d, p_manifest, + c.begin_tick, c.end_tick, c.epoch, c.tick_microseconds, c.versions, + c.build_identity, c.payload_bytes, c.payload_sha256); + UPDATE evolution.region_heads r + SET commit_sequence = c.commit_sequence, commit_id = c.commit_id, digest = d, + end_tick = c.end_tick, published_epoch = c.epoch + WHERE r.world_id = c.world_id AND r.region_id = c.region_id; + RETURN QUERY SELECT frame, c.commit_sequence, c.commit_id, encode(d, 'hex'), c.end_tick; +END $$; +REVOKE ALL ON FUNCTION evolution.publish_cellular_profile_migration(bytea) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION evolution.publish_cellular_profile_migration(bytea) TO :"runtime_role"; + +CREATE FUNCTION evolution.load_cellular_profile_migration(p_world evolution.u64, + p_region evolution.u64, p_holder uuid, p_epoch evolution.u64, p_commit uuid) +RETURNS TABLE(source_sequence evolution.u64, source_commit uuid, + source_manifest_sha256 bytea, source_payload_sha256 bytea, source_versions bytea, + target_sequence evolution.u64, target_commit uuid, target_manifest_sha256 bytea, + target_payload_sha256 bytea, target_versions bytea) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +BEGIN + PERFORM evolution.check_region_lease(p_world, p_region, p_holder, p_epoch); + RETURN QUERY SELECT m.source_sequence, m.source_commit, m.source_manifest_sha256, + m.source_payload_sha256, m.source_versions, m.target_sequence, m.target_commit, + m.target_manifest_sha256, m.target_payload_sha256, m.target_versions + FROM evolution.cellular_profile_migrations m + WHERE m.world_id = p_world AND m.region_id = p_region AND m.target_commit = p_commit; +END $$; +REVOKE ALL ON FUNCTION evolution.load_cellular_profile_migration( + evolution.u64,evolution.u64,uuid,evolution.u64,uuid) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION evolution.load_cellular_profile_migration( + evolution.u64,evolution.u64,uuid,evolution.u64,uuid) TO :"runtime_role"; + +INSERT INTO evolution.schema_migrations(version, sha256) VALUES (26, :'migration_sha_26'); diff --git a/migrations/apply.sql b/migrations/apply.sql index 03b46b5..46f9216 100644 --- a/migrations/apply.sql +++ b/migrations/apply.sql @@ -99,7 +99,9 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') SELECT right(:'history', length(',23:' || :'migration_sha_23')) = ',23:' || :'migration_sha_23' AS at_v23 \gset SELECT right(:'history', length(',24:' || :'migration_sha_24')) = ',24:' || :'migration_sha_24' AS at_v24 \gset SELECT right(:'history', length(',25:' || :'migration_sha_25')) = ',25:' || :'migration_sha_25' AS at_v25 \gset - \if :at_v25 + SELECT right(:'history', length(',26:' || :'migration_sha_26')) = ',26:' || :'migration_sha_26' AS at_v26 \gset + \if :at_v26 + \elif :at_v25 \elif :at_v24 \elif :at_v23 \ir 0024_primitive_life_owner_lookup.sql @@ -435,16 +437,20 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0023_cellular_birth_publication.sql \ir 0024_primitive_life_owner_lookup.sql \endif --- The versioned branch above always leaves an admitted schema at v24. Apply +-- The versioned branch above leaves an admitted schema at v24 or newer. Apply -- additive follow-ups from that exact prefix, including on a fresh bootstrap. SELECT coalesce(max(version), 0) = 24 AS at_v24, - coalesce(max(version), 0) = 25 AS at_v25 + coalesce(max(version), 0) = 25 AS at_v25, + coalesce(max(version), 0) = 26 AS at_v26 FROM evolution.schema_migrations \gset \if :at_v24 \ir 0025_cellular_v2_bundle.sql + \ir 0026_cellular_profile_migration.sql \elif :at_v25 + \ir 0026_cellular_profile_migration.sql +\elif :at_v26 \else - \echo 'Unknown or changed migration history after v24' + \echo 'Unknown or changed migration history after v26' DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$; \endif -- Effective-privilege audit on every run, including no-ops (ADR-0060). Attributes diff --git a/src/persistence/cellular_checkpoint_store.cpp b/src/persistence/cellular_checkpoint_store.cpp index bbfcb1a..da091b4 100644 --- a/src/persistence/cellular_checkpoint_store.cpp +++ b/src/persistence/cellular_checkpoint_store.cpp @@ -23,6 +23,11 @@ CellularPublicationFailure failure(PublicationError kind, ObjectError object = { CellularPreparedCommit prepared = {}) { return {kind, store, object, std::move(prepared)}; } +CellularProfileMigrationFailure migration_failure(PublicationError kind, ObjectError object = {}, + LeaseStoreFailure store = {}, + CellularProfileMigrationPrepared prepared = {}) { + return {kind, store, object, std::move(prepared)}; +} std::expected put_verified(const FilesystemObjectStore& objects, std::span bytes, VerifiedObject expected) { @@ -138,6 +143,95 @@ provision_cellular_region(Postgres& owner, const FilesystemObjectStore& objects, return *confirmed; } +std::expected +retry_cellular_profile_migration(Postgres& db, const CellularProfileMigrationPrepared& prepared, + milliseconds timeout) { + const auto manifest = decode_manifest(prepared.manifest); + if (!manifest || manifest->key != prepared.key || manifest->commit != prepared.commit || + manifest->profile != CheckpointProfile::cellular_lives_v1 || + manifest->cellular_versions != cellular_v2_versions || + manifest->begin_tick != manifest->end_tick || manifest->batch != BatchDigests{}) + return std::unexpected{migration_failure(PublicationError::candidate, {}, {}, prepared)}; + + const auto confirm_commit = [&](const RegionHead& head) { + const auto digest = sha256(prepared.manifest); + if (!digest || head.sequence != manifest->sequence || head.commit != manifest->commit || + head.digest != *digest || head.end_tick != manifest->end_tick || + head.frame != manifest->frame) + return std::expected{ + std::unexpected{migration_failure(PublicationError::receipt, {}, {}, prepared)}}; + return std::expected{ + CommittedCheckpoint{head, prepared.manifest}}; + }; + + auto published = publish_cellular_profile_migration(db, prepared.manifest, timeout); + if (published) + return confirm_commit(*published); + + if (published.error().kind == LeaseStoreError::unknown) { + const auto stored = load_region_commit(db, prepared.key, manifest->sequence, timeout); + if (stored && stored->manifest == prepared.manifest) + return confirm_commit(stored->head); + if (!stored && stored.error().kind != LeaseStoreError::not_provisioned) + return std::unexpected{ + migration_failure(PublicationError::unknown, {}, stored.error(), prepared)}; + return std::unexpected{ + migration_failure(PublicationError::unknown, {}, published.error(), prepared)}; + } + return std::unexpected{ + migration_failure(PublicationError::rejected, {}, published.error(), prepared)}; +} + +std::expected +migrate_cellular_profile_to_v2(Postgres& db, const FilesystemObjectStore& objects, + const LeaseGrant& lease, const CommittedCheckpoint& predecessor, + Uuid commit, const Sha256Digest& build, milliseconds timeout) { + const auto previous = decode_manifest(predecessor.manifest); + const auto previous_digest = sha256(predecessor.manifest); + if (!previous || !previous_digest || + previous->profile != CheckpointProfile::cellular_lives_v1 || + previous->cellular_versions != cellular_v1_versions || + predecessor.head.digest != *previous_digest || + predecessor.head.sequence != previous->sequence || + predecessor.head.commit != previous->commit || + predecessor.head.end_tick != previous->end_tick || lease.frame != previous->frame || + previous->sequence == 0 || previous->end_tick == 0) + return std::unexpected{migration_failure(PublicationError::candidate)}; + + auto source_payload = objects.get({previous->payload_sha256, previous->payload_bytes}); + if (!source_payload) + return std::unexpected{ + migration_failure(PublicationError::predecessor, source_payload.error())}; + const auto source_digest = sha256(*source_payload); + if (!source_digest || *source_digest != previous->payload_sha256) + return std::unexpected{ + migration_failure(PublicationError::predecessor, ObjectError::corrupt)}; + auto source = restore_cellular_payload(*source_payload, *previous); + if (!source || (*source)->format_version() != server::cellular_lives_legacy_format || + (*source)->rules_version() != 1) + return std::unexpected{migration_failure(PublicationError::predecessor)}; + auto migrated = (*source)->migrate_to_v2(); + if (!migrated || (*migrated)->tick() != (*source)->tick() || + (*migrated)->current_total() != (*source)->current_total()) + return std::unexpected{migration_failure(PublicationError::candidate)}; + + const auto payload = (*migrated)->snapshot(); + const auto payload_digest = sha256(payload); + if (!payload_digest) + return std::unexpected{migration_failure(PublicationError::digest)}; + auto manifest = cellular_profile_migration_manifest(*previous, *previous_digest, commit, lease, + build, payload.size(), *payload_digest); + auto encoded = manifest ? encode_manifest(*manifest) + : std::expected, CheckpointError>{ + std::unexpected{CheckpointError::format}}; + if (!encoded || !restore_cellular_payload(payload, *manifest)) + return std::unexpected{migration_failure(PublicationError::candidate)}; + if (auto stored = put_verified(objects, payload, {*payload_digest, payload.size()}); !stored) + return std::unexpected{migration_failure(stored.error().kind, stored.error().object)}; + return retry_cellular_profile_migration( + db, CellularProfileMigrationPrepared{previous->key, commit, std::move(*encoded)}, timeout); +} + std::expected retry_cellular_prepared(Postgres& db, const CellularPreparedCommit& prepared, milliseconds timeout) { @@ -268,6 +362,61 @@ recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects, return fail_with(RecoveryError::payload); std::vector actions; + if (manifest->sequence != 0 && manifest->begin_tick == manifest->end_tick) { + auto migration = load_cellular_profile_migration(db, key, lease, manifest->commit, timeout); + if (!migration) + return std::unexpected{RecoveryFailure{RecoveryError::store, migration.error()}}; + if (!*migration || (*migration)->target_sequence != manifest->sequence || + (*migration)->target_commit != manifest->commit || + (*migration)->target_manifest_sha256 != *digest || + (*migration)->target_payload_sha256 != manifest->payload_sha256 || + (*migration)->target_sequence == 0) + return fail_with(RecoveryError::history); + auto predecessor = load_region_commit(db, key, manifest->sequence - 1, timeout); + if (!predecessor) + return std::unexpected{RecoveryFailure{RecoveryError::tail, predecessor.error()}}; + const auto source_manifest_digest = sha256(predecessor->manifest); + auto source_manifest = decode_manifest(predecessor->manifest); + const auto source_manifest_bytes = + encode_manifest(source_manifest ? *source_manifest : CheckpointManifest{}); + const auto target_manifest_bytes = encode_manifest(*manifest); + if (!source_manifest_digest || !source_manifest || !source_manifest_bytes || + !target_manifest_bytes || *source_manifest_digest != manifest->predecessor_digest || + predecessor->head.digest != *source_manifest_digest || + predecessor->head.commit != manifest->predecessor_commit || + predecessor->head.sequence != manifest->sequence - 1 || + source_manifest->cellular_versions != cellular_v1_versions || + source_manifest->profile != CheckpointProfile::cellular_lives_v1 || + source_manifest->end_tick != manifest->end_tick || + (*migration)->source_sequence != source_manifest->sequence || + (*migration)->source_commit != source_manifest->commit || + (*migration)->source_manifest_sha256 != *source_manifest_digest || + (*migration)->source_payload_sha256 != source_manifest->payload_sha256 || + (*migration)->source_versions.size() != 36 || + !std::ranges::equal((*migration)->source_versions, + std::span{*source_manifest_bytes}.subspan(160, 36)) || + (*migration)->target_versions.size() != 36 || + !std::ranges::equal((*migration)->target_versions, + std::span{*target_manifest_bytes}.subspan(160, 36))) + return fail_with(RecoveryError::history); + auto source_bytes = + objects.get({source_manifest->payload_sha256, source_manifest->payload_bytes}); + if (!source_bytes) + return fail_with(source_bytes.error() == ObjectError::io ? RecoveryError::store + : RecoveryError::tail); + const auto source_payload_digest = sha256(*source_bytes); + if (!source_payload_digest || *source_payload_digest != source_manifest->payload_sha256) + return fail_with(RecoveryError::tail); + auto source_state = restore_cellular_payload(*source_bytes, *source_manifest); + if (!source_state || + (*source_state)->format_version() != server::cellular_lives_legacy_format) + return fail_with(RecoveryError::tail); + auto migrated = (*source_state)->migrate_to_v2(); + if (!migrated || (*migrated)->tick() != manifest->end_tick || + (*migrated)->snapshot() != *payload) + return fail_with(RecoveryError::history); + return RecoveredCellularCheckpoint{std::move(*committed), *manifest, std::move(*state), {}}; + } if (manifest->sequence != 0) { auto action_bytes = load_cellular_publication_actions(db, key, lease, manifest->commit, timeout); diff --git a/src/persistence/region_lease_store.cpp b/src/persistence/region_lease_store.cpp index b4dd8a6..6d16d37 100644 --- a/src/persistence/region_lease_store.cpp +++ b/src/persistence/region_lease_store.cpp @@ -4,6 +4,7 @@ #include "evo/persistence/probe_store.hpp" #include #include +#include #include #include #include @@ -356,6 +357,63 @@ publish_cellular_checkpoint(Postgres& db, std::span manifest return *head; } +std::expected +publish_cellular_profile_migration(Postgres& db, std::span manifest, + milliseconds timeout) { + const std::array parameters{bytea(manifest)}; + auto rows = db.query("SELECT * FROM evolution.publish_cellular_profile_migration($1::bytea)", + parameters, timeout); + if (!rows) + return std::unexpected{classify(rows.error())}; + const auto head = parse_head(*rows); + if (!head) + return std::unexpected{malformed(true)}; + return *head; +} + +std::expected, LeaseStoreFailure> +load_cellular_profile_migration(Postgres& db, RegionKey key, const LeaseGrant& lease, + const Uuid& commit, milliseconds timeout) { + const std::array parameters{std::to_string(key.world), + std::to_string(key.region), uuid_text(lease.holder), + std::to_string(lease.epoch), uuid_text(commit)}; + auto rows = db.query( + "SELECT source_sequence, source_commit, encode(source_manifest_sha256, 'hex'), " + "encode(source_payload_sha256, 'hex'), encode(source_versions, 'hex'), target_sequence, " + "target_commit, encode(target_manifest_sha256, 'hex'), " + "encode(target_payload_sha256, 'hex'), encode(target_versions, 'hex') " + "FROM evolution.load_cellular_profile_migration($1::evolution.u64, " + "$2::evolution.u64, $3::uuid, $4::evolution.u64, $5::uuid)", + parameters, timeout); + if (!rows) + return std::unexpected{classify(rows.error())}; + if (rows->empty()) + return std::optional{}; + if (rows->size() != 1 || (*rows)[0].size() != 10) + return std::unexpected{malformed(false)}; + const auto& row = (*rows)[0]; + const auto source_sequence = parse_integer(row[0]); + const auto source_commit = parse_uuid(row[1]); + const auto source_manifest = row[2] ? unhex<32>(*row[2]) : std::nullopt; + const auto source_payload = row[3] ? unhex<32>(*row[3]) : std::nullopt; + const auto source_versions = row[4] ? unhex_bytes(*row[4]) : std::nullopt; + const auto target_sequence = parse_integer(row[5]); + const auto target_commit = parse_uuid(row[6]); + const auto target_manifest = row[7] ? unhex<32>(*row[7]) : std::nullopt; + const auto target_payload = row[8] ? unhex<32>(*row[8]) : std::nullopt; + const auto target_versions = row[9] ? unhex_bytes(*row[9]) : std::nullopt; + if (!source_sequence || !source_commit || !source_manifest || !source_payload || + !source_versions || source_versions->size() != 36 || !target_sequence || !target_commit || + !target_manifest || !target_payload || !target_versions || target_versions->size() != 36 || + *source_sequence == std::numeric_limits::max() || + *target_sequence != *source_sequence + 1) + return std::unexpected{malformed(false)}; + return std::optional{CellularProfileMigrationRecord{ + *source_sequence, *source_commit, *source_manifest, *source_payload, + std::move(*source_versions), *target_sequence, *target_commit, *target_manifest, + *target_payload, std::move(*target_versions)}}; +} + std::expected resolve_region_commit(Postgres& db, RegionKey key, const Uuid& commit, std::span manifest, diff --git a/tests/cellular_publication_tests.cpp b/tests/cellular_publication_tests.cpp index 0df1afd..232476e 100644 --- a/tests/cellular_publication_tests.cpp +++ b/tests/cellular_publication_tests.cpp @@ -45,6 +45,13 @@ std::string hex(std::span bytes) { } return out; } +void refresh_cellular_checksum(std::vector& bytes) { + std::uint64_t value = 14695981039346656037ULL; + for (const auto byte : std::span{bytes}.first(bytes.size() - 8)) + value = (value ^ byte) * 1099511628211ULL; + for (unsigned index = 0; index < 8; ++index) + bytes[bytes.size() - 8 + index] = static_cast(value >> (index * 8U)); +} } // namespace namespace { @@ -138,10 +145,108 @@ void cellular_birth_publication_is_durable() { restarted->state()->snapshot() == (*candidate)->snapshot()); restarted->drain(); } + +void cellular_profile_migration_is_durable_and_recovers_without_a_tick() { + using namespace evo::biology; + constexpr RegionKey key{9755, 1}; + auto owner = connect_as(env("PGUSER")); + const auto build = filled<32>(0xD9); + CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " + "(decode(repeat('d9',32),'hex')) ON CONFLICT DO NOTHING", + {}, 5s)); + auto endowment = primitive_endowment(); + CHECK(endowment); + auto funding = endowment->body; + funding.energy.value = endowment->reserve.energy.value + endowment->construction.value; + auto primitive = PrimitiveLives::create({key.world, key.region, 1, 17, funding, {{0, 0}}}); + CHECK(primitive); + LifeRequest founder; + founder.account = filled<16>(0x5A); + founder.action = 9; + CHECK((*primitive)->spawn(founder, {1, 101, 201, 301, {0}})); + auto habitat = CellularHabitat::create({key.world, key.region, 1, 1'000, 1'000, {}}); + CHECK(habitat); + auto current = + CellularLives::create(**primitive, std::move(*habitat), {{1'000, 1'000}, 0, 293'150}); + CHECK(current); + auto legacy_bytes = (*current)->snapshot(); + legacy_bytes.erase(legacy_bytes.begin() + 220 + 168, legacy_bytes.begin() + 220 + 480); + constexpr std::uint64_t legacy_magic = 0x3130564C43564541ULL; + for (unsigned index = 0; index < 8; ++index) + legacy_bytes[index] = static_cast(legacy_magic >> (index * 8U)); + legacy_bytes[8] = 1; + legacy_bytes[9] = legacy_bytes[10] = legacy_bytes[11] = 0; + legacy_bytes[12] = 1; + legacy_bytes[13] = legacy_bytes[14] = legacy_bytes[15] = 0; + refresh_cellular_checksum(legacy_bytes); + auto legacy = CellularLives::restore(legacy_bytes, key.world, key.region, 1, 0); + CHECK(legacy && (*legacy)->format_version() == cellular_lives_legacy_format); + + auto objects = FilesystemObjectStore::open(env("PGHOST"), key, max_cellular_lives_bytes); + CHECK(objects); + auto provisioned = + provision_cellular_region(*owner, **objects, key, 1, **legacy, filled<16>(0x50), build, 5s); + CHECK(provisioned); + const auto connect = [&]() -> std::expected, PgFailure> { + return Postgres::connect({env("PGHOST"), + {}, + env("PGPORT"), + env("PGDATABASE"), + env("EVO_PG_RUNTIME_USER"), + {}, + {}}, + 5s); + }; + CellularHostConfig config{key, 1, {}, build, {build}}; + auto migration_objects = + FilesystemObjectStore::open(env("PGHOST"), key, max_cellular_lives_bytes); + CHECK(migration_objects); + auto made = CellularRegionHost::create(config, connect, std::move(*objects)); + CHECK(made); + auto host = std::move(*made); + CHECK(host->step().ready); + const auto authority = host->authority(); + CHECK(authority); + const auto predecessor = host->committed(); + CHECK(predecessor && predecessor->head.sequence == 0); + + CellularBatch batch; + const auto predecessor_digest = sha256(predecessor->manifest); + CHECK(predecessor_digest); + batch.header = {key.world, key.region, 1, predecessor->head.commit, *predecessor_digest, 0, 1}; + batch.ticks.resize(1); + for (const auto& life : host->state()->lives()) + batch.ticks[0].lives.push_back({life.identity.life, {}}); + const auto published = host->publish(batch); + CHECK(published && published->committed.head.sequence == 1); + auto migrated = migrate_cellular_profile_to_v2( + *owner, **migration_objects, *authority, published->committed, filled<16>(0x51), build, 5s); + CHECK(migrated && migrated->head.sequence == 2 && migrated->head.end_tick == 1); + CHECK( + retry_cellular_profile_migration(*owner, {key, filled<16>(0x51), migrated->manifest}, 5s)); + host->drain(); + host.reset(); + + auto recovery_objects = + FilesystemObjectStore::open(env("PGHOST"), key, max_cellular_lives_bytes); + CHECK(recovery_objects); + const auto target_manifest = decode_manifest(migrated->manifest); + CHECK(target_manifest); + const auto target_payload = + (*recovery_objects)->get({target_manifest->payload_sha256, target_manifest->payload_bytes}); + CHECK(target_payload); + const auto recovered = recover_cellular_checkpoint(*owner, **recovery_objects, key, *authority, + std::span{&build, 1}, 5s); + CHECK(recovered && recovered->checkpoint.head.sequence == 2 && + recovered->checkpoint.head.end_tick == 1 && recovered->state->tick() == 1 && + recovered->state->format_version() == cellular_lives_format && + recovered->state->snapshot() == *target_payload); +} } // namespace void run_cellular_publication_tests() { cellular_birth_publication_is_durable(); + cellular_profile_migration_is_durable_and_recovers_without_a_tick(); auto owner = connect_as(env("PGUSER")); const auto build = filled<32>(0xD9); CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " diff --git a/tests/region_migration_tests.cpp b/tests/region_migration_tests.cpp index 9d4e580..c80d330 100644 --- a/tests/region_migration_tests.cpp +++ b/tests/region_migration_tests.cpp @@ -191,6 +191,7 @@ void upgrade_across_migration_and_barrier() { " -v migration_sha_23=$(s 0023_cellular_birth_publication.sql)" " -v migration_sha_24=$(s 0024_primitive_life_owner_lookup.sql)" " -v migration_sha_25=$(s 0025_cellular_v2_bundle.sql)" + " -v migration_sha_26=$(s 0026_cellular_profile_migration.sql)" " -v runtime_role=evo_runtime_test -f migrations/apply.sql" " > /dev/null 2>&1") != 0); CHECK(history(*owner_db) == "1,2,3,4,5,6,7"); @@ -207,7 +208,7 @@ void upgrade_across_migration_and_barrier() { " EVO_PG_RUNTIME_ROLE=evo_runtime_test bash \"$EVO_SOURCE_ROOT/" "tools/migrate-postgres.sh\" > /dev/null") == 0); CHECK(history(*owner_db) == - "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25"); + "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26"); CHECK(scalar(*owner_db, "SELECT (to_regprocedure('evolution.primitive_life_owner(" "evolution.u64,evolution.u64,uuid,evolution.u64,uuid)') IS NOT NULL)::text") == diff --git a/tools/ci/postgres-check.sh b/tools/ci/postgres-check.sh index a9363b2..c2dfc0f 100644 --- a/tools/ci/postgres-check.sh +++ b/tools/ci/postgres-check.sh @@ -69,7 +69,7 @@ PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" -f "$root/migrations/0001_region_bootstrap.sql" > /dev/null test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1 PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25 +test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26 # v2 heads have no payload binding: v2->v3 fails closed and leaves history at v2. v2_sha="$(sha256sum "$root/migrations/0002_region_fencing.sql")" v2_sha="${v2_sha%% *}" @@ -90,7 +90,7 @@ PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '01000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32), 'hex'))" > /dev/null PGDATABASE=v3_commits bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 25:1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25 +test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 26:1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26 test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) FROM evolution.region_commits")" = 1 # Review P1: a v4 database holding a published (possibly terminal) checkpoint has # no durable decisions for it. The v5 upgrade must refuse before changing anything. @@ -144,7 +144,7 @@ PGDATABASE=v6_populated psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '02000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32) || repeat('00', 80), 'hex'))" > /dev/null PGDATABASE=v6_populated bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25 +test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT count(*) FROM evolution.region_commits c, LATERAL evolution.parse_checkpoint_manifest(c.manifest) m WHERE m.commit_sequence = 0 AND m.predecessor_commit IS NULL")" = 1 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(conname, ',') FROM pg_constraint WHERE conrelid = 'evolution.region_heads'::regclass AND contype = 'c' AND pg_get_constraintdef(oid) LIKE '%end_tick%'")" = region_heads_published_tick # Inherited privileges bypass fencing, so every migrate run audits effective rights. @@ -190,9 +190,9 @@ expect_audit_failure 'GRANT SELECT ON evolution.account_controls TO evo_runtime_ expect_audit_failure 'REVOKE EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) FROM evo_runtime_test' 'GRANT EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) TO evo_runtime_test' bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again. # Unknown future versions and checksum drift must not be repaired silently. -psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=26 WHERE version=25' +psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=27 WHERE version=26' if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi -psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=25 WHERE version=26" +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=26 WHERE version=27" psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=14" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=13" diff --git a/tools/migrate-postgres.sh b/tools/migrate-postgres.sh index 9d62f85..7722a20 100644 --- a/tools/migrate-postgres.sh +++ b/tools/migrate-postgres.sh @@ -62,6 +62,8 @@ digest_24="$(sha256sum "$root/migrations/0024_primitive_life_owner_lookup.sql")" digest_24="${digest_24%% *}" digest_25="$(sha256sum "$root/migrations/0025_cellular_v2_bundle.sql")" digest_25="${digest_25%% *}" +digest_26="$(sha256sum "$root/migrations/0026_cellular_profile_migration.sql")" +digest_26="${digest_26%% *}" psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_2=$digest_2" --set="migration_sha_3=$digest_3" \ --set="migration_sha_4=$digest_4" --set="migration_sha_5=$digest_5" \ @@ -80,5 +82,6 @@ psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_23=$digest_23" \ --set="migration_sha_24=$digest_24" \ --set="migration_sha_25=$digest_25" \ + --set="migration_sha_26=$digest_26" \ --set="runtime_role=$runtime_role" \ --file="$root/migrations/apply.sql"