From a479ce2d9511138db4129432f52c16d92157dceb Mon Sep 17 00:00:00 2001 From: dingo Date: Sun, 11 Oct 2026 06:13:17 -0400 Subject: [PATCH] Add cellular genome revision profile migration --- docs/biology/GENOME_MODEL.md | 6 + docs/engineering/M11_4_DELIVERY_PLAN.md | 7 +- docs/engineering/VALIDATION.md | 8 + include/evo/biology/lineage.hpp | 39 +++- include/evo/persistence/checkpoint_store.hpp | 5 + include/evo/persistence/region_checkpoint.hpp | 6 +- include/evo/processes/lineage_region.hpp | 2 +- .../evo/server/lifecycle/cellular_lives.hpp | 4 + .../evo/server/lifecycle/primitive_lives.hpp | 2 +- migrations/0027_cellular_v3_migration.sql | 142 +++++++++++++++ migrations/apply.sql | 14 +- schemas/cellular-lives-v3.md | 26 +++ schemas/lineage-archive-v2.md | 36 ++++ schemas/region-commit.md | 30 ++-- src/biology/lineage.cpp | 167 ++++++++++++++++-- src/persistence/cellular_checkpoint_store.cpp | 64 ++++++- src/persistence/region_checkpoint.cpp | 41 +++-- src/server/lifecycle/cellular_lives.cpp | 77 ++++++-- src/server/lifecycle/primitive_lives.cpp | 2 +- tests/cellular_lives_tests.cpp | 18 ++ tests/cellular_publication_tests.cpp | 12 +- tests/lineage_tests.cpp | 69 +++++++- tests/region_checkpoint_tests.cpp | 18 ++ tools/ci/postgres-check.sh | 6 +- tools/migrate-postgres.sh | 3 + 25 files changed, 725 insertions(+), 79 deletions(-) create mode 100644 migrations/0027_cellular_v3_migration.sql create mode 100644 schemas/cellular-lives-v3.md create mode 100644 schemas/lineage-archive-v2.md diff --git a/docs/biology/GENOME_MODEL.md b/docs/biology/GENOME_MODEL.md index 74d3983..e354906 100644 --- a/docs/biology/GENOME_MODEL.md +++ b/docs/biology/GENOME_MODEL.md @@ -272,6 +272,12 @@ whole archive canonically for history-capable region checkpoints. NPC mutation stays undecided, because the archive records an origin event and never an actor. +M11.4c adds the cellular-only append-only genome revision history under +[ADR-0122](../adr/0122-append-only-cellular-genome-revisions.md) and +[lineage-archive-v2](../../schemas/lineage-archive-v2.md). Birth-genome records +remain immutable; the current genome is derived from the organism's revision +chain. Environmental and primitive checkpoint profiles retain archive v1. + ## What a genome decides, end to end A genome now determines behaviour all the way into an ecosystem. `strategy_for` diff --git a/docs/engineering/M11_4_DELIVERY_PLAN.md b/docs/engineering/M11_4_DELIVERY_PLAN.md index 967bf0d..e3bdafc 100644 --- a/docs/engineering/M11_4_DELIVERY_PLAN.md +++ b/docs/engineering/M11_4_DELIVERY_PLAN.md @@ -66,4 +66,9 @@ pending; no gameplay balance is treated as final by this plan. then defines selection and atomic publication. Affected modules are biology archive API/codec, cellular life owner/snapshot, checkpoint version bundle/migration, replay trace, SQL action publication, host/session and - crash/retry tests. Implementation starts from this versioned history boundary. + crash/retry tests. The archive API/codec, owner v3 snapshot and explicit + no-tick v2-to-v3 manifest/SQL migration are implemented locally with focused + tests. Durable selection, trace replay and atomic action publication remain + to be implemented; no 11.4c or live mutation path is claimed complete. + Windows build and PostgreSQL checks are pending the next available Linux CI + run (see [validation log](VALIDATION.md)). diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index c3e6008..c55febc 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -6771,3 +6771,11 @@ the dependent image jobs had not started. The Gitea host then returned HTTP 503 for the run page, workflow page and a Git remote dry-run. The final result could not be verified in this environment. No full-CI pass is claimed for the 11.4b migration/recovery work; rerun or inspect this gate when the host is available. + +For the M11.4c archive-revision and v3 checkpoint-migration changes, clang-format +and `git diff --check` pass. The targeted Windows Debug build for lineage, cellular +lives and checkpoint tests again produced no output for 60 seconds and was +stopped. Earlier direct compiler setup lacked the Visual C++ standard-library +environment, and WSL is unavailable in this session. These tests and the +PostgreSQL migration/publication path therefore remain unverified here; rely on +the Linux CI run after the repository host is reachable. diff --git a/include/evo/biology/lineage.hpp b/include/evo/biology/lineage.hpp index bed0817..bcf3161 100644 --- a/include/evo/biology/lineage.hpp +++ b/include/evo/biology/lineage.hpp @@ -3,6 +3,8 @@ #include "evo/biology/genome.hpp" #include "evo/core/types.hpp" +#include + namespace evo::biology { // What the archive can establish about itself. There is deliberately no third // value: terminal extinction is a policy judgement over a reconciled world @@ -25,11 +27,12 @@ enum class HistoryError : std::uint8_t { unknown_organism, already_dead, invalid_identity, - invalid_tick + invalid_tick, + invalid_revision }; inline constexpr std::size_t max_archived_genomes = 4'096; inline constexpr std::size_t max_archived_organisms = 65'536; -// Whole-archive codec v1 (schemas/lineage-archive-v1.md, ADR-0062). +// Whole-archive codecs v1/v2 (schemas/lineage-archive-v1.md and -v2.md). enum class ArchiveCodecError : std::uint8_t { format, // Magic, lengths or field forms. version, // Unknown archive format or genome schema. @@ -40,10 +43,21 @@ enum class ArchiveCodecError : std::uint8_t { noncanonical, // Valid content in a non-canonical order or form. }; inline constexpr std::uint32_t lineage_archive_format = 1; +inline constexpr std::uint32_t lineage_archive_revision_format = 2; inline constexpr std::size_t archived_organism_bytes = 33; -inline constexpr std::size_t max_archive_bytes = +inline constexpr std::size_t archived_genome_revision_bytes = 40; +inline constexpr std::size_t max_archive_legacy_bytes = 32 + (max_archived_genomes * (2 + max_genome_bytes)) + (max_archived_organisms * archived_organism_bytes) + 8; +inline constexpr std::size_t max_archive_bytes = + 40 + (max_archived_genomes * (2 + max_genome_bytes)) + + (max_archived_organisms * archived_organism_bytes) + + (max_archived_genomes * archived_genome_revision_bytes) + 8; +struct OrganismGenomeRevision { + std::uint64_t organism{}, previous_genome{}, successor_genome{}, origin_event{}; + Tick at{}; + auto operator<=>(const OrganismGenomeRevision&) const = default; +}; // Retains genomes and the organisms that carried them. There is no removal // operation: not a guarded one, not a private one. Retention is a property of @@ -70,10 +84,18 @@ class LineageArchive { std::uint64_t genome, Tick when); // Terminal for the organism, and nothing to do with the genome it carried. [[nodiscard]] std::expected record_death(std::uint64_t organism, Tick when); + // Adds a successor genome and binds it as the next genome carried by a living + // organism. Birth records remain immutable; the revision chain is append-only. + [[nodiscard]] std::expected + record_mutation(std::uint64_t organism, const Genome& successor, Tick when); + // Explicitly upgrades a revision-free archive during a cellular profile migration. + void upgrade_to_revision_format() noexcept; [[nodiscard]] std::expected find(std::uint64_t genome) const; [[nodiscard]] std::expected organism(std::uint64_t identity) const; + [[nodiscard]] std::expected + current_genome(std::uint64_t organism) const; // Oldest first, ending at the founder. Still resolves after every carrier dies. [[nodiscard]] std::expected, HistoryError> ancestry(std::uint64_t genome) const; @@ -90,11 +112,17 @@ class LineageArchive { [[nodiscard]] std::size_t archived() const noexcept { return genomes_.size(); } + [[nodiscard]] std::uint32_t format_version() const noexcept { + return format_version_; + } [[nodiscard]] std::size_t recorded() const noexcept { return organisms_.size(); } // Ascending identities of organisms not yet recorded dead. [[nodiscard]] std::vector living_organisms() const; + [[nodiscard]] std::span revisions() const noexcept { + return revisions_; + } // Largest recorded identities, zero when none: allocator high-water checks. [[nodiscard]] std::uint64_t highest_genome() const noexcept; [[nodiscard]] std::uint64_t highest_organism() const noexcept; @@ -121,5 +149,10 @@ class LineageArchive { // ADR-0030, so an aggregate holding one would have a throwing move. std::vector genomes_; std::vector organisms_; + std::vector revisions_; + std::uint32_t format_version_{lineage_archive_format}; + [[nodiscard]] std::expected + append_revision(std::uint64_t organism, std::uint64_t previous, std::uint64_t successor, + std::uint64_t origin_event, Tick when); }; } // namespace evo::biology diff --git a/include/evo/persistence/checkpoint_store.hpp b/include/evo/persistence/checkpoint_store.hpp index ffdc567..b93becd 100644 --- a/include/evo/persistence/checkpoint_store.hpp +++ b/include/evo/persistence/checkpoint_store.hpp @@ -239,6 +239,11 @@ migrate_cellular_profile_to_v2(Postgres& db, const FilesystemObjectStore& object Uuid commit, const Sha256Digest& build, std::chrono::milliseconds timeout); [[nodiscard]] std::expected +migrate_cellular_profile_to_v3(Postgres& db, const FilesystemObjectStore& objects, + const LeaseGrant& lease, const CommittedCheckpoint& predecessor, + Uuid commit, const Sha256Digest& build, + std::chrono::milliseconds timeout); +[[nodiscard]] std::expected retry_cellular_profile_migration(Postgres& db, const CellularProfileMigrationPrepared& prepared, std::chrono::milliseconds timeout); diff --git a/include/evo/persistence/region_checkpoint.hpp b/include/evo/persistence/region_checkpoint.hpp index fbf1da1..9c00349 100644 --- a/include/evo/persistence/region_checkpoint.hpp +++ b/include/evo/persistence/region_checkpoint.hpp @@ -31,6 +31,7 @@ inline constexpr std::uint32_t lineage_history_profile = 1; // Archive format 1, genome schema 2, development 2, founder-life rules 1. inline constexpr std::uint32_t primitive_history_profile = 2; inline constexpr std::uint32_t cellular_history_profile = 3; +inline constexpr std::uint32_t cellular_revision_history_profile = 4; // Declared durability, verified at deployment. Development: one volume, no host-loss claim. enum class DurabilityPolicy : std::uint8_t { unknown = 0, single_volume_development = 1 }; @@ -51,6 +52,7 @@ struct CellularVersionBundle { }; inline constexpr CellularVersionBundle cellular_v1_versions{}; inline constexpr CellularVersionBundle cellular_v2_versions{2, 2, 1, 1, 1, 2, 1, 1, 1}; +inline constexpr CellularVersionBundle cellular_v3_versions{2, 3, 1, 1, 1, 2, 1, 2, 1}; // Batch evidence bound by manifest v2 (ADR-0063); all zero at genesis. struct BatchDigests { @@ -140,8 +142,8 @@ successor_manifest(const CheckpointManifest& predecessor, const Sha256Digest& pr Uuid commit, const LeaseGrant& lease, std::uint64_t end_tick, const Sha256Digest& build, std::uint64_t payload_bytes, const Sha256Digest& payload_sha256, const BatchDigests& batch); -// The only supported cellular profile transition: a leased, no-tick v1-to-v2 -// representation migration. SQL publication must independently verify the pair. +// Leased, no-tick cellular representation migration. SQL publication independently +// verifies the exact adjacent profile transition. [[nodiscard]] std::expected cellular_profile_migration_manifest(const CheckpointManifest& predecessor, const Sha256Digest& predecessor_digest, Uuid commit, diff --git a/include/evo/processes/lineage_region.hpp b/include/evo/processes/lineage_region.hpp index 82b834e..ea57611 100644 --- a/include/evo/processes/lineage_region.hpp +++ b/include/evo/processes/lineage_region.hpp @@ -29,7 +29,7 @@ enum class LineageRegionError : std::uint8_t { }; inline constexpr std::uint32_t lineage_region_format = 1; inline constexpr std::size_t max_lineage_region_bytes = - 72 + EnvironmentalCommunity::max_snapshot_bytes + biology::max_archive_bytes; + 72 + EnvironmentalCommunity::max_snapshot_bytes + biology::max_archive_legacy_bytes; class LineageRegion { public: diff --git a/include/evo/server/lifecycle/cellular_lives.hpp b/include/evo/server/lifecycle/cellular_lives.hpp index 3c470f4..888db97 100644 --- a/include/evo/server/lifecycle/cellular_lives.hpp +++ b/include/evo/server/lifecycle/cellular_lives.hpp @@ -16,6 +16,7 @@ namespace evo::server { inline constexpr std::uint32_t cellular_lives_legacy_format = 1; inline constexpr std::uint32_t cellular_lives_format = 2; +inline constexpr std::uint32_t cellular_lives_revision_format = 3; inline constexpr std::uint32_t cellular_lives_rules = 2; inline constexpr std::size_t max_cellular_lives_bytes = std::size_t{16} * 1024 * 1024; inline constexpr std::uint64_t cellular_mutation_surplus_seconds = 60; @@ -135,6 +136,9 @@ class CellularLives { // Detached no-tick representation upgrade. Only a fully restored v1 owner can migrate. [[nodiscard]] std::expected, CellularLivesError> migrate_to_v2() const; + // Detached no-tick v2-to-v3 profile upgrade enabling genome revisions. + [[nodiscard]] std::expected, CellularLivesError> + migrate_to_v3() const; // Returns a detached post-birth candidate for an allocator-reserved request. // The caller supplies the durable identity; the owner derives founder biology, diff --git a/include/evo/server/lifecycle/primitive_lives.hpp b/include/evo/server/lifecycle/primitive_lives.hpp index 3b08554..a50908b 100644 --- a/include/evo/server/lifecycle/primitive_lives.hpp +++ b/include/evo/server/lifecycle/primitive_lives.hpp @@ -25,7 +25,7 @@ inline constexpr std::size_t max_primitive_lives = 256; inline constexpr std::int32_t spawn_coordinate_limit_um = 1'000'000'000; inline constexpr std::uint32_t primitive_lives_format = 1; inline constexpr std::size_t max_primitive_lives_bytes = - 200 + 184 * max_primitive_lives + biology::max_archive_bytes; + 200 + 184 * max_primitive_lives + biology::max_archive_legacy_bytes; enum class LifeCodecError : std::uint8_t { format, version, checksum, closure }; struct NurseryConfig { std::uint64_t world{}, region{}, frame{}, source{}; diff --git a/migrations/0027_cellular_v3_migration.sql b/migrations/0027_cellular_v3_migration.sql new file mode 100644 index 0000000..c3f3412 --- /dev/null +++ b/migrations/0027_cellular_v3_migration.sql @@ -0,0 +1,142 @@ +-- M11.4c, ADR-0122: allow one leased, no-tick cellular-v2 to cellular-v3 link. +-- The previous profile-4 v2 parser and publisher remain the authority for v1-to-v2. + +ALTER FUNCTION evolution.parse_checkpoint_manifest(bytea) + RENAME TO parse_checkpoint_manifest_v26; + +CREATE FUNCTION evolution.parse_checkpoint_manifest(m bytea) +RETURNS evolution.checkpoint_manifest LANGUAGE plpgsql IMMUTABLE +SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + r evolution.checkpoint_manifest; + normalized bytea; + v2_bundle constant bytea := '\x020000000200000001000000010000000100000002000000010000000100000001000000'; + v3_bundle constant bytea := '\x020000000300000001000000010000000100000002000000010000000200000001000000'; +BEGIN + IF m IS NOT NULL AND octet_length(m) = 360 + AND evolution.manifest_uint(m, 12, 4) = 4 + AND substring(m FROM 161 FOR 36) = v3_bundle + AND evolution.manifest_uint(m, 228, 4) = 0 + AND evolution.manifest_uint(m, 232, 4) = 4 + AND evolution.manifest_uint(m, 236, 4) = 1 THEN + normalized := overlay(m PLACING v2_bundle FROM 161 FOR 36); + normalized := overlay(normalized PLACING evolution.manifest_u64le(3) FROM 233 FOR 4); + r := evolution.parse_checkpoint_manifest_v26(normalized); + r.versions := v3_bundle; + RETURN r; + END IF; + RETURN evolution.parse_checkpoint_manifest_v26(m); +END $$; +REVOKE ALL ON FUNCTION evolution.parse_checkpoint_manifest(bytea) FROM PUBLIC; + +ALTER FUNCTION evolution.publish_cellular_profile_migration(bytea) + RENAME TO publish_cellular_profile_migration_v26; +REVOKE ALL ON FUNCTION evolution.publish_cellular_profile_migration_v26(bytea) + FROM PUBLIC, :"runtime_role"; + +CREATE FUNCTION evolution.publish_cellular_profile_migration(p_manifest bytea) +RETURNS TABLE(region_frame evolution.u64, head_sequence evolution.u64, head_commit uuid, + head_digest text, head_tick evolution.u64) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + c evolution.checkpoint_manifest := evolution.parse_checkpoint_manifest(p_manifest); + s evolution.checkpoint_manifest; + h evolution.region_heads%ROWTYPE; + p evolution.region_commits%ROWTYPE; + old evolution.region_commits%ROWTYPE; + frame evolution.u64; + d bytea := sha256(p_manifest); + v2_bundle constant bytea := '\x020000000200000001000000010000000100000002000000010000000100000001000000'; + v3_bundle constant bytea := '\x020000000300000001000000010000000100000002000000010000000200000001000000'; + zero32 constant bytea := decode(repeat('00', 32), 'hex'); +BEGIN + IF c.versions <> v3_bundle THEN + RETURN QUERY SELECT * FROM evolution.publish_cellular_profile_migration_v26(p_manifest); + RETURN; + END IF; + IF c.profile <> 4 OR c.commit_sequence = 0 OR c.begin_tick <> c.end_tick + OR c.end_tick = 0 OR c.versions <> v3_bundle + OR evolution.manifest_uint(p_manifest, 228, 4) <> 0 + OR evolution.manifest_uint(p_manifest, 232, 4) <> 4 + OR evolution.manifest_uint(p_manifest, 236, 4) <> 1 + OR c.trace_bytes <> 0 OR c.trace_sha256 <> zero32 + OR c.decision_count <> 0 OR c.decisions_sha256 <> zero32 THEN + RAISE EXCEPTION 'invalid cellular-v3 profile migration' USING ERRCODE = '22023'; + END IF; + PERFORM evolution.check_region_lease(c.world_id, c.region_id, c.holder, c.epoch); + SELECT * INTO old FROM evolution.region_commits r WHERE r.commit_id = c.commit_id; + IF FOUND THEN + IF old.manifest <> p_manifest THEN + RAISE EXCEPTION 'cellular profile migration identity reused' USING ERRCODE = 'EVL07'; + END IF; + SELECT * INTO p FROM evolution.region_commits r + WHERE r.world_id = old.world_id AND r.region_id = old.region_id + AND r.commit_sequence = old.commit_sequence - 1; + IF NOT FOUND OR NOT EXISTS ( + SELECT 1 FROM evolution.cellular_profile_migrations m + WHERE m.target_commit = c.commit_id AND m.world_id = c.world_id + AND m.region_id = c.region_id AND m.source_sequence = p.commit_sequence + AND m.source_commit = p.commit_id AND m.source_manifest_sha256 = p.commit_digest + AND m.source_payload_sha256 = p.payload_sha256 AND m.source_versions = p.versions + AND m.target_sequence = old.commit_sequence AND m.target_manifest_sha256 = d + AND m.target_payload_sha256 = old.payload_sha256 AND m.target_versions = old.versions + AND m.source_commit = c.predecessor_commit + AND m.source_manifest_sha256 = c.predecessor_digest) THEN + RAISE EXCEPTION 'cellular profile migration identity reused' USING ERRCODE = 'EVL07'; + END IF; + SELECT l.frame_id INTO frame FROM evolution.region_leases l + WHERE l.world_id = c.world_id AND l.region_id = c.region_id; + RETURN QUERY SELECT frame, old.commit_sequence, old.commit_id, + encode(old.commit_digest, 'hex'), old.end_tick; + RETURN; + END IF; + SELECT * INTO h FROM evolution.region_heads r + WHERE r.world_id = c.world_id AND r.region_id = c.region_id FOR UPDATE; + IF NOT FOUND OR h.commit_sequence = 18446744073709551615 + OR c.commit_sequence <> h.commit_sequence + 1 OR c.predecessor_commit <> h.commit_id + OR c.predecessor_digest <> h.digest OR c.begin_tick <> h.end_tick THEN + RAISE EXCEPTION 'region head moved' USING ERRCODE = 'EVL03'; + END IF; + SELECT * INTO p FROM evolution.region_commits r + WHERE r.world_id = c.world_id AND r.region_id = c.region_id + AND r.commit_sequence = h.commit_sequence; + IF NOT FOUND THEN + RAISE EXCEPTION 'missing migration predecessor' USING ERRCODE = 'XX000'; + END IF; + s := evolution.parse_checkpoint_manifest(p.manifest); + IF s.profile <> 4 OR s.versions <> v2_bundle OR p.versions <> v2_bundle + OR c.frame_id <> s.frame_id OR c.tick_microseconds <> s.tick_microseconds + OR s.commit_id <> c.predecessor_commit OR s.end_tick <> c.end_tick + OR evolution.manifest_uint(p.manifest, 232, 4) <> 3 + OR NOT EXISTS (SELECT 1 FROM evolution.retained_builds b + WHERE b.build_identity = c.build_identity) THEN + RAISE EXCEPTION 'unsupported cellular-v3 profile migration' USING ERRCODE = 'EVL06'; + END IF; + SELECT l.frame_id INTO frame FROM evolution.region_leases l + WHERE l.world_id = c.world_id AND l.region_id = c.region_id; + IF c.frame_id <> frame THEN + RAISE EXCEPTION 'foreign frame' USING ERRCODE = '22023'; + END IF; + INSERT INTO evolution.cellular_profile_migrations(target_commit, world_id, region_id, + source_sequence, source_commit, source_manifest_sha256, source_payload_sha256, + source_versions, target_sequence, target_manifest_sha256, target_payload_sha256, + target_versions) + VALUES (c.commit_id, c.world_id, c.region_id, p.commit_sequence, p.commit_id, + p.commit_digest, p.payload_sha256, p.versions, c.commit_sequence, d, + c.payload_sha256, c.versions); + INSERT INTO evolution.region_commits(world_id, region_id, commit_sequence, commit_id, + commit_digest, manifest, begin_tick, end_tick, published_epoch, tick_microseconds, + versions, build_identity, payload_bytes, payload_sha256) + VALUES (c.world_id, c.region_id, c.commit_sequence, c.commit_id, d, p_manifest, + c.begin_tick, c.end_tick, c.epoch, c.tick_microseconds, c.versions, + c.build_identity, c.payload_bytes, c.payload_sha256); + UPDATE evolution.region_heads r + SET commit_sequence = c.commit_sequence, commit_id = c.commit_id, digest = d, + end_tick = c.end_tick, published_epoch = c.epoch + WHERE r.world_id = c.world_id AND r.region_id = c.region_id; + RETURN QUERY SELECT frame, c.commit_sequence, c.commit_id, encode(d, 'hex'), c.end_tick; +END $$; +REVOKE ALL ON FUNCTION evolution.publish_cellular_profile_migration(bytea) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION evolution.publish_cellular_profile_migration(bytea) TO :"runtime_role"; + +INSERT INTO evolution.schema_migrations(version, sha256) VALUES (27, :'migration_sha_27'); diff --git a/migrations/apply.sql b/migrations/apply.sql index 46f9216..d6854dd 100644 --- a/migrations/apply.sql +++ b/migrations/apply.sql @@ -100,7 +100,10 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') SELECT right(:'history', length(',24:' || :'migration_sha_24')) = ',24:' || :'migration_sha_24' AS at_v24 \gset SELECT right(:'history', length(',25:' || :'migration_sha_25')) = ',25:' || :'migration_sha_25' AS at_v25 \gset SELECT right(:'history', length(',26:' || :'migration_sha_26')) = ',26:' || :'migration_sha_26' AS at_v26 \gset - \if :at_v26 + SELECT right(:'history', length(',27:' || :'migration_sha_27')) = ',27:' || :'migration_sha_27' AS at_v27 \gset + \if :at_v27 + \elif :at_v26 + \ir 0027_cellular_v3_migration.sql \elif :at_v25 \elif :at_v24 \elif :at_v23 @@ -441,16 +444,21 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') -- additive follow-ups from that exact prefix, including on a fresh bootstrap. SELECT coalesce(max(version), 0) = 24 AS at_v24, coalesce(max(version), 0) = 25 AS at_v25, - coalesce(max(version), 0) = 26 AS at_v26 + coalesce(max(version), 0) = 26 AS at_v26, + coalesce(max(version), 0) = 27 AS at_v27 FROM evolution.schema_migrations \gset \if :at_v24 \ir 0025_cellular_v2_bundle.sql \ir 0026_cellular_profile_migration.sql + \ir 0027_cellular_v3_migration.sql \elif :at_v25 \ir 0026_cellular_profile_migration.sql + \ir 0027_cellular_v3_migration.sql \elif :at_v26 + \ir 0027_cellular_v3_migration.sql +\elif :at_v27 \else - \echo 'Unknown or changed migration history after v26' + \echo 'Unknown or changed migration history after v27' DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$; \endif -- Effective-privilege audit on every run, including no-ops (ADR-0060). Attributes diff --git a/schemas/cellular-lives-v3.md b/schemas/cellular-lives-v3.md new file mode 100644 index 0000000..ba29b5c --- /dev/null +++ b/schemas/cellular-lives-v3.md @@ -0,0 +1,26 @@ +# Cellular lives checkpoint v3 + +Authority: ADR-0122. Cellular profile 4 v3 adds append-only genome revisions +while retaining the v2 life records and mutation-offer state. Cellular-v1 and +v2 checkpoints remain readable with their original version bundles. + +The envelope magic is ASCII `AEVCLV03`, format is 3, and cellular rules remain +2. Its manifest bundle is `(2,3,1,1,1,2,1,2,1)`: rules, envelope, habitat, +coordinate, genome, development, founder, archive and mutation state. The +tick quantum remains 50,000 μs and the payload cap remains 16 MiB. The header, +spawn sites, life records and corpse records keep the v2 layout. The nested +lineage archive uses format 2 as specified in +[lineage-archive-v2](lineage-archive-v2.md). + +For a living life, `identity.genome` is the current genome reached by replaying +that organism's revision chain; the archived birth record remains immutable. +For a corpse, closure derives the genome at its death tick from the same chain. +Restore replays and validates the archive before checking the life, corpse, +habitat and inventory closure. + +The only v2-to-v3 conversion is a leased, no-tick profile migration. It upgrades +each format-1 archive to format 2 with an empty revision list and preserves the +tick, lives, outstanding offers, corpses, habitat and conserved inventory. SQL +records the source and target as one immutable migration link. Ordinary v3 +successors retain the exact v3 bundle; a mutation is published with an +advancing v3 tick. diff --git a/schemas/lineage-archive-v2.md b/schemas/lineage-archive-v2.md new file mode 100644 index 0000000..75640f2 --- /dev/null +++ b/schemas/lineage-archive-v2.md @@ -0,0 +1,36 @@ +# Lineage archive v2 (M11.4c) + +Authority: ADR-0122. The encoder and decoder are +`LineageArchive::encode/decode` in `src/biology/lineage.cpp`. All integers are +little-endian; no native struct serialization is used. V1 bytes and decoding +remain unchanged. V2 is used by cellular-v3 checkpoints. + +| Offset | Size | Field | +|---|---|---| +| 0 | 8 | magic `AEVOLAR1` | +| 8 | 4 | archive format = 2 | +| 12 | 4 | highest genome schema present | +| 16 | 4 | genome count G (≤ 4096) | +| 20 | 4 | birth-record organism count O (≤ 65536) | +| 24 | 4 | genome-revision count R (≤ 4096) | +| 28 | 4 | reserved = 0 | +| 32 | 8 | total bytes, including checksum | +| 40 | … | G × (u16 length 1..256, canonical genome record), ordered by (ancestry depth, genome id) | +| … | 33·O | birth records ordered by organism id: organism, birth genome, born tick, died tick, alive flag | +| … | 40·R | revisions ordered by (effective tick, organism): organism, previous genome, successor genome, origin event, effective tick (all u64) | +| end−8 | 8 | FNV-1a64 of all preceding bytes | + +The maximum size is 3,383,344 bytes. Counts and total size are checked before +allocation. A revision's successor is already in the genome table, has exactly +the previous genome as its parent, and carries the same nonzero value as its +genome ID and origin event. The organism must be alive at that tick, the tick +must be after birth and before death, and at most one revision per organism per +tick is permitted. Revisions form a continuous chain from the immutable birth +genome. `OrganismRecord.genome` always remains the birth genome; the current +genome is resolved by replaying the revisions. + +Format 2 is append-only. Decoding admits genomes and birth/death records, then +replays revisions against the organism's current genome. The result must +re-encode to identical bytes. A format-2 archive with no revisions is used by +the explicit cellular-v2-to-v3 migration. Other profiles retain their v1 +payload caps and bytes. diff --git a/schemas/region-commit.md b/schemas/region-commit.md index 5ed0c11..b4ff5d1 100644 --- a/schemas/region-commit.md +++ b/schemas/region-commit.md @@ -5,8 +5,11 @@ using manifest v2. M11.3 adds profile 4, the [cellular lives profile v1](cellular-lives-v1.md), with its own version bundle, payload bound and restore validator. M11.4 adds the compatible cellular-v2 [checkpoint](cellular-lives-v2.md) and its explicit no-tick version migration -(ADR-0119). These formats do not change profiles 1–3. PostgreSQL admission and -atomic publication are separate migration requirements. +(ADR-0119). M11.4c adds cellular-v3 +[checkpoints](cellular-lives-v3.md) for append-only genome revision history +(ADR-0122), with a separate no-tick v2-to-v3 migration. These formats do not +change profiles 1–3. PostgreSQL admission and atomic publication are separate +migration requirements. ## Canonical manifest encoding v1/v2 (ADR-0061, ADR-0063) @@ -33,7 +36,7 @@ record, and each commit row stores it. | 160 | 36 | nine u32 versions: environmental/primitive bundles as below; profile 4 has cellular rules, envelope, habitat, coordinate, genome, development, founder, archive, then reserved zero | | 196 | 32 | retained build identity (nonzero; `retained_builds` row required) | | 228 | 4 | rng_streams = 0 (these rules use no randomness) | -| 232 | 4 | history_profile: 0 with profile 1; 1 with profile 2; 2 with profile 3; 3 with profile 4 | +| 232 | 4 | history_profile: 0 with profile 1; 1 with profile 2; 2 with profile 3; 3 with cellular-v1/v2; 4 with cellular-v3 | | 236 | 4 | durability_policy = 1 (single-volume development) | | 240 | 8 | payload_bytes: profile 1 is 1..1 MiB; profile 2 is 1..4,268,144; profile 3 is 1..3,266,800; profile 4 is 1..16 MiB | | 248 | 32 | payload SHA-256 (nonzero) | @@ -51,15 +54,18 @@ founder/archive in the first eight u32 version slots. Version bundle 1 is `(1,1,1,1,1,2,1,1,0)` and uses [cellular-lives-v1](cellular-lives-v1.md). Version bundle 2 is `(2,2,1,1,1,2,1,1,1)` and uses [cellular-lives-v2](cellular-lives-v2.md); the ninth slot pins mutation state. -The tick quantum remains 50,000 μs. Both bundles use the bounded profile-4 trace -and bind their decision blocks transactionally. A region's ordinary successor -keeps its exact version bundle. Only the explicit leased migration commit in -ADR-0119 may transition v1 to v2 at an unchanged tick, with an immutable source -and target link; generic cellular publication cannot change versions or publish -a zero-tick commit. Profile-4 clone genesis remains unsupported until cellular -identity remapping is specified. Migration 18 admits profile 4, and the -dedicated cellular publisher binds tick evidence. Profile 3 retains its -creation-only trace and zero-decision rules. +Version bundle 3 is `(2,3,1,1,1,2,1,2,1)` and uses +[cellular-lives-v3](cellular-lives-v3.md); its archive slot selects revision +history format 2. The tick quantum remains 50,000 μs. All three bundles use the +bounded profile-4 trace and bind their decision blocks transactionally. A +region's ordinary successor keeps its exact version bundle. Only the explicit +leased migration commits in ADR-0119 and ADR-0122 may transition v1 to v2 or v2 +to v3 at an unchanged tick, with immutable source and target links; generic +cellular publication cannot change versions or publish a zero-tick commit. +Profile-4 clone genesis remains unsupported until cellular identity remapping +is specified. Migration 18 admits profile 4, and the dedicated cellular +publisher binds tick evidence. Profile 3 retains its creation-only trace and +zero-decision rules. Profile 2 payloads are schemas/environmental-lineage-v1.md containers. A region never changes profile, history, RNG or durability declarations; schema v4 enforces this against the predecessor. For profile 1, the payload is the exact diff --git a/src/biology/lineage.cpp b/src/biology/lineage.cpp index e941811..cc89b15 100644 --- a/src/biology/lineage.cpp +++ b/src/biology/lineage.cpp @@ -1,6 +1,7 @@ #include "evo/biology/lineage.hpp" #include +#include namespace evo::biology { namespace { @@ -69,12 +70,65 @@ std::expected LineageArchive::record_death(std::uint64_t org if (when < found->record.born) { return std::unexpected{HistoryError::invalid_tick}; } + if (std::ranges::any_of(revisions_, [&](const auto& revision) { + return revision.organism == organism && revision.at >= when; + })) + return std::unexpected{HistoryError::invalid_tick}; found->record.alive = false; found->record.died = when; // The organism is gone. Its genome, ancestry and origin are untouched. return {}; } +std::expected +LineageArchive::record_mutation(std::uint64_t organism, const Genome& successor, Tick when) { + const auto organism_record = locate(organisms_, organism, by_organism); + if (organism_record == organisms_.end() || organism_record->id != organism || + !organism_record->record.alive) + return std::unexpected{HistoryError::invalid_revision}; + if (when <= organism_record->record.born) + return std::unexpected{HistoryError::invalid_tick}; + if (genomes_.size() >= max_archived_genomes) + return std::unexpected{HistoryError::archive_full}; + if (successor.id() == 0 || successor.origin_event() != successor.id() || + successor.parents().size() != 1 || holds(genomes_, successor.id(), by_genome) || + std::ranges::any_of(genomes_, [&](const auto& genome) { + return genome.origin_event() == successor.origin_event(); + })) + return std::unexpected{HistoryError::invalid_revision}; + const auto current = current_genome(organism); + if (!current || successor.parents()[0] != *current) + return std::unexpected{HistoryError::invalid_revision}; + if (std::ranges::any_of(revisions_, [&](const auto& revision) { + return revision.organism == organism && revision.at >= when; + })) + return std::unexpected{HistoryError::invalid_tick}; + if (std::ranges::any_of(revisions_, [&](const auto& revision) { + return revision.origin_event == successor.origin_event(); + })) + return std::unexpected{HistoryError::invalid_revision}; + + revisions_.reserve(revisions_.size() + 1); + genomes_.reserve(genomes_.size() + 1); + const auto genome_slot = locate(genomes_, successor.id(), by_genome); + genomes_.insert(genome_slot, successor); + const OrganismGenomeRevision revision{organism, *current, successor.id(), + successor.origin_event(), when}; + const auto revision_slot = std::ranges::lower_bound( + revisions_, revision, + [](const OrganismGenomeRevision& left, const OrganismGenomeRevision& right) { + return std::tie(left.at.value, left.organism) < + std::tie(right.at.value, right.organism); + }); + revisions_.insert(revision_slot, revision); + format_version_ = lineage_archive_revision_format; + return {}; +} + +void LineageArchive::upgrade_to_revision_format() noexcept { + format_version_ = lineage_archive_revision_format; +} + std::expected LineageArchive::find(std::uint64_t genome) const { const auto found = locate(genomes_, genome, by_genome); if (found == genomes_.end() || found->id() != genome) { @@ -91,6 +145,58 @@ std::expected LineageArchive::organism(std::uint64 return found->record; } +std::expected +LineageArchive::current_genome(std::uint64_t identity) const { + const auto found = locate(organisms_, identity, by_organism); + if (found == organisms_.end() || found->id != identity) + return std::unexpected{HistoryError::unknown_organism}; + auto current = found->record.genome; + for (const auto& revision : revisions_) { + if (revision.organism != identity) + continue; + if (revision.previous_genome != current) + return std::unexpected{HistoryError::invalid_revision}; + current = revision.successor_genome; + } + return current; +} + +std::expected +LineageArchive::append_revision(std::uint64_t identity, std::uint64_t previous, + std::uint64_t successor, std::uint64_t origin_event, Tick when) { + const auto record = locate(organisms_, identity, by_organism); + const auto child = find(successor); + const auto current = current_genome(identity); + if (record == organisms_.end() || record->id != identity || !child || !current || + *current != previous || child->parents().size() != 1 || child->parents()[0] != previous || + child->origin_event() != origin_event || origin_event != successor || + when <= record->record.born || (!record->record.alive && when >= record->record.died)) + return std::unexpected{HistoryError::invalid_revision}; + if (std::ranges::any_of(revisions_, [&](const auto& revision) { + return revision.organism == identity && revision.at >= when; + })) + return std::unexpected{HistoryError::invalid_tick}; + if (std::ranges::any_of(revisions_, + [&](const auto& revision) { + return revision.successor_genome == successor || + revision.origin_event == origin_event; + }) || + std::ranges::any_of(genomes_, [&](const auto& genome) { + return genome.id() != successor && genome.origin_event() == origin_event; + })) + return std::unexpected{HistoryError::invalid_revision}; + const OrganismGenomeRevision revision{identity, previous, successor, origin_event, when}; + const auto slot = std::ranges::lower_bound( + revisions_, revision, + [](const OrganismGenomeRevision& left, const OrganismGenomeRevision& right) { + return std::tie(left.at.value, left.organism) < + std::tie(right.at.value, right.organism); + }); + revisions_.insert(slot, revision); + format_version_ = lineage_archive_revision_format; + return {}; +} + std::expected, HistoryError> LineageArchive::ancestry(std::uint64_t genome) const { if (!holds(genomes_, genome, by_genome)) { @@ -178,6 +284,7 @@ std::uint64_t LineageArchive::highest_organism() const noexcept { namespace { constexpr std::uint64_t archive_magic = 0x3152414C4F564541ULL; // "AEVOLAR1" constexpr std::size_t archive_header_bytes = 32; +constexpr std::size_t archive_revision_header_bytes = 40; std::uint64_t fnv(std::span bytes) noexcept { std::uint64_t value = 14695981039346656037ULL; for (const auto byte : bytes) @@ -229,11 +336,15 @@ std::vector LineageArchive::encode() const { // Genomes are id-sorted already, so a stable sort by depth yields (depth, id). std::ranges::stable_sort(order, {}, [&](std::size_t i) { return depth[i]; }); + const auto version = format_version_; + const auto header_bytes = + version == lineage_archive_format ? archive_header_bytes : archive_revision_header_bytes; std::vector out; - out.reserve(archive_header_bytes + (genomes_.size() * (2 + max_genome_bytes)) + - (organisms_.size() * archived_organism_bytes) + 8); + out.reserve(header_bytes + (genomes_.size() * (2 + max_genome_bytes)) + + (organisms_.size() * archived_organism_bytes) + + (revisions_.size() * archived_genome_revision_bytes) + 8); put(out, archive_magic, 8); - put(out, lineage_archive_format, 4); + put(out, version, 4); // The highest genome schema held: an archive of schema 1 records keeps its bytes. std::uint32_t schema = genome_schema_v1; for (const auto& g : genomes_) @@ -241,6 +352,10 @@ std::vector LineageArchive::encode() const { put(out, schema, 4); put(out, genomes_.size(), 4); put(out, organisms_.size(), 4); + if (version == lineage_archive_revision_format) { + put(out, revisions_.size(), 4); + put(out, 0, 4); + } put(out, 0, 8); // Total length, patched below. for (const auto i : order) { const auto record = genomes_[i].encode(); @@ -254,9 +369,17 @@ std::vector LineageArchive::encode() const { put(out, entry.record.alive ? 0 : entry.record.died.value, 8); put(out, entry.record.alive ? 1 : 0, 1); } + for (const auto& revision : revisions_) { + put(out, revision.organism, 8); + put(out, revision.previous_genome, 8); + put(out, revision.successor_genome, 8); + put(out, revision.origin_event, 8); + put(out, revision.at.value, 8); + } const auto total = out.size() + 8; for (unsigned b = 0; b < 8; ++b) - out[24 + b] = static_cast((total >> (8U * b)) & 0xFFU); + out[(version == lineage_archive_format ? 24 : 32) + b] = + static_cast((total >> (8U * b)) & 0xFFU); put(out, fnv(out), 8); return out; } @@ -265,24 +388,37 @@ std::expected LineageArchive::decode(std::span bytes) { if (bytes.size() < archive_header_bytes + 8 || get(bytes, 0, 8) != archive_magic) return std::unexpected{ArchiveCodecError::format}; + const auto format = get(bytes, 8, 4); + const auto header_bytes = + format == lineage_archive_format ? archive_header_bytes : archive_revision_header_bytes; + if (format != lineage_archive_format && format != lineage_archive_revision_format) + return std::unexpected{ArchiveCodecError::version}; + if (bytes.size() < header_bytes + 8) + return std::unexpected{ArchiveCodecError::format}; const auto schema = get(bytes, 12, 4); - if (get(bytes, 8, 4) != lineage_archive_format || schema < genome_schema_v1 || - schema > genome_schema_version) + if (schema < genome_schema_v1 || schema > genome_schema_version) return std::unexpected{ArchiveCodecError::version}; const auto genomes = get(bytes, 16, 4); const auto organisms = get(bytes, 20, 4); - if (bytes.size() > max_archive_bytes || genomes > max_archived_genomes || - organisms > max_archived_organisms) + const auto revisions = format == lineage_archive_revision_format ? get(bytes, 24, 4) : 0; + if ((format == lineage_archive_format && bytes.size() > max_archive_legacy_bytes) || + (format == lineage_archive_revision_format && bytes.size() > max_archive_bytes) || + genomes > max_archived_genomes || organisms > max_archived_organisms || + revisions > max_archived_genomes) return std::unexpected{ArchiveCodecError::bounds}; - if (get(bytes, 24, 8) != bytes.size()) + if ((format == lineage_archive_revision_format && get(bytes, 28, 4) != 0) || + genomes > max_archived_genomes || organisms > max_archived_organisms) + return std::unexpected{ArchiveCodecError::format}; + if (get(bytes, format == lineage_archive_format ? 24 : 32, 8) != bytes.size()) return std::unexpected{ArchiveCodecError::format}; const auto body = bytes.first(bytes.size() - 8); if (get(bytes, bytes.size() - 8, 8) != fnv(body)) return std::unexpected{ArchiveCodecError::checksum}; LineageArchive archive; + archive.format_version_ = static_cast(format); std::uint32_t highest = genome_schema_v1; - std::size_t at = archive_header_bytes; + std::size_t at = header_bytes; for (std::uint64_t g = 0; g < genomes; ++g) { if (at + 2 > body.size()) return std::unexpected{ArchiveCodecError::format}; @@ -301,7 +437,9 @@ LineageArchive::decode(std::span bytes) { // Canonical: the header names exactly the highest schema held (ADR-0099). if (highest != schema) return std::unexpected{ArchiveCodecError::version}; - if (body.size() - at != organisms * archived_organism_bytes) + const auto expected_tail = + organisms * archived_organism_bytes + revisions * archived_genome_revision_bytes; + if (body.size() - at != expected_tail) return std::unexpected{ArchiveCodecError::format}; for (std::uint64_t o = 0; o < organisms; ++o, at += archived_organism_bytes) { const auto id = get(body, at, 8); @@ -314,6 +452,13 @@ LineageArchive::decode(std::span bytes) { (alive == 0 && !archive.record_death(id, died))) return std::unexpected{ArchiveCodecError::history}; } + for (std::uint64_t r = 0; r < revisions; ++r, at += archived_genome_revision_bytes) { + const auto recorded = + archive.append_revision(get(body, at, 8), get(body, at + 8, 8), get(body, at + 16, 8), + get(body, at + 24, 8), Tick{get(body, at + 32, 8)}); + if (!recorded) + return std::unexpected{ArchiveCodecError::history}; + } if (!std::ranges::equal(archive.encode(), bytes)) return std::unexpected{ArchiveCodecError::noncanonical}; return archive; diff --git a/src/persistence/cellular_checkpoint_store.cpp b/src/persistence/cellular_checkpoint_store.cpp index da091b4..75064ec 100644 --- a/src/persistence/cellular_checkpoint_store.cpp +++ b/src/persistence/cellular_checkpoint_store.cpp @@ -149,7 +149,8 @@ retry_cellular_profile_migration(Postgres& db, const CellularProfileMigrationPre const auto manifest = decode_manifest(prepared.manifest); if (!manifest || manifest->key != prepared.key || manifest->commit != prepared.commit || manifest->profile != CheckpointProfile::cellular_lives_v1 || - manifest->cellular_versions != cellular_v2_versions || + (manifest->cellular_versions != cellular_v2_versions && + manifest->cellular_versions != cellular_v3_versions) || manifest->begin_tick != manifest->end_tick || manifest->batch != BatchDigests{}) return std::unexpected{migration_failure(PublicationError::candidate, {}, {}, prepared)}; @@ -232,6 +233,56 @@ migrate_cellular_profile_to_v2(Postgres& db, const FilesystemObjectStore& object db, CellularProfileMigrationPrepared{previous->key, commit, std::move(*encoded)}, timeout); } +std::expected +migrate_cellular_profile_to_v3(Postgres& db, const FilesystemObjectStore& objects, + const LeaseGrant& lease, const CommittedCheckpoint& predecessor, + Uuid commit, const Sha256Digest& build, milliseconds timeout) { + const auto previous = decode_manifest(predecessor.manifest); + const auto previous_digest = sha256(predecessor.manifest); + if (!previous || !previous_digest || + previous->profile != CheckpointProfile::cellular_lives_v1 || + previous->cellular_versions != cellular_v2_versions || + predecessor.head.digest != *previous_digest || + predecessor.head.sequence != previous->sequence || + predecessor.head.commit != previous->commit || + predecessor.head.end_tick != previous->end_tick || lease.frame != previous->frame || + previous->sequence == 0 || previous->end_tick == 0) + return std::unexpected{migration_failure(PublicationError::candidate)}; + + auto source_payload = objects.get({previous->payload_sha256, previous->payload_bytes}); + if (!source_payload) + return std::unexpected{ + migration_failure(PublicationError::predecessor, source_payload.error())}; + const auto source_digest = sha256(*source_payload); + if (!source_digest || *source_digest != previous->payload_sha256) + return std::unexpected{ + migration_failure(PublicationError::predecessor, ObjectError::corrupt)}; + auto source = restore_cellular_payload(*source_payload, *previous); + if (!source || (*source)->format_version() != server::cellular_lives_format || + (*source)->rules_version() != server::cellular_lives_rules) + return std::unexpected{migration_failure(PublicationError::predecessor)}; + auto migrated = (*source)->migrate_to_v3(); + if (!migrated || (*migrated)->tick() != (*source)->tick() || + (*migrated)->current_total() != (*source)->current_total()) + return std::unexpected{migration_failure(PublicationError::candidate)}; + + const auto payload = (*migrated)->snapshot(); + const auto payload_digest = sha256(payload); + if (!payload_digest) + return std::unexpected{migration_failure(PublicationError::digest)}; + auto manifest = cellular_profile_migration_manifest(*previous, *previous_digest, commit, lease, + build, payload.size(), *payload_digest); + auto encoded = manifest ? encode_manifest(*manifest) + : std::expected, CheckpointError>{ + std::unexpected{CheckpointError::format}}; + if (!encoded || !restore_cellular_payload(payload, *manifest)) + return std::unexpected{migration_failure(PublicationError::candidate)}; + if (auto stored = put_verified(objects, payload, {*payload_digest, payload.size()}); !stored) + return std::unexpected{migration_failure(stored.error().kind, stored.error().object)}; + return retry_cellular_profile_migration( + db, CellularProfileMigrationPrepared{previous->key, commit, std::move(*encoded)}, timeout); +} + std::expected retry_cellular_prepared(Postgres& db, const CellularPreparedCommit& prepared, milliseconds timeout) { @@ -385,8 +436,11 @@ recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects, predecessor->head.digest != *source_manifest_digest || predecessor->head.commit != manifest->predecessor_commit || predecessor->head.sequence != manifest->sequence - 1 || - source_manifest->cellular_versions != cellular_v1_versions || source_manifest->profile != CheckpointProfile::cellular_lives_v1 || + (source_manifest->cellular_versions != cellular_v1_versions && + source_manifest->cellular_versions != cellular_v2_versions) || + ((source_manifest->cellular_versions == cellular_v1_versions) != + (manifest->cellular_versions == cellular_v2_versions)) || source_manifest->end_tick != manifest->end_tick || (*migration)->source_sequence != source_manifest->sequence || (*migration)->source_commit != source_manifest->commit || @@ -409,9 +463,11 @@ recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects, return fail_with(RecoveryError::tail); auto source_state = restore_cellular_payload(*source_bytes, *source_manifest); if (!source_state || - (*source_state)->format_version() != server::cellular_lives_legacy_format) + (*source_state)->format_version() != source_manifest->cellular_versions.envelope) return fail_with(RecoveryError::tail); - auto migrated = (*source_state)->migrate_to_v2(); + auto migrated = source_manifest->cellular_versions == cellular_v1_versions + ? (*source_state)->migrate_to_v2() + : (*source_state)->migrate_to_v3(); if (!migrated || (*migrated)->tick() != manifest->end_tick || (*migrated)->snapshot() != *payload) return fail_with(RecoveryError::history); diff --git a/src/persistence/region_checkpoint.cpp b/src/persistence/region_checkpoint.cpp index 044f8d8..67a5165 100644 --- a/src/persistence/region_checkpoint.cpp +++ b/src/persistence/region_checkpoint.cpp @@ -61,12 +61,15 @@ bool structural(const CheckpointManifest& m) noexcept { const auto& b = m.batch; const bool primitive = m.profile == CheckpointProfile::primitive_lives_v1; const bool cellular = m.profile == CheckpointProfile::cellular_lives_v1; - const bool cellular_migration = - cellular && m.sequence > 0 && m.cellular_versions == cellular_v2_versions && - m.begin_tick == m.end_tick && m.end_tick > 0 && b == BatchDigests{}; + const bool cellular_migration = cellular && m.sequence > 0 && + (m.cellular_versions == cellular_v2_versions || + m.cellular_versions == cellular_v3_versions) && + m.begin_tick == m.end_tick && m.end_tick > 0 && + b == BatchDigests{}; if ((!cellular && m.cellular_versions != cellular_v1_versions) || (cellular && m.cellular_versions != cellular_v1_versions && - m.cellular_versions != cellular_v2_versions) || + m.cellular_versions != cellular_v2_versions && + m.cellular_versions != cellular_v3_versions) || (cellular && m.versions != environmental_v1_versions)) return false; if (primitive && (m.manifest_version != 2 || m.tick_microseconds != 50000 || @@ -139,12 +142,18 @@ bool supported(const CheckpointManifest& m) noexcept { return m.manifest_version == 2 && m.history_profile == primitive_history_profile && m.versions == primitive_v1_versions && m.tick_microseconds == 50000 && m.rng_streams == 0 && m.durability == DurabilityPolicy::single_volume_development; - if (m.profile == CheckpointProfile::cellular_lives_v1) - return m.manifest_version == 2 && m.history_profile == cellular_history_profile && + if (m.profile == CheckpointProfile::cellular_lives_v1) { + const bool history_matches = (m.cellular_versions != cellular_v3_versions && + m.history_profile == cellular_history_profile) || + (m.cellular_versions == cellular_v3_versions && + m.history_profile == cellular_revision_history_profile); + return m.manifest_version == 2 && history_matches && (m.cellular_versions == cellular_v1_versions || - m.cellular_versions == cellular_v2_versions) && + m.cellular_versions == cellular_v2_versions || + m.cellular_versions == cellular_v3_versions) && m.versions == environmental_v1_versions && m.tick_microseconds == 50000 && m.rng_streams == 0 && m.durability == DurabilityPolicy::single_volume_development; + } const bool profile = (m.profile == CheckpointProfile::environmental_v1 && m.history_profile == 0) || (m.profile == CheckpointProfile::environmental_lineage_v1 && @@ -354,10 +363,12 @@ std::expected cellular_profile_migration_ma if (!supported(p)) return std::unexpected{CheckpointError::unsupported}; if (!structural(p) || p.profile != CheckpointProfile::cellular_lives_v1 || - p.cellular_versions != cellular_v1_versions || p.sequence == 0 || p.sequence == maximum || - p.end_tick == 0 || lease.frame != p.frame || lease.holder == Uuid{} || lease.epoch == 0 || - predecessor_digest == Sha256Digest{} || commit == Uuid{} || commit == p.commit || - build == Sha256Digest{} || payload_bytes == 0 || payload_sha256 == Sha256Digest{}) + (p.cellular_versions != cellular_v1_versions && + p.cellular_versions != cellular_v2_versions) || + p.sequence == 0 || p.sequence == maximum || p.end_tick == 0 || lease.frame != p.frame || + lease.holder == Uuid{} || lease.epoch == 0 || predecessor_digest == Sha256Digest{} || + commit == Uuid{} || commit == p.commit || build == Sha256Digest{} || payload_bytes == 0 || + payload_sha256 == Sha256Digest{}) return std::unexpected{CheckpointError::chain}; CheckpointManifest m = p; m.commit = commit; @@ -371,7 +382,13 @@ std::expected cellular_profile_migration_ma m.build = build; m.payload_bytes = payload_bytes; m.payload_sha256 = payload_sha256; - m.cellular_versions = cellular_v2_versions; + if (p.cellular_versions == cellular_v1_versions) { + m.cellular_versions = cellular_v2_versions; + m.history_profile = cellular_history_profile; + } else { + m.cellular_versions = cellular_v3_versions; + m.history_profile = cellular_revision_history_profile; + } m.batch = {}; if (!structural(m)) return std::unexpected{CheckpointError::format}; diff --git a/src/server/lifecycle/cellular_lives.cpp b/src/server/lifecycle/cellular_lives.cpp index 368b638..1f3d4e6 100644 --- a/src/server/lifecycle/cellular_lives.cpp +++ b/src/server/lifecycle/cellular_lives.cpp @@ -12,6 +12,7 @@ namespace evo::server { namespace { constexpr std::uint64_t cellular_lives_v1_magic = 0x3130564C43564541ULL; // AEVCLV01 constexpr std::uint64_t cellular_lives_v2_magic = 0x3230564C43564541ULL; // AEVCLV02 +constexpr std::uint64_t cellular_lives_v3_magic = 0x3330564C43564541ULL; // AEVCLV03 constexpr std::size_t cellular_lives_header_bytes = 212; constexpr std::size_t cellular_site_bytes = 8; constexpr std::size_t cellular_life_v1_bytes = 168; @@ -222,8 +223,9 @@ validate_state(std::uint64_t world, std::uint64_t region, std::uint64_t frame, lives[j].identity.organism == life.identity.organism) return std::unexpected{CellularLivesError::duplicate_life}; const auto record = archive.organism(life.identity.organism); - if (!record || !record->alive || record->genome != life.identity.genome || - record->born != life.identity.born) + const auto current_genome = archive.current_genome(life.identity.organism); + if (!record || !record->alive || !current_genome || + *current_genome != life.identity.genome || record->born != life.identity.born) return std::unexpected{CellularLivesError::archive}; const auto budget = budget_for(archive, life, config.ambient_temperature_millikelvin); if (!budget || budget->body != life.body || life.body.energy.value != 0 || @@ -248,7 +250,10 @@ validate_state(std::uint64_t world, std::uint64_t region, std::uint64_t frame, const auto record = archive.organism(corpse.organism); if (!record || record->alive || record->died.value != corpse.died_tick) return std::unexpected{CellularLivesError::archive}; - const auto genome = archive.find(record->genome); + const auto current_genome = archive.current_genome(corpse.organism); + if (!current_genome) + return std::unexpected{CellularLivesError::archive}; + const auto genome = archive.find(*current_genome); if (!genome) return std::unexpected{CellularLivesError::archive}; const auto budget = biology::metabolic_budget(*genome, corpse.growth, @@ -347,13 +352,13 @@ CellularLives::create(const PrimitiveLives& opening, opening.balances()[body_index], opening.balances()[reserve_index]}); } + auto archive = opening.archive(); std::ranges::sort(lives, {}, [](const auto& life) { return life.identity.life; }); for (std::size_t i = 0; i < lives.size(); ++i) { const auto& life = lives[i]; if (i != 0 && lives[i - 1].identity.life == life.identity.life) return std::unexpected{CellularLivesError::duplicate_life}; - const auto budget = - budget_for(opening.archive(), life, config.ambient_temperature_millikelvin); + const auto budget = budget_for(archive, life, config.ambient_temperature_millikelvin); if (!budget || budget->phenotype.extent == 0 || budget->phenotype.extent > config.area.width_um || budget->phenotype.extent > config.area.height_um || life.body.energy.value != 0 || @@ -368,9 +373,9 @@ CellularLives::create(const PrimitiveLives& opening, const auto nursery = opening.balances()[0]; nursery_config.funding = nursery; const auto heat = opening.balances()[1]; - const auto current = validate_state(nursery_config.world, nursery_config.region, - nursery_config.frame, config, nursery_config, *habitat, - lives, {}, opening.archive(), nursery, heat, opening_total); + const auto current = + validate_state(nursery_config.world, nursery_config.region, nursery_config.frame, config, + nursery_config, *habitat, lives, {}, archive, nursery, heat, opening_total); if (!current) return std::unexpected{current.error()}; @@ -379,8 +384,8 @@ CellularLives::create(const PrimitiveLives& opening, const auto frame = nursery_config.frame; auto state = std::make_unique(world, region, frame, config, std::move(nursery_config), std::move(habitat), std::move(lives), - std::vector{}, opening.archive(), - nursery, heat, opening_total, *current); + std::vector{}, archive, nursery, heat, + opening_total, *current); return std::unique_ptr{new CellularLives{std::move(state)}}; } @@ -458,8 +463,11 @@ std::vector CellularLives::snapshot() const { std::vector bytes; bytes.reserve(total_bytes); Writer out{bytes}; - out.put(state_->format_version == cellular_lives_legacy_format ? cellular_lives_v1_magic - : cellular_lives_v2_magic); + const auto magic = + state_->format_version == cellular_lives_legacy_format ? cellular_lives_v1_magic + : state_->format_version == cellular_lives_revision_format ? cellular_lives_v3_magic + : cellular_lives_v2_magic; + out.put(magic); out.put(state_->format_version, 4); out.put(state_->rules_version, 4); out.put(state_->world); @@ -595,6 +603,32 @@ CellularLives::migrate_to_v2() const { return candidate; } +std::expected, CellularLivesError> +CellularLives::migrate_to_v3() const { + if (state_->format_version != cellular_lives_format || + state_->rules_version != cellular_lives_rules || + state_->archive.format_version() != biology::lineage_archive_format) + return std::unexpected{CellularLivesError::version}; + auto archive = state_->archive; + archive.upgrade_to_revision_format(); + auto habitat = biology::CellularHabitat::restore(state_->habitat->snapshot(), state_->world, + state_->region, state_->frame); + if (!habitat) + return std::unexpected{CellularLivesError::habitat}; + const auto current = + validate_state(state_->world, state_->region, state_->frame, state_->config, + state_->nursery_config, **habitat, state_->lives, state_->corpses, archive, + state_->nursery, state_->heat, state_->opening_total); + if (!current || *current != state_->current_total) + return std::unexpected{current ? CellularLivesError::conservation : current.error()}; + auto candidate = std::unique_ptr{new CellularLives{std::make_unique( + state_->world, state_->region, state_->frame, state_->config, state_->nursery_config, + std::move(*habitat), state_->lives, state_->corpses, archive, state_->nursery, state_->heat, + state_->opening_total, state_->current_total, cellular_lives_revision_format, + cellular_lives_rules)}}; + return candidate; +} + std::expected, CellularLivesError> CellularLives::birth_candidate(LifeRequest request, LifeAllocation allocation) const { if (request.action == 0 || @@ -723,10 +757,14 @@ CellularLives::restore(std::span bytes, std::uint64_t world, const auto rules_version = static_cast(in.get(4)); const bool legacy = magic == cellular_lives_v1_magic && format_version == cellular_lives_legacy_format && rules_version == 1; - const bool current_version = magic == cellular_lives_v2_magic && - format_version == cellular_lives_format && - rules_version == cellular_lives_rules; - if (!legacy && !current_version) + const bool offer_version = magic == cellular_lives_v2_magic && + format_version == cellular_lives_format && + rules_version == cellular_lives_rules; + const bool revision_version = magic == cellular_lives_v3_magic && + format_version == cellular_lives_revision_format && + rules_version == cellular_lives_rules; + const bool mutation_version = offer_version || revision_version; + if (!legacy && !mutation_version) return std::unexpected{CellularLivesError::version}; const auto saved_world = in.get(); const auto saved_region = in.get(); @@ -785,7 +823,7 @@ CellularLives::restore(std::span bytes, std::uint64_t world, life.position = {in.get(), in.get()}; life.body = in.inventory(); life.reserve = in.inventory(); - if (current_version) { + if (mutation_version) { auto& mutation = life.mutation; mutation.fed_energy_credit_pj = in.get(); mutation.event_index = in.get(); @@ -873,7 +911,10 @@ CellularLives::restore(std::span bytes, std::uint64_t world, return std::unexpected{CellularLivesError::noncanonical}; return std::unexpected{CellularLivesError::archive}; } - if (current_version) { + if (archive->format_version() != (revision_version ? biology::lineage_archive_revision_format + : biology::lineage_archive_format)) + return std::unexpected{CellularLivesError::version}; + if (mutation_version) { for (const auto& life : lives) { if (!life.mutation.offer) continue; diff --git a/src/server/lifecycle/primitive_lives.cpp b/src/server/lifecycle/primitive_lives.cpp index 39173a3..d85ea7f 100644 --- a/src/server/lifecycle/primitive_lives.cpp +++ b/src/server/lifecycle/primitive_lives.cpp @@ -256,7 +256,7 @@ PrimitiveLives::restore(std::span bytes, std::uint64_t world c.funding = in.inventory(); const auto sites = in.get(4), count = in.get(4), archive_bytes = in.get(4); if (in.get(4) != 0 || sites == 0 || sites > max_primitive_lives || count > sites || - archive_bytes > biology::max_archive_bytes || + archive_bytes > biology::max_archive_legacy_bytes || bytes.size() != life_header_bytes + 8 * sites + life_record_bytes * count + 40 * (2 + 2 * sites) + archive_bytes + 8) return std::unexpected{LifeCodecError::format}; diff --git a/tests/cellular_lives_tests.cpp b/tests/cellular_lives_tests.cpp index 0c50200..a6c34c1 100644 --- a/tests/cellular_lives_tests.cpp +++ b/tests/cellular_lives_tests.cpp @@ -212,6 +212,24 @@ void cellular_profile_migration_is_explicit_and_no_tick() { CHECK((*migrated)->advance_tick(migrated_idle) && (*migrated)->format_version() == cellular_lives_format && (*migrated)->rules_version() == cellular_lives_rules && (*migrated)->tick() == 3); + + const auto v2_before = (*migrated)->snapshot(); + const auto revision_profile = (*migrated)->migrate_to_v3(); + CHECK(revision_profile && + (*revision_profile)->format_version() == cellular_lives_revision_format && + (*revision_profile)->rules_version() == cellular_lives_rules && + (*revision_profile)->tick() == (*migrated)->tick() && + (*revision_profile)->current_total() == (*migrated)->current_total() && + std::ranges::equal((*revision_profile)->lives(), (*migrated)->lives()) && + (*revision_profile)->archive().format_version() == + evo::biology::lineage_archive_revision_format && + (*revision_profile)->archive().revisions().empty() && + (*migrated)->snapshot() == v2_before); + const auto v3_bytes = (*revision_profile)->snapshot(); + const auto v3_restored = + CellularLives::restore(v3_bytes, 91, 1, 1, (*revision_profile)->tick()); + CHECK(v3_restored && (*v3_restored)->snapshot() == v3_bytes && + (*v3_restored)->format_version() == cellular_lives_revision_format); } void retained_feeding_earns_a_persistent_mutation_offer() { diff --git a/tests/cellular_publication_tests.cpp b/tests/cellular_publication_tests.cpp index d825abd..6d58b2c 100644 --- a/tests/cellular_publication_tests.cpp +++ b/tests/cellular_publication_tests.cpp @@ -224,13 +224,19 @@ void cellular_profile_migration_is_durable_and_recovers_without_a_tick() { CHECK(migrated && migrated->head.sequence == 2 && migrated->head.end_tick == 1); CHECK( retry_cellular_profile_migration(*owner, {key, filled<16>(0x51), migrated->manifest}, 5s)); + auto revision_migrated = migrate_cellular_profile_to_v3(*owner, **migration_objects, *authority, + *migrated, filled<16>(0x52), build, 5s); + CHECK(revision_migrated && revision_migrated->head.sequence == 3 && + revision_migrated->head.end_tick == 1); + CHECK(retry_cellular_profile_migration( + *owner, {key, filled<16>(0x52), revision_migrated->manifest}, 5s)); host->drain(); host.reset(); auto recovery_objects = FilesystemObjectStore::open(env("PGHOST"), key, max_cellular_lives_bytes); CHECK(recovery_objects); - const auto target_manifest = decode_manifest(migrated->manifest); + const auto target_manifest = decode_manifest(revision_migrated->manifest); CHECK(target_manifest); const auto target_payload = (*recovery_objects)->get({target_manifest->payload_sha256, target_manifest->payload_bytes}); @@ -238,9 +244,9 @@ void cellular_profile_migration_is_durable_and_recovers_without_a_tick() { auto restarted_made = CellularRegionHost::create(config, connect, std::move(*recovery_objects)); CHECK(restarted_made); auto restarted = std::move(*restarted_made); - CHECK(restarted->step().ready && restarted->committed()->head.sequence == 2 && + CHECK(restarted->step().ready && restarted->committed()->head.sequence == 3 && restarted->committed()->head.end_tick == 1 && restarted->state()->tick() == 1 && - restarted->state()->format_version() == cellular_lives_format && + restarted->state()->format_version() == cellular_lives_revision_format && restarted->state()->snapshot() == *target_payload); restarted->drain(); } diff --git a/tests/lineage_tests.cpp b/tests/lineage_tests.cpp index 322adc9..8748173 100644 --- a/tests/lineage_tests.cpp +++ b/tests/lineage_tests.cpp @@ -334,7 +334,7 @@ void the_archive_decoder_refuses_damage_and_disorder() { }; CHECK(refused([](auto& b) { b[40] ^= 1U; }, ArchiveCodecError::checksum, false)); CHECK(refused([](auto& b) { b[0] ^= 1U; }, ArchiveCodecError::format)); - CHECK(refused([](auto& b) { b[8] = 2; }, ArchiveCodecError::version)); + CHECK(refused([](auto& b) { b[8] = 3; }, ArchiveCodecError::version)); CHECK(refused([](auto& b) { b[12] = 2; }, ArchiveCodecError::version)); CHECK(refused([](auto& b) { set(b, 16, max_archived_genomes + 1, 4); }, ArchiveCodecError::bounds)); @@ -382,6 +382,65 @@ void the_archive_decoder_refuses_damage_and_disorder() { CHECK(!LineageArchive::decode(std::span{bytes}.first(39))); } +void genome_revisions_preserve_birth_and_replay_current_genome() { + LineageArchive archive; + CHECK(archive.admit(descend(10, 0))); + CHECK(archive.record_birth(50, 10, evo::Tick{100})); + CHECK(archive.current_genome(50) && *archive.current_genome(50) == 10); + const auto successor = descend(11, 10, 420, genome_schema_version); + CHECK(archive.record_mutation(50, successor, evo::Tick{101})); + CHECK(archive.revisions().size() == 1); + CHECK(archive.current_genome(50) && *archive.current_genome(50) == 11); + const auto birth = archive.organism(50); + CHECK(birth && birth->genome == 10 && birth->born.value == 100 && birth->alive); + CHECK(archive.find(11) && archive.find(11)->parents()[0] == 10); + CHECK(archive.ancestry(11) && (*archive.ancestry(11) == std::vector{10, 11})); + + auto bytes = archive.encode(); + CHECK(field(bytes, 8, 4) == lineage_archive_revision_format); + CHECK(field(bytes, 24, 4) == 1 && field(bytes, 32, 8) == bytes.size()); + const auto decoded = LineageArchive::decode(bytes); + CHECK(decoded && decoded->encode() == bytes && decoded->revisions().size() == 1); + CHECK(decoded->current_genome(50) && *decoded->current_genome(50) == 11); + CHECK(decoded->organism(50) && decoded->organism(50)->genome == 10); + + CHECK(archive.record_death(50, evo::Tick{102})); + CHECK(archive.current_genome(50) && *archive.current_genome(50) == 11); + CHECK(!archive.record_mutation(50, descend(12, 11), evo::Tick{103})); + const auto died_bytes = archive.encode(); + const auto died = LineageArchive::decode(died_bytes); + CHECK(died && died->current_genome(50) && *died->current_genome(50) == 11); + CHECK(!died->record_death(50, evo::Tick{101})); +} + +void mutation_history_upgrade_and_failures_are_explicit() { + LineageArchive archive; + CHECK(archive.admit(descend(10, 0))); + CHECK(archive.record_birth(50, 10, evo::Tick{10})); + const auto format1 = archive.encode(); + archive.upgrade_to_revision_format(); + const auto format2 = archive.encode(); + CHECK(field(format1, 8, 4) == lineage_archive_format); + CHECK(field(format2, 8, 4) == lineage_archive_revision_format); + CHECK(field(format2, 24, 4) == 0); + const auto migrated = LineageArchive::decode(format2); + CHECK(migrated && migrated->encode() == format2 && migrated->revisions().empty()); + + auto fresh = LineageArchive{}; + CHECK(fresh.admit(descend(10, 0)) && fresh.record_birth(50, 10, evo::Tick{10})); + const auto wrong_parent = descend(11, 9); + CHECK(!fresh.record_mutation(50, wrong_parent, evo::Tick{11})); + CHECK(fresh.archived() == 1 && fresh.current_genome(50) && *fresh.current_genome(50) == 10); + CHECK(!fresh.record_mutation(50, descend(12, 10), evo::Tick{10})); + CHECK(!fresh.record_mutation(50, descend(13, 10), evo::Tick{9})); + CHECK(fresh.record_mutation(50, descend(11, 10), evo::Tick{11})); + CHECK(!fresh.record_mutation(50, descend(14, 11), evo::Tick{11})); + CHECK(fresh.record_mutation(50, descend(14, 11), evo::Tick{12})); + CHECK(!fresh.record_death(50, evo::Tick{12})); + CHECK(fresh.record_death(50, evo::Tick{13})); + CHECK(fresh.current_genome(50) && *fresh.current_genome(50) == 14); +} + // ADR-0099: the header names the highest genome schema held, so an archive of schema 1 // records keeps its exact bytes, and a mixed archive round-trips. void schemas_mix_without_rewriting() { @@ -416,11 +475,11 @@ void a_full_archive_round_trips() { CHECK(archive.record_death(o, evo::Tick{o + 1})); } const auto bytes = archive.encode(); - CHECK(bytes.size() <= max_archive_bytes); + CHECK(bytes.size() <= max_archive_legacy_bytes); const auto decoded = LineageArchive::decode(bytes); CHECK(decoded && decoded->encode() == bytes && decoded->recorded() == max_archived_organisms); - std::cout << "lineage_archive_full_bytes=" << bytes.size() << " max=" << max_archive_bytes - << '\n'; + std::cout << "lineage_archive_full_bytes=" << bytes.size() + << " max=" << max_archive_legacy_bytes << '\n'; } } // namespace int main() try { @@ -435,6 +494,8 @@ int main() try { capacity_refuses_loudly_rather_than_dropping(); the_archive_round_trips_canonically(); the_archive_decoder_refuses_damage_and_disorder(); + genome_revisions_preserve_birth_and_replay_current_genome(); + mutation_history_upgrade_and_failures_are_explicit(); a_full_archive_round_trips(); schemas_mix_without_rewriting(); std::cout << "PASS lineage retention, branching ancestry and provisional absence\n"; diff --git a/tests/region_checkpoint_tests.cpp b/tests/region_checkpoint_tests.cpp index f527a44..8307dda 100644 --- a/tests/region_checkpoint_tests.cpp +++ b/tests/region_checkpoint_tests.cpp @@ -481,6 +481,24 @@ void cellular_profile() { successor_manifest(*migration, filled<32>(0x21), filled<16>(0x22), lease(), 2, build, payload.size(), filled<32>(0x23), batch_for(1, 1)); CHECK(after_migration && after_migration->cellular_versions == cellular_v2_versions); + const auto revision_migration = + cellular_profile_migration_manifest(*after_migration, filled<32>(0x24), filled<16>(0x25), + lease(), build, payload.size(), filled<32>(0x26)); + CHECK(revision_migration && supported(*revision_migration) && + revision_migration->history_profile == cellular_revision_history_profile && + revision_migration->cellular_versions == cellular_v3_versions && + revision_migration->begin_tick == after_migration->end_tick && + revision_migration->end_tick == after_migration->end_tick && + revision_migration->batch == BatchDigests{}); + const auto revision_migration_bytes = encode_manifest(*revision_migration); + CHECK(revision_migration_bytes && + decode_manifest(*revision_migration_bytes) == *revision_migration); + const auto after_revision_migration = successor_manifest( + *revision_migration, filled<32>(0x27), filled<16>(0x28), lease(), + revision_migration->end_tick + 1, build, payload.size(), filled<32>(0x29), batch_for(1, 1)); + CHECK(after_revision_migration && + after_revision_migration->cellular_versions == cellular_v3_versions && + after_revision_migration->history_profile == cellular_revision_history_profile); auto bad_identity = *manifest; bad_identity.key.region = 4; diff --git a/tools/ci/postgres-check.sh b/tools/ci/postgres-check.sh index c2dfc0f..b613938 100644 --- a/tools/ci/postgres-check.sh +++ b/tools/ci/postgres-check.sh @@ -69,7 +69,7 @@ PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" -f "$root/migrations/0001_region_bootstrap.sql" > /dev/null test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1 PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26 +test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27 # v2 heads have no payload binding: v2->v3 fails closed and leaves history at v2. v2_sha="$(sha256sum "$root/migrations/0002_region_fencing.sql")" v2_sha="${v2_sha%% *}" @@ -90,7 +90,7 @@ PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '01000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32), 'hex'))" > /dev/null PGDATABASE=v3_commits bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 26:1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26 +test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 27:1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27 test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) FROM evolution.region_commits")" = 1 # Review P1: a v4 database holding a published (possibly terminal) checkpoint has # no durable decisions for it. The v5 upgrade must refuse before changing anything. @@ -144,7 +144,7 @@ PGDATABASE=v6_populated psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '02000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32) || repeat('00', 80), 'hex'))" > /dev/null PGDATABASE=v6_populated bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26 +test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT count(*) FROM evolution.region_commits c, LATERAL evolution.parse_checkpoint_manifest(c.manifest) m WHERE m.commit_sequence = 0 AND m.predecessor_commit IS NULL")" = 1 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(conname, ',') FROM pg_constraint WHERE conrelid = 'evolution.region_heads'::regclass AND contype = 'c' AND pg_get_constraintdef(oid) LIKE '%end_tick%'")" = region_heads_published_tick # Inherited privileges bypass fencing, so every migrate run audits effective rights. diff --git a/tools/migrate-postgres.sh b/tools/migrate-postgres.sh index 7722a20..d663a51 100644 --- a/tools/migrate-postgres.sh +++ b/tools/migrate-postgres.sh @@ -64,6 +64,8 @@ digest_25="$(sha256sum "$root/migrations/0025_cellular_v2_bundle.sql")" digest_25="${digest_25%% *}" digest_26="$(sha256sum "$root/migrations/0026_cellular_profile_migration.sql")" digest_26="${digest_26%% *}" +digest_27="$(sha256sum "$root/migrations/0027_cellular_v3_migration.sql")" +digest_27="${digest_27%% *}" psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_2=$digest_2" --set="migration_sha_3=$digest_3" \ --set="migration_sha_4=$digest_4" --set="migration_sha_5=$digest_5" \ @@ -83,5 +85,6 @@ psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_24=$digest_24" \ --set="migration_sha_25=$digest_25" \ --set="migration_sha_26=$digest_26" \ + --set="migration_sha_27=$digest_27" \ --set="runtime_role=$runtime_role" \ --file="$root/migrations/apply.sql"