From a2c4543df26e9d81c5819ef6d763b7e6b4ecb00d Mon Sep 17 00:00:00 2001 From: dingo Date: Mon, 28 Sep 2026 16:43:10 -0400 Subject: [PATCH] Complete M7.4b lineage checkpoint profile; M7.4 done ADR-0062 adds a canonical LineageArchive codec (parents before children, organisms by id, FNV) that decodes by replaying admit/record_birth/ record_death and requires byte-identical re-encoding; it never overwrites an existing archive. LineageRegion binds the environmental producer and consumer accounts to archived organisms with identity high-waters, enforces closure (lifecycle vs deaths, exact living set, tick bounds) and records deaths in the tick that causes them. Checkpoint profile 2 (environmental-lineage-v1, history profile 1) publishes that container; schema v4 admits it with a 4,268,144-byte cap and requires each commit to keep its predecessor's profile, RNG, history and durability declarations. Linux adds provision/publish/recover for the profile on the shared publication path. Qualification found WSL2 stepping CLOCK_REALTIME back ~0.5 s every ~32 s, which made two 1 s-lease tests flaky under ASan. Tests now wait on database-time expiry; ADR-0060 records the slewed-clock requirement. Validated: Windows CTest 57/57; Linux GCC Debug/Release, clang-tidy and ASan/UBSan CTest 57/57 each; PostgreSQL suites 2x Release, 2x analysis, 5x ASan; SQL and C++ mutations each caught. No Docker/Gitea CI, remote store, process kill or deployment. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 7 +- .../postgres-migration/job.yaml.example | 2 +- docs/ROADMAP.md | 13 ++ docs/STATUS.md | 24 +++ docs/adr/0034-lineage-archive.md | 4 +- docs/adr/0060-fenced-region-ownership.md | 14 +- docs/adr/0062-lineage-checkpoint-profile.md | 96 ++++++++++ docs/adr/README.md | 1 + docs/architecture/REGION_DURABILITY.md | 10 +- docs/biology/GENOME_MODEL.md | 3 +- docs/engineering/VALIDATION.md | 55 ++++++ docs/infrastructure/POSTGRESQL.md | 5 + include/evo/biology/lineage.hpp | 31 +++ include/evo/persistence/checkpoint_store.hpp | 22 +++ include/evo/persistence/region_checkpoint.hpp | 39 +++- include/evo/processes/lineage_region.hpp | 79 ++++++++ migrations/0004_lineage_profile.sql | 131 +++++++++++++ migrations/apply.sql | 11 +- schemas/environmental-lineage-v1.md | 38 ++++ schemas/lineage-archive-v1.md | 29 +++ schemas/region-commit.md | 11 +- src/biology/lineage.cpp | 147 ++++++++++++++ src/persistence/checkpoint_store.cpp | 154 +++++++++++---- src/persistence/region_checkpoint.cpp | 47 ++++- src/processes/lineage_region.cpp | 179 ++++++++++++++++++ tests/fixtures/environmental_region.hpp | 42 ++++ tests/lineage_region_tests.cpp | 146 ++++++++++++++ tests/lineage_tests.cpp | 148 +++++++++++++++ tests/region_checkpoint_store_tests.cpp | 67 +++++++ tests/region_checkpoint_tests.cpp | 59 ++++++ tests/region_lease_store_tests.cpp | 53 +++++- tools/ci/postgres-check.sh | 21 +- tools/migrate-postgres.sh | 3 + 33 files changed, 1622 insertions(+), 69 deletions(-) create mode 100644 docs/adr/0062-lineage-checkpoint-profile.md create mode 100644 include/evo/processes/lineage_region.hpp create mode 100644 migrations/0004_lineage_profile.sql create mode 100644 schemas/environmental-lineage-v1.md create mode 100644 schemas/lineage-archive-v1.md create mode 100644 src/processes/lineage_region.cpp create mode 100644 tests/lineage_region_tests.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index e6419fb..cc368ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,7 +62,7 @@ target_link_libraries(evo_ecology PUBLIC evo::accounting evo::biology) target_compile_features(evo_ecology PUBLIC cxx_std_23) evo_owned_target(evo_ecology) -add_library(evo_processes STATIC src/processes/producer.cpp src/processes/feeding.cpp src/processes/ecosystem.cpp src/processes/genome_strategy.cpp src/processes/ensemble.cpp src/processes/environmental_community.cpp src/processes/environmental_codec.cpp src/processes/environmental_schedule.cpp) +add_library(evo_processes STATIC src/processes/producer.cpp src/processes/feeding.cpp src/processes/ecosystem.cpp src/processes/genome_strategy.cpp src/processes/ensemble.cpp src/processes/environmental_community.cpp src/processes/environmental_codec.cpp src/processes/environmental_schedule.cpp src/processes/lineage_region.cpp) add_library(evo::processes ALIAS evo_processes) target_link_libraries(evo_processes PUBLIC evo::accounting evo::biology evo::world) evo_owned_target(evo_processes) @@ -118,6 +118,11 @@ if(BUILD_TESTING) evo_owned_target(evo-region-lease-tests) add_test(NAME region-lease-contracts COMMAND evo-region-lease-tests) set_tests_properties(region-lease-contracts PROPERTIES TIMEOUT 60) + add_executable(evo-lineage-region-tests tests/lineage_region_tests.cpp) + target_link_libraries(evo-lineage-region-tests PRIVATE evo::processes) + evo_owned_target(evo-lineage-region-tests) + add_test(NAME lineage-region-contracts COMMAND evo-lineage-region-tests) + set_tests_properties(lineage-region-contracts PROPERTIES TIMEOUT 60) add_executable(evo-region-checkpoint-tests tests/region_checkpoint_tests.cpp) target_link_libraries(evo-region-checkpoint-tests PRIVATE evo::region_checkpoint) evo_owned_target(evo-region-checkpoint-tests) diff --git a/deploy/kubernetes/postgres-migration/job.yaml.example b/deploy/kubernetes/postgres-migration/job.yaml.example index 94a3aad..a95a4eb 100644 --- a/deploy/kubernetes/postgres-migration/job.yaml.example +++ b/deploy/kubernetes/postgres-migration/job.yaml.example @@ -3,7 +3,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: evolution-db-migrate-v3 + name: evolution-db-migrate-v4 namespace: evolution-dev spec: backoffLimit: 0 diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 3074c2b..f09558f 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -529,3 +529,16 @@ codec and history-capable binding). M7.4a is implemented and locally qualified retained-build allowlists and restored-trajectory equality are covered. Next: M7.4b. M7.4 stays open until it passes. + +M7.4b implemented and locally qualified (2026-09-28); M7.4 is complete. ADR-0062 +adds: +- A canonical lineage archive codec that decodes by replaying the archive's own + rules. +- `LineageRegion`, which binds the environmental accounts to archived organisms, + keeps identity high-waters, enforces closure, and records deaths in the + causing tick. +- Checkpoint profile 2, and schema v4 (profile-specific caps; the profile is + fixed per region). +Recovery restores retained genomes, ancestry and dead organisms, and continues +identically. Next: M7.5 durable decisions, history and outbox rows in the same +transaction, and commit-ID resolution. diff --git a/docs/STATUS.md b/docs/STATUS.md index c7020f0..21d0226 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -788,3 +788,27 @@ builds. Windows ran 56/56; Linux Release, clang-tidy and ASan/UBSan PostgreSQL suites passed in WSL. M7.4 stays open for M7.4b (lineage archive codec and history-capable binding). No real world was provisioned; there is no deployment, host or process-kill test yet. + +## M7.4b lineage checkpoint profile; M7.4 complete (2026-09-28) + +ADR-0062 adds a canonical `LineageArchive` codec that decodes by replaying the +archive's own admission, birth and death rules. It also adds `LineageRegion`, +which: +- binds the two environmental accounts to archived organisms; +- keeps identity high-waters; +- enforces closure over lifecycle, the living set and ticks; +- records deaths in the tick that causes them. + +Checkpoint profile 2 (`environmental-lineage-v1`) and schema v4 publish and recover +it. A region can never switch profile. Recovery restores retained genomes, +ancestry and dead organisms, and continues identically. A full archive encodes to +2,543,648 of at most 3,219,496 bytes. + +While qualifying, WSL2 was measured stepping the wall clock back about 0.5 s every +~32 s. Four database-time tests now wait on the database's own expiry judgement +instead of assuming a local sleep equals database time; ADR-0060 records the +deployment requirement. Windows ran 57/57. Linux GCC Debug/Release, clang-tidy +and ASan/UBSan CTest each ran 57/57. The PostgreSQL suites passed 2/2/5 times. + +Next: M7.5 durable decisions, history and outbox rows, and commit-ID resolution. +Retained ancestry still needs M7.5 history and M7.7/M7.9 kill-recovery evidence. diff --git a/docs/adr/0034-lineage-archive.md b/docs/adr/0034-lineage-archive.md index d929287..19d24e0 100644 --- a/docs/adr/0034-lineage-archive.md +++ b/docs/adr/0034-lineage-archive.md @@ -2,7 +2,9 @@ ## Status -Adopted for M3.5. It does not settle NPC mutation, which the roadmap lists as a +Adopted for M3.5. ADR-0062 (M7.4b) adds the whole-archive codec, which +decodes by replaying these same admission, birth and death rules and never +overwrites an existing archive. It does not settle NPC mutation, which the roadmap lists as a dependency and OPEN_QUESTIONS leaves open until M12: the archive records an origin event and never an actor, so whether an NPC may diverge changes nothing here. Durable storage, retention tiers and cross-version migration remain M7 and M8. diff --git a/docs/adr/0060-fenced-region-ownership.md b/docs/adr/0060-fenced-region-ownership.md index 6ff526d..5a3444d 100644 --- a/docs/adr/0060-fenced-region-ownership.md +++ b/docs/adr/0060-fenced-region-ownership.md @@ -89,7 +89,19 @@ pre-created by an operator. This adds a second migration and a chain of prefix c ## Risks Grants made after the last migrate run are not audited until it runs again, so -operators must rerun it after role changes. A frozen VM/hypervisor can stop all +operators must rerun it after role changes. + +Database time is the server's wall clock (`clock_timestamp()`), so steps count. +M7.4b measured WSL2 stepping it back about 0.5 s every ~32 s: 3.66 s over 240 s, +roughly a 1.5% rate difference from CLOCK_MONOTONIC. +- Backward steps only lengthen leases in database time. Exclusivity holds, and a + holder's monotonic deadline still fences it early. +- Forward steps shorten leases. A successor can then acquire while the old holder + still believes it has authority. Commits are still rejected, but provisional + output could overlap. +Database hosts therefore need slewed, not stepped, time. A forward step larger +than the safety margin is outside the declared failure model. Tests wait on +database-time expiry instead of assuming a local sleep equals database time. A frozen VM/hypervisor can stop all guest clocks. Database commits stay fenced, but provisional output could then be late. Gateways still need bounded-validity registration (ADR-0058). The bare head kernel accepts any 32-byte digest until M7.4 diff --git a/docs/adr/0062-lineage-checkpoint-profile.md b/docs/adr/0062-lineage-checkpoint-profile.md new file mode 100644 index 0000000..a61f8ab --- /dev/null +++ b/docs/adr/0062-lineage-checkpoint-profile.md @@ -0,0 +1,96 @@ +# ADR-0062: Lineage archive codec and history-capable checkpoint profile + +## Status +Accepted for M7.4b, 2026-09-28. With ADR-0061 this completes M7.4. Durable +decision and history rows are still M7.5. The M7 retained-ancestry exit claim +still needs M7.7/M7.9 process-kill recovery evidence. + +## Context +REGION_DURABILITY.md requires a history-capable profile before retained ancestry +can be claimed. That profile must validate: +- organism-to-genome bindings and parent closure; +- lifecycle agreement between the simulation and the archive; +- identity high-waters and archived dead members. + +`LineageArchive` (ADR-0034) had no codec and deliberately has no assignment, so +an older copy cannot be written back. The M6 environmental fixture has two +organism accounts but no genome linkage, and it uses no randomness. + +## Decision +**Archive codec v1** (`LineageArchive::encode/decode`, schemas/lineage-archive-v1.md): +- Genome v1 records are ordered by (ancestry depth, id), so parents precede + children. Organisms follow in id order, as 33-byte records. +- A trailing FNV-1a64 detects corruption. The caps are the existing 4096 genomes + and 65536 organisms, at most 3,219,496 bytes, checked before allocation. +- Decoding constructs a new archive by replaying `admit`, `record_birth` and + `record_death`, so retention, parent presence, death finality and tick order are + re-established by the archive's own rules. A byte-identical re-encoding is + required, so equal content has exactly one encoding. +- Decoding never overwrites an existing archive. Refusing an older checkpoint is + the committed chain's job (ADR-0061): recovery only ever loads the head. + +**`LineageRegion`** (processes, headless) holds the environmental community, its +archive, the binding of the producer and consumer accounts to archived organisms, +and the next organism and genome identities. + +*Closure* requires: +- both bound organisms are recorded; +- each terminal flag equals the archive's death state; +- no birth or death is after the current tick; +- the archive's living organisms are exactly the bound, non-terminal ones, so + there is no unowned living state; +- both high-waters are above every recorded identity. + +`advance` records each terminal transition as an archive death at that tick. A +checkpoint therefore cannot split a death from the state that caused it. The +container is schemas/environmental-lineage-v1.md: header, binding, envelope, +archive and FNV. + +**Checkpoint profile 2** is `environmental-lineage-v1`, with history profile 1 +(archive 1 + genome schema 1). It uses the same nine version slots and zero RNG +streams, and its payload cap is 72 + 1 MiB + 3,219,496 bytes. The manifest +structure is unchanged, and profile and history profile must agree. + +Schema v4 replaces the SQL parser to admit profile 2 with its cap. Publication +additionally requires the predecessor's exact profile, RNG, history and durability +declarations. Grants are unchanged. Linux adds +`provision/publish/recover_lineage_checkpoint` on the same publication and +recovery path as profile 1; each path refuses the other profile. + +## Alternatives Considered +- **Extending environmental-v1 with optional sections**: forbidden, because empty + sections cannot stand in for missing history. +- **Admission-order encoding**: it is not recoverable from the archive and would + make equal content encode differently. +- **Deaths recorded by the host after the tick** could be split from the state + across a checkpoint. +- **A separate history object** would need its own atomic publication; one + container shares the manifest's digest and fence. + +## Benefits +- Retained genomes, ancestry and dead organisms now survive checkpoint recovery + and must agree with the simulation state that references them. +- Decode reuses the archive's invariants rather than duplicating them. +- Existing environmental-v1 regions are unaffected. + +## Costs +- The whole archive is rewritten with every batch. A full archive is about 2.5 MiB + in the measured test, up to about 3.2 MiB. +- Decode is O(genomes²) in the worst case because of sorted insertion. +- Archive memory is bounded by the fixed caps rather than charged to a budget. + +## Risks +- The environmental fixture has no reproduction, so births within a batch are not + exercised. A reproducing profile needs its own closure rules. +- Durable history rows (M7.5) must agree with the checkpoint's archive. Only the + checkpoint side exists yet. +- FNV is not authentication; SHA-256 at the manifest covers the payload. + +## Consequences +- M7.5 adds death decisions, history rows and outbox rows derived from these + transitions, in the same transaction. +- M7.7 recovers this profile after a process kill. +- M7.9 retains builds that can decode archive 1 and genome schema 1, and + qualifies retained ancestry end to end. +- A new genome schema or archive format needs a new history profile value and a + migration. diff --git a/docs/adr/README.md b/docs/adr/README.md index 8beba09..45ca502 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -72,3 +72,4 @@ design. No unadopted candidate is a link dependency of the proof. - [ADR-0059: PostgreSQL bootstrap](0059-postgresql-bootstrap.md) - accepted for M7.2 optional adapter, pinned server/client, migrations and Secret boundary. - [ADR-0060: Fenced region ownership](0060-fenced-region-ownership.md) - accepted for M7.3 database-time leases, guarded head advance, runtime grants and selected timings. - [ADR-0061: Checkpoint publication](0061-checkpoint-publication.md) - accepted for M7.4a manifest v1, immutable filesystem objects, OpenSSL SHA-256 and verified recovery. +- [ADR-0062: Lineage checkpoint profile](0062-lineage-checkpoint-profile.md) - accepted for M7.4b archive codec, LineageRegion closure and history-capable profile 2. diff --git a/docs/architecture/REGION_DURABILITY.md b/docs/architecture/REGION_DURABILITY.md index 7e2c01f..39e48a8 100644 --- a/docs/architecture/REGION_DURABILITY.md +++ b/docs/architecture/REGION_DURABILITY.md @@ -179,6 +179,10 @@ a local filesystem object store: - Unsupported builds are refused by both the database and recovery. - Recovery never creates genesis or picks an older head. -Two parts remain open. M7.4b adds the lineage archive codec and a history-capable -profile; until then the history rows of this audit stay open. No process-kill -test was run yet (7.7). +M7.4b (ADR-0062) closes the checkpoint side of the history audit. The lineage +archive codec and the `environmental-lineage-v1` profile validate: +- organism-to-genome bindings and parent closure (by archive replay); +- lifecycle agreement, with deaths recorded in the causing tick; +- the living set, identity high-waters and retained dead members. +Recovery restores all of these. Durable history rows (M7.5) and process-kill +recovery (M7.7/M7.9) remain before the retained-ancestry exit claim. diff --git a/docs/biology/GENOME_MODEL.md b/docs/biology/GENOME_MODEL.md index da1c2aa..e72a8f7 100644 --- a/docs/biology/GENOME_MODEL.md +++ b/docs/biology/GENOME_MODEL.md @@ -259,7 +259,8 @@ API. Absence is also reversible: a lineage with no carrier can gain one again. Capacity is bounded and admission fails loudly at the limit. Refusing to record is not deleting; a silent drop would be. Durable storage, retention tiers and -cross-version migration are M7 and M8 work, and nothing here is serialized. NPC +cross-version migration are M7 and M8 work. M7.4b (ADR-0062) serializes the +whole archive canonically for history-capable region checkpoints. NPC mutation stays undecided, because the archive records an origin event and never an actor. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index f6a4379..43c0dee 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -2158,3 +2158,58 @@ Only clean-to-v1 and exact-v1 restart are supported migration paths at this gate These are local socket and local disk only. - Not executed: Docker/Gitea CI, a remote object store or PVC, power-loss fsync semantics, read-back verification fault injection, process kill, and TSan. + +## M7.4b lineage checkpoint profile - 2026-09-28 + +- Windows MSVC Release CTest passed 57/57, including the new `lineage-region-contracts` + and extended lineage and region-checkpoint contracts. The repository clang-format + 19 check passed. +- WSL Ubuntu (GCC 14.2.0, clang 19.1.1, PostgreSQL 18.6, OpenSSL 3.0.13): + - The Linux `checks` build set (separate build dirs) passed CTest 57/57 in each + of GCC14 Debug, GCC14 Release, clang19 analysis and clang19 ASan/UBSan. + - The PostgreSQL integration suite passed 2× Release, 2× analysis and 5× + ASan/UBSan. +- Archive codec coverage: + - canonical encoding regardless of admission order, including a child with a + smaller ID than its parent; + - checksum, magic, version, count cap, length and organism-count framing; + - genome damage, unknown genome, invalid alive/died forms, death before birth, + reordered organisms (non-canonical) and a child encoded before its parent; + - a full 4096-genome/65536-organism round trip at 2,543,648 bytes. +- `LineageRegion` coverage: + - closure refusals: missing, identical or dead binding; low high-waters; stray + living organism; future birth; archive death while the community is active; + - the death recorded at tick 20 in the same advance; a refused tick changes + nothing; + - restore plus 36 continuation ticks identical to the original; + - container, binding, envelope and archive damage refused. +- Profile 2 coverage: + - the 4,268,144-byte cap is asserted in C++ and enforced by SQL (the maximum is + accepted, +1 refused); the profile-1 1 MiB+1 is refused; + - mismatched profile/history is EVL06; + - lineage publish (death included), recover and continue, and a second + publication; + - the environmental path refuses a lineage predecessor before upload; + - the database refuses a profile switch; + - a host of the other profile reports unsupported. +- Migrations: clean→v4; v1→v4; v3 with a committed genesis upgraded to v4 with + the commit retained; v2 heads still block. Tampered v2/v3/v4 checksums and an + unknown v5 are refused. +- Mutations (restored afterwards), each failing at the expected line: removing + the SQL profile/history continuity check (store test 392), the living-set + closure check (lineage region 58), or death recording (lineage region 78). +- Flake found and fixed. The ASan PostgreSQL suite failed intermittently: 1 of 3, + then 1 of 5 runs, at the queued-writer and expired-renewal assertions. + - Cause: a 4-minute probe measured WSL2 stepping CLOCK_REALTIME back 470-596 ms + every ~32 s (3.66 s total, roughly 1.5% versus CLOCK_MONOTONIC). + - PostgreSQL `clock_timestamp()` follows that wall clock, so a 1.2 s local sleep + could be under 1 s of database time. + - Tests now poll `clock_timestamp() >= expires_at`, holding the row lock where + relevant. Afterwards the suite ran 5/5 under ASan. + - The fencing logic itself was unchanged; the requirement is recorded in + ADR-0060. +- Local measurements, Release: + - manifest publish p50 0.72 ms and p99 1.24 ms; + - checkpoint publication 3.5-5.0 ms for 23-37 KiB payloads. +- Not executed: Docker/Gitea CI, remote storage, process kill, reproduction within + a batch (the fixture has none), durable history rows (M7.5) and TSan. diff --git a/docs/infrastructure/POSTGRESQL.md b/docs/infrastructure/POSTGRESQL.md index a699c99..f57cc4c 100644 --- a/docs/infrastructure/POSTGRESQL.md +++ b/docs/infrastructure/POSTGRESQL.md @@ -107,6 +107,11 @@ Operator steps: connection. It uploads and verifies the tick-0 payload, then calls the owner-only `provision_region_checkpoint(manifest)`. +Schema v4 (ADR-0062) additionally admits profile 2 (`environmental-lineage-v1`, +payload ≤ 4,268,144 bytes) and requires every commit to keep its predecessor's +profile, history, RNG and durability declarations. Provision history-capable +regions with `provision_lineage_region`; hosts use `publish/recover_lineage_checkpoint`. + There is no operator CLI for provisioning yet, and no real world has been provisioned. Migrating v2→v3 fails closed if any v2 head exists, because v2 heads have no payload. diff --git a/include/evo/biology/lineage.hpp b/include/evo/biology/lineage.hpp index 20c6107..bed0817 100644 --- a/include/evo/biology/lineage.hpp +++ b/include/evo/biology/lineage.hpp @@ -29,6 +29,21 @@ enum class HistoryError : std::uint8_t { }; inline constexpr std::size_t max_archived_genomes = 4'096; inline constexpr std::size_t max_archived_organisms = 65'536; +// Whole-archive codec v1 (schemas/lineage-archive-v1.md, ADR-0062). +enum class ArchiveCodecError : std::uint8_t { + format, // Magic, lengths or field forms. + version, // Unknown archive format or genome schema. + checksum, // Trailing FNV-1a64 mismatch (corruption, not forgery). + bounds, // Counts or bytes above the archive caps; checked before allocation. + genome, // A record fails Genome v1 validation. + history, // Admission/birth/death invariants refuse the record sequence. + noncanonical, // Valid content in a non-canonical order or form. +}; +inline constexpr std::uint32_t lineage_archive_format = 1; +inline constexpr std::size_t archived_organism_bytes = 33; +inline constexpr std::size_t max_archive_bytes = + 32 + (max_archived_genomes * (2 + max_genome_bytes)) + + (max_archived_organisms * archived_organism_bytes) + 8; // Retains genomes and the organisms that carried them. There is no removal // operation: not a guarded one, not a private one. Retention is a property of @@ -78,6 +93,22 @@ class LineageArchive { [[nodiscard]] std::size_t recorded() const noexcept { return organisms_.size(); } + // Ascending identities of organisms not yet recorded dead. + [[nodiscard]] std::vector living_organisms() const; + // Largest recorded identities, zero when none: allocator high-water checks. + [[nodiscard]] std::uint64_t highest_genome() const noexcept; + [[nodiscard]] std::uint64_t highest_organism() const noexcept; + + // Canonical bytes: genomes by (ancestry depth, id), so parents precede children, + // then organisms by id. Equal archives encode identically whatever order their + // records were admitted in. + [[nodiscard]] std::vector encode() const; + // Constructs a new archive by replaying admit/record_birth/record_death, so + // every retention invariant is re-established, then requires a byte-identical + // re-encoding. It never overwrites an existing archive; refusing stale older + // copies is the durable commit chain's job (ADR-0061/0062). + [[nodiscard]] static std::expected + decode(std::span bytes); private: struct ArchivedOrganism { diff --git a/include/evo/persistence/checkpoint_store.hpp b/include/evo/persistence/checkpoint_store.hpp index d552592..42e9359 100644 --- a/include/evo/persistence/checkpoint_store.hpp +++ b/include/evo/persistence/checkpoint_store.hpp @@ -44,6 +44,11 @@ struct RecoveredCheckpoint { CheckpointManifest manifest; std::unique_ptr state; }; +struct RecoveredLineageCheckpoint { + CommittedCheckpoint committed; + CheckpointManifest manifest; + std::unique_ptr state; +}; // Owner/operator connection only: uploads a tick-0 state and records genesis. [[nodiscard]] std::expected provision_environmental_region( @@ -65,4 +70,21 @@ publish_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& obje recover_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, std::span retained_builds, std::size_t restore_budget, std::chrono::milliseconds timeout); + +// environmental-lineage-v1 (ADR-0062): the same order and guarantees, with the +// LineageRegion container as payload. A region's profile never changes; each call +// refuses a predecessor/committed head of the other profile. +[[nodiscard]] std::expected +provision_lineage_region(Postgres& owner, const FilesystemObjectStore& objects, RegionKey key, + std::uint64_t frame, const processes::LineageRegion& opening, Uuid commit, + const Sha256Digest& build, std::size_t restore_budget, + std::chrono::milliseconds timeout); +[[nodiscard]] std::expected publish_lineage_checkpoint( + Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease, + const CommittedCheckpoint& predecessor, const processes::LineageRegion& state, Uuid commit, + const Sha256Digest& build, std::size_t restore_budget, std::chrono::milliseconds timeout); +[[nodiscard]] std::expected +recover_lineage_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, + std::span retained_builds, + std::size_t restore_budget, std::chrono::milliseconds timeout); } // namespace evo::persistence diff --git a/include/evo/persistence/region_checkpoint.hpp b/include/evo/persistence/region_checkpoint.hpp index c02c8c7..c0f9006 100644 --- a/include/evo/persistence/region_checkpoint.hpp +++ b/include/evo/persistence/region_checkpoint.hpp @@ -1,6 +1,7 @@ #pragma once #include "evo/persistence/region_lease.hpp" #include "evo/processes/environmental_community.hpp" +#include "evo/processes/lineage_region.hpp" #include #include #include @@ -8,12 +9,19 @@ #include #include -// Region checkpoint manifest v1 and environmental-v1 payload validation (ADR-0061, -// schemas/region-commit.md). Portable and storage-free: digests are computed and -// objects are stored by the Linux adapters; this only encodes and validates. +// Region checkpoint manifest v1 and payload validation for environmental-v1 +// (ADR-0061) and environmental-lineage-v1 (ADR-0062), schemas/region-commit.md. Portable and +// storage-free: digests are computed and objects are stored by the Linux adapters; this only +// encodes and validates. namespace evo::persistence { // Encoded as u32 on the wire; values above 255 decode as unknown (0), never truncated. -enum class CheckpointProfile : std::uint8_t { unknown = 0, environmental_v1 = 1 }; +enum class CheckpointProfile : std::uint8_t { + unknown = 0, + environmental_v1 = 1, // Environmental envelope only; history_profile 0. + environmental_lineage_v1 = 2, // LineageRegion container; history_profile 1. +}; +// history_profile 1: lineage archive format 1 with genome schema 1. +inline constexpr std::uint32_t lineage_history_profile = 1; // Declared durability, verified at deployment. Development: one volume, no host-loss claim. enum class DurabilityPolicy : std::uint8_t { unknown = 0, single_volume_development = 1 }; @@ -40,8 +48,8 @@ struct CheckpointManifest { std::uint64_t tick_microseconds{}; VersionBundle versions{}; Sha256Digest build{}; // Retained immutable build artifact identity. - std::uint32_t rng_streams{}; // environmental-v1 rules use no randomness: 0. - std::uint32_t history_profile{}; // No lineage/genome state in environmental-v1: 0. + std::uint32_t rng_streams{}; // These rules use no randomness: 0. + std::uint32_t history_profile{}; // 0 environmental-v1; 1 environmental-lineage-v1. DurabilityPolicy durability{DurabilityPolicy::single_volume_development}; std::uint64_t payload_bytes{}; Sha256Digest payload_sha256{}; @@ -50,6 +58,13 @@ struct CheckpointManifest { inline constexpr std::size_t checkpoint_manifest_bytes = 280; inline constexpr std::size_t max_environmental_payload_bytes = std::size_t{1024} * 1024; +inline constexpr std::size_t max_lineage_payload_bytes = processes::max_lineage_region_bytes; +// Payload cap by profile; zero for an unknown profile. +[[nodiscard]] constexpr std::size_t max_payload_bytes(CheckpointProfile profile) noexcept { + return profile == CheckpointProfile::environmental_v1 ? max_environmental_payload_bytes + : profile == CheckpointProfile::environmental_lineage_v1 ? max_lineage_payload_bytes + : 0; +} inline constexpr std::uint64_t max_checkpoint_ticks = 64; enum class CheckpointError : std::uint8_t { @@ -65,15 +80,17 @@ enum class CheckpointError : std::uint8_t { encode_manifest(const CheckpointManifest& manifest); [[nodiscard]] std::expected decode_manifest(std::span bytes); -// Profile, exact versions, zero RNG/history declarations and durability policy. +// Profile with its history profile, exact versions, zero RNG and durability policy. [[nodiscard]] bool supported(const CheckpointManifest& manifest) noexcept; // Genesis for operator provisioning only; recovery never creates one. [[nodiscard]] std::expected genesis_manifest(RegionKey key, std::uint64_t frame, Uuid commit, std::uint64_t tick_microseconds, const Sha256Digest& build, std::uint64_t payload_bytes, - const Sha256Digest& payload_sha256); + const Sha256Digest& payload_sha256, + CheckpointProfile profile = CheckpointProfile::environmental_v1); // Next link from the committed predecessor and its manifest digest under a lease. +// Profile, versions and quantum are inherited: a region never changes them. [[nodiscard]] std::expected successor_manifest(const CheckpointManifest& predecessor, const Sha256Digest& predecessor_digest, Uuid commit, const LeaseGrant& lease, std::uint64_t end_tick, @@ -86,4 +103,10 @@ successor_manifest(const CheckpointManifest& predecessor, const Sha256Digest& pr [[nodiscard]] std::expected, CheckpointError> restore_environmental_payload(std::span payload, const CheckpointManifest& manifest, std::size_t budget); +// environmental-lineage-v1: restores the container (envelope replay audit, archive +// replay through its invariants, closure) with the same identity/tick/quantum, +// nested envelope format 2 and canonical checks. +[[nodiscard]] std::expected, CheckpointError> +restore_lineage_payload(std::span payload, const CheckpointManifest& manifest, + std::size_t budget); } // namespace evo::persistence diff --git a/include/evo/processes/lineage_region.hpp b/include/evo/processes/lineage_region.hpp new file mode 100644 index 0000000..40c9e39 --- /dev/null +++ b/include/evo/processes/lineage_region.hpp @@ -0,0 +1,79 @@ +#pragma once +#include "evo/biology/lineage.hpp" +#include "evo/processes/environmental_community.hpp" +#include +#include +#include +#include +#include + +// History-capable environmental region (ADR-0062, schemas/environmental-lineage-v1.md): +// the environmental community, the lineage archive that retains its organisms' +// genomes and deaths, the binding of its two accounts to archived organisms, and +// identity high-waters. One owner; every advance keeps archive deaths in the same +// tick as the community's terminal transitions, so a checkpoint cannot split them. +namespace evo::processes { +struct LineageBinding { + std::uint64_t producer{}, consumer{}; // Archived organism identities. + std::uint64_t next_organism{}, next_genome{}; // Never reissue below these. + auto operator<=>(const LineageBinding&) const = default; +}; +enum class LineageRegionError : std::uint8_t { + community, // The community refused the tick or its envelope; nothing changed. + closure, // Binding, lifecycle, living set or high-waters disagree. + archive, // Archive bytes refused (see ArchiveCodecError via restore). + format, // Container framing, lengths or reserved fields. + version, // Unknown container format. + checksum, // Container FNV-1a64 mismatch. + poisoned, // A prior advance failed after committing the community tick. +}; +inline constexpr std::uint32_t lineage_region_format = 1; +inline constexpr std::size_t max_lineage_region_bytes = + 72 + EnvironmentalCommunity::max_snapshot_bytes + biology::max_archive_bytes; + +class LineageRegion { + public: + // Validates closure: both bound organisms recorded; terminal flags equal the + // archive's death records; the archive's living organisms are exactly the + // bound, non-terminal ones; births/deaths not after the current tick; and both + // high-waters above every recorded identity. + [[nodiscard]] static std::expected, LineageRegionError> + create(std::unique_ptr community, + const biology::LineageArchive& archive, LineageBinding binding); + LineageRegion(const LineageRegion&) = delete; + LineageRegion& operator=(const LineageRegion&) = delete; + LineageRegion(LineageRegion&&) = delete; + LineageRegion& operator=(LineageRegion&&) = delete; + ~LineageRegion() = default; + + // Advances the community one tick and records each terminal transition as an + // archive death at that tick. + [[nodiscard]] std::expected + advance(std::uint64_t tick, const EnvironmentalInput& input); + [[nodiscard]] std::expected snapshot() const; + // Restores the envelope (full replay audit) and the archive, validates closure + // and requires a byte-identical canonical re-encoding. The budget is charged to + // the community; archive memory is bounded by the archive's fixed caps. + [[nodiscard]] static std::expected, LineageRegionError> + restore(std::span bytes, world::WorldIdentity identity, std::size_t budget); + + [[nodiscard]] const EnvironmentalCommunity& community() const noexcept { + return *community_; + } + [[nodiscard]] const biology::LineageArchive& archive() const noexcept { + return archive_; + } + [[nodiscard]] const LineageBinding& binding() const noexcept { + return binding_; + } + + private: + LineageRegion(std::unique_ptr community, + const biology::LineageArchive& archive, LineageBinding binding); + [[nodiscard]] bool closed() const; + std::unique_ptr community_; + biology::LineageArchive archive_; + LineageBinding binding_; + bool poisoned_{}; +}; +} // namespace evo::processes diff --git a/migrations/0004_lineage_profile.sql b/migrations/0004_lineage_profile.sql new file mode 100644 index 0000000..7dd89d9 --- /dev/null +++ b/migrations/0004_lineage_profile.sql @@ -0,0 +1,131 @@ +-- M7.4b history-capable profile (ADR-0062). Admits environmental-lineage-v1 +-- (profile 2, history profile 1: lineage archive 1 + genome schema 1) beside +-- environmental-v1, with per-profile payload caps, and makes publication require the +-- predecessor's exact profile, RNG, history and durability declarations. +-- Grants are unchanged: CREATE OR REPLACE keeps each function's owner and ACL. + +-- Mirrors decode_manifest + supported() in src/persistence/region_checkpoint.cpp. +-- 22023: malformed; EVL06: parsable but unsupported by this schema. +CREATE OR REPLACE FUNCTION evolution.parse_checkpoint_manifest(m bytea) +RETURNS evolution.checkpoint_manifest LANGUAGE plpgsql IMMUTABLE +SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + r evolution.checkpoint_manifest; + zero16 constant bytea := '\x00000000000000000000000000000000'; + zero32 constant bytea := decode(repeat('00', 32), 'hex'); + profile numeric; + history numeric; + cap numeric; +BEGIN + IF m IS NULL OR octet_length(m) <> 280 OR substring(m FROM 1 FOR 8) <> '\x4145564f52434d31' THEN + RAISE EXCEPTION 'malformed checkpoint manifest' USING ERRCODE = '22023'; + END IF; + profile := evolution.manifest_uint(m, 12, 4); + history := evolution.manifest_uint(m, 232, 4); + IF evolution.manifest_uint(m, 8, 4) <> 1 + OR NOT ((profile = 1 AND history = 0) OR (profile = 2 AND history = 1)) + OR substring(m FROM 161 FOR 36) <> '\x020000000200000001000000010000000100000001000000010000000100000001000000' + OR evolution.manifest_uint(m, 228, 4) <> 0 OR evolution.manifest_uint(m, 236, 4) <> 1 THEN + RAISE EXCEPTION 'unsupported checkpoint manifest' USING ERRCODE = 'EVL06'; + END IF; + -- Payload caps: 1 MiB envelope; 72 + 1 MiB + lineage archive cap (3219496) container. + cap := CASE profile WHEN 1 THEN 1048576 ELSE 4268144 END; + r.world_id := evolution.manifest_uint(m, 16, 8); + r.region_id := evolution.manifest_uint(m, 24, 8); + r.frame_id := evolution.manifest_uint(m, 32, 8); + r.commit_id := encode(substring(m FROM 41 FOR 16), 'hex')::uuid; + r.commit_sequence := evolution.manifest_uint(m, 56, 8); + r.predecessor_commit := encode(substring(m FROM 65 FOR 16), 'hex')::uuid; + r.predecessor_digest := substring(m FROM 81 FOR 32); + r.begin_tick := evolution.manifest_uint(m, 112, 8); + r.end_tick := evolution.manifest_uint(m, 120, 8); + r.holder := encode(substring(m FROM 129 FOR 16), 'hex')::uuid; + r.epoch := evolution.manifest_uint(m, 144, 8); + r.tick_microseconds := evolution.manifest_uint(m, 152, 8); + r.versions := substring(m FROM 161 FOR 36); + r.build_identity := substring(m FROM 197 FOR 32); + r.payload_bytes := evolution.manifest_uint(m, 240, 8); + r.payload_sha256 := substring(m FROM 249 FOR 32); + IF r.world_id = 0 OR r.region_id = 0 OR r.frame_id = 0 + OR substring(m FROM 41 FOR 16) = zero16 OR r.tick_microseconds = 0 + OR r.build_identity = zero32 OR r.payload_bytes < 1 OR r.payload_bytes > cap + OR r.payload_sha256 = zero32 THEN + RAISE EXCEPTION 'malformed checkpoint manifest' USING ERRCODE = '22023'; + END IF; + IF r.commit_sequence = 0 THEN + IF substring(m FROM 65 FOR 16) <> zero16 OR r.predecessor_digest <> zero32 + OR r.begin_tick <> 0 OR r.end_tick <> 0 OR substring(m FROM 129 FOR 16) <> zero16 + OR r.epoch <> 0 THEN + RAISE EXCEPTION 'malformed checkpoint manifest' USING ERRCODE = '22023'; + END IF; + r.predecessor_commit := NULL; + r.holder := NULL; + ELSIF substring(m FROM 65 FOR 16) = zero16 OR r.predecessor_commit = r.commit_id + OR r.predecessor_digest = zero32 OR substring(m FROM 129 FOR 16) = zero16 + OR r.epoch = 0 OR r.end_tick <= r.begin_tick OR r.end_tick - r.begin_tick > 64 THEN + RAISE EXCEPTION 'malformed checkpoint manifest' USING ERRCODE = '22023'; + END IF; + RETURN r; +END $$; + +CREATE OR REPLACE FUNCTION evolution.publish_region_checkpoint(p_manifest bytea) +RETURNS TABLE (region_frame evolution.u64, head_sequence evolution.u64, head_commit uuid, + head_digest text, head_tick evolution.u64) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + c evolution.checkpoint_manifest := evolution.parse_checkpoint_manifest(p_manifest); + h evolution.region_heads%ROWTYPE; + p evolution.region_commits%ROWTYPE; + frame evolution.u64; + d bytea := sha256(p_manifest); +BEGIN + IF c.commit_sequence = 0 THEN + RAISE EXCEPTION 'genesis is provisioned, not published' USING ERRCODE = '22023'; + END IF; + PERFORM evolution.check_region_lease(c.world_id, c.region_id, c.holder, c.epoch); + SELECT * INTO h FROM evolution.region_heads r + WHERE r.world_id = c.world_id AND r.region_id = c.region_id FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'region not provisioned' USING ERRCODE = 'EVL04'; + END IF; + IF h.commit_sequence = 18446744073709551615 THEN + RAISE EXCEPTION 'commit sequences exhausted' USING ERRCODE = 'EVL05'; + END IF; + IF c.commit_sequence <> h.commit_sequence + 1 OR c.predecessor_commit <> h.commit_id + OR c.predecessor_digest <> h.digest OR c.begin_tick <> h.end_tick THEN + RAISE EXCEPTION 'region head moved' USING ERRCODE = 'EVL03'; + END IF; + SELECT l.frame_id INTO frame FROM evolution.region_leases l + WHERE l.world_id = c.world_id AND l.region_id = c.region_id; + IF c.frame_id <> frame THEN + RAISE EXCEPTION 'foreign frame' USING ERRCODE = '22023'; + END IF; + SELECT * INTO p FROM evolution.region_commits r + WHERE r.world_id = c.world_id AND r.region_id = c.region_id + AND r.commit_sequence = h.commit_sequence; + -- Profile, RNG/history/durability declarations, rules, formats and quantum are + -- fixed for a region; changing any of them is a migration, never a commit. + IF c.versions <> p.versions OR c.tick_microseconds <> p.tick_microseconds + OR substring(p_manifest FROM 13 FOR 4) <> substring(p.manifest FROM 13 FOR 4) + OR substring(p_manifest FROM 229 FOR 12) <> substring(p.manifest FROM 229 FOR 12) + OR NOT EXISTS (SELECT 1 FROM evolution.retained_builds b + WHERE b.build_identity = c.build_identity) THEN + RAISE EXCEPTION 'unsupported checkpoint' USING ERRCODE = 'EVL06'; + END IF; + IF EXISTS (SELECT 1 FROM evolution.region_commits r WHERE r.commit_id = c.commit_id) THEN + RAISE EXCEPTION 'commit identity reused' USING ERRCODE = 'EVL07'; + END IF; + INSERT INTO evolution.region_commits(world_id, region_id, commit_sequence, commit_id, + commit_digest, manifest, begin_tick, end_tick, published_epoch, tick_microseconds, + versions, build_identity, payload_bytes, payload_sha256) + VALUES (c.world_id, c.region_id, c.commit_sequence, c.commit_id, d, p_manifest, + c.begin_tick, c.end_tick, c.epoch, c.tick_microseconds, c.versions, + c.build_identity, c.payload_bytes, c.payload_sha256); + UPDATE evolution.region_heads r + SET commit_sequence = c.commit_sequence, commit_id = c.commit_id, digest = d, + end_tick = c.end_tick, published_epoch = c.epoch + WHERE r.world_id = c.world_id AND r.region_id = c.region_id; + RETURN QUERY SELECT frame, c.commit_sequence, c.commit_id, encode(d, 'hex'), c.end_tick; +END $$; + +INSERT INTO evolution.schema_migrations(version, sha256) VALUES (4, :'migration_sha_4'); diff --git a/migrations/apply.sql b/migrations/apply.sql index b5bb35d..c966f62 100644 --- a/migrations/apply.sql +++ b/migrations/apply.sql @@ -19,13 +19,19 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') SELECT :'history' = '1:' || :'migration_sha' AS at_v1, :'history' = '1:' || :'migration_sha' || ',2:' || :'migration_sha_2' AS at_v2, :'history' = '1:' || :'migration_sha' || ',2:' || :'migration_sha_2' - || ',3:' || :'migration_sha_3' AS at_v3 \gset - \if :at_v3 + || ',3:' || :'migration_sha_3' AS at_v3, + :'history' = '1:' || :'migration_sha' || ',2:' || :'migration_sha_2' + || ',3:' || :'migration_sha_3' || ',4:' || :'migration_sha_4' AS at_v4 \gset + \if :at_v4 + \elif :at_v3 + \ir 0004_lineage_profile.sql \elif :at_v2 \ir 0003_region_checkpoints.sql + \ir 0004_lineage_profile.sql \elif :at_v1 \ir 0002_region_fencing.sql \ir 0003_region_checkpoints.sql + \ir 0004_lineage_profile.sql \else \echo 'Unknown or changed migration history' DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$; @@ -34,6 +40,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0001_region_bootstrap.sql \ir 0002_region_fencing.sql \ir 0003_region_checkpoints.sql + \ir 0004_lineage_profile.sql \endif -- Effective-privilege audit on every run, including no-ops (ADR-0060). Attributes -- alone are insufficient: predefined roles such as pg_write_all_data, PUBLIC diff --git a/schemas/environmental-lineage-v1.md b/schemas/environmental-lineage-v1.md new file mode 100644 index 0000000..d5a5d3f --- /dev/null +++ b/schemas/environmental-lineage-v1.md @@ -0,0 +1,38 @@ +# Environmental lineage region container v1 (M7.4b) + +Authority: ADR-0062. This is the payload of checkpoint profile 2 (history profile 1). +The encoder and decoder are `LineageRegion::snapshot/restore` in +`src/processes/lineage_region.cpp`. All integers are little-endian. + +| Offset | Size | Field | +|---|---|---| +| 0 | 8 | magic `AEVOELR1` | +| 8 | 4 | container format = 1 | +| 12 | 4 | reserved, must be 0 | +| 16 | 8 | producer account's archived organism id | +| 24 | 8 | consumer account's archived organism id | +| 32 | 8 | next organism id (> every recorded organism) | +| 40 | 8 | next genome id (> every archived genome) | +| 48 | 8 | envelope length E (1..1 MiB) | +| 56 | E | environmental community envelope, format 2 | +| 56+E | 8 | archive length A | +| 64+E | A | lineage archive v1 | +| end−8 | 8 | FNV-1a64 of all previous bytes | + +The maximum is 72 + 1 MiB + 3,219,496 = 4,268,144 bytes. PostgreSQL enforces this +same value as the profile-2 payload cap. + +Restore checks, in order: +1. The container framing. +2. The full envelope restore, with its replay audit. +3. The archive replay (lineage-archive-v1). +4. Closure (ADR-0062): + - both organisms are recorded; + - terminal flags equal the archive's death state; + - no birth or death is after the current tick; + - the living organisms are exactly the bound, non-terminal ones; + - the high-waters are above every recorded identity. +5. A byte-identical re-encoding. + +Account order is the envelope's producer, then consumer. The manifest's region +maps these region-local accounts to organisms; no global ownership transfer exists. diff --git a/schemas/lineage-archive-v1.md b/schemas/lineage-archive-v1.md new file mode 100644 index 0000000..9117765 --- /dev/null +++ b/schemas/lineage-archive-v1.md @@ -0,0 +1,29 @@ +# Lineage archive v1 (M7.4b) + +Authority: ADR-0062 and ADR-0034. The encoder and decoder are +`LineageArchive::encode/decode` in `src/biology/lineage.cpp`. All integers are +little-endian, with no native struct serialization. + +| Offset | Size | Field | +|---|---|---| +| 0 | 8 | magic `AEVOLAR1` | +| 8 | 4 | archive format = 1 | +| 12 | 4 | genome schema = 1 | +| 16 | 4 | genome count G (≤ 4096) | +| 20 | 4 | organism count O (≤ 65536) | +| 24 | 8 | total bytes, including the checksum | +| 32 | … | G × (u16 length 1..256, Genome v1 record) ordered by (ancestry depth, genome id) | +| … | 33·O | organism records ordered by id: id u64, genome u64, born u64, died u64, alive u8 | +| end−8 | 8 | FNV-1a64 of all previous bytes | + +The maximum size is 3,219,496 bytes, and the counts are checked before any record +is decoded. A living organism has `died` = 0 and `alive` = 1. A dead organism has +`alive` = 0 and `died` ≥ `born`. + +Decoding admits each genome and replays births and deaths through the archive's +own operations, so a child before its parent, a duplicate, an unknown genome or a +second death is refused. The result must re-encode to identical bytes; otherwise +the input is non-canonical. + +An empty archive is 40 bytes. Unknown formats or genome schemas are unsupported. +There is no migration API; a new genome schema needs a new history profile. diff --git a/schemas/region-commit.md b/schemas/region-commit.md index be85575..76bf6b7 100644 --- a/schemas/region-commit.md +++ b/schemas/region-commit.md @@ -12,7 +12,7 @@ record, and each commit row stores it. |---|---|---| | 0 | 8 | magic `AEVORCM1` | | 8 | 4 | manifest_version = 1 (other values are unsupported) | -| 12 | 4 | profile: 1 = environmental-v1 | +| 12 | 4 | profile: 1 = environmental-v1, 2 = environmental-lineage-v1 (ADR-0062) | | 16 | 24 | world, region, frame (nonzero u64 each) | | 40 | 16 | commit_id (nonzero; globally unique in the database) | | 56 | 8 | commit_sequence | @@ -25,15 +25,18 @@ record, and each commit row stores it. | 160 | 36 | nine u32 versions: rules 2, envelope 2, resources/fields/terrain/coordinate/generator/material/interval-adapter 1 | | 196 | 32 | retained build identity (nonzero; `retained_builds` row required) | | 228 | 4 | rng_streams = 0 (these rules use no randomness) | -| 232 | 4 | history_profile = 0 (no genome/lineage state) | +| 232 | 4 | history_profile: 0 with profile 1; 1 (lineage archive 1 + genome schema 1) with profile 2 | | 236 | 4 | durability_policy = 1 (single-volume development) | -| 240 | 8 | payload_bytes (1..1 MiB) | +| 240 | 8 | payload_bytes: profile 1 is 1..1 MiB; profile 2 is 1..4,268,144 | | 248 | 32 | payload SHA-256 (nonzero) | A structurally invalid record is malformed (SQL 22023). A record that parses but has an unsupported version, profile, RNG, history or durability declaration is unsupported (EVL06). Values above 255 are never truncated onto a supported enum. -The payload is the exact environmental envelope format 2; its object key is the +Profile 2 payloads are schemas/environmental-lineage-v1.md containers. A region +never changes profile, history, RNG or durability declarations; schema v4 +enforces this against the predecessor. For profile 1, the payload is the exact +environmental envelope format 2; its object key is the hex payload SHA-256 in the region's namespace. Decisions, trace and outbox are not encoded here: M7.5 adds them under a new manifest version. diff --git a/src/biology/lineage.cpp b/src/biology/lineage.cpp index 0534229..a665095 100644 --- a/src/biology/lineage.cpp +++ b/src/biology/lineage.cpp @@ -161,6 +161,153 @@ LineageArchive::living_carriers(std::uint64_t genome) const { return carrying; } +std::vector LineageArchive::living_organisms() const { + std::vector living; + for (const auto& entry : organisms_) + if (entry.record.alive) + living.push_back(entry.id); + return living; +} +std::uint64_t LineageArchive::highest_genome() const noexcept { + return genomes_.empty() ? 0 : genomes_.back().id(); +} +std::uint64_t LineageArchive::highest_organism() const noexcept { + return organisms_.empty() ? 0 : organisms_.back().id; +} + +namespace { +constexpr std::uint64_t archive_magic = 0x3152414C4F564541ULL; // "AEVOLAR1" +constexpr std::size_t archive_header_bytes = 32; +std::uint64_t fnv(std::span bytes) noexcept { + std::uint64_t value = 14695981039346656037ULL; + for (const auto byte : bytes) + value = (value ^ byte) * 1099511628211ULL; + return value; +} +void put(std::vector& out, std::uint64_t value, unsigned width) { + for (unsigned i = 0; i < width; ++i) { + out.push_back(static_cast(value & 0xFFU)); + value >>= 8U; + } +} +std::uint64_t get(std::span in, std::size_t at, unsigned width) { + std::uint64_t value{}; + for (unsigned i = 0; i < width; ++i) + value |= static_cast(in[at + i]) << (8U * i); + return value; +} +} // namespace + +std::vector LineageArchive::encode() const { + // Depth from the founder, memoized; the parent graph is acyclic by admission. + std::vector depth(genomes_.size(), 0); + std::vector known(genomes_.size(), false); + const auto index = [&](std::uint64_t id) { + return static_cast(locate(genomes_, id, by_genome) - genomes_.begin()); + }; + for (std::size_t i = 0; i < genomes_.size(); ++i) { + std::vector chain; + auto at = i; + while (!known[at]) { + chain.push_back(at); + if (genomes_[at].parents().empty()) + break; + at = index(genomes_[at].parents()[0]); + } + std::uint32_t d = known[at] ? depth[at] + 1 : 0; + for (auto it = chain.rbegin(); it != chain.rend(); ++it) { + if (known[*it]) + continue; + depth[*it] = genomes_[*it].parents().empty() ? 0 : d; + known[*it] = true; + d = depth[*it] + 1; + } + } + std::vector order(genomes_.size()); + for (std::size_t i = 0; i < order.size(); ++i) + order[i] = i; + // Genomes are id-sorted already, so a stable sort by depth yields (depth, id). + std::ranges::stable_sort(order, {}, [&](std::size_t i) { return depth[i]; }); + + std::vector out; + out.reserve(archive_header_bytes + (genomes_.size() * (2 + max_genome_bytes)) + + (organisms_.size() * archived_organism_bytes) + 8); + put(out, archive_magic, 8); + put(out, lineage_archive_format, 4); + put(out, genome_schema_version, 4); + put(out, genomes_.size(), 4); + put(out, organisms_.size(), 4); + put(out, 0, 8); // Total length, patched below. + for (const auto i : order) { + const auto record = genomes_[i].encode(); + put(out, record.size(), 2); + out.insert(out.end(), record.begin(), record.end()); + } + for (const auto& entry : organisms_) { + put(out, entry.id, 8); + put(out, entry.record.genome, 8); + put(out, entry.record.born.value, 8); + put(out, entry.record.alive ? 0 : entry.record.died.value, 8); + put(out, entry.record.alive ? 1 : 0, 1); + } + const auto total = out.size() + 8; + for (unsigned b = 0; b < 8; ++b) + out[24 + b] = static_cast((total >> (8U * b)) & 0xFFU); + put(out, fnv(out), 8); + return out; +} + +std::expected +LineageArchive::decode(std::span bytes) { + if (bytes.size() < archive_header_bytes + 8 || get(bytes, 0, 8) != archive_magic) + return std::unexpected{ArchiveCodecError::format}; + if (get(bytes, 8, 4) != lineage_archive_format || get(bytes, 12, 4) != genome_schema_version) + return std::unexpected{ArchiveCodecError::version}; + const auto genomes = get(bytes, 16, 4); + const auto organisms = get(bytes, 20, 4); + if (bytes.size() > max_archive_bytes || genomes > max_archived_genomes || + organisms > max_archived_organisms) + return std::unexpected{ArchiveCodecError::bounds}; + if (get(bytes, 24, 8) != bytes.size()) + return std::unexpected{ArchiveCodecError::format}; + const auto body = bytes.first(bytes.size() - 8); + if (get(bytes, bytes.size() - 8, 8) != fnv(body)) + return std::unexpected{ArchiveCodecError::checksum}; + + LineageArchive archive; + std::size_t at = archive_header_bytes; + for (std::uint64_t g = 0; g < genomes; ++g) { + if (at + 2 > body.size()) + return std::unexpected{ArchiveCodecError::format}; + const auto length = static_cast(get(body, at, 2)); + at += 2; + if (length == 0 || length > max_genome_bytes || at + length > body.size()) + return std::unexpected{ArchiveCodecError::format}; + const auto genome = Genome::decode(body.subspan(at, length)); + if (!genome) + return std::unexpected{ArchiveCodecError::genome}; + if (!archive.admit(*genome)) + return std::unexpected{ArchiveCodecError::history}; + at += length; + } + if (body.size() - at != organisms * archived_organism_bytes) + return std::unexpected{ArchiveCodecError::format}; + for (std::uint64_t o = 0; o < organisms; ++o, at += archived_organism_bytes) { + const auto id = get(body, at, 8); + const Tick born{get(body, at + 16, 8)}; + const Tick died{get(body, at + 24, 8)}; + const auto alive = get(body, at + 32, 1); + if (alive > 1 || (alive == 1 && died.value != 0)) + return std::unexpected{ArchiveCodecError::format}; + if (!archive.record_birth(id, get(body, at + 8, 8), born) || + (alive == 0 && !archive.record_death(id, died))) + return std::unexpected{ArchiveCodecError::history}; + } + if (!std::ranges::equal(archive.encode(), bytes)) + return std::unexpected{ArchiveCodecError::noncanonical}; + return archive; +} + std::expected LineageArchive::status(std::uint64_t genome) const { const auto carrying = living_carriers(genome); if (!carrying) { diff --git a/src/persistence/checkpoint_store.cpp b/src/persistence/checkpoint_store.cpp index adee2ad..983bbbf 100644 --- a/src/persistence/checkpoint_store.cpp +++ b/src/persistence/checkpoint_store.cpp @@ -7,16 +7,56 @@ namespace evo::persistence { namespace { using std::chrono::milliseconds; + +// Per-profile payload operations; the publication and recovery paths are shared. +struct EnvironmentalCodec { + using State = processes::EnvironmentalCommunity; + using Recovered = RecoveredCheckpoint; + static constexpr auto profile = CheckpointProfile::environmental_v1; + static auto snapshot(const State& s) { + return s.snapshot(); + } + static std::uint64_t tick(const State& s) { + return s.progress().tick; + } + static std::uint64_t quantum(const State& s) { + return s.tick_microseconds(); + } + static auto restore(std::span payload, const CheckpointManifest& m, + std::size_t budget) { + return restore_environmental_payload(payload, m, budget); + } +}; +struct LineageCodec { + using State = processes::LineageRegion; + using Recovered = RecoveredLineageCheckpoint; + static constexpr auto profile = CheckpointProfile::environmental_lineage_v1; + static auto snapshot(const State& s) { + return s.snapshot(); + } + static std::uint64_t tick(const State& s) { + return s.community().progress().tick; + } + static std::uint64_t quantum(const State& s) { + return s.community().tick_microseconds(); + } + static auto restore(std::span payload, const CheckpointManifest& m, + std::size_t budget) { + return restore_lineage_payload(payload, m, budget); + } +}; + PublicationFailure fail(PublicationError kind) { return {kind, {}, {}}; } -// Encodes the candidate, validates the exact payload bytes as recovery would, -// uploads and reads back. Returns the manifest bytes ready for the database. +// Validates the exact payload bytes as recovery would, uploads and reads back. +// Returns the manifest bytes ready for the database. +template std::expected, PublicationFailure> stage(const FilesystemObjectStore& objects, const std::vector& payload, const CheckpointManifest& manifest, std::size_t budget) { auto bytes = encode_manifest(manifest); - if (!bytes || !restore_environmental_payload(payload, manifest, budget)) + if (!bytes || manifest.profile != Codec::profile || !Codec::restore(payload, manifest, budget)) return std::unexpected{fail(PublicationError::candidate)}; auto stored = objects.put(payload); if (!stored) @@ -45,23 +85,23 @@ confirm(const std::expected& published, return std::unexpected{fail(PublicationError::receipt)}; return CommittedCheckpoint{*published, std::move(manifest_bytes)}; } -} // namespace -std::expected provision_environmental_region( - Postgres& owner, const FilesystemObjectStore& objects, RegionKey key, std::uint64_t frame, - const processes::EnvironmentalCommunity& opening, Uuid commit, const Sha256Digest& build, - std::size_t restore_budget, milliseconds timeout) { - auto payload = opening.snapshot(); - if (!payload || opening.progress().tick != 0) +template +std::expected +provision(Postgres& owner, const FilesystemObjectStore& objects, RegionKey key, std::uint64_t frame, + const typename Codec::State& opening, Uuid commit, const Sha256Digest& build, + std::size_t budget, milliseconds timeout) { + auto payload = Codec::snapshot(opening); + if (!payload || Codec::tick(opening) != 0) return std::unexpected{fail(PublicationError::candidate)}; const auto digest = sha256(*payload); if (!digest) return std::unexpected{fail(PublicationError::digest)}; - auto manifest = genesis_manifest(key, frame, commit, opening.tick_microseconds(), build, - payload->size(), *digest); + auto manifest = genesis_manifest(key, frame, commit, Codec::quantum(opening), build, + payload->size(), *digest, Codec::profile); if (!manifest) return std::unexpected{fail(PublicationError::candidate)}; - auto bytes = stage(objects, *payload, *manifest, restore_budget); + auto bytes = stage(objects, *payload, *manifest, budget); if (!bytes) return std::unexpected{bytes.error()}; constexpr char digits[] = "0123456789abcdef"; @@ -73,14 +113,11 @@ std::expected provision_environmental_r const std::array p{parameter}; auto done = owner.query("SELECT evolution.provision_region_checkpoint($1::bytea)", p, timeout); if (!done) { - const auto kind = done.error().kind == PgError::unknown ? PublicationError::unknown - : PublicationError::rejected; - return std::unexpected{ - PublicationFailure{kind, - {done.error().kind == PgError::unknown ? LeaseStoreError::unknown - : LeaseStoreError::store, - done.error()}, - {}}}; + const bool unknown = done.error().kind == PgError::unknown; + return std::unexpected{PublicationFailure{ + unknown ? PublicationError::unknown : PublicationError::rejected, + {unknown ? LeaseStoreError::unknown : LeaseStoreError::store, done.error()}, + {}}}; } auto committed = load_region_checkpoint(owner, key, timeout); if (!committed) @@ -89,32 +126,33 @@ std::expected provision_environmental_r return confirm(committed->head, std::move(*bytes), *manifest); } -std::expected publish_environmental_checkpoint( - Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease, - const CommittedCheckpoint& predecessor, const processes::EnvironmentalCommunity& state, - Uuid commit, const Sha256Digest& build, std::size_t restore_budget, milliseconds timeout) { +template +std::expected +publish(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease, + const CommittedCheckpoint& predecessor, const typename Codec::State& state, Uuid commit, + const Sha256Digest& build, std::size_t budget, milliseconds timeout) { const auto previous = decode_manifest(predecessor.manifest); - auto payload = state.snapshot(); - if (!previous || !payload) + auto payload = Codec::snapshot(state); + if (!previous || !payload || previous->profile != Codec::profile) return std::unexpected{fail(PublicationError::candidate)}; const auto digest = sha256(*payload); if (!digest) return std::unexpected{fail(PublicationError::digest)}; auto manifest = successor_manifest(*previous, predecessor.head.digest, commit, lease, - state.progress().tick, build, payload->size(), *digest); + Codec::tick(state), build, payload->size(), *digest); if (!manifest) return std::unexpected{fail(PublicationError::candidate)}; - auto bytes = stage(objects, *payload, *manifest, restore_budget); + auto bytes = stage(objects, *payload, *manifest, budget); if (!bytes) return std::unexpected{bytes.error()}; auto published = publish_region_checkpoint(db, *bytes, timeout); return confirm(published, std::move(*bytes), *manifest); } -std::expected -recover_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, - std::span retained_builds, - std::size_t restore_budget, milliseconds timeout) { +template +std::expected +recover(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, + std::span retained_builds, std::size_t budget, milliseconds timeout) { auto committed = load_region_checkpoint(db, key, timeout); if (!committed) return std::unexpected{ @@ -138,7 +176,7 @@ recover_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& obje manifest->sequence != head.sequence || manifest->commit != head.commit || manifest->end_tick != head.end_tick) return fail_with(RecoveryError::manifest); - if (!supported(*manifest) || + if (!supported(*manifest) || manifest->profile != Codec::profile || std::ranges::find(retained_builds, manifest->build) == retained_builds.end()) return fail_with(RecoveryError::unsupported); auto payload = objects.get({manifest->payload_sha256, manifest->payload_bytes}); @@ -146,9 +184,53 @@ recover_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& obje return fail_with(payload.error() == ObjectError::missing ? RecoveryError::missing_payload : payload.error() == ObjectError::corrupt ? RecoveryError::corrupt_payload : RecoveryError::store); - auto state = restore_environmental_payload(*payload, *manifest, restore_budget); + auto state = Codec::restore(*payload, *manifest, budget); if (!state) return fail_with(RecoveryError::payload); - return RecoveredCheckpoint{std::move(*committed), *manifest, std::move(*state)}; + return typename Codec::Recovered{std::move(*committed), *manifest, std::move(*state)}; +} +} // namespace + +std::expected provision_environmental_region( + Postgres& owner, const FilesystemObjectStore& objects, RegionKey key, std::uint64_t frame, + const processes::EnvironmentalCommunity& opening, Uuid commit, const Sha256Digest& build, + std::size_t restore_budget, milliseconds timeout) { + return provision(owner, objects, key, frame, opening, commit, build, + restore_budget, timeout); +} +std::expected publish_environmental_checkpoint( + Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease, + const CommittedCheckpoint& predecessor, const processes::EnvironmentalCommunity& state, + Uuid commit, const Sha256Digest& build, std::size_t restore_budget, milliseconds timeout) { + return publish(db, objects, lease, predecessor, state, commit, build, + restore_budget, timeout); +} +std::expected +recover_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, + std::span retained_builds, + std::size_t restore_budget, milliseconds timeout) { + return recover(db, objects, key, retained_builds, restore_budget, timeout); +} + +std::expected +provision_lineage_region(Postgres& owner, const FilesystemObjectStore& objects, RegionKey key, + std::uint64_t frame, const processes::LineageRegion& opening, Uuid commit, + const Sha256Digest& build, std::size_t restore_budget, + milliseconds timeout) { + return provision(owner, objects, key, frame, opening, commit, build, + restore_budget, timeout); +} +std::expected publish_lineage_checkpoint( + Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease, + const CommittedCheckpoint& predecessor, const processes::LineageRegion& state, Uuid commit, + const Sha256Digest& build, std::size_t restore_budget, milliseconds timeout) { + return publish(db, objects, lease, predecessor, state, commit, build, + restore_budget, timeout); +} +std::expected +recover_lineage_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, + std::span retained_builds, + std::size_t restore_budget, milliseconds timeout) { + return recover(db, objects, key, retained_builds, restore_budget, timeout); } } // namespace evo::persistence diff --git a/src/persistence/region_checkpoint.cpp b/src/persistence/region_checkpoint.cpp index d8bf4c8..67576ad 100644 --- a/src/persistence/region_checkpoint.cpp +++ b/src/persistence/region_checkpoint.cpp @@ -51,9 +51,12 @@ class Reader { // Every field, independent of which versions this build supports. bool structural(const CheckpointManifest& m) noexcept { + // An unknown profile is unsupported, not malformed: bound it by the largest cap. + const auto cap = max_payload_bytes(m.profile); + const auto limit = cap == 0 ? max_lineage_payload_bytes : cap; if (m.key.world == 0 || m.key.region == 0 || m.frame == 0 || zero(m.commit) || m.tick_microseconds == 0 || zero(m.build) || m.payload_bytes == 0 || - m.payload_bytes > max_environmental_payload_bytes || zero(m.payload_sha256)) + m.payload_bytes > limit || zero(m.payload_sha256)) return false; if (m.sequence == 0) return zero(m.predecessor_commit) && zero(m.predecessor_digest) && m.begin_tick == 0 && @@ -65,9 +68,12 @@ bool structural(const CheckpointManifest& m) noexcept { } // namespace bool supported(const CheckpointManifest& m) noexcept { - return m.profile == CheckpointProfile::environmental_v1 && - m.versions == environmental_v1_versions && m.rng_streams == 0 && - m.history_profile == 0 && m.durability == DurabilityPolicy::single_volume_development; + const bool profile = + (m.profile == CheckpointProfile::environmental_v1 && m.history_profile == 0) || + (m.profile == CheckpointProfile::environmental_lineage_v1 && + m.history_profile == lineage_history_profile); + return profile && m.versions == environmental_v1_versions && m.rng_streams == 0 && + m.durability == DurabilityPolicy::single_volume_development; } std::expected, CheckpointError> @@ -150,8 +156,11 @@ decode_manifest(std::span bytes) { std::expected genesis_manifest(RegionKey key, std::uint64_t frame, Uuid commit, std::uint64_t tick_microseconds, const Sha256Digest& build, std::uint64_t payload_bytes, - const Sha256Digest& payload_sha256) { + const Sha256Digest& payload_sha256, CheckpointProfile profile) { CheckpointManifest m; + m.profile = profile; + m.history_profile = + profile == CheckpointProfile::environmental_lineage_v1 ? lineage_history_profile : 0; m.key = key; m.frame = frame; m.commit = commit; @@ -196,7 +205,7 @@ restore_environmental_payload(std::span payload, const Check std::size_t budget) { if (!structural(m)) return std::unexpected{CheckpointError::format}; - if (!supported(m)) + if (!supported(m) || m.profile != CheckpointProfile::environmental_v1) return std::unexpected{CheckpointError::unsupported}; if (payload.size() != m.payload_bytes || payload.size() < 16) return std::unexpected{CheckpointError::payload}; @@ -217,4 +226,30 @@ restore_environmental_payload(std::span payload, const Check return std::unexpected{CheckpointError::payload}; return std::move(*restored); } + +std::expected, CheckpointError> +restore_lineage_payload(std::span payload, const CheckpointManifest& m, + std::size_t budget) { + if (!structural(m)) + return std::unexpected{CheckpointError::format}; + if (!supported(m) || m.profile != CheckpointProfile::environmental_lineage_v1) + return std::unexpected{CheckpointError::unsupported}; + // Container header (56 bytes) and the nested envelope's magic and format. + if (payload.size() != m.payload_bytes || payload.size() < 72) + return std::unexpected{CheckpointError::payload}; + Reader container{payload.subspan(8)}; + Reader nested{payload.subspan(64)}; + if (container.u(4) != processes::lineage_region_format || nested.u(8) != m.versions.envelope) + return std::unexpected{CheckpointError::unsupported}; + auto restored = processes::LineageRegion::restore(payload, {m.key.world, m.frame}, budget); + if (!restored) + return std::unexpected{restored.error() == processes::LineageRegionError::version + ? CheckpointError::unsupported + : CheckpointError::payload}; + const auto& community = (*restored)->community(); + if (community.progress().tick != m.end_tick || + community.tick_microseconds() != m.tick_microseconds) + return std::unexpected{CheckpointError::payload}; + return std::move(*restored); +} } // namespace evo::persistence diff --git a/src/processes/lineage_region.cpp b/src/processes/lineage_region.cpp new file mode 100644 index 0000000..ba3aa40 --- /dev/null +++ b/src/processes/lineage_region.cpp @@ -0,0 +1,179 @@ +#include "evo/processes/lineage_region.hpp" +#include +#include +#include + +namespace evo::processes { +namespace { +constexpr std::uint64_t region_magic = 0x31524C454F564541ULL; // "AEVOELR1" +constexpr std::size_t region_header_bytes = 56; +std::uint64_t fnv(std::span bytes) noexcept { + std::uint64_t value = 14695981039346656037ULL; + for (const auto byte : bytes) + value = (value ^ byte) * 1099511628211ULL; + return value; +} +// Fills an exactly sized buffer; the caller checks the final cursor (M6.5c pattern, +// which also avoids GCC 14 reserve/append false positives). +class Cursor { + public: + explicit Cursor(world::WorldBytes& out) : out_(out) {} + void put(std::uint64_t value, unsigned width) { + for (unsigned i = 0; i < width; ++i, value >>= 8U) + out_[at_++] = static_cast(value & 0xFFU); + } + void bytes(std::span value) { + std::ranges::copy(value, out_.begin() + static_cast(at_)); + at_ += value.size(); + } + [[nodiscard]] std::size_t at() const noexcept { + return at_; + } + + private: + world::WorldBytes& out_; + std::size_t at_{}; +}; +std::uint64_t get(std::span in, std::size_t at, unsigned width) { + std::uint64_t value{}; + for (unsigned i = 0; i < width; ++i) + value |= static_cast(in[at + i]) << (8U * i); + return value; +} +} // namespace + +LineageRegion::LineageRegion(std::unique_ptr community, + const biology::LineageArchive& archive, LineageBinding binding) + : community_(std::move(community)), archive_(archive), binding_(binding) {} + +bool LineageRegion::closed() const { + const auto& b = binding_; + if (!community_ || b.producer == 0 || b.consumer == 0 || b.producer == b.consumer) + return false; + const auto producer = archive_.organism(b.producer); + const auto consumer = archive_.organism(b.consumer); + if (!producer || !consumer) + return false; + const auto progress = community_->progress(); + const Tick now{progress.tick}; + // Terminal lifecycle and archive death agree, and neither is in the future. + for (const auto& [record, terminal] : {std::pair{*producer, progress.producer_terminal}, + std::pair{*consumer, progress.consumer_terminal}}) { + if (terminal > 1 || record.alive != (terminal == 0) || record.born > now || + (!record.alive && record.died > now)) + return false; + } + // No living organism exists outside this region's accounts. + std::array expected{}; + std::size_t count = 0; + if (producer->alive) + expected[count++] = b.producer; + if (consumer->alive) + expected[count++] = b.consumer; + if (count == 2 && expected[0] > expected[1]) + std::swap(expected[0], expected[1]); + const auto living = archive_.living_organisms(); + return std::ranges::equal(living, std::span{expected}.first(count)) && + b.next_organism > archive_.highest_organism() && + b.next_genome > archive_.highest_genome(); +} + +std::expected, LineageRegionError> +LineageRegion::create(std::unique_ptr community, + const biology::LineageArchive& archive, LineageBinding binding) { + if (!community) + return std::unexpected{LineageRegionError::community}; + std::unique_ptr region( + new LineageRegion(std::move(community), archive, binding)); + if (!region->closed()) + return std::unexpected{LineageRegionError::closure}; + return region; +} + +std::expected +LineageRegion::advance(std::uint64_t tick, const EnvironmentalInput& input) { + if (poisoned_) + return std::unexpected{LineageRegionError::poisoned}; + auto report = community_->advance(tick, input); + if (!report) + return std::unexpected{LineageRegionError::community}; + // Closure guarantees both organisms are alive until their transition and born + // no later than the previous tick, so these deaths cannot be refused. + const std::pair deaths[] = { + {report->terminal_transition, binding_.producer}, + {report->consumer_transition, binding_.consumer}}; + for (const auto& [transition, organism] : deaths) + if (transition == 1 && !archive_.record_death(organism, Tick{tick})) { + poisoned_ = true; + return std::unexpected{LineageRegionError::poisoned}; + } + return *report; +} + +std::expected LineageRegion::snapshot() const { + if (poisoned_) + return std::unexpected{LineageRegionError::poisoned}; + auto envelope = community_->snapshot(); + if (!envelope) + return std::unexpected{LineageRegionError::community}; + const auto archive = archive_.encode(); + world::WorldBytes out(region_header_bytes + envelope->size() + 8 + archive.size() + 8); + Cursor w{out}; + w.put(region_magic, 8); + w.put(lineage_region_format, 4); + w.put(0, 4); + w.put(binding_.producer, 8); + w.put(binding_.consumer, 8); + w.put(binding_.next_organism, 8); + w.put(binding_.next_genome, 8); + w.put(envelope->size(), 8); + w.bytes(*envelope); + w.put(archive.size(), 8); + w.bytes(archive); + w.put(fnv(std::span{out}.first(w.at())), 8); + if (w.at() != out.size()) + return std::unexpected{LineageRegionError::format}; + return out; +} + +std::expected, LineageRegionError> +LineageRegion::restore(std::span bytes, world::WorldIdentity identity, + std::size_t budget) { + if (bytes.size() < region_header_bytes + 24 || bytes.size() > max_lineage_region_bytes || + get(bytes, 0, 8) != region_magic) + return std::unexpected{LineageRegionError::format}; + if (get(bytes, 8, 4) != lineage_region_format) + return std::unexpected{LineageRegionError::version}; + if (get(bytes, 12, 4) != 0) + return std::unexpected{LineageRegionError::format}; + const auto body = bytes.first(bytes.size() - 8); + if (get(bytes, bytes.size() - 8, 8) != fnv(body)) + return std::unexpected{LineageRegionError::checksum}; + const LineageBinding binding{get(bytes, 16, 8), get(bytes, 24, 8), get(bytes, 32, 8), + get(bytes, 40, 8)}; + // Lengths are checked against the remaining bytes before any sub-span is taken. + const auto envelope_bytes = get(bytes, 48, 8); + if (envelope_bytes == 0 || envelope_bytes > EnvironmentalCommunity::max_snapshot_bytes || + envelope_bytes > body.size() - region_header_bytes - 8) + return std::unexpected{LineageRegionError::format}; + const auto envelope = + body.subspan(region_header_bytes, static_cast(envelope_bytes)); + const auto after = region_header_bytes + envelope.size(); + const auto archive_bytes = get(body, after, 8); + if (archive_bytes != body.size() - after - 8) + return std::unexpected{LineageRegionError::format}; + auto community = EnvironmentalCommunity::restore(envelope, identity, budget); + if (!community) + return std::unexpected{LineageRegionError::community}; + auto archive = biology::LineageArchive::decode(body.subspan(after + 8)); + if (!archive) + return std::unexpected{LineageRegionError::archive}; + auto region = create(std::move(*community), *archive, binding); + if (!region) + return std::unexpected{region.error()}; + const auto again = (*region)->snapshot(); + if (!again || !std::ranges::equal(*again, bytes)) + return std::unexpected{LineageRegionError::format}; + return std::move(*region); +} +} // namespace evo::processes diff --git a/tests/fixtures/environmental_region.hpp b/tests/fixtures/environmental_region.hpp index 5d03af6..883aefd 100644 --- a/tests/fixtures/environmental_region.hpp +++ b/tests/fixtures/environmental_region.hpp @@ -1,6 +1,7 @@ #pragma once // Rules-2 environmental community used by region checkpoint tests (M7.4). #include "evo/processes/environmental_community.hpp" +#include "evo/processes/lineage_region.hpp" #include #include #include @@ -81,4 +82,45 @@ inline void advance_region(processes::EnvironmentalCommunity& o, std::uint64_t t require(o.advance(tick, region_input(tick)).has_value(), "advance"); } } + +// History-capable fixture: founder genome 1 and descendant 2; a long-dead organism +// 5 (genome 1); producer organism 10 (genome 1) and consumer 11 (genome 2). +inline biology::Genome region_genome(std::uint64_t id, std::uint64_t parent) { + biology::GenomeDraft draft; + draft.id = id; + draft.origin_event = id; + draft.development_version = 1; + if (parent != 0) + draft.parents = {parent}; + draft.capabilities = {{biology::Capability::energy_conversion, 400, 0}, + {biology::Capability::membrane, 120, 0}, + {biology::Capability::target_size, 8'000'000, 0}}; + auto genome = biology::Genome::create(draft); + require(genome.has_value(), "genome"); + return *genome; +} +inline biology::LineageArchive region_archive() { + biology::LineageArchive archive; + require(archive.admit(region_genome(1, 0)).has_value(), "admit 1"); + require(archive.admit(region_genome(2, 1)).has_value(), "admit 2"); + require(archive.record_birth(5, 1, Tick{0}).has_value(), "birth 5"); + require(archive.record_death(5, Tick{0}).has_value(), "death 5"); + require(archive.record_birth(10, 1, Tick{0}).has_value(), "birth 10"); + require(archive.record_birth(11, 2, Tick{0}).has_value(), "birth 11"); + return archive; +} +inline constexpr processes::LineageBinding region_binding{10, 11, 12, 3}; +inline std::unique_ptr lineage_region() { + auto region = + processes::LineageRegion::create(region_community(), region_archive(), region_binding); + require(region.has_value(), "lineage region"); + return std::move(*region); +} +inline void advance_lineage(processes::LineageRegion& r, std::uint64_t ticks, + bool consumer_dies = false) { + for (std::uint64_t i = 0; i < ticks; ++i) { + const auto tick = r.community().progress().tick + 1; + require(r.advance(tick, region_input(tick, consumer_dies)).has_value(), "advance lineage"); + } +} } // namespace evo::test diff --git a/tests/lineage_region_tests.cpp b/tests/lineage_region_tests.cpp new file mode 100644 index 0000000..1db7310 --- /dev/null +++ b/tests/lineage_region_tests.cpp @@ -0,0 +1,146 @@ +#include "evo/processes/lineage_region.hpp" +#include "fixtures/environmental_region.hpp" + +#include +#include +#include +#include + +namespace { +using namespace evo::processes; +using evo::Tick; +using evo::biology::LineageArchive; +using evo::test::advance_lineage; +using evo::test::lineage_region; +using evo::test::region_archive; +using evo::test::region_binding; +using evo::test::region_community; +using evo::test::region_identity; +void check(bool ok, std::source_location at = std::source_location::current()) { + if (!ok) + throw std::runtime_error("lineage region line " + std::to_string(at.line())); +} +#define CHECK(...) check(static_cast(__VA_ARGS__)) +bool refused(const LineageArchive& archive, LineageBinding binding, LineageRegionError error) { + auto r = LineageRegion::create(region_community(), archive, binding); + return !r && r.error() == error; +} +void set(evo::world::WorldBytes& b, std::size_t at, std::uint64_t value) { + for (unsigned i = 0; i < 8; ++i) + b[at + i] = static_cast((value >> (8U * i)) & 0xFFU); +} +void reseal(evo::world::WorldBytes& b) { + std::uint64_t h = 14695981039346656037ULL; + for (std::size_t i = 0; i + 8 < b.size(); ++i) + h = (h ^ b[i]) * 1099511628211ULL; + set(b, b.size() - 8, h); +} + +void closure_is_required() { + CHECK(lineage_region()); + CHECK(!LineageRegion::create(nullptr, region_archive(), region_binding)); + const auto with = [](auto change) { + auto b = region_binding; + change(b); + return b; + }; + const auto closure = LineageRegionError::closure; + CHECK(refused(region_archive(), with([](auto& b) { b.producer = 0; }), closure)); + CHECK(refused(region_archive(), with([](auto& b) { b.consumer = b.producer; }), closure)); + CHECK(refused(region_archive(), with([](auto& b) { b.consumer = 99; }), closure)); // Unknown. + CHECK(refused(region_archive(), with([](auto& b) { b.consumer = 5; }), closure)); // Dead. + CHECK(refused(region_archive(), with([](auto& b) { b.next_organism = 11; }), closure)); + CHECK(refused(region_archive(), with([](auto& b) { b.next_genome = 2; }), closure)); + // A living organism that no account owns is unowned state, not history. + auto stray = region_archive(); + CHECK(stray.record_birth(12, 2, Tick{0})); + CHECK(refused(stray, region_binding, closure)); + CHECK(refused(stray, with([](auto& b) { b.next_organism = 13; }), closure)); + // Born in the future relative to the community tick. + LineageArchive future; + CHECK(future.admit(evo::test::region_genome(1, 0)) && + future.admit(evo::test::region_genome(2, 1))); + CHECK(future.record_birth(10, 1, Tick{3}) && future.record_birth(11, 2, Tick{0})); + CHECK(refused(future, region_binding, closure)); + // Archive says dead while the community's producer is active. + auto dead = region_archive(); + CHECK(dead.record_death(10, Tick{0})); + CHECK(refused(dead, region_binding, closure)); +} + +void deaths_are_recorded_in_the_same_tick() { + auto region = lineage_region(); + advance_lineage(*region, 19, true); + CHECK(region->archive().organism(11)->alive); + auto report = region->advance(20, evo::test::region_input(20, true)); + CHECK(report && report->consumer_transition == 1); + const auto consumer = region->archive().organism(11); + CHECK(consumer && !consumer->alive && consumer->died.value == 20); + CHECK(region->community().progress().consumer_terminal == 1); + CHECK(region->archive().living_organisms() == std::vector{10}); + // The genome outlives its only carrier; ancestry still resolves. + CHECK(*region->archive().living_carriers(2) == 0 && region->archive().ancestry(2)->size() == 2); + // A refused tick changes neither the community nor the archive. + const auto before = region->snapshot(); + CHECK(!region->advance(20, evo::test::region_input(20))); + CHECK(region->snapshot() == before); +} + +void snapshots_restore_and_continue() { + auto original = lineage_region(); + advance_lineage(*original, 24, true); + const auto bytes = original->snapshot(); + CHECK(bytes && bytes->size() <= max_lineage_region_bytes); + auto restored = LineageRegion::restore(*bytes, region_identity, 8000000); + CHECK(restored && (*restored)->binding() == region_binding); + CHECK((*restored)->archive().encode() == original->archive().encode()); + CHECK(!(*restored)->archive().organism(11)->alive); + for (std::uint64_t tick = 25; tick <= 60; ++tick) { + auto a = original->advance(tick, evo::test::region_input(tick)); + auto b = (*restored)->advance(tick, evo::test::region_input(tick)); + CHECK(a && b && *a == *b); + } + CHECK(original->snapshot() == (*restored)->snapshot()); + + const auto refused_with = [&](auto change, LineageRegionError error, bool seal = true) { + auto b = *bytes; + change(b); + if (seal) + reseal(b); + auto r = LineageRegion::restore(b, region_identity, 8000000); + return !r && r.error() == error; + }; + CHECK(refused_with([](auto& b) { b[100] ^= 1U; }, LineageRegionError::checksum, false)); + CHECK(refused_with([](auto& b) { b[0] ^= 1U; }, LineageRegionError::format)); + CHECK(refused_with([](auto& b) { b[8] = 2; }, LineageRegionError::version)); + CHECK(refused_with([](auto& b) { b[12] = 1; }, LineageRegionError::format)); + CHECK(refused_with([](auto& b) { set(b, 48, 0); }, LineageRegionError::format)); + CHECK(refused_with([&](auto& b) { set(b, 48, b.size()); }, LineageRegionError::format)); + // Binding edits that break closure: a stale high-water or a swapped account. + CHECK(refused_with([](auto& b) { set(b, 32, 11); }, LineageRegionError::closure)); + CHECK(refused_with([](auto& b) { set(b, 24, 10); }, LineageRegionError::closure)); + // Nested damage: envelope byte and archive byte (each checked by its own codec). + CHECK(refused_with([](auto& b) { b[80] ^= 1U; }, LineageRegionError::community)); + CHECK(refused_with([](auto& b) { b[b.size() - 20] ^= 1U; }, LineageRegionError::archive)); + // An older checkpoint with the consumer still alive cannot pass as this one: + // the restored bytes are what the chain committed, nothing else is consulted. + auto earlier = lineage_region(); + advance_lineage(*earlier, 10, true); + auto older = LineageRegion::restore(*earlier->snapshot(), region_identity, 8000000); + CHECK(older && (*older)->archive().organism(11)->alive); + CHECK(!LineageRegion::restore(*bytes, {7, 10}, 8000000)); // Foreign frame. +} +} // namespace + +int main() { + try { + closure_is_required(); + deaths_are_recorded_in_the_same_tick(); + snapshots_restore_and_continue(); + std::puts("lineage region contracts passed"); + return 0; + } catch (const std::exception& e) { + std::fprintf(stderr, "%s\n", e.what()); + return 1; + } +} diff --git a/tests/lineage_tests.cpp b/tests/lineage_tests.cpp index d8e4db1..7177017 100644 --- a/tests/lineage_tests.cpp +++ b/tests/lineage_tests.cpp @@ -4,8 +4,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -252,6 +254,149 @@ void capacity_refuses_loudly_rather_than_dropping() { CHECK(deep->front() == 1 && deep->back() == max_archived_genomes); CHECK(*archive.common_ancestor(max_archived_genomes, 1) == 1); } +std::uint64_t field(const std::vector& b, std::size_t at, unsigned width) { + std::uint64_t value{}; + for (unsigned i = 0; i < width; ++i) + value |= static_cast(b[at + i]) << (8U * i); + return value; +} +void set(std::vector& b, std::size_t at, std::uint64_t value, unsigned width) { + for (unsigned i = 0; i < width; ++i) + b[at + i] = static_cast((value >> (8U * i)) & 0xFFU); +} +// Recomputes the trailing FNV so a test reaches the check it is aimed at. +void reseal(std::vector& b) { + std::uint64_t h = 14695981039346656037ULL; + for (std::size_t i = 0; i + 8 < b.size(); ++i) + h = (h ^ b[i]) * 1099511628211ULL; + set(b, b.size() - 8, h, 8); +} +LineageArchive with_history() { + auto archive = branching(); + CHECK(archive.record_birth(100, 4, evo::Tick{10})); + CHECK(archive.record_birth(101, 5, evo::Tick{12})); + CHECK(archive.record_birth(90, 1, evo::Tick{0})); + CHECK(archive.record_death(90, evo::Tick{0})); + CHECK(archive.record_death(100, evo::Tick{30})); + return archive; +} + +void the_archive_round_trips_canonically() { + const auto archive = with_history(); + const auto bytes = archive.encode(); + CHECK(field(bytes, 0, 8) == 0x3152414C4F564541ULL && field(bytes, 8, 4) == 1); + CHECK(field(bytes, 16, 4) == 5 && field(bytes, 20, 4) == 3 && + field(bytes, 24, 8) == bytes.size()); + const auto decoded = LineageArchive::decode(bytes); + CHECK(decoded && decoded->encode() == bytes); + CHECK(decoded->archived() == 5 && decoded->recorded() == 3); + CHECK(*decoded->find(4) == *archive.find(4)); + const auto dead = decoded->organism(100); + CHECK(dead && !dead->alive && dead->died.value == 30 && dead->born.value == 10); + CHECK(decoded->ancestry(4) == archive.ancestry(4)); + CHECK(decoded->living_organisms() == std::vector{101}); + CHECK(decoded->highest_genome() == 5 && decoded->highest_organism() == 101); + // A clone decodes into a new archive; the original is untouched. + CHECK(archive.recorded() == 3 && archive.encode() == bytes); + // Admission order does not change the bytes, only the content does. + LineageArchive reordered; + for (const auto& [id, parent, c] : + {std::tuple{1, 0, 400U}, std::tuple{3, 1, 391U}, std::tuple{2, 1, 409U}, + std::tuple{5, 2, 400U}, std::tuple{4, 2, 418U}}) + CHECK(reordered.admit( + descend(static_cast(id), static_cast(parent), c))); + CHECK(reordered.record_birth(90, 1, evo::Tick{0})); + CHECK(reordered.record_birth(101, 5, evo::Tick{12})); + CHECK(reordered.record_birth(100, 4, evo::Tick{10})); + CHECK(reordered.record_death(100, evo::Tick{30})); + CHECK(reordered.record_death(90, evo::Tick{0})); + CHECK(reordered.encode() == bytes); + // A child with a smaller identity than its parent still follows its parent. + LineageArchive inverted; + CHECK(inverted.admit(descend(50, 0)) && inverted.admit(descend(7, 50))); + const auto inverted_bytes = inverted.encode(); + CHECK(LineageArchive::decode(inverted_bytes)); + LineageArchive empty; + CHECK(LineageArchive::decode(empty.encode()) && empty.encode().size() == 40); +} + +void the_archive_decoder_refuses_damage_and_disorder() { + const auto bytes = with_history().encode(); + const auto refused = [&](auto change, ArchiveCodecError error, bool seal = true) { + auto b = bytes; + change(b); + if (seal) + reseal(b); + const auto d = LineageArchive::decode(b); + return !d && d.error() == error; + }; + CHECK(refused([](auto& b) { b[40] ^= 1U; }, ArchiveCodecError::checksum, false)); + CHECK(refused([](auto& b) { b[0] ^= 1U; }, ArchiveCodecError::format)); + CHECK(refused([](auto& b) { b[8] = 2; }, ArchiveCodecError::version)); + CHECK(refused([](auto& b) { b[12] = 2; }, ArchiveCodecError::version)); + CHECK(refused([](auto& b) { set(b, 16, max_archived_genomes + 1, 4); }, + ArchiveCodecError::bounds)); + CHECK(refused([](auto& b) { set(b, 20, max_archived_organisms + 1, 4); }, + ArchiveCodecError::bounds)); + CHECK(refused([](auto& b) { set(b, 24, b.size() + 1, 8); }, ArchiveCodecError::format)); + CHECK(refused( + [](auto& b) { + b.insert(b.end() - 8, 0); + set(b, 24, b.size(), 8); + }, + ArchiveCodecError::format)); + CHECK(refused([](auto& b) { set(b, 20, 2, 4); }, ArchiveCodecError::format)); + // Genome body damage (its own digest), then an organism record naming no genome. + CHECK(refused([](auto& b) { b[40] ^= 1U; }, ArchiveCodecError::genome)); + const auto organisms = bytes.size() - 8 - (3 * archived_organism_bytes); + CHECK(refused([&](auto& b) { set(b, organisms + 8, 77, 8); }, ArchiveCodecError::history)); + CHECK(refused([&](auto& b) { b[organisms + 32] = 2; }, ArchiveCodecError::format)); + // Alive with a death tick; death before birth; organisms out of order. + const auto alive = organisms + (2 * archived_organism_bytes); // Organism 101. + CHECK(refused([&](auto& b) { set(b, alive + 24, 5, 8); }, ArchiveCodecError::format)); + const auto dead = organisms + archived_organism_bytes; // Organism 100. + CHECK(refused([&](auto& b) { set(b, dead + 24, 9, 8); }, ArchiveCodecError::history)); + CHECK(refused( + [&](auto& b) { + std::swap_ranges(b.begin() + static_cast(organisms), + b.begin() + static_cast(organisms + 33), + b.begin() + static_cast(dead)); + }, + ArchiveCodecError::noncanonical)); + // A child encoded before its parent is refused by admission itself. + LineageArchive pair; + CHECK(pair.admit(descend(1, 0)) && pair.admit(descend(2, 1))); + auto swapped = pair.encode(); + const auto first = field(swapped, 32, 2); + std::vector parent(swapped.begin() + 32, + swapped.begin() + 34 + static_cast(first)); + std::vector child(swapped.begin() + 34 + static_cast(first), + swapped.end() - 8); + std::ranges::copy(child, swapped.begin() + 32); + std::ranges::copy(parent, swapped.begin() + 32 + static_cast(child.size())); + reseal(swapped); + const auto dangling = LineageArchive::decode(swapped); + CHECK(!dangling && dangling.error() == ArchiveCodecError::history); + CHECK(!LineageArchive::decode(std::span{bytes}.first(39))); +} + +void a_full_archive_round_trips() { + LineageArchive archive; + CHECK(archive.admit(descend(1, 0))); + for (std::uint64_t id = 2; id <= max_archived_genomes; ++id) + CHECK(archive.admit(descend(id, id - 1))); + for (std::uint64_t o = 1; o <= max_archived_organisms; ++o) { + CHECK(archive.record_birth(o, 1 + (o % max_archived_genomes), evo::Tick{o})); + if (o % 3 == 0) + CHECK(archive.record_death(o, evo::Tick{o + 1})); + } + const auto bytes = archive.encode(); + CHECK(bytes.size() <= max_archive_bytes); + const auto decoded = LineageArchive::decode(bytes); + CHECK(decoded && decoded->encode() == bytes && decoded->recorded() == max_archived_organisms); + std::cout << "lineage_archive_full_bytes=" << bytes.size() << " max=" << max_archive_bytes + << '\n'; +} } // namespace int main() try { a_genome_outlives_every_organism_that_carried_it(); @@ -263,6 +408,9 @@ int main() try { the_archive_cannot_be_replaced_or_rewound(); the_archive_offers_no_way_to_forget(); capacity_refuses_loudly_rather_than_dropping(); + the_archive_round_trips_canonically(); + the_archive_decoder_refuses_damage_and_disorder(); + a_full_archive_round_trips(); std::cout << "PASS lineage retention, branching ancestry and provisional absence\n"; return 0; } catch (...) { diff --git a/tests/region_checkpoint_store_tests.cpp b/tests/region_checkpoint_store_tests.cpp index 57a1ba9..f63fb4d 100644 --- a/tests/region_checkpoint_store_tests.cpp +++ b/tests/region_checkpoint_store_tests.cpp @@ -347,6 +347,72 @@ void failures_leave_the_previous_head() { recover_environmental_checkpoint(*r.runtime, *r.objects, {7, 99}, retained, budget, 5s); CHECK(!none && none.error().kind == RecoveryError::not_provisioned); } +// History-capable profile: deaths commit with the state that caused them, the +// archive survives recovery, and a region never switches profile. +void lineage_publish_and_recover() { + const RegionKey key{7, 6}; + auto owner = connect_as(env("PGUSER")); + auto runtime = connect_as(env("EVO_PG_RUNTIME_USER")); + auto objects = FilesystemObjectStore::open(scratch("lineage"), key, max_lineage_payload_bytes); + CHECK(objects); + CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " + "(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING", + {}, 5s)); + auto opening = evo::test::lineage_region(); + auto genesis = + provision_lineage_region(*owner, **objects, key, evo::test::region_identity.frame, *opening, + filled<16>(0xE0), build, budget, 5s); + CHECK(genesis); + CHECK(decode_manifest(genesis->manifest)->profile == + CheckpointProfile::environmental_lineage_v1); + auto lease = acquire_region_lease(*runtime, key, 20000ms, 5s); + CHECK(lease); + + auto region = evo::test::lineage_region(); + evo::test::advance_lineage(*region, 24, true); // Consumer dies at tick 20. + auto first = publish_lineage_checkpoint(*runtime, **objects, *lease, *genesis, *region, + filled<16>(0xE1), build, budget, 5s); + CHECK(first && first->head.end_tick == 24); + // The environmental path refuses a lineage predecessor before uploading. + auto plain = evo::test::region_community(); + evo::test::advance_region(*plain, 30); + auto crossed = publish_environmental_checkpoint(*runtime, **objects, *lease, *first, *plain, + filled<16>(0xE2), build, budget, 5s); + CHECK(!crossed && crossed.error().kind == PublicationError::candidate); + // The database refuses a profile switch even when the manifest is well formed. + auto switched = *decode_manifest(first->manifest); + auto next = successor_manifest(switched, first->head.digest, filled<16>(0xE3), *lease, 30, + build, 100, filled<32>(0xA2)); + CHECK(next); + next->profile = CheckpointProfile::environmental_v1; + next->history_profile = 0; + auto switched_bytes = encode_manifest(*next); + CHECK(switched_bytes); + auto refused = publish_region_checkpoint(*runtime, *switched_bytes, 5s); + CHECK(!refused && refused.error().kind == LeaseStoreError::unsupported); + + const std::array retained{build}; + auto recovered = recover_lineage_checkpoint(*runtime, **objects, key, retained, budget, 5s); + CHECK(recovered && recovered->committed.head == first->head); + const auto consumer = recovered->state->archive().organism(11); + CHECK(consumer && !consumer->alive && consumer->died.value == 20); + CHECK(recovered->state->archive().ancestry(2)->size() == 2); // Retained ancestry. + CHECK(recovered->state->snapshot() == region->snapshot()); + for (std::uint64_t tick = 25; tick <= 60; ++tick) { + auto a = region->advance(tick, evo::test::region_input(tick)); + auto b = recovered->state->advance(tick, evo::test::region_input(tick)); + CHECK(a && b && *a == *b); + } + CHECK(recovered->state->snapshot() == region->snapshot()); + auto second = + publish_lineage_checkpoint(*runtime, **objects, *lease, recovered->committed, + *recovered->state, filled<16>(0xE4), build, budget, 5s); + CHECK(second && second->head.sequence == 2 && second->head.end_tick == 60); + // A host restoring the other profile refuses rather than misreading the payload. + auto wrong = recover_environmental_checkpoint(*runtime, **objects, key, retained, budget, 5s); + CHECK(!wrong && wrong.error().kind == RecoveryError::unsupported); + CHECK(release_region_lease(*runtime, key, *lease, 5s)); +} } // namespace void run_region_checkpoint_store_tests() { @@ -354,4 +420,5 @@ void run_region_checkpoint_store_tests() { object_store_is_immutable_and_verified(); publish_recover_and_continue(); failures_leave_the_previous_head(); + lineage_publish_and_recover(); } diff --git a/tests/region_checkpoint_tests.cpp b/tests/region_checkpoint_tests.cpp index c24d5cc..ba22af9 100644 --- a/tests/region_checkpoint_tests.cpp +++ b/tests/region_checkpoint_tests.cpp @@ -182,6 +182,64 @@ void payload_validation_and_restored_trajectory() { CHECK(refused([](auto& b, auto&) { b[8] = 1; }, CheckpointError::unsupported)); CHECK(!restore_environmental_payload(*payload, *m, 1000)); // Budget still charged. } +void lineage_profile() { + // The SQL parser hard-codes this cap (migrations/0004_lineage_profile.sql). + static_assert(max_lineage_payload_bytes == 4268144); + static_assert(max_payload_bytes(CheckpointProfile::unknown) == 0); + const RegionKey region{evo::test::region_identity.world, 5}; + auto state = evo::test::lineage_region(); + auto opening = state->snapshot(); + CHECK(opening); + auto g = genesis_manifest(region, evo::test::region_identity.frame, filled<16>(1), 50000, build, + opening->size(), filled<32>(0xA0), + CheckpointProfile::environmental_lineage_v1); + CHECK(g && g->history_profile == 1 && supported(*g)); + auto bytes = encode_manifest(*g); + CHECK(bytes && at(*bytes, 12, 4) == 2 && at(*bytes, 232, 4) == 1); + CHECK(decode_manifest(*bytes) == *g); + // Profile and history profile travel together. + auto mixed = *g; + mixed.history_profile = 0; + CHECK(!supported(mixed)); + mixed = genesis(); + mixed.history_profile = 1; + CHECK(!supported(mixed)); + // Per-profile payload caps are structural. + CHECK(!genesis_manifest(region, 9, filled<16>(1), 50000, build, (1024 * 1024) + 1, + filled<32>(0xA0))); + CHECK(genesis_manifest(region, 9, filled<16>(1), 50000, build, (1024 * 1024) + 1, + filled<32>(0xA0), CheckpointProfile::environmental_lineage_v1)); + CHECK(!genesis_manifest(region, 9, filled<16>(1), 50000, build, 4268145, filled<32>(0xA0), + CheckpointProfile::environmental_lineage_v1)); + + auto restored = restore_lineage_payload(*opening, *g, 8000000); + CHECK(restored && (*restored)->archive().recorded() == 3); + CHECK(restore_environmental_payload(*opening, *g, 8000000).error() == + CheckpointError::unsupported); + auto plain = evo::test::region_community()->snapshot(); + auto env = + genesis_manifest(region, 9, filled<16>(1), 50000, build, plain->size(), filled<32>(0xA0)); + CHECK(restore_lineage_payload(*plain, *env, 8000000).error() == CheckpointError::unsupported); + + // Successors inherit the profile; a lineage batch with a death restores exactly. + evo::test::advance_lineage(*state, 24, true); + auto payload = state->snapshot(); + auto lease_at_frame = lease(); + lease_at_frame.frame = evo::test::region_identity.frame; + auto m = successor_manifest(*g, filled<32>(0xD0), filled<16>(2), lease_at_frame, 24, build, + payload->size(), filled<32>(0xA1)); + CHECK(m && m->profile == CheckpointProfile::environmental_lineage_v1 && + m->history_profile == 1); + auto later = restore_lineage_payload(*payload, *m, 8000000); + CHECK(later && !(*later)->archive().organism(11)->alive); + auto wrong_tick = *m; + wrong_tick.end_tick = 23; + CHECK(restore_lineage_payload(*payload, wrong_tick, 8000000).error() == + CheckpointError::payload); + auto nested = *payload; + nested[64] = 1; // Nested envelope format 1 is not this profile's envelope. + CHECK(restore_lineage_payload(nested, *m, 8000000).error() == CheckpointError::unsupported); +} } // namespace int main() { @@ -190,6 +248,7 @@ int main() { malformed_and_unsupported(); successor_chain(); payload_validation_and_restored_trajectory(); + lineage_profile(); std::puts("region checkpoint contracts passed"); return 0; } catch (const std::exception& e) { diff --git a/tests/region_lease_store_tests.cpp b/tests/region_lease_store_tests.cpp index e698e02..db345a2 100644 --- a/tests/region_lease_store_tests.cpp +++ b/tests/region_lease_store_tests.cpp @@ -126,6 +126,24 @@ std::vector patched(const CheckpointManifest& m, std::size_t offse std::ranges::copy(value, bytes->begin() + static_cast(offset)); return std::move(*bytes); } +// Expiry is decided by database time, which is the server's wall clock. A local +// sleep does not prove it: a VM can step that clock back (WSL2 was measured +// stepping ~0.5 s every ~32 s). Wait on the database's own judgement instead. +// Owner-only read; works inside a transaction that holds the row lock. +void wait_until_expired(Postgres& owner_db, RegionKey key) { + const std::array region{std::to_string(key.region)}; + for (int attempt = 0; attempt < 400; ++attempt) { + auto rows = owner_db.query("SELECT (clock_timestamp() >= expires_at)::text FROM " + "evolution.region_leases WHERE world_id = 1 AND region_id = " + "$1::evolution.u64", + region, 5s); + CHECK(rows && rows->size() == 1); + if ((*rows)[0][0] == "true") + return; + std::this_thread::sleep_for(50ms); + } + CHECK(false); // Not expired after 20 s of database time polling. +} bool failed(const auto& result, LeaseStoreError kind) { return !result && result.error().kind == kind; } @@ -245,6 +263,7 @@ void database_time_expiry_and_paused_writer() { std::this_thread::sleep_for(1200ms); // Worker A is paused past its lease. CHECK(a.state(LeaseClock::now()) == LeaseState::fenced); // Expiry is decided by database time; the successor acquires a newer epoch. + wait_until_expired(*admin, key); auto lease_b = acquire_region_lease(*worker_b, key, 20000ms, 5s); CHECK(lease_b && lease_b->epoch == lease_a->epoch + 1); auto head_b = publish(*worker_b, link(genesis, *lease_b, 31, 3)); @@ -262,7 +281,7 @@ void database_time_expiry_and_paused_writer() { CHECK(release_region_lease(*worker_b, key, *lease_b, 5s)); auto lease_c = acquire_region_lease(*worker_a, key, 1000ms, 5s); CHECK(lease_c && lease_c->epoch == lease_b->epoch + 1); - std::this_thread::sleep_for(1200ms); + wait_until_expired(*admin, key); CHECK( failed(renew_region_lease(*worker_a, key, *lease_c, 1000ms, 5s), LeaseStoreError::fenced)); } @@ -285,7 +304,8 @@ void queued_writer_uses_post_lock_database_time() { std::expected queued = std::unexpected{LeaseStoreFailure{}}; std::thread writer([&] { queued = publish(*worker, link(genesis, *lease, 41, 1)); }); - std::this_thread::sleep_for(1300ms); + std::this_thread::sleep_for(300ms); // Let the writer queue behind the lock. + wait_until_expired(*blocker, key); // Still holding the row lock. CHECK(blocker->query("ROLLBACK", {}, 5s)); writer.join(); CHECK(failed(queued, LeaseStoreError::fenced)); @@ -318,6 +338,8 @@ void partitioned_renewal_fences_locally() { CHECK(local_deadline <= sent + fast.duration - fast.safety_margin); std::this_thread::sleep_for(1300ms); CHECK(!authority.may_originate(LeaseClock::now())); + // Release the orphaned renewal only after database expiry, so it must be refused. + wait_until_expired(*blocker, key); CHECK(blocker->query("ROLLBACK", {}, 5s)); // The orphaned renewal reached the row only after database expiry: rejected. auto successor = runtime(); @@ -412,6 +434,32 @@ void exhaustion_fails_closed() { CHECK(failed(publish(*db, beyond), LeaseStoreError::exhausted)); } +// The SQL parser's per-profile payload caps match the C++ constants exactly. +void payload_caps_follow_the_profile() { + auto admin = owner(); + retain_build(*admin); + const auto provision_raw = [&](std::vector bytes) { + const std::array p{bytea(bytes)}; + return admin->query("SELECT evolution.provision_region_checkpoint($1::bytea)", p, 5s); + }; + auto lineage = genesis_manifest({1, 90}, 7, id(90), 50000, build, max_lineage_payload_bytes, + digest(0xA0), CheckpointProfile::environmental_lineage_v1); + CHECK(lineage); + CHECK(provision_raw(*encode_manifest(*lineage))); + lineage->key.region = 91; + lineage->commit = id(91); + const std::array over_lineage{0x71, 0x20, 0x41, 0, 0, 0, 0, 0}; // 4268145 + CHECK(sqlstate(provision_raw(patched(*lineage, 240, over_lineage))) == "22023"); + auto plain = genesis_manifest({1, 92}, 7, id(92), 50000, build, max_environmental_payload_bytes, + digest(0xA0)); + CHECK(plain); + const std::array over_plain{0x01, 0x00, 0x10, 0, 0, 0, 0, 0}; // 1 MiB + 1 + CHECK(sqlstate(provision_raw(patched(*plain, 240, over_plain))) == "22023"); + // Profile and history profile must match: a lineage profile claiming no history. + const std::array no_history{0}; + CHECK(sqlstate(provision_raw(patched(*lineage, 232, no_history))) == "EVL06"); +} + struct Summary { std::vector samples; void print(const char* name) { @@ -472,5 +520,6 @@ void run_region_lease_store_tests() { partitioned_renewal_fences_locally(); contenders_serialize_on_the_database(); exhaustion_fails_closed(); + payload_caps_follow_the_profile(); measure_round_trips(); } diff --git a/tools/ci/postgres-check.sh b/tools/ci/postgres-check.sh index 4a8c7e0..9aab5a6 100644 --- a/tools/ci/postgres-check.sh +++ b/tools/ci/postgres-check.sh @@ -43,7 +43,7 @@ PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" -f "$root/migrations/0001_region_bootstrap.sql" > /dev/null test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1 PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3 +test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4 # v2 heads have no payload binding: v2->v3 fails closed and leaves history at v2. v2_sha="$(sha256sum "$root/migrations/0002_region_fencing.sql")" v2_sha="${v2_sha%% *}" @@ -53,6 +53,19 @@ PGDATABASE=v2_heads psql -X -v ON_ERROR_STOP=1 -c "SELECT evolution.provision_ if PGDATABASE=v2_heads bash "$root/tools/migrate-postgres.sh"; then echo 'Expected v2 heads to block v3' >&2; exit 1 fi +v3_sha="$(sha256sum "$root/migrations/0003_region_checkpoints.sql")" +v3_sha="${v3_sha%% *}" +createdb v3_commits +PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" \ + -v "migration_sha_2=$v2_sha" -v "migration_sha_3=$v3_sha" -v runtime_role=evo_runtime_test \ + -f "$root/migrations/0001_region_bootstrap.sql" -f "$root/migrations/0002_region_fencing.sql" \ + -f "$root/migrations/0003_region_checkpoints.sql" > /dev/null +PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 \ + -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ + -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '01000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32), 'hex'))" > /dev/null +PGDATABASE=v3_commits bash "$root/tools/migrate-postgres.sh" +test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 4:1,2,3,4 +test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) FROM evolution.region_commits")" = 1 test "$(PGDATABASE=v2_heads psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2 # Inherited privileges bypass fencing, so every migrate run audits effective rights. psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_bad_runtime NOLOGIN IN ROLE pg_write_all_data' @@ -86,9 +99,11 @@ expect_audit_failure 'GRANT EXECUTE ON FUNCTION evolution.provision_region_check expect_audit_failure 'GRANT SELECT ON evolution.region_commits TO evo_runtime_test' 'REVOKE SELECT ON evolution.region_commits FROM evo_runtime_test' bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again. # Unknown future versions and checksum drift must not be repaired silently. -psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=4 WHERE version=3' +psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=5 WHERE version=4' +if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=4 WHERE version=5" +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=4" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi -psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=3 WHERE version=4" psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=3" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=2" diff --git a/tools/migrate-postgres.sh b/tools/migrate-postgres.sh index 585594a..db4d4d3 100644 --- a/tools/migrate-postgres.sh +++ b/tools/migrate-postgres.sh @@ -18,7 +18,10 @@ digest_2="$(sha256sum "$root/migrations/0002_region_fencing.sql")" digest_2="${digest_2%% *}" digest_3="$(sha256sum "$root/migrations/0003_region_checkpoints.sql")" digest_3="${digest_3%% *}" +digest_4="$(sha256sum "$root/migrations/0004_lineage_profile.sql")" +digest_4="${digest_4%% *}" psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_2=$digest_2" --set="migration_sha_3=$digest_3" \ + --set="migration_sha_4=$digest_4" \ --set="runtime_role=$runtime_role" \ --file="$root/migrations/apply.sql"