From 627411983fd6cb73cebeaecb811acd0892250461 Mon Sep 17 00:00:00 2001 From: dingo Date: Sat, 17 Oct 2026 17:10:17 -0400 Subject: [PATCH] M12.3: publish and recover durable exact populations --- CMakeLists.txt | 13 +- README.md | 10 +- apps/evo-server/main.cpp | 48 +- docs/STATUS.md | 15 + .../0142-durable-exact-population-profile.md | 86 ++++ docs/architecture/REGION_DURABILITY.md | 9 + docs/engineering/M12_DELIVERY_PLAN.md | 2 +- docs/engineering/M12_DURABLE_POPULATIONS.md | 68 +++ docs/engineering/M12_REMAINING_GATES.md | 35 +- docs/engineering/VALIDATION.md | 46 ++ .../population_checkpoint_store.hpp | 54 ++ include/evo/persistence/region_checkpoint.hpp | 22 +- .../evo/persistence/region_lease_store.hpp | 19 + include/evo/server/cellular_region_host.hpp | 52 +- .../server/lifecycle/cellular_population.hpp | 3 + .../lifecycle/cellular_population_batch.hpp | 49 ++ .../0044_exact_population_publication.sql | 338 ++++++++++++ migrations/apply.sql | 19 +- schemas/cellular-population-batch.md | 40 ++ schemas/cellular-population.md | 11 +- src/persistence/cellular_checkpoint_store.cpp | 11 +- src/persistence/population_checkpoint.cpp | 78 +++ .../population_checkpoint_store.cpp | 315 ++++++++++++ src/persistence/region_checkpoint.cpp | 31 +- src/persistence/region_lease_store.cpp | 100 +++- src/server/cellular_region_host.cpp | 233 ++++++++- .../lifecycle/cellular_population_batch.cpp | 296 +++++++++++ tests/cellular_population_tests.cpp | 27 +- tests/fixtures/cellular_ecology.hpp | 22 +- tests/population_checkpoint_tests.cpp | 159 ++++++ tests/population_publication_tests.cpp | 486 ++++++++++++++++++ tests/postgres_tests.cpp | 6 + tests/region_migration_tests.cpp | 16 +- tools/ci/postgres-check.sh | 17 +- tools/migrate-postgres.sh | 3 + 35 files changed, 2637 insertions(+), 102 deletions(-) create mode 100644 docs/adr/0142-durable-exact-population-profile.md create mode 100644 docs/engineering/M12_DURABLE_POPULATIONS.md create mode 100644 include/evo/persistence/population_checkpoint_store.hpp create mode 100644 include/evo/server/lifecycle/cellular_population_batch.hpp create mode 100644 migrations/0044_exact_population_publication.sql create mode 100644 schemas/cellular-population-batch.md create mode 100644 src/persistence/population_checkpoint.cpp create mode 100644 src/persistence/population_checkpoint_store.cpp create mode 100644 src/server/lifecycle/cellular_population_batch.cpp create mode 100644 tests/population_checkpoint_tests.cpp create mode 100644 tests/population_publication_tests.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8beba04..0c124fd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,7 +90,7 @@ add_library(evo_lifecycle STATIC src/server/lifecycle/primitive_lives.cpp src/server/lifecycle/cellular_batch.cpp src/server/lifecycle/wildlife_controller.cpp src/server/lifecycle/cellular_play_summary.cpp src/server/lifecycle/cellular_population.cpp src/server/lifecycle/cellular_population_codec.cpp src/server/lifecycle/cellular_population_policy.cpp - src/server/lifecycle/cellular_schedule.cpp) + src/server/lifecycle/cellular_schedule.cpp src/server/lifecycle/cellular_population_batch.cpp) add_library(evo::lifecycle ALIAS evo_lifecycle) target_link_libraries(evo_lifecycle PUBLIC evo::biology) evo_owned_target(evo_lifecycle) @@ -124,7 +124,8 @@ target_include_directories(evo_ops PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") target_compile_features(evo_ops PUBLIC cxx_std_23) evo_owned_target(evo_ops) # Portable checkpoint manifest codec and environmental-v1 payload validation. -add_library(evo_region_checkpoint STATIC src/persistence/region_checkpoint.cpp src/persistence/region_batch.cpp) +add_library(evo_region_checkpoint STATIC src/persistence/region_checkpoint.cpp src/persistence/region_batch.cpp + src/persistence/population_checkpoint.cpp) add_library(evo::region_checkpoint ALIAS evo_region_checkpoint) target_link_libraries(evo_region_checkpoint PUBLIC evo::region_lease evo::processes evo::lifecycle) evo_owned_target(evo_region_checkpoint) @@ -250,7 +251,7 @@ if(EVO_BUILD_POSTGRES) find_package(OpenSSL 3 REQUIRED COMPONENTS Crypto) add_library(evo_region_store STATIC src/persistence/sha256.cpp src/persistence/object_store.cpp src/persistence/checkpoint_store.cpp src/persistence/primitive_checkpoint_store.cpp - src/persistence/cellular_checkpoint_store.cpp + src/persistence/cellular_checkpoint_store.cpp src/persistence/population_checkpoint_store.cpp src/persistence/region_clone.cpp) add_library(evo::region_store ALIAS evo_region_store) target_link_libraries(evo_region_store PUBLIC evo::postgres evo::region_checkpoint @@ -302,6 +303,7 @@ if(EVO_BUILD_POSTGRES) # Contention tests use one thread and one connection per contender. add_executable(evo-postgres-tests tests/postgres_tests.cpp tests/region_lease_store_tests.cpp tests/cellular_publication_tests.cpp + tests/population_publication_tests.cpp tests/account_control_store_tests.cpp tests/probe_handoff_store_tests.cpp tests/life_reservation_store_tests.cpp tests/primitive_publication_tests.cpp @@ -677,6 +679,11 @@ if(BUILD_TESTING) target_link_libraries(evo-cellular-schedule-tests PRIVATE evo::lifecycle) evo_owned_target(evo-cellular-schedule-tests) add_test(NAME cellular-unattended-schedule COMMAND evo-cellular-schedule-tests) + add_executable(evo-population-checkpoint-tests tests/population_checkpoint_tests.cpp) + target_link_libraries(evo-population-checkpoint-tests PRIVATE evo::region_checkpoint) + evo_owned_target(evo-population-checkpoint-tests) + add_test(NAME population-checkpoint-contract COMMAND evo-population-checkpoint-tests) + set_tests_properties(population-checkpoint-contract PROPERTIES TIMEOUT 180) endif() add_executable(evo-chunk-layout experiments/world/chunk_layout.cpp) diff --git a/README.md b/README.md index cf46bd6..269d8c7 100644 --- a/README.md +++ b/README.md @@ -132,10 +132,12 @@ process fixtures remain separate from organism biology. and reporting. `tests` holds contract, fuzz and measurement programs. `src/server/lifecycle` owns funded primitive founder creation and an in-memory cellular tick owner over the finite habitat. -The cellular owner now has durable profile-4 recovery/publication and an -adapter for reliable v7 session actions. `evo-server` still lacks the cellular -scheduler, player-life entry operation, and cellular view replication, so this -is not yet a playable client path. `evo::simulation` remains +The cellular owner has durable profile-4 recovery/publication and the cellular +worker/session path. Opt-in exact population profile 5 is described in +[M12 durable populations](docs/engineering/M12_DURABLE_POPULATIONS.md); +`--cellular-populations on|off` defaults to off. Durable interval admission, +environmental coupling and population qualification remain in progress. The +microscope client is the current M11 playtest path. `evo::simulation` remains graphics-, network- and database-independent. World0 is persistent state. No normal configure, build, test or CI job may create, diff --git a/apps/evo-server/main.cpp b/apps/evo-server/main.cpp index 30c98bb..78fdd50 100644 --- a/apps/evo-server/main.cpp +++ b/apps/evo-server/main.cpp @@ -113,6 +113,7 @@ struct Options { std::uint64_t objects_reserve_mib{1024}; server_app::SessionOptions session; server_app::M10Options m10; + bool cellular_populations{}; struct PrimitiveSessionOptions { std::string listen, certificate, key, issuer, environment; [[nodiscard]] bool any() const noexcept { @@ -273,6 +274,8 @@ std::optional parse(int argc, char** argv) { o.m10.life_routes.emplace_back(value); else if (flag == "--m10-load-governor" && (value == "on" || value == "off")) o.m10.load_governor = value == "on"; + else if (flag == "--cellular-populations" && (value == "on" || value == "off")) + o.cellular_populations = value == "on"; else if (flag == "--primitive-listen") o.primitive_session.listen = value; else if (flag == "--primitive-cert") @@ -304,6 +307,8 @@ std::optional parse(int argc, char** argv) { o.pg.port.empty() || o.pg.database.empty() || o.pg.user.empty() || o.objects.empty() || o.host.build == persistence::Sha256Digest{}) return std::nullopt; + if (o.cellular_populations && o.profile != "cellular") + return std::nullopt; // Rehearsal controls exist only for the synthetic driver (clones, drills). if ((o.synthetic_until != 0 || o.accelerate != 1) && !o.synthetic) return std::nullopt; @@ -898,7 +903,7 @@ int serve_primitive(Options& o) { #ifdef EVO_SERVER_M10 int serve_cellular(Options& o) { auto objects = persistence::FilesystemObjectStore::open( - o.objects, o.host.key, server::max_cellular_lives_bytes, nullptr, + o.objects, o.host.key, server::max_cellular_population_bytes, nullptr, o.objects_reserve_mib * 1024 * 1024); if (!objects) { std::puts("error objects"); @@ -906,7 +911,9 @@ int serve_cellular(Options& o) { } const auto pg = o.pg; auto made = server::CellularRegionHost::create( - {o.host.key, o.host.frame, o.host.lease, o.host.build, o.host.retained_builds}, + {o.host.key, o.host.frame, o.host.lease, o.host.build, o.host.retained_builds, + o.cellular_populations ? std::optional{server::CellularTierPolicy{true, 1200, 1800}} + : std::nullopt}, [pg] { return persistence::Postgres::connect(pg, 2s); }, std::move(*objects)); if (!made) { std::puts("error config"); @@ -1060,6 +1067,17 @@ int serve_cellular(Options& o) { held_births.push_back(birth); } const bool birth_waiting = !held_births.empty(); + const bool want_population = + o.cellular_populations && state && + state->rules_version() == server::cellular_lives_foraging_rules && !player_present && + !birth_waiting && host.queued_mutations().empty() && + std::ranges::any_of(state->lives(), [](const auto& life) { + return life.account == server::LifeAccount{}; + }); + if (status.ready && state && want_population != host.population_mode()) { + [[maybe_unused]] const auto changed = host.set_population_mode(want_population); + state = host.state(); + } const bool needed = state && (player_present || birth_waiting || server::cellular_unattended_tick_needed(*state)); const auto monotonic_us = [] { @@ -1076,7 +1094,29 @@ int serve_cellular(Options& o) { return finish(4); } if (status.ready && state && needed) { - if (paced->due && state->tick() != UINT64_MAX) { + if (host.population_mode()) { + // Collect one second of due time before an unattended publication. + // Overload services at most 64 fixed steps per bounded batch. Debt + // remains visible and is consumed only by a confirmed head. + if (paced->due && paced->debt >= 20) { + const auto& head = host.committed()->head; + const server::CellularBatchHeader header{ + o.host.key.world, + o.host.key.region, + o.host.frame, + head.commit, + head.digest, + head.end_tick, + head.end_tick + std::min(paced->debt, 64)}; + const auto inputs = server::unattended_population_inputs(*state, header); + const auto end = header.end_tick; + if (inputs) { + [[maybe_unused]] const auto published = host.publish(*inputs); + } + if (!host.state() || host.state()->tick() < end) + schedule.defer(monotonic_us()); + } + } else if (paced->due && state->tick() != UINT64_MAX) { const auto tick = state->tick() + 1; std::vector baseline; baseline.reserve(state->lives().size()); @@ -1201,7 +1241,7 @@ int main(int argc, char** argv) { "[--m10-handoff-key F [--m10-neighbour " "REGION=X0,Y0,X1,Y1=HOST:PORT=NAME=PUBFILE]...] " "[--life-route REGION=HOST:PORT=TLS_NAME ...] " - "[--m10-load-governor on|off]]\\n", + "[--m10-load-governor on|off]] [--cellular-populations on|off]\\n", stderr); return 2; } diff --git a/docs/STATUS.md b/docs/STATUS.md index 93b36db..916228c 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1,5 +1,20 @@ # Project status and next steps +## M12.3b durable exact populations - 2026-10-17 + +ADR-0142 and schema 44 add dedicated population profile 5, trace 1, immutable +publication/transition evidence and exact source/target recovery. The owning +cellular host recovers either representation, materializes before player +operations and serializes mutation reservations against profile entry. The +optional `--cellular-populations on` path batches unattended work; entry is off +by default. No biological version, golden, deployment or live region changed. + +[Scope and evidence](engineering/M12_DURABLE_POPULATIONS.md) distinguish portable +contracts, real PostgreSQL ambiguity/process-loss tests and host recreation from +remaining executable/network fault drills and target-host qualification. Durable +interval admission/backlog is the next M12.4 segment. The dependent ecological, +multi-day and returning-player gates remain open; M11.9 stays closed for now. + ## M12.4a current-worker unattended scheduling - 2026-10-17 The cellular worker now keeps ticking for funded wildlife after players suspend, diff --git a/docs/adr/0142-durable-exact-population-profile.md b/docs/adr/0142-durable-exact-population-profile.md new file mode 100644 index 0000000..f2dbc56 --- /dev/null +++ b/docs/adr/0142-durable-exact-population-profile.md @@ -0,0 +1,86 @@ +# ADR-0142: Durable exact population profile and publication + +## Status + +Accepted for M12's opt-in engineering implementation, 2026-10-17. Extends +ADR-0139/0140 with a dedicated durable profile. Production enabling and target-host +qualification remain separate; no deployment or World0 operation is authorized. + +## Context + +The exact cohort envelope carries membership and policy beyond raw cellular +bytes. Reusing profile 4 would make retained manifests ambiguous. Portable +restoration alone cannot prove atomic publication, current authority or recovery +after a lost database reply. + +## Decision + +Checkpoint profile 5 contains population envelope 1 and nested cellular bundle 9, +with history profile 5 identifying exact retained membership. Profile 4 retains +its raw cellular bytes and meaning. No biological version or golden is replaced. + +Use explicit fenced, no-tick profile transitions: rules-8 profile 4 to population +5 wraps the same biology with empty membership, sequence zero and a persisted +policy; population 5 to profile 4 materializes the retained individuals by +unwrapping the identical biology. Both require the exact current predecessor, +current region fence, retained build and immutable transition evidence. Recovery +checks the complete source/target pair, not just their aggregate inventory. +Conversion sequence starts at zero for each entry generation; the region commit +sequence remains monotonic and the transition records identify every generation. + +Initially profile 5 admits unattended ecological batches only: controlled cells +receive frozen intentions; player births, mutation selections and session action +receipts remain profile-4 operations. A host must finish the reverse transition +before processing those operations. This restriction avoids bypassing their +existing account-control/reservation transactions. It does not freeze wildlife +or erase controlled/suspended stock. An explicit profile selector is opt-in. +The cellular worker accepts `--cellular-populations on|off` (default off). It may +enter only for unattended rules-8 wildlife, with no waiting player birth or +mutation. Session demand or a player operation completes the reverse transition +first. SQL serializes profile entry and mutation reservation under the same +region lease: entry refuses outstanding choices, and new choices require profile +4 in the current host. Historical reservation contracts remain unchanged outside +profile 5. An existing profile-5 head remains recoverable with entry disabled. +The worker collects at least 20 due ticks (one second) and services at most 64 +per unattended publication, consuming local debt only on confirmation. This is +bounded publication batching; durable interval admission/restart backlog is the +next M12.4 segment, not supplied by a wall-clock inference on recovery. + +Population batch format 1 records the unchanged canonical cellular intentions +and an exact membership/sequence boundary for each tick. Biological replay reads +the recorded intentions, never reruns the wildlife controller. It verifies each +recorded boundary against the persisted deterministic policy. Manual conversion +records operate on an entire wildlife genome group at the same biological tick. +No random materialization sample exists in this exact model. Restoration installs +the recorded checkpoint; validation cannot resample a different member set. + +Store immutable payload and trace objects with SHA-256/read-back verification, +then commit the manifest and publication evidence in one database transaction +under the existing serialized lease/head guard. Unknown replies retain the exact +prepared request; read-only resolution never grants a stale writer permission to +continue. Dedicated evidence triggers prohibit generic publication bypasses. +Retain all old formats, histories and source checkpoints. Do not prune objects. + +## Validation and limits + +Require malformed/profile-confusion rejection, whole-state source/target equality, +conversion oscillation, trace/payload mismatch refusal, frozen controlled biology, +duplicate/conflicting retries, expired writers and missing/corrupt objects. Exercise +real PostgreSQL and process death before publication and after commit/receipt. +Portable codec success alone does not close the durable gate. + +This profile supplies a place for bounded unattended intervals and later escrow; +it does not itself implement biological migration or environmental coupling. +Existing player actions, NPC heredity and fixed 50 ms biology retain their rules. + +## Alternatives, costs and consequences + +Relabeling the raw profile breaks retained recovery; persisting only aggregate +stock loses exact members. Reconstructing policy on restart can choose a different +representation. Keep explicit profiles and recorded boundaries instead. + +This retains every member and full trace, so it adds durable bytes and replay +work and makes no compression claim. SQL parsers, grants and source/target +verification must evolve together. Target-host cost and executable/network fault +qualification remain risks before production enablement. All old objects and +histories remain retained; no garbage collection or compatibility reset follows. diff --git a/docs/architecture/REGION_DURABILITY.md b/docs/architecture/REGION_DURABILITY.md index d537b27..932524e 100644 --- a/docs/architecture/REGION_DURABILITY.md +++ b/docs/architecture/REGION_DURABILITY.md @@ -31,6 +31,15 @@ the narrower environmental profile. M7.1 completes the audit, not those codecs. ## Identity, authority and watermarks +ADR-0142 adds opt-in exact population profile 5 (envelope 1, nested cellular +bundle 9, history profile 5). Raw cellular profile 4 is unchanged. Fenced no-tick +transitions retain complete biology and source/target digests; normal population +publication binds recorded intentions and exact membership boundaries. Generic +publishers cannot bypass dedicated immutable evidence. The host has one owning +state, materializes before player operations and never treats read-only commit +resolution as current authority. See [the contract](../../schemas/cellular-population-batch.md) +and [qualification limits](../engineering/M12_DURABLE_POPULATIONS.md). + World/region/frame are nonzero durable IDs. No ID is a Pod name. The host records its unique holder ID and monotonically increasing nonzero lease epoch. Exhaustion of IDs, epochs, commit sequences or ticks fails closed; never wrap/reuse them. diff --git a/docs/engineering/M12_DELIVERY_PLAN.md b/docs/engineering/M12_DELIVERY_PLAN.md index 198858d..cad1837 100644 --- a/docs/engineering/M12_DELIVERY_PLAN.md +++ b/docs/engineering/M12_DELIVERY_PLAN.md @@ -10,7 +10,7 @@ Commit and push each validated scoped segment separately. |---|---|---| | 12.1 | Engineering evidence policy and explicit deferrals documented | ADR-0133, identity/time contract and worked histories; no new game rules or runtime versions. | | 12.2 | Exact local model selected and paired qualification implemented (12.2d); production cost gate open | ADR-0139, exact canonical equality across 196,608 paired ticks per platform. Regional approximation disabled; no compression claim. Deployment-host capacity and fuller interaction coverage remain open. | -| 12.3 | Portable exact conversion closure, automatic policy and process recovery implemented | ADR-0140, population envelope 1. Dedicated PostgreSQL profile/trace/publication adapter and fault injection remain required before enabling the live selector. | +| 12.3 | Exact population profile/trace/store and opt-in host integration implemented (12.3b) | ADR-0142, schema 44; exact publication/recovery, mutation race guards, ambiguous replies and process kills. Executable/network population fault drills and target-host qualification remain open. | | 12.4 | Current-worker unattended demand and confirmed-head pacing implemented (12.4a) | ADR-0141. Population durable intervals and biological migration escrow remain unimplemented. | | 12.5 | Blocked on 12.2-12.4 | Finite M6 process fixtures do not qualify coupled persistent cellular/cohort atmosphere/water budgets. | | 12.6 | Existing local archive retained; integrated population history open | World-level durable event/query/client path remains required. | diff --git a/docs/engineering/M12_DURABLE_POPULATIONS.md b/docs/engineering/M12_DURABLE_POPULATIONS.md new file mode 100644 index 0000000..ef55012 --- /dev/null +++ b/docs/engineering/M12_DURABLE_POPULATIONS.md @@ -0,0 +1,68 @@ +# M12.3b durable exact populations + +ADR-0142 implements the persistence boundary identified in the remaining-gate +audit. This is an opt-in engineering path; no live region was migrated or enabled. +The optional cellular worker accepts `--cellular-populations on|off`, default off. +When unattended rules-8 wildlife is present, entry can wrap its exact biological +state as profile 5. A returning player, queued birth or mutation requires the +durable reverse transition before processing player operations. + +## State and publication + +There is exactly one owning biological store in the host. Profile 4 owns raw +CellularLives; profile 5 owns a CellularPopulation containing that store and its +exact membership tags. Common callers receive only a const biological view. +Entering, leaving and oscillating cannot heal, move, reproduce or fund a life. +Every profile-5 generation starts membership sequence zero, while the immutable +region publication sequence and source/target evidence preserve its identity. + +Migration 44 retains all old parsers, histories and raw cellular formats. It adds +dedicated immutable population publication and profile-transition records, fenced +publish/resolve/load functions and generic-publication bypass guards. Entry and +new mutation reservations serialize on the same region lease. Entry refuses +outstanding player choices; profile 5 refuses new choices. Existing unrelated +reservation contracts retain their original behavior. + +Payloads and traces are immutable content-addressed objects. The publisher checks +their SHA-256 and complete read-back before the atomic SQL head/evidence update. +Recovery verifies the exact predecessor and full source/target bytes. Biological +traces record intentions and exact membership boundaries; replay does not rerun +the wildlife controller. Unknown replies stop publication until exact-byte +resolution/retry succeeds under current authority. + +## Evidence and qualification limits + +The portable contract covers canonical codec/manifest/profile confusion, sampled +truncations, appended bytes, membership/sequence tampering, controlled-input +refusal, manual conversions and full biological replay equality. Native Release +tests run under NDEBUG; sanitizer and static-analysis evidence is recorded in +[VALIDATION](VALIDATION.md). + +The private PostgreSQL fixture creates real reserved founders and waits for a +funded player-derived descendant. It exercises durable membership oscillation, +fresh-connection recovery, reverse materialization, generic bypass rejection, +mutation/profile race guards, duplicates/conflicts and stale authority. Missing +and corrupt payloads refuse recovery. Lease-row contention creates real adapter +timeouts: one backend commits after its client disconnects; another is terminated +before commit. Both resolve/retry the retained exact request. + +SIGKILL tests use forked publisher processes at the pre-transaction boundary +(after immutable uploads), after SQL commit and after a pipe acknowledgement. +The parent restores through a new database connection and compares every retained +byte. These are real process kills, but the pipe acknowledgement is a fixture +signal, not an actual network client acknowledgement or an independently exec'd +worker restart. The actual CellularRegionHost is separately drained/recreated +and restores a profile-5 head with entry disabled, then materializes before a +returning-player operation. Previous raw worker process drills remain regression +coverage, not additional population-specific fault evidence. + +The worker collects one second of due unattended time before publication and +services at most 64 fixed steps per batch. The portable input builder matches an +independent per-tick fixture. This removes per-tick SQL publication from the new +population path; it does not yet make local pacing debt a durable admission queue. + +Still open: profile-5 executable/network fault drills, deployment-host tail costs, +durable ecological interval admission and biological migration escrow, jointly +owned environmental budgets, world-level history/extinction reconciliation and +actual multi-day/returning-player evidence. This work changes neither biological +versions nor goldens, and does not establish useful compression or M13 readiness. diff --git a/docs/engineering/M12_REMAINING_GATES.md b/docs/engineering/M12_REMAINING_GATES.md index a3c8bae..a94ad02 100644 --- a/docs/engineering/M12_REMAINING_GATES.md +++ b/docs/engineering/M12_REMAINING_GATES.md @@ -5,31 +5,24 @@ all of M12. M11.9 remains closed for now. M12.2 decisions are delegated and reco in ADR-0139; no additional product approval is inferred as a coding prerequisite. Completed scoped commits do not mean their entire roadmap gates have passed. -## Persistence boundary before expansion +## Persistence boundary resolved by 12.3b -The new exact population envelope cannot be labeled as the existing profile-4 -CellularLives payload. The live publisher/recovery path, immutable manifest, -database validators and trace currently agree on that raw cellular format. A -population checkpoint containing ownership tags and policy needs a dedicated -profile and versioned transition/publication adapter. Reinterpreting profile 4 -would violate REGION_DURABILITY and break retained recovery evidence. - -Resolution: add an opt-in population profile and an explicit no-tick migration -from the accepted rules-8 cellular state, retaining identities, archive, funding -and balances. Publish its complete immutable payload, bound trace and permanent -events under the current region fence. Replay must retain recorded intentions -and exact membership samples, rather than rerunning a decision after an uncertain -commit. Verify lost replies and pre-upload/pre-transaction/post-commit/post-ACK -process death against a private PostgreSQL fixture. Keep production disabled until -these and target-host budgets pass. A portable codec is not a substitute for this -adapter. This is remaining M12.3 implementation, not a user-permission hold. +ADR-0142 adds dedicated profile 5, population trace 1 and schema 44; raw profile 4 +retains its meaning. Explicit no-tick migrations preserve the complete biology. +The publisher, replay verifier, immutable SQL evidence, recovery and owning host +now support exact population membership with entry disabled by default. Private +PostgreSQL tests cover ambiguity, fencing, profile races and publisher process +kills. [Scope and limits](M12_DURABLE_POPULATIONS.md) distinguish those tests from +executable/network fault drills and deployment-host qualification still required. +This resolves the architectural profile conflict and permits the next durable +interval work. It does not close all of M12.3 or authorize production enablement. ## Dependent ecological integration | Gate | Existing useful basis | Work/evidence still required | |---|---|---| | 12.2 | Exact cellular model and paired matrix; local samples through 256 lives | Deployment-host tail costs including publication; larger wildlife cohorts, predation/twins and contention coverage; useful compression remains unimplemented. | -| 12.3 | Complete population codec, exact conversion candidates, automatic policy and fresh-process continuation | Dedicated durable profile/trace/store/host integration and fault-injected atomic publication described above. | +| 12.3 | Dedicated population profile/trace/store/owning host; exact recovery and private PostgreSQL fault tests | Population-specific executable/network fault drills and target-host publication/capacity qualification. | | 12.4 | Current worker advances unattended wildlife and preserves running scheduling debt | Durable ecological interval admission/backlog recovery; exact biological source/destination escrow, partition/duplicate settlement and stock-in-transit audits. | | 12.5 | Finite cellular contact feeding/decay and separate M6 producer/consumer resource owner | One jointly owned cellular/environment budget, explicit effective depth and source entitlements, versioned water/atmosphere species mapping and producer coefficients, plus closed/open and nutrient-exhaustion comparisons. | | 12.6 | Retained genome/organism archive with ancestry and common-ancestor queries | Append-only population census/history events in the same durable commit, bounded world history queries/indexing and client presentation; no account power or invented precise statistical ancestry. | @@ -46,9 +39,9 @@ and photosynthesis steps B/C do not silently become accepted M12 mechanics. The engineering constitution requires stopping expansion at this architectural boundary and documenting the resolution. That does not prevent the independent -worker scheduling correction, which uses the existing raw cellular publication -path. The outstanding population integration is the next substantive coding -segment; none of the rows above is closed by adding a placeholder service. +worker scheduling correction or the dedicated exact population adapter above. +Durable ecological intervals are the next substantive coding segment; none of +the rows above is closed by adding a placeholder service. Current expansion readiness: **no-go for M13 integration or World0 promotion** while these M12 integration/evidence gates remain open. This records the present diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index fb97bf1..e8e35c1 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -1,5 +1,51 @@ # Validation report +## 2026-10-17 - M12.3b durable exact population publication + +ADR-0142 affects portable lifecycle batches/checkpoint manifests, optional +PostgreSQL adapters/schema 44, the owning cellular host and opt-in worker batching. +No biological version or golden changes; no deployment or World0 operation. + +Executed: + +- MSVC 19.44 Release configure/build and **76/76 CTests** (final run 9.53 s). +- GCC 14.2 actual QUIC/PostgreSQL worker and integration-test build. Final private + `postgres-check.sh --population-only` **passed, exit 0, 5 s** after checked-access + refinements. Real reserved founders fund the wildlife fixture; tests exercise + exact membership oscillation, duplicate/conflicting requests, mutation/profile + race guards, missing/corrupt payloads, stale authority, host recreation and + materialization before player operations. +- Real PostgreSQL lease-row contention makes the adapter time out. One queued + request later commits and another backend is terminated before commit; both + resolve/retry the retained bytes. Forked publisher SIGKILLs occur before the + transaction, after commit and after a pipe ACK, with fresh-connection recovery. + This does not claim a population-specific network ACK or exec'd worker kill. +- Clang 19 actual server build; explicit clang-tidy 19 on changed optional store, + SQL adapter, cellular recovery, host, population integration tests and worker. + Final refinements use separately checked stable optional values; those follow-up + tidy runs pass. Analysis-preset portable checks and repository format check pass. +- Portable analysis CTests pass **7/7** (29.70 s); final batch-builder/checkpoint + checks pass **2/2** (3.73 s). ASan/UBSan population checkpoint/replay checks pass + (final 12.50 s), with no reports. Tests use explicit checks under NDEBUG. + +The broad regression first exposed an over-broad mutation guard and the schema-44 +upgrade expectation; both were corrected. The historical failed-migration drill +now supplies all required checksums, so it reaches its intended bad-v8-checksum +failure rather than an unrelated missing psql variable. The C++ suite, backup and +DR passed in the next run, but a concurrent local relink interrupted the upgrade +drill with `Text file busy`; this is not counted as a passing full run. Final +qualification uses a frozen copy of the executables to avoid that interference. + +Final frozen-binary `postgres-check.sh --full` **passed, exit 0, 496 s**: complete +C++ suite (421 s), independent reservation connections, backup/restore, host-loss +DR, upgrade, migration history, permissions and checksum drift. The final optional +checked-access refinements additionally passed the targeted private population +suite above; all populations remain opt-in and no live region was changed. + +Remaining scope: [M12_DURABLE_POPULATIONS](M12_DURABLE_POPULATIONS.md) and +[M12_REMAINING_GATES](M12_REMAINING_GATES.md). No compression, deployment-host +capacity, durable interval admission or multi-day ecological acceptance is inferred. + ## 2026-10-17 - M12.4a unattended demand and confirmed-head pacing ADR-0141 changes the existing cellular worker's demand/pacing and metrics, with diff --git a/include/evo/persistence/population_checkpoint_store.hpp b/include/evo/persistence/population_checkpoint_store.hpp new file mode 100644 index 0000000..cbf1c27 --- /dev/null +++ b/include/evo/persistence/population_checkpoint_store.hpp @@ -0,0 +1,54 @@ +#pragma once + +#include "evo/persistence/checkpoint_store.hpp" + +namespace evo::persistence { +struct PopulationPreparedCommit { + RegionKey key{}; + Uuid commit{}; + std::vector manifest, trace; // Empty trace means a profile transition. +}; +struct PopulationPublicationFailure { + PublicationError kind{}; + LeaseStoreFailure store{}; + ObjectError object{}; + PopulationPreparedCommit prepared; +}; +struct RecoveredPopulationCheckpoint { + CommittedCheckpoint committed; + CheckpointManifest manifest; + std::unique_ptr state; +}; +// Explicit no-tick transitions, under the same lease/head fence as publication. +[[nodiscard]] std::expected +migrate_cellular_to_population(Postgres&, const FilesystemObjectStore&, const LeaseGrant&, + const CommittedCheckpoint&, Uuid commit, const Sha256Digest& build, + server::CellularTierPolicy, std::chrono::milliseconds timeout, + const PublishHooks& hooks = {}); +[[nodiscard]] std::expected +materialize_population_profile(Postgres&, const FilesystemObjectStore&, const LeaseGrant&, + const CommittedCheckpoint&, Uuid commit, const Sha256Digest& build, + std::chrono::milliseconds timeout, const PublishHooks& hooks = {}); +[[nodiscard]] std::expected +publish_population_batch(Postgres&, const FilesystemObjectStore&, const LeaseGrant&, + const CommittedCheckpoint&, const server::CellularPopulationBatch&, + const server::CellularPopulation& candidate, Uuid commit, + const Sha256Digest& build, std::chrono::milliseconds timeout, + const PublishHooks& hooks = {}); +[[nodiscard]] std::expected +retry_population_prepared(Postgres&, const PopulationPreparedCommit&, + std::chrono::milliseconds timeout); +[[nodiscard]] std::expected +resolve_population_prepared(Postgres&, const PopulationPreparedCommit&, + std::chrono::milliseconds timeout); +[[nodiscard]] std::expected +recover_population_checkpoint(Postgres&, const FilesystemObjectStore&, RegionKey, const LeaseGrant&, + std::span retained_builds, + std::chrono::milliseconds timeout); +// Used by raw cellular recovery after a population -> individual profile transition. +[[nodiscard]] std::expected +verify_population_profile_transition(Postgres&, const FilesystemObjectStore&, const LeaseGrant&, + const CommittedCheckpoint&, const CheckpointManifest&, + std::span payload, + std::chrono::milliseconds timeout); +} // namespace evo::persistence diff --git a/include/evo/persistence/region_checkpoint.hpp b/include/evo/persistence/region_checkpoint.hpp index 4103d31..5cfa193 100644 --- a/include/evo/persistence/region_checkpoint.hpp +++ b/include/evo/persistence/region_checkpoint.hpp @@ -5,6 +5,7 @@ #include "evo/processes/lineage_region.hpp" #include "evo/server/lifecycle/cellular_batch.hpp" #include "evo/server/lifecycle/cellular_lives.hpp" +#include "evo/server/lifecycle/cellular_population_batch.hpp" #include "evo/server/lifecycle/primitive_batch.hpp" #include #include @@ -25,12 +26,14 @@ enum class CheckpointProfile : std::uint8_t { environmental_lineage_v1 = 2, // LineageRegion container; history_profile 1. primitive_lives_v1 = 3, // Funded creation state; history_profile 2. cellular_lives_v1 = 4, // Complete cellular state; history_profile 3. + cellular_population_v1 = 5, // Population envelope 1; nested cellular bundle 9. }; // history_profile 1: lineage archive format 1 with genome schema 1. inline constexpr std::uint32_t lineage_history_profile = 1; // Archive format 1, genome schema 2, development 2, founder-life rules 1. inline constexpr std::uint32_t primitive_history_profile = 2; inline constexpr std::uint32_t cellular_history_profile = 3; +inline constexpr std::uint32_t population_history_profile = 5; inline constexpr std::uint32_t cellular_revision_history_profile = 4; // Declared durability, verified at deployment. Development: one volume, no host-loss claim. enum class DurabilityPolicy : std::uint8_t { unknown = 0, single_volume_development = 1 }; @@ -119,7 +122,9 @@ inline constexpr std::size_t max_lineage_payload_bytes = processes::max_lineage_ : profile == CheckpointProfile::environmental_lineage_v1 ? max_lineage_payload_bytes : profile == CheckpointProfile::primitive_lives_v1 ? server::max_primitive_lives_bytes : profile == CheckpointProfile::cellular_lives_v1 ? server::max_cellular_lives_bytes - : 0; + : profile == CheckpointProfile::cellular_population_v1 + ? server::max_cellular_population_bytes + : 0; } inline constexpr std::uint64_t max_checkpoint_ticks = 64; @@ -189,4 +194,19 @@ restore_primitive_payload(std::span payload, const CheckpointManifest& manifest); [[nodiscard]] std::expected, CheckpointError> restore_cellular_payload(std::span payload, const CheckpointManifest& manifest); +[[nodiscard]] std::expected, CheckpointError> +restore_population_payload(std::span payload, + const CheckpointManifest& manifest); +// Dedicated profile 4/bundle 9 <-> profile 5 transition, no biological tick. +[[nodiscard]] std::expected +population_profile_transition_manifest(const CheckpointManifest& predecessor, + const Sha256Digest& predecessor_digest, Uuid commit, + const LeaseGrant& lease, const Sha256Digest& build, + std::uint64_t payload_bytes, + const Sha256Digest& payload_sha256); +// Population manual conversion may publish at the same tick with a nonempty trace. +[[nodiscard]] std::expected population_successor_manifest( + const CheckpointManifest& predecessor, const Sha256Digest& predecessor_digest, Uuid commit, + const LeaseGrant& lease, std::uint64_t end_tick, const Sha256Digest& build, + std::uint64_t payload_bytes, const Sha256Digest& payload_sha256, const BatchDigests& batch); } // namespace evo::persistence diff --git a/include/evo/persistence/region_lease_store.hpp b/include/evo/persistence/region_lease_store.hpp index f7962dc..9e4e545 100644 --- a/include/evo/persistence/region_lease_store.hpp +++ b/include/evo/persistence/region_lease_store.hpp @@ -217,4 +217,23 @@ pending_region_outbox(Postgres& db, RegionKey key, std::uint32_t limit, [[nodiscard]] std::expected acknowledge_region_outbox(Postgres& db, RegionKey key, const BatchDecision& decision, std::chrono::milliseconds timeout); +struct PopulationEvidence { + std::vector trace; + std::optional source_commit; + Sha256Digest source_manifest_sha256{}, source_payload_sha256{}; +}; +[[nodiscard]] std::expected +publish_population_checkpoint(Postgres&, std::span manifest, + std::span trace, + std::chrono::milliseconds timeout); +[[nodiscard]] std::expected +publish_population_profile_transition(Postgres&, std::span manifest, + std::chrono::milliseconds timeout); +[[nodiscard]] std::expected +resolve_population_commit(Postgres&, RegionKey, const Uuid& commit, + std::span manifest, + std::span trace, std::chrono::milliseconds timeout); +[[nodiscard]] std::expected, LeaseStoreFailure> +load_population_evidence(Postgres&, RegionKey, const LeaseGrant&, const Uuid& commit, + std::chrono::milliseconds timeout); } // namespace evo::persistence diff --git a/include/evo/server/cellular_region_host.hpp b/include/evo/server/cellular_region_host.hpp index 9d28536..a6bdb39 100644 --- a/include/evo/server/cellular_region_host.hpp +++ b/include/evo/server/cellular_region_host.hpp @@ -3,6 +3,7 @@ #include "evo/network/session_protocol.hpp" #include "evo/persistence/checkpoint_store.hpp" #include "evo/persistence/life_reservation_store.hpp" +#include "evo/persistence/population_checkpoint_store.hpp" #include "evo/server/lifecycle/cellular_batch.hpp" #include "evo/server/ops.hpp" @@ -13,6 +14,7 @@ #include #include #include +#include #include namespace evo::server { @@ -24,6 +26,7 @@ struct CellularHostConfig { persistence::LeasePolicy lease{}; persistence::Sha256Digest build{}; std::vector retained_builds; + std::optional population_policy{}; // Opt-in profile-5 entry. }; enum class CellularHostError : std::uint8_t { configuration, @@ -74,7 +77,7 @@ struct CellularMutationReply { [[nodiscard]] network::CellularMutationOffer offer_view(const CellularLives& state, const CellularOwnedLife& life, std::uint64_t tick); -// Single-writer profile-4 authority. The caller supplies a complete bounded trace; +// Single-writer profile-4/5 authority. The caller supplies a complete bounded trace; // the host replays it from its recovered state and advances only on a confirmed head. class CellularRegionHost { public: @@ -87,6 +90,9 @@ class CellularRegionHost { // Acquires/renews authority, recovers fully, or resolves the exact pending commit. const HostStatus& step(); + // Explicit no-tick fenced transition. Uncertain publication pauses the host. + [[nodiscard]] std::expected set_population_mode(bool enabled); + [[nodiscard]] bool population_mode() const noexcept; [[nodiscard]] std::expected publish(const CellularBatch& batch, std::span actions = {}, std::span mutation_actions = {}); @@ -140,6 +146,9 @@ class CellularRegionHost { void upkeep(); void resolve(); void refresh(); + bool recover_state(const persistence::LeaseGrant&); + [[nodiscard]] std::expected + publish_population(const CellularBatch&); void finalize_births(std::span births); void queue_birth(const CellularTickIntents::Birth& birth); void reload_mutations(); @@ -158,7 +167,46 @@ class CellularRegionHost { std::unique_ptr objects_; std::unique_ptr db_; persistence::LeaseAuthority authority_; - std::unique_ptr state_; + // Exactly one owner. The common interface is a const biological view, never + // a second independently spendable cellular copy alongside a population. + struct OwnedState { + std::unique_ptr individual; + std::unique_ptr population; + OwnedState() = default; + OwnedState(std::unique_ptr state) : individual(std::move(state)) {} + OwnedState(std::unique_ptr state) : population(std::move(state)) {} + OwnedState& operator=(std::unique_ptr state) { + population.reset(); + individual = std::move(state); + return *this; + } + OwnedState& operator=(std::unique_ptr state) { + individual.reset(); + population = std::move(state); + return *this; + } + void reset() { + individual.reset(); + population.reset(); + } + const CellularLives* get() const { + return population ? &population->state() : individual.get(); + } + const CellularLives* operator->() const { + return get(); + } + const CellularLives& operator*() const { + return *get(); + } + explicit operator bool() const { + return get() != nullptr; + } + }; + [[nodiscard]] std::expected + accept_population_publication( + std::expected, + OwnedState candidate); + OwnedState state_; std::optional committed_; std::unique_ptr pending_; std::vector finalized_actions_; diff --git a/include/evo/server/lifecycle/cellular_population.hpp b/include/evo/server/lifecycle/cellular_population.hpp index 2558e4e..d327137 100644 --- a/include/evo/server/lifecycle/cellular_population.hpp +++ b/include/evo/server/lifecycle/cellular_population.hpp @@ -10,6 +10,7 @@ #include namespace evo::server { +struct CellularPopulationBatch; // ADR-0139: exact local cohorts retain individuals, never mean reserves or samples. // The single owned CellularLives store backs BOTH representations, exactly once. struct CellularCohortMember { @@ -72,6 +73,8 @@ class CellularPopulation { std::span selections = {}); private: + friend std::expected, CellularPopulationError> + execute_population_batch(const CellularPopulation&, CellularPopulationBatch&, bool); explicit CellularPopulation(std::unique_ptr state, CellularTierPolicy policy); void apply_policy(); std::unique_ptr state_; diff --git a/include/evo/server/lifecycle/cellular_population_batch.hpp b/include/evo/server/lifecycle/cellular_population_batch.hpp new file mode 100644 index 0000000..38f85f1 --- /dev/null +++ b/include/evo/server/lifecycle/cellular_population_batch.hpp @@ -0,0 +1,49 @@ +#pragma once + +#include "evo/server/lifecycle/cellular_batch.hpp" +#include "evo/server/lifecycle/cellular_population.hpp" + +namespace evo::server { +struct CellularPopulationBoundary { + std::uint64_t sequence{}; + std::vector members; + bool operator==(const CellularPopulationBoundary&) const = default; +}; +struct CellularPopulationConversion { + std::uint64_t genome{}; + bool to_cohort{}; + bool operator==(const CellularPopulationConversion&) const = default; +}; +struct CellularPopulationBatch { + CellularBatchHeader header; + // Exactly one: 1..64 unattended biological ticks, or a no-tick whole-group conversion. + CellularBatch biology; + std::optional conversion; + std::vector boundaries; + bool operator==(const CellularPopulationBatch&) const = default; +}; +inline constexpr std::size_t population_batch_header_bytes = 136; +inline constexpr std::size_t max_population_batch_bytes = + population_batch_header_bytes + max_cellular_batch_bytes + + max_cellular_batch_ticks * (16 + 16 * biology::max_cellular_tick_lives); +struct CellularPopulationExecution { + CellularPopulationBatch batch; + std::unique_ptr state; +}; +// Build an unattended interval from a detached biological copy. Controlled cells +// receive frozen-only intentions; the unchanged wildlife controller supplies the +// rest. The returned recorded intentions replay without rerunning that controller. +[[nodiscard]] std::expected +unattended_population_inputs(const CellularLives&, const CellularBatchHeader&); +[[nodiscard]] std::expected +prepare_population_batch(const CellularPopulation& base, CellularBatch biology); +[[nodiscard]] std::expected +prepare_population_conversion(const CellularPopulation& base, CellularBatchHeader header, + CellularPopulationConversion conversion); +[[nodiscard]] std::expected, CellularPopulationError> +replay_population_batch(const CellularPopulation& base, const CellularPopulationBatch& batch); +[[nodiscard]] std::expected, CellularPopulationError> +encode_population_batch(const CellularPopulationBatch& batch); +[[nodiscard]] std::expected +decode_population_batch(std::span bytes); +} // namespace evo::server diff --git a/migrations/0044_exact_population_publication.sql b/migrations/0044_exact_population_publication.sql new file mode 100644 index 0000000..17f3f60 --- /dev/null +++ b/migrations/0044_exact_population_publication.sql @@ -0,0 +1,338 @@ +-- ADR-0142: exact population profile 5, immutable evidence and fenced publishers. +-- All historical profile-4 bytes and parsers retain their original meanings. +ALTER FUNCTION evolution.parse_checkpoint_manifest(bytea) RENAME TO parse_checkpoint_manifest_v43; +CREATE FUNCTION evolution.parse_checkpoint_manifest(m bytea) +RETURNS evolution.checkpoint_manifest LANGUAGE plpgsql IMMUTABLE +SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + r evolution.checkpoint_manifest; + n bytea; + tick_begin numeric; + tick_end numeric; + payload_size numeric; + trace_size numeric; + trace_digest bytea; + zero32 constant bytea := decode(repeat('00',32),'hex'); + bundle9 constant bytea := '\x080000000600000002000000010000000100000004000000030000000200000001000000'; +BEGIN + IF m IS NULL OR octet_length(m) <> 360 OR evolution.manifest_uint(m,12,4) <> 5 THEN + RETURN evolution.parse_checkpoint_manifest_v43(m); + END IF; + tick_begin := evolution.manifest_uint(m,112,8); + tick_end := evolution.manifest_uint(m,120,8); + payload_size := evolution.manifest_uint(m,240,8); + trace_size := evolution.manifest_uint(m,280,8); + trace_digest := substring(m FROM 289 FOR 32); + IF substring(m FROM 161 FOR 36) <> bundle9 OR evolution.manifest_uint(m,232,4) <> 5 + OR evolution.manifest_uint(m,228,4) <> 0 OR evolution.manifest_uint(m,236,4) <> 1 + OR evolution.manifest_uint(m,56,8) = 0 OR tick_end < tick_begin OR tick_end - tick_begin > 64 + OR payload_size = 0 OR payload_size > 16781400 OR evolution.manifest_uint(m,320,4) <> 0 + OR substring(m FROM 329 FOR 32) <> zero32 + OR (trace_size = 0 AND (trace_digest <> zero32 OR tick_begin <> tick_end OR tick_end = 0)) + OR (trace_size > 0 AND (trace_size < 152 OR trace_size > 1968376 OR trace_size % 8 <> 0 + OR trace_digest = zero32)) THEN + RAISE EXCEPTION 'unsupported or malformed population manifest' USING ERRCODE = '22023'; + END IF; + -- Reuse historical common-field validation, with only parser-local normalized + -- profile/history/size/trace declarations. Stored and returned bytes stay profile 5. + n := overlay(m PLACING '\x04000000'::bytea FROM 13 FOR 4); + n := overlay(n PLACING '\x04000000'::bytea FROM 233 FOR 4); + n := overlay(n PLACING evolution.manifest_u64le(least(payload_size,16777216)) FROM 241 FOR 8); + n := overlay(n PLACING evolution.manifest_u64le(CASE WHEN tick_end = tick_begin THEN 0 + ELSE 112 + 16 * (tick_end - tick_begin) END) FROM 281 FOR 8); + IF tick_end = tick_begin THEN + n := overlay(n PLACING zero32 FROM 289 FOR 32); + END IF; + r := evolution.parse_checkpoint_manifest_v43(n); + r.profile := 5; + r.payload_bytes := payload_size; + r.trace_bytes := trace_size; + r.trace_sha256 := trace_digest; + RETURN r; +END $$; +REVOKE ALL ON FUNCTION evolution.parse_checkpoint_manifest(bytea) FROM PUBLIC; + +CREATE TABLE evolution.population_publications ( + commit_id uuid PRIMARY KEY REFERENCES evolution.region_commits(commit_id) DEFERRABLE INITIALLY DEFERRED, + trace bytea NOT NULL CHECK (octet_length(trace) BETWEEN 152 AND 1968376) +); +CREATE TABLE evolution.population_profile_transitions ( + target_commit uuid PRIMARY KEY REFERENCES evolution.region_commits(commit_id) DEFERRABLE INITIALLY DEFERRED, + source_commit uuid NOT NULL REFERENCES evolution.region_commits(commit_id), + source_manifest_sha256 bytea NOT NULL CHECK (octet_length(source_manifest_sha256) = 32), + source_payload_sha256 bytea NOT NULL CHECK (octet_length(source_payload_sha256) = 32), + source_profile integer NOT NULL CHECK (source_profile IN (4,5)), + target_profile integer NOT NULL CHECK (target_profile IN (4,5)), + CHECK (source_profile <> target_profile AND source_commit <> target_commit) +); +REVOKE ALL ON evolution.population_publications, evolution.population_profile_transitions FROM PUBLIC; +CREATE TRIGGER population_publications_immutable BEFORE UPDATE OR DELETE ON evolution.population_publications + FOR EACH ROW EXECUTE FUNCTION evolution.refuse_history_rewrite(); +CREATE TRIGGER population_publications_no_truncate BEFORE TRUNCATE ON evolution.population_publications + FOR EACH STATEMENT EXECUTE FUNCTION evolution.refuse_history_rewrite(); +CREATE TRIGGER population_transitions_immutable BEFORE UPDATE OR DELETE ON evolution.population_profile_transitions + FOR EACH ROW EXECUTE FUNCTION evolution.refuse_history_rewrite(); +CREATE TRIGGER population_transitions_no_truncate BEFORE TRUNCATE ON evolution.population_profile_transitions + FOR EACH STATEMENT EXECUTE FUNCTION evolution.refuse_history_rewrite(); + +CREATE OR REPLACE FUNCTION evolution.refuse_unpublished_cellular() RETURNS trigger +LANGUAGE plpgsql SECURITY INVOKER SET search_path = pg_catalog, pg_temp AS $$ +BEGIN + IF evolution.manifest_uint(NEW.manifest,12,4) = 4 AND NEW.commit_sequence > 0 + AND NOT EXISTS (SELECT 1 FROM evolution.cellular_publications p WHERE p.commit_id = NEW.commit_id) + AND NOT EXISTS (SELECT 1 FROM evolution.cellular_profile_migrations p WHERE p.target_commit = NEW.commit_id) + AND NOT EXISTS (SELECT 1 FROM evolution.population_profile_transitions p + WHERE p.target_commit = NEW.commit_id AND p.target_profile = 4) THEN + RAISE EXCEPTION 'cellular commit requires fenced publisher' USING ERRCODE = 'EVL06'; + END IF; + RETURN NEW; +END $$; +CREATE FUNCTION evolution.refuse_unpublished_population() RETURNS trigger +LANGUAGE plpgsql SECURITY INVOKER SET search_path = pg_catalog, pg_temp AS $$ +BEGIN + IF evolution.manifest_uint(NEW.manifest,12,4) = 5 AND + (NEW.commit_sequence = 0 OR + (NOT EXISTS (SELECT 1 FROM evolution.population_publications p WHERE p.commit_id = NEW.commit_id) + AND NOT EXISTS (SELECT 1 FROM evolution.population_profile_transitions p + WHERE p.target_commit = NEW.commit_id AND p.target_profile = 5))) THEN + RAISE EXCEPTION 'population commit requires fenced publisher' USING ERRCODE = 'EVL06'; + END IF; + RETURN NEW; +END $$; +REVOKE ALL ON FUNCTION evolution.refuse_unpublished_population() FROM PUBLIC; +CREATE TRIGGER region_population_evidence BEFORE INSERT ON evolution.region_commits + FOR EACH ROW EXECUTE FUNCTION evolution.refuse_unpublished_population(); + +CREATE OR REPLACE FUNCTION evolution.publish_region_checkpoint(p_manifest bytea,p_decisions bytea) +RETURNS TABLE(region_frame evolution.u64,head_sequence evolution.u64,head_commit uuid, + head_digest text,head_tick evolution.u64) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE c evolution.checkpoint_manifest := evolution.parse_checkpoint_manifest(p_manifest); +BEGIN + IF c.profile IN (4,5) THEN + RAISE EXCEPTION 'cellular/population requires dedicated publisher' USING ERRCODE = 'EVL06'; + END IF; + RETURN QUERY SELECT * FROM evolution.publish_region_checkpoint_core(p_manifest,p_decisions); +END $$; + +-- Structural validation only. C++ additionally replays all biology and verifies +-- the exact policy/sample, including controlled presence and source state closure. +CREATE FUNCTION evolution.check_population_trace(m bytea,t bytea) RETURNS void +LANGUAGE plpgsql IMMUTABLE SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + c evolution.checkpoint_manifest := evolution.parse_checkpoint_manifest(m); + kind numeric; n numeric; biological numeric; genome numeric; to_cohort numeric; + off integer; bio_end integer; intents numeric; members numeric; previous numeric; + sequence numeric; last_sequence numeric := -1; life numeric; +BEGIN + IF c.profile <> 5 OR t IS NULL OR octet_length(t) <> c.trace_bytes OR sha256(t) <> c.trace_sha256 + OR octet_length(t) < 152 OR substring(t FROM 1 FOR 8) <> '\x4145564f50425431' + OR evolution.manifest_uint(t,8,4) <> 1 OR evolution.manifest_uint(t,12,4) > 1 + OR substring(t FROM 17 FOR 88) <> (substring(m FROM 17 FOR 24) || + substring(m FROM 65 FOR 48) || substring(m FROM 113 FOR 16)) + OR evolution.manifest_uint(t,108,4) <> 0 THEN + RAISE EXCEPTION 'invalid population trace binding' USING ERRCODE = '22023'; + END IF; + kind := evolution.manifest_uint(t,12,4); n := evolution.manifest_uint(t,104,4); + biological := evolution.manifest_uint(t,112,8); genome := evolution.manifest_uint(t,120,8); + to_cohort := evolution.manifest_uint(t,128,8); + IF n < 1 OR n > 64 OR biological > 1705072 OR biological > octet_length(t) - 136 + OR to_cohort > 1 OR (kind = 1 AND (biological <> 0 OR genome = 0 OR n <> 1 + OR c.begin_tick <> c.end_tick OR c.end_tick = 0)) + OR (kind = 0 AND (genome <> 0 OR to_cohort <> 0 OR biological < 112 + OR c.end_tick - c.begin_tick <> n)) THEN + RAISE EXCEPTION 'invalid population trace shape' USING ERRCODE = '22023'; + END IF; + off := 136; + IF kind = 0 THEN + IF substring(t FROM 137 FOR 8) <> '\x45564c4342543031' + OR evolution.manifest_uint(t,144,4) <> 3 OR evolution.manifest_uint(t,148,4) <> 4 + OR substring(t FROM 153 FOR 88) <> substring(t FROM 17 FOR 88) + OR evolution.manifest_uint(t,240,4) <> n OR evolution.manifest_uint(t,244,4) <> 0 THEN + RAISE EXCEPTION 'invalid population biological binding' USING ERRCODE = '22023'; + END IF; + off := 248; bio_end := (136 + biological)::integer; + FOR i IN 1..n::integer LOOP + IF off + 16 > bio_end THEN RAISE EXCEPTION 'truncated population tick' USING ERRCODE = '22023'; END IF; + intents := evolution.manifest_uint(t,off,4); + IF intents > 256 OR evolution.manifest_uint(t,off+4,4) <> 0 + OR evolution.manifest_uint(t,off+8,4) <> 0 OR evolution.manifest_uint(t,off+12,4) <> 0 + OR off + 16 + intents * 24 > bio_end THEN + RAISE EXCEPTION 'unsupported population player operation' USING ERRCODE = '22023'; + END IF; + off := off + 16; previous := 0; + FOR j IN 1..intents::integer LOOP + life := evolution.manifest_uint(t,off,8); + IF life <= previous THEN RAISE EXCEPTION 'unordered population inputs' USING ERRCODE = '22023'; END IF; + previous := life; off := off + 24; + END LOOP; + END LOOP; + IF off <> bio_end THEN RAISE EXCEPTION 'population biological length differs' USING ERRCODE = '22023'; END IF; + END IF; + FOR i IN 1..n::integer LOOP + IF off + 16 > octet_length(t) THEN RAISE EXCEPTION 'truncated population sample' USING ERRCODE = '22023'; END IF; + sequence := evolution.manifest_uint(t,off,8); members := evolution.manifest_uint(t,off+8,8); + IF members > 256 OR off + 16 + members * 16 > octet_length(t) + OR (last_sequence >= 0 AND (sequence < last_sequence OR sequence > last_sequence + 1)) THEN + RAISE EXCEPTION 'invalid population sample' USING ERRCODE = '22023'; + END IF; + last_sequence := sequence; off := off + 16; previous := 0; + FOR j IN 1..members::integer LOOP + life := evolution.manifest_uint(t,off,8); + IF life <= previous OR evolution.manifest_uint(t,off+8,8) = 0 THEN + RAISE EXCEPTION 'invalid population member' USING ERRCODE = '22023'; + END IF; + previous := life; off := off + 16; + END LOOP; + END LOOP; + IF off <> octet_length(t) THEN RAISE EXCEPTION 'trailing population bytes' USING ERRCODE = '22023'; END IF; +END $$; +REVOKE ALL ON FUNCTION evolution.check_population_trace(bytea,bytea) FROM PUBLIC; + +CREATE FUNCTION evolution.publish_population_checkpoint(m bytea,t bytea) +RETURNS TABLE(region_frame evolution.u64,head_sequence evolution.u64,head_commit uuid, + head_digest text,head_tick evolution.u64) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE c evolution.checkpoint_manifest := evolution.parse_checkpoint_manifest(m); existing bytea; +BEGIN + PERFORM evolution.check_population_trace(m,t); + -- Lease then head lock follows the common writer's order. Trace evidence is in + -- this same transaction; any core rejection rolls the insertion back. + PERFORM evolution.check_region_lease(c.world_id,c.region_id,c.holder,c.epoch); + SELECT p.trace INTO existing FROM evolution.population_publications p WHERE p.commit_id = c.commit_id; + IF FOUND THEN + IF existing <> t THEN RAISE EXCEPTION 'population identity reused' USING ERRCODE = 'EVL07'; END IF; + ELSE + INSERT INTO evolution.population_publications VALUES(c.commit_id,t); + END IF; + RETURN QUERY SELECT * FROM evolution.publish_region_checkpoint_core(m,''::bytea); +END $$; +REVOKE ALL ON FUNCTION evolution.publish_population_checkpoint(bytea,bytea) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION evolution.publish_population_checkpoint(bytea,bytea) TO :"runtime_role"; + +CREATE FUNCTION evolution.publish_population_profile_transition(m bytea) +RETURNS TABLE(region_frame evolution.u64,head_sequence evolution.u64,head_commit uuid, + head_digest text,head_tick evolution.u64) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + c evolution.checkpoint_manifest := evolution.parse_checkpoint_manifest(m); + s evolution.checkpoint_manifest; p evolution.region_commits%ROWTYPE; + h evolution.region_heads%ROWTYPE; old evolution.region_commits%ROWTYPE; + d bytea := sha256(m); frame evolution.u64; + bundle9 constant bytea := '\x080000000600000002000000010000000100000004000000030000000200000001000000'; +BEGIN + IF c.profile NOT IN (4,5) OR c.versions <> bundle9 OR c.commit_sequence = 0 + OR c.begin_tick <> c.end_tick OR c.end_tick = 0 OR c.trace_bytes <> 0 THEN + RAISE EXCEPTION 'invalid population profile transition' USING ERRCODE = '22023'; + END IF; + PERFORM evolution.check_region_lease(c.world_id,c.region_id,c.holder,c.epoch); + SELECT * INTO old FROM evolution.region_commits r WHERE r.commit_id = c.commit_id; + IF FOUND THEN + IF old.manifest <> m OR NOT EXISTS (SELECT 1 FROM evolution.population_profile_transitions x + WHERE x.target_commit = c.commit_id AND x.source_commit = c.predecessor_commit + AND x.source_manifest_sha256 = c.predecessor_digest AND x.target_profile = c.profile) THEN + RAISE EXCEPTION 'population transition identity reused' USING ERRCODE = 'EVL07'; + END IF; + SELECT l.frame_id INTO frame FROM evolution.region_leases l WHERE l.world_id = c.world_id AND l.region_id = c.region_id; + RETURN QUERY SELECT frame,old.commit_sequence,old.commit_id,encode(old.commit_digest,'hex'),old.end_tick; + RETURN; + END IF; + SELECT * INTO h FROM evolution.region_heads r WHERE r.world_id = c.world_id AND r.region_id = c.region_id FOR UPDATE; + IF NOT FOUND OR h.commit_sequence = 18446744073709551615 OR c.commit_sequence <> h.commit_sequence + 1 + OR c.predecessor_commit <> h.commit_id OR c.predecessor_digest <> h.digest OR c.begin_tick <> h.end_tick THEN + RAISE EXCEPTION 'population transition head moved' USING ERRCODE = 'EVL03'; + END IF; + SELECT * INTO p FROM evolution.region_commits r WHERE r.commit_id = h.commit_id; + IF NOT FOUND THEN RAISE EXCEPTION 'missing population source' USING ERRCODE = 'XX000'; END IF; + s := evolution.parse_checkpoint_manifest(p.manifest); + IF c.profile = 5 AND EXISTS (SELECT 1 FROM evolution.cellular_mutation_reservations x + WHERE x.world_id = c.world_id AND x.region_id = c.region_id AND NOT EXISTS + (SELECT 1 FROM evolution.cellular_mutation_action_publications a + WHERE a.world_id = x.world_id AND a.account_id = x.account_id AND a.action_id = x.action_id)) THEN + RAISE EXCEPTION 'pending player mutation blocks population transition' USING ERRCODE = 'EVL18'; + END IF; + IF s.profile NOT IN (4,5) OR s.profile = c.profile OR s.versions <> bundle9 + OR s.frame_id <> c.frame_id OR s.tick_microseconds <> c.tick_microseconds + OR s.end_tick <> c.end_tick OR NOT EXISTS (SELECT 1 FROM evolution.retained_builds b WHERE b.build_identity = c.build_identity) THEN + RAISE EXCEPTION 'unsupported population source' USING ERRCODE = 'EVL06'; + END IF; + INSERT INTO evolution.population_profile_transitions VALUES(c.commit_id,p.commit_id,p.commit_digest,p.payload_sha256,s.profile,c.profile); + INSERT INTO evolution.region_commits(world_id,region_id,commit_sequence,commit_id,commit_digest,manifest, + begin_tick,end_tick,published_epoch,tick_microseconds,versions,build_identity,payload_bytes,payload_sha256) + VALUES(c.world_id,c.region_id,c.commit_sequence,c.commit_id,d,m,c.begin_tick,c.end_tick,c.epoch, + c.tick_microseconds,c.versions,c.build_identity,c.payload_bytes,c.payload_sha256); + UPDATE evolution.region_heads r SET commit_sequence = c.commit_sequence,commit_id = c.commit_id, + digest = d,end_tick = c.end_tick,published_epoch = c.epoch WHERE r.world_id = c.world_id AND r.region_id = c.region_id; + RETURN QUERY SELECT c.frame_id,c.commit_sequence,c.commit_id,encode(d,'hex'),c.end_tick; +END $$; +REVOKE ALL ON FUNCTION evolution.publish_population_profile_transition(bytea) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION evolution.publish_population_profile_transition(bytea) TO :"runtime_role"; + +CREATE FUNCTION evolution.resolve_population_commit(w evolution.u64,r evolution.u64,id uuid,m bytea,t bytea) +RETURNS TABLE(status text,region_frame evolution.u64,head_sequence evolution.u64,head_commit uuid, + head_digest text,head_tick evolution.u64) +LANGUAGE plpgsql STABLE SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE c evolution.checkpoint_manifest := evolution.parse_checkpoint_manifest(m); result record; +BEGIN + IF c.world_id <> w OR c.region_id <> r OR c.commit_id <> id OR t IS NULL + OR c.profile NOT IN (4,5) THEN RAISE EXCEPTION 'invalid population resolution' USING ERRCODE = '22023'; END IF; + IF octet_length(t) > 0 THEN PERFORM evolution.check_population_trace(m,t); + ELSIF c.trace_bytes <> 0 OR c.begin_tick <> c.end_tick THEN + RAISE EXCEPTION 'invalid transition resolution' USING ERRCODE = '22023'; + END IF; + SELECT * INTO result FROM evolution.resolve_region_commit(w,r,id,m,''::bytea); + IF result.status = 'committed' AND + ((octet_length(t) = 0 AND NOT EXISTS (SELECT 1 FROM evolution.population_profile_transitions x WHERE x.target_commit = id)) + OR (octet_length(t) > 0 AND NOT EXISTS (SELECT 1 FROM evolution.population_publications x WHERE x.commit_id = id AND x.trace = t))) THEN + RETURN QUERY SELECT 'conflict'::text,NULL::evolution.u64,NULL::evolution.u64,NULL::uuid,NULL::text,NULL::evolution.u64; + ELSE + RETURN QUERY SELECT result.status,result.region_frame,result.head_sequence,result.head_commit,result.head_digest,result.head_tick; + END IF; +END $$; +REVOKE ALL ON FUNCTION evolution.resolve_population_commit(evolution.u64,evolution.u64,uuid,bytea,bytea) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION evolution.resolve_population_commit(evolution.u64,evolution.u64,uuid,bytea,bytea) TO :"runtime_role"; + +CREATE FUNCTION evolution.load_population_evidence(w evolution.u64,r evolution.u64,holder uuid,epoch evolution.u64,id uuid) +RETURNS TABLE(trace bytea,source_commit uuid,source_manifest_sha256 bytea,source_payload_sha256 bytea) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +BEGIN + PERFORM evolution.check_region_lease(w,r,holder,epoch); + RETURN QUERY SELECT p.trace,NULL::uuid,NULL::bytea,NULL::bytea FROM evolution.population_publications p + JOIN evolution.region_commits c ON c.commit_id = p.commit_id WHERE c.world_id = w AND c.region_id = r AND c.commit_id = id + UNION ALL SELECT ''::bytea,p.source_commit,p.source_manifest_sha256,p.source_payload_sha256 FROM evolution.population_profile_transitions p + JOIN evolution.region_commits c ON c.commit_id = p.target_commit WHERE c.world_id = w AND c.region_id = r AND c.commit_id = id; +END $$; +REVOKE ALL ON FUNCTION evolution.load_population_evidence(evolution.u64,evolution.u64,uuid,evolution.u64,uuid) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION evolution.load_population_evidence(evolution.u64,evolution.u64,uuid,evolution.u64,uuid) TO :"runtime_role"; + +-- Region lease serialization prevents a new player choice from racing entry into +-- the unattended profile. Exact old action retries retain their original result. +ALTER FUNCTION evolution.reserve_cellular_mutation(evolution.u64,evolution.u64,uuid,evolution.u64, + uuid,evolution.u64,evolution.u64,evolution.u64,bytea) RENAME TO reserve_cellular_mutation_v43; +REVOKE ALL ON FUNCTION evolution.reserve_cellular_mutation_v43(evolution.u64,evolution.u64,uuid,evolution.u64, + uuid,evolution.u64,evolution.u64,evolution.u64,bytea) FROM PUBLIC, :"runtime_role"; +CREATE FUNCTION evolution.reserve_cellular_mutation(p_world evolution.u64,p_region evolution.u64, + p_holder uuid,p_epoch evolution.u64,p_account uuid,p_generation evolution.u64,p_action evolution.u64, + p_target_tick evolution.u64,p_input bytea) +RETURNS TABLE(successor_identity evolution.u64,action_region evolution.u64) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE m bytea; +BEGIN + PERFORM evolution.check_region_lease(p_world,p_region,p_holder,p_epoch); + IF NOT EXISTS (SELECT 1 FROM evolution.cellular_mutation_reservations x + WHERE x.world_id = p_world AND x.account_id = p_account AND x.action_id = p_action) THEN + SELECT c.manifest INTO m FROM evolution.region_heads h JOIN evolution.region_commits c + ON c.commit_id = h.commit_id WHERE h.world_id = p_world AND h.region_id = p_region; + IF m IS NOT NULL AND (evolution.parse_checkpoint_manifest(m)).profile = 5 THEN + RAISE EXCEPTION 'population profile refuses player mutation' USING ERRCODE = 'EVL06'; + END IF; + END IF; + RETURN QUERY SELECT * FROM evolution.reserve_cellular_mutation_v43(p_world,p_region,p_holder,p_epoch, + p_account,p_generation,p_action,p_target_tick,p_input); +END $$; +REVOKE ALL ON FUNCTION evolution.reserve_cellular_mutation(evolution.u64,evolution.u64,uuid,evolution.u64, + uuid,evolution.u64,evolution.u64,evolution.u64,bytea) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION evolution.reserve_cellular_mutation(evolution.u64,evolution.u64,uuid,evolution.u64, + uuid,evolution.u64,evolution.u64,evolution.u64,bytea) TO :"runtime_role"; + +INSERT INTO evolution.schema_migrations(version,sha256) VALUES(44,:'migration_sha_44'); diff --git a/migrations/apply.sql b/migrations/apply.sql index 9a616d3..b76996e 100644 --- a/migrations/apply.sql +++ b/migrations/apply.sql @@ -119,6 +119,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') SELECT :'history_v40' || ',41:' || :'migration_sha_41' AS history_v41 \gset SELECT :'history_v41' || ',42:' || :'migration_sha_42' AS history_v42 \gset SELECT :'history_v42' || ',43:' || :'migration_sha_43' AS history_v43 \gset + SELECT :'history_v43' || ',44:' || :'migration_sha_44' AS history_v44 \gset SELECT :'history' = :'history_v22' AS at_v22, :'history' = :'history_v23' AS at_v23, :'history' = :'history_v24' AS at_v24, @@ -140,8 +141,10 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') :'history' = :'history_v40' AS at_v40, :'history' = :'history_v41' AS at_v41, :'history' = :'history_v42' AS at_v42, - :'history' = :'history_v43' AS at_v43 \gset - \if :at_v43 + :'history' = :'history_v43' AS at_v43, + :'history' = :'history_v44' AS at_v44 \gset + \if :at_v44 + \elif :at_v43 \elif :at_v42 \elif :at_v41 \elif :at_v40 @@ -521,7 +524,8 @@ SELECT coalesce(max(version), 0) = 24 AS at_v24, coalesce(max(version), 0) = 40 AS at_v40, coalesce(max(version), 0) = 41 AS at_v41, coalesce(max(version), 0) = 42 AS at_v42, - coalesce(max(version), 0) = 43 AS at_v43 + coalesce(max(version), 0) = 43 AS at_v43, + coalesce(max(version), 0) = 44 AS at_v44 FROM evolution.schema_migrations \gset \if :at_v24 \ir 0025_cellular_v2_bundle.sql @@ -733,10 +737,15 @@ SELECT coalesce(max(version), 0) = 24 AS at_v24, \elif :at_v42 \ir 0043_cellular_v9_foraging_bundle.sql \elif :at_v43 +\elif :at_v44 \else \echo 'Unknown or changed migration history after v43' DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$; \endif +SELECT coalesce(max(version), 0) = 43 AS needs_v44 FROM evolution.schema_migrations \gset +\if :needs_v44 + \ir 0044_exact_population_publication.sql +\endif -- Effective-privilege audit on every run, including no-ops (ADR-0060). Attributes -- alone are insufficient: predefined roles such as pg_write_all_data, PUBLIC -- grants or SET-able memberships would bypass the fenced functions. Checks the @@ -756,6 +765,10 @@ BEGIN UNION SELECT m.member FROM pg_catalog.pg_auth_members m JOIN closure c ON m.roleid = c.oid ), fenced(oid) AS (VALUES + ('evolution.publish_population_checkpoint(bytea,bytea)'::regprocedure::oid), + ('evolution.publish_population_profile_transition(bytea)'::regprocedure::oid), + ('evolution.resolve_population_commit(evolution.u64,evolution.u64,uuid,bytea,bytea)'::regprocedure::oid), + ('evolution.load_population_evidence(evolution.u64,evolution.u64,uuid,evolution.u64,uuid)'::regprocedure::oid), ('evolution.publish_primitive_checkpoint(bytea,bytea)'::regprocedure::oid), ('evolution.load_primitive_lives(evolution.u64,evolution.u64)'::regprocedure::oid), ('evolution.query_primitive_life(evolution.u64,evolution.u64,uuid,evolution.u64,uuid,evolution.u64,evolution.u64)'::regprocedure::oid), diff --git a/schemas/cellular-population-batch.md b/schemas/cellular-population-batch.md new file mode 100644 index 0000000..7f42970 --- /dev/null +++ b/schemas/cellular-population-batch.md @@ -0,0 +1,40 @@ +# Exact population batch 1 + +Authority: ADR-0142. All integers little-endian; no padding. Maximum 1,968,376 +bytes. This trace belongs only to checkpoint profile 5, nested cellular bundle 9 +and history profile 5. SHA-256 binds the complete immutable trace to its manifest. + +| Offset | Bytes | Field | +|---:|---:|---| +| 0 | 8 | Magic `AEVOPBT1` | +| 8 | 4 | Format 1 | +| 12 | 4 | Kind: 0 biological interval, 1 manual conversion | +| 16 | 24 | World, region, frame as u64 | +| 40 | 16 | Predecessor commit UUID | +| 56 | 32 | Predecessor manifest SHA-256 | +| 88 | 16 | Begin and end tick as u64 | +| 104 | 4 | Boundary count, 1..64 | +| 108 | 4 | Reserved, zero | +| 112 | 8 | Nested biological trace length | +| 120 | 8 | Manual conversion genome, zero for kind 0 | +| 128 | 8 | Manual destination: 0 individual, 1 cohort; zero for kind 0 | +| 136 | declared length | Unchanged cellular batch format 3 (kind 0 only) | +| after biology | variable | Ordered boundaries | + +Each boundary is conversion sequence u64, member count u64 (0..256), then sorted +unique `(life u64, genome u64)` pairs. Exact end of input and canonical re-encoding +are required. Kind 0 has one boundary per tick and advances 1..64 fixed 50 ms +ticks. Both headers must agree exactly. Birth and mutation arrays are empty; +every controlled cell has exactly its frozen-only intent. Recorded wildlife +intentions replay through the same biological kernel. The persisted policy must +reproduce the exact boundary, including sequence and every member. + +Kind 1 contains no biological trace, one boundary and a nonzero genome; begin and +end tick are equal and nonzero. Conversion moves the whole eligible wildlife +genome group. It cannot change biological bytes or select a random sample. + +Profile transitions 4 to 5 and 5 to 4 are separate no-tick commits with no trace. +Their immutable SQL evidence binds both source digests and profiles; recovery +loads and verifies both payloads. A biological interval or manual conversion +instead requires full recorded trace evidence, replay and exact payload equality. +Runtime cannot publish either case through the generic region publisher. diff --git a/schemas/cellular-population.md b/schemas/cellular-population.md index d5af4cc..0c5b143 100644 --- a/schemas/cellular-population.md +++ b/schemas/cellular-population.md @@ -1,6 +1,6 @@ # Exact cellular population envelope 1 -Authority: ADR-0139/0140. Little-endian integers, no padding. Size is bounded by +Authority: ADR-0139/0140/0142. Little-endian integers, no padding. Size is bounded by `max_cellular_population_bytes`. No changes to existing cellular owner envelopes. | Offset | Bytes | Field | @@ -30,5 +30,10 @@ conversion. No random seed or residual is missing: the representation is exact. The checksum detects corruption, not malicious alteration. Production object publication must also bind SHA-256, predecessor, current fence and immutable -manifest. This format is not yet an accepted PostgreSQL checkpoint profile; it -must never be labeled as the existing cellular profile-4 payload. +manifest. Migration 44 admits this envelope as checkpoint profile 5, nested +cellular bundle 9 and history profile 5. It must never be labeled as the existing +raw cellular profile-4 payload. Entry wraps identical rules-8 biology with empty +membership and sequence zero; reverse entry unwraps the identical biology. +Conversion sequence is local to each profile-5 generation. The immutable region +commit sequence and transition evidence distinguish generations and never reset. +See [population batch format 1](cellular-population-batch.md). diff --git a/src/persistence/cellular_checkpoint_store.cpp b/src/persistence/cellular_checkpoint_store.cpp index f4720d2..9ccc274 100644 --- a/src/persistence/cellular_checkpoint_store.cpp +++ b/src/persistence/cellular_checkpoint_store.cpp @@ -1,4 +1,5 @@ #include "evo/persistence/checkpoint_store.hpp" +#include "evo/persistence/population_checkpoint_store.hpp" #include "evo/persistence/sha256.hpp" #include @@ -537,7 +538,15 @@ recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects, std::vector actions; std::vector mutation_actions_recovered; std::vector mutation_results_recovered; - if (manifest->sequence != 0 && manifest->begin_tick == manifest->end_tick) { + if (manifest->sequence != 0 && manifest->begin_tick == manifest->end_tick && + manifest->cellular_versions == cellular_v9_versions) { + auto verified = verify_population_profile_transition(db, objects, lease, *committed, + *manifest, *payload, timeout); + if (!verified) + return std::unexpected(verified.error()); + return RecoveredCellularCheckpoint{ + std::move(*committed), *manifest, std::move(*state), {}, {}, {}}; + } else if (manifest->sequence != 0 && manifest->begin_tick == manifest->end_tick) { auto migration = load_cellular_profile_migration(db, key, lease, manifest->commit, timeout); if (!migration) return std::unexpected{RecoveryFailure{RecoveryError::store, migration.error()}}; diff --git a/src/persistence/population_checkpoint.cpp b/src/persistence/population_checkpoint.cpp new file mode 100644 index 0000000..28afe7c --- /dev/null +++ b/src/persistence/population_checkpoint.cpp @@ -0,0 +1,78 @@ +#include "evo/persistence/region_checkpoint.hpp" + +namespace evo::persistence { +namespace { +std::expected +next_population(const CheckpointManifest& previous, const Sha256Digest& previous_digest, + Uuid commit, const LeaseGrant& lease, std::uint64_t end_tick, + const Sha256Digest& build, std::uint64_t bytes, const Sha256Digest& digest, + BatchDigests batch, bool transition) { + if (!supported(previous) || !encode_manifest(previous) || previous.sequence == 0 || + previous.sequence == ~std::uint64_t{0} || previous.end_tick == 0 || + previous.cellular_versions != cellular_v9_versions || previous_digest == Sha256Digest{} || + lease.frame != previous.frame || lease.epoch == 0 || lease.holder == Uuid{} || + commit == Uuid{} || commit == previous.commit || end_tick < previous.end_tick || + end_tick - previous.end_tick > max_checkpoint_ticks || + (previous.profile != CheckpointProfile::cellular_lives_v1 && + previous.profile != CheckpointProfile::cellular_population_v1) || + (!transition && previous.profile != CheckpointProfile::cellular_population_v1)) + return std::unexpected(CheckpointError::chain); + auto next = previous; + if (transition) { + next.profile = previous.profile == CheckpointProfile::cellular_lives_v1 + ? CheckpointProfile::cellular_population_v1 + : CheckpointProfile::cellular_lives_v1; + next.history_profile = next.profile == CheckpointProfile::cellular_population_v1 + ? population_history_profile + : cellular_revision_history_profile; + } + next.commit = commit; + ++next.sequence; + next.predecessor_commit = previous.commit; + next.predecessor_digest = previous_digest; + next.begin_tick = previous.end_tick; + next.end_tick = end_tick; + next.holder = lease.holder; + next.epoch = lease.epoch; + next.build = build; + next.payload_bytes = bytes; + next.payload_sha256 = digest; + next.batch = batch; + if (!encode_manifest(next)) + return std::unexpected(CheckpointError::format); + return next; +} +} // namespace + +std::expected +population_profile_transition_manifest(const CheckpointManifest& previous, + const Sha256Digest& previous_digest, Uuid commit, + const LeaseGrant& lease, const Sha256Digest& build, + std::uint64_t bytes, const Sha256Digest& digest) { + return next_population(previous, previous_digest, commit, lease, previous.end_tick, build, + bytes, digest, {}, true); +} +std::expected population_successor_manifest( + const CheckpointManifest& previous, const Sha256Digest& previous_digest, Uuid commit, + const LeaseGrant& lease, std::uint64_t end_tick, const Sha256Digest& build, std::uint64_t bytes, + const Sha256Digest& digest, const BatchDigests& batch) { + if (batch.trace_bytes == 0) + return std::unexpected(CheckpointError::chain); + return next_population(previous, previous_digest, commit, lease, end_tick, build, bytes, digest, + batch, false); +} +std::expected, CheckpointError> +restore_population_payload(std::span payload, const CheckpointManifest& m) { + if (!supported(m) || m.profile != CheckpointProfile::cellular_population_v1) + return std::unexpected(CheckpointError::unsupported); + if (!encode_manifest(m) || payload.size() != m.payload_bytes) + return std::unexpected(CheckpointError::format); + auto state = server::CellularPopulation::restore(payload, m.key.world, m.key.region, m.frame, + m.end_tick); + if (!state) + return std::unexpected(state.error() == server::CellularPopulationError::unsupported + ? CheckpointError::unsupported + : CheckpointError::payload); + return std::move(*state); +} +} // namespace evo::persistence diff --git a/src/persistence/population_checkpoint_store.cpp b/src/persistence/population_checkpoint_store.cpp new file mode 100644 index 0000000..e311259 --- /dev/null +++ b/src/persistence/population_checkpoint_store.cpp @@ -0,0 +1,315 @@ +#include "evo/persistence/population_checkpoint_store.hpp" +#include "evo/persistence/sha256.hpp" + +#include +#include + +namespace evo::persistence { +namespace { +using std::chrono::milliseconds; +PopulationPublicationFailure failure(PublicationError kind, ObjectError object = {}, + LeaseStoreFailure store = {}, + PopulationPreparedCommit prepared = {}) { + return {kind, store, object, std::move(prepared)}; +} +bool bound(const CommittedCheckpoint& checkpoint, const CheckpointManifest& m) { + const auto digest = sha256(checkpoint.manifest); + return digest && *digest == checkpoint.head.digest && m.commit == checkpoint.head.commit && + m.sequence == checkpoint.head.sequence && m.end_tick == checkpoint.head.end_tick && + m.frame == checkpoint.head.frame; +} +bool canonical(const PopulationPreparedCommit& prepared, const CheckpointManifest& m) { + if (!supported(m) || m.key != prepared.key || m.commit != prepared.commit || m.sequence == 0 || + m.cellular_versions != cellular_v9_versions || !encode_manifest(m)) + return false; + if (prepared.trace.empty()) + return (m.profile == CheckpointProfile::cellular_lives_v1 || + m.profile == CheckpointProfile::cellular_population_v1) && + m.begin_tick == m.end_tick && m.batch == BatchDigests{}; + const auto trace = server::decode_population_batch(prepared.trace); + const auto digest = sha256(prepared.trace); + if (!trace || !digest || m.profile != CheckpointProfile::cellular_population_v1 || + m.batch != BatchDigests{prepared.trace.size(), *digest, 0, {}}) + return false; + const auto& h = trace->header; + return h.world == m.key.world && h.region == m.key.region && h.frame == m.frame && + h.predecessor == m.predecessor_commit && h.predecessor_digest == m.predecessor_digest && + h.begin_tick == m.begin_tick && h.end_tick == m.end_tick; +} +std::expected put(const FilesystemObjectStore& objects, + std::span bytes) { + const auto digest = sha256(bytes); + if (!digest) + return std::unexpected(failure(PublicationError::digest)); + auto object = objects.put(bytes); + if (!object) + return std::unexpected(failure(PublicationError::upload, object.error())); + auto readback = objects.get(*object); + if (!readback || object->bytes != bytes.size() || object->sha256 != *digest || + !std::ranges::equal(*readback, bytes)) + return std::unexpected( + failure(PublicationError::verify, readback ? ObjectError::corrupt : readback.error())); + return {}; +} +std::expected +transition(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease, + const CommittedCheckpoint& predecessor, Uuid commit, const Sha256Digest& build, + server::CellularTierPolicy policy, bool to_population, milliseconds timeout, + const PublishHooks& hooks) { + auto previous = decode_manifest(predecessor.manifest); + const auto previous_digest = sha256(predecessor.manifest); + if (!previous || !previous_digest || !bound(predecessor, *previous) || + previous->profile != (to_population ? CheckpointProfile::cellular_lives_v1 + : CheckpointProfile::cellular_population_v1) || + previous->cellular_versions != cellular_v9_versions) + return std::unexpected(failure(PublicationError::predecessor)); + auto source = objects.get({previous->payload_sha256, previous->payload_bytes}); + if (!source) + return std::unexpected(failure(PublicationError::predecessor, source.error())); + std::vector payload; + if (to_population) { + auto state = restore_cellular_payload(*source, *previous); + if (!state) + return std::unexpected(failure(PublicationError::predecessor)); + auto population = server::CellularPopulation::create(std::move(*state), policy); + if (!population || (*population)->state().snapshot() != *source) + return std::unexpected(failure(PublicationError::candidate)); + payload = (*population)->snapshot(); + } else { + auto state = restore_population_payload(*source, *previous); + if (!state) + return std::unexpected(failure(PublicationError::predecessor)); + payload = (*state)->state().snapshot(); + } + const auto digest = sha256(payload); + if (!digest) + return std::unexpected(failure(PublicationError::digest)); + auto manifest = population_profile_transition_manifest(*previous, *previous_digest, commit, + lease, build, payload.size(), *digest); + if (!manifest) + return std::unexpected(failure(PublicationError::candidate)); + auto encoded = encode_manifest(*manifest); + if (!encoded) + return std::unexpected(failure(PublicationError::candidate)); + if (auto uploaded = put(objects, payload); !uploaded) + return std::unexpected(uploaded.error()); + if (hooks.before_transaction) + hooks.before_transaction(); + return retry_population_prepared(db, {previous->key, commit, std::move(*encoded), {}}, timeout); +} +} // namespace + +std::expected migrate_cellular_to_population( + Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease, + const CommittedCheckpoint& predecessor, Uuid commit, const Sha256Digest& build, + server::CellularTierPolicy policy, milliseconds timeout, const PublishHooks& hooks) { + return transition(db, objects, lease, predecessor, commit, build, policy, true, timeout, hooks); +} +std::expected +materialize_population_profile(Postgres& db, const FilesystemObjectStore& objects, + const LeaseGrant& lease, const CommittedCheckpoint& predecessor, + Uuid commit, const Sha256Digest& build, milliseconds timeout, + const PublishHooks& hooks) { + return transition(db, objects, lease, predecessor, commit, build, {}, false, timeout, hooks); +} +std::expected +retry_population_prepared(Postgres& db, const PopulationPreparedCommit& prepared, + milliseconds timeout) { + const auto m = decode_manifest(prepared.manifest); + if (!m || !canonical(prepared, *m)) + return std::unexpected(failure(PublicationError::candidate)); + auto result = + prepared.trace.empty() + ? publish_population_profile_transition(db, prepared.manifest, timeout) + : publish_population_checkpoint(db, prepared.manifest, prepared.trace, timeout); + if (!result) + return std::unexpected(failure(result.error().kind == LeaseStoreError::unknown + ? PublicationError::unknown + : PublicationError::rejected, + {}, result.error(), prepared)); + const auto digest = sha256(prepared.manifest); + if (!digest || result->digest != *digest || result->commit != m->commit || + result->sequence != m->sequence || result->end_tick != m->end_tick || + result->frame != m->frame) + return std::unexpected(failure(PublicationError::receipt)); + return CommittedCheckpoint{*result, prepared.manifest}; +} +std::expected +resolve_population_prepared(Postgres& db, const PopulationPreparedCommit& prepared, + milliseconds timeout) { + const auto manifest = decode_manifest(prepared.manifest); + if (!manifest || !canonical(prepared, *manifest)) + return std::unexpected(LeaseStoreFailure{LeaseStoreError::invalid, {}}); + return resolve_population_commit(db, prepared.key, prepared.commit, prepared.manifest, + prepared.trace, timeout); +} +std::expected publish_population_batch( + Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease, + const CommittedCheckpoint& predecessor, const server::CellularPopulationBatch& batch, + const server::CellularPopulation& candidate, Uuid commit, const Sha256Digest& build, + milliseconds timeout, const PublishHooks& hooks) { + const auto previous = decode_manifest(predecessor.manifest); + const auto previous_digest = sha256(predecessor.manifest); + const auto trace = server::encode_population_batch(batch); + if (!previous || !previous_digest || !trace || !bound(predecessor, *previous) || + previous->profile != CheckpointProfile::cellular_population_v1) + return std::unexpected(failure(PublicationError::predecessor)); + auto source = objects.get({previous->payload_sha256, previous->payload_bytes}); + if (!source) + return std::unexpected(failure(PublicationError::predecessor, source.error())); + auto base = restore_population_payload(*source, *previous); + if (!base) + return std::unexpected(failure(PublicationError::predecessor)); + auto replay = server::replay_population_batch(**base, batch); + const auto payload = candidate.snapshot(); + if (!replay || (*replay)->snapshot() != payload) + return std::unexpected(failure(PublicationError::candidate)); + const auto payload_digest = sha256(payload), trace_digest = sha256(*trace); + if (!payload_digest || !trace_digest) + return std::unexpected(failure(PublicationError::digest)); + auto m = population_successor_manifest(*previous, *previous_digest, commit, lease, + batch.header.end_tick, build, payload.size(), + *payload_digest, {trace->size(), *trace_digest, 0, {}}); + if (!m) + return std::unexpected(failure(PublicationError::candidate)); + auto encoded = encode_manifest(*m); + if (!encoded) + return std::unexpected(failure(PublicationError::candidate)); + PopulationPreparedCommit prepared{previous->key, commit, std::move(*encoded), *trace}; + if (!canonical(prepared, *m)) + return std::unexpected(failure(PublicationError::candidate)); + if (auto uploaded = put(objects, payload); !uploaded) + return std::unexpected(uploaded.error()); + if (auto uploaded = put(objects, *trace); !uploaded) + return std::unexpected(uploaded.error()); + if (hooks.before_transaction) + hooks.before_transaction(); + return retry_population_prepared(db, prepared, timeout); +} + +std::expected +verify_population_profile_transition(Postgres& db, const FilesystemObjectStore& objects, + const LeaseGrant& lease, const CommittedCheckpoint& committed, + const CheckpointManifest& m, + std::span payload, milliseconds timeout) { + const auto fail = [](RecoveryError kind) { return std::unexpected(RecoveryFailure{kind, {}}); }; + if (!bound(committed, m) || m.sequence == 0 || m.begin_tick != m.end_tick || + m.batch != BatchDigests{}) + return fail(RecoveryError::manifest); + auto evidence = load_population_evidence(db, m.key, lease, m.commit, timeout); + if (!evidence) + return std::unexpected(RecoveryFailure{RecoveryError::store, evidence.error()}); + const auto record_optional = std::move(*evidence); + if (!record_optional.has_value()) + return fail(RecoveryError::history); + const auto& record = *record_optional; + if (!record.trace.empty() || !record.source_commit.has_value()) + return fail(RecoveryError::history); + auto predecessor = load_region_commit(db, m.key, m.sequence - 1, timeout); + if (!predecessor) + return std::unexpected(RecoveryFailure{RecoveryError::tail, predecessor.error()}); + auto source = decode_manifest(predecessor->manifest); + if (!source || !bound(*predecessor, *source) || !supported(*source) || + source->commit != m.predecessor_commit || + predecessor->head.digest != m.predecessor_digest || source->key != m.key || + source->frame != m.frame || source->end_tick != m.end_tick || + source->cellular_versions != cellular_v9_versions || + m.cellular_versions != cellular_v9_versions || record.source_commit != source->commit || + record.source_manifest_sha256 != predecessor->head.digest || + record.source_payload_sha256 != source->payload_sha256) + return fail(RecoveryError::tail); + auto bytes = objects.get({source->payload_sha256, source->payload_bytes}); + if (!bytes) + return fail(RecoveryError::tail); + if (m.profile == CheckpointProfile::cellular_population_v1 && + source->profile == CheckpointProfile::cellular_lives_v1) { + auto raw = restore_cellular_payload(*bytes, *source); + auto population = restore_population_payload(payload, m); + if (!raw || !population) + return fail(RecoveryError::payload); + auto expected = + server::CellularPopulation::create(std::move(*raw), (*population)->policy()); + if (!expected || !std::ranges::equal((*expected)->snapshot(), payload)) + return fail(RecoveryError::history); + } else if (m.profile == CheckpointProfile::cellular_lives_v1 && + source->profile == CheckpointProfile::cellular_population_v1) { + auto population = restore_population_payload(*bytes, *source); + if (!population || !restore_cellular_payload(payload, m) || + !std::ranges::equal((*population)->state().snapshot(), payload)) + return fail(RecoveryError::history); + } else + return fail(RecoveryError::unsupported); + return {}; +} +std::expected +recover_population_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, + const LeaseGrant& lease, + std::span retained_builds, milliseconds timeout) { + const auto fail = [](RecoveryError kind) { return std::unexpected(RecoveryFailure{kind, {}}); }; + auto committed = load_region_checkpoint(db, key, timeout); + if (!committed) + return std::unexpected( + RecoveryFailure{committed.error().kind == LeaseStoreError::not_provisioned + ? RecoveryError::not_provisioned + : RecoveryError::store, + committed.error()}); + auto m = decode_manifest(committed->manifest); + if (!m || !bound(*committed, *m) || m->key != key) + return fail(RecoveryError::manifest); + if (!supported(*m) || m->profile != CheckpointProfile::cellular_population_v1 || + lease.frame != m->frame || + std::ranges::find(retained_builds, m->build) == retained_builds.end()) + return fail(RecoveryError::unsupported); + auto bytes = objects.get({m->payload_sha256, m->payload_bytes}); + if (!bytes) + return fail(bytes.error() == ObjectError::io ? RecoveryError::store + : bytes.error() == ObjectError::missing ? RecoveryError::missing_payload + : RecoveryError::corrupt_payload); + auto population = restore_population_payload(*bytes, *m); + if (!population) + return fail(RecoveryError::payload); + if (m->batch == BatchDigests{}) { + if (auto verified = verify_population_profile_transition(db, objects, lease, *committed, *m, + *bytes, timeout); + !verified) + return std::unexpected(verified.error()); + } else { + auto evidence = load_population_evidence(db, key, lease, m->commit, timeout); + if (!evidence) + return std::unexpected(RecoveryFailure{RecoveryError::store, evidence.error()}); + const auto record_optional = std::move(*evidence); + if (!record_optional.has_value()) + return fail(RecoveryError::history); + const auto& record = *record_optional; + if (record.source_commit.has_value()) + return fail(RecoveryError::history); + auto trace_bytes = objects.get({m->batch.trace_sha256, m->batch.trace_bytes}); + if (!trace_bytes || *trace_bytes != record.trace) + return fail(RecoveryError::tail); + PopulationPreparedCommit prepared{key, m->commit, committed->manifest, *trace_bytes}; + if (!canonical(prepared, *m)) + return fail(RecoveryError::tail); + auto predecessor = load_region_commit(db, key, m->sequence - 1, timeout); + if (!predecessor) + return std::unexpected(RecoveryFailure{RecoveryError::tail, predecessor.error()}); + auto previous = decode_manifest(predecessor->manifest); + if (!previous || !bound(*predecessor, *previous) || + previous->commit != m->predecessor_commit || + predecessor->head.digest != m->predecessor_digest || previous->key != key || + previous->frame != m->frame || previous->sequence + 1 != m->sequence || + previous->end_tick != m->begin_tick) + return fail(RecoveryError::tail); + auto source_bytes = objects.get({previous->payload_sha256, previous->payload_bytes}); + if (!source_bytes) + return fail(RecoveryError::tail); + auto base = restore_population_payload(*source_bytes, *previous); + auto trace = server::decode_population_batch(*trace_bytes); + if (!base || !trace) + return fail(RecoveryError::tail); + auto replay = server::replay_population_batch(**base, *trace); + if (!replay || (*replay)->snapshot() != *bytes) + return fail(RecoveryError::tail); + } + return RecoveredPopulationCheckpoint{std::move(*committed), *m, std::move(*population)}; +} +} // namespace evo::persistence diff --git a/src/persistence/region_checkpoint.cpp b/src/persistence/region_checkpoint.cpp index 46941a7..5103dbb 100644 --- a/src/persistence/region_checkpoint.cpp +++ b/src/persistence/region_checkpoint.cpp @@ -60,12 +60,20 @@ bool structural(const CheckpointManifest& m) noexcept { return false; const auto& b = m.batch; const bool primitive = m.profile == CheckpointProfile::primitive_lives_v1; - const bool cellular = m.profile == CheckpointProfile::cellular_lives_v1; + const bool population = m.profile == CheckpointProfile::cellular_population_v1; + const bool cellular = m.profile == CheckpointProfile::cellular_lives_v1 || population; const bool cellular_migration = cellular && m.sequence > 0 && (m.cellular_versions == cellular_v2_versions || - m.cellular_versions == cellular_v3_versions) && + m.cellular_versions == cellular_v3_versions || + m.cellular_versions == cellular_v9_versions) && m.begin_tick == m.end_tick && m.end_tick > 0 && b == BatchDigests{}; + const bool population_conversion = population && m.begin_tick == m.end_tick && m.end_tick > 0 && + b.trace_bytes >= server::population_batch_header_bytes + 16; + if (population && (m.sequence == 0 || m.cellular_versions != cellular_v9_versions || + m.history_profile != population_history_profile || b.decision_count != 0 || + !zero(b.decisions_sha256))) + return false; if ((!cellular && m.cellular_versions != cellular_v1_versions) || (cellular && m.cellular_versions != cellular_v1_versions && m.cellular_versions != cellular_v2_versions && @@ -103,7 +111,7 @@ bool structural(const CheckpointManifest& m) noexcept { } const bool link = !zero(m.predecessor_commit) && m.predecessor_commit != m.commit && !zero(m.predecessor_digest) && !zero(m.holder) && m.epoch != 0 && - (m.end_tick > m.begin_tick || cellular_migration) && + (m.end_tick > m.begin_tick || cellular_migration || population_conversion) && m.end_tick - m.begin_tick <= max_checkpoint_ticks; if (m.manifest_version == 1) return link && no_batch; @@ -146,6 +154,9 @@ bool structural(const CheckpointManifest& m) noexcept { (b.trace_bytes - server::primitive_batch_header_bytes) % server::primitive_birth_bytes == 0 + : population + ? b.trace_bytes >= server::population_batch_header_bytes + 16 && + b.trace_bytes <= server::max_population_batch_bytes && b.trace_bytes % 8 == 0 : cellular ? (cellular_current_trace || cellular_v2_trace || cellular_legacy_trace) : b.trace_bytes == trace_bytes_for(ticks); return link && trace_shape && !zero(b.trace_sha256) && @@ -155,6 +166,11 @@ bool structural(const CheckpointManifest& m) noexcept { } // namespace bool supported(const CheckpointManifest& m) noexcept { + if (m.profile == CheckpointProfile::cellular_population_v1) + return m.manifest_version == 2 && m.history_profile == population_history_profile && + m.cellular_versions == cellular_v9_versions && + m.versions == environmental_v1_versions && m.tick_microseconds == 50000 && + m.rng_streams == 0 && m.durability == DurabilityPolicy::single_volume_development; if (m.profile == CheckpointProfile::primitive_lives_v1) return m.manifest_version == 2 && m.history_profile == primitive_history_profile && m.versions == primitive_v1_versions && m.tick_microseconds == 50000 && @@ -208,7 +224,8 @@ encode_manifest(const CheckpointManifest& m) { w.u(m.epoch, 8); w.u(m.tick_microseconds, 8); const auto versions = - m.profile == CheckpointProfile::cellular_lives_v1 + (m.profile == CheckpointProfile::cellular_lives_v1 || + m.profile == CheckpointProfile::cellular_population_v1) ? std::array{m.cellular_versions.rules, m.cellular_versions.envelope, m.cellular_versions.habitat, m.cellular_versions.coordinate, m.cellular_versions.genome, m.cellular_versions.development, @@ -265,7 +282,8 @@ decode_manifest(std::span bytes) { std::array versions{}; for (auto& v : versions) v = static_cast(r.u(4)); - if (m.profile == CheckpointProfile::cellular_lives_v1) { + if (m.profile == CheckpointProfile::cellular_lives_v1 || + m.profile == CheckpointProfile::cellular_population_v1) { m.cellular_versions = {versions[0], versions[1], versions[2], versions[3], versions[4], versions[5], versions[6], versions[7], versions[8]}; } else { @@ -326,7 +344,8 @@ clone_manifest(const CheckpointManifest& source, const Sha256Digest& source_dige RegionKey target, Uuid commit, std::uint64_t payload_bytes, const Sha256Digest& payload_sha256, const BatchDigests& inherited) { if (!supported(source) || source.profile == CheckpointProfile::primitive_lives_v1 || - source.profile == CheckpointProfile::cellular_lives_v1) + source.profile == CheckpointProfile::cellular_lives_v1 || + source.profile == CheckpointProfile::cellular_population_v1) return std::unexpected{CheckpointError::unsupported}; if (!structural(source) || target.world == source.key.world || commit == source.commit || inherited.trace_bytes != 0 || !zero(inherited.trace_sha256)) diff --git a/src/persistence/region_lease_store.cpp b/src/persistence/region_lease_store.cpp index afe9b29..63b965b 100644 --- a/src/persistence/region_lease_store.cpp +++ b/src/persistence/region_lease_store.cpp @@ -463,7 +463,7 @@ std::expected resolve_cellular_region_commit( p, timeout); if (!rows) return std::unexpected{classify(rows.error())}; - if (rows->size() != 1 || (*rows)[0].size() != 6 || !(*rows)[0][0]) + if (rows->size() != 1 || (*rows)[0].size() != 6 || !(*rows)[0][0].has_value()) return std::unexpected{malformed(false)}; const auto status = (*rows)[0][0].value_or(std::string{}); if (status == "absent") @@ -1502,4 +1502,100 @@ std::expected revoke_account_grants(Postgres& db, st return std::unexpected{AccountIdentityError::unknown}; return *revoked == "true"; } -} // namespace evo::persistence \ No newline at end of file +std::expected +publish_population_checkpoint(Postgres& db, std::span manifest, + std::span trace, milliseconds timeout) { + const std::array parameters{bytea(manifest), bytea(trace)}; + auto rows = + db.query("SELECT * FROM evolution.publish_population_checkpoint($1::bytea,$2::bytea)", + parameters, timeout); + if (!rows) + return std::unexpected(classify(rows.error())); + auto head = parse_head(*rows); + if (!head) + return std::unexpected(malformed(true)); + return *head; +} +std::expected +publish_population_profile_transition(Postgres& db, std::span manifest, + milliseconds timeout) { + const std::array parameters{bytea(manifest)}; + auto rows = db.query("SELECT * FROM evolution.publish_population_profile_transition($1::bytea)", + parameters, timeout); + if (!rows) + return std::unexpected(classify(rows.error())); + auto head = parse_head(*rows); + if (!head) + return std::unexpected(malformed(true)); + return *head; +} +std::expected +resolve_population_commit(Postgres& db, RegionKey key, const Uuid& commit, + std::span manifest, + std::span trace, milliseconds timeout) { + const std::array parameters{std::to_string(key.world), + std::to_string(key.region), uuid_text(commit), + bytea(manifest), bytea(trace)}; + auto rows = db.query("SELECT * FROM evolution.resolve_population_commit($1::evolution.u64," + "$2::evolution.u64,$3::uuid,$4::bytea,$5::bytea)", + parameters, timeout); + if (!rows) + return std::unexpected(classify(rows.error())); + if (rows->size() != 1 || (*rows)[0].size() != 6 || !(*rows)[0][0]) + return std::unexpected(malformed(false)); + const auto status_field = (*rows)[0][0]; + if (!status_field.has_value()) + return std::unexpected(malformed(false)); + const auto& status = *status_field; + if (status == "absent") + return ResolvedCommit{Resolution::absent, {}}; + if (status == "conflict") + return ResolvedCommit{Resolution::conflict, {}}; + PgRows receipt{PgRows::value_type((*rows)[0].begin() + 1, (*rows)[0].end())}; + const auto head = status == "committed" ? parse_head(receipt) : std::nullopt; + if (!head) + return std::unexpected(malformed(false)); + return ResolvedCommit{Resolution::committed, *head}; +} +std::expected, LeaseStoreFailure> +load_population_evidence(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& commit, + milliseconds timeout) { + const std::array parameters{std::to_string(key.world), + std::to_string(key.region), uuid_text(lease.holder), + std::to_string(lease.epoch), uuid_text(commit)}; + auto rows = db.query( + "SELECT encode(trace,'hex'),source_commit::text,encode(source_manifest_sha256,'hex')," + "encode(source_payload_sha256,'hex') FROM " + "evolution.load_population_evidence($1::evolution.u64," + "$2::evolution.u64,$3::uuid,$4::evolution.u64,$5::uuid)", + parameters, timeout); + if (!rows) + return std::unexpected(classify(rows.error())); + if (rows->empty()) + return std::optional{}; + if (rows->size() != 1 || (*rows)[0].size() != 4 || !(*rows)[0][0].has_value()) + return std::unexpected(malformed(false)); + const auto& row = (*rows)[0]; + const auto trace_field = row[0]; + if (!trace_field.has_value()) + return std::unexpected(malformed(false)); + auto trace = unhex_bytes(*trace_field); + if (!trace) + return std::unexpected(malformed(false)); + PopulationEvidence evidence{std::move(*trace), {}, {}, {}}; + if (evidence.trace.empty()) { + const auto source = parse_uuid(row[1]); + const auto manifest_field = row[2], payload_field = row[3]; + const auto manifest = + manifest_field.has_value() ? unhex<32>(*manifest_field) : std::nullopt; + const auto payload = payload_field.has_value() ? unhex<32>(*payload_field) : std::nullopt; + if (!source || !manifest || !payload) + return std::unexpected(malformed(false)); + evidence.source_commit = *source; + evidence.source_manifest_sha256 = *manifest; + evidence.source_payload_sha256 = *payload; + } else if (row[1] || row[2] || row[3]) + return std::unexpected(malformed(false)); + return std::optional{std::move(evidence)}; +} +} // namespace evo::persistence diff --git a/src/server/cellular_region_host.cpp b/src/server/cellular_region_host.cpp index 853742f..9373966 100644 --- a/src/server/cellular_region_host.cpp +++ b/src/server/cellular_region_host.cpp @@ -12,10 +12,11 @@ namespace evo::server { struct CellularRegionHost::Pending { persistence::CellularPreparedCommit prepared; - std::unique_ptr candidate; + OwnedState candidate; std::vector actions; std::vector births; std::vector mutation_actions; + std::optional population_prepared{}; }; namespace { @@ -196,34 +197,72 @@ void CellularRegionHost::acquire() { status_.phase = HostPhase::quarantined; return; } - auto recovered = persistence::recover_cellular_checkpoint(*db_, *objects_, config_.key, *lease, - config_.retained_builds, - config_.lease.commit_timeout); - if (!recovered) { - if (recovered.error().kind == persistence::RecoveryError::store) { - db_.reset(); - ++status_.store_failures; - return; - } - authority_.fence(); - status_.phase = HostPhase::quarantined; + if (!recover_state(*lease) || !committed_.has_value() || !state_) return; - } - state_ = std::move(recovered->state); - committed_ = std::move(recovered->committed); queued_mutations_.clear(); - mutations_uncertain_ = true; // Reserved choices are reloaded under the new lease. - // ADR-0127: every founder corpse is an ended life; record any ending a crash lost. + mutations_uncertain_ = true; pending_endings_.clear(); - for (const auto& corpse : state_->corpses()) - if (corpse.parent_life == 0) - pending_endings_.push_back({committed_->head.commit, corpse.life_id}); + // Controlled cells cannot die in unattended profile 5; their earlier raw + // endings were drained before entry. Do not invent a new death commit. + if (!population_mode()) + for (const auto& corpse : state_->corpses()) + if (corpse.parent_life == 0) + pending_endings_.push_back({committed_->head.commit, corpse.life_id}); status_.phase = HostPhase::running; status_.simulated_tick = committed_->head.end_tick; status_.committed_tick = committed_->head.end_tick; status_.committed_sequence = committed_->head.sequence; } +bool CellularRegionHost::recover_state(const persistence::LeaseGrant& lease) { + auto head = + persistence::load_region_checkpoint(*db_, config_.key, config_.lease.commit_timeout); + if (!head) { + db_.reset(); + ++status_.store_failures; + return false; + } + const auto manifest = persistence::decode_manifest(head->manifest); + if (!manifest) { + authority_.fence(); + status_.phase = HostPhase::quarantined; + return false; + } + if (manifest->profile == persistence::CheckpointProfile::cellular_population_v1) { + auto recovered = persistence::recover_population_checkpoint(*db_, *objects_, config_.key, + lease, config_.retained_builds, + config_.lease.commit_timeout); + if (!recovered) { + if (recovered.error().kind == persistence::RecoveryError::store) { + db_.reset(); + ++status_.store_failures; + } else { + authority_.fence(); + status_.phase = HostPhase::quarantined; + } + return false; + } + state_ = std::move(recovered->state); + committed_ = std::move(recovered->committed); + return true; + } + auto recovered = persistence::recover_cellular_checkpoint( + *db_, *objects_, config_.key, lease, config_.retained_builds, config_.lease.commit_timeout); + if (!recovered) { + if (recovered.error().kind == persistence::RecoveryError::store) { + db_.reset(); + ++status_.store_failures; + return false; + } + authority_.fence(); + status_.phase = HostPhase::quarantined; + return false; + } + state_ = std::move(recovered->state); + committed_ = std::move(recovered->committed); + return true; +} + void CellularRegionHost::upkeep() { if (authority_.state(LeaseClock::now()) == persistence::LeaseState::fenced) { authority_.fence(); @@ -280,8 +319,12 @@ void CellularRegionHost::resolve() { ++status_.fences; return; } - const auto resolved = persistence::resolve_cellular_prepared(*db_, pending_->prepared, - config_.lease.commit_timeout); + const auto& population_prepared = pending_->population_prepared; + const auto resolved = population_prepared.has_value() + ? persistence::resolve_population_prepared( + *db_, *population_prepared, config_.lease.commit_timeout) + : persistence::resolve_cellular_prepared( + *db_, pending_->prepared, config_.lease.commit_timeout); if (!resolved) { if (resolved.error().kind == LeaseStoreError::fenced) { authority_.fence(); @@ -309,8 +352,18 @@ void CellularRegionHost::resolve() { if (resolved->status == persistence::Resolution::absent && !authority_.may_begin_commit(LeaseClock::now())) return; - auto retried = persistence::retry_cellular_prepared(*db_, pending_->prepared, + auto retried = [&]() -> std::expected { + if (!pending_->population_prepared) + return persistence::retry_cellular_prepared(*db_, pending_->prepared, config_.lease.commit_timeout); + auto result = persistence::retry_population_prepared(*db_, *pending_->population_prepared, + config_.lease.commit_timeout); + if (!result) + return std::unexpected(persistence::CellularPublicationFailure{ + result.error().kind, result.error().store, result.error().object, {}}); + return std::move(*result); + }(); if (!retried) { if (retried.error().kind == persistence::PublicationError::unknown) { db_.reset(); @@ -483,6 +536,8 @@ CellularRegionHost::select_mutation(const persistence::Uuid& account, std::uint6 if (!status_.ready || mutations_uncertain_ || pending_ || !state_ || !committed_ || !authority_.may_begin_commit(LeaseClock::now()) || !database()) return {not_ready, 0, {}}; + if (population_mode() && !set_population_mode(false)) + return {not_ready, 0, {}}; const auto tick = committed_->head.end_tick + 1; const CellularMutationActionRecord record{account, action_id, tick, input}; const auto bytes = encode_cellular_mutation_input(record); @@ -552,6 +607,12 @@ CellularRegionHost::publish(const CellularBatch& batch, if (!status_.ready || mutations_uncertain_ || !state_ || !committed_ || !grant || !authority_.may_begin_commit(LeaseClock::now())) return std::unexpected{CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}}; + if (population_mode()) { + if (!actions.empty() || !mutation_actions.empty()) + return std::unexpected( + CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}}); + return publish_population(batch); + } auto candidate = replay_cellular_batch(*state_, batch); if (!candidate) return std::unexpected{ @@ -722,6 +783,8 @@ network::PrimitiveLifeOutcome CellularRegionHost::create_primitive_life(const persistence::Uuid& account, std::uint64_t generation, std::uint64_t request) { using network::PrimitiveLifeResult; + if (population_mode() && !set_population_mode(false)) + return {request, PrimitiveLifeResult::unknown, 0, 0}; const auto existing = query_birth(account, generation, request, true); if (existing.result != PrimitiveLifeResult::unknown) return existing; @@ -754,6 +817,8 @@ CellularRegionHost::create_primitive_life(const persistence::Uuid& account, network::PrimitiveLifeOutcome CellularRegionHost::query_primitive_life(const persistence::Uuid& account, std::uint64_t generation, std::uint64_t request) { + if (population_mode() && !set_population_mode(false)) + return {request, network::PrimitiveLifeResult::unknown, 0, 0}; return query_birth(account, generation, request, true); } @@ -784,4 +849,126 @@ std::optional CellularRegionHost::authority() const { return std::nullopt; return authority_.grant(); } +bool CellularRegionHost::population_mode() const noexcept { + return state_.population != nullptr; +} + +std::expected +CellularRegionHost::accept_population_publication( + std::expected + result, + OwnedState candidate) { + if (!result) { + const auto& error = result.error(); + if (error.kind == persistence::PublicationError::unknown) { + pending_ = std::make_unique( + Pending{{}, std::move(candidate), {}, {}, {}, error.prepared}); + status_.phase = HostPhase::resolving; + status_.ready = false; + ++status_.unknown_outcomes; + db_.reset(); + } else if (error.store.kind == LeaseStoreError::fenced) { + authority_.fence(); + state_.reset(); + committed_.reset(); + status_.phase = HostPhase::acquiring; + status_.ready = false; + ++status_.fences; + } else if (error.kind == persistence::PublicationError::predecessor || + error.kind == persistence::PublicationError::verify || + error.kind == persistence::PublicationError::receipt) { + authority_.fence(); + state_.reset(); + committed_.reset(); + status_.phase = HostPhase::quarantined; + status_.ready = false; + } else { + mutations_uncertain_ = true; + } + return std::unexpected(CellularPublishFailure{CellularHostError::publication, error.kind, + error.store, error.object}); + } + note_endings(*state_, *candidate, result->head.commit); + state_ = std::move(candidate); + committed_ = *result; + ++status_.commits; + status_.committed_sequence = committed_->head.sequence; + status_.committed_tick = status_.simulated_tick = committed_->head.end_tick; + refresh(); + return std::move(*result); +} + +std::expected CellularRegionHost::set_population_mode(bool enabled) { + if (!status_.ready || pending_ || mutations_uncertain_ || !state_ || !committed_.has_value() || + !authority_.may_begin_commit(LeaseClock::now()) || !database()) + return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}); + if (population_mode() == enabled) + return {}; + if (enabled && + (!config_.population_policy || !queued_mutations_.empty() || !pending_endings_.empty() || + state_->rules_version() != cellular_lives_foraging_rules)) + return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}); + const auto policy = config_.population_policy.value_or(CellularTierPolicy{}); + std::array marker{}; + marker[0] = enabled ? 1 : 0; + marker[1] = policy.automatic ? 1 : 0; + for (unsigned i = 0; i < 4; ++i) { + marker[2 + i] = static_cast(policy.materialize_radius_um >> (i * 8U)); + marker[6 + i] = static_cast(policy.collapse_radius_um >> (i * 8U)); + } + const auto id = commit_id(*committed_, marker, {}); + auto raw = CellularLives::restore(state_->snapshot(), state_->world(), state_->region(), + state_->frame(), state_->tick()); + if (!id || !raw) + return std::unexpected( + CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}}); + OwnedState candidate; + if (enabled) { + auto population = CellularPopulation::create(std::move(*raw), policy); + if (!population) + return std::unexpected( + CellularPublishFailure{CellularHostError::configuration, {}, {}, {}}); + candidate = std::move(*population); + } else + candidate = std::move(*raw); + const auto grant = authority_.grant(); + if (!grant.has_value()) + return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}); + auto result = + enabled ? persistence::migrate_cellular_to_population(*db_, *objects_, *grant, *committed_, + *id, config_.build, policy, + config_.lease.commit_timeout) + : persistence::materialize_population_profile(*db_, *objects_, *grant, *committed_, + *id, config_.build, + config_.lease.commit_timeout); + auto accepted = accept_population_publication(std::move(result), std::move(candidate)); + if (!accepted) + return std::unexpected(accepted.error()); + return {}; +} + +std::expected +CellularRegionHost::publish_population(const CellularBatch& batch) { + const auto grant = authority_.grant(); + if (!state_.population || !queued_mutations_.empty() || !committed_.has_value() || + !grant.has_value() || !database()) + return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}); + auto execution = prepare_population_batch(*state_.population, batch); + if (!execution) + return std::unexpected( + CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}}); + const auto trace = encode_population_batch(execution->batch); + const auto id = trace ? commit_id(*committed_, *trace, {}) : std::nullopt; + if (!id) + return std::unexpected( + CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}}); + auto result = persistence::publish_population_batch( + *db_, *objects_, *grant, *committed_, execution->batch, *execution->state, *id, + config_.build, config_.lease.commit_timeout); + auto accepted = + accept_population_publication(std::move(result), OwnedState(std::move(execution->state))); + if (!accepted) + return std::unexpected(accepted.error()); + return persistence::PublishedCellularBatch{std::move(*accepted), {}}; +} } // namespace evo::server diff --git a/src/server/lifecycle/cellular_population_batch.cpp b/src/server/lifecycle/cellular_population_batch.cpp new file mode 100644 index 0000000..7d9933e --- /dev/null +++ b/src/server/lifecycle/cellular_population_batch.cpp @@ -0,0 +1,296 @@ +#include "evo/server/lifecycle/cellular_population_batch.hpp" +#include "evo/server/lifecycle/wildlife_controller.hpp" + +#include +#include + +namespace evo::server { +namespace { +constexpr std::uint64_t magic = 0x315442504F564541ULL; // "AEVOPBT1" +bool shape(const CellularPopulationBatch& batch, bool recording) { + const auto& h = batch.header; + const auto zero = [](const auto& bytes) { + return std::ranges::all_of(bytes, [](auto byte) { return byte == 0; }); + }; + if (h.world == 0 || h.region == 0 || h.frame == 0 || zero(h.predecessor) || + zero(h.predecessor_digest) || h.end_tick < h.begin_tick) + return false; + const auto count = batch.conversion ? std::size_t{1} : batch.biology.ticks.size(); + if (count == 0 || count > max_cellular_batch_ticks || + (!recording && batch.boundaries.size() != count)) + return false; + if (batch.conversion) { + if (h.begin_tick == 0 || h.begin_tick != h.end_tick || batch.conversion->genome == 0 || + !batch.biology.ticks.empty() || batch.biology.header != CellularBatchHeader{}) + return false; + } else { + if (batch.biology.header != h || batch.biology.format_version != cellular_batch_format || + h.end_tick - h.begin_tick != count || !encode_cellular_batch(batch.biology)) + return false; + for (const auto& tick : batch.biology.ticks) + if (!tick.births.empty() || !tick.mutations.empty()) + return false; + } + for (const auto& boundary : batch.boundaries) { + if (boundary.members.size() > biology::max_cellular_tick_lives) + return false; + std::uint64_t previous = 0; + for (const auto& member : boundary.members) { + if (member.life <= previous || member.genome == 0) + return false; + previous = member.life; + } + } + return true; +} +void put(std::vector& out, std::uint64_t value, unsigned width = 8) { + for (unsigned i = 0; i < width; ++i) { + out.push_back(static_cast(value & 255U)); + value >>= 8U; + } +} +class Reader { + public: + explicit Reader(std::span bytes) : bytes_(bytes) {} + std::uint64_t get(unsigned width = 8) { + std::uint64_t value = 0; + for (unsigned i = 0; i < width; ++i) + value |= std::uint64_t{bytes_[at_++]} << (8U * i); + return value; + } + template std::array array() { + std::array value{}; + for (auto& byte : value) + byte = bytes_[at_++]; + return value; + } + std::size_t at() const { + return at_; + } + + private: + std::span bytes_; + std::size_t at_{}; +}; +} // namespace + +std::expected +unattended_population_inputs(const CellularLives& base, const CellularBatchHeader& header) { + if (header.world != base.world() || header.region != base.region() || + header.frame != base.frame() || header.begin_tick != base.tick() || + header.end_tick <= header.begin_tick || + header.end_tick - header.begin_tick > max_cellular_batch_ticks) + return std::unexpected(CellularPopulationError::identity); + auto working = CellularLives::restore(base.snapshot(), base.world(), base.region(), + base.frame(), base.tick()); + if (!working) + return std::unexpected(CellularPopulationError::format); + CellularBatch batch{header, {}}; + while ((*working)->tick() < header.end_tick) { + auto intents = wildlife_intents(**working, (*working)->tick() + 1); + if (!intents) + return std::unexpected(CellularPopulationError::ineligible); + for (const auto& life : (*working)->lives()) { + if (life.account == LifeAccount{}) + continue; + CellularLifeIntent frozen{life.identity.life, {}}; + frozen.intent.frozen = true; + intents->push_back(frozen); + } + std::ranges::sort(*intents, {}, &CellularLifeIntent::life_id); + if (!(*working)->advance_tick(*intents)) + return std::unexpected(CellularPopulationError::ineligible); + batch.ticks.push_back({std::move(*intents), {}, {}}); + } + if (!encode_cellular_batch(batch)) + return std::unexpected(CellularPopulationError::format); + return batch; +} + +std::expected, CellularPopulationError> +execute_population_batch(const CellularPopulation& base, CellularPopulationBatch& batch, + bool record) { + if (!shape(batch, record) || batch.header.world != base.state().world() || + batch.header.region != base.state().region() || + batch.header.frame != base.state().frame() || + batch.header.begin_tick != base.state().tick()) + return std::unexpected(CellularPopulationError::identity); + auto candidate = + CellularPopulation::restore(base.snapshot(), base.state().world(), base.state().region(), + base.state().frame(), base.state().tick()); + if (!candidate) + return std::unexpected(candidate.error()); + if (record) + batch.boundaries.clear(); + const auto check_boundary = [&](std::size_t index) { + CellularPopulationBoundary actual{(*candidate)->conversion_sequence_, + (*candidate)->members_}; + if (record) { + batch.boundaries.push_back(std::move(actual)); + return true; + } + return batch.boundaries[index] == actual; + }; + if (batch.conversion) { + const auto converted = batch.conversion->to_cohort + ? (*candidate)->collapse(batch.conversion->genome) + : (*candidate)->materialize(batch.conversion->genome); + if (!converted || !check_boundary(0)) + return std::unexpected(CellularPopulationError::ineligible); + } else { + for (std::size_t index = 0; index < batch.biology.ticks.size(); ++index) { + const auto& input = batch.biology.ticks[index]; + for (const auto& life : (*candidate)->state().lives()) { + if (life.account == LifeAccount{}) + continue; + const auto intent = std::ranges::find(input.lives, life.identity.life, + &CellularLifeIntent::life_id); + biology::CellularStepIntent frozen; + frozen.frozen = true; + if (intent == input.lives.end() || intent->intent != frozen) + return std::unexpected(CellularPopulationError::ineligible); + } + CellularBatch one{batch.biology.header, {input}, cellular_batch_format}; + one.header.begin_tick = (*candidate)->state().tick(); + one.header.end_tick = one.header.begin_tick + 1; + auto biology = replay_cellular_batch((*candidate)->state(), one); + if (!biology) + return std::unexpected(CellularPopulationError::ineligible); + (*candidate)->state_ = std::move(*biology); + std::erase_if((*candidate)->members_, [&](const auto& member) { + return std::ranges::find((*candidate)->state().lives(), member.life, + [](const auto& life) { return life.identity.life; }) == + (*candidate)->state().lives().end(); + }); + // Verify the persisted policy's exact result against the recorded sample; + // no controller or random materialization is run during replay. + if ((*candidate)->policy_.automatic) { + if ((*candidate)->conversion_sequence_ == ~std::uint64_t{0}) + return std::unexpected(CellularPopulationError::overflow); + (*candidate)->apply_policy(); + } + if (!check_boundary(index)) + return std::unexpected(CellularPopulationError::ineligible); + } + } + return candidate; +} + +std::expected +prepare_population_batch(const CellularPopulation& base, CellularBatch biology) { + CellularPopulationBatch batch{biology.header, std::move(biology), {}, {}}; + auto state = execute_population_batch(base, batch, true); + if (!state) + return std::unexpected(state.error()); + return CellularPopulationExecution{std::move(batch), std::move(*state)}; +} +std::expected +prepare_population_conversion(const CellularPopulation& base, CellularBatchHeader header, + CellularPopulationConversion conversion) { + CellularPopulationBatch batch{header, {}, conversion, {}}; + auto state = execute_population_batch(base, batch, true); + if (!state) + return std::unexpected(state.error()); + return CellularPopulationExecution{std::move(batch), std::move(*state)}; +} +std::expected, CellularPopulationError> +replay_population_batch(const CellularPopulation& base, const CellularPopulationBatch& batch) { + auto copy = batch; + return execute_population_batch(base, copy, false); +} + +std::expected, CellularPopulationError> +encode_population_batch(const CellularPopulationBatch& batch) { + if (!shape(batch, false)) + return std::unexpected(CellularPopulationError::format); + std::vector biological; + if (!batch.conversion) { + auto encoded = encode_cellular_batch(batch.biology); + if (!encoded) + return std::unexpected(CellularPopulationError::format); + biological = std::move(*encoded); + } + std::vector out; + put(out, magic); + put(out, 1, 4); + put(out, batch.conversion ? 1 : 0, 4); + const auto& h = batch.header; + for (const auto value : {h.world, h.region, h.frame}) + put(out, value); + out.insert(out.end(), h.predecessor.begin(), h.predecessor.end()); + out.insert(out.end(), h.predecessor_digest.begin(), h.predecessor_digest.end()); + put(out, h.begin_tick); + put(out, h.end_tick); + put(out, batch.boundaries.size(), 4); + put(out, 0, 4); + put(out, biological.size()); + put(out, batch.conversion ? batch.conversion->genome : 0); + put(out, batch.conversion && batch.conversion->to_cohort ? 1 : 0); + out.insert(out.end(), biological.begin(), biological.end()); + for (const auto& boundary : batch.boundaries) { + put(out, boundary.sequence); + put(out, boundary.members.size()); + for (const auto& member : boundary.members) { + put(out, member.life); + put(out, member.genome); + } + } + return out; +} + +std::expected +decode_population_batch(std::span bytes) { + if (bytes.size() < population_batch_header_bytes || bytes.size() > max_population_batch_bytes) + return std::unexpected(CellularPopulationError::format); + Reader in(bytes); + if (in.get() != magic || in.get(4) != 1) + return std::unexpected(CellularPopulationError::unsupported); + const auto kind = in.get(4); + CellularPopulationBatch batch; + auto& h = batch.header; + h.world = in.get(); + h.region = in.get(); + h.frame = in.get(); + h.predecessor = in.array<16>(); + h.predecessor_digest = in.array<32>(); + h.begin_tick = in.get(); + h.end_tick = in.get(); + const auto count = in.get(4); + if (in.get(4) != 0 || kind > 1 || count == 0 || count > max_cellular_batch_ticks) + return std::unexpected(CellularPopulationError::format); + const auto size = in.get(), genome = in.get(), to_cohort = in.get(); + if (size > max_cellular_batch_bytes || size > bytes.size() - population_batch_header_bytes || + to_cohort > 1 || (kind == 0 && (genome != 0 || to_cohort != 0)) || (kind == 1 && size != 0)) + return std::unexpected(CellularPopulationError::format); + if (kind == 1) + batch.conversion = {genome, to_cohort == 1}; + else { + auto biological = decode_cellular_batch( + bytes.subspan(population_batch_header_bytes, static_cast(size))); + if (!biological) + return std::unexpected(CellularPopulationError::format); + batch.biology = std::move(*biological); + } + std::size_t at = population_batch_header_bytes + static_cast(size); + for (std::uint64_t index = 0; index < count; ++index) { + if (bytes.size() - at < 16) + return std::unexpected(CellularPopulationError::format); + Reader boundary_reader(bytes.subspan(at)); + CellularPopulationBoundary boundary; + boundary.sequence = boundary_reader.get(); + const auto members = boundary_reader.get(); + if (members > biology::max_cellular_tick_lives || members * 16 > bytes.size() - at - 16) + return std::unexpected(CellularPopulationError::format); + for (std::uint64_t member = 0; member < members; ++member) + boundary.members.push_back({boundary_reader.get(), boundary_reader.get()}); + at += boundary_reader.at(); + batch.boundaries.push_back(std::move(boundary)); + } + if (at != bytes.size() || !shape(batch, false)) + return std::unexpected(CellularPopulationError::format); + const auto encoded = encode_population_batch(batch); + if (!encoded || !std::ranges::equal(*encoded, bytes)) + return std::unexpected(CellularPopulationError::format); + return batch; +} +} // namespace evo::server diff --git a/tests/cellular_population_tests.cpp b/tests/cellular_population_tests.cpp index fa3cc51..8960f6b 100644 --- a/tests/cellular_population_tests.cpp +++ b/tests/cellular_population_tests.cpp @@ -12,23 +12,6 @@ using namespace evo::server; using namespace evo::fixture; using Population = std::unique_ptr; -Owner descendants() { - auto state = opening(1, true); - std::set mutated; - for (unsigned step = 0; step < 8192; ++step) { - const auto input = inputs(*state, 1, mutated); - const auto executed = - checked(state->advance_tick_with_mutations(input.lives, input.mutations), - "descendant fixture tick"); - for (const auto& mutation : executed.mutations) - if (mutation.result == CellularMutationResult::applied) - mutated.insert(mutation.life_id); - } - require(std::ranges::any_of(state->lives(), - [](const auto& life) { return life.account == LifeAccount{}; }), - "fixture has no funded descendant"); - return state; -} Population restored(std::span bytes, const CellularLives& state) { return checked(CellularPopulation::restore(bytes, state.world(), state.region(), state.frame(), state.tick()), @@ -64,7 +47,8 @@ void refresh(std::vector& bytes) { put(bytes, bytes.size() - 8, hash); } void conversions_and_codec() { - auto population = checked(CellularPopulation::create(descendants()), "population create"); + auto population = + checked(CellularPopulation::create(cellular_descendants()), "population create"); const auto genome = wildlife_genome(*population); const auto before = population->snapshot(); const auto biology = population->state().snapshot(); @@ -129,7 +113,7 @@ void conversions_and_codec() { } void automatic_and_restart() { - auto initial = descendants(); + auto initial = cellular_descendants(); auto policy = checked(CellularPopulation::create(restore(*initial), {true, 1200, 1800}), "automatic owner"); auto exact = checked(CellularPopulation::create(restore(*initial)), "individual control"); @@ -170,8 +154,9 @@ void write(const char* path, const CellularPopulation& population) { require(!stream.fail(), "test checkpoint write failed"); } void checkpoint(const char* path, const char* expected) { - auto population = checked(CellularPopulation::create(descendants(), {true, 1200, 1800}), - "process checkpoint owner"); + auto population = + checked(CellularPopulation::create(cellular_descendants(), {true, 1200, 1800}), + "process checkpoint owner"); for (unsigned tick = 0; tick < 250; ++tick) frozen_tick(*population); require(!population->members().empty(), "process checkpoint has no cohort"); diff --git a/tests/fixtures/cellular_ecology.hpp b/tests/fixtures/cellular_ecology.hpp index cbd03c2..9973ad5 100644 --- a/tests/fixtures/cellular_ecology.hpp +++ b/tests/fixtures/cellular_ecology.hpp @@ -44,7 +44,8 @@ inline Owner restore(const CellularLives& state) { "reference restore"); } -inline Owner opening(std::uint64_t seed, bool rich, std::uint32_t count = 4) { +inline Owner opening(std::uint64_t seed, bool rich, std::uint32_t count = 4, + bool populated = true) { require(count > 0 && count <= biology::max_cellular_tick_lives, "fixture population bound"); const std::uint32_t side = count == 4 ? 6'000 : 48'000; const auto world = 200 + seed; @@ -106,6 +107,8 @@ inline Owner opening(std::uint64_t seed, bool rich, std::uint32_t count = 4) { checked(CellularLives::create(*primitive, std::move(habitat), {{side, side}, 0, 293'150}, CellularLivesProfile::foraging), "reference cellular owner"); + if (!populated) + return state; require(state->advance_tick(std::span{}).has_value(), "reference genesis tick"); for (std::uint64_t index = 0; index < count; ++index) { @@ -177,4 +180,21 @@ inline CellularTickIntents inputs(const CellularLives& state, std::uint64_t seed return tick; } +inline Owner cellular_descendants() { + auto state = opening(1, true); + std::set mutated; + for (unsigned step = 0; step < 8192; ++step) { + const auto input = inputs(*state, 1, mutated); + const auto executed = + checked(state->advance_tick_with_mutations(input.lives, input.mutations), + "descendant fixture tick"); + for (const auto& mutation : executed.mutations) + if (mutation.result == CellularMutationResult::applied) + mutated.insert(mutation.life_id); + } + require(std::ranges::any_of(state->lives(), + [](const auto& life) { return life.account == LifeAccount{}; }), + "fixture has no funded descendant"); + return state; +} } // namespace evo::fixture diff --git a/tests/population_checkpoint_tests.cpp b/tests/population_checkpoint_tests.cpp new file mode 100644 index 0000000..69b1699 --- /dev/null +++ b/tests/population_checkpoint_tests.cpp @@ -0,0 +1,159 @@ +#include "evo/persistence/region_checkpoint.hpp" +#include "evo/server/lifecycle/cellular_population_batch.hpp" +#include "fixtures/cellular_ecology.hpp" + +namespace { +using namespace evo; +using namespace evo::server; +using namespace evo::persistence; +using namespace evo::fixture; +template std::array filled(std::uint8_t byte) { + std::array out{}; + out.fill(byte); + return out; +} +CellularBatchHeader header(const CellularPopulation& state, std::uint64_t ticks) { + const auto& s = state.state(); + return {s.world(), s.region(), s.frame(), filled<16>(1), + filled<32>(2), s.tick(), s.tick() + ticks}; +} +CellularBatch inputs_batch(const CellularPopulation& base) { + CellularBatch batch{header(base, 64), {}}; + auto biological = restore(base.state()); + for (unsigned tick = 0; tick < 64; ++tick) { + auto inputs = + checked(wildlife_intents(*biological, biological->tick() + 1), "batch wildlife"); + for (const auto& life : biological->lives()) { + if (life.account == LifeAccount{}) + continue; + CellularLifeIntent frozen{life.identity.life, {}}; + frozen.intent.frozen = true; + inputs.push_back(frozen); + } + std::ranges::sort(inputs, {}, &CellularLifeIntent::life_id); + batch.ticks.push_back({inputs, {}, {}}); + checked(biological->advance_tick(inputs), "build biological input fixture"); + } + return batch; +} +void traces() { + auto population = checked( + CellularPopulation::create(cellular_descendants(), {true, 1200, 1800}), "batch population"); + std::uint64_t converted = 0; + for (unsigned interval = 0; interval < 6; ++interval) { + auto inputs = inputs_batch(*population); + require(checked(unattended_population_inputs(population->state(), inputs.header), + "unattended worker batch") == inputs, + "worker interval changed frozen/wildlife input contract"); + const auto predecessor = population->snapshot(); + auto execution = checked(prepare_population_batch(*population, inputs), "prepare trace"); + const auto bytes = checked(encode_population_batch(execution.batch), "encode trace"); + const auto decoded = checked(decode_population_batch(bytes), "decode trace"); + require(decoded == execution.batch, "population trace round trip"); + auto replay = checked(replay_population_batch(*population, decoded), "replay trace"); + require(replay->snapshot() == execution.state->snapshot() && + population->snapshot() == predecessor, + "trace changed predecessor or replay result"); + auto altered = decoded; + ++altered.boundaries.back().sequence; + require(!replay_population_batch(*population, altered), "altered sequence admitted"); + altered = decoded; + altered.biology.ticks.front().lives.front().intent.frozen = false; + require(!replay_population_batch(*population, altered), + "unfrozen controlled step admitted"); + for (std::size_t length = 0; length < bytes.size(); length += 113) + require(!decode_population_batch(std::span{bytes}.first(length)), + "truncated trace admitted"); + auto extended = bytes; + extended.push_back(0); + require(!decode_population_batch(extended), "appended trace bytes admitted"); + converted += replay->members().size(); + population = std::move(replay); + } + require(converted > 0 && !population->members().empty(), "batch test has no cohorts"); + const auto genome = population->members().front().genome; + const auto before = population->state().snapshot(); + auto conversion = + checked(prepare_population_conversion(*population, header(*population, 0), {genome, false}), + "manual trace"); + const auto encoded = checked(encode_population_batch(conversion.batch), "manual encode"); + const auto decoded = checked(decode_population_batch(encoded), "manual decode"); + auto replay = checked(replay_population_batch(*population, decoded), "manual replay"); + require(replay->state().snapshot() == before && replay->members().empty(), + "manual materialization changed biology"); +} +void manifests() { + auto state = cellular_descendants(); + auto population = checked(CellularPopulation::create(restore(*state), {true, 1200, 1800}), + "manifest population"); + CheckpointManifest source; + source.profile = CheckpointProfile::cellular_lives_v1; + source.cellular_versions = cellular_v9_versions; + source.history_profile = cellular_revision_history_profile; + source.key = {state->world(), state->region()}; + source.frame = state->frame(); + source.tick_microseconds = 50000; + source.sequence = 2; + source.commit = filled<16>(1); + source.predecessor_commit = filled<16>(2); + source.predecessor_digest = filled<32>(3); + source.holder = filled<16>(4); + source.epoch = 1; + source.begin_tick = state->tick() - 1; + source.end_tick = state->tick(); + source.build = filled<32>(5); + source.payload_bytes = state->snapshot().size(); + source.payload_sha256 = filled<32>(6); + source.batch = {128, filled<32>(7), 0, {}}; + require(encode_manifest(source).has_value(), "source manifest fixture"); + LeaseGrant lease{filled<16>(4), 1, state->frame(), std::chrono::milliseconds{20000}}; + auto next = checked(population_profile_transition_manifest( + source, filled<32>(8), filled<16>(9), lease, source.build, + population->snapshot().size(), filled<32>(10)), + "profile transition"); + require(next.profile == CheckpointProfile::cellular_population_v1 && supported(next), + "wrong profile"); + const auto encoded = checked(encode_manifest(next), "population manifest"); + require(checked(decode_manifest(encoded), "manifest decode") == next, + "manifest profile changed"); + auto restored_population = + checked(restore_population_payload(population->snapshot(), next), "profile payload"); + require(restored_population->snapshot() == population->snapshot(), + "population payload changed"); + require(!restore_cellular_payload(population->snapshot(), next), + "population mislabeled as cellular"); + auto back = checked(population_profile_transition_manifest( + next, filled<32>(11), filled<16>(12), lease, source.build, + state->snapshot().size(), filled<32>(13)), + "reverse transition"); + require(back.end_tick == source.end_tick && back.profile == source.profile && + checked(restore_cellular_payload(state->snapshot(), back), "unwrapped payload") + ->snapshot() == state->snapshot(), + "profile transition changed biology/time"); + auto execution = checked(prepare_population_batch(*population, inputs_batch(*population)), + "manifest tick fixture"); + const auto trace = checked(encode_population_batch(execution.batch), "manifest trace fixture"); + const auto advancing = population_successor_manifest( + next, filled<32>(11), filled<16>(12), lease, execution.state->state().tick(), source.build, + execution.state->snapshot().size(), filled<32>(13), {trace.size(), filled<32>(14), 0, {}}); + require(advancing.has_value() && + restore_population_payload(execution.state->snapshot(), *advancing).has_value(), + "population successor refused"); + require(!clone_manifest(next, filled<32>(1), {9999, 1}, filled<16>(3), 100, filled<32>(4), {}), + "population clone admitted without identity remapping"); + auto malformed = next; + malformed.sequence = 0; + require(!encode_manifest(malformed), "population genesis admitted"); + malformed = next; + malformed.cellular_versions = cellular_v8_versions; + require(!supported(malformed), "unsupported nested rules admitted"); +} +} // namespace +int main() try { + traces(); + manifests(); + return 0; +} catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; +} diff --git a/tests/population_publication_tests.cpp b/tests/population_publication_tests.cpp new file mode 100644 index 0000000..c853eb2 --- /dev/null +++ b/tests/population_publication_tests.cpp @@ -0,0 +1,486 @@ +#include "evo/persistence/account_control_store.hpp" +#include "evo/persistence/population_checkpoint_store.hpp" +#include "evo/persistence/sha256.hpp" +#include "evo/server/cellular_region_host.hpp" +#include "fixtures/cellular_ecology.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +void run_population_publication_tests(); +namespace { +using namespace evo; +using namespace evo::server; +using namespace evo::persistence; +using namespace evo::fixture; +using namespace std::chrono_literals; +template T take(std::expected value, const char* where) { + if (!value) { + std::string message{where}; + if constexpr (requires { value.error().cause.sqlstate; }) + message += " SQLSTATE=" + std::string(value.error().cause.sqlstate.data(), 5); + if constexpr (requires { value.error().store.cause.sqlstate; }) + message += " SQLSTATE=" + std::string(value.error().store.cause.sqlstate.data(), 5); + throw std::runtime_error(message); + } + if constexpr (!std::is_void_v) + return std::move(*value); +} +template T take(std::optional value, const char* where) { + if (!value.has_value()) + throw std::runtime_error(where); + return std::move(*value); +} +std::string environment(const char* name) { + const auto* value = std::getenv(name); + require(value != nullptr, "missing private test environment"); + return value; +} +std::unique_ptr connection(bool owner = false) { + return take(Postgres::connect({environment("PGHOST"), + {}, + environment("PGPORT"), + environment("PGDATABASE"), + environment(owner ? "PGUSER" : "EVO_PG_RUNTIME_USER"), + {}, + {}}, + 5s), + "population connection"); +} +Uuid identity(unsigned number) { + Uuid id{}; + id[0] = 0xDA; + id[1] = 0x12; + id[14] = static_cast(number >> 8U); + id[15] = static_cast(number); + return id; +} +std::string hexadecimal(std::span bytes, bool prefix = true) { + std::string out = prefix ? "\\x" : ""; + constexpr char digits[] = "0123456789abcdef"; + for (const auto b : bytes) { + out.push_back(digits[b >> 4U]); + out.push_back(digits[b & 15U]); + } + return out; +} +struct Fixture { + RegionKey key{30201, 1}; + std::unique_ptr owner = connection(true), db = connection(); + std::unique_ptr objects; + LeaseGrant lease; + Sha256Digest build{}; + CommittedCheckpoint head; + std::unique_ptr raw; + std::unique_ptr population; + unsigned next_id = 1; + Uuid id() { + return identity(next_id++); + } + CellularBatchHeader header(std::uint64_t count) const { + return {key.world, + key.region, + 1, + head.head.commit, + head.head.digest, + head.head.end_tick, + head.head.end_tick + count}; + } +}; +void funded_source(Fixture& f) { + f.build.fill(0xDA); + const std::array build{hexadecimal(f.build)}; + take(f.owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES($1::bytea) " + "ON CONFLICT DO NOTHING", + build, 5s), + "retain test build"); + take(f.owner->query("INSERT INTO evolution.life_allocators VALUES(30201,100)", {}, 5s), + "test allocator"); + f.objects = take( + FilesystemObjectStore::open(environment("PGHOST"), f.key, max_cellular_population_bytes), + "population object namespace"); + f.raw = opening(30001, true, 4, false); + f.head = + take(provision_cellular_region(*f.owner, *f.objects, f.key, 1, *f.raw, f.id(), f.build, 5s), + "empty cellular genesis"); + f.lease = + take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "population test lease"); + CellularBatch birth{f.header(1), {{}}}; + for (std::uint8_t index = 1; index <= 4; ++index) { + Uuid account{}; + account.back() = index; + const auto control = + take(claim_account_control(*f.db, f.key, f.lease, account, 1, index, 60000ms, 5s), + "founder account control"); + const auto reserved = take( + reserve_primitive_life(*f.db, f.key, f.lease, account, control.generation, 1, 1, 5s), + "funded founder reservation"); + birth.ticks.front().births.push_back({account, 1, control.generation, reserved.identity}); + } + f.raw = take(replay_cellular_batch(*f.raw, birth), "funded founder replay"); + f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, birth, *f.raw, f.id(), + f.build, 5s), + "funded founder publication") + .committed; + while (f.raw->next_bud_sequence() == 1 && f.raw->tick() < 20000) { + CellularBatch batch{f.header(1), {}}; + for (unsigned tick = 0; tick < 64; ++tick) { + auto intents = + take(wildlife_intents(*f.raw, f.raw->tick() + 1), "funded source wildlife"); + for (const auto& life : f.raw->lives()) { + if (life.account == LifeAccount{}) + continue; + CellularLifeIntent intent{life.identity.life, {}}; + intent.intent.bud = (f.raw->tick() + life.identity.life) % 32 == 0; + intents.push_back(intent); + } + std::ranges::sort(intents, {}, &CellularLifeIntent::life_id); + take(f.raw->advance_tick(intents), "funded source biology"); + batch.ticks.push_back({intents, {}, {}}); + if (f.raw->next_bud_sequence() > 1) + break; + } + batch.header.end_tick = f.raw->tick(); + f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, batch, *f.raw, + f.id(), f.build, 5s), + "funded source batch") + .committed; + take(renew_region_lease(*f.db, f.key, f.lease, 60000ms, 5s), "source lease renewal"); + } + require(std::ranges::any_of(f.raw->lives(), + [](const auto& life) { return life.account == LifeAccount{}; }), + "durable fixture has no funded player-derived wildlife"); +} +CellularPopulationExecution unattended(Fixture& f) { + auto working = restore(f.population->state()); + CellularBatch batch{f.header(64), {}}; + for (unsigned tick = 0; tick < 64; ++tick) { + auto intents = take(wildlife_intents(*working, working->tick() + 1), "unattended wildlife"); + for (const auto& life : working->lives()) { + if (life.account == LifeAccount{}) + continue; + CellularLifeIntent frozen{life.identity.life, {}}; + frozen.intent.frozen = true; + intents.push_back(frozen); + } + std::ranges::sort(intents, {}, &CellularLifeIntent::life_id); + take(working->advance_tick(intents), "unattended fixture step"); + batch.ticks.push_back({intents, {}, {}}); + } + return take(prepare_population_batch(*f.population, std::move(batch)), + "population recorded interval"); +} +void recover(Fixture& f) { + const std::array builds{f.build}; + auto db = connection(); + auto recovered = + take(recover_population_checkpoint(*db, *f.objects, f.key, f.lease, builds, 5s), + "population durable recovery"); + f.head = std::move(recovered.committed); + f.population = std::move(recovered.state); +} +void conversions_and_retries(Fixture& f) { + const auto biology = f.raw->snapshot(); + Uuid account{}; + account.back() = 1; + const auto input = + encode_cellular_mutation_input({account, 9991, f.head.head.end_tick + 1, {100, 1, 1, 1}}); + take(f.db->query("BEGIN", {}, 5s), "reservation guard transaction"); + take(reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 9991, input, 5s), + "reserve pending player choice"); + const auto blocked = migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), + f.build, {true, 1200, 1800}, 5s); + require(!blocked && std::string(blocked.error().store.cause.sqlstate.data(), 5) == "EVL18", + "population entry stranded a reserved player choice"); + take(f.db->query("ROLLBACK", {}, 5s), "rollback guard fixture"); + f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), + f.build, {true, 1200, 1800}, 5s), + "initial population transition"); + recover(f); + require(f.population->state().snapshot() == biology && f.population->members().empty(), + "transition changed biology"); + const auto refused = + reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 9991, input, 5s); + require(!refused && std::string(refused.error().cause.sqlstate.data(), 5) == "EVL06", + "population profile admitted a player mutation reservation"); + const auto wildlife = std::ranges::find_if(f.population->state().lives(), [](const auto& life) { + return life.account == LifeAccount{}; + }); + require(wildlife != f.population->state().lives().end(), "missing durable descendant"); + const auto genome = wildlife->identity.genome; + auto conversion = + take(prepare_population_conversion(*f.population, f.header(0), {genome, true}), + "collapse sample"); + auto bad = conversion.batch; + ++bad.boundaries.front().sequence; + require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, bad, *conversion.state, + f.id(), f.build, 5s), + "altered sample published"); + const auto commit = f.id(); + f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, conversion.batch, + *conversion.state, commit, f.build, 5s), + "durable collapse"); + const PopulationPreparedCommit prepared{ + f.key, commit, f.head.manifest, + take(encode_population_batch(conversion.batch), "prepared trace")}; + require(take(retry_population_prepared(*f.db, prepared, 5s), "idempotent retry").head == + f.head.head, + "retry advanced head"); + require(take(resolve_population_prepared(*f.db, prepared, 5s), "read-only resolve").status == + Resolution::committed, + "commit unresolved"); + auto changed = prepared; + auto changed_manifest = take(decode_manifest(changed.manifest), "conflict manifest"); + changed_manifest.payload_sha256[0] ^= 1; + changed.manifest = take(encode_manifest(changed_manifest), "conflict encoding"); + require(take(resolve_population_prepared(*f.db, changed, 5s), "conflicting resolve").status == + Resolution::conflict, + "changed commit was accepted"); + require(!publish_region_checkpoint(*f.db, f.head.manifest, {}, 5s), + "generic publisher bypassed profile gate"); + const auto expected = conversion.state->snapshot(); + recover(f); + require(f.population->snapshot() == expected && !f.population->members().empty(), + "committed sample not recovered"); + require(f.population->state().snapshot() == biology, "collapse healed/aged/moved biology"); + for (unsigned oscillation = 0; oscillation < 8; ++oscillation) { + const bool collapse = oscillation % 2 != 0; + auto next = + take(prepare_population_conversion(*f.population, f.header(0), {genome, collapse}), + "conversion oscillation"); + f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, next.batch, + *next.state, f.id(), f.build, 5s), + "publish oscillation"); + recover(f); + require(f.population->state().snapshot() == biology, + "durable conversion changed exact state"); + } +} +void process_loss(Fixture& f, int point) { + auto next = unattended(f); + const auto predecessor = f.head.head; + const auto expected = next.state->snapshot(); + const auto commit = f.id(); + int channel[2]{}; + require(::pipe(channel) == 0, "process acknowledgement pipe"); + const auto child = ::fork(); + require(child >= 0, "population fork"); + if (child == 0) { + ::close(channel[0]); + try { + auto db = connection(); + PublishHooks hooks; + if (point == 0) + hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); }; + const auto published = + publish_population_batch(*db, *f.objects, f.lease, f.head, next.batch, *next.state, + commit, f.build, 5s, hooks); + if (!published) + ::_exit(20); + if (point == 2) { + const char ack = 'A'; + if (::write(channel[1], &ack, 1) != 1) + ::_exit(21); + } + ::kill(::getpid(), SIGKILL); + } catch (...) { + ::_exit(22); + } + ::_exit(23); + } + ::close(channel[1]); + char ack{}; + const auto acknowledged = ::read(channel[0], &ack, 1); + ::close(channel[0]); + int status{}; + require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) && + WTERMSIG(status) == SIGKILL, + "child missed population crash boundary"); + require((point == 2 && acknowledged == 1 && ack == 'A') || (point != 2 && acknowledged == 0), + "unexpected permanent acknowledgement"); + recover(f); + if (point == 0) + require(f.head.head == predecessor, "uncommitted process published a head"); + else + require(f.head.head.commit == commit && f.population->snapshot() == expected, + "committed process loss changed state"); +} +void ambiguous_publication(Fixture& f, bool terminate) { + auto next = unattended(f); + const auto expected = next.state->snapshot(); + auto locker = connection(true); + take(locker->query("BEGIN", {}, 5s), "population lease row transaction"); + const std::array key{std::to_string(f.key.world), std::to_string(f.key.region)}; + take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=$1::evolution.u64 " + "AND region_id=$2::evolution.u64 FOR UPDATE", + key, 5s), + "hold population publication"); + auto timed = connection(); + auto unknown = publish_population_batch(*timed, *f.objects, f.lease, f.head, next.batch, + *next.state, f.id(), f.build, 300ms); + require(!unknown && unknown.error().kind == PublicationError::unknown, + "population publication failed to retain ambiguous request"); + if (terminate) { + const std::array user{environment("EVO_PG_RUNTIME_USER")}; + take(locker->query("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE " + "usename=$1 AND wait_event_type='Lock'", + user, 5s), + "terminate uncommitted population writer"); + } + take(locker->query("ROLLBACK", {}, 5s), "release population publication"); + auto resolver = connection(); + bool resolved = false; + for (unsigned attempt = 0; attempt < 200; ++attempt) { + auto result = take(resolve_population_prepared(*resolver, unknown.error().prepared, 5s), + "resolve ambiguous population publication"); + if (result.status == (terminate ? Resolution::absent : Resolution::committed)) { + resolved = true; + break; + } + std::this_thread::sleep_for(25ms); + } + require(resolved, "ambiguous population publication did not resolve"); + f.head = take(retry_population_prepared(*resolver, unknown.error().prepared, 5s), + "retry exact population request"); + recover(f); + require(f.population->snapshot() == expected, "ambiguous publication changed retained sample"); +} +void isolation_and_reverse(Fixture& f) { + auto interval = unattended(f); + const auto old_lease = f.lease; + take(release_region_lease(*f.db, f.key, f.lease, 5s), "release old authority"); + auto new_db = connection(); + f.lease = take(acquire_region_lease(*new_db, f.key, f.build, 60000ms, 5s), + "new population authority"); + require(f.lease.epoch > old_lease.epoch, "authority epoch did not advance"); + const auto old_head = f.head.head; + auto stale = publish_population_batch(*f.db, *f.objects, old_lease, f.head, interval.batch, + *interval.state, f.id(), f.build, 5s); + require(!stale && stale.error().store.kind == LeaseStoreError::fenced, + "stale population writer published"); + recover(f); + require(f.head.head == old_head, "fenced writer changed head"); + const auto expected = f.population->state().snapshot(); + f.head = take( + materialize_population_profile(*new_db, *f.objects, f.lease, f.head, f.id(), f.build, 5s), + "return to cellular profile"); + const std::array builds{f.build}; + auto recovered = + take(recover_cellular_checkpoint(*new_db, *f.objects, f.key, f.lease, builds, 5s), + "cellular recovery after materialization"); + require(recovered.state->snapshot() == expected && + recovered.manifest.profile == CheckpointProfile::cellular_lives_v1, + "reverse transition changed biology or time"); + f.raw = std::move(recovered.state); +} +void missing_and_corrupt_objects(Fixture& f) { + const auto manifest = take(decode_manifest(f.head.manifest), "object test manifest"); + const auto path = f.objects->directory() / hexadecimal(manifest.payload_sha256, false); + const auto backup = path.string() + ".population-test"; + const std::array builds{f.build}; + std::filesystem::rename(path, backup); + auto missing = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s); + std::filesystem::rename(backup, path); + require(!missing && missing.error().kind == RecoveryError::missing_payload, + "missing population payload did not fail closed"); + std::filesystem::rename(path, backup); + { + std::ofstream corrupt(path, std::ios::binary); + corrupt << 'X'; + } + auto corrupt = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s); + std::filesystem::remove(path); + std::filesystem::rename(backup, path); + require(!corrupt && corrupt.error().kind == RecoveryError::corrupt_payload, + "corrupt population payload did not fail closed"); + recover(f); +} +void host_ownership_and_admission(Fixture& f) { + take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture host authority"); + const auto make_host = [&](bool enabled) { + auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), f.key, + max_cellular_population_bytes), + "host objects"); + CellularHostConfig config{f.key, 1, {}, f.build, {f.build}}; + if (enabled) + config.population_policy = CellularTierPolicy{true, 1200, 1800}; + return take(CellularRegionHost::create( + std::move(config), + [] { + return Postgres::connect({environment("PGHOST"), + {}, + environment("PGPORT"), + environment("PGDATABASE"), + environment("EVO_PG_RUNTIME_USER"), + {}, + {}}, + 5s); + }, + std::move(objects)), + "population host"); + }; + auto host = make_host(false); + require(host->step().ready && !host->population_mode(), "raw host recovery"); + require(!host->set_population_mode(true), "default host enabled population entry"); + host->drain(); + host = make_host(true); + require(host->step().ready, "opt-in host recovery"); + const auto before = host->state()->snapshot(); + const auto tick = host->state()->tick(); + take(host->set_population_mode(true), "host population entry"); + require(host->population_mode() && host->state()->snapshot() == before, + "host entry changed biology"); + auto population = + take(CellularPopulation::create( + take(CellularLives::restore(before, f.key.world, f.key.region, 1, tick), + "host reference restore"), + {true, 1200, 1800}), + "host reference population"); + f.head = take(host->committed(), "host population head"); + auto next = unattended(f); // uses the same deterministic frozen/wildlife input contract. + // The fixture's old envelope may have a different membership history; biology is shared. + auto expected = + take(prepare_population_batch(*population, next.batch.biology), "host reference batch"); + take(host->publish(next.batch.biology), "host population publication"); + require(host->population_mode() && + host->state()->snapshot() == expected.state->state().snapshot(), + "host population continuation differs"); + const auto committed = take(host->committed(), "host continued head"); + const auto continued = host->state()->snapshot(); + host->drain(); + host = make_host(false); + require(host->step().ready && host->population_mode() && + take(host->committed(), "host recovered head").head == committed.head && + host->state()->snapshot() == continued, + "restarted host failed to recover population head"); + Uuid account{}; + account.back() = 1; + // A returning player's operation completes the durable reverse transition before lookup. + [[maybe_unused]] const auto queried = host->query_primitive_life(account, 1, 1); + require(!host->population_mode() && host->state()->snapshot() == continued && + host->state()->tick() == tick + 64, + "player operation did not materialize first"); + host->drain(); +} +} // namespace +void run_population_publication_tests() { + Fixture fixture; + funded_source(fixture); + conversions_and_retries(fixture); + ambiguous_publication(fixture, false); + ambiguous_publication(fixture, true); + process_loss(fixture, 0); + process_loss(fixture, 1); + process_loss(fixture, 2); + missing_and_corrupt_objects(fixture); + isolation_and_reverse(fixture); + host_ownership_and_admission(fixture); + std::puts("population publication, exact recovery, process loss and fencing passed"); +} diff --git a/tests/postgres_tests.cpp b/tests/postgres_tests.cpp index d61f9a2..4debaf3 100644 --- a/tests/postgres_tests.cpp +++ b/tests/postgres_tests.cpp @@ -9,6 +9,7 @@ void run_account_control_store_tests(); void run_life_reservation_store_tests(); void run_primitive_publication_tests(); void run_cellular_publication_tests(); +void run_population_publication_tests(); void run_probe_handoff_store_tests(); void run_region_action_tests(); void run_region_checkpoint_store_tests(); @@ -125,6 +126,10 @@ void run() { } // namespace int main(int argc, char** argv) { try { + if (argc == 2 && std::string_view(argv[1]) == "--population-only") { + run_population_publication_tests(); + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--life-reservations-only") { run_life_reservation_store_tests(); run_primitive_publication_tests(); @@ -140,6 +145,7 @@ int main(int argc, char** argv) { run_life_reservation_store_tests(); run_primitive_publication_tests(); run_cellular_publication_tests(); + run_population_publication_tests(); run_probe_handoff_store_tests(); run_region_action_tests(); std::puts("region lease fencing integration passed"); diff --git a/tests/region_migration_tests.cpp b/tests/region_migration_tests.cpp index d274d76..81b01a3 100644 --- a/tests/region_migration_tests.cpp +++ b/tests/region_migration_tests.cpp @@ -196,6 +196,20 @@ void upgrade_across_migration_and_barrier() { " -v migration_sha_28=$(s 0028_cellular_v3_trace_bounds.sql)" " -v migration_sha_29=$(s 0029_cellular_mutation_reservations.sql)" " -v migration_sha_30=$(s 0030_cellular_birth_publication_binding.sql)" + " -v migration_sha_31=$(s 0031_fix_cellular_birth_publication_record.sql)" + " -v migration_sha_32=$(s 0032_fix_cellular_v3_manifest_normalization.sql)" + " -v migration_sha_33=$(s 0033_fix_cellular_action_tick_offset.sql)" + " -v migration_sha_34=$(s 0034_cellular_v4_engulf_bundle.sql)" + " -v migration_sha_35=$(s 0035_cellular_mutation_publication.sql)" + " -v migration_sha_36=$(s 0036_cellular_v5_budding_bundle.sql)" + " -v migration_sha_37=$(s 0037_cellular_life_endings.sql)" + " -v migration_sha_38=$(s 0038_cellular_v6_presence_bundle.sql)" + " -v migration_sha_39=$(s 0039_account_identities.sql)" + " -v migration_sha_40=$(s 0040_cellular_mutation_decline.sql)" + " -v migration_sha_41=$(s 0041_cellular_v7_balance_bundle.sql)" + " -v migration_sha_42=$(s 0042_cellular_v8_reproduction_bundle.sql)" + " -v migration_sha_43=$(s 0043_cellular_v9_foraging_bundle.sql)" + " -v migration_sha_44=$(s 0044_exact_population_publication.sql)" " -v runtime_role=evo_runtime_test -f migrations/apply.sql" " > /dev/null 2>&1") != 0); CHECK(history(*owner_db) == "1,2,3,4,5,6,7"); @@ -212,7 +226,7 @@ void upgrade_across_migration_and_barrier() { " EVO_PG_RUNTIME_ROLE=evo_runtime_test bash \"$EVO_SOURCE_ROOT/" "tools/migrate-postgres.sh\" > /dev/null") == 0); CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24," - "25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43"); + "25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44"); CHECK(scalar(*owner_db, "SELECT (c.condeferrable AND c.condeferred AND " "c.confrelid = 'evolution.region_commits'::regclass)::text " "FROM pg_catalog.pg_constraint c WHERE c.conrelid = " diff --git a/tools/ci/postgres-check.sh b/tools/ci/postgres-check.sh index 518945f..42926aa 100644 --- a/tools/ci/postgres-check.sh +++ b/tools/ci/postgres-check.sh @@ -2,6 +2,7 @@ set -euo pipefail # Usage: postgres-check.sh [--full|--sanitizer|--suite-only] # postgres-check.sh --life-reservations-only [] +# postgres-check.sh --population-only # --full (default) is the complete qualification: the C++ suite, the backup, DR and # upgrade process checks, then the SQL-only migration, permission and checksum checks. # --sanitizer keeps everything that runs the binaries built beside the test binary @@ -11,6 +12,7 @@ set -euo pipefail root="$(cd "$(dirname "$0")/../.." && pwd)" mode=full case "${1:-}" in + --population-only) mode=population; shift ;; --full) shift ;; --sanitizer) mode=sanitizer; shift ;; --suite-only) mode=suite-only; shift ;; @@ -56,6 +58,11 @@ if [ "$mode" = life-reservations ]; then bash "$root/tools/ci/life-reservations-concurrency.sh" exit 0 fi +if [ "$mode" = population ]; then + phase population + "$1" --population-only + exit 0 +fi # M7.9 clone target: a separate database with its own runtime group and login, # so a clone shares neither the source's database nor its runtime credentials. psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_clone_runtime NOLOGIN' -c 'CREATE ROLE evo_clone_login LOGIN IN ROLE evo_clone_runtime' @@ -122,7 +129,7 @@ PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" -f "$root/migrations/0001_region_bootstrap.sql" > /dev/null test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1 PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43 +test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44 # v2 heads have no payload binding: v2->v3 fails closed and leaves history at v2. v2_sha="$(sha256sum "$root/migrations/0002_region_fencing.sql")" v2_sha="${v2_sha%% *}" @@ -143,7 +150,7 @@ PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '01000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32), 'hex'))" > /dev/null PGDATABASE=v3_commits bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 43:1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43 +test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 44:1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44 test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) FROM evolution.region_commits")" = 1 # Review P1: a v4 database holding a published (possibly terminal) checkpoint has # no durable decisions for it. The v5 upgrade must refuse before changing anything. @@ -197,7 +204,7 @@ PGDATABASE=v6_populated psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '02000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32) || repeat('00', 80), 'hex'))" > /dev/null PGDATABASE=v6_populated bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43 +test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT count(*) FROM evolution.region_commits c, LATERAL evolution.parse_checkpoint_manifest(c.manifest) m WHERE m.commit_sequence = 0 AND m.predecessor_commit IS NULL")" = 1 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(conname, ',') FROM pg_constraint WHERE conrelid = 'evolution.region_heads'::regclass AND contype = 'c' AND pg_get_constraintdef(oid) LIKE '%end_tick%'")" = region_heads_published_tick phase permission-audit @@ -250,9 +257,9 @@ expect_audit_failure 'REVOKE EXECUTE ON FUNCTION evolution.claim_account_control bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again. phase checksum-drift # Unknown future versions and checksum drift must not be repaired silently. -psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=44 WHERE version=43' +psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=45 WHERE version=44' if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi -psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=43 WHERE version=44" +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=44 WHERE version=45" psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=14" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=13" diff --git a/tools/migrate-postgres.sh b/tools/migrate-postgres.sh index 6b072ca..3996ebd 100644 --- a/tools/migrate-postgres.sh +++ b/tools/migrate-postgres.sh @@ -98,6 +98,8 @@ digest_42="$(sha256sum "$root/migrations/0042_cellular_v8_reproduction_bundle.sq digest_42="${digest_42%% *}" digest_43="$(sha256sum "$root/migrations/0043_cellular_v9_foraging_bundle.sql")" digest_43="${digest_43%% *}" +digest_44="$(sha256sum "$root/migrations/0044_exact_population_publication.sql")" +digest_44="${digest_44%% *}" psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_2=$digest_2" --set="migration_sha_3=$digest_3" \ --set="migration_sha_4=$digest_4" --set="migration_sha_5=$digest_5" \ @@ -134,5 +136,6 @@ psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_41=$digest_41" \ --set="migration_sha_42=$digest_42" \ --set="migration_sha_43=$digest_43" \ + --set="migration_sha_44=$digest_44" \ --set="runtime_role=$runtime_role" \ --file="$root/migrations/apply.sql"