From 7631f9d2e24d724ec95d6de238594e7e87359deb Mon Sep 17 00:00:00 2001 From: dingo Date: Sun, 18 Oct 2026 15:56:58 -0400 Subject: [PATCH] M12.6: capture consistent regional census snapshots --- CMakeLists.txt | 1 + README.md | 6 +- docs/STATUS.md | 16 ++ ...53-consistent-cellular-census-snapshots.md | 66 +++++ docs/adr/README.md | 2 + docs/architecture/PERSISTENCE_ARCHITECTURE.md | 9 + docs/biology/ECOLOGICAL_IDENTITY_AND_TIME.md | 7 + docs/engineering/M12_DELIVERY_PLAN.md | 2 +- docs/engineering/M12_REMAINING_GATES.md | 9 +- docs/engineering/TEST_STRATEGY.md | 12 + docs/engineering/VALIDATION.md | 68 +++++ .../persistence/cellular_census_snapshot.hpp | 50 ++++ include/evo/persistence/postgres.hpp | 3 + .../consistent-cellular-census-snapshot-v1.md | 48 ++++ src/persistence/cellular_census_snapshot.cpp | 102 ++++++++ src/persistence/postgres.cpp | 3 + tests/population_publication_tests.cpp | 247 ++++++++++++++++++ 17 files changed, 648 insertions(+), 3 deletions(-) create mode 100644 docs/adr/0153-consistent-cellular-census-snapshots.md create mode 100644 include/evo/persistence/cellular_census_snapshot.hpp create mode 100644 schemas/ecology/consistent-cellular-census-snapshot-v1.md create mode 100644 src/persistence/cellular_census_snapshot.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 5fa63df..85e0699 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -258,6 +258,7 @@ if(EVO_BUILD_POSTGRES) src/persistence/checkpoint_store.cpp src/persistence/primitive_checkpoint_store.cpp src/persistence/cellular_checkpoint_store.cpp src/persistence/population_checkpoint_store.cpp src/persistence/cellular_history_store.cpp + src/persistence/cellular_census_snapshot.cpp src/persistence/region_clone.cpp src/persistence/cellular_migration_store.cpp) add_library(evo::region_store ALIAS evo_region_store) target_link_libraries(evo_region_store PUBLIC evo::postgres evo::region_checkpoint diff --git a/README.md b/README.md index bb52483..59d1cd5 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,11 @@ current cellular owners. Retained departed history is excluded from owned stock. derives bounded, fully replayed census pages from the existing atomic biological journal. [Scoped multi-region queries](schemas/ecology/scoped-cellular-census-set-v1.md) require exact declared coverage/boundaries, consistent immutable genomes and -disjoint resident identities before summing. Complete world/branch reconciliation, +disjoint resident identities before summing. +[Consistent scoped snapshots](schemas/ecology/consistent-cellular-census-snapshot-v1.md) +capture declared current heads in one short database snapshot, then fully replay +the immutable cuts; concurrent atomic migration cannot split a selected pair. +Complete world/branch reconciliation, client queries and terminal extinction remain later gates. `apps/evo-sim` owns CLI/file I/O and reporting. `tests` holds contract, diff --git a/docs/STATUS.md b/docs/STATUS.md index 84cc0cb..610c90b 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1,5 +1,21 @@ # Project status and next steps +## M12.6d consistent scoped census snapshots - 2026-10-18 + +Scoped census reads now select all declared regional heads from one short +read-only database snapshot, then fully replay the exact immutable cuts after +the metadata transaction ends. A concurrent atomic migration cannot split the +selected source/destination heads. Missing or failed evidence returns no partial +census; unsafe capture cleanup closes its connection. + +Private raw/cohort migration races and TLS BEGIN/COMMIT request/reply-loss tests +pass in Release and ASan/UBSan, alongside existing history and migration/storage +faults. Native default Release passes **79/79**; Clang analysis and formatting +pass. [Evidence and limits](engineering/VALIDATION.md) retain durable world/branch +coverage, outside transfer/viability reconciliation, client access, target-host +query costs and later ecology/soak/product gates. A caller-declared consistent +cut cannot accept terminal extinction. No deployment or World0 action. + ## M12.6c scoped multi-region census queries - 2026-10-18 The portable census now queries 1..64 explicitly declared regions at an exact diff --git a/docs/adr/0153-consistent-cellular-census-snapshots.md b/docs/adr/0153-consistent-cellular-census-snapshots.md new file mode 100644 index 0000000..5d25e8f --- /dev/null +++ b/docs/adr/0153-consistent-cellular-census-snapshots.md @@ -0,0 +1,66 @@ +# ADR-0153: Consistent database cuts for scoped cellular census queries + +## Status + +Accepted for M12.6d, 2026-10-18. Backend read adapter under ADR-0151/0152; +no durable world catalog, external branch identity or extinction acceptance. + +## Context and decision + +Equal-tick regional observations can straddle an atomic migration. A source-after +and destination-before selection can omit stock without duplicating an organism. +The scoped portable query cannot detect that omission from detached totals alone. + +Capture the current committed heads of 1..64 declared regions in one explicitly +owned PostgreSQL `REPEATABLE READ READ ONLY` transaction on an idle connection. +Requests are nonzero, distinct, ascending region IDs in one world, with object +stores and grants for the existing replay readers. Keep exact head/manifest pairs, +then commit the short metadata transaction before reading biological objects. +An atomic migration committed between head reads is either visible at both +endpoints or neither when both are in the declared set. + +After capture, use ADR-0151's existing exact-cut history reader for each selected +head. Verify the returned head and manifest still match the captured cut. Complete +semantic replay, object hashes and fenced evidence checks remain mandatory; no +cached census or older-head fallback. Combine observations through ADR-0152's +coverage/boundary/genome/resident checks. Different head ticks fail rather than +advancing biology or silently selecting an older common tick. Return exact cuts +alongside the scoped census only after every step succeeds. + +The adapter refuses all caller-owned transactions, including read-only ones. +It rolls back failed metadata capture; failed completion/cleanup closes the +connection. An exception during capture also closes the connection immediately, +releasing server resources when disconnect is observed. Add an explicit noexcept +disconnect method to the libpq owner for this cleanup. No automatic retry. +Per-I/O deadlines retain existing semantics, not a whole-query latency guarantee. + +## Alternatives, benefits, costs and risks + +Independent head reads can mix the sides of a paired publication. Holding one +transaction through full recovery is unnecessarily long, and existing fenced +evidence functions take lease row locks that a read-only transaction forbids. +Short read-only metadata capture plus immutable replay avoids that coupling. +Ordinary replay still briefly takes existing per-call lease locks; this does not +eliminate all contention or establish a deployment capacity claim. + +Capture uses indexed regional head lookups. Work and retained observations remain +bounded by 64 regions; full replay/object cost is inherited from ADR-0151. A long +metadata snapshot can retain old database versions until completion. No new SQL +objects, cache/index, durable writes, biological rule/codec or dependency. + +## Consequences and qualification + +Require an actual paired migration on an independent connection between the first +and second head reads: both captured cuts must remain before the pair while a +later snapshot sees both after it. Recover the earlier cuts after publication, +compare exact census stock, and repeat for raw/cohort histories. Missing regions, +stale/missing/corrupt replay evidence, mismatched boundaries, invalid/over-bound +requests and caller-owned transactions must return no partial census. Confirm +transaction cleanup/connection closure, unchanged heads and no object writes. + +This is one connected database's consistent committed cut for a caller-declared +region set. It neither proves that the list includes every world owner nor +reconciles branches, outside transfers, dormant/reserve viability or global +freshness. Current grants support recovery, not new observation authority. No +terminal-extinction outcome or public client permission follows. Durable +world/branch coverage and complete M12.7 reconciliation remain open. diff --git a/docs/adr/README.md b/docs/adr/README.md index 0f2b2ab..0c66dc2 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -1,5 +1,7 @@ # Architecture decision records +- [ADR-0153: Consistent scoped cellular census snapshots](0153-consistent-cellular-census-snapshots.md) - short read-only repeatable-read head capture followed by exact immutable replay; atomic pairs cannot straddle a declared-region cut, without claiming world/branch completeness. + - [ADR-0152: Scoped multi-region cellular census queries](0152-scoped-cellular-census-sets.md) - bounded declared coverage, exact boundaries, complete shared-genome equality and disjoint resident identities; no world-completeness or extinction authority. - [ADR-0151: Committed cellular census history](0151-committed-cellular-census-history.md) - derives bounded, exact-cut census pages from the existing atomic immutable biological journal, with full historical replay; no duplicate census writes or world-extinction authority. diff --git a/docs/architecture/PERSISTENCE_ARCHITECTURE.md b/docs/architecture/PERSISTENCE_ARCHITECTURE.md index 03cff4c..316fb64 100644 --- a/docs/architecture/PERSISTENCE_ARCHITECTURE.md +++ b/docs/architecture/PERSISTENCE_ARCHITECTURE.md @@ -1,5 +1,14 @@ # Persistence and region authority +M12.6d captures declared regional heads in one short read-only repeatable-read +transaction, closes it, then fully replays each exact immutable cut before +constructing a scoped census. Atomic migration commits cannot split the selected +head set when both endpoints are declared. Failed metadata capture rolls back; +failed cleanup or an exception disconnects the session. Full replay remains +outside the metadata transaction, with existing per-call evidence locks. +See [ADR-0153](../adr/0153-consistent-cellular-census-snapshots.md). The caller's +list is still not complete world coverage or external branch identity. + M12.6c's portable [scoped census set](../../schemas/ecology/scoped-cellular-census-set-v1.md) combines a bounded declared set of regional observations, including replayed history entries. It verifies exact common boundaries, complete shared genome diff --git a/docs/biology/ECOLOGICAL_IDENTITY_AND_TIME.md b/docs/biology/ECOLOGICAL_IDENTITY_AND_TIME.md index b30ef69..7e72e81 100644 --- a/docs/biology/ECOLOGICAL_IDENTITY_AND_TIME.md +++ b/docs/biology/ECOLOGICAL_IDENTITY_AND_TIME.md @@ -22,6 +22,13 @@ It refuses missing/stale/conflicting evidence. The list remains caller-declared; neither a durable world catalog nor reconciled migration cuts or branch proof is supplied, so this query cannot accept world absence or terminal extinction. +M12.6d's [consistent scoped snapshots](../../schemas/ecology/consistent-cellular-census-snapshot-v1.md) +select current regional heads from one database snapshot, then verify each exact +immutable cut through full recovery. A pair cannot commit between selected sides +when both regions are declared. Durable required-world coverage, outside transfer +and reserve reconciliation, external branch identity and extinction acceptance +remain separate; this is no complete world evidence envelope. + ## Identity and classification An organism has a permanent identity and terminal death. An immutable genome has diff --git a/docs/engineering/M12_DELIVERY_PLAN.md b/docs/engineering/M12_DELIVERY_PLAN.md index c309aa5..9ec58bd 100644 --- a/docs/engineering/M12_DELIVERY_PLAN.md +++ b/docs/engineering/M12_DELIVERY_PLAN.md @@ -13,7 +13,7 @@ Commit and push each validated scoped segment separately. | 12.3 | Exact population profile/trace/store and opt-in host integration implemented (12.3b) | ADR-0142, schema 44; exact publication/recovery, mutation race guards, ambiguous replies and process kills. Executable/network population fault drills and target-host qualification remain open. | | 12.4 | Owned intervals and durable paired migration/fault qualification implemented in scoped segments through 12.4l | ADR-0143–0149/schema 49; bounded admitted work, source freeze, staged ghost, atomic pair and exact recovery, local TLS and storage faults. Multi-host/orchestrator, linked ecology and integrated stock audits remain open. | | 12.5 | Blocked on 12.2-12.4 | Finite M6 process fixtures do not qualify coupled persistent cellular/cohort atmosphere/water budgets. | -| 12.6 | Regional census/lineage queries, replayed commit-bound history pages and scoped multi-region queries implemented (12.6a-c) | ADR-0150–0152; atomic biological commits are the durable events. Fixed regional cuts and full replay bind observations; declared sets reject missing/stale/conflicting identity or duplicate residents. Durable world/branch coverage, reconciled migration cuts, target-host query/index costs and client path remain required. | +| 12.6 | Regional queries, replayed history, scoped multi-region queries and consistent database head snapshots implemented (12.6a-d) | ADR-0150–0153; short read-only snapshot capture prevents atomic pairs from straddling declared head sets, with exact immutable replay. Durable world/branch coverage, outside transfer/viability reconciliation, target-host query/index costs and client path remain required. | | 12.7 | Evidence policy defined; reconciler open | Complete durable world watermark and every viable/in-flight store remain required. | | 12.8 | Not qualified | Actual multi-day integrated ecology/restart/clone evidence remains required. | | 12.9 | Not accepted | Returning-player product evidence and expansion review remain required; World0 stays separately blocked. | diff --git a/docs/engineering/M12_REMAINING_GATES.md b/docs/engineering/M12_REMAINING_GATES.md index 7a44c12..2eab042 100644 --- a/docs/engineering/M12_REMAINING_GATES.md +++ b/docs/engineering/M12_REMAINING_GATES.md @@ -27,7 +27,7 @@ or authorize production enablement. | 12.3 | Dedicated population profile/trace/store/owning host; exact recovery and private PostgreSQL fault tests | Population-specific executable/network fault drills and target-host publication/capacity qualification. | | 12.4 | Owned intervals; exact transfer/accounting; durable atomic pair/recovery; independent loopback TLS path loss and owner expiry/replacement/heal; local object sync/read-failure matrix | Multi-host/kernel-network, storage-host and orchestrator drills, linked-boundary contracts and integrated ownership/stock audits at scale. | | 12.5 | Finite cellular contact feeding/decay and separate M6 producer/consumer resource owner | One jointly owned cellular/environment budget, explicit effective depth and source entitlements, versioned water/atmosphere species mapping and producer coefficients, plus closed/open and nutrient-exhaustion comparisons. | -| 12.6 | Retained archive, regional census/lineage queries, replayed commit-bound history and bounded scoped multi-region queries (12.6a-c) | Durable world/branch history context and required-region coverage, reconciled migration cuts, target-host query/index qualification and client permission/presentation; no account power or invented precise statistical ancestry. | +| 12.6 | Retained archive, regional census/lineage queries, replayed history, scoped multi-region queries and consistent database head snapshots (12.6a-d) | Durable world/branch history context and required-region coverage, outside transfer/viability reconciliation, target-host query/index qualification and client permission/presentation; no account power or invented precise statistical ancestry. | | 12.7 | ADR-0133's viability/unknown semantics | Durable region-set/watermark reconciliation covering every owned representation, suspended/viable stock and migration escrow. Missing/stale regions must prevent terminal extinction. | | 12.8 | Deterministic seed/regime drivers and separate-process exact continuation | Reproducible integrated worlds running for actual days, real restarts/clone comparisons and retained accounting/diversity/cost audits. Accelerated tests cannot supply elapsed days. | | 12.9 | M11 feedback and current UI review retained | Returning-player samples on the integrated ecology and explicit product/3D readiness decision after soak evidence. No M12 product acceptance or World0 promotion is inferred. | @@ -114,6 +114,13 @@ before querying. Nonresident historical records add no living stock. A required list supplied by a caller is still not durable world coverage; equal ticks alone cannot prove a consistent migration cut or branch. This does not close M12.7. +M12.6d's [consistent scoped snapshot](../../schemas/ecology/consistent-cellular-census-snapshot-v1.md) +now captures all declared heads in one database read snapshot before exact replay. +An atomic pair cannot straddle that selected set when both endpoints are declared. +The short metadata transaction ends before object I/O and fenced recovery; failed +evidence still returns no partial census. Durable world/branch identity, required +owner coverage, outside transfers and viable-store reconciliation remain open. + Current expansion readiness: **no-go for M13 integration or World0 promotion** while these M12 integration/evidence gates remain open. This records the present engineering evidence, not a substitute for the later maintainer product review. diff --git a/docs/engineering/TEST_STRATEGY.md b/docs/engineering/TEST_STRATEGY.md index d50481d..d6168f7 100644 --- a/docs/engineering/TEST_STRATEGY.md +++ b/docs/engineering/TEST_STRATEGY.md @@ -1,5 +1,17 @@ # Validation strategy +M12.6d's private PostgreSQL snapshot tests publish a real atomic migration from +an independent connection between the first and second captured head reads. +Both selected cuts must remain before the pair; the next snapshot sees both +after it. Recover the earlier cuts after publication and compare full regional +rows plus exact/descendant count/body/reserve, in both directions and raw/cohort +profiles. Verify actual repeatable-read/read-only settings, no metadata transaction +held over replay, exact cut binding and unchanged biological stock. Invalid, +missing, unretained, stale, unreadable or caller-transaction evidence returns no +partial result. Capture errors roll back; lost completion/throwing hooks close +the session. Sanitize/inspect the new adapter and existing libpq owner together. +This does not qualify complete world/branch coverage or terminal extinction. + M12.6c's scoped census-set contract compares declared-region queries with direct resident count/body/reserve oracles. Require missing/extra/duplicate regions, every boundary field, unknown/invalid scope, conflicting immutable successor IDs, diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index d5f12c6..875e205 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -1,5 +1,73 @@ # Validation report +## 2026-10-18 - M12.6d consistent scoped census snapshots + +[ADR-0153](../adr/0153-consistent-cellular-census-snapshots.md) captures the current +heads of 1..64 declared regions in one owned read-only repeatable-read database +transaction. It completes that short metadata transaction before fully replaying +each immutable selected cut through the existing history reader. Exact cut binding +and the scoped census-set checks remain mandatory. Failed capture rolls back; +failed completion/cleanup or a capture exception disconnects the session. No +partial result or older-head fallback. See +[the contract](../../schemas/ecology/consistent-cellular-census-snapshot-v1.md). + +Executed locally: + +- Native MSVC **19.44.35217.0** default Release build/CTest **79/79 passed**, + **13.76 s**, with optional Linux persistence disabled. +- GCC 14 Release private population/history integration passed **17 s** (15 s + population phase). Final combined snapshot/history/migration **TLS and storage + fault qualification passed, exit 0, 39 s** (38 s population phase). +- Clang 19 Linux **ASan/UBSan** private population integration passed **46 s**; + the final combined TLS/storage/snapshot matrix passed **exit 0, 70 s** (66 s + population phase), without sanitizer reports. +- Explicit clang-tidy 19 covered the libpq owner, snapshot adapter and integration + tests, then rechecked the final TLS assertions; no owned warnings. Repository + clang-format 19 and `git diff --check` passed. + +Each raw/cohort migration race uses an independent publishing connection after +the first captured head read. Actual database settings are checked as repeatable +read and read-only. Both captured heads remain before the atomic pair; a later +snapshot sees both new heads. Earlier cuts fully replay after publication, retain +identical complete regional rows, and preserve exact/descendant resident counts, +body and reserve in both migration directions. The metadata transaction is closed +before the existing idle-only historical reader runs. + +Tests also retain exact head/manifest binding on repeated reads, direct biology +oracles, request bounds/order/null validation, missing regions, incompatible +current ticks/owner versions, stale grants, unretained builds, missing/unreadable +payloads and unchanged current heads. Caller-owned read-only transactions and a +writer's pending publication are refused without taking over its transaction. +Capture failure rolls back; throwing hooks and failed completion close the +connection. Disconnect is idempotent and later queries return `closed`. + +Actual loopback TCP with end-to-end TLS independently drops requests and replies +at both BEGIN and COMMIT. Every uncertain outcome returns no census, closes the +query session and preserves observed heads. Existing migration owner expiry, +replacement/heal, exact recovery and local object-failure checks still pass. +All database runners used fresh private temporary clusters and stopped them on +exit; no live database, deployment or World0 action. + +The test allowlist initially needed an explicit nested digest-array type; analysis +also required consuming the expected result in the throwing-hook test. Both were +corrected and the final matrix rebuilt. No checks/goldens were suppressed or +rewritten. No SQL schema, biological rule/codec, dependency or ownership change. + +Limits: this is a consistent cut of a caller-declared set in one connected +database, not durable world coverage or external branch identity. Outside-region +transfers, reserve viability and complete global freshness/reconciliation remain +open. SQL deadlines do not preempt local object reads or promise whole-query +latency. Existing replay evidence calls retain their per-call lease locks. No +deployment-host capacity claim, new TSan, complete SQL/backup/DR rerun, remote CI, +multi-host drill or actual multi-day soak. Prior full persistence regression is +retained in M12.6b. No terminal-extinction acceptance or client endpoint. + +Logs: `build/m126d-native.log`, `build/m126d-postgres.log`, +`build/m126d-asan.log`, `build/m126d-final-postgres-build.log`, +`build/m126d-final-postgres.log`, `build/m126d-final-asan-build.log`, +`build/m126d-final-asan.log`, `build/m126d-final-tidy.log`, +`build/m126d-final-integration-tidy.log`, `build/m126d-final-format.log`. + ## 2026-10-18 - M12.6c scoped multi-region census queries [ADR-0152](../adr/0152-scoped-cellular-census-sets.md) adds a portable detached diff --git a/include/evo/persistence/cellular_census_snapshot.hpp b/include/evo/persistence/cellular_census_snapshot.hpp new file mode 100644 index 0000000..c56162a --- /dev/null +++ b/include/evo/persistence/cellular_census_snapshot.hpp @@ -0,0 +1,50 @@ +#pragma once + +#include "evo/persistence/cellular_history_store.hpp" +#include "evo/server/lifecycle/cellular_census_set.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace evo::persistence { +struct CellularCensusRegionRequest { + std::uint64_t region{}; + LeaseGrant lease{}; + const FilesystemObjectStore* objects{}; // Non-owning, valid throughout the call. +}; +struct CellularCensusSnapshot { + std::vector cuts; // Sorted regions, exact captured head/manifest pairs. + server::CellularCensusSet census; +}; +enum class CellularSnapshotError : std::uint8_t { invalid_request, transaction, history, census }; +struct CellularSnapshotFailure { + CellularSnapshotError kind{}; + std::uint64_t region{}; // Failed region, or zero for a request/transaction/set error. + PgFailure transaction{}; // Meaningful for transaction. + CellularHistoryFailure history{}; // Meaningful for history. + server::CellularCensusSetError census{}; // Meaningful for census. +}; +// Test/fault seam after each head has been captured (zero-based request index). +// Production callers pass nothing. A throwing hook closes the capture connection. +struct CellularCensusSnapshotHooks { + std::function after_head; +}; +// Idle connected session only. Requests: 1..64, nonzero strictly ascending regions +// in one world; all stores nonnull. Capture current heads in a short READ ONLY +// REPEATABLE READ transaction, finish it, then fully replay each immutable cut. +// All cuts must share frame/tick/owner/rules/quantum. No partial census on error. +// Per-I/O timeout 1..30000 ms; no automatic retry or whole-query latency promise. +// Metadata failures roll back; failed cleanup or an exception closes the session. +// Caller-declared coverage, NOT world/branch completeness or live authority. +[[nodiscard]] std::expected +load_cellular_census_snapshot(Postgres&, std::uint64_t world, + std::span, + std::span retained_builds, + std::chrono::milliseconds timeout, + const CellularCensusSnapshotHooks& hooks = {}); +} // namespace evo::persistence diff --git a/include/evo/persistence/postgres.hpp b/include/evo/persistence/postgres.hpp index 4d7918a..c6b00dc 100644 --- a/include/evo/persistence/postgres.hpp +++ b/include/evo/persistence/postgres.hpp @@ -40,6 +40,9 @@ class Postgres { // True only for a connected, idle session outside an explicit transaction. // Read-only status inspection; no query, lock or transaction is started. [[nodiscard]] bool transaction_idle() const noexcept; + // Drop the owned connection and any server-side transaction. Idempotent; + // future queries return closed. No SQL, allocation or automatic reconnect. + void disconnect() noexcept; [[nodiscard]] std::expected query(const std::string& sql, std::span parameters, std::chrono::milliseconds timeout); diff --git a/schemas/ecology/consistent-cellular-census-snapshot-v1.md b/schemas/ecology/consistent-cellular-census-snapshot-v1.md new file mode 100644 index 0000000..9c10668 --- /dev/null +++ b/schemas/ecology/consistent-cellular-census-snapshot-v1.md @@ -0,0 +1,48 @@ +# Consistent scoped cellular census snapshot v1 + +Semantic backend read contract under +[ADR-0153](../../docs/adr/0153-consistent-cellular-census-snapshots.md), not a +persistent/wire codec. Linux API: `persistence::load_cellular_census_snapshot`. + +Input names one nonzero world and 1..64 requests with strictly ascending nonzero +region IDs. Each request retains a non-owning object-store pointer and a grant +for the existing historical recovery checks. Stores must remain alive throughout +the call. The connected PostgreSQL session must be idle; all caller transactions, +including read-only transactions and pending publications, are refused unchanged. +The timeout for each PostgreSQL query is 1..30000 ms; local object reads retain +their existing blocking semantics. No automatic retry or whole-query SLO. + +1. Begin an owned `REPEATABLE READ READ ONLY` transaction and read every requested + current committed head plus exact manifest. The first read establishes the + snapshot; later head reads use the same database view. Retain request order. +2. Commit the metadata transaction before any object restoration/replay. Metadata + errors roll back. Failure to begin/finish/clean up safely closes a non-idle + connection; an exception during capture also disconnects it. Results from a + failed capture never escape as a partial census. +3. On the now-idle session, recover each exact selected cut through the committed + history reader. Validate full manifest binding, retained builds, strong object + digests, complete payload/trace/transition/paired replay and existing fenced + evidence. The returned head and manifest must equal their captured values. +4. Build the scoped census set, requiring all captured states to agree on frame, + completed tick, owner format, rules and quantum, with consistent complete + shared-genome records and disjoint resident organism identities. Different + current ticks fail; no catch-up or earlier common-tick selection is inferred. +5. Return exact captured head/manifest pairs and the detached scoped census only + after all regions pass. Any later failure discards the entire result. There is + no fallback, skipped region or missing-as-zero behavior. + +When both endpoints are declared, one atomic migration committed between head +reads appears at neither endpoint of that captured cut; a later capture sees +both new heads. More generally each atomic pair is visible at both endpoints or +neither in the selected database snapshot. Replay remains pinned to those exact +immutable cuts even if the current heads advance before replay begins. Frozen +source stock remains owned; staged ghosts and uncommitted uploads are excluded. +Existing per-call lease locks still apply during recovery outside the metadata +transaction. No long read transaction is held over payload I/O. + +Returned references are historical evidence, not current lease/activation proof. +The region list remains caller-declared and need not include outside migration +endpoints or all world owners. This supplies neither a durable world catalog nor +external branch/rewind identity, reserve viability reconciliation, freshness or +complete global watermark. No world absence/terminal-extinction acceptance, +account advantage, new client permissions or public endpoint is provided. diff --git a/src/persistence/cellular_census_snapshot.cpp b/src/persistence/cellular_census_snapshot.cpp new file mode 100644 index 0000000..437f977 --- /dev/null +++ b/src/persistence/cellular_census_snapshot.cpp @@ -0,0 +1,102 @@ +#include "evo/persistence/cellular_census_snapshot.hpp" + +#include + +namespace evo::persistence { +namespace { +using Failure = CellularSnapshotFailure; +using Error = CellularSnapshotError; +auto transaction_failure(PgFailure error) { + return std::unexpected(Failure{Error::transaction, 0, error, {}, {}}); +} +auto history_failure(std::uint64_t region, CellularHistoryFailure error) { + return std::unexpected(Failure{Error::history, region, {}, error, {}}); +} +// Exception-safe capture ownership: explicit completion disarms this guard. +// Closing cannot allocate/throw or leave a caller with an open failed transaction. +struct CaptureGuard { + Postgres& db; + bool active = true; + ~CaptureGuard() { + if (active) + db.disconnect(); + } +}; +std::expected, Failure> +capture(Postgres& db, std::uint64_t world, std::span requests, + std::chrono::milliseconds timeout, const CellularCensusSnapshotHooks& hooks) { + const auto begun = db.query("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY", {}, timeout); + if (!begun) { + if (!db.transaction_idle()) + db.disconnect(); + return transaction_failure(begun.error()); + } + CaptureGuard guard{db}; + auto result = [&]() -> std::expected, Failure> { + std::vector cuts; + cuts.reserve(requests.size()); + for (std::size_t i = 0; i < requests.size(); ++i) { + auto head = load_region_checkpoint(db, {world, requests[i].region}, timeout); + if (!head) + return history_failure(requests[i].region, {CellularHistoryError::recovery, + {RecoveryError::store, head.error()}}); + cuts.push_back(std::move(*head)); + if (hooks.after_head) + hooks.after_head(i); + } + return cuts; + }(); + const auto done = db.query(result ? "COMMIT" : "ROLLBACK", {}, timeout); + if (!done) + return transaction_failure(done.error()); + if (!db.transaction_idle()) + return transaction_failure({PgError::sql, {}}); + guard.active = false; + return result; +} +} // namespace + +std::expected load_cellular_census_snapshot( + Postgres& db, std::uint64_t world, std::span requests, + std::span builds, std::chrono::milliseconds timeout, + const CellularCensusSnapshotHooks& hooks) { + if (!db.transaction_idle() || world == 0 || requests.empty() || + requests.size() > server::max_cellular_census_regions || timeout.count() <= 0 || + timeout.count() > 30000) + return std::unexpected(Failure{Error::invalid_request, 0, {}, {}, {}}); + std::vector required; + required.reserve(requests.size()); + for (const auto& request : requests) { + if (!request.objects || request.region == 0 || request.lease.frame == 0 || + (!required.empty() && request.region <= required.back())) + return std::unexpected(Failure{Error::invalid_request, 0, {}, {}, {}}); + required.push_back(request.region); + } + auto cuts = capture(db, world, requests, timeout, hooks); + if (!cuts) + return std::unexpected(cuts.error()); + std::vector observations; + observations.reserve(requests.size()); + for (std::size_t i = 0; i < requests.size(); ++i) { + const auto& request = requests[i]; + const auto& cut = (*cuts)[i]; + auto page = load_cellular_census_history(db, *request.objects, {world, request.region}, + request.lease, cut.head, cut.head.sequence, 1, + builds, timeout); + if (!page) + return history_failure(request.region, page.error()); + if (page->entries.size() != 1 || page->entries.front().committed.head != cut.head || + page->entries.front().committed.manifest != cut.manifest) + return history_failure(request.region, {CellularHistoryError::cut_mismatch, {}}); + observations.push_back(std::move(page->entries.front().census)); + } + const auto& first = observations.front().boundary(); + auto census = server::CellularCensusSet::observe({first.world, first.frame, first.tick, + first.owner_format, first.rules, + first.quantum_microseconds}, + required, observations); + if (!census) + return std::unexpected(Failure{Error::census, 0, {}, {}, census.error()}); + return CellularCensusSnapshot{std::move(*cuts), std::move(*census)}; +} +} // namespace evo::persistence diff --git a/src/persistence/postgres.cpp b/src/persistence/postgres.cpp index 10403b3..8424b7f 100644 --- a/src/persistence/postgres.cpp +++ b/src/persistence/postgres.cpp @@ -119,6 +119,9 @@ struct Postgres::Impl { }; Postgres::Postgres(std::unique_ptr impl) : impl_(std::move(impl)) {} Postgres::~Postgres() = default; +void Postgres::disconnect() noexcept { + impl_->connection.reset(); +} bool Postgres::transaction_idle() const noexcept { return impl_->connection && PQstatus(impl_->connection.get()) == CONNECTION_OK && PQtransactionStatus(impl_->connection.get()) == PQTRANS_IDLE; diff --git a/tests/population_publication_tests.cpp b/tests/population_publication_tests.cpp index 0409396..5a75e12 100644 --- a/tests/population_publication_tests.cpp +++ b/tests/population_publication_tests.cpp @@ -1,4 +1,5 @@ #include "evo/persistence/account_control_store.hpp" +#include "evo/persistence/cellular_census_snapshot.hpp" #include "evo/persistence/cellular_history_store.hpp" #include "evo/persistence/population_checkpoint_store.hpp" #include "evo/persistence/population_interval_store.hpp" @@ -790,6 +791,158 @@ void history_pages(Fixture& f) { "history transaction requirement changed for read-only callers"); std::puts("committed census history pagination, replay, transaction and failure guards passed"); } +void snapshot_transport_faults(Fixture& f) { + if (!std::getenv("EVO_MIGRATION_TCP_PORT")) + return; + const std::array builds{f.build}; + const std::array requests{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()}}; + for (unsigned mode = 0; mode < 4; ++mode) { + test::PostgresRelay relay(migration_backend_port()); + auto db = migration_tls_connection(relay); + const bool at_completion = mode >= 2; + const bool lost_request = mode % 2 == 0; + const auto partition = [&] { + if (lost_request) + relay.drop_requests(true); + else + relay.drop_responses(true); + }; + if (!at_completion) + partition(); + const CellularCensusSnapshotHooks hook{[&](std::size_t) { + if (at_completion) + partition(); + }}; + const auto lost = + load_cellular_census_snapshot(*db, f.key.world, requests, builds, 300ms, hook); + require(!lost && lost.error().kind == CellularSnapshotError::transaction && + lost.error().transaction.kind == PgError::unknown && !db->transaction_idle() && + relay.healthy() && + (lost_request ? relay.dropped_requests() : relay.dropped_responses()) > 0 && + take(load_region_checkpoint(*f.db, f.key, 5s), "head after lost capture I/O") + .head == f.head.head, + "lost snapshot BEGIN/COMMIT request/reply returned evidence or kept a session"); + } + std::puts("snapshot TLS BEGIN/COMMIT request/reply loss fails closed"); +} +void snapshot_guards(Fixture& f) { + const std::array builds{f.build}; + const std::array requests{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()}}; + const auto read = [&] { + return load_cellular_census_snapshot(*f.db, f.key.world, requests, builds, 5s); + }; + const auto before = f.head; + const auto first = take(read(), "single-region captured snapshot"); + const auto second = take(read(), "repeat captured snapshot"); + require(first.cuts.size() == 1 && first.cuts[0].head == before.head && + first.cuts[0].manifest == before.manifest && + second.cuts[0].head == first.cuts[0].head && + std::ranges::equal( + first.census.regions()[0].rows(), + take(CellularCensus::observe(*f.raw), "snapshot direct oracle").rows()) && + f.db->transaction_idle(), + "captured snapshot lost its exact committed cut/biology or retained a transaction"); + const auto invalid = [&](auto region_requests, std::uint64_t world = 30201, + std::chrono::milliseconds timeout = 5s) { + const auto result = + load_cellular_census_snapshot(*f.db, world, region_requests, builds, timeout); + require(!result && result.error().kind == CellularSnapshotError::invalid_request && + f.db->transaction_idle(), + "invalid snapshot request started a transaction or returned a census"); + }; + invalid(std::vector{}); + invalid(requests, 0); + invalid(requests, f.key.world, 0ms); + invalid(requests, f.key.world, 30001ms); + auto bad = requests; + bad[0].objects = nullptr; + invalid(bad); + bad = requests; + bad[0].region = 0; + invalid(bad); + bad = requests; + bad[0].lease.frame = 0; + invalid(bad); + invalid(std::array{requests[0], requests[0]}); + auto higher = requests[0]; + ++higher.region; + invalid(std::array{higher, requests[0]}); + invalid(std::vector(max_cellular_census_regions + 1, requests[0])); + higher.region = 999'999; + const auto missing = load_cellular_census_snapshot(*f.db, f.key.world, + std::array{requests[0], higher}, builds, 5s); + require(!missing && missing.error().kind == CellularSnapshotError::history && + missing.error().region == higher.region && f.db->transaction_idle(), + "missing region became partial coverage or left capture open"); + for (const auto* begin : {"BEGIN", "BEGIN READ ONLY"}) { + take(f.db->query(begin, {}, 5s), "caller snapshot transaction"); + if (std::string_view{begin} == "BEGIN") + take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, + {false, 1200, 1800}, 5s), + "pending caller snapshot publication"); + const auto refused = read(); + require(!refused && refused.error().kind == CellularSnapshotError::invalid_request && + !f.db->transaction_idle(), + "snapshot took ownership of a caller transaction"); + take(f.db->query("ROLLBACK", {}, 5s), "caller snapshot rollback"); + } + const auto manifest = take(decode_manifest(before.manifest), "snapshot retained payload"); + const auto path = f.objects->directory() / hexadecimal(manifest.payload_sha256, false); + const auto saved = path.string() + ".snapshot-test"; + std::filesystem::rename(path, saved); + const auto missing_object = read(); + std::filesystem::rename(saved, path); + require(!missing_object && missing_object.error().kind == CellularSnapshotError::history && + f.db->transaction_idle(), + "missing replay payload became a snapshot or left capture open"); + { + const UnreadableObject denied{path}; + const auto unreadable = read(); + require(!unreadable && unreadable.error().kind == CellularSnapshotError::history && + f.db->transaction_idle(), + "unreadable captured payload became a census"); + } + auto stale = requests; + --stale[0].lease.epoch; + const auto fenced = load_cellular_census_snapshot(*f.db, f.key.world, stale, builds, 5s); + require(!fenced && fenced.error().history.recovery.store.kind == LeaseStoreError::fenced, + "snapshot ignored stale evidence authority"); + const std::array foreign_builds{}; + require(!load_cellular_census_snapshot(*f.db, f.key.world, requests, foreign_builds, 5s), + "snapshot accepted unretained build evidence"); + auto closed = connection(); + CellularCensusSnapshotHooks close_hook{[&](std::size_t) { closed->disconnect(); }}; + const auto lost = + load_cellular_census_snapshot(*closed, f.key.world, requests, builds, 5s, close_hook); + closed->disconnect(); + const auto closed_query = closed->query("SELECT 1", {}, 5s); + require(!lost && lost.error().kind == CellularSnapshotError::transaction && + !closed->transaction_idle() && !closed_query && + closed_query.error().kind == PgError::closed, + "failed capture completion reused a dead session or returned a partial census"); + auto throwing = connection(); + bool threw = false; + bool hook_called = false; + try { + const CellularCensusSnapshotHooks hook{[&](std::size_t) { + hook_called = true; + throw std::runtime_error("capture hook test exception"); + }}; + const auto result = + load_cellular_census_snapshot(*throwing, f.key.world, requests, builds, 5s, hook); + require(!result, "throwing capture hook returned a snapshot"); + } catch (const std::runtime_error&) { + threw = hook_called; + } + const auto after_throw = throwing->query("SELECT 1", {}, 5s); + require(threw && !after_throw && after_throw.error().kind == PgError::closed && + !throwing->transaction_idle() && + take(load_region_checkpoint(*f.db, f.key, 5s), "head after snapshot guards").head == + before.head, + "capture exception retained its connection/transaction or changed a head"); + std::puts("consistent census snapshot guards and connection cleanup passed"); + snapshot_transport_faults(f); +} void boundary_upgrades(Fixture& f) { const std::array builds{f.build}; const auto original = f.head; @@ -1180,6 +1333,87 @@ void migration_owner_partitions(Fixture& f, RegionKey destination_key, std::puts("migration independent TLS owner partitions, expiry, replacement and heal passed"); } +void migration_snapshot_race(Fixture& f, RegionKey destination_key, + FilesystemObjectStore& destination_objects, + const LeaseGrant& destination_lease, + CommittedCheckpoint& destination_head, std::uint64_t genome) { + const std::array builds{f.build}; + const std::array requests{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()}, + CellularCensusRegionRequest{destination_key.region, destination_lease, + &destination_objects}}; + auto reader = connection(); + for (const bool reverse : {false, true}) { + auto& sh = reverse ? destination_head : f.head; + auto& dh = reverse ? f.head : destination_head; + auto& so = reverse ? destination_objects : *f.objects; + auto& dst = reverse ? *f.objects : destination_objects; + const auto& sl = reverse ? destination_lease : f.lease; + const auto& dl = reverse ? f.lease : destination_lease; + const auto sk = reverse ? destination_key : f.key; + const auto dk = reverse ? f.key : destination_key; + const auto before = + take(load_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s), + "before snapshot migration race"); + const CellularMigrationRequest request{f.id(), + {sk.region, sh.head.commit, f.id(), sh.head.digest}, + {dk.region, dh.head.commit, f.id(), dh.head.digest}, + genome}; + const auto frozen = + take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s), + "snapshot race source freeze"); + take(stage_cellular_migration(*f.db, so, dst, dl, sh, dh, frozen.trace, 5s), + "snapshot race destination stage"); + std::optional published; + const CellularCensusSnapshotHooks hook{[&](std::size_t index) { + if (index != 0) + return; + const auto settings = + take(reader->query("SELECT current_setting('transaction_isolation'), " + "current_setting('transaction_read_only')", + {}, 5s), + "capture transaction properties"); + require(settings.size() == 1 && settings[0].size() == 2 && + take(settings[0][0], "capture isolation") == "repeatable read" && + take(settings[0][1], "capture access") == "on", + "capture did not own a repeatable read read-only snapshot"); + auto publisher = connection(); + published = take(publish_cellular_migration(*publisher, so, dst, sl, dl, sh, dh, + frozen.trace, f.build, 5s), + "publish real pair between snapshot head reads"); + }}; + const auto raced = + take(load_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s, hook), + "coherent census across a concurrent pair"); + const auto committed = + take(std::move(published), "pair was not published in snapshot hook"); + sh = committed.source; + dh = committed.destination; + require(reader->transaction_idle() && raced.cuts.size() == 2 && before.cuts.size() == 2, + "raced capture retained its transaction or lost coverage"); + for (std::size_t i = 0; i < before.cuts.size(); ++i) + require(raced.cuts[i].head == before.cuts[i].head && + raced.cuts[i].manifest == before.cuts[i].manifest && + std::ranges::equal(raced.census.regions()[i].rows(), + before.census.regions()[i].rows()), + "atomic pair split a captured cut or changed historical replay"); + const auto after = + take(load_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s), + "snapshot after pair publication"); + require(after.cuts[0].head == f.head.head && after.cuts[1].head == destination_head.head && + after.cuts[0].head.sequence == before.cuts[0].head.sequence + 1 && + after.cuts[1].head.sequence == before.cuts[1].head.sequence + 1, + "later snapshot omitted one committed migration endpoint"); + for (const auto scope : + {CellularCensusScope::exact_genome, CellularCensusScope::descendants}) { + const auto initial = take(before.census.query(genome, scope), "before pair lineage"); + const auto captured = take(raced.census.query(genome, scope), "captured pair lineage"); + const auto settled = take(after.census.query(genome, scope), "settled pair lineage"); + require(initial == captured && initial.resident == settled.resident, + "snapshot migration race lost/duplicated living count/body/reserve"); + } + } + std::puts("consistent raw/cohort snapshots across atomic migration commits passed"); +} void migration_publication(Fixture& f, const RegionKey destination_key, FilesystemObjectStore& destination_objects, LeaseGrant& destination_lease, CommittedCheckpoint& destination_head) { @@ -1693,6 +1927,8 @@ void migration_publication(Fixture& f, const RegionKey destination_key, "return migration rewrote earlier committed ownership observations"); migration_owner_partitions(f, destination_key, destination_objects, destination_lease, destination_head, genome); + migration_snapshot_race(f, destination_key, destination_objects, destination_lease, + destination_head, genome); } std::puts("migration object sync/read failures, quarantine and exact recovery passed"); // Race the actual two-head transaction against abort on independent connections. @@ -1796,6 +2032,16 @@ void migration_freezes(Fixture& f) { auto destination_lease = take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s), "migration destination authority"); + const auto unequal_cut = load_cellular_census_snapshot( + *f.db, f.key.world, + std::array{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()}, + CellularCensusRegionRequest{destination_key.region, destination_lease, + destination_objects.get()}}, + std::array{f.build}, 5s); + require(!unequal_cut && unequal_cut.error().kind == CellularSnapshotError::census && + unequal_cut.error().census == CellularCensusSetError::boundary && + f.db->transaction_idle(), + "snapshot silently reconciled different current ticks/owner versions"); while (destination->tick() < f.raw->tick()) { const auto count = std::min(std::uint64_t{64}, f.raw->tick() - destination->tick()); CellularBatch inputs{{f.key.world, 2, 1, destination_head.head.commit, @@ -2606,6 +2852,7 @@ void run_population_publication_tests() { Fixture fixture; funded_source(fixture); history_pages(fixture); + snapshot_guards(fixture); conversions_and_retries(fixture); interval_contract(fixture); ambiguous_publication(fixture, false);