From 27ec5729bf86e665b020a97edffe7bd3cb4d4ce4 Mon Sep 17 00:00:00 2001 From: dingo Date: Sun, 18 Oct 2026 21:14:53 -0400 Subject: [PATCH] M12.7: reconcile pending migration evidence in census snapshots --- README.md | 2 + docs/STATUS.md | 16 ++ .../0156-census-pending-migration-evidence.md | 68 ++++++++ docs/adr/README.md | 2 + docs/architecture/PERSISTENCE_ARCHITECTURE.md | 8 + docs/engineering/M12_DELIVERY_PLAN.md | 2 +- docs/engineering/M12_REMAINING_GATES.md | 15 +- docs/engineering/TEST_STRATEGY.md | 10 ++ docs/engineering/VALIDATION.md | 79 +++++++++ .../persistence/cellular_census_snapshot.hpp | 11 +- .../persistence/cellular_migration_store.hpp | 4 + migrations/0052_census_pending_migrations.sql | 36 +++++ migrations/apply.sql | 16 +- .../ecology/census-pending-migrations-v1.md | 33 ++++ .../registered-world-census-coverage-v1.md | 2 + src/persistence/cellular_census_snapshot.cpp | 49 +++++- .../cellular_migration_observation_detail.hpp | 20 +++ src/persistence/cellular_migration_store.cpp | 70 ++++++++ tests/population_publication_tests.cpp | 152 +++++++++++++++++- tests/region_migration_tests.cpp | 3 +- tools/ci/history-domain-check.sh | 3 +- tools/ci/postgres-check.sh | 11 +- tools/migrate-postgres.sh | 3 + 23 files changed, 592 insertions(+), 23 deletions(-) create mode 100644 docs/adr/0156-census-pending-migration-evidence.md create mode 100644 migrations/0052_census_pending_migrations.sql create mode 100644 schemas/ecology/census-pending-migrations-v1.md create mode 100644 src/persistence/cellular_migration_observation_detail.hpp diff --git a/README.md b/README.md index f50194e..61218e2 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,8 @@ requires every region in the durable authority registry, including unavailable owners, within the same selected database view. Schema 51's [history domain](schemas/ecology/database-history-domain-v1.md) identifies that database history and forks supported restores with parent/exact-cut provenance. +Schema 52 also returns [verified pending handoffs](schemas/ecology/census-pending-migrations-v1.md) +from that same view; staged ghosts never add census stock. Complete world/branch reconciliation, client queries and terminal extinction remain later gates. `apps/evo-sim` owns CLI/file I/O diff --git a/docs/STATUS.md b/docs/STATUS.md index 105875b..3cc4b8e 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1,5 +1,21 @@ # Project status and next steps +## M12.7c registered census pending handoff evidence - 2026-10-18 + +Schema 52 captures active biological migration pointers with history domain, +registry and heads. Both predecessor manifests are bound to retained builds; +source capsules and staged destination ghosts receive complete paired replay +outside the metadata transaction. Capsules/ghosts add no stock. Prepare/stage/ +abort/commit interleavings preserve the selected view; unavailable or false evidence +fails the entire result. This accepts no terminal extinction or live authority. + +Native Release **79/79**, final raw/population allowlist cases, ASan/UBSan TLS/storage +faults, Clang analysis and formatting pass. Final private persistence/process/ +backup/DR/migration/permission qualification passed in **558 s**; results and limits +are recorded in [validation](engineering/VALIDATION.md). Other viable +stores, cross-domain clone provenance, accepted global reconciliation/freshness +and ecology/soak/product gates remain open. + ## M12.7b database history domains and restore provenance - 2026-10-18 Schema 51 binds registered census evidence to a database history-domain UUID, diff --git a/docs/adr/0156-census-pending-migration-evidence.md b/docs/adr/0156-census-pending-migration-evidence.md new file mode 100644 index 0000000..e09c1b2 --- /dev/null +++ b/docs/adr/0156-census-pending-migration-evidence.md @@ -0,0 +1,68 @@ +# ADR-0156: Pending migration evidence in registered census snapshots + +## Status + +Accepted for scoped M12.7c, 2026-10-18. Extends ADR-0133/0147–0149/0153–0155. +No biological format, rule, activation workflow or terminal-extinction decision. + +## Decision + +A registered census must expose its pending biological handoffs, rather than +letting a caller mistake a staged ghost for additional owned life or infer that +no transfer exists. Schema 52 reads the union of source/destination pending +pointers for one world, ordered by request UUID, at most 65 requests. The 65th +detects overflow of the 64-request work bound; never truncate a complete result. +This is the existing active-pointer inventory, not an audit of all historical +preparations or a new source of ownership. No duplicate state table/write. + +Capture it in the same read-only repeatable-read view as domain, registry and +heads. Keep both retained predecessor manifests, canonical trace and, when staged, +ghost digest/length. Require internally coherent pointers, un-aborted/uncommitted +state and metadata identities; malformed/missing evidence fails the entire query. +Both predecessor builds must belong to the caller's retained build allowlist, +including an optimistic destination predecessor that is no longer its current head. +Every endpoint must be registered. The source predecessor must equal its selected +head; a staged destination must also equal its selected predecessor. A merely +prepared destination is an optimistic observation and may have advanced, as +ADR-0147 permits. Common census-boundary checks still apply independently. + +After metadata capture ends, fully replay every captured handoff from its retained +source-side predecessor/capsule objects. For a stage, independently replay the +destination-side capsules and compare the complete derived ghost bytes with the +verified staged object. Reuse the existing migration codec, binding and paired +replay logic. Missing/corrupt/false capsules or ghosts fail with no partial census. +This is historical evidence verification, not a fenced live-owner lookup; a +concurrent abort/commit does not invalidate an already captured immutable view. + +Return verified trace/member/material evidence and a staged flag alongside the +registered census. Source stock is already counted at its unchanged committed +head. Capsules and ghosts add **zero** census stock. After atomic commit, the next +view sees settled heads and no pending pointers; abort clears both pointers with +unchanged heads. Neither timeout nor lease expiry settles a workflow. + +## Alternatives, costs and risks + +Reading current fenced loaders after capture could combine newer pointer state +with older heads and also needs write locks. Reading inside the read-only view +cannot use those locking functions. Capture immutable evidence references without +locks, close the metadata transaction, then replay exactly those references. +Counting a ghost twice breaks conservation; ignoring unavailable evidence hides +uncertainty. The conservative error is an unavailable registered result. + +This adds one bounded metadata read and complete pair/ghost replay per pending +request, plus transient bounded evidence buffers. No whole-query latency promise, +new cache, aggregate speedup, dependency or target-host capacity claim. The trusted +owner can bypass table invariants; this does not authenticate an untrusted owner +or invent stock outside the current committed owner model. + +## Qualification and remaining scope + +Test raw/population prepare/stage/abort/commit, unchanged census stock, corrupt or +unavailable source capsules/ghosts, malformed pointers, bounds, expired authority, +and capture interleavings with settlement/new preparation. Run full persistence, +backup/DR, migration/privilege checks, sanitizers, analysis and default CTest. +Prior migrations and biological goldens remain unchanged. + +This reconciles the existing pending migration evidence with registered owner +cuts. Cross-domain clone ancestry, other viable-store coverage, accepted durable +global reconciliation/freshness, coupled ecology and actual days remain open. diff --git a/docs/adr/README.md b/docs/adr/README.md index 6f7200e..84fa301 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -1,5 +1,7 @@ # Architecture decision records +- [ADR-0156: Census pending migration evidence](0156-census-pending-migration-evidence.md) - schema 52 captures active handoff pointers with registered heads and fully verifies retained capsules/ghosts; source owns stock and ghosts add none, including across settlement races. + - [ADR-0155: Database history domains](0155-database-history-domain.md) - schema 51 binds registered census evidence to a database namespace; supported quarantined restores retain parent domains and exact fork cuts before readmission. Cross-domain clone provenance and accepted extinction remain open. - [ADR-0154: Registered-world census coverage](0154-registered-world-census-coverage.md) - schema 50 uses the existing durable authority registry as the required regional set in a consistent snapshot; omitted/unavailable/over-bound owners fail without a new catalog table or extinction authority. diff --git a/docs/architecture/PERSISTENCE_ARCHITECTURE.md b/docs/architecture/PERSISTENCE_ARCHITECTURE.md index fc4f49d..6974d44 100644 --- a/docs/architecture/PERSISTENCE_ARCHITECTURE.md +++ b/docs/architecture/PERSISTENCE_ARCHITECTURE.md @@ -1,5 +1,13 @@ # Persistence and region authority +M12.7c/schema 52 adds [pending migration evidence](../../schemas/ecology/census-pending-migrations-v1.md) +to registered snapshots. Pending source/destination pointers, namespace, registry +and heads share one metadata view. Exact captured capsules/ghosts are fully +replayed after capture, even if settlement commits meanwhile. The source head +already owns prepared/staged life; ghosts add no census stock. Missing/false +evidence fails the entire result. This is existing handoff evidence reconciliation, +not coverage of every viable store or an accepted terminal-extinction record. + M12.7b/schema 51 adds [database history domains](../../schemas/ecology/database-history-domain-v1.md). Registered census evidence identifies a world by `(history_domain, world_id)`; domain, registry and heads share one metadata view. Supported restores fork the diff --git a/docs/engineering/M12_DELIVERY_PLAN.md b/docs/engineering/M12_DELIVERY_PLAN.md index 2512679..c6cdfc1 100644 --- a/docs/engineering/M12_DELIVERY_PLAN.md +++ b/docs/engineering/M12_DELIVERY_PLAN.md @@ -14,7 +14,7 @@ Commit and push each validated scoped segment separately. | 12.4 | Owned intervals and durable paired migration/fault qualification implemented in scoped segments through 12.4l | ADR-0143–0149/schema 49; bounded admitted work, source freeze, staged ghost, atomic pair and exact recovery, local TLS and storage faults. Multi-host/orchestrator, linked ecology and integrated stock audits remain open. | | 12.5 | Blocked on 12.2-12.4 | Finite M6 process fixtures do not qualify coupled persistent cellular/cohort atmosphere/water budgets. | | 12.6 | Regional queries, replayed history, scoped multi-region queries and consistent database head snapshots implemented (12.6a-d) | ADR-0150–0153; short read-only snapshot capture prevents atomic pairs from straddling declared head sets, with exact immutable replay. Durable world/branch coverage, outside transfer/viability reconciliation, target-host query/index costs and client path remain required. | -| 12.7 | Evidence policy, registered coverage and database history domains implemented (12.7a-b) | ADR-0154/0155/schema 51 captures namespace/registry/heads together; supported quarantined restores fork with exact-cut provenance. Cross-domain clone provenance, other viable/in-flight stores and durable accepted global reconciliation remain required. | +| 12.7 | Evidence policy, registered coverage, history domains and verified pending handoff evidence implemented (12.7a-c) | ADR-0154–0156/schema 52 captures namespace/registry/heads/pending handoffs together, with exact capsule/ghost replay and supported restore forks. Cross-domain clone provenance, other viable-store coverage and durable accepted global reconciliation/freshness remain required. | | 12.8 | Not qualified | Actual multi-day integrated ecology/restart/clone evidence remains required. | | 12.9 | Not accepted | Returning-player product evidence and expansion review remain required; World0 stays separately blocked. | diff --git a/docs/engineering/M12_REMAINING_GATES.md b/docs/engineering/M12_REMAINING_GATES.md index 5e96e55..3f0d156 100644 --- a/docs/engineering/M12_REMAINING_GATES.md +++ b/docs/engineering/M12_REMAINING_GATES.md @@ -28,7 +28,7 @@ or authorize production enablement. | 12.4 | Owned intervals; exact transfer/accounting; durable atomic pair/recovery; independent loopback TLS path loss and owner expiry/replacement/heal; local object sync/read-failure matrix | Multi-host/kernel-network, storage-host and orchestrator drills, linked-boundary contracts and integrated ownership/stock audits at scale. | | 12.5 | Finite cellular contact feeding/decay and separate M6 producer/consumer resource owner | One jointly owned cellular/environment budget, explicit effective depth and source entitlements, versioned water/atmosphere species mapping and producer coefficients, plus closed/open and nutrient-exhaustion comparisons. | | 12.6 | Retained archive, regional census/lineage queries, replayed history, scoped multi-region queries and consistent database head snapshots (12.6a-d); registered coverage/history domains follow in 12.7a-b | Cross-domain clone provenance, retained historical coverage/other viable stores, target-host query/index qualification and client permission/presentation; no account power or invented precise statistical ancestry. | -| 12.7 | Viability/unknown policy; consistent cuts, registered-region coverage and database history domains with supported restore forks (12.7a-b/schema 51) | Cross-domain clone provenance, other viable-store/transfer reconciliation and durable accepted global evidence records. Missing/stale/unsupported owners must prevent terminal extinction. | +| 12.7 | Viability/unknown policy; consistent cuts, registered coverage, history domains/restore forks and verified pending handoff evidence (12.7a-c/schema 52) | Cross-domain clone provenance, other viable-store coverage and durable accepted global evidence records/freshness. Missing/stale/unsupported owners must prevent terminal extinction. | | 12.8 | Deterministic seed/regime drivers and separate-process exact continuation | Reproducible integrated worlds running for actual days, real restarts/clone comparisons and retained accounting/diversity/cost audits. Accelerated tests cannot supply elapsed days. | | 12.9 | M11 feedback and current UI review retained | Returning-player samples on the integrated ecology and explicit product/3D readiness decision after soak evidence. No M12 product acceptance or World0 promotion is inferred. | @@ -93,6 +93,11 @@ their subadult realization while reproduction requires a fully realized parent. Finite extra food alone does not close repeated wildlife generations. A versioned growth contract and real multi-generation evidence remain necessary; transfer tests must not fabricate adult wildlife or silently change the rules to pass. +The existing metabolic construction budget already derives a channelwise +affordable growth transition with explicit intake/release, but cellular ticks do +not apply it. It is a concrete basis for a separately versioned funded maturation +contract that also resolves expanded-body collision, tick ordering and offline +freeze. Its existence alone does not admit a new growth rule. M12.6a's [regional census projection](../../schemas/ecology/regional-cellular-census-v1.md) independently implements local current-genome and descendant-closure queries. @@ -138,6 +143,14 @@ unknown original context; direct privileged database copies can bypass the workflow. Cross-domain clone provenance, other viable/in-flight stores and durable accepted reconciliation remain open. This still does not close M12.7. +M12.7c/schema 52's [pending handoff evidence](../../schemas/ecology/census-pending-migrations-v1.md) +now captures active source/destination pointers with domain/registry/heads, binds +them to the selected cuts and fully verifies retained source capsules and staged +ghost biology after capture. Source-owned life is already in the census; ghosts +add none. Prepare/stage/abort/commit races preserve the selected view. Unavailable +or false evidence fails the whole query. Other viable stores, external clone +provenance and accepted durable global reconciliation/freshness remain open. + Current expansion readiness: **no-go for M13 integration or World0 promotion** while these M12 integration/evidence gates remain open. This records the present engineering evidence, not a substitute for the later maintainer product review. diff --git a/docs/engineering/TEST_STRATEGY.md b/docs/engineering/TEST_STRATEGY.md index 5f7011e..7695aa0 100644 --- a/docs/engineering/TEST_STRATEGY.md +++ b/docs/engineering/TEST_STRATEGY.md @@ -1,5 +1,15 @@ # Validation strategy +M12.7c/schema 52 captures pending handoffs with registered heads, then fully +replays source capsules and destination ghosts outside the metadata transaction. +For raw/population owners, race prepare/stage/abort/commit with capture; verify +unchanged source-owned census totals and zero extra ghost stock. Deny capsule/ +ghost reads and reject false transfer quantities or real objects with false +ghost biology. Owner fault seams test malformed pending pointers and an injected +65-row response (bounds before decoding). Retain full immutable replay, TLS and +cleanup checks, migration/checksum/permissions, backup/DR, sanitizers and default +CTest. These tests do not establish every viable store or terminal extinction. + M12.7b/schema 51 checks database-domain adoption/no-op stability, restore-only quarantine, transaction rollback, exact fork cuts, idempotent/stale requests and runtime read-only privileges. Registered snapshots retain their domain during an diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index cdc02bd..4540eff 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -1,5 +1,84 @@ # Validation report +## 2026-10-18 - M12.7c registered census pending handoff evidence + +[ADR-0156](../adr/0156-census-pending-migration-evidence.md)/schema 52 captures +the union of active source/destination migration pointers in the same read-only +repeatable-read view as domain, complete registry and current heads. The source +predecessor must match its selected head; a staged destination must also match. +Both predecessor builds must be allowlisted, including an older optimistic +destination cut. After capture, full source capsule and destination ghost replay +verifies exact members/material/bytes, without adding any ghost stock to census. + +Executed locally: + +- MSVC default Release full CTest **79/79 passed, 14.14 s**. +- GCC 14 Release final semantic-failure population/history/migration/census matrix + passed **exit 0, 18 s** (16 s population). The final predecessor-build allowlist + guard and the complete affected cases passed **exit 0, 17 s** (16 s population). +- Clang 19 **ASan/UBSan** complete affected raw/population matrix, real loopback + TLS request/reply loss and local storage faults passed **exit 0, 77 s** + (73 s population), without sanitizer reports, before the final allowlist guard. + The final rerun passed **exit 0, 71 s** (69 s population), without reports. +- clang-tidy 19 passed on both changed persistence adapters and integration test + files; final semantic-failure assertions passed a separate recheck. Final + allowlist analysis/format also passed. Changed shell scripts pass `bash -n`. +- Initial full private persistence/process/backup/DR and SQL qualification passed + **exit 0, 539 s** before the final allowlist guard. The final frozen-binary rerun + passed **exit 0, 558 s**: C++ integration/process/load/history **435 s**, migration + TLS/storage faults **42 s**, life reservations **4 s**, domain/legacy/backup/ + restore **31 s**, host-loss DR **8 s**, upgrade **10 s**, migration history + **5 s**, runtime permission audit **13 s**, checksum/future rejection **8 s**, + plus setup/migration. The final completion sentinel records zero and the log + contains the all-phase success summary. The new getter's missing-grant case + correctly fails the effective-privilege audit before restoring the grant. +- Final `git diff --check` passed; private test clusters stop on exit and preserve + forensic directories. No biological golden or prior numbered migration changed. + +Both representations, in both directions, use real player-origin wildlife and +independent connections. Capture before preparation sees no pending request; +the next view sees source-only evidence. A concurrent stage leaves an earlier +view prepared and the next staged. A concurrent abort or paired commit preserves +the captured staged evidence; the next view sees no pointers and, for commit, +both new heads. All pending views retain the original source-owned census stock. +No receipt, ghost or transfer amount is added as another resident population. + +Actual read denial of the source's destination capsule and the destination ghost +fails the registered result without fallback to live peer state. Metadata-only +false transferred material and a digest-valid object with the wrong complete +ghost biology are rejected by full replay. Trusted owner fault seams move a +source pointer to the wrong region and inject 65 backend rows; these fail without +partial evidence. The overflow fixture is a response-bound test, not 65 valid +simultaneous migrations or a qualified world size. The original SQL function and +pointer are restored before continuing the real workflow. + +Schema 52 adds one least-privilege read function and no state table/index, +registration/publication writes, biological format/rule/golden or dependency. +Prior numbered migrations 1..51 are unchanged. Latest-prefix assertions include +52; unknown-future rejection uses 53. Runtime audit requires the new read grant. +The pre-context backup fixture removes both schema-51 metadata and schema-52 +getter to reconstruct the exact old prefix; old external ancestry is not invented. + +Limits: active-pointer evidence is not an audit of every historical preparation, +coverage of every viable store, accepted durable global freshness or terminal +extinction. No target-host capacity, new TSan, remote CI, multi-host/storage-host +failure, cellular clone/pending-migration backup restore, actual multi-day ecology +or product acceptance is claimed. Generic backup/DR regression remains separate +from cellular restore qualification. No live database, deployment or World0 action. + +Logs: `build/m127c-build.log`, `build/m127c-rebuild.log`, +`build/m127c-final-build.log`, `build/m127c-native.log`, +`build/m127c-population.log`, `build/m127c-final-population.log`, +`build/m127c-asan-build.log`, `build/m127c-asan.log`, +`build/m127c-tidy.log`, `build/m127c-format.log`, +`build/m127c-final-tidy.log`, `build/m127c-final-format.log`, +`build/m127c-allowlist-build.log`, `build/m127c-allowlist.log`, +`build/m127c-final-asan-build.log`, `build/m127c-final-asan.log`, +`build/m127c-allowlist-tidy.log`, `build/m127c-allowlist-format.log`, +`build/m127c-shell.log`, `build/m127c-full.log`, +`build/m127c-full.exit`, `build/m127c-final-full.log`, +`build/m127c-final-full.exit` (completion only). + ## 2026-10-18 - M12.7b database history domains and restore provenance [ADR-0155](../adr/0155-database-history-domain.md)/schema 51 adds a database diff --git a/include/evo/persistence/cellular_census_snapshot.hpp b/include/evo/persistence/cellular_census_snapshot.hpp index cc1d805..9f8da41 100644 --- a/include/evo/persistence/cellular_census_snapshot.hpp +++ b/include/evo/persistence/cellular_census_snapshot.hpp @@ -1,6 +1,7 @@ #pragma once #include "evo/persistence/cellular_history_store.hpp" +#include "evo/persistence/cellular_migration_store.hpp" #include "evo/server/lifecycle/cellular_census_set.hpp" #include @@ -27,7 +28,8 @@ enum class CellularSnapshotError : std::uint8_t { history, census, coverage, - context + context, + migration }; struct RegisteredCellularRegion { std::uint64_t region{}, frame{}; @@ -49,6 +51,7 @@ struct RegisteredCellularCensusSnapshot { std::vector catalog; // Exact registry view used for head selection. CellularCensusSnapshot snapshot; Uuid history_domain{}; // Schema 51 database namespace captured with catalog and heads. + std::vector migrations; // Schema 52 verified pending handoffs. }; struct CellularSnapshotFailure { CellularSnapshotError kind{}; @@ -57,6 +60,7 @@ struct CellularSnapshotFailure { CellularHistoryFailure history{}; // Meaningful for history. server::CellularCensusSetError census{}; // Meaningful for census. CellularCoverageFailure coverage{}; // Meaningful for coverage. + RecoveryFailure migration{}; // Meaningful for migration; no partial result on failure. }; // Test/fault seam after each head has been captured (zero-based request index). // Production callers pass nothing. A throwing hook closes the capture connection. @@ -76,9 +80,10 @@ load_cellular_census_snapshot(Postgres&, std::uint64_t world, std::span retained_builds, std::chrono::milliseconds timeout, const CellularCensusSnapshotHooks& hooks = {}); -// Schema 51: require resources for EVERY registered region of this world, +// Schema 52: require resources for EVERY registered region of this world, // including expired/released holders and regions with unavailable checkpoints. -// Capture the registry and all heads in the SAME short database snapshot. +// Capture domain, registry, pending handoffs and heads in the SAME database view. +// Fully verify retained handoff capsules/ghosts after capture. Ghosts add no stock. // Empty/over-bound registries or any coverage/frame mismatch fail, never zero. // Completeness is only for registered regions of this connected database view; // Domain/world identifies this database history, including supported restore forks. diff --git a/include/evo/persistence/cellular_migration_store.hpp b/include/evo/persistence/cellular_migration_store.hpp index a858847..0d7ab74 100644 --- a/include/evo/persistence/cellular_migration_store.hpp +++ b/include/evo/persistence/cellular_migration_store.hpp @@ -4,6 +4,10 @@ #include "evo/server/lifecycle/cellular_migration_trace.hpp" namespace evo::persistence { +struct CellularMigrationObservation { + server::CellularMigrationTrace trace; + bool staged{}; // Ghost/capsules are verified evidence, never additional owned stock. +}; struct CellularMigrationPrepared { RegionKey key{}; // Source region. std::vector trace; diff --git a/migrations/0052_census_pending_migrations.sql b/migrations/0052_census_pending_migrations.sql new file mode 100644 index 0000000..ba999d5 --- /dev/null +++ b/migrations/0052_census_pending_migrations.sql @@ -0,0 +1,36 @@ +-- Read-only active handoff evidence, not another ownership store (ADR-0156). +CREATE FUNCTION evolution.load_world_cellular_migrations_v1(w evolution.u64) +RETURNS TABLE(trace bytea,source_manifest bytea,destination_manifest bytea, + ghost_sha256 bytea,ghost_bytes evolution.u64,coherent boolean) +LANGUAGE plpgsql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ +BEGIN + IF w IS NULL OR w=0 THEN + RAISE EXCEPTION 'nonzero world required' USING ERRCODE='22023'; + END IF; + RETURN QUERY + WITH ids AS ( + SELECT x.request_id FROM evolution.cellular_migration_pending x WHERE x.world_id=w + UNION + SELECT x.request_id FROM evolution.cellular_migration_destination_pending x WHERE x.world_id=w + ) + SELECT p.trace,s.manifest,d.manifest,g.ghost_sha256,g.ghost_bytes, + coalesce(p.world_id=w AND s.world_id=w AND d.world_id=w + AND s.region_id=p.source_region AND d.region_id=p.destination_region + AND substring(p.trace FROM 49 FOR 16)=pg_catalog.uuid_send(p.request_id) + AND EXISTS(SELECT 1 FROM evolution.cellular_migration_pending x + WHERE x.world_id=w AND x.region_id=p.source_region AND x.request_id=p.request_id) + AND (CASE WHEN g.request_id IS NULL THEN + NOT EXISTS(SELECT 1 FROM evolution.cellular_migration_destination_pending x WHERE x.request_id=p.request_id) + ELSE EXISTS(SELECT 1 FROM evolution.cellular_migration_destination_pending x + WHERE x.world_id=w AND x.region_id=p.destination_region AND x.request_id=p.request_id) END) + AND NOT EXISTS(SELECT 1 FROM evolution.cellular_migration_aborts x WHERE x.request_id=p.request_id) + AND NOT EXISTS(SELECT 1 FROM evolution.cellular_migration_commits x WHERE x.request_id=p.request_id),false) + FROM ids LEFT JOIN evolution.cellular_migration_preparations p USING(request_id) + LEFT JOIN evolution.region_commits s ON s.commit_id=p.source_commit + LEFT JOIN evolution.region_commits d ON d.commit_id=p.destination_commit + LEFT JOIN evolution.cellular_migration_stages g USING(request_id) + ORDER BY ids.request_id LIMIT 65; +END $$; +REVOKE ALL ON FUNCTION evolution.load_world_cellular_migrations_v1(evolution.u64) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION evolution.load_world_cellular_migrations_v1(evolution.u64) TO :"runtime_role"; +INSERT INTO evolution.schema_migrations(version,sha256) VALUES(52,:'migration_sha_52'); diff --git a/migrations/apply.sql b/migrations/apply.sql index 1bc18dd..df14f3d 100644 --- a/migrations/apply.sql +++ b/migrations/apply.sql @@ -127,6 +127,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') SELECT :'history_v48' || ',49:' || :'migration_sha_49' AS history_v49 \gset SELECT :'history_v49' || ',50:' || :'migration_sha_50' AS history_v50 \gset SELECT :'history_v50' || ',51:' || :'migration_sha_51' AS history_v51 \gset + SELECT :'history_v51' || ',52:' || :'migration_sha_52' AS history_v52 \gset SELECT :'history' = :'history_v22' AS at_v22, :'history' = :'history_v23' AS at_v23, :'history' = :'history_v24' AS at_v24, @@ -156,8 +157,10 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') :'history' = :'history_v48' AS at_v48, :'history' = :'history_v49' AS at_v49, :'history' = :'history_v50' AS at_v50, - :'history' = :'history_v51' AS at_v51 \gset - \if :at_v51 + :'history' = :'history_v51' AS at_v51, + :'history' = :'history_v52' AS at_v52 \gset + \if :at_v52 + \elif :at_v51 \elif :at_v50 \elif :at_v49 \elif :at_v48 @@ -553,7 +556,8 @@ SELECT coalesce(max(version), 0) = 24 AS at_v24, coalesce(max(version), 0) = 48 AS at_v48, coalesce(max(version), 0) = 49 AS at_v49, coalesce(max(version), 0) = 50 AS at_v50, - coalesce(max(version), 0) = 51 AS at_v51 + coalesce(max(version), 0) = 51 AS at_v51, + coalesce(max(version), 0) = 52 AS at_v52 FROM evolution.schema_migrations \gset \if :at_v24 \ir 0025_cellular_v2_bundle.sql @@ -773,6 +777,7 @@ SELECT coalesce(max(version), 0) = 24 AS at_v24, \elif :at_v49 \elif :at_v50 \elif :at_v51 +\elif :at_v52 \else \echo 'Unknown or changed migration history after v43' DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$; @@ -809,6 +814,10 @@ SELECT coalesce(max(version), 0) = 50 AS needs_v51 FROM evolution.schema_migrati \if :needs_v51 \ir 0051_database_history_domain.sql \endif +SELECT coalesce(max(version), 0) = 51 AS needs_v52 FROM evolution.schema_migrations \gset +\if :needs_v52 + \ir 0052_census_pending_migrations.sql +\endif -- Effective-privilege audit on every run, including no-ops (ADR-0060). Attributes -- alone are insufficient: predefined roles such as pg_write_all_data, PUBLIC -- grants or SET-able memberships would bypass the fenced functions. Checks the @@ -828,6 +837,7 @@ BEGIN UNION SELECT m.member FROM pg_catalog.pg_auth_members m JOIN closure c ON m.roleid = c.oid ), fenced(oid) AS (VALUES + ('evolution.load_world_cellular_migrations_v1(evolution.u64)'::regprocedure::oid), ('evolution.load_history_domain_v1()'::regprocedure::oid), ('evolution.load_world_region_catalog_v1(evolution.u64)'::regprocedure::oid), ('evolution.publish_cellular_migration(uuid,bytea,bytea,uuid,evolution.u64,uuid,evolution.u64)'::regprocedure::oid), diff --git a/schemas/ecology/census-pending-migrations-v1.md b/schemas/ecology/census-pending-migrations-v1.md new file mode 100644 index 0000000..0dcf0b6 --- /dev/null +++ b/schemas/ecology/census-pending-migrations-v1.md @@ -0,0 +1,33 @@ +# Pending migration evidence in registered census snapshots v1 + +Semantic adapter contract, schema 52; +[ADR-0156](../../docs/adr/0156-census-pending-migration-evidence.md). +No biological codec/rule change or client endpoint. + +- Capture source/destination pending pointer union, history domain, complete + region registry and exact current heads in one read-only repeatable-read view. +- Inventory order is ascending request UUID. Maximum 64 requests; a 65th response + fails the whole query. Missing or inconsistent metadata never becomes empty. +- Every request must have its expected source pointer, a destination pointer iff + staged, neither abort nor commit, and consistent retained trace/manifests. +- Endpoints must belong to the captured registry/frame. Source manifest must + equal the captured source head's manifest; a staged destination must also match. + Prepared destination cuts remain optimistic and may differ from its latest head. + Both predecessor builds must be in the caller's retained build allowlist. +- After capture, replay source-side predecessor/capsule bytes exactly. For a stage, + independently replay destination-side predecessor/capsule bytes and compare the + complete derived ghost with its verified retained object. Metadata hashes or + matching aggregate inventory alone are insufficient. +- Return the verified canonical trace (members/material included) and staged flag. + External request references use the returned history domain/world/request tuple; + a copied request UUID alone does not identify a branch. + Capsules/ghosts own no additional stock; census totals remain the regional heads' + totals. Error returns no partial census/inventory. SQL/capture failure preserves + transaction cleanup; missing/corrupt/false objects remain uncertainty. +- Concurrent prepare/stage/abort/commit belongs to a later view. Already captured + immutable evidence remains verifiable after settlement. A later read sees its + new pointer state and heads atomically. No timeout/lease-expiry settlement. +- This inventories the existing active pointers, not every historical preparation + or every possible viable store. No live authority, accepted global freshness or + terminal-extinction record is created. Caller-declared snapshots remain scoped + observations without this registered-inventory guarantee. diff --git a/schemas/ecology/registered-world-census-coverage-v1.md b/schemas/ecology/registered-world-census-coverage-v1.md index 606650e..5cd49b6 100644 --- a/schemas/ecology/registered-world-census-coverage-v1.md +++ b/schemas/ecology/registered-world-census-coverage-v1.md @@ -41,6 +41,8 @@ authority, including when a registered holder is released or expired. The caller-declared API remains available with its previous scoped semantics. Schema 51 additionally binds registered results to a [database history domain](database-history-domain-v1.md) captured in the same view. +Schema 52 also binds [pending migration evidence](census-pending-migrations-v1.md) +to that view and fully verifies retained capsules/ghosts without adding ghost stock. Registered coverage alone adds no viable-reserve or outside-store reconciliation, durable accepted extinction record, client endpoint or permission. A consistent registered-region cut alone cannot accept terminal diff --git a/src/persistence/cellular_census_snapshot.cpp b/src/persistence/cellular_census_snapshot.cpp index f7dc014..0e05622 100644 --- a/src/persistence/cellular_census_snapshot.cpp +++ b/src/persistence/cellular_census_snapshot.cpp @@ -1,5 +1,7 @@ #include "evo/persistence/cellular_census_snapshot.hpp" +#include "cellular_migration_observation_detail.hpp" +#include #include #include #include @@ -19,10 +21,14 @@ auto history_failure(std::uint64_t region, CellularHistoryFailure error) { auto coverage_failure(CellularCoverageError kind, PgFailure cause = {}) { return std::unexpected(Failure{Error::coverage, 0, {}, {}, {}, {kind, cause}}); } +auto migration_failure(RecoveryFailure cause, std::uint64_t region = 0) { + return std::unexpected(Failure{Error::migration, region, {}, {}, {}, {}, cause}); +} struct Captured { std::vector catalog; std::vector cuts; Uuid history_domain{}; + std::vector migrations; }; std::expected history_domain(Postgres& db, std::chrono::milliseconds timeout) { const auto rows = db.query("SELECT pg_catalog.encode(pg_catalog.uuid_send(" @@ -120,6 +126,10 @@ std::expected capture(Postgres& db, std::uint64_t world, return coverage_failure(CellularCoverageError::frame); } captured.catalog = std::move(*required); + auto pending = detail::capture_cellular_migrations(db, world, timeout); + if (!pending) + return migration_failure(pending.error()); + captured.migrations = std::move(*pending); } captured.cuts.reserve(requests.size()); for (std::size_t i = 0; i < requests.size(); ++i) { @@ -163,6 +173,42 @@ load_snapshot(Postgres& db, std::uint64_t world, auto cuts = capture(db, world, requests, timeout, hooks, registered); if (!cuts) return std::unexpected(cuts.error()); + std::vector migrations; + for (const auto& pending : cuts->migrations) { + const auto source_manifest = decode_manifest(pending.source); + const auto destination_manifest = decode_manifest(pending.destination); + if (!source_manifest || !destination_manifest) + return migration_failure({RecoveryError::manifest}); + if (std::ranges::find(builds, source_manifest->build) == builds.end() || + std::ranges::find(builds, destination_manifest->build) == builds.end()) + return migration_failure({RecoveryError::unsupported}); + const auto trace = server::decode_migration_trace(pending.trace); + if (!trace || trace->world != world || + (!migrations.empty() && + trace->request.request <= migrations.back().trace.request.request)) + return migration_failure({RecoveryError::history}); + const auto endpoint = [&](std::uint64_t region) { + return std::ranges::lower_bound(requests, region, {}, + &CellularCensusRegionRequest::region); + }; + const auto s = endpoint(trace->request.source.region); + const auto d = endpoint(trace->request.destination.region); + if (s == requests.end() || d == requests.end() || + s->region != trace->request.source.region || + d->region != trace->request.destination.region || s->lease.frame != trace->frame || + d->lease.frame != trace->frame) + return migration_failure({RecoveryError::history}); + const auto si = static_cast(s - requests.begin()); + const auto di = static_cast(d - requests.begin()); + if (cuts->cuts[si].manifest != pending.source || + (pending.ghost && cuts->cuts[di].manifest != pending.destination)) + return migration_failure({RecoveryError::history}, s->region); + auto verified = + detail::verify_captured_cellular_migration(pending, *s->objects, *d->objects); + if (!verified) + return migration_failure(verified.error(), s->region); + migrations.push_back(std::move(*verified)); + } std::vector observations; observations.reserve(requests.size()); for (std::size_t i = 0; i < requests.size(); ++i) { @@ -187,7 +233,8 @@ load_snapshot(Postgres& db, std::uint64_t world, return std::unexpected(Failure{Error::census, 0, {}, {}, census.error()}); return RegisteredCellularCensusSnapshot{std::move(cuts->catalog), {std::move(cuts->cuts), std::move(*census)}, - cuts->history_domain}; + cuts->history_domain, + std::move(migrations)}; } } // namespace diff --git a/src/persistence/cellular_migration_observation_detail.hpp b/src/persistence/cellular_migration_observation_detail.hpp new file mode 100644 index 0000000..61f0a6d --- /dev/null +++ b/src/persistence/cellular_migration_observation_detail.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include "evo/persistence/cellular_migration_store.hpp" + +#include + +namespace evo::persistence::detail { +struct CapturedCellularMigration { + std::vector trace, source, destination; + std::optional ghost; +}; +// Internal census adapter: caller owns the read-only metadata snapshot. +[[nodiscard]] std::expected, RecoveryFailure> +capture_cellular_migrations(Postgres&, std::uint64_t world, std::chrono::milliseconds); +// Immutable historical evidence only; no live authority/activation or SQL access. +[[nodiscard]] std::expected +verify_captured_cellular_migration(const CapturedCellularMigration&, + const FilesystemObjectStore& source, + const FilesystemObjectStore& destination); +} // namespace evo::persistence::detail diff --git a/src/persistence/cellular_migration_store.cpp b/src/persistence/cellular_migration_store.cpp index 0e913ae..9706370 100644 --- a/src/persistence/cellular_migration_store.cpp +++ b/src/persistence/cellular_migration_store.cpp @@ -1,5 +1,7 @@ #include "evo/persistence/cellular_migration_store.hpp" +#include "cellular_migration_observation_detail.hpp" #include "evo/persistence/sha256.hpp" +#include "evo/server/lifecycle/cellular_census_set.hpp" #include #include @@ -441,6 +443,74 @@ load_cellular_migration_stage(Postgres& db, const FilesystemObjectStore& objects return fail(RecoveryError::history); return result; } +namespace detail { +std::expected, RecoveryFailure> +capture_cellular_migrations(Postgres& db, std::uint64_t world, milliseconds timeout) { + const std::array args{std::to_string(world)}; + const auto rows = + db.query("SELECT encode(trace,'hex'),encode(source_manifest,'hex')," + "encode(destination_manifest,'hex'),encode(ghost_sha256,'hex')," + "pg_catalog.trunc(ghost_bytes)::text,coherent::text " + "FROM evolution.load_world_cellular_migrations_v1($1::evolution.u64)", + args, timeout); + if (!rows) + return std::unexpected(RecoveryFailure{RecoveryError::store, failure(rows.error())}); + if (rows->size() > server::max_cellular_census_regions) + return std::unexpected( + RecoveryFailure{RecoveryError::store, {LeaseStoreError::invalid, {PgError::bounds}}}); + std::vector result; + for (const auto& row : *rows) { + if (row.size() != 6 || row[5].value_or("") != "true") + return std::unexpected(RecoveryFailure{RecoveryError::history}); + auto cut = parse({row[0], row[1], row[2], std::string{"false"}}); + if (!cut) + return std::unexpected(RecoveryFailure{RecoveryError::history}); + std::optional ghost; + if (row[3] || row[4]) { + auto stage = + parse_stage({row[0], row[1], row[2], std::string{"false"}, row[3], row[4]}); + if (!stage) + return std::unexpected(RecoveryFailure{RecoveryError::history}); + ghost = stage->ghost; + } + result.push_back( + {std::move(cut->trace), std::move(cut->source), std::move(cut->destination), ghost}); + } + return result; +} +std::expected +verify_captured_cellular_migration(const CapturedCellularMigration& record, + const FilesystemObjectStore& source_objects, + const FilesystemObjectStore& destination_objects) { + const auto fail = [](RecoveryError kind) { return std::unexpected(RecoveryFailure{kind}); }; + auto trace = server::decode_migration_trace(record.trace); + const auto s = decode_manifest(record.source), d = decode_manifest(record.destination); + const auto sd = sha256(record.source), dd = sha256(record.destination); + if (!trace || !s || !d || !sd || !dd || !cuts(*s, *d, *sd, *dd, *trace)) + return fail(RecoveryError::history); + const auto source = source_objects.get({s->payload_sha256, s->payload_bytes}); + const auto capsule = source_objects.get({d->payload_sha256, d->payload_bytes}); + if (!source || !capsule) + return fail(RecoveryError::tail); + const auto replay = derive(*s, *d, *source, *capsule, trace->request); + if (!replay || *replay != *trace) + return fail(RecoveryError::history); + if (record.ghost) { + if (record.ghost->sha256 == Sha256Digest{} || record.ghost->bytes == 0 || + record.ghost->bytes > max_payload_bytes(d->profile)) + return fail(RecoveryError::history); + const auto sc = destination_objects.get({s->payload_sha256, s->payload_bytes}); + const auto dc = destination_objects.get({d->payload_sha256, d->payload_bytes}); + const auto ghost = destination_objects.get(*record.ghost); + if (!sc || !dc || !ghost) + return fail(RecoveryError::tail); + const auto candidate = ghost_candidate(*s, *d, *sc, *dc, *trace); + if (!candidate || *candidate != *ghost) + return fail(RecoveryError::history); + } + return CellularMigrationObservation{std::move(*trace), record.ghost.has_value()}; +} +} // namespace detail namespace { constexpr auto commit_columns = "encode(trace,'hex'),encode(source_predecessor,'hex')," diff --git a/tests/population_publication_tests.cpp b/tests/population_publication_tests.cpp index d51a571..8b76a67 100644 --- a/tests/population_publication_tests.cpp +++ b/tests/population_publication_tests.cpp @@ -1545,13 +1545,133 @@ void migration_snapshot_race(Fixture& f, RegionKey destination_key, load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s), "before snapshot migration race"); const auto& before = before_registered.snapshot; - const CellularMigrationRequest request{f.id(), - {sk.region, sh.head.commit, f.id(), sh.head.digest}, - {dk.region, dh.head.commit, f.id(), dh.head.digest}, - genome}; - const auto frozen = - take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s), - "snapshot race source freeze"); + CellularMigrationRequest request{f.id(), + {sk.region, sh.head.commit, f.id(), sh.head.digest}, + {dk.region, dh.head.commit, f.id(), dh.head.digest}, + genome}; + const auto read = [&] { + return load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds, + 5s); + }; + const auto unchanged_stock = [&](const auto& snapshot) { + require(take(snapshot.snapshot.census.query(genome, CellularCensusScope::descendants), + "pending lineage") == + take(before.census.query(genome, CellularCensusScope::descendants), + "before pending lineage"), + "pending evidence added stock"); + }; + const auto refused = [&](const auto& result, RecoveryError error) { + require(!result && result.error().kind == CellularSnapshotError::migration && + result.error().migration.kind == error && reader->transaction_idle(), + "bad pending evidence returned a partial census or retained a transaction"); + }; + std::optional preparation; + const CellularCensusSnapshotHooks prepare_hook{[&](std::size_t i) { + if (i == 0) + preparation = + take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s), + "capture prepare"); + }}; + const auto before_prepare = + take(load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds, + 5s, prepare_hook), + "before concurrent prepare"); + auto frozen = take(std::move(preparation), "preparation hook ran"); + require(before_registered.migrations.empty() && before_prepare.migrations.empty(), + "concurrent preparation escaped its metadata view"); + const auto prepared = take(read(), "prepared snapshot inventory"); + require(prepared.migrations.size() == 1 && !prepared.migrations[0].staged && + prepared.migrations[0].trace == frozen.trace, + "source-only inventory missing or treated as staged"); + unchanged_stock(prepared); + const auto unretained = load_registered_cellular_census_snapshot( + *reader, f.key.world, requests, std::span{}, 5s); + refused(unretained, RecoveryError::unsupported); + const auto dm = take(decode_manifest(dh.manifest), "pending destination capsule"); + { + UnreadableObject denied(so.directory() / hexadecimal(dm.payload_sha256, false)); + refused(read(), RecoveryError::tail); + } + // Trusted owner fault seam: a pointer must not hide a foreign endpoint. + const std::array pointer{std::to_string(f.key.world), + std::to_string(sk.region)}; + take(f.owner->query("UPDATE evolution.cellular_migration_pending SET region_id=99999 " + "WHERE world_id=$1::evolution.u64 AND region_id=$2::evolution.u64", + pointer, 5s), + "malformed pending pointer seam"); + const auto malformed = read(); + take(f.owner->query( + "UPDATE evolution.cellular_migration_pending SET region_id=$2::evolution.u64 " + "WHERE world_id=$1::evolution.u64 AND region_id=99999", + pointer, 5s), + "restore pending pointer seam"); + refused(malformed, RecoveryError::history); + if (!reverse) { + const auto definition = + take(f.owner->query("SELECT " + "pg_get_functiondef('evolution.load_world_cellular_migrations_" + "v1(evolution.u64)'::regprocedure)", + {}, 5s), + "inventory definition"); + const auto original = take(definition.at(0).at(0), "inventory definition value"); + take( + f.owner->query( + "CREATE OR REPLACE FUNCTION evolution.load_world_cellular_migrations_v1(w " + "evolution.u64) " + "RETURNS TABLE(trace bytea,source_manifest bytea,destination_manifest bytea," + "ghost_sha256 bytea,ghost_bytes evolution.u64,coherent boolean) " + "LANGUAGE sql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ " + "SELECT " + "NULL::bytea,NULL::bytea,NULL::bytea,NULL::bytea,NULL::evolution.u64,true " + "FROM pg_catalog.generate_series(1,65) $$", + {}, 5s), + "overflow response seam"); + const auto overflow = read(); + take(f.owner->query(original, {}, 5s), "restore bounded inventory function"); + refused(overflow, RecoveryError::store); + require(overflow.error().migration.store.cause.kind == PgError::bounds, + "65th response was not rejected before parsing"); + } + std::optional stage; + const CellularCensusSnapshotHooks stage_hook{[&](std::size_t i) { + if (i == 0) + stage = take(stage_cellular_migration(*f.db, so, dst, dl, sh, dh, frozen.trace, 5s), + "capture stage"); + }}; + const auto before_stage = take(load_registered_cellular_census_snapshot( + *reader, f.key.world, requests, builds, 5s, stage_hook), + "before concurrent stage"); + const auto staged = take(read(), "staged snapshot inventory"); + require(before_stage.migrations.size() == 1 && !before_stage.migrations[0].staged && + staged.migrations.size() == 1 && staged.migrations[0].staged, + "concurrent stage escaped the capture view"); + unchanged_stock(staged); + const auto ghost = take(std::move(stage), "stage hook ran").ghost; + const auto ghost_path = dst.directory() / hexadecimal(ghost.sha256, false); + { + UnreadableObject denied(ghost_path); + refused(read(), RecoveryError::tail); + } + const CellularCensusSnapshotHooks abort_hook{[&](std::size_t i) { + if (i == 0) + take(abort_cellular_migration_freeze(*f.db, sk, sl, frozen.trace.request.request, + 5s), + "capture abort"); + }}; + const auto before_abort = take(load_registered_cellular_census_snapshot( + *reader, f.key.world, requests, builds, 5s, abort_hook), + "before concurrent abort"); + const auto after_abort = take(read(), "after abort inventory"); + require(before_abort.migrations.size() == 1 && before_abort.migrations[0].staged && + after_abort.migrations.empty(), + "abort escaped the selected inventory view"); + unchanged_stock(before_abort); + unchanged_stock(after_abort); + request.request = f.id(); + request.source.commit = f.id(); + request.destination.commit = f.id(); + frozen = take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s), + "snapshot race source freeze"); take(stage_cellular_migration(*f.db, so, dst, dl, sh, dh, frozen.trace, 5s), "snapshot race destination stage"); std::optional published; @@ -1592,6 +1712,10 @@ void migration_snapshot_race(Fixture& f, RegionKey destination_key, load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s), "snapshot after pair publication"); const auto& after = after_registered.snapshot; + require(raced_registered.migrations.size() == 1 && raced_registered.migrations[0].staged && + raced_registered.migrations[0].trace == frozen.trace && + after_registered.migrations.empty(), + "commit escaped the captured inventory or retained settled pointers"); require(before_registered.catalog == raced_registered.catalog && after_registered.catalog == before_registered.catalog && before_registered.catalog == @@ -2620,6 +2744,19 @@ void migration_freezes(Fixture& f) { load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s); require(!false_replay && false_replay.error().kind == RecoveryError::history, "freeze recovery trusted declared material without complete biological replay"); + const auto refuse_false_inventory = [&] { + const std::array resources{ + CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()}, + CellularCensusRegionRequest{destination_key.region, destination_lease, + destination_objects.get()}}; + const auto result = + load_registered_cellular_census_snapshot(*f.db, f.key.world, resources, builds, 5s); + require(!result && result.error().kind == CellularSnapshotError::migration && + result.error().migration.kind == RecoveryError::history && + f.db->transaction_idle(), + "registered census accepted semantically false pending biology"); + }; + refuse_false_inventory(); take( abort_cellular_migration_freeze(*f.db, f.key, f.lease, false_trace.request.request, 5s), "explicitly abort false preparation without moving stock"); @@ -2658,6 +2795,7 @@ void migration_freezes(Fixture& f) { *f.db, *destination_objects, destination_key, destination_lease, 5s); require(!false_ghost_replay && false_ghost_replay.error().kind == RecoveryError::history, "stage recovery accepted a real object with the wrong complete biology"); + refuse_false_inventory(); take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, false_stage_freeze.trace.request.request, 5s), "abort false ghost evidence"); diff --git a/tests/region_migration_tests.cpp b/tests/region_migration_tests.cpp index 9156cc1..c45394e 100644 --- a/tests/region_migration_tests.cpp +++ b/tests/region_migration_tests.cpp @@ -217,6 +217,7 @@ void upgrade_across_migration_and_barrier() { " -v migration_sha_49=$(s 0049_cellular_migration_publication.sql)" " -v migration_sha_50=$(s 0050_world_region_catalog.sql)" " -v migration_sha_51=$(s 0051_database_history_domain.sql)" + " -v migration_sha_52=$(s 0052_census_pending_migrations.sql)" " -v runtime_role=evo_runtime_test -f migrations/apply.sql" " > /dev/null 2>&1") != 0); CHECK(history(*owner_db) == "1,2,3,4,5,6,7"); @@ -234,7 +235,7 @@ void upgrade_across_migration_and_barrier() { "tools/migrate-postgres.sh\" > /dev/null") == 0); CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24," - "25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51"); + "25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52"); CHECK(scalar(*owner_db, "SELECT (c.condeferrable AND c.condeferred AND " "c.confrelid = 'evolution.region_commits'::regclass)::text " "FROM pg_catalog.pg_constraint c WHERE c.conrelid = " diff --git a/tools/ci/history-domain-check.sh b/tools/ci/history-domain-check.sh index 5d85d9b..f1b8c89 100644 --- a/tools/ci/history-domain-check.sh +++ b/tools/ci/history-domain-check.sh @@ -60,9 +60,10 @@ done # Restore the resulting real legacy backup through both supported workflows. sql > /dev/null <<'SQL' DROP FUNCTION evolution.fork_restored_history_v1(uuid); +DROP FUNCTION evolution.load_world_cellular_migrations_v1(evolution.u64); DROP FUNCTION evolution.load_history_domain_v1(); DROP TABLE evolution.history_domain_cuts, evolution.current_history_domain, evolution.history_domains; -DELETE FROM evolution.schema_migrations WHERE version=51; +DELETE FROM evolution.schema_migrations WHERE version>=51; UPDATE evolution.retained_builds SET retired_at=NULL; SQL bash "$root/tools/backup/backup-world.sh" --objects "$objects" --out "$work/legacy" --label legacy-context > /dev/null diff --git a/tools/ci/postgres-check.sh b/tools/ci/postgres-check.sh index 8a458d3..cc9096e 100644 --- a/tools/ci/postgres-check.sh +++ b/tools/ci/postgres-check.sh @@ -154,7 +154,7 @@ PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" -f "$root/migrations/0001_region_bootstrap.sql" > /dev/null test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1 PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51 +test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52 # v2 heads have no payload binding: v2->v3 fails closed and leaves history at v2. v2_sha="$(sha256sum "$root/migrations/0002_region_fencing.sql")" v2_sha="${v2_sha%% *}" @@ -175,7 +175,7 @@ PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '01000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32), 'hex'))" > /dev/null PGDATABASE=v3_commits bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 51:1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51 +test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 52:1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52 test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) FROM evolution.region_commits")" = 1 # Review P1: a v4 database holding a published (possibly terminal) checkpoint has # no durable decisions for it. The v5 upgrade must refuse before changing anything. @@ -229,7 +229,7 @@ PGDATABASE=v6_populated psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '02000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32) || repeat('00', 80), 'hex'))" > /dev/null PGDATABASE=v6_populated bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51 +test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT count(*) FROM evolution.region_commits c, LATERAL evolution.parse_checkpoint_manifest(c.manifest) m WHERE m.commit_sequence = 0 AND m.predecessor_commit IS NULL")" = 1 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(conname, ',') FROM pg_constraint WHERE conrelid = 'evolution.region_heads'::regclass AND contype = 'c' AND pg_get_constraintdef(oid) LIKE '%end_tick%'")" = region_heads_published_tick phase permission-audit @@ -283,12 +283,13 @@ bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit # Schema 50: registry access is granted only through the bounded read function. expect_audit_failure 'REVOKE EXECUTE ON FUNCTION evolution.load_world_region_catalog_v1(evolution.u64) FROM evo_runtime_test' 'GRANT EXECUTE ON FUNCTION evolution.load_world_region_catalog_v1(evolution.u64) TO evo_runtime_test' expect_audit_failure 'REVOKE EXECUTE ON FUNCTION evolution.load_history_domain_v1() FROM evo_runtime_test' 'GRANT EXECUTE ON FUNCTION evolution.load_history_domain_v1() TO evo_runtime_test' +expect_audit_failure 'REVOKE EXECUTE ON FUNCTION evolution.load_world_cellular_migrations_v1(evolution.u64) FROM evo_runtime_test' 'GRANT EXECUTE ON FUNCTION evolution.load_world_cellular_migrations_v1(evolution.u64) TO evo_runtime_test' expect_audit_failure 'GRANT EXECUTE ON FUNCTION evolution.fork_restored_history_v1(uuid) TO evo_runtime_test' 'REVOKE EXECUTE ON FUNCTION evolution.fork_restored_history_v1(uuid) FROM evo_runtime_test' phase checksum-drift # Unknown future versions and checksum drift must not be repaired silently. -psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=52 WHERE version=51' +psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=53 WHERE version=52' if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi -psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=51 WHERE version=52" +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=52 WHERE version=53" psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=14" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=13" diff --git a/tools/migrate-postgres.sh b/tools/migrate-postgres.sh index 89acc55..fb93e13 100644 --- a/tools/migrate-postgres.sh +++ b/tools/migrate-postgres.sh @@ -114,6 +114,8 @@ digest_50="$(sha256sum "$root/migrations/0050_world_region_catalog.sql")" digest_50="${digest_50%% *}" digest_51="$(sha256sum "$root/migrations/0051_database_history_domain.sql")" digest_51="${digest_51%% *}" +digest_52="$(sha256sum "$root/migrations/0052_census_pending_migrations.sql")" +digest_52="${digest_52%% *}" psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_2=$digest_2" --set="migration_sha_3=$digest_3" \ --set="migration_sha_4=$digest_4" --set="migration_sha_5=$digest_5" \ @@ -158,5 +160,6 @@ psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_49=$digest_49" \ --set="migration_sha_50=$digest_50" \ --set="migration_sha_51=$digest_51" \ + --set="migration_sha_52=$digest_52" \ --set="runtime_role=$runtime_role" \ --file="$root/migrations/apply.sql"