diff --git a/docs/STATUS.md b/docs/STATUS.md index 868b8a4..4b11698 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1,5 +1,23 @@ # Project status and next steps +## M12.4j atomic paired biological publication - 2026-10-18 + +ADR-0149/schema 49 commits both exact migration results and one immutable common +receipt under both current fences. Second-insert failure rolls back both heads +and pointer settlement. Commit and abort have one exclusive outcome; exact +retries preserve both original manifests under fresh endpoint authority. Full +recovery replays the pair from retained local capsules before a host adopts a +changed head or becomes ready. The source owns stock until commit; afterwards +only the destination owns the migrants. + +Native Release passes 78/78; focused private PostgreSQL, ASan/UBSan, Clang +analysis, format and shell syntax checks pass. Complete private persistence, +process-fault, backup/DR and migration/permission qualification passes in 507 s. +[Evidence and limits](engineering/VALIDATION.md) +retain independent network-partition drills, linked ecology, maturation and later +M12 integration/soak gates. All of M12 remains in progress. No automatic route, +biological rule/format, deployment or live region changed. + ## M12.4i durable read-only destination staging - 2026-10-18 ADR-0148/schema 48 stages the destination under current authority at its exact diff --git a/docs/adr/0149-atomic-cellular-migration-publication.md b/docs/adr/0149-atomic-cellular-migration-publication.md new file mode 100644 index 0000000..ab47e66 --- /dev/null +++ b/docs/adr/0149-atomic-cellular-migration-publication.md @@ -0,0 +1,62 @@ +# ADR-0149: Atomic paired biological migration publication + +## Status + +Accepted and locally qualified for scoped M12.4j implementation, 2026-10-18. +Executed evidence and limits are in [VALIDATION](../engineering/VALIDATION.md). +No automatic routing, ecological boundary or deployment is +selected. + +## Decision + +Publish ADR-0144's two exact candidates in one PostgreSQL transaction after +ADR-0147 preparation and ADR-0148 destination staging. Lock endpoint lease rows +in increasing region order, require both current authorities and unchanged +captured heads, and bind the destination payload to the immutable staged ghost. +The immutable shared receipt names the preparation and both result commits. +Both result manifests preserve their profiles, bundle 10, history, quantum and +completed tick while advancing each commit sequence exactly once. They use no +ordinary biological batch or invented birth/death event. + +Restore both verified predecessor capsules, replay the exact trace and upload/ +read back both complete results before publication. Clear both pending pointers, +insert the common receipt, insert both commits and update both heads in the same +transaction. Any failure rolls everything back, including a failure at the second +commit. Deferred receipt foreign keys allow evidence guards to recognize this +dedicated publisher without disabling triggers or permitting generic bypasses. + +Before this commit the source owns all outgoing stock and the ghost owns none. +After it only the destination owns the transferred organisms/material. Current +regional budgets and explicit cumulative boundary flows retain their meanings. +Keep both predecessor histories/capsules, stage and receipt. No stock account is +created for the receipt, and no terminal extinction inference follows from it. + +Explicit source abort and paired publication serialize on the source lease row. +Abort wins only before commit; a committed request can never be aborted. If abort +wins, commit is refused. Exact committed retries compare both complete manifests +and return the original pair under current endpoint authority, including newer +epochs; changed bytes conflict. Unknown replies retain that exact prepared pair. +Lease replacement never changes its bytes or fabricates a second transfer. + +Recovery loads the shared receipt under current local authority, verifies both +predecessor/result manifests and their links, replays both candidates from local +retained capsules, compares both result digests and the complete local payload, +and requires the staged destination digest/length. Missing or false evidence +fails closed. A paused host whose pointers disappeared reloads its actual head +and performs this recovery before readiness; it never resumes stale source stock +or activates a ghost based solely on pointer absence. + +## Alternatives, risks and qualification + +Two ordinary publications can leave half a transfer committed. Treating timeout +as rollback can duplicate stock. Shared append-only evidence plus one fenced +transaction gives one outcome, at the cost of temporarily requiring both owners +and one metadata authority. This is not a general distributed transaction across +independent databases. + +Require both representations, complete stock/member equality, exact/conflicting +retries, abort/commit races, stale/replacement authorities, failed second inserts, +ambiguous replies, process loss before/after commit, missing/corrupt/false replay +evidence, host adoption/restart and subsequent biological continuation. Linked +geometry, independent network partitions, actual multi-day ecology and target +host capacity remain separate gates. diff --git a/docs/adr/README.md b/docs/adr/README.md index 7774ce8..5d543bf 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -1,6 +1,8 @@ # Architecture decision records -- [ADR-0148: Durable destination staging without activation](0148-durable-cellular-migration-staging.md) - schema 48 freezes the destination at its exact cut, retains a verified read-only ghost and replays it on recovery; source abort releases both pauses. Paired publication/activation remain open. +- [ADR-0149: Atomic paired biological migration publication](0149-atomic-cellular-migration-publication.md) - schema 49 binds both current fences, heads and exact candidates in one immutable receipt/transaction; recovery replays the pair and hosts verify the changed head before readiness. + +- [ADR-0148: Durable destination staging without activation](0148-durable-cellular-migration-staging.md) - schema 48 freezes the destination at its exact cut, retains a verified read-only ghost and replays it on recovery; source abort releases both pauses. - [ADR-0147: Durable biological preparation and source freeze](0147-durable-cellular-migration-freeze.md) - schema 47 retains exact preparation/capsules and explicit abort; the source stays authoritative and paused across worker replacement. Paired activation remains open. diff --git a/docs/architecture/PERSISTENCE_ARCHITECTURE.md b/docs/architecture/PERSISTENCE_ARCHITECTURE.md index 0c22664..e4d47cc 100644 --- a/docs/architecture/PERSISTENCE_ARCHITECTURE.md +++ b/docs/architecture/PERSISTENCE_ARCHITECTURE.md @@ -24,7 +24,13 @@ destination ghost stay read-only in the destination namespace. Both roles pause publication, interval/player admission and host readiness across replacement. Explicit source abort releases both pointers atomically; stage/abort races cannot activate or strand stock. See [ADR-0148](../adr/0148-durable-cellular-migration-staging.md). -Atomic paired result publication, its common receipt and activation remain required. +M12.4j/schema 49 now publishes both results and an immutable common receipt in +one transaction under both current fences. Abort is exclusive with commit; +second-insert failure rolls back both heads and pointer settlement. Recovery +replays both cuts and binds both results to the stage. Frozen hosts reload and +verify a changed head before readiness, rather than inferring activation from +pointer absence. See [ADR-0149](../adr/0149-atomic-cellular-migration-publication.md). +Independent network partitions and linked ecological boundaries remain separate. M7.1 now scopes the first implementation in [ADR-0058](../adr/0058-region-durability-contract.md) and [REGION_DURABILITY.md](REGION_DURABILITY.md): full checkpoint batches, not diff --git a/docs/architecture/REGION_DURABILITY.md b/docs/architecture/REGION_DURABILITY.md index 350a10a..135f72d 100644 --- a/docs/architecture/REGION_DURABILITY.md +++ b/docs/architecture/REGION_DURABILITY.md @@ -76,7 +76,14 @@ destination head remains unchanged and is durably paused alongside the source. Recovery verifies the complete ghost without activating it; explicit source abort releases both pointers atomically. Stage/abort races serialize on source authority. See [the staging contract](../../schemas/ecology/cellular-migration-stage-v1.md). -Atomic paired publication, commit/abort races and activation remain open. +ADR-0149/schema 49 now atomically publishes both exact heads and a common immutable +receipt under both current fences. Failure at either insert rolls back both +heads and both pending pointers. Commit and abort have one exclusive outcome; +exact committed retries preserve both original manifests under fresh authority. +Recovery replays both candidates from local capsules and checks both result +digests and the complete local payload. A frozen host verifies its changed head +before readiness. See [the paired commit contract](../../schemas/ecology/cellular-migration-commit-v1.md). +No atlas, independent-database protocol or automatic ecological movement follows. World/region/frame are nonzero durable IDs. No ID is a Pod name. The host records its unique holder ID and monotonically increasing nonzero lease epoch. Exhaustion diff --git a/docs/engineering/M12_EXACT_BIOLOGICAL_TRANSFER.md b/docs/engineering/M12_EXACT_BIOLOGICAL_TRANSFER.md index d803025..9f342e1 100644 --- a/docs/engineering/M12_EXACT_BIOLOGICAL_TRANSFER.md +++ b/docs/engineering/M12_EXACT_BIOLOGICAL_TRANSFER.md @@ -163,3 +163,30 @@ Atomic paired publication/common receipt, commit/abort races, recovery/activatio and linked ecology remain subsequent work. The ghost cannot run biology or contribute owned census/extinction stock. No biological format, rule, route, automatic movement, live region or deployment changes. + +## M12.4j atomic paired ownership publication + +ADR-0149/schema 49 publishes both same-profile bundle-10 no-tick candidates and +their immutable common receipt in one transaction under both current fences, +captured heads and the staged destination binding. Both complete objects are +uploaded/read back first. Deferred receipt references permit evidence guards +without disabling triggers. Second-insert failure rolls back both heads, receipt +and pointer settlement. Generic publishers cannot admit a half. + +Before commit the source owns outgoing stock; afterwards only the destination +owns it. Abort and commit have one exclusive outcome. Exact retries retain both +original manifests under newer current leases. Unknown replies retain the exact +prepared pair, without inferring rollback or rebasing. + +Recovery verifies both predecessor/result manifests, trace and staged ghost, +replays both candidates from local capsules, checks both result digests and +compares complete local biology. Missing or false evidence fails closed. A paused +host verifies a changed head before readiness, including after replacement; +pointer absence never authorizes stale source stock or an orphan ghost. Both +representations and return transfers use real player-origin wildlife. Existing +durable biological continuation runs afterwards. + +See [the paired commit contract](../../schemas/ecology/cellular-migration-commit-v1.md). +Independent network partitions, linked geometry/environment, maturation, world +census/cuts, actual elapsed-day ecology and target-host capacity remain open. +No automatic route, biological rule/format, golden or deployment changes. diff --git a/docs/engineering/M12_REMAINING_GATES.md b/docs/engineering/M12_REMAINING_GATES.md index ee1d09b..2b87a77 100644 --- a/docs/engineering/M12_REMAINING_GATES.md +++ b/docs/engineering/M12_REMAINING_GATES.md @@ -25,7 +25,7 @@ or authorize production enablement. |---|---|---| | 12.2 | Exact cellular model and paired matrix; local samples through 256 lives | Deployment-host tail costs including publication; larger wildlife cohorts, predation/twins and contention coverage; useful compression remains unimplemented. | | 12.3 | Dedicated population profile/trace/store/owning host; exact recovery and private PostgreSQL fault tests | Population-specific executable/network fault drills and target-host publication/capacity qualification. | -| 12.4 | Current-worker demand/debt, owned bounded intervals/restart recovery, and portable exact biological transfer candidates with boundary accounting | Durable biological source/destination escrow, partition/duplicate settlement, linked-boundary contracts and stock-in-transit audits. | +| 12.4 | Owned bounded intervals; exact transfer/accounting; durable preparation/staging, atomic paired publication and replayed host recovery | Independent network-partition drills, linked-boundary contracts and integrated ownership/stock audits at scale. | | 12.5 | Finite cellular contact feeding/decay and separate M6 producer/consumer resource owner | One jointly owned cellular/environment budget, explicit effective depth and source entitlements, versioned water/atmosphere species mapping and producer coefficients, plus closed/open and nutrient-exhaustion comparisons. | | 12.6 | Retained genome/organism archive with ancestry and common-ancestor queries | Append-only population census/history events in the same durable commit, bounded world history queries/indexing and client presentation; no account power or invented precise statistical ancestry. | | 12.7 | ADR-0133's viability/unknown semantics | Durable region-set/watermark reconciliation covering every owned representation, suspended/viable stock and migration escrow. Missing/stale regions must prevent terminal extinction. | @@ -67,9 +67,14 @@ its exact head through worker replacement; no timeout or drain releases it. ADR-0148/schema 48 now supplies destination-authorized staging at its exact captured head, verified read-only ghost/capsule retention and full destination recovery. Both endpoints stay paused; explicit source abort atomically releases -both pointers and races safely with staging. Atomic paired commit, its common -receipt, commit/abort races and activation remain open. Preparation remains an -optimistic destination observation; first staging rechecks both captured heads. +both pointers and races safely with staging. ADR-0149/schema 49 now commits both +exact results and an immutable common receipt under both current fences. Failed +second inserts roll back both heads; commit/abort races have one exclusive +outcome. Exact retries preserve both manifests under fresh authority. Hosts +replay and verify changed heads before readiness. Preparation remains an +optimistic destination observation; staging/publication recheck captured heads. +Independent network-partition drills and linked ecological boundaries remain +open; shared-database fault tests do not supply those proofs. The transfer tests also expose the current rules-8 maturation gap: buds retain their subadult realization while reproduction requires a fully realized parent. Finite extra food alone does not close repeated wildlife generations. A versioned diff --git a/docs/engineering/TEST_STRATEGY.md b/docs/engineering/TEST_STRATEGY.md index e2ae94a..4466310 100644 --- a/docs/engineering/TEST_STRATEGY.md +++ b/docs/engineering/TEST_STRATEGY.md @@ -45,6 +45,19 @@ resumes only its original biology after confirmed source abort. Full schema, permission, backup/DR and historical regression remain required. These tests do not qualify paired commit, destination activation or linked ecological movement. +M12.4j's private paired-publication matrix uses real wildlife, both representations +and return transfers. It requires complete source/destination equality with the +portable pair, one shared receipt and unchanged completed ticks. An injected +second-insert failure must roll back both heads, receipt and pointer settlement. +Wrong publishers, unstaged/aborted requests, stale fences and conflicting exact +retries fail. Tests cover genuine lock-timeout ambiguity, SIGKILL before/after +commit, fresh-epoch exact retry, independent-connection commit/abort races and +receipt recovery that rejects missing counterpart capsules and valid but false +biological objects. Owning hosts must reload both actual heads before readiness, +retain complete recovery after replacement and continue biology afterwards. +Independent network partitions, linked ecology and actual elapsed-day stability +remain separate evidence; full schema/permission/backup/DR regression is required. + Tests defend contracts, not directory coverage. All checks remain active in Release builds. Current proof has no actual network, biology or distributed recovery, so those categories below are milestone gates rather than passing diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index f247795..7e526b2 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -1,5 +1,60 @@ # Validation report +## 2026-10-18 - M12.4j atomic paired biological publication and recovery + +ADR-0149/schema 49 commits both exact bundle-10 no-tick results and one immutable +shared receipt under both current endpoint fences and heads. Both objects are +uploaded/read back first, and the destination binds its retained staged ghost. +The transaction settles both pointers atomically. Abort is exclusive with commit; +fresh-epoch retries preserve both original manifests. Recovery replays both +candidates from retained local capsules and verifies both result digests plus the +complete local payload. A paused host reloads/verifies its actual changed head +before readiness; pointer absence cannot resume stale source stock. + +Executed locally: + +- Native MSVC Release **78/78 passed**, 12.63 s. +- Actual GCC 14 optional server/persistence targets rebuilt; final private + PostgreSQL population/paired suite passed, **16 s** (13 s test phase). +- Linux Clang 19 ASan/UBSan private population/paired suite passed, **38 s** + (35 s test phase), with no sanitizer reports. +- Explicit clang-tidy 19 on the migration adapter, no-tick recovery adapter, + owning host and PostgreSQL population tests passed with no owned warnings. + Repository clang-format 19 and changed shell scripts' syntax checks passed. +- Complete frozen-binary private PostgreSQL qualification passed with **exit 0**, + **507 s**: C++ integration/process/load/history (431 s), life reservations, + backup/restore, host-loss DR, upgrade, migration history/rollback, runtime + permission audit and checksum/future-version rejection. The detached runner + recorded its completion code. + +Both representations use actual player-origin wildlife and return transfers. +Complete results equal the portable pair; committed tick identity is unchanged. +An owner-only test trigger fails the second result insert, proving both heads, +receipt and pointer settlement roll back. Unstaged requests, wrong publishers, +conflicting bytes and stale authority are refused. Real lock-timeout ambiguity +retains both manifests. SIGKILL before the transaction preserves both heads; +SIGKILL after commit is recovered through the exact pair. Fresh host epochs return +the original receipt. Abort-before-commit is refused by publication; committed +requests refuse abort; independent connections race the two operations to one +outcome without stranded pointers. A valid old object declared as the result +fails complete recovery in a rolled-back metadata-only adversarial fixture. +Missing counterpart capsules and false local bytes fail replay. Both owning hosts +adopt only verified heads, survive replacement and the existing durable biological +continuation matrix runs after the transfers. + +The first static-analysis pass found an intentionally discarded expected result +in a kill-test child; the child now checks it before its unexpected-return exit. +The final analysis pass is clean. No assertion or warning suppression was added. + +Limits: this is the shared-database ownership boundary. Independent network +partitions, linked ecological geometry/resources, maturation, global census/cuts, +actual elapsed-day soak and target-host capacity remain open. Backup/DR regression +uses its existing supported fixtures; it does not qualify cellular history cloning +or a new cellular restore procedure. No live database, World0, dependency, +biological rule/format, golden or deployment changed. The earlier captured +`Build_Errors.txt` and unrelated local work remain untracked; remote CI confirmation +of the prior runner repair remains separate. + ## 2026-10-18 - M12.4i durable read-only destination staging ADR-0148/schema 48 retains a destination-authorized staging record and bounded diff --git a/include/evo/persistence/cellular_migration_store.hpp b/include/evo/persistence/cellular_migration_store.hpp index 81426d8..a858847 100644 --- a/include/evo/persistence/cellular_migration_store.hpp +++ b/include/evo/persistence/cellular_migration_store.hpp @@ -74,4 +74,35 @@ retry_cellular_migration_stage(Postgres&, const LeaseGrant&, const CellularMigra [[nodiscard]] std::expected, RecoveryFailure> load_cellular_migration_stage(Postgres&, const FilesystemObjectStore&, RegionKey, const LeaseGrant&, std::chrono::milliseconds timeout); +struct CellularMigrationCommitPrepared { + Uuid request{}; + std::vector source_manifest, destination_manifest; +}; +struct CellularMigrationCommitted { + CommittedCheckpoint source, destination; +}; +struct CellularMigrationCommitFailure { + PublicationError kind{}; + LeaseStoreFailure store{}; + ObjectError object{}; + CellularMigrationCommitPrepared prepared; +}; +[[nodiscard]] std::expected +publish_cellular_migration(Postgres&, const FilesystemObjectStore& source_objects, + const FilesystemObjectStore& destination_objects, + const LeaseGrant& source_lease, const LeaseGrant& destination_lease, + const CommittedCheckpoint& source, + const CommittedCheckpoint& destination, + const server::CellularMigrationTrace&, const Sha256Digest& build, + std::chrono::milliseconds timeout, const PublishHooks& hooks = {}); +// Both CURRENT fences are separate from the retained manifests' original writers. +[[nodiscard]] std::expected +retry_cellular_migration_commit(Postgres&, const LeaseGrant& source_lease, + const LeaseGrant& destination_lease, + const CellularMigrationCommitPrepared&, + std::chrono::milliseconds timeout); +// True: full paired receipt/replay verified. False: this is another no-tick kind. +[[nodiscard]] std::expected verify_cellular_migration_checkpoint( + Postgres&, const FilesystemObjectStore&, const LeaseGrant&, const CommittedCheckpoint&, + std::span payload, std::chrono::milliseconds timeout); } // namespace evo::persistence diff --git a/migrations/0049_cellular_migration_publication.sql b/migrations/0049_cellular_migration_publication.sql new file mode 100644 index 0000000..76cd86b --- /dev/null +++ b/migrations/0049_cellular_migration_publication.sql @@ -0,0 +1,163 @@ +-- ADR-0149: one shared receipt and two heads in one fenced transaction. +CREATE TABLE evolution.cellular_migration_commits ( + request_id uuid PRIMARY KEY REFERENCES evolution.cellular_migration_stages, + source_result uuid UNIQUE NOT NULL REFERENCES evolution.region_commits(commit_id) DEFERRABLE INITIALLY DEFERRED, + destination_result uuid UNIQUE NOT NULL REFERENCES evolution.region_commits(commit_id) DEFERRABLE INITIALLY DEFERRED, + CHECK(source_result<>destination_result) +); +REVOKE ALL ON evolution.cellular_migration_commits FROM PUBLIC; +CREATE TRIGGER cellular_migration_commits_immutable BEFORE UPDATE OR DELETE ON evolution.cellular_migration_commits + FOR EACH ROW EXECUTE FUNCTION evolution.refuse_history_rewrite(); +CREATE TRIGGER cellular_migration_commits_no_truncate BEFORE TRUNCATE ON evolution.cellular_migration_commits + FOR EACH STATEMENT EXECUTE FUNCTION evolution.refuse_history_rewrite(); + +CREATE FUNCTION evolution.cellular_migration_commit_record(id uuid) +RETURNS TABLE(trace bytea,source_predecessor bytea,destination_predecessor bytea,source_result bytea,destination_result bytea,ghost_sha256 bytea,ghost_bytes evolution.u64) +LANGUAGE sql STABLE SET search_path=pg_catalog,pg_temp AS $$ + SELECT p.trace,s.manifest,d.manifest,sr.manifest,dr.manifest,g.ghost_sha256,g.ghost_bytes + FROM evolution.cellular_migration_commits x JOIN evolution.cellular_migration_preparations p USING(request_id) + JOIN evolution.cellular_migration_stages g USING(request_id) + JOIN evolution.region_commits s ON s.commit_id=p.source_commit + JOIN evolution.region_commits d ON d.commit_id=p.destination_commit + JOIN evolution.region_commits sr ON sr.commit_id=x.source_result + JOIN evolution.region_commits dr ON dr.commit_id=x.destination_result WHERE x.request_id=id +$$; +REVOKE ALL ON FUNCTION evolution.cellular_migration_commit_record(uuid) FROM PUBLIC; + +CREATE FUNCTION evolution.publish_cellular_migration(id uuid,sm bytea,dm bytea,sh uuid,se evolution.u64,dh uuid,de evolution.u64) +RETURNS TABLE(trace bytea,source_predecessor bytea,destination_predecessor bytea,source_result bytea,destination_result bytea,ghost_sha256 bytea,ghost_bytes evolution.u64) +LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ +DECLARE p evolution.cellular_migration_preparations%ROWTYPE; g evolution.cellular_migration_stages%ROWTYPE; + old evolution.cellular_migration_commits%ROWTYPE; + s evolution.checkpoint_manifest := evolution.parse_checkpoint_manifest(sm); + d evolution.checkpoint_manifest := evolution.parse_checkpoint_manifest(dm); + predecessor evolution.checkpoint_manifest; h evolution.region_heads%ROWTYPE; + m bytea; c evolution.checkpoint_manifest; result_digest bytea; + bundle constant bytea := '\x080000000700000002000000010000000100000004000000030000000200000001000000'; + zero32 constant bytea := decode(repeat('00',32),'hex'); +BEGIN + SELECT * INTO p FROM evolution.cellular_migration_preparations x WHERE x.request_id=id; + IF NOT FOUND THEN RAISE EXCEPTION 'unknown migration preparation' USING ERRCODE='EVL04'; END IF; + IF s.world_id<>p.world_id OR d.world_id<>p.world_id OR s.region_id<>p.source_region OR d.region_id<>p.destination_region + OR s.commit_id<>encode(substring(p.trace FROM 121 FOR 16),'hex')::uuid + OR d.commit_id<>encode(substring(p.trace FROM 193 FOR 16),'hex')::uuid THEN + RAISE EXCEPTION 'foreign migration results' USING ERRCODE='22023'; + END IF; + PERFORM 1 FROM evolution.region_leases x WHERE x.world_id=p.world_id AND x.region_id IN (p.source_region,p.destination_region) + ORDER BY x.region_id FOR UPDATE; + PERFORM evolution.check_region_lease(p.world_id,p.source_region,sh,se); + PERFORM evolution.check_region_lease(p.world_id,p.destination_region,dh,de); + SELECT * INTO old FROM evolution.cellular_migration_commits x WHERE x.request_id=id; + IF FOUND THEN + IF NOT EXISTS(SELECT 1 FROM evolution.region_commits x WHERE x.commit_id=old.source_result AND x.manifest=sm) + OR NOT EXISTS(SELECT 1 FROM evolution.region_commits x WHERE x.commit_id=old.destination_result AND x.manifest=dm) THEN + RAISE EXCEPTION 'migration commit request reused' USING ERRCODE='EVL07'; + END IF; + RETURN QUERY SELECT * FROM evolution.cellular_migration_commit_record(id); + RETURN; + END IF; + SELECT * INTO g FROM evolution.cellular_migration_stages x WHERE x.request_id=id; + IF NOT FOUND OR EXISTS(SELECT 1 FROM evolution.cellular_migration_aborts x WHERE x.request_id=id) + OR NOT EXISTS(SELECT 1 FROM evolution.cellular_migration_pending x WHERE x.world_id=p.world_id AND x.region_id=p.source_region AND x.request_id=id) + OR NOT EXISTS(SELECT 1 FROM evolution.cellular_migration_destination_pending x WHERE x.world_id=p.world_id AND x.region_id=p.destination_region AND x.request_id=id) THEN + RAISE EXCEPTION 'migration not staged or was aborted' USING ERRCODE='EVL18'; + END IF; + IF s.holder<>sh OR s.epoch<>se OR d.holder<>dh OR d.epoch<>de + OR d.payload_sha256<>g.ghost_sha256 OR d.payload_bytes<>g.ghost_bytes THEN + RAISE EXCEPTION 'migration writer or staged ghost differs' USING ERRCODE='EVL07'; + END IF; + FOREACH m IN ARRAY ARRAY[sm,dm] LOOP + c := evolution.parse_checkpoint_manifest(m); + SELECT * INTO h FROM evolution.region_heads x WHERE x.world_id=c.world_id AND x.region_id=c.region_id FOR UPDATE; + IF NOT FOUND OR h.commit_sequence=18446744073709551615 OR c.commit_sequence<>h.commit_sequence+1 + OR c.predecessor_commit<>h.commit_id OR c.predecessor_digest<>h.digest OR c.begin_tick<>h.end_tick + OR c.end_tick<>h.end_tick OR c.end_tick<>evolution.manifest_uint(p.trace,32,8) + OR h.commit_id<>(CASE WHEN c.region_id=p.source_region THEN p.source_commit ELSE p.destination_commit END) THEN + RAISE EXCEPTION 'migration endpoint head moved' USING ERRCODE='EVL03'; + END IF; + predecessor := evolution.parse_checkpoint_manifest((SELECT x.manifest FROM evolution.region_commits x WHERE x.commit_id=h.commit_id)); + IF c.profile NOT IN (4,5) OR c.profile<>predecessor.profile OR c.profile<>s.profile OR c.versions<>bundle + OR c.versions<>predecessor.versions OR c.frame_id<>predecessor.frame_id + OR c.tick_microseconds<>predecessor.tick_microseconds OR c.frame_id<>evolution.manifest_uint(p.trace,24,8) + OR c.frame_id<>(SELECT x.frame_id FROM evolution.region_leases x WHERE x.world_id=c.world_id AND x.region_id=c.region_id) + OR evolution.manifest_uint(m,228,4)<>0 + OR evolution.manifest_uint(m,232,4)<>(CASE c.profile WHEN 4 THEN 4 ELSE 5 END) + OR evolution.manifest_uint(m,236,4)<>1 OR c.trace_bytes<>0 OR c.trace_sha256<>zero32 + OR c.decision_count<>0 OR c.decisions_sha256<>zero32 + OR NOT EXISTS(SELECT 1 FROM evolution.retained_builds x WHERE x.build_identity=c.build_identity) + OR EXISTS(SELECT 1 FROM evolution.population_interval_pending x WHERE x.world_id=c.world_id AND x.region_id=c.region_id) THEN + RAISE EXCEPTION 'unsupported paired migration result' USING ERRCODE='EVL06'; + END IF; + END LOOP; + INSERT INTO evolution.cellular_migration_commits VALUES(id,s.commit_id,d.commit_id); + DELETE FROM evolution.cellular_migration_pending x WHERE x.request_id=id; + DELETE FROM evolution.cellular_migration_destination_pending x WHERE x.request_id=id; + FOREACH m IN ARRAY ARRAY[sm,dm] LOOP + c := evolution.parse_checkpoint_manifest(m); result_digest := sha256(m); + INSERT INTO evolution.region_commits(world_id,region_id,commit_sequence,commit_id,commit_digest,manifest, + begin_tick,end_tick,published_epoch,tick_microseconds,versions,build_identity,payload_bytes,payload_sha256) + VALUES(c.world_id,c.region_id,c.commit_sequence,c.commit_id,result_digest,m,c.begin_tick,c.end_tick,c.epoch, + c.tick_microseconds,c.versions,c.build_identity,c.payload_bytes,c.payload_sha256); + UPDATE evolution.region_heads x SET commit_sequence=c.commit_sequence,commit_id=c.commit_id,digest=result_digest, + end_tick=c.end_tick,published_epoch=c.epoch WHERE x.world_id=c.world_id AND x.region_id=c.region_id; + END LOOP; + RETURN QUERY SELECT * FROM evolution.cellular_migration_commit_record(id); +END $$; +REVOKE ALL ON FUNCTION evolution.publish_cellular_migration(uuid,bytea,bytea,uuid,evolution.u64,uuid,evolution.u64) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION evolution.publish_cellular_migration(uuid,bytea,bytea,uuid,evolution.u64,uuid,evolution.u64) TO :"runtime_role"; + +CREATE FUNCTION evolution.load_cellular_migration_commit(w evolution.u64,r evolution.u64,holder uuid,epoch evolution.u64,target uuid) +RETURNS TABLE(trace bytea,source_predecessor bytea,destination_predecessor bytea,source_result bytea,destination_result bytea,ghost_sha256 bytea,ghost_bytes evolution.u64) +LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ +DECLARE id uuid; +BEGIN + PERFORM evolution.check_region_lease(w,r,holder,epoch); + SELECT x.request_id INTO id FROM evolution.cellular_migration_commits x + JOIN evolution.cellular_migration_preparations p USING(request_id) + WHERE p.world_id=w AND ((p.source_region=r AND x.source_result=target) OR (p.destination_region=r AND x.destination_result=target)); + IF FOUND THEN RETURN QUERY SELECT * FROM evolution.cellular_migration_commit_record(id); END IF; +END $$; +REVOKE ALL ON FUNCTION evolution.load_cellular_migration_commit(evolution.u64,evolution.u64,uuid,evolution.u64,uuid) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION evolution.load_cellular_migration_commit(evolution.u64,evolution.u64,uuid,evolution.u64,uuid) TO :"runtime_role"; + +ALTER FUNCTION evolution.abort_cellular_migration_freeze(evolution.u64,evolution.u64,uuid,evolution.u64,uuid) RENAME TO abort_cellular_migration_freeze_v48; +REVOKE ALL ON FUNCTION evolution.abort_cellular_migration_freeze_v48(evolution.u64,evolution.u64,uuid,evolution.u64,uuid) FROM :"runtime_role",PUBLIC; +CREATE FUNCTION evolution.abort_cellular_migration_freeze(w evolution.u64,r evolution.u64,holder uuid,epoch evolution.u64,id uuid) +RETURNS TABLE(trace bytea,source_manifest bytea,destination_manifest bytea,aborted boolean) +LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ +BEGIN + PERFORM evolution.check_region_lease(w,r,holder,epoch); + IF EXISTS(SELECT 1 FROM evolution.cellular_migration_commits x JOIN evolution.cellular_migration_preparations p USING(request_id) + WHERE x.request_id=id AND p.world_id=w AND p.source_region=r) THEN + RAISE EXCEPTION 'committed migration cannot abort' USING ERRCODE='EVL07'; + END IF; + RETURN QUERY SELECT * FROM evolution.abort_cellular_migration_freeze_v48(w,r,holder,epoch,id); +END $$; +REVOKE ALL ON FUNCTION evolution.abort_cellular_migration_freeze(evolution.u64,evolution.u64,uuid,evolution.u64,uuid) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION evolution.abort_cellular_migration_freeze(evolution.u64,evolution.u64,uuid,evolution.u64,uuid) TO :"runtime_role"; + +CREATE OR REPLACE FUNCTION evolution.refuse_unpublished_cellular() RETURNS trigger +LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ +BEGIN + IF evolution.manifest_uint(NEW.manifest,12,4)=4 AND NEW.commit_sequence>0 + AND NOT EXISTS(SELECT 1 FROM evolution.cellular_publications x WHERE x.commit_id=NEW.commit_id) + AND NOT EXISTS(SELECT 1 FROM evolution.cellular_profile_migrations x WHERE x.target_commit=NEW.commit_id) + AND NOT EXISTS(SELECT 1 FROM evolution.population_profile_transitions x WHERE x.target_commit=NEW.commit_id AND x.target_profile=4) + AND NOT EXISTS(SELECT 1 FROM evolution.cellular_migration_commits x WHERE x.source_result=NEW.commit_id OR x.destination_result=NEW.commit_id) THEN + RAISE EXCEPTION 'cellular commit requires fenced publisher' USING ERRCODE='EVL06'; + END IF; + RETURN NEW; +END $$; +CREATE OR REPLACE FUNCTION evolution.refuse_unpublished_population() RETURNS trigger +LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ +BEGIN + IF evolution.manifest_uint(NEW.manifest,12,4)=5 AND (NEW.commit_sequence=0 OR + (NOT EXISTS(SELECT 1 FROM evolution.population_publications x WHERE x.commit_id=NEW.commit_id) + AND NOT EXISTS(SELECT 1 FROM evolution.population_profile_transitions x WHERE x.target_commit=NEW.commit_id AND x.target_profile=5) + AND NOT EXISTS(SELECT 1 FROM evolution.cellular_profile_migrations x WHERE x.target_commit=NEW.commit_id) + AND NOT EXISTS(SELECT 1 FROM evolution.cellular_migration_commits x WHERE x.source_result=NEW.commit_id OR x.destination_result=NEW.commit_id))) THEN + RAISE EXCEPTION 'population commit requires fenced publisher' USING ERRCODE='EVL06'; + END IF; + RETURN NEW; +END $$; +INSERT INTO evolution.schema_migrations(version,sha256) VALUES(49,:'migration_sha_49'); diff --git a/migrations/apply.sql b/migrations/apply.sql index d4d866f..f9ef31e 100644 --- a/migrations/apply.sql +++ b/migrations/apply.sql @@ -124,6 +124,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') SELECT :'history_v45' || ',46:' || :'migration_sha_46' AS history_v46 \gset SELECT :'history_v46' || ',47:' || :'migration_sha_47' AS history_v47 \gset SELECT :'history_v47' || ',48:' || :'migration_sha_48' AS history_v48 \gset + SELECT :'history_v48' || ',49:' || :'migration_sha_49' AS history_v49 \gset SELECT :'history' = :'history_v22' AS at_v22, :'history' = :'history_v23' AS at_v23, :'history' = :'history_v24' AS at_v24, @@ -150,8 +151,10 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') :'history' = :'history_v45' AS at_v45, :'history' = :'history_v46' AS at_v46, :'history' = :'history_v47' AS at_v47, - :'history' = :'history_v48' AS at_v48 \gset - \if :at_v48 + :'history' = :'history_v48' AS at_v48, + :'history' = :'history_v49' AS at_v49 \gset + \if :at_v49 + \elif :at_v48 \elif :at_v47 \elif :at_v46 \elif :at_v45 @@ -541,7 +544,8 @@ SELECT coalesce(max(version), 0) = 24 AS at_v24, coalesce(max(version), 0) = 45 AS at_v45, coalesce(max(version), 0) = 46 AS at_v46, coalesce(max(version), 0) = 47 AS at_v47, - coalesce(max(version), 0) = 48 AS at_v48 + coalesce(max(version), 0) = 48 AS at_v48, + coalesce(max(version), 0) = 49 AS at_v49 FROM evolution.schema_migrations \gset \if :at_v24 \ir 0025_cellular_v2_bundle.sql @@ -758,6 +762,7 @@ SELECT coalesce(max(version), 0) = 24 AS at_v24, \elif :at_v46 \elif :at_v47 \elif :at_v48 +\elif :at_v49 \else \echo 'Unknown or changed migration history after v43' DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$; @@ -782,6 +787,10 @@ SELECT coalesce(max(version), 0) = 47 AS needs_v48 FROM evolution.schema_migrati \if :needs_v48 \ir 0048_cellular_migration_staging.sql \endif +SELECT coalesce(max(version), 0) = 48 AS needs_v49 FROM evolution.schema_migrations \gset +\if :needs_v49 + \ir 0049_cellular_migration_publication.sql +\endif -- Effective-privilege audit on every run, including no-ops (ADR-0060). Attributes -- alone are insufficient: predefined roles such as pg_write_all_data, PUBLIC -- grants or SET-able memberships would bypass the fenced functions. Checks the @@ -801,6 +810,8 @@ BEGIN UNION SELECT m.member FROM pg_catalog.pg_auth_members m JOIN closure c ON m.roleid = c.oid ), fenced(oid) AS (VALUES + ('evolution.publish_cellular_migration(uuid,bytea,bytea,uuid,evolution.u64,uuid,evolution.u64)'::regprocedure::oid), + ('evolution.load_cellular_migration_commit(evolution.u64,evolution.u64,uuid,evolution.u64,uuid)'::regprocedure::oid), ('evolution.stage_cellular_migration(evolution.u64,evolution.u64,uuid,evolution.u64,bytea,bytea,evolution.u64)'::regprocedure::oid), ('evolution.load_cellular_migration_stage(evolution.u64,evolution.u64,uuid,evolution.u64)'::regprocedure::oid), ('evolution.prepare_cellular_migration_freeze(evolution.u64,evolution.u64,uuid,evolution.u64,bytea)'::regprocedure::oid), diff --git a/schemas/ecology/cellular-migration-commit-v1.md b/schemas/ecology/cellular-migration-commit-v1.md new file mode 100644 index 0000000..dce9700 --- /dev/null +++ b/schemas/ecology/cellular-migration-commit-v1.md @@ -0,0 +1,56 @@ +# Atomic paired biological migration publication + +Authority: ADR-0149, schema 49. Requires retained +[source preparation](cellular-migration-freeze-v1.md) and +[destination staging](cellular-migration-stage-v1.md). Biological/checkpoint/trace +formats retain their meanings. + +The publisher restores both digest-verified predecessor cuts, replays the exact +retained trace and uploads/reads back both complete candidates. Two same-profile +bundle-10 no-tick manifests each advance sequence once and bind their own +predecessor commit/SHA, world/region/frame, completed tick, quantum and history. +Ordinary batch evidence is zero: migration is neither a biological interval nor +a profile/version conversion. Result IDs come from the retained trace; writer +fields name the respective current endpoint authorities. + +Exact retry input is request UUID plus both canonical manifests. Both CURRENT +lease grants are supplied separately. SQL locks endpoint lease rows in increasing +region order and checks both grants, captured heads, pending pointers, stage and +its destination payload SHA/length. Unstaged, aborted, stale or incompatible +requests fail. Ordinary publishers cannot publish either migration half. + +One transaction inserts immutable `cellular_migration_commits` (preparation/stage +request UUID and unique source/destination result references), clears both pending +pointers, inserts both commits and updates both heads. Deferred result foreign +keys allow evidence guards to remain enabled. Any failure, including the second +insert, rolls back everything. This requires one shared metadata authority, +without implying a cross-database protocol. + +Before commit the source owns all outgoing organisms/material; its staged peer +owns none. Afterwards only the destination owns them. Receipt rows, historical +source observations and cumulative flows are evidence, not spendable stock or +extra census entries. No birth, death or local tick is invented. + +Abort and commit serialize on the source lease row. Abort before commit releases +both pauses and permanently refuses publication. Commit before abort permanently +refuses abort. Committed retries compare both exact manifests and return the +original pair under current fences, even with newer epochs. No old bytes change +and no second transfer occurs. Unknown replies retain the prepared pair and +cannot infer rollback. + +Recovery loads the receipt under current local authority and verifies the trace, +both predecessor/result manifests, adjacent links and staged destination object +reference. Local retained capsules rederive both results. Their full lengths and +SHA-256 must match both manifests; complete local bytes must equal the restored +payload. Missing/corrupt capsules or a valid object with false biology fail +closed, without live-peer or older-head fallback. Retain all evidence; no GC. + +A paused host whose pointers disappear reloads its actual head. A changed head +requires full recovery before readiness or advancement, including after restart. +Pointer absence alone cannot resume the old source or activate an uploaded ghost. +Unknown abort that loses to commit reconciles the actual committed head. + +Independent network partitions, linked geometry/resources, world-wide cuts/census, +target-host costs and actual elapsed-day ecology remain separate gates. Backup/DR +regression retains its supported fixtures and does not qualify cellular history +cloning or a new cellular restore procedure. No automatic migration or deployment. diff --git a/schemas/ecology/cellular-migration-stage-v1.md b/schemas/ecology/cellular-migration-stage-v1.md index 034fa43..80e8622 100644 --- a/schemas/ecology/cellular-migration-stage-v1.md +++ b/schemas/ecology/cellular-migration-stage-v1.md @@ -49,6 +49,7 @@ abort releases both pauses. Exact retries of a retained stage after abort return `aborted` and do not reopen either role. Neither expiry, drain nor timeout implies abort. Keep staged objects/capsules and journals after abort; no GC is authorized. -Atomic paired result publication, its immutable common receipt, commit/abort -races, recovery/activation and linked ecological geometry remain subsequent -work. This segment enables no automatic movement, route, atlas or deployment. +Schema 49/ADR-0149 now adds [atomic paired publication](cellular-migration-commit-v1.md), +its common receipt, exclusive commit/abort outcome and full recovery before host +readiness. Linked ecology and independent network partitions remain separate +gates. No automatic movement, route, atlas or deployment is enabled. diff --git a/src/persistence/cellular_checkpoint_store.cpp b/src/persistence/cellular_checkpoint_store.cpp index d2b3f56..3582a6f 100644 --- a/src/persistence/cellular_checkpoint_store.cpp +++ b/src/persistence/cellular_checkpoint_store.cpp @@ -1,3 +1,4 @@ +#include "evo/persistence/cellular_migration_store.hpp" #include "evo/persistence/checkpoint_store.hpp" #include "evo/persistence/population_checkpoint_store.hpp" #include "evo/persistence/sha256.hpp" @@ -553,6 +554,14 @@ verify_cellular_no_tick_checkpoint(Postgres& db, const FilesystemObjectStore& ob m.frame != committed.head.frame || lease.frame != m.frame || m.sequence == 0 || m.begin_tick != m.end_tick || m.batch != BatchDigests{}) return fail(RecoveryError::manifest); + if (m.cellular_versions == cellular_v10_versions) { + auto paired = + verify_cellular_migration_checkpoint(db, objects, lease, committed, payload, timeout); + if (!paired) + return std::unexpected(paired.error()); + if (*paired) + return {}; + } auto evidence = load_cellular_profile_migration(db, m.key, lease, m.commit, timeout); if (!evidence) return std::unexpected(RecoveryFailure{RecoveryError::store, evidence.error()}); diff --git a/src/persistence/cellular_migration_store.cpp b/src/persistence/cellular_migration_store.cpp index 07eb1be..0e913ae 100644 --- a/src/persistence/cellular_migration_store.cpp +++ b/src/persistence/cellular_migration_store.cpp @@ -3,6 +3,7 @@ #include #include +#include #include namespace evo::persistence { @@ -289,10 +290,14 @@ std::optional checked_stage(const StageRecord& record, R return std::nullopt; return CellularMigrationStage{*freeze, record.ghost}; } -std::optional> -ghost_candidate(const CheckpointManifest& s, const CheckpointManifest& d, - std::span source, std::span destination, - const server::CellularMigrationTrace& trace) { +struct MigrationPayloads { + std::vector source, destination; +}; +std::optional migration_payloads(const CheckpointManifest& s, + const CheckpointManifest& d, + std::span source, + std::span destination, + const server::CellularMigrationTrace& trace) { if (s.profile == CheckpointProfile::cellular_lives_v1) { auto src = restore_cellular_payload(source, s), dst = restore_cellular_payload(destination, d); @@ -301,7 +306,7 @@ ghost_candidate(const CheckpointManifest& s, const CheckpointManifest& d, auto pair = server::replay_migration_trace(**src, **dst, trace.request, trace); if (!pair) return std::nullopt; - return pair->destination->snapshot(); + return MigrationPayloads{pair->source->snapshot(), pair->destination->snapshot()}; } auto src = restore_population_payload(source, s), dst = restore_population_payload(destination, d); @@ -310,7 +315,16 @@ ghost_candidate(const CheckpointManifest& s, const CheckpointManifest& d, auto pair = server::replay_migration_trace(**src, **dst, trace.request, trace); if (!pair) return std::nullopt; - return pair->destination->snapshot(); + return MigrationPayloads{pair->source->snapshot(), pair->destination->snapshot()}; +} +std::optional> +ghost_candidate(const CheckpointManifest& s, const CheckpointManifest& d, + std::span source, std::span destination, + const server::CellularMigrationTrace& trace) { + auto pair = migration_payloads(s, d, source, destination, trace); + if (!pair) + return std::nullopt; + return std::move(pair->destination); } CellularMigrationStageFailure stage_failed(PublicationError kind, LeaseStoreFailure store = {}, ObjectError object = {}, @@ -427,4 +441,212 @@ load_cellular_migration_stage(Postgres& db, const FilesystemObjectStore& objects return fail(RecoveryError::history); return result; } +namespace { +constexpr auto commit_columns = + "encode(trace,'hex'),encode(source_predecessor,'hex')," + "encode(destination_predecessor,'hex'),encode(source_result,'hex')," + "encode(destination_result,'hex'),encode(ghost_sha256,'hex'),ghost_bytes::text"; +std::optional +migration_manifest(const CheckpointManifest& p, const Sha256Digest& digest, const Uuid& commit, + const LeaseGrant& lease, const Sha256Digest& build, VerifiedObject object) { + if (!supported(p) || p.cellular_versions != cellular_v10_versions || p.sequence == 0 || + p.sequence == std::numeric_limits::max() || lease.frame != p.frame) + return std::nullopt; + auto m = p; + m.commit = commit; + ++m.sequence; + m.predecessor_commit = p.commit; + m.predecessor_digest = digest; + m.begin_tick = m.end_tick; + m.holder = lease.holder; + m.epoch = lease.epoch; + m.build = build; + m.payload_bytes = object.bytes; + m.payload_sha256 = object.sha256; + m.batch = {}; + if (!encode_manifest(m)) + return std::nullopt; + return m; +} +std::optional checkpoint(std::vector bytes) { + const auto m = decode_manifest(bytes); + const auto digest = sha256(bytes); + if (!m || !digest) + return std::nullopt; + return CommittedCheckpoint{{m->frame, m->sequence, m->commit, *digest, m->end_tick}, + std::move(bytes)}; +} +struct CommitRecord { + StageRecord stage; + CellularMigrationCommitted result; +}; +std::optional parse_commit(const PgRows::value_type& row) { + if (row.size() != 7) + return std::nullopt; + auto stage = parse_stage({row[0], row[1], row[2], std::string{"false"}, row[5], row[6]}); + auto s = unhex(row[3].value_or(""), checkpoint_manifest_bytes), + d = unhex(row[4].value_or(""), checkpoint_manifest_bytes); + auto source = s ? checkpoint(std::move(*s)) : std::nullopt; + auto destination = d ? checkpoint(std::move(*d)) : std::nullopt; + if (!stage || !source || !destination) + return std::nullopt; + return CommitRecord{std::move(*stage), {std::move(*source), std::move(*destination)}}; +} +bool checked_commit(const CommitRecord& record) { + const auto t = server::decode_migration_trace(record.stage.cut.trace); + const auto s = decode_manifest(record.stage.cut.source), + d = decode_manifest(record.stage.cut.destination); + const auto sr = decode_manifest(record.result.source.manifest), + dr = decode_manifest(record.result.destination.manifest); + const auto sd = sha256(record.stage.cut.source), dd = sha256(record.stage.cut.destination); + if (!t || !s || !d || !sr || !dr || !sd || !dd || !cuts(*s, *d, *sd, *dd, *t) || + record.stage.ghost != VerifiedObject{dr->payload_sha256, dr->payload_bytes}) + return false; + const auto expected_s = migration_manifest(*s, *sd, t->request.source.commit, + {sr->holder, sr->epoch, sr->frame, milliseconds{0}}, + sr->build, {sr->payload_sha256, sr->payload_bytes}); + const auto expected_d = migration_manifest(*d, *dd, t->request.destination.commit, + {dr->holder, dr->epoch, dr->frame, milliseconds{0}}, + dr->build, {dr->payload_sha256, dr->payload_bytes}); + return expected_s && expected_d && *expected_s == *sr && *expected_d == *dr; +} +CellularMigrationCommitFailure commit_failed(PublicationError kind, LeaseStoreFailure store = {}, + ObjectError object = {}, + CellularMigrationCommitPrepared prepared = {}) { + return {kind, store, object, std::move(prepared)}; +} +} // namespace + +std::expected +retry_cellular_migration_commit(Postgres& db, const LeaseGrant& source_lease, + const LeaseGrant& destination_lease, + const CellularMigrationCommitPrepared& prepared, + milliseconds timeout) { + const auto s = decode_manifest(prepared.source_manifest), + d = decode_manifest(prepared.destination_manifest); + if (!s || !d || prepared.request == Uuid{} || s->frame != source_lease.frame || + d->frame != destination_lease.frame) + return std::unexpected(commit_failed(PublicationError::candidate)); + const std::array args{uuid(prepared.request), + "\\x" + hex(prepared.source_manifest), + "\\x" + hex(prepared.destination_manifest), + uuid(source_lease.holder), + std::to_string(source_lease.epoch), + uuid(destination_lease.holder), + std::to_string(destination_lease.epoch)}; + auto rows = db.query( + std::string("SELECT ") + commit_columns + + " FROM evolution.publish_cellular_migration($1::uuid,$2::bytea,$3::bytea,$4::uuid," + "$5::evolution.u64,$6::uuid,$7::evolution.u64)", + args, timeout); + if (!rows) { + const auto store = failure(rows.error()); + return std::unexpected(commit_failed(store.kind == LeaseStoreError::unknown + ? PublicationError::unknown + : PublicationError::rejected, + store, {}, prepared)); + } + auto record = rows->size() == 1 ? parse_commit(rows->front()) : std::nullopt; + if (!record || !checked_commit(*record) || + record->result.source.manifest != prepared.source_manifest || + record->result.destination.manifest != prepared.destination_manifest) + return std::unexpected(commit_failed(PublicationError::unknown, {}, {}, prepared)); + const auto trace = server::decode_migration_trace(record->stage.cut.trace); + if (!trace || trace->request.request != prepared.request) + return std::unexpected(commit_failed(PublicationError::unknown, {}, {}, prepared)); + return std::move(record->result); +} + +std::expected +publish_cellular_migration(Postgres& db, const FilesystemObjectStore& source_objects, + const FilesystemObjectStore& destination_objects, + const LeaseGrant& source_lease, const LeaseGrant& destination_lease, + const CommittedCheckpoint& source, + const CommittedCheckpoint& destination, + const server::CellularMigrationTrace& trace, const Sha256Digest& build, + milliseconds timeout, const PublishHooks& hooks) { + const auto s = decode_manifest(source.manifest), d = decode_manifest(destination.manifest); + if (!s || !d || !bound(source, *s) || !bound(destination, *d) || + !cuts(*s, *d, source.head.digest, destination.head.digest, trace)) + return std::unexpected(commit_failed(PublicationError::predecessor)); + auto src = source_objects.get({s->payload_sha256, s->payload_bytes}), + dst = destination_objects.get({d->payload_sha256, d->payload_bytes}); + if (!src || !dst) + return std::unexpected( + commit_failed(PublicationError::predecessor, {}, !src ? src.error() : dst.error())); + const auto pair = migration_payloads(*s, *d, *src, *dst, trace); + if (!pair) + return std::unexpected(commit_failed(PublicationError::candidate)); + const auto so = source_objects.put(pair->source), + dest = destination_objects.put(pair->destination); + if (!so || !dest) + return std::unexpected( + commit_failed(PublicationError::upload, {}, !so ? so.error() : dest.error())); + const auto sb = source_objects.get(*so), dbb = destination_objects.get(*dest); + if (!sb || !dbb || *sb != pair->source || *dbb != pair->destination) + return std::unexpected(commit_failed(PublicationError::verify)); + const auto sm = migration_manifest(*s, source.head.digest, trace.request.source.commit, + source_lease, build, *so), + dm = + migration_manifest(*d, destination.head.digest, trace.request.destination.commit, + destination_lease, build, *dest); + if (!sm || !dm) + return std::unexpected(commit_failed(PublicationError::candidate)); + const auto sm_bytes = encode_manifest(*sm), dm_bytes = encode_manifest(*dm); + if (!sm_bytes || !dm_bytes) + return std::unexpected(commit_failed(PublicationError::candidate)); + if (hooks.before_transaction) + hooks.before_transaction(); + return retry_cellular_migration_commit(db, source_lease, destination_lease, + {trace.request.request, *sm_bytes, *dm_bytes}, timeout); +} + +std::expected +verify_cellular_migration_checkpoint(Postgres& db, const FilesystemObjectStore& objects, + const LeaseGrant& lease, const CommittedCheckpoint& committed, + std::span payload, milliseconds timeout) { + const auto fail = [](RecoveryError kind) { return std::unexpected(RecoveryFailure{kind, {}}); }; + const auto m = decode_manifest(committed.manifest); + if (!m || !bound(committed, *m) || lease.frame != m->frame) + return fail(RecoveryError::manifest); + const std::array args{std::to_string(m->key.world), + std::to_string(m->key.region), uuid(lease.holder), + std::to_string(lease.epoch), uuid(m->commit)}; + auto rows = db.query( + std::string("SELECT ") + commit_columns + + " FROM evolution.load_cellular_migration_commit($1::evolution.u64,$2::evolution.u64," + "$3::uuid,$4::evolution.u64,$5::uuid)", + args, timeout); + if (!rows) + return std::unexpected(RecoveryFailure{RecoveryError::store, failure(rows.error())}); + if (rows->empty()) + return false; + const auto record = rows->size() == 1 ? parse_commit(rows->front()) : std::nullopt; + if (!record || !checked_commit(*record)) + return fail(RecoveryError::history); + const auto s = decode_manifest(record->stage.cut.source), + d = decode_manifest(record->stage.cut.destination); + const auto sr = decode_manifest(record->result.source.manifest), + dr = decode_manifest(record->result.destination.manifest); + const auto trace = server::decode_migration_trace(record->stage.cut.trace); + if (!s || !d || !sr || !dr || !trace) + return fail(RecoveryError::history); + const bool is_source = m->key == s->key; + if (committed.manifest != + (is_source ? record->result.source.manifest : record->result.destination.manifest)) + return fail(RecoveryError::history); + const auto src = objects.get({s->payload_sha256, s->payload_bytes}), + dst = objects.get({d->payload_sha256, d->payload_bytes}); + if (!src || !dst) + return fail(RecoveryError::tail); + const auto pair = migration_payloads(*s, *d, *src, *dst, *trace); + if (!pair) + return fail(RecoveryError::history); + const auto sd = sha256(pair->source), dd = sha256(pair->destination); + if (!sd || !dd || *sd != sr->payload_sha256 || *dd != dr->payload_sha256 || + pair->source.size() != sr->payload_bytes || pair->destination.size() != dr->payload_bytes || + !std::ranges::equal(is_source ? pair->source : pair->destination, payload)) + return fail(RecoveryError::history); + return true; +} } // namespace evo::persistence diff --git a/src/server/cellular_region_host.cpp b/src/server/cellular_region_host.cpp index 397c4f7..845bc4a 100644 --- a/src/server/cellular_region_host.cpp +++ b/src/server/cellular_region_host.cpp @@ -495,12 +495,15 @@ void CellularRegionHost::reconcile_migration_freeze() { ++status_.unknown_outcomes; return; } - authority_.fence(); - status_.phase = result.error().kind == LeaseStoreError::fenced ? HostPhase::acquiring - : HostPhase::quarantined; - state_.reset(); - committed_.reset(); - return; + if (result.error().kind != LeaseStoreError::commit_conflict) { + authority_.fence(); + status_.phase = result.error().kind == LeaseStoreError::fenced + ? HostPhase::acquiring + : HostPhase::quarantined; + state_.reset(); + committed_.reset(); + return; + } } migration_abort_.reset(); } @@ -534,6 +537,27 @@ void CellularRegionHost::reconcile_migration_freeze() { } migration_frozen_ = staged->has_value(); } + if (!migration_frozen_) { + migration_reload_ = true; + const auto head = + persistence::load_region_checkpoint(*db_, config_.key, config_.lease.commit_timeout); + if (!head) { + db_.reset(); + ++status_.store_failures; + return; + } + if (!committed_ || head->head != committed_->head) { + // Pointer release alone cannot resume the old owner. A paired commit + // changed this head; recover and verify its shared receipt first. + if (!recover_state(*grant) || !committed_) + return; + status_.simulated_tick = committed_->head.end_tick; + status_.committed_tick = committed_->head.end_tick; + status_.committed_sequence = committed_->head.sequence; + interval_reload_ = population_mode(); + mutations_uncertain_ = true; + } + } migration_reload_ = false; } diff --git a/tests/population_publication_tests.cpp b/tests/population_publication_tests.cpp index 5b90b5d..156500b 100644 --- a/tests/population_publication_tests.cpp +++ b/tests/population_publication_tests.cpp @@ -746,6 +746,373 @@ void boundary_upgrades(Fixture& f) { require(f.raw->format_version() == cellular_lives_migration_format, "reverse transition downgraded accounting owner"); } +void migration_publication(Fixture& f, const RegionKey destination_key, + FilesystemObjectStore& destination_objects, + LeaseGrant& destination_lease, CommittedCheckpoint& destination_head) { + const std::array builds{f.build}; + const auto wildlife = std::ranges::find_if( + f.raw->lives(), [](const auto& life) { return life.account == LifeAccount{}; }); + require(wildlife != f.raw->lives().end(), "paired publication needs real wildlife"); + const auto genome = wildlife->identity.genome; + const auto make_host = [&](RegionKey key) { + auto objects = take( + FilesystemObjectStore::open(environment("PGHOST"), key, max_cellular_population_bytes), + "paired host objects"); + CellularHostConfig config{key, 1, {}, f.build, {f.build}}; + config.lease.commit_timeout = 300ms; + return take( + CellularRegionHost::create( + config, + [] { return std::expected, PgFailure>{connection()}; }, + std::move(objects)), + "paired host"); + }; + for (unsigned representation = 0; representation < 2; ++representation) { + if (representation == 0) { + f.head = take(materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), + f.build, 5s), + "paired raw source"); + destination_head = + take(materialize_population_profile(*f.db, destination_objects, destination_lease, + destination_head, f.id(), f.build, 5s), + "paired raw destination"); + } else { + f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), + f.build, {true, 1200, 1800}, 5s), + "paired population source"); + destination_head = take(migrate_cellular_to_population( + *f.db, destination_objects, destination_lease, + destination_head, f.id(), f.build, {true, 1200, 1800}, 5s), + "paired population destination"); + } + for (unsigned direction = 0; direction < 2; ++direction) { + auto& sh = direction == 0 ? f.head : destination_head; + auto& dh = direction == 0 ? destination_head : f.head; + auto& so = direction == 0 ? *f.objects : destination_objects; + auto& dst = direction == 0 ? destination_objects : *f.objects; + auto& sl = direction == 0 ? f.lease : destination_lease; + auto& dl = direction == 0 ? destination_lease : f.lease; + const auto sk = direction == 0 ? f.key : destination_key; + const auto dk = direction == 0 ? destination_key : f.key; + take(release_region_lease(*f.db, sk, sl, 5s), "release paired source to host"); + take(release_region_lease(*f.db, dk, dl, 5s), "release paired destination to host"); + auto source_host = make_host(sk), destination_host = make_host(dk); + require(source_host->step().ready && destination_host->step().ready, + "paired hosts before preparation"); + sl = take(source_host->authority(), "current source authority before pause"); + dl = take(destination_host->authority(), "current destination authority before pause"); + const auto source_before = sh, destination_before = dh; + const auto s = take(decode_manifest(sh.manifest), "paired predecessor source"); + const auto d = take(decode_manifest(dh.manifest), "paired predecessor destination"); + const auto sb = + take(so.get({s.payload_sha256, s.payload_bytes}), "paired source bytes"); + const auto dbb = + take(dst.get({d.payload_sha256, d.payload_bytes}), "paired destination bytes"); + const CellularMigrationRequest request{ + f.id(), + {sk.region, sh.head.commit, f.id(), sh.head.digest}, + {dk.region, dh.head.commit, f.id(), dh.head.digest}, + genome}; + take(source_host->prepare_migration_freeze(dh, dst, request), + "owning paired source freeze"); + const auto frozen = take(take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s), + "paired source evidence"), + "missing paired freeze"); + // Preparation alone cannot authorize either result. + require(!publish_cellular_migration(*f.db, so, dst, sl, dl, sh, dh, frozen.trace, + f.build, 5s), + "unstaged migration published"); + take(destination_host->stage_migration(sh, so, frozen.trace), "owning paired stage"); + std::vector expected_source, expected_destination; + if (representation == 0) { + auto a = take(restore_cellular_payload(sb, s), "paired raw source oracle"); + auto b = take(restore_cellular_payload(dbb, d), "paired raw destination oracle"); + auto pair = take(replay_migration_trace(*a, *b, request, frozen.trace), + "paired raw oracle"); + expected_source = pair.source->snapshot(); + expected_destination = pair.destination->snapshot(); + } else { + auto a = take(restore_population_payload(sb, s), "paired population source oracle"); + auto b = take(restore_population_payload(dbb, d), + "paired population destination oracle"); + auto pair = take(replay_migration_trace(*a, *b, request, frozen.trace), + "paired population oracle"); + expected_source = pair.source->snapshot(); + expected_destination = pair.destination->snapshot(); + } + // Fail specifically on the SECOND result insert, after the first and receipt. + take(f.owner->query( + "CREATE FUNCTION evolution.test_refuse_second_migration() RETURNS trigger " + "LANGUAGE plpgsql AS $$ BEGIN IF NEW.commit_id='" + + hexadecimal(request.destination.commit, false) + + "'::uuid THEN RAISE EXCEPTION 'injected second insert' USING " + "ERRCODE='EVL06'; END IF; RETURN NEW; END $$", + {}, 5s), + "second insert fault function"); + take(f.owner->query( + "CREATE TRIGGER test_refuse_second_migration BEFORE INSERT ON " + "evolution.region_commits " + "FOR EACH ROW EXECUTE FUNCTION evolution.test_refuse_second_migration()", + {}, 5s), + "second insert fault trigger"); + const auto rejected = publish_cellular_migration(*f.db, so, dst, sl, dl, sh, dh, + frozen.trace, f.build, 5s); + take(f.owner->query( + "DROP TRIGGER test_refuse_second_migration ON evolution.region_commits", {}, + 5s), + "remove second insert fault"); + take(f.owner->query("DROP FUNCTION evolution.test_refuse_second_migration()", {}, 5s), + "remove fault function"); + require( + !rejected && rejected.error().store.kind == LeaseStoreError::unsupported && + take(load_region_checkpoint(*f.db, sk, 5s), "rollback source head").head == + source_before.head && + take(load_region_checkpoint(*f.db, dk, 5s), "rollback destination head").head == + destination_before.head && + take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s), + "rollback source pause") + .has_value() && + take(load_cellular_migration_stage(*f.db, dst, dk, dl, 5s), + "rollback destination pause") + .has_value(), + "failed second publication left half a migration or released a pause"); + const auto prepared = rejected.error().prepared; + take(f.db->query("BEGIN", {}, 5s), "false paired biology transaction"); + auto false_pair = prepared; + auto false_source = + take(decode_manifest(false_pair.source_manifest), "false source manifest"); + false_source.payload_sha256 = s.payload_sha256; + false_source.payload_bytes = s.payload_bytes; + false_pair.source_manifest = + take(encode_manifest(false_source), "false source canonical bytes"); + take(retry_cellular_migration_commit(*f.db, sl, dl, false_pair, 5s), + "metadata-only false pair"); + const auto false_recovery = verify_cellular_migration_checkpoint( + *f.db, so, sl, take(load_region_checkpoint(*f.db, sk, 5s), "false paired head"), sb, + 5s); + take(f.db->query("ROLLBACK", {}, 5s), "rollback false paired biology fixture"); + require(!false_recovery && false_recovery.error().kind == RecoveryError::history, + "paired recovery accepted a valid old object as the migrated biology"); + const std::array generic{hexadecimal(prepared.source_manifest)}; + require(!f.db->query("SELECT * FROM evolution.publish_region_checkpoint($1::bytea)", + generic, 5s), + "generic publisher admitted a migration half"); + const auto child = ::fork(); + require(child >= 0, "pre-transaction paired fork"); + if (child == 0) { + try { + auto db = connection(); + PublishHooks hooks{[] { ::kill(::getpid(), SIGKILL); }}; + if (!publish_cellular_migration(*db, so, dst, sl, dl, sh, dh, frozen.trace, + f.build, 5s, hooks)) + ::_exit(52); + } catch (...) { + ::_exit(51); + } + ::_exit(52); + } + int status{}; + require( + ::waitpid(child, &status, 0) == child && WIFSIGNALED(status) && + WTERMSIG(status) == SIGKILL && + take(load_region_checkpoint(*f.db, sk, 5s), "killed pair source").head == + source_before.head && + take(load_region_checkpoint(*f.db, dk, 5s), "killed pair destination").head == + destination_before.head, + "pre-transaction paired death changed a head"); + if (direction == 0) { + auto locker = connection(true), timed = connection(); + take(locker->query("BEGIN", {}, 5s), "pair ambiguity transaction"); + take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND " + "region_id=" + + std::to_string(sk.region) + " FOR UPDATE", + {}, 5s), + "hold paired reply"); + const auto unknown = + retry_cellular_migration_commit(*timed, sl, dl, prepared, 300ms); + require(!unknown && unknown.error().kind == PublicationError::unknown && + unknown.error().prepared.source_manifest == prepared.source_manifest && + unknown.error().prepared.destination_manifest == + prepared.destination_manifest, + "ambiguous pair lost its exact manifests"); + take(locker->query("ROLLBACK", {}, 5s), "release paired ambiguity"); + } else { + const auto committed_child = ::fork(); + require(committed_child >= 0, "post-commit paired fork"); + if (committed_child == 0) { + try { + auto db = connection(); + if (!retry_cellular_migration_commit(*db, sl, dl, prepared, 5s)) + ::_exit(53); + ::kill(::getpid(), SIGKILL); + } catch (...) { + ::_exit(54); + } + ::_exit(55); + } + require(::waitpid(committed_child, &status, 0) == committed_child && + WIFSIGNALED(status) && WTERMSIG(status) == SIGKILL, + "paired publisher failed before post-commit kill"); + } + const auto committed = + take(retry_cellular_migration_commit(*f.db, sl, dl, prepared, 5s), + "resolve exact paired commit"); + sh = committed.source; + dh = committed.destination; + require(!abort_cellular_migration_freeze(*f.db, sk, sl, request.request, 5s), + "committed migration was aborted"); + auto changed = prepared; + auto altered = + take(decode_manifest(changed.destination_manifest), "conflicting committed pair"); + altered.payload_sha256.back() ^= 1; + changed.destination_manifest = take(encode_manifest(altered), "conflicting pair bytes"); + const auto conflict = retry_cellular_migration_commit(*f.db, sl, dl, changed, 5s); + auto stale = sl; + --stale.epoch; + require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict && + !retry_cellular_migration_commit(*f.db, stale, dl, prepared, 5s), + "changed pair or stale source authority was accepted"); + for (unsigned attempt = 0; + attempt < 200 && (!source_host->step().ready || !destination_host->step().ready); + ++attempt) + std::this_thread::sleep_for(25ms); + require(source_host->status().ready && destination_host->status().ready && + take(source_host->committed(), "adopted source").head == sh.head && + take(destination_host->committed(), "adopted destination").head == dh.head, + "pointer release resumed a stale owner instead of recovering the pair"); + const auto check_bytes = [&](const CommittedCheckpoint& h, RegionKey key, + const FilesystemObjectStore& objects, + const LeaseGrant& lease) { + require(take(load_region_checkpoint(*f.db, key, 5s), "current paired recovery head") + .head == h.head, + "paired recovery selected a different head"); + if (representation == 0) + return take(recover_cellular_checkpoint(*f.db, objects, key, lease, builds, 5s), + "recover raw paired result") + .state->snapshot(); + return take(recover_population_checkpoint(*f.db, objects, key, lease, builds, 5s), + "recover cohort paired result") + .state->snapshot(); + }; + require(check_bytes(sh, sk, so, sl) == expected_source && + check_bytes(dh, dk, dst, dl) == expected_destination, + "committed biology/member/material differs from the exact pair"); + const auto capsule_path = so.directory() / hexadecimal(d.payload_sha256, false); + const auto backup = capsule_path.string() + ".paired-test"; + std::filesystem::rename(capsule_path, backup); + const auto missing = + verify_cellular_migration_checkpoint(*f.db, so, sl, sh, expected_source, 5s); + std::filesystem::rename(backup, capsule_path); + require(!missing && missing.error().kind == RecoveryError::tail && + !verify_cellular_migration_checkpoint(*f.db, so, sl, sh, sb, 5s), + "pair recovery ignored missing counterpart evidence or false local biology"); + source_host->drain(); + destination_host->drain(); + source_host = make_host(sk); + destination_host = make_host(dk); + require(source_host->step().ready && destination_host->step().ready, + "replacement paired hosts failed complete recovery"); + sl = take(source_host->authority(), "replacement source fence"); + dl = take(destination_host->authority(), "replacement destination fence"); + require(take(retry_cellular_migration_commit(*f.db, sl, dl, prepared, 5s), + "new epoch exact paired retry") + .source.head == sh.head, + "fresh authority changed a committed retry"); + source_host->drain(); + destination_host->drain(); + sl = take(acquire_region_lease(*f.db, sk, f.build, 60000ms, 5s), + "return paired source authority"); + dl = take(acquire_region_lease(*f.db, dk, f.build, 60000ms, 5s), + "return paired destination authority"); + } + } + // Race the actual two-head transaction against abort on independent connections. + const auto prepare_pair = [&](bool reverse) { + auto& sh = reverse ? destination_head : f.head; + auto& dh = reverse ? f.head : destination_head; + auto& so = reverse ? destination_objects : *f.objects; + auto& dst = reverse ? *f.objects : destination_objects; + auto& sl = reverse ? destination_lease : f.lease; + auto& dl = reverse ? f.lease : destination_lease; + const auto sk = reverse ? destination_key : f.key; + const auto dk = reverse ? f.key : destination_key; + CellularMigrationRequest request{f.id(), + {sk.region, sh.head.commit, f.id(), sh.head.digest}, + {dk.region, dh.head.commit, f.id(), dh.head.digest}, + genome}; + const auto freeze = + take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s), + "commit/abort source preparation"); + take(stage_cellular_migration(*f.db, so, dst, dl, sh, dh, freeze.trace, 5s), + "commit/abort destination stage"); + return freeze.trace; + }; + const auto aborted_trace = prepare_pair(false); + take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, aborted_trace.request.request, 5s), + "abort wins before paired commit"); + const auto after_abort = publish_cellular_migration( + *f.db, *f.objects, destination_objects, f.lease, destination_lease, f.head, + destination_head, aborted_trace, f.build, 5s); + require(!after_abort && after_abort.error().store.kind == LeaseStoreError::commit_conflict, + "paired publication reopened an aborted workflow"); + for (unsigned race = 0; race < 4; ++race) { + const auto trace = prepare_pair(false); + auto committing = connection(), aborting = connection(); + std::barrier start(2); + std::optional> + published; + std::optional> aborted; + std::thread commit_thread([&] { + start.arrive_and_wait(); + published = publish_cellular_migration(*committing, *f.objects, destination_objects, + f.lease, destination_lease, f.head, + destination_head, trace, f.build, 5s); + }); + std::thread abort_thread([&] { + start.arrive_and_wait(); + aborted = abort_cellular_migration_freeze(*aborting, f.key, f.lease, + trace.request.request, 5s); + }); + commit_thread.join(); + abort_thread.join(); + require(published.has_value() && aborted.has_value() && + (published->has_value() != aborted->has_value()), + "commit/abort race had two outcomes or no definitive outcome"); + if (published->has_value()) { + require(aborted->error().kind == LeaseStoreError::commit_conflict, + "losing abort was not refused"); + f.head = (*published)->source; + destination_head = (*published)->destination; + const auto returning = prepare_pair(true); + const auto returned = + take(publish_cellular_migration(*f.db, destination_objects, *f.objects, + destination_lease, f.lease, destination_head, + f.head, returning, f.build, 5s), + "return race winner's stock"); + destination_head = returned.source; + f.head = returned.destination; + } else { + require(published->error().store.kind == LeaseStoreError::commit_conflict, + "losing paired publication was not refused"); + } + require( + take(load_region_checkpoint(*f.db, f.key, 5s), "race source head").head == + f.head.head && + take(load_region_checkpoint(*f.db, destination_key, 5s), "race destination head") + .head == destination_head.head && + !take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s), + "race source settled") + .has_value() && + !take(load_cellular_migration_stage(*f.db, destination_objects, destination_key, + destination_lease, 5s), + "race destination settled") + .has_value(), + "commit/abort race left half a head or a pending pause"); + } + recover(f); + f.raw = restore(f.population->state()); +} + void migration_freezes(Fixture& f) { const RegionKey destination_key{f.key.world, 2}; auto destination_objects = @@ -1426,6 +1793,8 @@ void migration_freezes(Fixture& f) { destination_host->drain(); destination_lease = take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s), "return staged destination fixture authority"); + migration_publication(f, destination_key, *destination_objects, destination_lease, + destination_head); f.head = take( materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s), "resume frozen population as raw owner"); diff --git a/tests/region_migration_tests.cpp b/tests/region_migration_tests.cpp index c877457..58dad0e 100644 --- a/tests/region_migration_tests.cpp +++ b/tests/region_migration_tests.cpp @@ -214,6 +214,7 @@ void upgrade_across_migration_and_barrier() { " -v migration_sha_46=$(s 0046_boundary_accounting_upgrade.sql)" " -v migration_sha_47=$(s 0047_cellular_migration_freeze.sql)" " -v migration_sha_48=$(s 0048_cellular_migration_staging.sql)" + " -v migration_sha_49=$(s 0049_cellular_migration_publication.sql)" " -v runtime_role=evo_runtime_test -f migrations/apply.sql" " > /dev/null 2>&1") != 0); CHECK(history(*owner_db) == "1,2,3,4,5,6,7"); @@ -231,7 +232,7 @@ void upgrade_across_migration_and_barrier() { "tools/migrate-postgres.sh\" > /dev/null") == 0); CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24," - "25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48"); + "25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49"); CHECK(scalar(*owner_db, "SELECT (c.condeferrable AND c.condeferred AND " "c.confrelid = 'evolution.region_commits'::regclass)::text " "FROM pg_catalog.pg_constraint c WHERE c.conrelid = " diff --git a/tools/ci/postgres-check.sh b/tools/ci/postgres-check.sh index 2f0f087..c43d050 100644 --- a/tools/ci/postgres-check.sh +++ b/tools/ci/postgres-check.sh @@ -129,7 +129,7 @@ PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" -f "$root/migrations/0001_region_bootstrap.sql" > /dev/null test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1 PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48 +test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49 # v2 heads have no payload binding: v2->v3 fails closed and leaves history at v2. v2_sha="$(sha256sum "$root/migrations/0002_region_fencing.sql")" v2_sha="${v2_sha%% *}" @@ -150,7 +150,7 @@ PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '01000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32), 'hex'))" > /dev/null PGDATABASE=v3_commits bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 48:1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48 +test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 49:1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49 test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) FROM evolution.region_commits")" = 1 # Review P1: a v4 database holding a published (possibly terminal) checkpoint has # no durable decisions for it. The v5 upgrade must refuse before changing anything. @@ -204,7 +204,7 @@ PGDATABASE=v6_populated psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '02000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32) || repeat('00', 80), 'hex'))" > /dev/null PGDATABASE=v6_populated bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48 +test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT count(*) FROM evolution.region_commits c, LATERAL evolution.parse_checkpoint_manifest(c.manifest) m WHERE m.commit_sequence = 0 AND m.predecessor_commit IS NULL")" = 1 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(conname, ',') FROM pg_constraint WHERE conrelid = 'evolution.region_heads'::regclass AND contype = 'c' AND pg_get_constraintdef(oid) LIKE '%end_tick%'")" = region_heads_published_tick phase permission-audit @@ -257,9 +257,9 @@ expect_audit_failure 'REVOKE EXECUTE ON FUNCTION evolution.claim_account_control bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again. phase checksum-drift # Unknown future versions and checksum drift must not be repaired silently. -psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=49 WHERE version=48' +psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=50 WHERE version=49' if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi -psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=48 WHERE version=49" +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=49 WHERE version=50" psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=14" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=13" diff --git a/tools/migrate-postgres.sh b/tools/migrate-postgres.sh index 202cd78..6b0972d 100644 --- a/tools/migrate-postgres.sh +++ b/tools/migrate-postgres.sh @@ -108,6 +108,8 @@ digest_47="$(sha256sum "$root/migrations/0047_cellular_migration_freeze.sql")" digest_47="${digest_47%% *}" digest_48="$(sha256sum "$root/migrations/0048_cellular_migration_staging.sql")" digest_48="${digest_48%% *}" +digest_49="$(sha256sum "$root/migrations/0049_cellular_migration_publication.sql")" +digest_49="${digest_49%% *}" psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_2=$digest_2" --set="migration_sha_3=$digest_3" \ --set="migration_sha_4=$digest_4" --set="migration_sha_5=$digest_5" \ @@ -149,5 +151,6 @@ psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_46=$digest_46" \ --set="migration_sha_47=$digest_47" \ --set="migration_sha_48=$digest_48" \ + --set="migration_sha_49=$digest_49" \ --set="runtime_role=$runtime_role" \ --file="$root/migrations/apply.sql"