From c625a4cb6871b9220e7f5ba53fb677edc1aac62f Mon Sep 17 00:00:00 2001 From: dingo Date: Sun, 18 Oct 2026 09:08:50 -0400 Subject: [PATCH] M12.4: retain migration source freezes and explicit aborts --- CMakeLists.txt | 2 +- docs/STATUS.md | 17 + .../0147-durable-cellular-migration-freeze.md | 62 +++ docs/adr/README.md | 2 + docs/architecture/PERSISTENCE_ARCHITECTURE.md | 9 + docs/architecture/REGION_DURABILITY.md | 22 +- .../M12_EXACT_BIOLOGICAL_TRANSFER.md | 20 + docs/engineering/M12_REMAINING_GATES.md | 6 + docs/engineering/TEST_STRATEGY.md | 12 + docs/engineering/VALIDATION.md | 54 +++ .../persistence/cellular_migration_store.hpp | 45 +++ include/evo/server/cellular_region_host.hpp | 14 + include/evo/server/ops.hpp | 1 + migrations/0047_cellular_migration_freeze.sql | 207 ++++++++++ migrations/apply.sql | 18 +- schemas/cellular-migration-trace-v1.md | 7 + .../ecology/cellular-migration-freeze-v1.md | 64 +++ src/persistence/cellular_migration_store.cpp | 255 ++++++++++++ src/server/cellular_region_host.cpp | 146 ++++++- src/server/ops.cpp | 2 + tests/ops_tests.cpp | 6 +- tests/population_publication_tests.cpp | 373 ++++++++++++++++++ tests/region_migration_tests.cpp | 3 +- tools/ci/postgres-check.sh | 10 +- tools/migrate-postgres.sh | 3 + 25 files changed, 1342 insertions(+), 18 deletions(-) create mode 100644 docs/adr/0147-durable-cellular-migration-freeze.md create mode 100644 include/evo/persistence/cellular_migration_store.hpp create mode 100644 migrations/0047_cellular_migration_freeze.sql create mode 100644 schemas/ecology/cellular-migration-freeze-v1.md create mode 100644 src/persistence/cellular_migration_store.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 29791fa..b7edf5c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -255,7 +255,7 @@ if(EVO_BUILD_POSTGRES) add_library(evo_region_store STATIC src/persistence/sha256.cpp src/persistence/object_store.cpp src/persistence/checkpoint_store.cpp src/persistence/primitive_checkpoint_store.cpp src/persistence/cellular_checkpoint_store.cpp src/persistence/population_checkpoint_store.cpp - src/persistence/region_clone.cpp) + src/persistence/region_clone.cpp src/persistence/cellular_migration_store.cpp) add_library(evo::region_store ALIAS evo_region_store) target_link_libraries(evo_region_store PUBLIC evo::postgres evo::region_checkpoint PRIVATE OpenSSL::Crypto) diff --git a/docs/STATUS.md b/docs/STATUS.md index 12643bb..f3770e5 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1,5 +1,22 @@ # Project status and next steps +## M12.4h durable source freeze and explicit abort - 2026-10-18 + +ADR-0147/schema 47 retains exact biological migration preparation, replay capsules +and a durable source freeze across worker replacement. Source publication and +new interval/player reservations are refused while pending; source stock and both +heads remain unchanged. Explicit current-source abort appends its outcome and +clears the pause, with exact retries. The host pauses readiness/advancement, +retains ambiguous requests and verifies complete biology before resuming. + +Native Release passes 78/78; focused private PostgreSQL, ASan/UBSan, Clang +analysis, format and shell syntax checks pass. The complete private persistence, +process-fault, backup/DR and migration/permission suite passes in 502 s. +[Evidence and limits](engineering/VALIDATION.md) +retain destination staging, atomic paired commit, commit/abort races, activation +and linked-boundary contracts as subsequent M12.4 work. All of M12 remains in +progress; no deployment or live region changed. + ## M12.4g durable accounting upgrade - 2026-10-18 ADR-0146/schema 46 extends the immutable version-upgrade journal to same-profile diff --git a/docs/adr/0147-durable-cellular-migration-freeze.md b/docs/adr/0147-durable-cellular-migration-freeze.md new file mode 100644 index 0000000..3ed1738 --- /dev/null +++ b/docs/adr/0147-durable-cellular-migration-freeze.md @@ -0,0 +1,62 @@ +# ADR-0147: Durable biological migration preparation and source freeze + +## Status + +Accepted and qualified for scoped M12.4h implementation, 2026-10-18. Executed +evidence and limits are in [VALIDATION](../engineering/VALIDATION.md). +Destination staging, atomic paired publication and activation +remain subsequent segments. No deployment or automatic migration is selected. + +## Decision + +Prepare ADR-0144's exact raw/population pair against two digest-verified bundle-10 +committed cuts, with the same representation, world/frame/tick and compatible +biology/geometry. Retain canonical migration trace 1 under its stable request +UUID. The source's current lease and exact head fence preparation. The destination +head is checked optimistically during preparation; it is neither locked nor +activated. This does not establish a simultaneous global cut at freeze commit. + +Before preparation, copy and verify the destination predecessor payload into the +source's immutable object namespace. This is a read-only replay capsule, not an +owned population or additional spendable stock. Both manifests stay retained in +the shared commit journal. Recovery can verify the entire recorded pair from the +source namespace even if the destination later advances or is unavailable. + +Schema 47 records immutable preparations and explicit source-authorized aborts, +plus one bounded pending pointer per source region. A prepared source retains all +its cells, inventory and history at its existing head. Coarsely freeze the whole +region: refuse new checkpoints, profile/version changes, interval admissions and +player mutation reservations until settlement. Preparation refuses pending +intervals and outstanding player choices; C++ requires fully suspended controlled +cells and an eligible complete wildlife group. No biological tick or transfer +occurs. Destination ghosts own nothing; this segment creates none. + +The owning host pauses readiness and advancement during preparation uncertainty +or a recovered freeze. It retries the exact retained request, renews authority, +and restores/validates the capsule before resuming any work. Replacing a worker +or expiring a lease never releases the freeze. Explicit abort under current source +authority appends its immutable outcome and removes the pending pointer in one +transaction, leaving both biological heads unchanged. Exact preparation/abort +retries retain their outcome; retrying an aborted preparation never reopens it. +Timeout is not an abort. A replacement owner can explicitly abort without needing +destination authority, since no ownership has transferred. + +Subsequent destination staging must recheck its exact captured head. A moved +destination requires an explicit source abort and a fresh workflow; do not rebase +the old trace silently. Future two-head commit must settle the source pointer +atomically with both exact results and immutable shared evidence. This segment +admits no migration result manifest and no destination activation. + +## Alternatives, costs and qualification + +An in-memory pause disappears after process death. Lease expiry is not an abort +decision. Copying cells into an active destination before paired commit duplicates +ownership. Keep the source authoritative and durably frozen instead. A whole +region pause is deliberately coarse; it is not a final ecological movement or +latency policy. Source-side capsule retention adds bounded storage and replay work. + +Require real player-origin wildlife, both representations, complete replay and +stock equality, pending-work refusal, publication/admission bypass rejection, +duplicate/conflicting preparation, explicit abort, restart/new-epoch recovery, +ambiguous replies, process loss, missing/corrupt capsules and host readiness. +No elapsed-day, atlas, environmental coupling or multigeneration claim follows. diff --git a/docs/adr/README.md b/docs/adr/README.md index 3199737..e2fbdfd 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -1,5 +1,7 @@ # Architecture decision records +- [ADR-0147: Durable biological preparation and source freeze](0147-durable-cellular-migration-freeze.md) - schema 47 retains exact preparation/capsules and explicit abort; the source stays authoritative and paused across worker replacement. Destination staging and paired activation remain open. + - [ADR-0146: Durable boundary-accounting upgrade admission](0146-durable-boundary-accounting-upgrade.md) - schema 46 extends immutable version-upgrade evidence to same-profile bundle 9->10, full upgrade replay, population batch 2 and preserved representation/interval guards; paired migration escrow remains open. - [ADR-0145: Explicit boundary-accounting checkpoint upgrade](0145-boundary-accounting-checkpoint-upgrade.md) - portable bundle 10, exact no-tick retained-source upgrade and representation-preserving continuation; SQL admission and durable upgrade evidence remain open. diff --git a/docs/architecture/PERSISTENCE_ARCHITECTURE.md b/docs/architecture/PERSISTENCE_ARCHITECTURE.md index ef75f29..af92684 100644 --- a/docs/architecture/PERSISTENCE_ARCHITECTURE.md +++ b/docs/architecture/PERSISTENCE_ARCHITECTURE.md @@ -10,6 +10,15 @@ upgrade automatically. Atomic paired transfer publication remains required. See [ADR-0145](../adr/0145-boundary-accounting-checkpoint-upgrade.md) and [ADR-0146](../adr/0146-durable-boundary-accounting-upgrade.md). +M12.4h/schema 47 retains source migration preparation, immutable explicit abort +and a bounded pending freeze across worker replacement. Complete recovery replay +uses both retained manifests and a verified destination capsule in the source's +object namespace. Both biological heads stay unchanged; destination inspection +is optimistic and grants no destination authority. Source publication and new +interval/player reservations are blocked until explicit settlement. See +[ADR-0147](../adr/0147-durable-cellular-migration-freeze.md). Destination staging, +atomic paired result publication and activation remain required. + M7.1 now scopes the first implementation in [ADR-0058](../adr/0058-region-durability-contract.md) and [REGION_DURABILITY.md](REGION_DURABILITY.md): full checkpoint batches, not recovery from semantic events or an unimplemented input journal. Candidate uploads diff --git a/docs/architecture/REGION_DURABILITY.md b/docs/architecture/REGION_DURABILITY.md index 779f00a..65d4d80 100644 --- a/docs/architecture/REGION_DURABILITY.md +++ b/docs/architecture/REGION_DURABILITY.md @@ -49,17 +49,29 @@ readiness. New work requires fully suspended controlled cells, leaving vulnerabl linger/death on profile 4. Earlier trace semantics remain readable. See the [interval contract](../../schemas/ecology/population-interval-v1.md). -ADR-0144 adds a portable exact transfer pair with explicit regional boundary -flows/nonresident history (owner 7, population envelope 2). These versions have -no admitted durable manifest bundle yet. Existing profile-4/5 publishers retain -their tuples; population trace 1 refuses envelope 2. A future adapter must freeze -the source and atomically commit both fenced heads before destination activation. +ADR-0144 adds an exact transfer pair with explicit regional boundary flows and +nonresident history (owner 7, population envelope 2). ADR-0145/0146 and schema 46 +admit bundle 10 through an explicit same-profile version upgrade. Population trace +1 retains bundle 9/envelope 1; trace 2 requires bundle 10/envelope 2. No worker +upgrades automatically. A paired publisher must commit both fenced heads before +destination activation. See [scope and remaining proof](../engineering/M12_EXACT_BIOLOGICAL_TRANSFER.md). Its bounded [paired migration trace](../../schemas/cellular-migration-trace-v1.md) replays from both source cuts and an independently trusted request, including complete cohort tags. Independent-process replay is qualified; no existing durable publisher admits that trace or treats it as a transfer receipt. +ADR-0147/schema 47 retains that trace as source preparation evidence, without +admitting either migration result. The source stays authoritative at its exact +head; a durable pending pointer blocks its new publications/admissions and +survives lease expiry or replacement. Explicit fenced abort appends its outcome +and releases the pointer with both biological heads unchanged. A verified +destination predecessor capsule in the source namespace allows complete pair +replay without consulting destination live state. The owning host pauses +readiness, retains unknown requests and never treats timeout/drain as abort. +See [the freeze contract](../../schemas/ecology/cellular-migration-freeze-v1.md). +Destination staging, paired commit/abort races and activation remain open. + World/region/frame are nonzero durable IDs. No ID is a Pod name. The host records its unique holder ID and monotonically increasing nonzero lease epoch. Exhaustion of IDs, epochs, commit sequences or ticks fails closed; never wrap/reuse them. diff --git a/docs/engineering/M12_EXACT_BIOLOGICAL_TRANSFER.md b/docs/engineering/M12_EXACT_BIOLOGICAL_TRANSFER.md index 64ed4d7..0219549 100644 --- a/docs/engineering/M12_EXACT_BIOLOGICAL_TRANSFER.md +++ b/docs/engineering/M12_EXACT_BIOLOGICAL_TRANSFER.md @@ -120,3 +120,23 @@ This admits the accounting format and its ordinary continuation. Paired migratio trace 1 still has no SQL publisher, source freeze, staged ghost or activation rights. Durable two-head escrow, abort/partition/duplicate settlement, linked geometry and ecology remain open. No deployment, live upgrade or golden changed. + +## M12.4h durable source preparation and abort + +ADR-0147/schema 47 retains canonical paired preparation under the source's current +lease and exact head. A verified destination predecessor capsule in the source +namespace supports complete recovery replay. The source keeps all stock at its +unchanged head; the destination is neither staged nor activated. New source +checkpoints/profile changes and interval/player reservations are refused while +pending. A replacement owner recovers the same freeze, without a timeout-based +release. Explicit current-source abort appends its immutable outcome and removes +the pending pointer with both biological heads unchanged. Retrying an aborted +preparation never reopens it. + +The owning host withholds readiness and advancement while frozen or uncertain, +retains exact requests, renews authority and polls paused work once per second. +A fixed migration-freeze gauge identifies known pauses. Missing/corrupt capsules +and canonically encoded false member/material declarations fail complete replay. +See [the contract](../../schemas/ecology/cellular-migration-freeze-v1.md). +Destination staging, atomic paired head publication, commit/abort races and +activation remain the next segment; no route or automatic migration is enabled. diff --git a/docs/engineering/M12_REMAINING_GATES.md b/docs/engineering/M12_REMAINING_GATES.md index 9cb4dae..f3515c4 100644 --- a/docs/engineering/M12_REMAINING_GATES.md +++ b/docs/engineering/M12_REMAINING_GATES.md @@ -61,6 +61,12 @@ version-upgrade evidence and complete replayed recovery. Both profile upgrades, population batch 2 and bundle-preserving profile changes are explicit; no worker upgrades automatically. This format admission does not publish a migration pair or close source-freeze/destination-activation and escrow qualification. +ADR-0147/schema 47 now adds durable source preparation/freeze, retained replay +capsules, explicit abort and host pause/recovery. Source stock remains owned at +its exact head through worker replacement; no timeout or drain releases it. +Destination staging, atomic paired commit, commit/abort races and activation +remain open. The destination head is only checked optimistically at preparation +and must be rechecked before any future stage/commit. The transfer tests also expose the current rules-8 maturation gap: buds retain their subadult realization while reproduction requires a fully realized parent. Finite extra food alone does not close repeated wildlife generations. A versioned diff --git a/docs/engineering/TEST_STRATEGY.md b/docs/engineering/TEST_STRATEGY.md index 75755e6..eb12f0d 100644 --- a/docs/engineering/TEST_STRATEGY.md +++ b/docs/engineering/TEST_STRATEGY.md @@ -20,6 +20,18 @@ Batch-1/2 cross-binding is refused in both directions. The process-loss and host-recreation/materialization matrix also runs with batch 2. Full historical migration/backup/DR/permission regression remains required for schema admission. +M12.4h's private freeze matrix uses funded player-derived wildlife and both +bundle-10 representations. It requires unchanged source/destination heads and +stock, exact request retries, conflicting/new-work refusal, pending-choice/ +interval guards, new-epoch recovery, explicit abort without reopening, and real +SIGKILL before/after preparation. Unknown preparation/abort replies pause the +owning host through exact reconciliation; drain/recreation retains the freeze. +Missing and same-length corrupt capsules fail, and canonical false material +fails full biological replay. SQL checksum rejection is checked independently of +the C++ decoder. The metric has one fixed unlabelled series and remains bounded. +These are preparation/abort checks, not proof of destination activation or atomic +paired publication; full historical persistence regression is also required. + Tests defend contracts, not directory coverage. All checks remain active in Release builds. Current proof has no actual network, biology or distributed recovery, so those categories below are milestone gates rather than passing diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index cb4fe37..254482a 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -1,5 +1,59 @@ # Validation report +## 2026-10-18 - M12.4h durable migration source freeze and explicit abort + +ADR-0147/schema 47 retains exact source-authorized preparation, both predecessor +manifests and a verified destination payload capsule in the source namespace. +Source stock remains at its unchanged head. Immutable explicit abort removes the +pending pointer without transferring ownership or advancing either head. The +owning host pauses readiness/advancement across uncertainty and replacement, +renews authority and verifies the complete retained biological pair. Destination +staging and paired result publication are subsequent work. + +Executed locally: + +- Native MSVC Release **78/78 passed**, 14.66 s. +- Actual GCC 14 optional persistence/server targets rebuilt; final private + PostgreSQL population/freeze suite passed, 9 s total (6 s test phase). +- Linux Clang 19 ASan/UBSan final private population/freeze suite passed, + **30 s** total (28 s test phase), with no sanitizer reports. +- Explicit clang-tidy 19 on the new persistence adapter, owning host and + PostgreSQL population tests passed with no owned warnings. Repository + clang-format 19 and both changed shell scripts' syntax checks passed. +- Complete frozen-binary private PostgreSQL qualification passed with **exit 0**, + **502 s**: C++ integration/process/load/history (424 s), life reservations, + backup/restore, host-loss DR, upgrade, migration history/rollback, runtime + permission audit and checksum/future-version rejection. The detached runner + recorded its completion code; no interrupted run is counted as a pass. + +Both raw and population representations exercise actual player-origin wildlife, +complete capsule replay, unchanged heads/material, duplicate/conflicting +requests, pending-work refusal and publication/admission bypass guards. Replacement +epochs retain freezes; stale aborts fail, explicit abort is idempotent and aborted +preparation retries never reopen a pointer. Real lock-timeout ambiguity retains +exact requests. Process kills before preparation and after commit distinguish +absence from a retained freeze. Missing and same-length corrupt capsules fail +closed even while the original destination object exists; canonical false +inventory declarations fail full biological replay. The owning-host matrix +covers restart/drain, ambiguous prepare/abort and resumption of identical biology. +The fixed unlabelled freeze metric is included in existing cardinality bounds. + +Initial compile/analysis issues (qualified lease-clock type, optional handling) +were corrected. The first new reservation fixture used an obsolete account +control generation and reached the existing epoch fence first. The successful +fixture claims current control inside a rolled-back transaction and reaches the +new freeze guard without weakening either fence. + +Limits: source preparation checks the destination head optimistically; it is not +a simultaneous world cut. There is no destination ghost, paired commit, +activation, linked geometry, elapsed-day or multigeneration qualification. Backup +inspection confirms all immutable namespace objects, including capsules, are +copied; existing backup/DR fixtures do not qualify cellular history cloning or a +new cellular restore procedure. No live database, World0, deployment, dependency, +biological rule or golden changed. The supplied `Build_Errors.txt` still contains +the earlier captured CI failures; remote confirmation of their committed runner +repair remains separate. That file and unrelated local work remain untracked. + ## 2026-10-18 - M12.4g durable boundary-accounting upgrade ADR-0146/schema 46 admits the explicit same-profile bundle-9 to 10 upgrade through diff --git a/include/evo/persistence/cellular_migration_store.hpp b/include/evo/persistence/cellular_migration_store.hpp new file mode 100644 index 0000000..51bfca3 --- /dev/null +++ b/include/evo/persistence/cellular_migration_store.hpp @@ -0,0 +1,45 @@ +#pragma once + +#include "evo/persistence/checkpoint_store.hpp" +#include "evo/server/lifecycle/cellular_migration_trace.hpp" + +namespace evo::persistence { +struct CellularMigrationPrepared { + RegionKey key{}; // Source region. + std::vector trace; +}; +struct CellularMigrationFreeze { + server::CellularMigrationTrace trace; + bool aborted{}; +}; +struct CellularMigrationFailure { + PublicationError kind{}; + LeaseStoreFailure store{}; + ObjectError object{}; + CellularMigrationPrepared prepared; // Exact request after unknown/rejection. +}; +// Verify both complete cuts and the paired biology, retain a verified destination +// replay capsule in the source namespace, then freeze the exact source head. +// Neither candidate is published or activated by this operation. +[[nodiscard]] std::expected +prepare_cellular_migration_freeze(Postgres&, const FilesystemObjectStore& source_objects, + const FilesystemObjectStore& destination_objects, + const LeaseGrant& source_lease, const CommittedCheckpoint& source, + const CommittedCheckpoint& destination, + const server::CellularMigrationRequest&, + std::chrono::milliseconds timeout, + const PublishHooks& hooks = {}); +[[nodiscard]] std::expected +retry_cellular_migration_freeze(Postgres&, const LeaseGrant&, const CellularMigrationPrepared&, + std::chrono::milliseconds timeout); +// Current source authority only. No timeout/lease-expiry auto-abort. The result +// repeats a retained abort and cannot reopen an old preparation. +[[nodiscard]] std::expected +abort_cellular_migration_freeze(Postgres&, RegionKey, const LeaseGrant&, const Uuid& request, + std::chrono::milliseconds timeout); +// Fenced lookup AND complete capsule replay. An empty result means no freeze; +// missing/corrupt evidence is an error, never implicit permission to advance. +[[nodiscard]] std::expected, RecoveryFailure> +load_cellular_migration_freeze(Postgres&, const FilesystemObjectStore&, RegionKey, + const LeaseGrant&, std::chrono::milliseconds timeout); +} // namespace evo::persistence diff --git a/include/evo/server/cellular_region_host.hpp b/include/evo/server/cellular_region_host.hpp index ed92c33..2c33de3 100644 --- a/include/evo/server/cellular_region_host.hpp +++ b/include/evo/server/cellular_region_host.hpp @@ -1,6 +1,7 @@ #pragma once #include "evo/network/session_protocol.hpp" +#include "evo/persistence/cellular_migration_store.hpp" #include "evo/persistence/checkpoint_store.hpp" #include "evo/persistence/life_reservation_store.hpp" #include "evo/persistence/population_checkpoint_store.hpp" @@ -91,6 +92,14 @@ class CellularRegionHost { // Acquires/renews authority, recovers fully, or resolves the exact pending commit. const HostStatus& step(); + // Explicit source-only preparation/abort. Preparation pauses the whole region; + // restart and lease expiry never imply abort or destination activation. + [[nodiscard]] std::expected + prepare_migration_freeze(const persistence::CommittedCheckpoint& destination, + const persistence::FilesystemObjectStore& destination_objects, + const CellularMigrationRequest&); + [[nodiscard]] std::expected + abort_migration_freeze(const persistence::Uuid& request); // Explicit no-tick fenced transition. Uncertain publication pauses the host. [[nodiscard]] std::expected set_population_mode(bool enabled); [[nodiscard]] bool population_mode() const noexcept; @@ -154,6 +163,7 @@ class CellularRegionHost { void refresh(); bool recover_state(const persistence::LeaseGrant&); void reconcile_interval(); + void reconcile_migration_freeze(); void finish_interval(); [[nodiscard]] std::expected publish_population(const CellularBatch&); @@ -215,6 +225,10 @@ class CellularRegionHost { std::expected, OwnedState candidate); OwnedState state_; + bool migration_reload_{}, migration_frozen_{}; + persistence::LeaseClock::time_point migration_poll_at_{}; + std::optional migration_request_; + std::optional migration_abort_; bool interval_reload_{}; std::optional admitted_interval_; // An uncertain admission must retry its exact source/target, never select again. diff --git a/include/evo/server/ops.hpp b/include/evo/server/ops.hpp index d73f2e4..b6399ed 100644 --- a/include/evo/server/ops.hpp +++ b/include/evo/server/ops.hpp @@ -38,6 +38,7 @@ struct HostStatus { std::uint64_t commits{}, store_failures{}, unknown_outcomes{}, fences{}; // Local lease safety: time until this host must stop originating (0 = none). std::uint64_t lease_remaining_ms{}; + bool migration_frozen{}; // Known durable source freeze; expiry is not an abort. }; // Process-level counters and bounds the exposition adds to the host status. diff --git a/migrations/0047_cellular_migration_freeze.sql b/migrations/0047_cellular_migration_freeze.sql new file mode 100644 index 0000000..88b1527 --- /dev/null +++ b/migrations/0047_cellular_migration_freeze.sql @@ -0,0 +1,207 @@ +-- ADR-0147: retain exact biological preparation; source owns all stock until paired commit. +CREATE TABLE evolution.cellular_migration_preparations ( + request_id uuid PRIMARY KEY, + world_id evolution.u64 NOT NULL, + source_region evolution.u64 NOT NULL, + destination_region evolution.u64 NOT NULL CHECK (destination_region <> source_region), + source_commit uuid NOT NULL REFERENCES evolution.region_commits(commit_id), + destination_commit uuid NOT NULL REFERENCES evolution.region_commits(commit_id), + trace bytea NOT NULL CHECK (octet_length(trace) BETWEEN 272 AND 2312), + prepared_holder uuid NOT NULL, + prepared_epoch evolution.u64 NOT NULL CHECK (prepared_epoch > 0), + FOREIGN KEY(world_id,source_region) REFERENCES evolution.region_leases, + FOREIGN KEY(world_id,destination_region) REFERENCES evolution.region_leases +); +CREATE TABLE evolution.cellular_migration_aborts ( + request_id uuid PRIMARY KEY REFERENCES evolution.cellular_migration_preparations, + holder uuid NOT NULL, epoch evolution.u64 NOT NULL CHECK (epoch > 0) +); +CREATE TABLE evolution.cellular_migration_pending ( + world_id evolution.u64 NOT NULL, region_id evolution.u64 NOT NULL, + request_id uuid UNIQUE NOT NULL REFERENCES evolution.cellular_migration_preparations, + PRIMARY KEY(world_id,region_id) +); +REVOKE ALL ON evolution.cellular_migration_preparations,evolution.cellular_migration_aborts, + evolution.cellular_migration_pending FROM PUBLIC; +CREATE TRIGGER cellular_migration_preparations_immutable BEFORE UPDATE OR DELETE ON evolution.cellular_migration_preparations + FOR EACH ROW EXECUTE FUNCTION evolution.refuse_history_rewrite(); +CREATE TRIGGER cellular_migration_preparations_no_truncate BEFORE TRUNCATE ON evolution.cellular_migration_preparations + FOR EACH STATEMENT EXECUTE FUNCTION evolution.refuse_history_rewrite(); +CREATE TRIGGER cellular_migration_aborts_immutable BEFORE UPDATE OR DELETE ON evolution.cellular_migration_aborts + FOR EACH ROW EXECUTE FUNCTION evolution.refuse_history_rewrite(); +CREATE TRIGGER cellular_migration_aborts_no_truncate BEFORE TRUNCATE ON evolution.cellular_migration_aborts + FOR EACH STATEMENT EXECUTE FUNCTION evolution.refuse_history_rewrite(); + +CREATE FUNCTION evolution.check_cellular_migration_trace(t bytea) RETURNS void +LANGUAGE plpgsql IMMUTABLE SET search_path = pg_catalog, pg_temp AS $$ +DECLARE n integer; members numeric; previous numeric := 0; life numeric; + hash numeric := 14695981039346656037; ids bytea[]; +BEGIN + IF t IS NULL OR octet_length(t) < 272 OR octet_length(t) > 2312 THEN + RAISE EXCEPTION 'invalid migration trace length' USING ERRCODE = '22023'; + END IF; + n := octet_length(t); members := evolution.manifest_uint(t,248,4); + ids := ARRAY[substring(t FROM 73 FOR 16),substring(t FROM 121 FOR 16), + substring(t FROM 145 FOR 16),substring(t FROM 193 FOR 16)]; + IF substring(t FROM 1 FOR 8) <> '\x4145564d49473031'::bytea OR evolution.manifest_uint(t,8,4) <> 1 + OR evolution.manifest_uint(t,12,4) NOT IN (1,2) OR evolution.manifest_uint(t,16,8) = 0 + OR evolution.manifest_uint(t,24,8) = 0 OR evolution.manifest_uint(t,32,8) = 0 + OR evolution.manifest_uint(t,40,8) = 0 OR substring(t FROM 49 FOR 16) = decode(repeat('00',16),'hex') + OR evolution.manifest_uint(t,64,8) NOT BETWEEN 1 AND 2147483647 + OR evolution.manifest_uint(t,136,8) NOT BETWEEN 1 AND 2147483647 + OR evolution.manifest_uint(t,64,8) = evolution.manifest_uint(t,136,8) + OR substring(t FROM 89 FOR 32) = decode(repeat('00',32),'hex') + OR substring(t FROM 161 FOR 32) = decode(repeat('00',32),'hex') + OR substring(t FROM 209 FOR 40) = decode(repeat('00',40),'hex') + OR members NOT BETWEEN 1 AND 256 OR n <> 264 + 8 * members + OR evolution.manifest_uint(t,252,4) <> 0 + OR (SELECT count(DISTINCT x) FROM unnest(ids) x) <> 4 + OR decode(repeat('00',16),'hex') = ANY(ids) THEN + RAISE EXCEPTION 'invalid migration trace shape' USING ERRCODE = '22023'; + END IF; + FOR i IN 0..members::integer-1 LOOP + life := evolution.manifest_uint(t,256+8*i,8); + IF life <= previous OR life < 9223372036854775808 + OR trunc((life-9223372036854775808)/4294967296) = 0 OR mod(life,4294967296) = 0 THEN + RAISE EXCEPTION 'invalid migration member' USING ERRCODE = '22023'; + END IF; + previous := life; + END LOOP; + -- Exact bounded FNV-1a64, corruption detection only; manifests/payloads use SHA-256. + FOR i IN 0..n-9 LOOP + hash := mod((hash-mod(hash,256)+(mod(hash,256)::integer # get_byte(t,i)))*1099511628211, + 18446744073709551616); + END LOOP; + IF hash <> evolution.manifest_uint(t,n-8,8) THEN + RAISE EXCEPTION 'migration trace checksum differs' USING ERRCODE = '22023'; + END IF; +END $$; +REVOKE ALL ON FUNCTION evolution.check_cellular_migration_trace(bytea) FROM PUBLIC; + +CREATE FUNCTION evolution.cellular_migration_record(w evolution.u64,r evolution.u64,id uuid) +RETURNS TABLE(trace bytea,source_manifest bytea,destination_manifest bytea,aborted boolean) +LANGUAGE sql STABLE SET search_path = pg_catalog, pg_temp AS $$ + SELECT p.trace,s.manifest,d.manifest,EXISTS(SELECT 1 FROM evolution.cellular_migration_aborts a WHERE a.request_id=p.request_id) + FROM evolution.cellular_migration_preparations p + JOIN evolution.region_commits s ON s.commit_id=p.source_commit + JOIN evolution.region_commits d ON d.commit_id=p.destination_commit + WHERE p.world_id=w AND p.source_region=r AND p.request_id=id +$$; +REVOKE ALL ON FUNCTION evolution.cellular_migration_record(evolution.u64,evolution.u64,uuid) FROM PUBLIC; + +CREATE FUNCTION evolution.prepare_cellular_migration_freeze(w evolution.u64,r evolution.u64,holder uuid,epoch evolution.u64,t bytea) +RETURNS TABLE(trace bytea,source_manifest bytea,destination_manifest bytea,aborted boolean) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE id uuid; src uuid; dst uuid; destination evolution.u64; h evolution.region_heads%ROWTYPE; + sm bytea; dm bytea; s evolution.checkpoint_manifest; d evolution.checkpoint_manifest; + old evolution.cellular_migration_preparations%ROWTYPE; + bundle10 constant bytea := '\x080000000700000002000000010000000100000004000000030000000200000001000000'; +BEGIN + PERFORM evolution.check_cellular_migration_trace(t); + IF evolution.manifest_uint(t,16,8) <> w OR evolution.manifest_uint(t,64,8) <> r THEN + RAISE EXCEPTION 'foreign migration source' USING ERRCODE = '22023'; + END IF; + id := encode(substring(t FROM 49 FOR 16),'hex')::uuid; + src := encode(substring(t FROM 73 FOR 16),'hex')::uuid; + dst := encode(substring(t FROM 145 FOR 16),'hex')::uuid; + destination := evolution.manifest_uint(t,136,8); + PERFORM evolution.check_region_lease(w,r,holder,epoch); + SELECT * INTO old FROM evolution.cellular_migration_preparations p WHERE p.request_id=id; + IF FOUND THEN + IF old.world_id<>w OR old.source_region<>r OR old.trace<>t THEN + RAISE EXCEPTION 'migration request reused' USING ERRCODE = 'EVL07'; + END IF; + RETURN QUERY SELECT * FROM evolution.cellular_migration_record(w,r,id); + RETURN; + END IF; + SELECT * INTO h FROM evolution.region_heads x WHERE x.world_id=w AND x.region_id=r FOR UPDATE; + IF NOT FOUND OR h.commit_id<>src OR h.digest<>substring(t FROM 89 FOR 32) + OR h.end_tick<>evolution.manifest_uint(t,32,8) THEN + RAISE EXCEPTION 'migration source moved' USING ERRCODE = 'EVL03'; + END IF; + SELECT c.manifest INTO sm FROM evolution.region_commits c WHERE c.commit_id=src; + -- Destination remains unowned by this workflow. A later stage must recheck its head. + SELECT c.manifest INTO dm FROM evolution.region_heads x JOIN evolution.region_commits c ON c.commit_id=x.commit_id + WHERE x.world_id=w AND x.region_id=destination AND x.commit_id=dst AND x.digest=substring(t FROM 161 FOR 32); + IF dm IS NULL THEN RAISE EXCEPTION 'migration destination moved' USING ERRCODE = 'EVL03'; END IF; + s := evolution.parse_checkpoint_manifest(sm); d := evolution.parse_checkpoint_manifest(dm); + IF s.profile NOT IN (4,5) OR d.profile<>s.profile OR s.versions<>bundle10 OR d.versions<>bundle10 + OR s.commit_sequence=0 OR d.commit_sequence=0 OR s.frame_id<>d.frame_id + OR s.frame_id<>evolution.manifest_uint(t,24,8) OR s.end_tick<>d.end_tick + OR s.tick_microseconds<>d.tick_microseconds + OR s.profile<>(CASE evolution.manifest_uint(t,12,4) WHEN 1 THEN 4 ELSE 5 END) THEN + RAISE EXCEPTION 'incompatible migration cuts' USING ERRCODE = 'EVL06'; + END IF; + IF EXISTS(SELECT 1 FROM evolution.cellular_migration_pending p WHERE p.world_id=w AND p.region_id=r) + OR EXISTS(SELECT 1 FROM evolution.population_interval_pending p WHERE p.world_id=w AND p.region_id=r) + OR EXISTS(SELECT 1 FROM evolution.cellular_mutation_reservations x WHERE x.world_id=w AND x.region_id=r + AND NOT EXISTS(SELECT 1 FROM evolution.cellular_mutation_action_publications a + WHERE a.world_id=x.world_id AND a.account_id=x.account_id AND a.action_id=x.action_id)) + OR EXISTS(SELECT 1 FROM evolution.life_reservations x WHERE x.world_id=w AND x.region_id=r AND x.rejection=0 AND x.ended_commit IS NULL + AND NOT EXISTS(SELECT 1 FROM evolution.primitive_life_bindings b + WHERE b.world_id=x.world_id AND b.account_id=x.account_id AND b.request_id=x.request_id)) THEN + RAISE EXCEPTION 'migration source has pending work' USING ERRCODE = 'EVL18'; + END IF; + INSERT INTO evolution.cellular_migration_preparations VALUES(id,w,r,destination,src,dst,t,holder,epoch); + INSERT INTO evolution.cellular_migration_pending VALUES(w,r,id); + RETURN QUERY SELECT * FROM evolution.cellular_migration_record(w,r,id); +END $$; +REVOKE ALL ON FUNCTION evolution.prepare_cellular_migration_freeze(evolution.u64,evolution.u64,uuid,evolution.u64,bytea) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION evolution.prepare_cellular_migration_freeze(evolution.u64,evolution.u64,uuid,evolution.u64,bytea) TO :"runtime_role"; + +CREATE FUNCTION evolution.abort_cellular_migration_freeze(w evolution.u64,r evolution.u64,holder uuid,epoch evolution.u64,id uuid) +RETURNS TABLE(trace bytea,source_manifest bytea,destination_manifest bytea,aborted boolean) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE p evolution.cellular_migration_preparations%ROWTYPE; +BEGIN + PERFORM evolution.check_region_lease(w,r,holder,epoch); + SELECT * INTO p FROM evolution.cellular_migration_preparations x WHERE x.request_id=id AND x.world_id=w AND x.source_region=r; + IF NOT FOUND THEN RAISE EXCEPTION 'unknown source migration' USING ERRCODE = 'EVL04'; END IF; + IF NOT EXISTS(SELECT 1 FROM evolution.cellular_migration_aborts a WHERE a.request_id=id) THEN + IF NOT EXISTS(SELECT 1 FROM evolution.cellular_migration_pending x WHERE x.world_id=w AND x.region_id=r AND x.request_id=id) + OR NOT EXISTS(SELECT 1 FROM evolution.region_heads h WHERE h.world_id=w AND h.region_id=r AND h.commit_id=p.source_commit + AND h.digest=substring(p.trace FROM 89 FOR 32) AND h.end_tick=evolution.manifest_uint(p.trace,32,8)) THEN + RAISE EXCEPTION 'migration freeze inconsistent' USING ERRCODE = 'EVL03'; + END IF; + INSERT INTO evolution.cellular_migration_aborts VALUES(id,holder,epoch); + DELETE FROM evolution.cellular_migration_pending x WHERE x.world_id=w AND x.region_id=r AND x.request_id=id; + END IF; + RETURN QUERY SELECT * FROM evolution.cellular_migration_record(w,r,id); +END $$; +REVOKE ALL ON FUNCTION evolution.abort_cellular_migration_freeze(evolution.u64,evolution.u64,uuid,evolution.u64,uuid) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION evolution.abort_cellular_migration_freeze(evolution.u64,evolution.u64,uuid,evolution.u64,uuid) TO :"runtime_role"; + +CREATE FUNCTION evolution.load_cellular_migration_freeze(w evolution.u64,r evolution.u64,holder uuid,epoch evolution.u64) +RETURNS TABLE(trace bytea,source_manifest bytea,destination_manifest bytea,aborted boolean) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE p evolution.cellular_migration_preparations%ROWTYPE; +BEGIN + PERFORM evolution.check_region_lease(w,r,holder,epoch); + SELECT a.* INTO p FROM evolution.cellular_migration_pending x JOIN evolution.cellular_migration_preparations a ON a.request_id=x.request_id + WHERE x.world_id=w AND x.region_id=r; + IF NOT FOUND THEN RETURN; END IF; + IF p.world_id<>w OR p.source_region<>r + OR EXISTS(SELECT 1 FROM evolution.cellular_migration_aborts a WHERE a.request_id=p.request_id) + OR NOT EXISTS(SELECT 1 FROM evolution.region_heads h WHERE h.world_id=w AND h.region_id=r AND h.commit_id=p.source_commit + AND h.digest=substring(p.trace FROM 89 FOR 32) AND h.end_tick=evolution.manifest_uint(p.trace,32,8)) THEN + RAISE EXCEPTION 'migration freeze inconsistent' USING ERRCODE = 'XX000'; + END IF; + RETURN QUERY SELECT * FROM evolution.cellular_migration_record(w,r,p.request_id); +END $$; +REVOKE ALL ON FUNCTION evolution.load_cellular_migration_freeze(evolution.u64,evolution.u64,uuid,evolution.u64) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION evolution.load_cellular_migration_freeze(evolution.u64,evolution.u64,uuid,evolution.u64) TO :"runtime_role"; + +CREATE FUNCTION evolution.refuse_frozen_cellular_work() RETURNS trigger +LANGUAGE plpgsql SET search_path = pg_catalog, pg_temp AS $$ +BEGIN + IF EXISTS(SELECT 1 FROM evolution.cellular_migration_pending p WHERE p.world_id=NEW.world_id AND p.region_id=NEW.region_id) THEN + RAISE EXCEPTION 'cellular source frozen for migration' USING ERRCODE = 'EVL18'; + END IF; + RETURN NEW; +END $$; +REVOKE ALL ON FUNCTION evolution.refuse_frozen_cellular_work() FROM PUBLIC; +CREATE TRIGGER region_cellular_migration_freeze BEFORE INSERT ON evolution.region_commits FOR EACH ROW EXECUTE FUNCTION evolution.refuse_frozen_cellular_work(); +CREATE TRIGGER interval_cellular_migration_freeze BEFORE INSERT ON evolution.population_interval_admissions FOR EACH ROW EXECUTE FUNCTION evolution.refuse_frozen_cellular_work(); +CREATE TRIGGER mutation_cellular_migration_freeze BEFORE INSERT ON evolution.cellular_mutation_reservations FOR EACH ROW EXECUTE FUNCTION evolution.refuse_frozen_cellular_work(); +CREATE TRIGGER birth_cellular_migration_freeze BEFORE INSERT ON evolution.life_reservations FOR EACH ROW EXECUTE FUNCTION evolution.refuse_frozen_cellular_work(); +INSERT INTO evolution.schema_migrations(version,sha256) VALUES(47,:'migration_sha_47'); diff --git a/migrations/apply.sql b/migrations/apply.sql index ac03d4c..e930b33 100644 --- a/migrations/apply.sql +++ b/migrations/apply.sql @@ -122,6 +122,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') SELECT :'history_v43' || ',44:' || :'migration_sha_44' AS history_v44 \gset SELECT :'history_v44' || ',45:' || :'migration_sha_45' AS history_v45 \gset SELECT :'history_v45' || ',46:' || :'migration_sha_46' AS history_v46 \gset + SELECT :'history_v46' || ',47:' || :'migration_sha_47' AS history_v47 \gset SELECT :'history' = :'history_v22' AS at_v22, :'history' = :'history_v23' AS at_v23, :'history' = :'history_v24' AS at_v24, @@ -146,8 +147,10 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') :'history' = :'history_v43' AS at_v43, :'history' = :'history_v44' AS at_v44, :'history' = :'history_v45' AS at_v45, - :'history' = :'history_v46' AS at_v46 \gset - \if :at_v46 + :'history' = :'history_v46' AS at_v46, + :'history' = :'history_v47' AS at_v47 \gset + \if :at_v47 + \elif :at_v46 \elif :at_v45 \elif :at_v44 \elif :at_v43 @@ -533,7 +536,8 @@ SELECT coalesce(max(version), 0) = 24 AS at_v24, coalesce(max(version), 0) = 43 AS at_v43, coalesce(max(version), 0) = 44 AS at_v44, coalesce(max(version), 0) = 45 AS at_v45, - coalesce(max(version), 0) = 46 AS at_v46 + coalesce(max(version), 0) = 46 AS at_v46, + coalesce(max(version), 0) = 47 AS at_v47 FROM evolution.schema_migrations \gset \if :at_v24 \ir 0025_cellular_v2_bundle.sql @@ -748,6 +752,7 @@ SELECT coalesce(max(version), 0) = 24 AS at_v24, \elif :at_v44 \elif :at_v45 \elif :at_v46 +\elif :at_v47 \else \echo 'Unknown or changed migration history after v43' DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$; @@ -764,6 +769,10 @@ SELECT coalesce(max(version), 0) = 45 AS needs_v46 FROM evolution.schema_migrati \if :needs_v46 \ir 0046_boundary_accounting_upgrade.sql \endif +SELECT coalesce(max(version), 0) = 46 AS needs_v47 FROM evolution.schema_migrations \gset +\if :needs_v47 + \ir 0047_cellular_migration_freeze.sql +\endif -- Effective-privilege audit on every run, including no-ops (ADR-0060). Attributes -- alone are insufficient: predefined roles such as pg_write_all_data, PUBLIC -- grants or SET-able memberships would bypass the fenced functions. Checks the @@ -783,6 +792,9 @@ BEGIN UNION SELECT m.member FROM pg_catalog.pg_auth_members m JOIN closure c ON m.roleid = c.oid ), fenced(oid) AS (VALUES + ('evolution.prepare_cellular_migration_freeze(evolution.u64,evolution.u64,uuid,evolution.u64,bytea)'::regprocedure::oid), + ('evolution.abort_cellular_migration_freeze(evolution.u64,evolution.u64,uuid,evolution.u64,uuid)'::regprocedure::oid), + ('evolution.load_cellular_migration_freeze(evolution.u64,evolution.u64,uuid,evolution.u64)'::regprocedure::oid), ('evolution.admit_population_interval(evolution.u64,evolution.u64,uuid,evolution.u64,uuid,bytea,evolution.u64)'::regprocedure::oid), ('evolution.load_population_interval(evolution.u64,evolution.u64,uuid,evolution.u64)'::regprocedure::oid), ('evolution.publish_population_checkpoint(bytea,bytea)'::regprocedure::oid), diff --git a/schemas/cellular-migration-trace-v1.md b/schemas/cellular-migration-trace-v1.md index 2b1f2fd..81f29ae 100644 --- a/schemas/cellular-migration-trace-v1.md +++ b/schemas/cellular-migration-trace-v1.md @@ -57,3 +57,10 @@ terminates, restores in a fresh process, replays and compares all four complete output snapshots. This qualifies portable continuation, not database atomicity, source freeze, ghost staging, abort, duplicate-delivery settlement or crash-safe activation. No schema 45 publisher admits this trace or the new owner tuple. + +M12.4h/ADR-0147/schema 47 subsequently retains trace 1 as durable source-freeze +preparation evidence. Both predecessor cuts are SHA-256 bound and replayed from +the source's retained capsule; SQL also checks canonical shape and FNV. The +source still owns every cell and the destination owns no ghost. Explicit abort +releases the freeze without changing either head. This is not result-manifest +admission or a paired ownership receipt. See [the freeze contract](ecology/cellular-migration-freeze-v1.md). diff --git a/schemas/ecology/cellular-migration-freeze-v1.md b/schemas/ecology/cellular-migration-freeze-v1.md new file mode 100644 index 0000000..b851992 --- /dev/null +++ b/schemas/ecology/cellular-migration-freeze-v1.md @@ -0,0 +1,64 @@ +# Biological migration preparation and source freeze + +Authority: ADR-0147, SQL schema 47. This retains preparation and explicit abort, +not a migration result or destination activation receipt. Checkpoint profiles, +rules 8, owner 7 and population batch 2 retain their existing bytes and meanings. + +One source-authorized request binds canonical +[migration trace 1](../cellular-migration-trace-v1.md), both retained predecessor +commits/manifests and their exact payloads. Both cuts use bundle 10, the same +representation, world/frame/completed tick and compatible geometry/biology. +The source lease/head is fenced; the destination head is checked optimistically +and remains free to advance. Later staging must check it again. Preparation +does not establish a simultaneous world cut or a physical route. + +The adapter restores both SHA-256-verified payloads and derives the complete +biological pair before publication. It copies/verifies the destination cut into +the source's immutable object namespace as a replay capsule. This copy owns no +stock. The source keeps every organism, reserve, body, nursery/habitat compartment, +heat and historical fact at its unchanged committed head. The destination also +keeps its existing head. Proposed transferred material in the trace is metadata, +not an additional account or an applied boundary flow. + +SQL retains: + +- Immutable `cellular_migration_preparations`, keyed by globally unique request + UUID: source/destination region and predecessor commit references, canonical + trace (272..2,312 bytes), preparing holder and epoch. +- Immutable `cellular_migration_aborts`, keyed by request, with the deciding + current source holder/epoch. +- One operational `cellular_migration_pending` pointer per source region. + +Preparation refuses an active source workflow, pending ecological interval, +unpublished player choice or outstanding successful life reservation. C++ also +requires fully suspended controlled cells and an eligible whole wildlife group. +SQL independently validates trace identity/length/member bounds, exact FNV-1a64, +both committed manifest bindings and the declared representation/tuple. +Checksum/UUID metadata alone does not qualify the biology: recovery restores both +cuts from the source namespace and rederives every recorded member/channel. + +While pending, indexed trigger guards refuse new region commits, interval +admissions, mutation reservations and life reservations at that source. Historical +exact operation retries keep their old meanings. The whole region is paused; +there is no per-member advancement or passive healing. The host withholds +readiness during preparation uncertainty and a recovered freeze, renews authority, +and polls paused work at most once per second. Normal unpaused stepping adds no +per-tick migration query. `evo_region_migration_frozen` is a fixed, unlabelled +0/1 gauge for a known durable freeze; it does not grant authority. + +Explicit source-authorized abort appends its outcome and removes the pending +pointer atomically, without changing either biological head. Exact preparation +and abort retries return the recorded outcome. An aborted preparation cannot +reopen its pointer. Process death, lease expiry, drain, missing evidence and +timeout never imply abort. Missing/corrupt capsules fail closed, without falling +back to the destination's live state. A new source holder may explicitly abort; +destination authority is unnecessary because no ownership has moved. + +Keep both predecessor histories and source-side capsule objects, including after +abort. The current backup copies all immutable namespace objects and the shared +database snapshot; no GC or pruning is authorized. This does not qualify cellular +history cloning/restore beyond the existing supported fixtures. + +Destination ghost staging, atomic two-head publication, commit/abort races and +activation remain subsequent work. This segment adds no automatic migration, +atlas, cross-edge movement, environmental coupling or census/extinction rule. diff --git a/src/persistence/cellular_migration_store.cpp b/src/persistence/cellular_migration_store.cpp new file mode 100644 index 0000000..f04b4ec --- /dev/null +++ b/src/persistence/cellular_migration_store.cpp @@ -0,0 +1,255 @@ +#include "evo/persistence/cellular_migration_store.hpp" +#include "evo/persistence/sha256.hpp" + +#include +#include +#include + +namespace evo::persistence { +namespace { +using std::chrono::milliseconds; +LeaseStoreFailure failure(const PgFailure& cause) { + const std::string_view state(cause.sqlstate.data(), 5); + auto kind = LeaseStoreError::store; + if (cause.kind == PgError::unknown) + kind = LeaseStoreError::unknown; + else if (state == "22023" || cause.kind == PgError::bounds || cause.kind == PgError::config) + kind = LeaseStoreError::invalid; + else if (state == "EVL02") + kind = LeaseStoreError::fenced; + else if (state == "EVL03") + kind = LeaseStoreError::head_mismatch; + else if (state == "EVL04") + kind = LeaseStoreError::not_provisioned; + else if (state == "EVL06") + kind = LeaseStoreError::unsupported; + else if (state == "EVL07" || state == "EVL18") + kind = LeaseStoreError::commit_conflict; + return {kind, cause}; +} +std::string hex(std::span bytes) { + constexpr std::string_view digits = "0123456789abcdef"; + std::string out; + for (const auto byte : bytes) { + out += digits[byte >> 4U]; + out += digits[byte & 15U]; + } + return out; +} +std::string uuid(const Uuid& id) { + const auto h = hex(id); + return h.substr(0, 8) + '-' + h.substr(8, 4) + '-' + h.substr(12, 4) + '-' + h.substr(16, 4) + + '-' + h.substr(20); +} +std::optional> unhex(std::string_view value, std::size_t cap) { + if (value.size() % 2 != 0 || value.size() / 2 > cap) + return std::nullopt; + std::vector out; + out.reserve(value.size() / 2); + for (std::size_t at = 0; at < value.size(); at += 2) { + unsigned byte{}; + const auto [end, error] = + std::from_chars(value.data() + at, value.data() + at + 2, byte, 16); + if (error != std::errc{} || end != value.data() + at + 2 || byte > 255) + return std::nullopt; + out.push_back(static_cast(byte)); + } + return out; +} +struct Record { + std::vector trace, source, destination; + bool aborted{}; +}; +std::optional parse(const PgRows::value_type& row) { + if (row.size() != 4 || (row[3].value_or("") != "true" && row[3].value_or("") != "false")) + return std::nullopt; + auto trace = unhex(row[0].value_or(""), server::max_migration_trace_bytes); + auto source = unhex(row[1].value_or(""), checkpoint_manifest_bytes); + auto destination = unhex(row[2].value_or(""), checkpoint_manifest_bytes); + if (!trace || !source || !destination) + return std::nullopt; + return Record{std::move(*trace), std::move(*source), std::move(*destination), + row[3].value_or("") == "true"}; +} +bool bound(const CommittedCheckpoint& c, const CheckpointManifest& m) { + const auto digest = sha256(c.manifest); + return digest && *digest == c.head.digest && m.commit == c.head.commit && + m.frame == c.head.frame && m.sequence == c.head.sequence && + m.end_tick == c.head.end_tick; +} +bool cuts(const CheckpointManifest& s, const CheckpointManifest& d, + const Sha256Digest& source_digest, const Sha256Digest& destination_digest, + const server::CellularMigrationTrace& t) { + const auto profile = t.representation == server::CellularMigrationRepresentation::individual + ? CheckpointProfile::cellular_lives_v1 + : CheckpointProfile::cellular_population_v1; + return supported(s) && supported(d) && encode_manifest(s) && encode_manifest(d) && + s.cellular_versions == cellular_v10_versions && + d.cellular_versions == cellular_v10_versions && s.profile == profile && + d.profile == profile && s.sequence > 0 && d.sequence > 0 && + s.key == RegionKey{t.world, t.request.source.region} && + d.key == RegionKey{t.world, t.request.destination.region} && s.frame == t.frame && + d.frame == t.frame && s.end_tick == t.tick && d.end_tick == t.tick && + s.commit == t.request.source.predecessor && + d.commit == t.request.destination.predecessor && + source_digest == t.request.source.predecessor_digest && + destination_digest == t.request.destination.predecessor_digest; +} +std::expected +derive(const CheckpointManifest& s, const CheckpointManifest& d, + std::span source, std::span destination, + const server::CellularMigrationRequest& request) { + if (s.profile == CheckpointProfile::cellular_lives_v1) { + auto src = restore_cellular_payload(source, s), + dst = restore_cellular_payload(destination, d); + if (!src || !dst) + return std::unexpected(CheckpointError::payload); + auto pair = server::prepare_migration_trace(**src, **dst, request); + if (!pair) + return std::unexpected(CheckpointError::payload); + return std::move(pair->trace); + } + auto src = restore_population_payload(source, s), + dst = restore_population_payload(destination, d); + if (!src || !dst) + return std::unexpected(CheckpointError::payload); + auto pair = server::prepare_migration_trace(**src, **dst, request); + if (!pair) + return std::unexpected(CheckpointError::payload); + return std::move(pair->trace); +} +std::optional checked_record(const Record& record, RegionKey key, + const LeaseGrant& lease) { + auto trace = server::decode_migration_trace(record.trace); + const auto source = decode_manifest(record.source), + destination = decode_manifest(record.destination); + const auto s = sha256(record.source), d = sha256(record.destination); + if (!trace || !source || !destination || !s || !d || + key != RegionKey{trace->world, trace->request.source.region} || + lease.frame != trace->frame || !cuts(*source, *destination, *s, *d, *trace)) + return std::nullopt; + return CellularMigrationFreeze{std::move(*trace), record.aborted}; +} +constexpr auto columns = "encode(trace,'hex'),encode(source_manifest,'hex')," + "encode(destination_manifest,'hex'),aborted::text"; +CellularMigrationFailure failed(PublicationError kind, LeaseStoreFailure store = {}, + ObjectError object = {}, CellularMigrationPrepared prepared = {}) { + return {kind, store, object, std::move(prepared)}; +} +} // namespace + +std::expected +retry_cellular_migration_freeze(Postgres& db, const LeaseGrant& lease, + const CellularMigrationPrepared& prepared, milliseconds timeout) { + const auto trace = server::decode_migration_trace(prepared.trace); + if (!trace || prepared.key != RegionKey{trace->world, trace->request.source.region} || + lease.frame != trace->frame) + return std::unexpected(failed(PublicationError::candidate)); + const std::array args{std::to_string(prepared.key.world), + std::to_string(prepared.key.region), uuid(lease.holder), + std::to_string(lease.epoch), "\\x" + hex(prepared.trace)}; + auto rows = + db.query(std::string("SELECT ") + columns + + " FROM evolution.prepare_cellular_migration_freeze(" + "$1::evolution.u64,$2::evolution.u64,$3::uuid,$4::evolution.u64,$5::bytea)", + args, timeout); + if (!rows) { + const auto store = failure(rows.error()); + return std::unexpected(failed(store.kind == LeaseStoreError::unknown + ? PublicationError::unknown + : PublicationError::rejected, + store, {}, prepared)); + } + const auto record = rows->size() == 1 ? parse(rows->front()) : std::nullopt; + const auto result = record ? checked_record(*record, prepared.key, lease) : std::nullopt; + if (!result || record->trace != prepared.trace) + return std::unexpected(failed(PublicationError::unknown, {}, {}, prepared)); + return *result; +} + +std::expected +prepare_cellular_migration_freeze(Postgres& db, const FilesystemObjectStore& source_objects, + const FilesystemObjectStore& destination_objects, + const LeaseGrant& lease, const CommittedCheckpoint& source, + const CommittedCheckpoint& destination, + const server::CellularMigrationRequest& request, + milliseconds timeout, const PublishHooks& hooks) { + const auto s = decode_manifest(source.manifest), d = decode_manifest(destination.manifest); + if (!s || !d || !bound(source, *s) || !bound(destination, *d)) + return std::unexpected(failed(PublicationError::predecessor)); + auto src = source_objects.get({s->payload_sha256, s->payload_bytes}); + auto dst = destination_objects.get({d->payload_sha256, d->payload_bytes}); + if (!src || !dst) + return std::unexpected( + failed(PublicationError::predecessor, {}, !src ? src.error() : dst.error())); + auto trace = derive(*s, *d, *src, *dst, request); + if (!trace || !cuts(*s, *d, source.head.digest, destination.head.digest, *trace) || + lease.frame != s->frame) + return std::unexpected(failed(PublicationError::candidate)); + auto encoded = server::encode_migration_trace(*trace); + if (!encoded) + return std::unexpected(failed(PublicationError::candidate)); + auto capsule = source_objects.put(*dst); + if (!capsule) + return std::unexpected(failed(PublicationError::upload, {}, capsule.error())); + auto readback = source_objects.get(*capsule); + if (!readback || *readback != *dst || + *capsule != VerifiedObject{d->payload_sha256, d->payload_bytes}) + return std::unexpected(failed(PublicationError::verify, {}, + readback ? ObjectError::corrupt : readback.error())); + if (hooks.before_transaction) + hooks.before_transaction(); + return retry_cellular_migration_freeze(db, lease, {s->key, std::move(*encoded)}, timeout); +} + +std::expected +abort_cellular_migration_freeze(Postgres& db, RegionKey key, const LeaseGrant& lease, + const Uuid& request, milliseconds timeout) { + const std::array args{std::to_string(key.world), std::to_string(key.region), + uuid(lease.holder), std::to_string(lease.epoch), + uuid(request)}; + auto rows = + db.query(std::string("SELECT ") + columns + + " FROM evolution.abort_cellular_migration_freeze(" + "$1::evolution.u64,$2::evolution.u64,$3::uuid,$4::evolution.u64,$5::uuid)", + args, timeout); + if (!rows) + return std::unexpected(failure(rows.error())); + const auto record = rows->size() == 1 ? parse(rows->front()) : std::nullopt; + auto result = record ? checked_record(*record, key, lease) : std::nullopt; + if (!result || !result->aborted || result->trace.request.request != request) + return std::unexpected(LeaseStoreFailure{LeaseStoreError::unknown, {}}); + return *result; +} + +std::expected, RecoveryFailure> +load_cellular_migration_freeze(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, + const LeaseGrant& lease, milliseconds timeout) { + const auto fail = [](RecoveryError kind) { return std::unexpected(RecoveryFailure{kind, {}}); }; + const std::array args{std::to_string(key.world), std::to_string(key.region), + uuid(lease.holder), std::to_string(lease.epoch)}; + auto rows = db.query(std::string("SELECT ") + columns + + " FROM evolution.load_cellular_migration_freeze(" + "$1::evolution.u64,$2::evolution.u64,$3::uuid,$4::evolution.u64)", + args, timeout); + if (!rows) + return std::unexpected(RecoveryFailure{RecoveryError::store, failure(rows.error())}); + if (rows->empty()) + return std::optional{}; + const auto record = rows->size() == 1 ? parse(rows->front()) : std::nullopt; + auto result = record ? checked_record(*record, key, lease) : std::nullopt; + if (!result || result->aborted) + return fail(RecoveryError::history); + const auto s = decode_manifest(record->source), d = decode_manifest(record->destination); + if (!s || !d) + return fail(RecoveryError::manifest); + auto src = objects.get({s->payload_sha256, s->payload_bytes}); + auto dst = objects.get({d->payload_sha256, d->payload_bytes}); + if (!src || !dst) + return fail(RecoveryError::tail); + const auto replay = derive(*s, *d, *src, *dst, result->trace.request); + if (!replay || *replay != result->trace) + return fail(RecoveryError::history); + return result; +} +} // namespace evo::persistence diff --git a/src/server/cellular_region_host.cpp b/src/server/cellular_region_host.cpp index 9dab731..d93dda0 100644 --- a/src/server/cellular_region_host.cpp +++ b/src/server/cellular_region_host.cpp @@ -199,6 +199,9 @@ void CellularRegionHost::acquire() { } if (!recover_state(*lease) || !committed_.has_value() || !state_) return; + migration_reload_ = true; + migration_frozen_ = false; + migration_poll_at_ = {}; interval_reload_ = population_mode(); admitted_interval_.reset(); interval_request_.reset(); @@ -412,7 +415,9 @@ void CellularRegionHost::refresh() { const auto& grant = authority_.grant(); status_.ready = status_.phase == HostPhase::running && state_ && committed_ && grant && authority_.may_originate(LeaseClock::now()) && !interval_reload_ && - !interval_request_.has_value(); + !interval_request_.has_value() && !migration_reload_ && !migration_frozen_ && + !migration_request_.has_value() && !migration_abort_.has_value(); + status_.migration_frozen = migration_frozen_; const auto& interval = admitted_interval_; status_.ecological_pending_ticks = interval.has_value() ? interval->end_tick - interval->begin_tick : 0; @@ -424,6 +429,139 @@ void CellularRegionHost::refresh() { : 0; } +void CellularRegionHost::reconcile_migration_freeze() { + if ((!migration_reload_ && !migration_frozen_ && !migration_request_ && !migration_abort_) || + LeaseClock::now() < migration_poll_at_) + return; + const auto grant = authority_.grant(); + if (!grant || !authority_.may_begin_commit(LeaseClock::now()) || !database()) + return; + migration_poll_at_ = LeaseClock::now() + std::chrono::seconds{1}; + if (migration_request_) { + auto result = persistence::retry_cellular_migration_freeze( + *db_, *grant, *migration_request_, config_.lease.commit_timeout); + if (!result) { + if (result.error().kind == persistence::PublicationError::unknown) { + db_.reset(); + ++status_.unknown_outcomes; + return; + } + if (result.error().store.kind == LeaseStoreError::fenced || + result.error().store.kind == LeaseStoreError::head_mismatch) { + authority_.fence(); + state_.reset(); + committed_.reset(); + status_.phase = HostPhase::acquiring; + } else { + authority_.fence(); + status_.phase = HostPhase::quarantined; + } + return; + } + migration_request_.reset(); + } + if (migration_abort_) { + auto result = persistence::abort_cellular_migration_freeze( + *db_, config_.key, *grant, *migration_abort_, config_.lease.commit_timeout); + if (!result) { + if (result.error().kind == LeaseStoreError::unknown) { + db_.reset(); + ++status_.unknown_outcomes; + return; + } + authority_.fence(); + status_.phase = result.error().kind == LeaseStoreError::fenced ? HostPhase::acquiring + : HostPhase::quarantined; + state_.reset(); + committed_.reset(); + return; + } + migration_abort_.reset(); + } + const auto frozen = persistence::load_cellular_migration_freeze( + *db_, *objects_, config_.key, *grant, config_.lease.commit_timeout); + if (!frozen) { + migration_reload_ = true; + if (frozen.error().kind == persistence::RecoveryError::store) { + db_.reset(); + ++status_.store_failures; + } else { + authority_.fence(); + status_.phase = HostPhase::quarantined; + } + return; + } + migration_frozen_ = frozen->has_value(); + migration_reload_ = false; +} + +std::expected CellularRegionHost::prepare_migration_freeze( + const persistence::CommittedCheckpoint& destination, + const persistence::FilesystemObjectStore& destination_objects, + const CellularMigrationRequest& request) { + const auto grant = authority_.grant(); + if (!grant.has_value() || !status_.ready || pending_ || !state_ || !committed_ || + !queued_births_.empty() || !queued_mutations_.empty() || mutations_uncertain_ || + !pending_endings_.empty() || interval_request_ || admitted_interval_ || interval_reload_ || + !population_interval_safe(*state_) || !authority_.may_begin_commit(LeaseClock::now()) || + !database()) + return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}); + migration_reload_ = true; + refresh(); + const auto result = persistence::prepare_cellular_migration_freeze( + *db_, *objects_, destination_objects, *grant, *committed_, destination, request, + config_.lease.commit_timeout); + if (!result) { + if (result.error().kind == persistence::PublicationError::unknown) { + migration_request_ = result.error().prepared; + db_.reset(); + ++status_.unknown_outcomes; + } else if (result.error().store.kind == LeaseStoreError::head_mismatch || + result.error().store.kind == LeaseStoreError::fenced) { + authority_.fence(); + state_.reset(); + committed_.reset(); + status_.phase = HostPhase::acquiring; + } + migration_poll_at_ = {}; + refresh(); + return std::unexpected(CellularPublishFailure{CellularHostError::publication, + result.error().kind, result.error().store, + result.error().object}); + } + migration_poll_at_ = {}; + reconcile_migration_freeze(); + refresh(); + return {}; +} + +std::expected +CellularRegionHost::abort_migration_freeze(const persistence::Uuid& request) { + const auto grant = authority_.grant(); + if (!grant || !state_ || pending_ || status_.phase != HostPhase::running || + !authority_.may_begin_commit(LeaseClock::now()) || !database()) + return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}); + migration_reload_ = true; + refresh(); + const auto result = persistence::abort_cellular_migration_freeze( + *db_, config_.key, *grant, request, config_.lease.commit_timeout); + if (!result) { + if (result.error().kind == LeaseStoreError::unknown) { + migration_abort_ = request; + db_.reset(); + ++status_.unknown_outcomes; + } + migration_poll_at_ = {}; + refresh(); + return std::unexpected( + CellularPublishFailure{CellularHostError::publication, {}, result.error(), {}}); + } + migration_poll_at_ = {}; + reconcile_migration_freeze(); + refresh(); + return {}; +} + const HostStatus& CellularRegionHost::step() { if (status_.phase == HostPhase::quarantined || status_.phase == HostPhase::stopped) return status_; @@ -433,6 +571,12 @@ const HostStatus& CellularRegionHost::step() { resolve(); else if (!state_) acquire(); + if (!pending_ && state_ && status_.phase == HostPhase::running) + reconcile_migration_freeze(); + if (migration_reload_ || migration_frozen_ || migration_request_ || migration_abort_) { + refresh(); + return status_; + } if (!pending_ && state_ && status_.phase == HostPhase::running) reconcile_interval(); if (!pending_ && state_ && mutations_uncertain_ && status_.phase == HostPhase::running) diff --git a/src/server/ops.cpp b/src/server/ops.cpp index 037ae27..d048e0e 100644 --- a/src/server/ops.cpp +++ b/src/server/ops.cpp @@ -74,6 +74,8 @@ std::string render_metrics(const HostStatus& s, const OpsCounters& c) { metric(out, "evo_region_tick_debt", "gauge", "Due ticks not yet simulated", s.tick_debt); metric(out, "evo_region_ecological_pending_ticks", "gauge", "Durable admitted ecological ticks", s.ecological_pending_ticks); + metric(out, "evo_region_migration_frozen", "gauge", "Known durable biological source freeze", + s.migration_frozen ? 1 : 0); metric(out, "evo_region_tick_debt_max", "gauge", "Largest tick debt since recovery", s.max_tick_debt); metric(out, "evo_region_queued_inputs", "gauge", "Admitted inputs awaiting a tick", s.queued); diff --git a/tests/ops_tests.cpp b/tests/ops_tests.cpp index 5996856..f137bcf 100644 --- a/tests/ops_tests.cpp +++ b/tests/ops_tests.cpp @@ -72,6 +72,7 @@ void exposition_is_bounded_and_well_formed() { s.committed_tick = 41; s.lease_remaining_ms = 1234; s.ecological_pending_ticks = 64; + s.migration_frozen = true; OpsCounters c; c.queue_capacity_inputs = 64; c.queue_capacity_bytes = 65536; @@ -93,10 +94,11 @@ void exposition_is_bounded_and_well_formed() { CHECK(text.find("evo_region_ready 1\n") != std::string::npos); CHECK(text.find("evo_region_commit_lag_ticks 9\n") != std::string::npos); CHECK(text.find("evo_region_ecological_pending_ticks 64\n") != std::string::npos); + CHECK(text.find("evo_region_migration_frozen 1\n") != std::string::npos); CHECK(text.find("evo_region_lease_remaining_milliseconds 1234\n") != std::string::npos); CHECK(text.find("evo_region_dropped_total{reason=\"fenced\"} 5\n") != std::string::npos); - // Series count is fixed: 6 phases + 4 reasons + 23 unlabelled families. - CHECK(samples.size() == 6 + 4 + 23); + // Series count is fixed: 6 phases + 4 reasons + 24 unlabelled families. + CHECK(samples.size() == 6 + 4 + 24); // Worst-case values keep the size bounded and never underflow the lag. HostStatus big; diff --git a/tests/population_publication_tests.cpp b/tests/population_publication_tests.cpp index 4b2c706..582284e 100644 --- a/tests/population_publication_tests.cpp +++ b/tests/population_publication_tests.cpp @@ -745,6 +745,378 @@ void boundary_upgrades(Fixture& f) { require(f.raw->format_version() == cellular_lives_migration_format, "reverse transition downgraded accounting owner"); } +void migration_freezes(Fixture& f) { + const RegionKey destination_key{f.key.world, 2}; + auto destination_objects = + take(FilesystemObjectStore::open(environment("PGHOST"), destination_key, + max_cellular_population_bytes), + "migration destination objects"); + auto destination = opening(30001, true, 4, false, 2); + auto destination_head = + take(provision_cellular_region(*f.owner, *destination_objects, destination_key, 1, + *destination, f.id(), f.build, 5s), + "migration empty destination genesis"); + auto destination_lease = + take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s), + "migration destination authority"); + while (destination->tick() < f.raw->tick()) { + const auto count = std::min(std::uint64_t{64}, f.raw->tick() - destination->tick()); + CellularBatch inputs{{f.key.world, 2, 1, destination_head.head.commit, + destination_head.head.digest, destination->tick(), + destination->tick() + count}, + {}}; + inputs.ticks.resize(static_cast(count)); + destination = + take(replay_cellular_batch(*destination, inputs), "empty destination biology"); + destination_head = take(publish_cellular_batch(*f.db, *destination_objects, + destination_lease, destination_head, inputs, + *destination, f.id(), f.build, 5s), + "empty destination tick publication") + .committed; + } + destination_head = + take(migrate_cellular_boundary_profile(*f.db, *destination_objects, destination_lease, + destination_head, f.id(), f.build, 5s), + "destination accounting upgrade"); + destination = take(destination->migration_upgrade_candidate(), "destination upgraded oracle"); + const std::array builds{f.build}; + const auto request_for = [&] { + const auto member = std::ranges::find_if( + f.raw->lives(), [](const auto& life) { return life.account == LifeAccount{}; }); + require(member != f.raw->lives().end(), "freeze fixture lacks real wildlife"); + return CellularMigrationRequest{ + f.id(), + {1, f.head.head.commit, f.id(), f.head.head.digest}, + {2, destination_head.head.commit, f.id(), destination_head.head.digest}, + member->identity.genome}; + }; + for (unsigned representation = 0; representation < 2; ++representation) { + if (representation == 1) { + f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), + f.build, {true, 1200, 1800}, 5s), + "freeze cohort source entry"); + recover(f); + destination_head = take(migrate_cellular_to_population( + *f.db, *destination_objects, destination_lease, + destination_head, f.id(), f.build, {true, 1200, 1800}, 5s), + "freeze cohort destination entry"); + } + const auto source_before = f.head; + const auto source_bytes = f.raw->snapshot(); + const auto destination_before = destination_head; + const auto request = request_for(); + const auto prepare = [&](const CellularMigrationRequest& requested, + std::chrono::milliseconds timeout = 5s) { + return prepare_cellular_migration_freeze(*f.db, *f.objects, *destination_objects, + f.lease, f.head, destination_head, requested, + timeout); + }; + if (representation == 0) { + Uuid account{}; + account.back() = 1; + take(f.db->query("BEGIN", {}, 5s), "pending player choice before freeze"); + const auto control = + take(claim_account_control(*f.db, f.key, f.lease, account, 2, 9900, 60000ms, 5s), + "pending choice current control"); + const auto input = encode_cellular_mutation_input( + {account, 9900, f.head.head.end_tick + 1, {100, 1, 1, 1}}); + take(reserve_cellular_mutation(*f.db, f.key, f.lease, account, control.generation, 9900, + input, 5s), + "reserve pre-freeze player choice"); + const auto pending = prepare(request); + take(f.db->query("ROLLBACK", {}, 5s), "rollback pre-freeze player choice"); + require(!pending && + std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18", + "preparation stranded a player choice"); + } else { + take(f.db->query("BEGIN", {}, 5s), "prepare with admitted work transaction"); + take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, + f.head.head.end_tick + 20, 5s), + "admitted work before freeze"); + const auto pending = prepare(request); + take(f.db->query("ROLLBACK", {}, 5s), "rollback pending-work preparation fixture"); + require(!pending && + std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18", + "preparation skipped durable ecological work"); + } + auto frozen = take(prepare(request), "durable source freeze"); + const CellularMigrationPrepared prepared{ + f.key, take(encode_migration_trace(frozen.trace), "frozen canonical trace")}; + auto malformed = prepared.trace; + malformed.back() ^= 1; + const std::array malformed_args{hexadecimal(f.lease.holder, false), + std::to_string(f.lease.epoch), + hexadecimal(malformed)}; + const auto malformed_sql = f.db->query("SELECT * FROM " + "evolution.prepare_cellular_migration_freeze(30201," + "1,$1::uuid,$2::evolution.u64,$3::bytea)", + malformed_args, 5s); + require(!malformed_sql && std::string(malformed_sql.error().sqlstate.data(), 5) == "22023", + "SQL accepted an invalid trace checksum before duplicate lookup"); + require( + !frozen.aborted && frozen.trace.request == request && + take(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s), + "replay frozen capsule"), + "missing durable freeze") + .trace == frozen.trace && + take(load_region_checkpoint(*f.db, f.key, 5s), "frozen source head").head == + source_before.head && + take(load_region_checkpoint(*f.db, destination_key, 5s), "unowned destination head") + .head == destination_before.head, + "preparation changed either owner or lost its full replay capsule"); + require( + !take(retry_cellular_migration_freeze(*f.db, f.lease, prepared, 5s), "duplicate freeze") + .aborted, + "duplicate freeze lost its state"); + auto conflict_trace = frozen.trace; + ++conflict_trace.transferred.energy.value; + auto conflict = retry_cellular_migration_freeze( + *f.db, f.lease, {f.key, take(encode_migration_trace(conflict_trace), "conflict trace")}, + 5s); + require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict && + !prepare(request_for()), + "freeze accepted changed bytes or replaced pending request"); + if (representation == 0) { + const auto h = f.header(1); + const auto inputs = take(unattended_population_inputs(*f.raw, h), "frozen raw inputs"); + auto advanced = take(replay_cellular_batch(*f.raw, inputs), "frozen detached future"); + auto denied = publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, inputs, + *advanced, f.id(), f.build, 5s); + require(!denied && + std::string(denied.error().store.cause.sqlstate.data(), 5) == "EVL18" && + !migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), + f.build, {}, 5s), + "raw source progressed or changed profile while frozen"); + Uuid account{}; + account.back() = 1; + const auto input = encode_cellular_mutation_input( + {account, 9901, f.head.head.end_tick + 1, {100, 1, 1, 1}}); + take(f.db->query("BEGIN", {}, 5s), "current control while frozen fixture"); + const auto control = + take(claim_account_control(*f.db, f.key, f.lease, account, 2, 9901, 60000ms, 5s), + "current frozen-source account control"); + const auto choice = reserve_cellular_mutation(*f.db, f.key, f.lease, account, + control.generation, 9901, input, 5s); + take(f.db->query("ROLLBACK", {}, 5s), "rollback frozen control fixture"); + require(!choice, "frozen raw owner admitted a new mutation reservation"); + if (std::string(choice.error().cause.sqlstate.data(), 5) != "EVL18") + throw std::runtime_error("frozen mutation guard SQLSTATE=" + + std::string(choice.error().cause.sqlstate.data(), 5)); + } else { + require(!admit_population_interval(*f.db, f.key, f.lease, f.head.head, + f.head.head.end_tick + 20, 5s) && + !materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), + f.build, 5s), + "population source admitted work or changed profile while frozen"); + } + const auto capsule_manifest = + take(decode_manifest(destination_head.manifest), "capsule manifest"); + const auto capsule_path = + f.objects->directory() / hexadecimal(capsule_manifest.payload_sha256, false); + const auto backup = capsule_path.string() + ".freeze-test"; + std::filesystem::rename(capsule_path, backup); + const auto missing = load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s); + std::filesystem::rename(backup, capsule_path); + require(!missing && missing.error().kind == RecoveryError::tail, + "freeze recovery fell back to destination live state without the retained capsule"); + char first{}; + { + std::fstream file(capsule_path, std::ios::binary | std::ios::in | std::ios::out); + file.get(first); + file.seekp(0); + file.put(static_cast(static_cast(first) ^ 1U)); + file.flush(); + require(file.good(), "corrupt same-length capsule fixture"); + } + const auto corrupt = load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s); + { + std::fstream file(capsule_path, std::ios::binary | std::ios::in | std::ios::out); + file.put(first); + file.flush(); + require(file.good(), "restore same-length capsule fixture"); + } + require(!corrupt && corrupt.error().kind == RecoveryError::tail, + "corrupt capsule bypassed strong object binding"); + const auto old_lease = f.lease; + take(release_region_lease(*f.db, f.key, f.lease, 5s), "release frozen source owner"); + f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), + "new frozen source owner"); + require(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s), + "recover freeze under new epoch") + .has_value() && + !abort_cellular_migration_freeze(*f.db, f.key, old_lease, request.request, 5s), + "new epoch released a freeze or old epoch aborted it"); + require( + take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, request.request, 5s), + "explicit source abort") + .aborted && + take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, request.request, 5s), + "duplicate source abort") + .aborted && + take(retry_cellular_migration_freeze(*f.db, f.lease, prepared, 5s), + "retry aborted preparation") + .aborted && + !take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s), + "aborted pointer load") + .has_value(), + "abort was not durable or preparation retry reopened it"); + require(take(load_region_checkpoint(*f.db, f.key, 5s), "aborted source head").head == + source_before.head && + f.raw->snapshot() == source_bytes, + "abort changed biological stock/time/history"); + + auto false_trace = frozen.trace; + false_trace.request.request = f.id(); + false_trace.request.source.commit = f.id(); + false_trace.request.destination.commit = f.id(); + ++false_trace.transferred.energy.value; + take(retry_cellular_migration_freeze(*f.db, f.lease, + {f.key, take(encode_migration_trace(false_trace), + "canonical false inventory trace")}, + 5s), + "metadata-only false preparation"); + const auto false_replay = + load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s); + require(!false_replay && false_replay.error().kind == RecoveryError::history, + "freeze recovery trusted declared material without complete biological replay"); + take( + abort_cellular_migration_freeze(*f.db, f.key, f.lease, false_trace.request.request, 5s), + "explicitly abort false preparation without moving stock"); + + // A genuine unknown reply retains exactly the chosen cut and group. + auto locker = connection(true); + take(locker->query("BEGIN", {}, 5s), "freeze ambiguity lock"); + take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND " + "region_id=1 FOR UPDATE", + {}, 5s), + "hold freeze reply"); + auto timed = connection(); + auto unknown = + prepare_cellular_migration_freeze(*timed, *f.objects, *destination_objects, f.lease, + f.head, destination_head, request_for(), 300ms); + require(!unknown && unknown.error().kind == PublicationError::unknown && + !unknown.error().prepared.trace.empty(), + "freeze ambiguity lost its immutable request"); + take(locker->query("ROLLBACK", {}, 5s), "release freeze ambiguity"); + frozen = take(retry_cellular_migration_freeze(*f.db, f.lease, unknown.error().prepared, 5s), + "reconcile unknown freeze"); + take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, frozen.trace.request.request, + 5s), + "abort reconciled freeze"); + + // Both sides of preparation's transaction boundary, with real process death. + for (unsigned point = 0; point < 2; ++point) { + const auto killed_request = request_for(); + const auto child = ::fork(); + require(child >= 0, "freeze publisher fork"); + if (child == 0) { + try { + auto db = connection(); + PublishHooks hooks; + if (point == 0) + hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); }; + if (!prepare_cellular_migration_freeze(*db, *f.objects, *destination_objects, + f.lease, f.head, destination_head, + killed_request, 5s, hooks)) + ::_exit(41); + ::kill(::getpid(), SIGKILL); + } catch (...) { + ::_exit(42); + } + ::_exit(43); + } + int status{}; + require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) && + WTERMSIG(status) == SIGKILL && + take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s), + "recover killed preparation") + .has_value() == (point == 1), + "publisher death changed source ownership freeze"); + if (point == 1) + take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, killed_request.request, + 5s), + "abort killed preparation"); + } + } + // The owning host must retain the freeze through drain/recreation with entry off. + take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture to frozen host"); + const auto make_host = [&] { + auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), f.key, + max_cellular_population_bytes), + "freeze host objects"); + CellularHostConfig config{f.key, 1, {}, f.build, {f.build}}; + config.lease.commit_timeout = 300ms; + return take( + CellularRegionHost::create( + config, + [] { return std::expected, PgFailure>{connection()}; }, + std::move(objects)), + "freeze host"); + }; + auto host = make_host(); + require(host->step().ready, "source host before freeze"); + const auto before = host->state()->snapshot(); + const auto host_request = request_for(); + take(host->prepare_migration_freeze(destination_head, *destination_objects, host_request), + "host source freeze"); + require(!host->status().ready && host->status().migration_frozen && + !host->advance_unattended(64) && host->state()->snapshot() == before, + "frozen host advanced or originated current output"); + host->drain(); + host = make_host(); + require(!host->step().ready && host->status().migration_frozen && + host->state()->snapshot() == before && !host->authority().has_value(), + "replacement host implicitly aborted or ignored its freeze"); + take(host->abort_migration_freeze(host_request.request), "replacement host explicit abort"); + require(host->step().ready && !host->status().migration_frozen && + host->state()->snapshot() == before, + "host failed to resume the identical source after confirmed abort"); + auto locker = connection(true); + take(locker->query("BEGIN", {}, 5s), "host unknown freeze lock"); + take( + locker->query( + "SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE", + {}, 5s), + "hold host freeze reply"); + const auto uncertain_request = request_for(); + const auto uncertain = + host->prepare_migration_freeze(destination_head, *destination_objects, uncertain_request); + require(!uncertain && !host->status().ready, "host originated during uncertain freeze"); + take(locker->query("ROLLBACK", {}, 5s), "release host freeze reply"); + for (unsigned attempt = 0; attempt < 200 && !host->status().migration_frozen; ++attempt) { + host->step(); + std::this_thread::sleep_for(25ms); + } + require(host->status().migration_frozen && !host->status().ready && + host->state()->snapshot() == before, + "host chose different biology after unknown preparation"); + take(locker->query("BEGIN", {}, 5s), "host unknown abort lock"); + take( + locker->query( + "SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE", + {}, 5s), + "hold host abort reply"); + const auto unknown_abort = host->abort_migration_freeze(uncertain_request.request); + require(!unknown_abort && !host->status().ready, "unconfirmed abort resumed the source"); + take(locker->query("ROLLBACK", {}, 5s), "release host abort reply"); + for (unsigned attempt = 0; attempt < 200 && !host->step().ready; ++attempt) + std::this_thread::sleep_for(25ms); + require(host->status().ready && !host->status().migration_frozen && + host->state()->snapshot() == before, + "host failed exact ambiguous-abort reconciliation"); + host->drain(); + f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), + "return freeze fixture authority"); + f.head = take( + materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s), + "resume frozen population as raw owner"); + f.raw = take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s), + "source after frozen profile materialization") + .state; + take(release_region_lease(*f.db, destination_key, destination_lease, 5s), + "release freeze destination"); +} + void host_ownership_and_admission(Fixture& f) { take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture host authority"); const auto make_host = [&](bool enabled) { @@ -886,6 +1258,7 @@ void run_population_publication_tests() { missing_and_corrupt_objects(fixture); isolation_and_reverse(fixture); boundary_upgrades(fixture); + migration_freezes(fixture); host_ownership_and_admission(fixture); std::puts("population publication, exact recovery, process loss and fencing passed"); } diff --git a/tests/region_migration_tests.cpp b/tests/region_migration_tests.cpp index bb22b2a..75001f8 100644 --- a/tests/region_migration_tests.cpp +++ b/tests/region_migration_tests.cpp @@ -212,6 +212,7 @@ void upgrade_across_migration_and_barrier() { " -v migration_sha_44=$(s 0044_exact_population_publication.sql)" " -v migration_sha_45=$(s 0045_owned_population_intervals.sql)" " -v migration_sha_46=$(s 0046_boundary_accounting_upgrade.sql)" + " -v migration_sha_47=$(s 0047_cellular_migration_freeze.sql)" " -v runtime_role=evo_runtime_test -f migrations/apply.sql" " > /dev/null 2>&1") != 0); CHECK(history(*owner_db) == "1,2,3,4,5,6,7"); @@ -229,7 +230,7 @@ void upgrade_across_migration_and_barrier() { "tools/migrate-postgres.sh\" > /dev/null") == 0); CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24," - "25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46"); + "25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47"); CHECK(scalar(*owner_db, "SELECT (c.condeferrable AND c.condeferred AND " "c.confrelid = 'evolution.region_commits'::regclass)::text " "FROM pg_catalog.pg_constraint c WHERE c.conrelid = " diff --git a/tools/ci/postgres-check.sh b/tools/ci/postgres-check.sh index cf7b845..b306edb 100644 --- a/tools/ci/postgres-check.sh +++ b/tools/ci/postgres-check.sh @@ -129,7 +129,7 @@ PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" -f "$root/migrations/0001_region_bootstrap.sql" > /dev/null test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1 PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46 +test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47 # v2 heads have no payload binding: v2->v3 fails closed and leaves history at v2. v2_sha="$(sha256sum "$root/migrations/0002_region_fencing.sql")" v2_sha="${v2_sha%% *}" @@ -150,7 +150,7 @@ PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '01000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32), 'hex'))" > /dev/null PGDATABASE=v3_commits bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 46:1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46 +test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 47:1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47 test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) FROM evolution.region_commits")" = 1 # Review P1: a v4 database holding a published (possibly terminal) checkpoint has # no durable decisions for it. The v5 upgrade must refuse before changing anything. @@ -204,7 +204,7 @@ PGDATABASE=v6_populated psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '02000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32) || repeat('00', 80), 'hex'))" > /dev/null PGDATABASE=v6_populated bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46 +test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT count(*) FROM evolution.region_commits c, LATERAL evolution.parse_checkpoint_manifest(c.manifest) m WHERE m.commit_sequence = 0 AND m.predecessor_commit IS NULL")" = 1 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(conname, ',') FROM pg_constraint WHERE conrelid = 'evolution.region_heads'::regclass AND contype = 'c' AND pg_get_constraintdef(oid) LIKE '%end_tick%'")" = region_heads_published_tick phase permission-audit @@ -257,9 +257,9 @@ expect_audit_failure 'REVOKE EXECUTE ON FUNCTION evolution.claim_account_control bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again. phase checksum-drift # Unknown future versions and checksum drift must not be repaired silently. -psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=47 WHERE version=46' +psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=48 WHERE version=47' if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi -psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=46 WHERE version=47" +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=47 WHERE version=48" psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=14" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=13" diff --git a/tools/migrate-postgres.sh b/tools/migrate-postgres.sh index ffcf6f1..a900957 100644 --- a/tools/migrate-postgres.sh +++ b/tools/migrate-postgres.sh @@ -104,6 +104,8 @@ digest_45="$(sha256sum "$root/migrations/0045_owned_population_intervals.sql")" digest_45="${digest_45%% *}" digest_46="$(sha256sum "$root/migrations/0046_boundary_accounting_upgrade.sql")" digest_46="${digest_46%% *}" +digest_47="$(sha256sum "$root/migrations/0047_cellular_migration_freeze.sql")" +digest_47="${digest_47%% *}" psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_2=$digest_2" --set="migration_sha_3=$digest_3" \ --set="migration_sha_4=$digest_4" --set="migration_sha_5=$digest_5" \ @@ -143,5 +145,6 @@ psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_44=$digest_44" \ --set="migration_sha_45=$digest_45" \ --set="migration_sha_46=$digest_46" \ + --set="migration_sha_47=$digest_47" \ --set="runtime_role=$runtime_role" \ --file="$root/migrations/apply.sql"