#include "evo/persistence/account_control_store.hpp" #include "evo/persistence/cellular_census_snapshot.hpp" #include "evo/persistence/cellular_history_store.hpp" #include "evo/persistence/population_checkpoint_store.hpp" #include "evo/persistence/population_interval_store.hpp" #include "evo/persistence/sha256.hpp" #include "evo/server/cellular_region_host.hpp" #include "evo/server/lifecycle/cellular_census_set.hpp" #include "fixtures/cellular_ecology.hpp" #include "fixtures/postgres_relay.hpp" #include #include #include #include #include #include #include #include #include #include #include #include void run_population_publication_tests(); namespace { using namespace evo; using namespace evo::server; using namespace evo::persistence; using namespace evo::fixture; using namespace std::chrono_literals; template T take(std::expected value, const char* where) { if (!value) { std::string message{where}; if constexpr (requires { value.error().cause.sqlstate; }) message += " SQLSTATE=" + std::string(value.error().cause.sqlstate.data(), 5); if constexpr (requires { value.error().store.cause.sqlstate; }) message += " SQLSTATE=" + std::string(value.error().store.cause.sqlstate.data(), 5); throw std::runtime_error(message); } if constexpr (!std::is_void_v) return std::move(*value); } template T take(std::optional value, const char* where) { if (!value.has_value()) throw std::runtime_error(where); return std::move(*value); } std::string environment(const char* name) { const auto* value = std::getenv(name); require(value != nullptr, "missing private test environment"); return value; } std::unique_ptr connection(bool owner = false) { return take(Postgres::connect({environment("PGHOST"), {}, environment("PGPORT"), environment("PGDATABASE"), environment(owner ? "PGUSER" : "EVO_PG_RUNTIME_USER"), {}, {}}, 5s), "population connection"); } Uuid identity(unsigned number) { Uuid id{}; id[0] = 0xDA; id[1] = 0x12; id[14] = static_cast(number >> 8U); id[15] = static_cast(number); return id; } std::string hexadecimal(std::span bytes, bool prefix = true) { std::string out = prefix ? "\\x" : ""; constexpr char digits[] = "0123456789abcdef"; for (const auto b : bytes) { out.push_back(digits[b >> 4U]); out.push_back(digits[b & 15U]); } return out; } struct Fixture { RegionKey key{30201, 1}; std::unique_ptr owner = connection(true), db = connection(); std::unique_ptr objects; LeaseGrant lease; Sha256Digest build{}; CommittedCheckpoint head; std::unique_ptr raw; std::unique_ptr population; unsigned next_id = 1; Uuid id() { return identity(next_id++); } CellularBatchHeader header(std::uint64_t count) const { return {key.world, key.region, 1, head.head.commit, head.head.digest, head.head.end_tick, head.head.end_tick + count}; } }; // Deny actual reads by the unprivileged fixture user, then restore permissions // even if a runtime requirement throws. This never changes immutable bytes. class UnreadableObject { public: explicit UnreadableObject(std::filesystem::path path) : path_(std::move(path)), permissions_(std::filesystem::status(path_).permissions()) { std::filesystem::permissions(path_, std::filesystem::perms::none); } ~UnreadableObject() { std::error_code error; std::filesystem::permissions(path_, permissions_, error); } UnreadableObject(const UnreadableObject&) = delete; UnreadableObject& operator=(const UnreadableObject&) = delete; private: std::filesystem::path path_; std::filesystem::perms permissions_; }; std::uint16_t migration_backend_port() { const auto port = environment("EVO_MIGRATION_TCP_PORT"); unsigned value{}; const auto [end, error] = std::from_chars(port.data(), port.data() + port.size(), value); require(error == std::errc{} && end == port.data() + port.size() && value > 0 && value <= 65535, "invalid private migration TLS port"); return static_cast(value); } PgOptions migration_tls_options(const test::PostgresRelay& relay) { return {"localhost", "127.0.0.1", relay.port(), environment("PGDATABASE"), environment("EVO_PG_RUNTIME_USER"), {}, environment("EVO_MIGRATION_CA")}; } std::unique_ptr migration_tls_connection(const test::PostgresRelay& relay) { auto db = take(Postgres::connect(migration_tls_options(relay), 2s), "migration verify-full TLS connection"); take(db->query("SELECT 1", {}, 2s), "migration TLS warmup"); return db; } void migration_transport_commit(Fixture& f, const CellularMigrationCommitPrepared& prepared, RegionKey sk, RegionKey dk, const LeaseGrant& sl, const LeaseGrant& dl, const RegionHead& source_before, const RegionHead& destination_before) { { test::PostgresRelay relay(migration_backend_port()); auto wrong = migration_tls_options(relay); wrong.host = "untrusted.evolution.test"; require(!Postgres::connect(wrong, 1s), "relay bypassed TLS hostname verification"); } { test::PostgresRelay relay(migration_backend_port()); auto db = migration_tls_connection(relay); relay.drop_requests(true); const auto lost = retry_cellular_migration_commit(*db, sl, dl, prepared, 300ms); require( !lost && lost.error().kind == PublicationError::unknown && lost.error().prepared.source_manifest == prepared.source_manifest && lost.error().prepared.destination_manifest == prepared.destination_manifest && relay.dropped_requests() > 0 && relay.healthy() && !db->transaction_idle() && take(load_region_checkpoint(*f.db, sk, 5s), "lost request source head").head == source_before && take(load_region_checkpoint(*f.db, dk, 5s), "lost request destination head").head == destination_before, "a dropped TLS request changed ownership or lost the exact pair"); } { test::PostgresRelay relay(migration_backend_port()); auto db = migration_tls_connection(relay); relay.drop_responses(true); const auto lost = retry_cellular_migration_commit(*db, sl, dl, prepared, 500ms); require( !lost && lost.error().kind == PublicationError::unknown && lost.error().prepared.source_manifest == prepared.source_manifest && lost.error().prepared.destination_manifest == prepared.destination_manifest && relay.dropped_responses() > 0 && relay.healthy() && !db->transaction_idle() && take(load_region_checkpoint(*f.db, sk, 5s), "lost reply source head").manifest == prepared.source_manifest && take(load_region_checkpoint(*f.db, dk, 5s), "lost reply destination head") .manifest == prepared.destination_manifest, "lost TLS receipt was treated as rollback or committed only one head"); } std::puts("migration TLS request/reply loss and hostname verification passed"); } void funded_source(Fixture& f) { f.build.fill(0xDA); const std::array build{hexadecimal(f.build)}; take(f.owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES($1::bytea) " "ON CONFLICT DO NOTHING", build, 5s), "retain test build"); take(f.owner->query("INSERT INTO evolution.life_allocators VALUES(30201,100)", {}, 5s), "test allocator"); f.objects = take( FilesystemObjectStore::open(environment("PGHOST"), f.key, max_cellular_population_bytes), "population object namespace"); f.raw = opening(30001, true, 4, false); f.head = take(provision_cellular_region(*f.owner, *f.objects, f.key, 1, *f.raw, f.id(), f.build, 5s), "empty cellular genesis"); f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "population test lease"); CellularBatch birth{f.header(1), {{}}}; for (std::uint8_t index = 1; index <= 4; ++index) { Uuid account{}; account.back() = index; const auto control = take(claim_account_control(*f.db, f.key, f.lease, account, 1, index, 60000ms, 5s), "founder account control"); const auto reserved = take( reserve_primitive_life(*f.db, f.key, f.lease, account, control.generation, 1, 1, 5s), "funded founder reservation"); birth.ticks.front().births.push_back({account, 1, control.generation, reserved.identity}); } f.raw = take(replay_cellular_batch(*f.raw, birth), "funded founder replay"); f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, birth, *f.raw, f.id(), f.build, 5s), "funded founder publication") .committed; while (f.raw->next_bud_sequence() == 1 && f.raw->tick() < 20000) { CellularBatch batch{f.header(1), {}}; for (unsigned tick = 0; tick < 64; ++tick) { auto intents = take(wildlife_intents(*f.raw, f.raw->tick() + 1), "funded source wildlife"); for (const auto& life : f.raw->lives()) { if (life.account == LifeAccount{}) continue; CellularLifeIntent intent{life.identity.life, {}}; intent.intent.bud = (f.raw->tick() + life.identity.life) % 32 == 0; intents.push_back(intent); } std::ranges::sort(intents, {}, &CellularLifeIntent::life_id); take(f.raw->advance_tick(intents), "funded source biology"); batch.ticks.push_back({intents, {}, {}}); if (f.raw->next_bud_sequence() > 1) break; } batch.header.end_tick = f.raw->tick(); f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, batch, *f.raw, f.id(), f.build, 5s), "funded source batch") .committed; take(renew_region_lease(*f.db, f.key, f.lease, 60000ms, 5s), "source lease renewal"); } require(std::ranges::any_of(f.raw->lives(), [](const auto& life) { return life.account == LifeAccount{}; }), "durable fixture has no funded player-derived wildlife"); require(!migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, {true, 1200, 1800}, 5s), "population entry bypassed vulnerable controlled life"); const auto suspended_at = f.raw->tick() + cellular_linger_ticks + 1; while (f.raw->tick() < suspended_at) { auto h = f.header(std::min(std::uint64_t{64}, suspended_at - f.raw->tick())); auto prefix = take(unattended_population_inputs(*f.raw, h), "raw linger inputs"); f.raw = take(replay_cellular_batch(*f.raw, prefix), "raw linger biology"); f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, prefix, *f.raw, f.id(), f.build, 5s), "raw linger publication") .committed; std::vector endings; for (const auto& corpse : f.raw->corpses()) if (corpse.parent_life == 0) endings.push_back(corpse.life_id); if (!endings.empty()) take(end_cellular_lives(*f.db, f.key, f.lease, f.head.head.commit, endings, 5s), "raw linger endings"); } require(population_interval_safe(*f.raw), "population source retained a vulnerable controlled life"); } CellularPopulationExecution unattended(Fixture& f) { auto working = restore(f.population->state()); CellularBatch batch{f.header(64), {}}; for (unsigned tick = 0; tick < 64; ++tick) { auto intents = take(wildlife_intents(*working, working->tick() + 1), "unattended wildlife"); for (const auto& life : working->lives()) { if (life.account == LifeAccount{}) continue; CellularLifeIntent frozen{life.identity.life, {}}; frozen.intent.frozen = true; intents.push_back(frozen); } std::ranges::sort(intents, {}, &CellularLifeIntent::life_id); take(working->advance_tick(intents), "unattended fixture step"); batch.ticks.push_back({intents, {}, {}}); } return take(prepare_population_batch(*f.population, std::move(batch)), "population recorded interval"); } void recover(Fixture& f) { const std::array builds{f.build}; auto db = connection(); auto recovered = take(recover_population_checkpoint(*db, *f.objects, f.key, f.lease, builds, 5s), "population durable recovery"); f.head = std::move(recovered.committed); f.population = std::move(recovered.state); } void conversions_and_retries(Fixture& f) { const auto biology = f.raw->snapshot(); Uuid account{}; account.back() = 1; const auto input = encode_cellular_mutation_input({account, 9991, f.head.head.end_tick + 1, {100, 1, 1, 1}}); take(f.db->query("BEGIN", {}, 5s), "reservation guard transaction"); take(reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 9991, input, 5s), "reserve pending player choice"); const auto blocked = migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, {true, 1200, 1800}, 5s); require(!blocked && std::string(blocked.error().store.cause.sqlstate.data(), 5) == "EVL18", "population entry stranded a reserved player choice"); take(f.db->query("ROLLBACK", {}, 5s), "rollback guard fixture"); f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, {true, 1200, 1800}, 5s), "initial population transition"); recover(f); require(f.population->state().snapshot() == biology && f.population->members().empty(), "transition changed biology"); const auto refused = reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 9991, input, 5s); require(!refused && std::string(refused.error().cause.sqlstate.data(), 5) == "EVL06", "population profile admitted a player mutation reservation"); const auto wildlife = std::ranges::find_if(f.population->state().lives(), [](const auto& life) { return life.account == LifeAccount{}; }); require(wildlife != f.population->state().lives().end(), "missing durable descendant"); const auto genome = wildlife->identity.genome; auto conversion = take(prepare_population_conversion(*f.population, f.header(0), {genome, true}), "collapse sample"); auto bad = conversion.batch; ++bad.boundaries.front().sequence; require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, bad, *conversion.state, f.id(), f.build, 5s), "altered sample published"); const auto commit = f.id(); f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, conversion.batch, *conversion.state, commit, f.build, 5s), "durable collapse"); const PopulationPreparedCommit prepared{ f.key, commit, f.head.manifest, take(encode_population_batch(conversion.batch), "prepared trace")}; require(take(retry_population_prepared(*f.db, prepared, 5s), "idempotent retry").head == f.head.head, "retry advanced head"); require(take(resolve_population_prepared(*f.db, prepared, 5s), "read-only resolve").status == Resolution::committed, "commit unresolved"); auto future_batch = conversion.batch; future_batch.format_version = 2; auto future = prepared; future.trace = take(encode_population_batch(future_batch), "future trace fixture"); auto future_manifest = take(decode_manifest(future.manifest), "future manifest fixture"); future_manifest.batch.trace_bytes = future.trace.size(); future_manifest.batch.trace_sha256 = take(sha256(future.trace), "future trace digest"); future.manifest = take(encode_manifest(future_manifest), "future manifest encoding"); auto rejected_future = retry_population_prepared(*f.db, future, 5s); require(!rejected_future && rejected_future.error().kind == PublicationError::candidate, "bundle 9 prepared retry admitted population trace 2"); require(!resolve_population_prepared(*f.db, future, 5s), "bundle 9 prepared resolve admitted population trace 2"); auto rejected_sql = publish_population_checkpoint(*f.db, future.manifest, future.trace, 5s); require(!rejected_sql && rejected_sql.error().kind == LeaseStoreError::invalid, "schema 45 did not reject the unsupported trace format before commit lookup"); const auto current_manifest = take(decode_manifest(f.head.manifest), "upgrade source manifest"); const auto boundary_payload = take(cellular_boundary_upgrade_payload(conversion.state->snapshot(), current_manifest), "portable boundary upgrade source"); const auto boundary_manifest = take( cellular_boundary_upgrade_manifest( current_manifest, f.head.head.digest, f.id(), f.lease, f.build, boundary_payload.size(), take(sha256(boundary_payload), "boundary payload digest")), "portable boundary upgrade manifest"); const PopulationPreparedCommit boundary{ f.key, boundary_manifest.commit, take(encode_manifest(boundary_manifest), "boundary candidate encoding"), {}}; auto rejected_boundary = retry_population_prepared(*f.db, boundary, 5s); require(!rejected_boundary && rejected_boundary.error().kind == PublicationError::rejected && take(resolve_population_prepared(*f.db, boundary, 5s), "upgrade through wrong journal resolution") .status == Resolution::absent, "accounting upgrade bypassed its dedicated version journal"); auto rejected_upgrade = publish_population_profile_transition(*f.db, boundary.manifest, 5s); require(!rejected_upgrade, "representation publisher admitted accounting upgrade without dedicated evidence"); require(!publish_region_checkpoint(*f.db, boundary.manifest, {}, 5s), "generic publisher admitted boundary upgrade"); require(take(load_region_checkpoint(*f.db, f.key, 5s), "head after future rejection").head == f.head.head, "future trace changed the retained durable head"); auto changed = prepared; auto changed_manifest = take(decode_manifest(changed.manifest), "conflict manifest"); changed_manifest.payload_sha256[0] ^= 1; changed.manifest = take(encode_manifest(changed_manifest), "conflict encoding"); require(take(resolve_population_prepared(*f.db, changed, 5s), "conflicting resolve").status == Resolution::conflict, "changed commit was accepted"); require(!publish_region_checkpoint(*f.db, f.head.manifest, {}, 5s), "generic publisher bypassed profile gate"); const auto expected = conversion.state->snapshot(); recover(f); require(f.population->snapshot() == expected && !f.population->members().empty(), "committed sample not recovered"); require(f.population->state().snapshot() == biology, "collapse healed/aged/moved biology"); for (unsigned oscillation = 0; oscillation < 8; ++oscillation) { const bool collapse = oscillation % 2 != 0; auto next = take(prepare_population_conversion(*f.population, f.header(0), {genome, collapse}), "conversion oscillation"); f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, next.batch, *next.state, f.id(), f.build, 5s), "publish oscillation"); recover(f); require(f.population->state().snapshot() == biology, "durable conversion changed exact state"); } } void process_loss(Fixture& f, int point) { auto next = unattended(f); take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, next.batch.header.end_tick, 5s), "admit process-loss interval"); const auto predecessor = f.head.head; const auto expected = next.state->snapshot(); const auto commit = f.id(); int channel[2]{}; require(::pipe(channel) == 0, "process acknowledgement pipe"); const auto child = ::fork(); require(child >= 0, "population fork"); if (child == 0) { ::close(channel[0]); try { auto db = connection(); PublishHooks hooks; if (point == 0) hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); }; const auto published = publish_population_batch(*db, *f.objects, f.lease, f.head, next.batch, *next.state, commit, f.build, 5s, hooks); if (!published) ::_exit(20); if (point == 2) { const char ack = 'A'; if (::write(channel[1], &ack, 1) != 1) ::_exit(21); } ::kill(::getpid(), SIGKILL); } catch (...) { ::_exit(22); } ::_exit(23); } ::close(channel[1]); char ack{}; const auto acknowledged = ::read(channel[0], &ack, 1); ::close(channel[0]); int status{}; require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) && WTERMSIG(status) == SIGKILL, "child missed population crash boundary"); require((point == 2 && acknowledged == 1 && ack == 'A') || (point != 2 && acknowledged == 0), "unexpected permanent acknowledgement"); recover(f); const auto pending = take(load_population_interval(*f.db, f.key, f.lease, 5s), "recover process interval"); require(pending.has_value() == (point == 0), "process loss changed interval settlement"); if (point == 0) require(f.head.head == predecessor, "uncommitted process published a head"); else require(f.head.head.commit == commit && f.population->snapshot() == expected, "committed process loss changed state"); } void ambiguous_publication(Fixture& f, bool terminate) { auto next = unattended(f); take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, next.batch.header.end_tick, 5s), "admit ambiguous interval"); const auto expected = next.state->snapshot(); auto locker = connection(true); take(locker->query("BEGIN", {}, 5s), "population lease row transaction"); const std::array key{std::to_string(f.key.world), std::to_string(f.key.region)}; take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=$1::evolution.u64 " "AND region_id=$2::evolution.u64 FOR UPDATE", key, 5s), "hold population publication"); auto timed = connection(); auto unknown = publish_population_batch(*timed, *f.objects, f.lease, f.head, next.batch, *next.state, f.id(), f.build, 300ms); require(!unknown && unknown.error().kind == PublicationError::unknown, "population publication failed to retain ambiguous request"); if (terminate) { const std::array user{environment("EVO_PG_RUNTIME_USER")}; take(locker->query("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE " "usename=$1 AND wait_event_type='Lock'", user, 5s), "terminate uncommitted population writer"); } take(locker->query("ROLLBACK", {}, 5s), "release population publication"); auto resolver = connection(); bool resolved = false; for (unsigned attempt = 0; attempt < 200; ++attempt) { auto result = take(resolve_population_prepared(*resolver, unknown.error().prepared, 5s), "resolve ambiguous population publication"); if (result.status == (terminate ? Resolution::absent : Resolution::committed)) { resolved = true; break; } std::this_thread::sleep_for(25ms); } require(resolved, "ambiguous population publication did not resolve"); f.head = take(retry_population_prepared(*resolver, unknown.error().prepared, 5s), "retry exact population request"); recover(f); require(f.population->snapshot() == expected, "ambiguous publication changed retained sample"); } void interval_contract(Fixture& f) { require(!take(load_population_interval(*f.db, f.key, f.lease, 5s), "empty interval journal") .has_value(), "settled source retained pending work"); auto next = unattended(f); const auto source = f.head.head; const auto admitted = take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s), "durable interval admission"); require(take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s), "duplicate interval admission") == admitted, "admission retry changed boundary"); auto changed = admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 63, 5s); require(!changed && changed.error().kind == LeaseStoreError::commit_conflict, "source admitted two different intervals"); const auto expected = next.state->snapshot(); require(!f.population->members().empty(), "interval fixture lost its exact cohort"); const auto genome = f.population->members().front().genome; auto conversion = take(prepare_population_conversion(*f.population, f.header(0), {genome, false}), "conversion during admitted work"); require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, conversion.batch, *conversion.state, f.id(), f.build, 5s), "conversion skipped an admitted interval"); require( !materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s), "profile exit skipped an admitted interval"); auto shortened_header = f.header(63); auto shortened = take(prepare_population_batch( *f.population, take(unattended_population_inputs(f.population->state(), shortened_header), "shortened input")), "shortened interval"); require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, shortened.batch, *shortened.state, f.id(), f.build, 5s), "publication partially consumed an interval"); take(release_region_lease(*f.db, f.key, f.lease, 5s), "release interval owner"); const auto epoch = f.lease.epoch; f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "replace interval owner"); require(f.lease.epoch > epoch && take(take(load_population_interval(*f.db, f.key, f.lease, 5s), "new owner interval load"), "retained admission") == admitted, "replacement owner lost admitted work"); f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, next.batch, *next.state, f.id(), f.build, 5s), "settle exact admitted interval"); recover(f); require(f.population->snapshot() == expected && !take(load_population_interval(*f.db, f.key, f.lease, 5s), "settled interval load") .has_value(), "settlement lost work or retained spendable pending state"); require(take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s), "settled admission retry") == admitted && !take(load_population_interval(*f.db, f.key, f.lease, 5s), "settled retry pending") .has_value(), "old admission retry reopened settled work"); require(!admit_population_interval(*f.db, f.key, f.lease, f.head.head, f.head.head.end_tick + 65, 5s), "oversized interval admitted"); const auto denied = f.db->query( "SELECT * FROM evolution.publish_population_checkpoint_v44(''::bytea,''::bytea)", {}, 5s); require(!denied && std::string(denied.error().sqlstate.data(), 5) == "42501", "runtime bypassed interval guard"); } void isolation_and_reverse(Fixture& f) { auto interval = unattended(f); const auto old_lease = f.lease; take(release_region_lease(*f.db, f.key, f.lease, 5s), "release old authority"); auto new_db = connection(); f.lease = take(acquire_region_lease(*new_db, f.key, f.build, 60000ms, 5s), "new population authority"); require(f.lease.epoch > old_lease.epoch, "authority epoch did not advance"); const auto old_head = f.head.head; auto stale = publish_population_batch(*f.db, *f.objects, old_lease, f.head, interval.batch, *interval.state, f.id(), f.build, 5s); require(!stale && stale.error().store.kind == LeaseStoreError::fenced, "stale population writer published"); recover(f); require(f.head.head == old_head, "fenced writer changed head"); const auto expected = f.population->state().snapshot(); f.head = take( materialize_population_profile(*new_db, *f.objects, f.lease, f.head, f.id(), f.build, 5s), "return to cellular profile"); const std::array builds{f.build}; auto recovered = take(recover_cellular_checkpoint(*new_db, *f.objects, f.key, f.lease, builds, 5s), "cellular recovery after materialization"); require(recovered.state->snapshot() == expected && recovered.manifest.profile == CheckpointProfile::cellular_lives_v1, "reverse transition changed biology or time"); f.raw = std::move(recovered.state); } void missing_and_corrupt_objects(Fixture& f) { const auto manifest = take(decode_manifest(f.head.manifest), "object test manifest"); const auto path = f.objects->directory() / hexadecimal(manifest.payload_sha256, false); const auto backup = path.string() + ".population-test"; const std::array builds{f.build}; std::filesystem::rename(path, backup); auto missing = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s); std::filesystem::rename(backup, path); require(!missing && missing.error().kind == RecoveryError::missing_payload, "missing population payload did not fail closed"); std::filesystem::rename(path, backup); { std::ofstream corrupt(path, std::ios::binary); corrupt << 'X'; } auto corrupt = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s); std::filesystem::remove(path); std::filesystem::rename(backup, path); require(!corrupt && corrupt.error().kind == RecoveryError::corrupt_payload, "corrupt population payload did not fail closed"); recover(f); } void history_pages(Fixture& f) { const std::array builds{f.build}; const auto upper = f.head; const auto biology = f.raw->snapshot(); const auto read = [&](const RegionHead& watermark, std::uint64_t first, std::size_t limit) { return load_cellular_census_history(*f.db, *f.objects, f.key, f.lease, watermark, first, limit, builds, 5s); }; auto first = take(read(upper.head, 0, 2), "history first page"); require(first.entries.size() == 2 && first.next_sequence == 2 && first.entries.front().committed.head.sequence == 0 && first.entries.front().census.rows().empty(), "history lost genesis or changed sequence pagination"); // Append two distinct no-tick cuts between requests; the old watermark stays pinned. const auto population = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, {false, 1200, 1800}, 5s), "history population cut"); f.head = take( materialize_population_profile(*f.db, *f.objects, f.lease, population, f.id(), f.build, 5s), "history raw cut"); std::uint64_t next = take(first.next_sequence, "history continuation cursor"); while (next <= upper.head.sequence) { auto page = take(read(upper.head, next, 2), "pinned historical page"); require(page.watermark == upper.head && !page.entries.empty() && page.entries.size() <= 2, "history changed its pinned upper cut"); for (const auto& entry : page.entries) { const auto stored = take(load_region_commit(*f.db, f.key, next, 5s), "history oracle cut"); const auto manifest = take(decode_manifest(stored.manifest), "history oracle manifest"); auto state = take(restore_cellular_payload( take(f.objects->get({manifest.payload_sha256, manifest.payload_bytes}), "history oracle payload"), manifest), "history oracle biology"); const auto oracle = take(CellularCensus::observe(*state), "history oracle census"); require(entry.committed.head == stored.head && entry.committed.manifest == stored.manifest && std::ranges::equal(entry.census.rows(), oracle.rows()) && entry.census.boundary() == oracle.boundary(), "history rebound old census to a newer or different committed cut"); ++next; } require( page.next_sequence == (next <= upper.head.sequence ? std::optional{next} : std::nullopt), "history cursor skipped or invented a commit"); } const auto transitions = take(read(f.head.head, upper.head.sequence + 1, 8), "no-tick history"); require(transitions.entries.size() == 2 && !transitions.next_sequence && transitions.entries[0].committed.head == population.head && transitions.entries[1].committed.head == f.head.head && transitions.entries[0].census.boundary().population_format == 1 && transitions.entries[1].census.boundary().population_format == 0 && population.head.end_tick == upper.head.end_tick && f.head.head.end_tick == upper.head.end_tick, "history collapsed distinct no-tick events or lost representation identity"); const auto repeat = take(read(upper.head, 0, 2), "repeat pinned history"); require(repeat.entries[0].committed.head == first.entries[0].committed.head && repeat.entries[1].committed.head == first.entries[1].committed.head && f.raw->snapshot() == biology && take(load_region_checkpoint(*f.db, f.key, 5s), "read-only history head").head == f.head.head, "history read changed biology, head or event identity"); for (const auto limit : {std::size_t{0}, max_cellular_history_page + 1, std::numeric_limits::max()}) require(!read(upper.head, 0, limit) && read(upper.head, 0, limit).error().kind == CellularHistoryError::invalid_request, "unbounded history request admitted"); require(!read(upper.head, upper.head.sequence + 1, 1), "history read beyond pinned cut"); auto wrong = upper.head; wrong.digest.back() ^= 1; require(!read(wrong, 0, 1) && read(wrong, 0, 1).error().kind == CellularHistoryError::cut_mismatch, "history accepted a false watermark digest"); wrong = upper.head; ++wrong.end_tick; require(!read(wrong, 0, 1), "history accepted a false watermark tick"); wrong = upper.head; ++wrong.frame; require(!read(wrong, 0, 1), "history merged foreign frames"); require(!load_cellular_census_history(*f.db, *f.objects, {f.key.world, 999}, f.lease, upper.head, 0, 1, builds, 5s) && !load_cellular_census_history(*f.db, *f.objects, f.key, f.lease, upper.head, 0, 1, {}, 5s), "foreign region or unretained build became an empty history"); const auto birth = take(load_region_commit(*f.db, f.key, 1, 5s), "historical birth cut"); const auto manifest = take(decode_manifest(birth.manifest), "historical birth manifest"); const auto path = f.objects->directory() / hexadecimal(manifest.payload_sha256, false); const auto saved = path.string() + ".history-test"; const auto bytes = take(f.objects->get({manifest.payload_sha256, manifest.payload_bytes}), "historical birth bytes"); std::filesystem::rename(path, saved); const auto missing = read(upper.head, 0, 2); // Genesis succeeded; second cut must fail the page. std::filesystem::rename(saved, path); require(!missing && missing.error().kind == CellularHistoryError::recovery, "missing old payload returned a partial page or zero census"); { std::ofstream file(path, std::ios::binary); file << 'X'; require(file.good(), "corrupt old history payload fixture"); } const auto corrupt = read(upper.head, 0, 2); { std::ofstream file(path, std::ios::binary); file.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); require(file.good(), "restore old history payload fixture"); } require(!corrupt && corrupt.error().kind == CellularHistoryError::recovery, "corrupt historical object returned a census"); const auto trace_path = f.objects->directory() / hexadecimal(manifest.batch.trace_sha256, false); const auto trace_saved = trace_path.string() + ".history-test"; std::filesystem::rename(trace_path, trace_saved); const auto absent_trace = read(upper.head, 1, 1); std::filesystem::rename(trace_saved, trace_path); require(!absent_trace && absent_trace.error().recovery.kind == RecoveryError::tail, "history trusted a payload without its replay evidence"); take(f.db->query("BEGIN", {}, 5s), "uncommitted history transaction"); const auto transactional = read(upper.head, 0, 1); require(!f.db->transaction_idle() && !transactional && transactional.error().kind == CellularHistoryError::invalid_request, "history admitted a caller-owned transaction"); const auto orphan = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, {false, 1200, 1800}, 5s), "uncommitted history cut"); auto observer = connection(); const auto invisible = load_cellular_census_history( *observer, *f.objects, f.key, f.lease, orphan.head, orphan.head.sequence, 1, builds, 5s); take(f.db->query("ROLLBACK", {}, 5s), "rollback uncommitted history cut"); require(f.db->transaction_idle() && !invisible && take(load_region_checkpoint(*f.db, f.key, 5s), "orphan history head").head == f.head.head, "uncommitted object/ghost became a durable history event"); auto old_lease = f.lease; --old_lease.epoch; const auto stale = load_cellular_census_history(*f.db, *f.objects, f.key, old_lease, upper.head, 1, 1, builds, 5s); require(!stale && stale.error().recovery.store.kind == LeaseStoreError::fenced && take(read(upper.head, 0, 2), "current history reader").entries[1].committed.head == birth.head, "historical evidence ignored a stale fence or rewrote original writers"); take(f.db->query("BEGIN READ ONLY", {}, 5s), "read-only history transaction"); const auto readonly = read(upper.head, 0, 1); take(f.db->query("ROLLBACK", {}, 5s), "close read-only history transaction"); require(!readonly && readonly.error().kind == CellularHistoryError::invalid_request, "history transaction requirement changed for read-only callers"); std::puts("committed census history pagination, replay, transaction and failure guards passed"); } void snapshot_transport_faults(Fixture& f) { if (!std::getenv("EVO_MIGRATION_TCP_PORT")) return; const std::array builds{f.build}; const std::array requests{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()}}; for (unsigned mode = 0; mode < 4; ++mode) { test::PostgresRelay relay(migration_backend_port()); auto db = migration_tls_connection(relay); const bool at_completion = mode >= 2; const bool lost_request = mode % 2 == 0; const auto partition = [&] { if (lost_request) relay.drop_requests(true); else relay.drop_responses(true); }; if (!at_completion) partition(); const CellularCensusSnapshotHooks hook{[&](std::size_t) { if (at_completion) partition(); }}; const auto lost = load_cellular_census_snapshot(*db, f.key.world, requests, builds, 300ms, hook); require(!lost && lost.error().kind == CellularSnapshotError::transaction && lost.error().transaction.kind == PgError::unknown && !db->transaction_idle() && relay.healthy() && (lost_request ? relay.dropped_requests() : relay.dropped_responses()) > 0 && take(load_region_checkpoint(*f.db, f.key, 5s), "head after lost capture I/O") .head == f.head.head, "lost snapshot BEGIN/COMMIT request/reply returned evidence or kept a session"); } std::puts("snapshot TLS BEGIN/COMMIT request/reply loss fails closed"); } void snapshot_guards(Fixture& f) { const std::array builds{f.build}; const std::array requests{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()}}; const auto read = [&] { return load_cellular_census_snapshot(*f.db, f.key.world, requests, builds, 5s); }; const auto before = f.head; const auto first = take(read(), "single-region captured snapshot"); const auto second = take(read(), "repeat captured snapshot"); require(first.cuts.size() == 1 && first.cuts[0].head == before.head && first.cuts[0].manifest == before.manifest && second.cuts[0].head == first.cuts[0].head && std::ranges::equal( first.census.regions()[0].rows(), take(CellularCensus::observe(*f.raw), "snapshot direct oracle").rows()) && f.db->transaction_idle(), "captured snapshot lost its exact committed cut/biology or retained a transaction"); const auto invalid = [&](auto region_requests, std::uint64_t world = 30201, std::chrono::milliseconds timeout = 5s) { const auto result = load_cellular_census_snapshot(*f.db, world, region_requests, builds, timeout); require(!result && result.error().kind == CellularSnapshotError::invalid_request && f.db->transaction_idle(), "invalid snapshot request started a transaction or returned a census"); }; invalid(std::vector{}); invalid(requests, 0); invalid(requests, f.key.world, 0ms); invalid(requests, f.key.world, 30001ms); auto bad = requests; bad[0].objects = nullptr; invalid(bad); bad = requests; bad[0].region = 0; invalid(bad); bad = requests; bad[0].lease.frame = 0; invalid(bad); invalid(std::array{requests[0], requests[0]}); auto higher = requests[0]; ++higher.region; invalid(std::array{higher, requests[0]}); invalid(std::vector(max_cellular_census_regions + 1, requests[0])); higher.region = 999'999; const auto missing = load_cellular_census_snapshot(*f.db, f.key.world, std::array{requests[0], higher}, builds, 5s); require(!missing && missing.error().kind == CellularSnapshotError::history && missing.error().region == higher.region && f.db->transaction_idle(), "missing region became partial coverage or left capture open"); for (const auto* begin : {"BEGIN", "BEGIN READ ONLY"}) { take(f.db->query(begin, {}, 5s), "caller snapshot transaction"); if (std::string_view{begin} == "BEGIN") take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, {false, 1200, 1800}, 5s), "pending caller snapshot publication"); const auto refused = read(); require(!refused && refused.error().kind == CellularSnapshotError::invalid_request && !f.db->transaction_idle(), "snapshot took ownership of a caller transaction"); take(f.db->query("ROLLBACK", {}, 5s), "caller snapshot rollback"); } const auto manifest = take(decode_manifest(before.manifest), "snapshot retained payload"); const auto path = f.objects->directory() / hexadecimal(manifest.payload_sha256, false); const auto saved = path.string() + ".snapshot-test"; std::filesystem::rename(path, saved); const auto missing_object = read(); std::filesystem::rename(saved, path); require(!missing_object && missing_object.error().kind == CellularSnapshotError::history && f.db->transaction_idle(), "missing replay payload became a snapshot or left capture open"); { const UnreadableObject denied{path}; const auto unreadable = read(); require(!unreadable && unreadable.error().kind == CellularSnapshotError::history && f.db->transaction_idle(), "unreadable captured payload became a census"); } auto stale = requests; --stale[0].lease.epoch; const auto fenced = load_cellular_census_snapshot(*f.db, f.key.world, stale, builds, 5s); require(!fenced && fenced.error().history.recovery.store.kind == LeaseStoreError::fenced, "snapshot ignored stale evidence authority"); const std::array foreign_builds{}; require(!load_cellular_census_snapshot(*f.db, f.key.world, requests, foreign_builds, 5s), "snapshot accepted unretained build evidence"); auto closed = connection(); CellularCensusSnapshotHooks close_hook{[&](std::size_t) { closed->disconnect(); }}; const auto lost = load_cellular_census_snapshot(*closed, f.key.world, requests, builds, 5s, close_hook); closed->disconnect(); const auto closed_query = closed->query("SELECT 1", {}, 5s); require(!lost && lost.error().kind == CellularSnapshotError::transaction && !closed->transaction_idle() && !closed_query && closed_query.error().kind == PgError::closed, "failed capture completion reused a dead session or returned a partial census"); auto throwing = connection(); bool threw = false; bool hook_called = false; try { const CellularCensusSnapshotHooks hook{[&](std::size_t) { hook_called = true; throw std::runtime_error("capture hook test exception"); }}; const auto result = load_cellular_census_snapshot(*throwing, f.key.world, requests, builds, 5s, hook); require(!result, "throwing capture hook returned a snapshot"); } catch (const std::runtime_error&) { threw = hook_called; } const auto after_throw = throwing->query("SELECT 1", {}, 5s); require(threw && !after_throw && after_throw.error().kind == PgError::closed && !throwing->transaction_idle() && take(load_region_checkpoint(*f.db, f.key, 5s), "head after snapshot guards").head == before.head, "capture exception retained its connection/transaction or changed a head"); std::puts("consistent census snapshot guards and connection cleanup passed"); snapshot_transport_faults(f); } void boundary_upgrades(Fixture& f) { const std::array builds{f.build}; const auto original = f.head; const auto expected_raw = take(f.raw->migration_upgrade_candidate(), "raw upgrade oracle")->snapshot(); // Rehearse the raw upgrade, recovery and ordinary trace-3 publication in one // transaction, then roll it back to retain a real bundle-9 source for profile 5. take(f.db->query("BEGIN", {}, 5s), "raw upgrade rollback rehearsal"); auto raw_head = take( migrate_cellular_boundary_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s), "raw accounting upgrade"); auto raw = take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s), "raw accounting recovery"); require(raw.state->snapshot() == expected_raw && raw.manifest.cellular_versions == cellular_v10_versions && raw_head.head.end_tick == original.head.end_tick, "raw accounting upgrade changed biology or time"); const CellularBatchHeader h{f.key.world, f.key.region, 1, raw_head.head.commit, raw_head.head.digest, raw_head.head.end_tick, raw_head.head.end_tick + 1}; const auto inputs = take(unattended_population_inputs(*raw.state, h), "upgraded raw inputs"); auto advanced = take(replay_cellular_batch(*raw.state, inputs), "upgraded raw oracle"); take(publish_cellular_batch(*f.db, *f.objects, f.lease, raw_head, inputs, *advanced, f.id(), f.build, 5s), "upgraded raw ordinary publication"); require(take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s), "upgraded raw ordinary recovery") .state->snapshot() == advanced->snapshot(), "raw accounting continuation differs"); take(f.db->query("ROLLBACK", {}, 5s), "roll back raw upgrade rehearsal"); require(take(load_region_checkpoint(*f.db, f.key, 5s), "raw rollback head").head == original.head, "rolled-back upgrade moved the durable head"); f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, {true, 1200, 1800}, 5s), "population accounting source entry"); recover(f); const auto member = std::ranges::find_if(f.population->state().lives(), [](const auto& life) { return life.account == LifeAccount{}; }); require(member != f.population->state().lives().end(), "accounting source has no wildlife"); auto collapsed = take( prepare_population_conversion(*f.population, f.header(0), {member->identity.genome, true}), "accounting source cohort"); f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, collapsed.batch, *collapsed.state, f.id(), f.build, 5s), "accounting source membership publication"); recover(f); require(!f.population->members().empty() && f.population->conversion_sequence() != 0, "accounting source lost cohort history"); auto interval = unattended(f); take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, interval.batch.header.end_tick, 5s), "upgrade pending interval"); auto pending = migrate_cellular_boundary_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s); require(!pending && std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18", "accounting upgrade discarded admitted work"); f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, interval.batch, *interval.state, f.id(), f.build, 5s), "settle pre-upgrade work"); recover(f); const auto before = f.head; const auto source = take(decode_manifest(before.manifest), "accounting source manifest"); const auto upgraded_population = take(f.population->migration_upgrade_candidate(), "cohort upgrade oracle"); const auto expected = upgraded_population->snapshot(); const auto altered = take(CellularPopulation::create(restore(upgraded_population->state()), {false, 1200, 1800}), "valid unequal upgrade target"); require(altered->state().current_total() == upgraded_population->state().current_total() && altered->snapshot() != expected, "unequal upgrade fixture must retain biological totals"); const auto altered_object = take(f.objects->put(altered->snapshot()), "unequal upgrade object"); const auto altered_manifest = take( cellular_boundary_upgrade_manifest(source, before.head.digest, f.id(), f.lease, f.build, altered_object.bytes, altered_object.sha256), "unequal upgrade manifest"); take(f.db->query("BEGIN", {}, 5s), "unequal upgrade recovery rehearsal"); take(publish_cellular_profile_migration( *f.db, take(encode_manifest(altered_manifest), "unequal upgrade encoding"), 5s), "metadata-only unequal upgrade publication"); const auto unequal = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s); take(f.db->query("ROLLBACK", {}, 5s), "undo unequal upgrade rehearsal"); require(!unequal && unequal.error().kind == RecoveryError::history && take(load_region_checkpoint(*f.db, f.key, 5s), "head after unequal upgrade").head == before.head, "upgrade recovery compared only aggregate stock or rewrote the retained source"); const auto child = ::fork(); require(child >= 0, "upgrade pre-transaction fork"); if (child == 0) { try { auto db = connection(); PublishHooks hooks; hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); }; [[maybe_unused]] const auto result = migrate_cellular_boundary_profile( *db, *f.objects, f.lease, before, identity(60000), f.build, 5s, hooks); } catch (...) { ::_exit(31); } ::_exit(32); } int status{}; require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) && WTERMSIG(status) == SIGKILL, "upgrade child missed pre-transaction kill"); recover(f); require(f.head.head == before.head, "unpublished upgrade became authoritative"); auto locker = connection(true); take(locker->query("BEGIN", {}, 5s), "accounting ambiguity lock transaction"); take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 " "AND region_id=1 FOR UPDATE", {}, 5s), "hold accounting reply"); auto timed = connection(); auto unknown = migrate_cellular_boundary_profile(*timed, *f.objects, f.lease, before, f.id(), f.build, 300ms); require(!unknown && unknown.error().kind == PublicationError::unknown && !unknown.error().prepared.manifest.empty(), "uncertain accounting upgrade lost its exact request"); take(locker->query("ROLLBACK", {}, 5s), "release accounting reply"); auto resolver = connection(); f.head = take(retry_cellular_profile_migration(*resolver, unknown.error().prepared, 5s), "retry exact accounting upgrade"); require(take(retry_cellular_profile_migration(*resolver, unknown.error().prepared, 5s), "duplicate accounting upgrade") .head == f.head.head, "duplicate accounting upgrade advanced head"); recover(f); require(f.population->snapshot() == expected && f.head.head.end_tick == before.head.end_tick && take(decode_manifest(f.head.manifest), "upgraded population manifest") .cellular_versions == cellular_v10_versions, "accounting upgrade changed biology, cohort history or time"); auto changed = unknown.error().prepared; auto changed_manifest = take(decode_manifest(changed.manifest), "changed accounting manifest"); changed_manifest.payload_sha256[0] ^= 1; changed.manifest = take(encode_manifest(changed_manifest), "changed accounting bytes"); auto conflict = retry_cellular_profile_migration(*resolver, changed, 5s); require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict, "accounting retry accepted changed bytes"); require( !migrate_cellular_boundary_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s), "accounting upgrade applied twice"); const auto path = f.objects->directory() / hexadecimal(source.payload_sha256, false); const auto backup = path.string() + ".upgrade-test"; std::filesystem::rename(path, backup); const auto missing = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s); std::filesystem::rename(backup, path); require(!missing && missing.error().kind == RecoveryError::tail, "accounting recovery trusted aggregate stock without its retained source"); missing_and_corrupt_objects(f); const auto stale = f.lease; take(release_region_lease(*f.db, f.key, f.lease, 5s), "release accounting writer"); f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "replace accounting writer"); auto stale_retry = retry_cellular_profile_migration(*f.db, unknown.error().prepared, 5s); require(!stale_retry && stale_retry.error().store.kind == LeaseStoreError::fenced && !recover_population_checkpoint(*f.db, *f.objects, f.key, stale, builds, 5s), "old accounting receipt granted current authority"); recover(f); require(f.population->snapshot() == expected, "new epoch changed accounting upgrade"); auto future_interval = unattended(f); auto old_trace = future_interval.batch; old_trace.format_version = 1; const auto old_bytes = take(encode_population_batch(old_trace), "wrong old trace fixture"); const auto upgraded_manifest = take(decode_manifest(f.head.manifest), "new accounting source"); auto wrong_trace_manifest = take(population_successor_manifest( upgraded_manifest, f.head.head.digest, f.id(), f.lease, old_trace.header.end_tick, f.build, future_interval.state->snapshot().size(), take(sha256(future_interval.state->snapshot()), "new payload SHA"), {old_bytes.size(), take(sha256(old_bytes), "old trace SHA"), 0, {}}), "wrong old trace manifest fixture"); const PopulationPreparedCommit wrong_trace{ f.key, wrong_trace_manifest.commit, take(encode_manifest(wrong_trace_manifest), "wrong old trace manifest"), old_bytes}; auto refused_old = retry_population_prepared(*f.db, wrong_trace, 5s); auto refused_old_sql = publish_population_checkpoint(*f.db, wrong_trace.manifest, old_bytes, 5s); require(!refused_old && refused_old.error().kind == PublicationError::candidate && !refused_old_sql && refused_old_sql.error().kind == LeaseStoreError::invalid && take(load_region_checkpoint(*f.db, f.key, 5s), "head after old trace").head == f.head.head, "bundle 10 reinterpreted retained population trace 1"); // Exercise the existing publication/process-loss matrix with batch 2 as well. process_loss(f, 0); process_loss(f, 1); process_loss(f, 2); isolation_and_reverse(f); require(f.raw->format_version() == cellular_lives_migration_format, "reverse transition downgraded accounting owner"); } void migration_owner_partitions(Fixture& f, RegionKey destination_key, FilesystemObjectStore& destination_objects, LeaseGrant& destination_lease, CommittedCheckpoint& destination_head, std::uint64_t genome) { if (!std::getenv("EVO_MIGRATION_TCP_PORT")) return; const std::array builds{f.build}; for (unsigned lost_source = 0; lost_source < 2; ++lost_source) { test::PostgresRelay source_relay(migration_backend_port()), destination_relay(migration_backend_port()); const auto make_host = [&](RegionKey key, const test::PostgresRelay& relay) { auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), key, max_cellular_population_bytes), "partition owner objects"); CellularHostConfig config{key, 1, {}, f.build, {f.build}}; config.lease = {3000ms, 400ms, 300ms, 150ms, 250ms}; const auto options = migration_tls_options(relay); return take(CellularRegionHost::create( config, [options] { return Postgres::connect(options, 500ms); }, std::move(objects)), "partition owner host"); }; take(release_region_lease(*f.db, f.key, f.lease, 5s), "release source to TLS owner"); take(release_region_lease(*f.db, destination_key, destination_lease, 5s), "release destination to TLS owner"); auto source_host = make_host(f.key, source_relay), destination_host = make_host(destination_key, destination_relay); require(source_host->step().ready && destination_host->step().ready, "TLS owners before freeze"); const auto sl = take(source_host->authority(), "TLS source fence"), dl = take(destination_host->authority(), "TLS destination fence"); const auto source_bytes = source_host->state()->snapshot(), destination_bytes = destination_host->state()->snapshot(); const auto source_before = f.head, destination_before = destination_head; const CellularMigrationRequest request{ f.id(), {f.key.region, f.head.head.commit, f.id(), f.head.head.digest}, {destination_key.region, destination_head.head.commit, f.id(), destination_head.head.digest}, genome}; source_relay.drop_responses(true); const auto unknown_prepare = source_host->prepare_migration_freeze(destination_head, destination_objects, request); require(!unknown_prepare && unknown_prepare.error().publication == PublicationError::unknown && !source_host->status().ready && source_relay.dropped_responses() > 0, "lost preparation receipt did not pause source ownership"); const auto frozen = take(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, sl, 5s), "independent committed preparation observation"), "missing preparation after TLS reply loss"); source_relay.drop_responses(false); const auto reconcile_source_until = LeaseClock::now() + 2s; while (!source_host->status().migration_frozen && LeaseClock::now() < reconcile_source_until) { source_host->step(); destination_host->step(); std::this_thread::sleep_for(25ms); } require(source_host->status().migration_frozen && !source_host->status().ready, "source lost the exact prepared request after TLS reconnect"); destination_relay.drop_responses(true); const auto unknown_stage = destination_host->stage_migration(f.head, *f.objects, frozen.trace); require(!unknown_stage && unknown_stage.error().publication == PublicationError::unknown && !destination_host->status().ready && destination_relay.dropped_responses() > 0 && take(load_cellular_migration_stage(*f.db, destination_objects, destination_key, dl, 5s), "independent committed stage observation") .has_value(), "lost stage receipt activated the ghost or lost retained work"); destination_relay.drop_responses(false); const auto reconcile_destination_until = LeaseClock::now() + 2s; while (!destination_host->status().migration_frozen && LeaseClock::now() < reconcile_destination_until) { source_host->step(); destination_host->step(); std::this_thread::sleep_for(25ms); } require(source_host->status().migration_frozen && destination_host->status().migration_frozen, "TLS owners failed exact stage reconciliation"); auto& isolated = lost_source ? source_host : destination_host; auto& healthy = lost_source ? destination_host : source_host; auto& relay = lost_source ? source_relay : destination_relay; const auto isolated_key = lost_source ? f.key : destination_key; auto& replacement = lost_source ? f.lease : destination_lease; const auto healthy_key = lost_source ? destination_key : f.key; auto& healthy_lease = lost_source ? destination_lease : f.lease; relay.drop_requests(true); const auto expire_until = LeaseClock::now() + 3500ms; while (LeaseClock::now() < expire_until) { isolated->step(); healthy->step(); std::this_thread::sleep_for(25ms); } require(!isolated->status().ready && !isolated->authority().has_value() && !isolated->advance_unattended(64) && relay.dropped_requests() > 0 && healthy->status().phase == HostPhase::running && healthy->status().epoch == (lost_source ? dl.epoch : sl.epoch) && healthy->status().migration_frozen, "isolated owner retained authority or healthy peer released a frozen ghost"); replacement = take(acquire_region_lease(*f.db, isolated_key, f.build, 60000ms, 5s), "replacement after actual TLS partition expiry"); require(replacement.epoch > (lost_source ? sl.epoch : dl.epoch), "replacement reused partitioned epoch"); if (lost_source) require(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, replacement, 5s), "replacement source retains freeze") .has_value(), "source lease expiry implicitly aborted"); else require(take(load_cellular_migration_stage(*f.db, destination_objects, destination_key, replacement, 5s), "replacement destination retains stage") .has_value(), "destination lease expiry activated ghost"); const auto stale = publish_cellular_migration(*f.db, *f.objects, destination_objects, sl, dl, f.head, destination_head, frozen.trace, f.build, 5s); require(!stale && stale.error().store.kind == LeaseStoreError::fenced && take(load_region_checkpoint(*f.db, f.key, 5s), "partition source head").head == source_before.head && take(load_region_checkpoint(*f.db, destination_key, 5s), "partition destination head") .head == destination_before.head, "stale partitioned fence published a migration half"); healthy->step(); const auto current_source = lost_source ? replacement : sl; if (lost_source) { test::PostgresRelay abort_relay(migration_backend_port()); auto db = migration_tls_connection(abort_relay); abort_relay.drop_responses(true); const auto unknown = abort_cellular_migration_freeze(*db, f.key, current_source, request.request, 300ms); require(!unknown && unknown.error().kind == LeaseStoreError::unknown && abort_relay.dropped_responses() > 0 && abort_relay.healthy(), "lost replacement abort receipt was not unknown"); take(abort_cellular_migration_freeze(*f.db, f.key, current_source, request.request, 5s), "exact replacement abort reconciliation"); } else { source_relay.drop_responses(true); const auto unknown = source_host->abort_migration_freeze(request.request); require(!unknown && !source_host->status().ready && source_relay.dropped_responses() > 0, "lost owning-source abort receipt resumed the region"); source_relay.drop_responses(false); } require(!take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, current_source, 5s), "independent abort source observation") .has_value(), "lost abort receipt reopened source stock"); const auto resume_until = LeaseClock::now() + 2s; while (!healthy->step().ready && LeaseClock::now() < resume_until) std::this_thread::sleep_for(25ms); require(healthy->status().ready && healthy->state()->snapshot() == (lost_source ? destination_bytes : source_bytes), "healthy owner resumed different stock after abort"); relay.drop_requests(false); isolated->step(); require(!isolated->status().ready && !isolated->advance_unattended(64), "healed stale owner resumed despite a replacement fence"); const auto isolated_manifest = take(decode_manifest(lost_source ? f.head.manifest : destination_head.manifest), "isolated head profile"); const auto& objects = lost_source ? *f.objects : destination_objects; std::vector recovered; if (isolated_manifest.profile == CheckpointProfile::cellular_population_v1) recovered = take(recover_population_checkpoint(*f.db, objects, isolated_key, replacement, builds, 5s), "replacement population replay") .state->state() .snapshot(); else recovered = take(recover_cellular_checkpoint(*f.db, objects, isolated_key, replacement, builds, 5s), "replacement raw replay") .state->snapshot(); require(recovered == (lost_source ? source_bytes : destination_bytes) && source_relay.healthy() && destination_relay.healthy(), "replacement or relay lost exact stock"); source_host->drain(); destination_host->drain(); healthy_lease = take(acquire_region_lease(*f.db, healthy_key, f.build, 60000ms, 5s), "return healthy TLS owner authority"); } std::puts("migration independent TLS owner partitions, expiry, replacement and heal passed"); } void migration_snapshot_race(Fixture& f, RegionKey destination_key, FilesystemObjectStore& destination_objects, const LeaseGrant& destination_lease, CommittedCheckpoint& destination_head, std::uint64_t genome) { const std::array builds{f.build}; const std::array requests{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()}, CellularCensusRegionRequest{destination_key.region, destination_lease, &destination_objects}}; auto reader = connection(); for (const bool reverse : {false, true}) { auto& sh = reverse ? destination_head : f.head; auto& dh = reverse ? f.head : destination_head; auto& so = reverse ? destination_objects : *f.objects; auto& dst = reverse ? *f.objects : destination_objects; const auto& sl = reverse ? destination_lease : f.lease; const auto& dl = reverse ? f.lease : destination_lease; const auto sk = reverse ? destination_key : f.key; const auto dk = reverse ? f.key : destination_key; const auto before = take(load_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s), "before snapshot migration race"); const CellularMigrationRequest request{f.id(), {sk.region, sh.head.commit, f.id(), sh.head.digest}, {dk.region, dh.head.commit, f.id(), dh.head.digest}, genome}; const auto frozen = take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s), "snapshot race source freeze"); take(stage_cellular_migration(*f.db, so, dst, dl, sh, dh, frozen.trace, 5s), "snapshot race destination stage"); std::optional published; const CellularCensusSnapshotHooks hook{[&](std::size_t index) { if (index != 0) return; const auto settings = take(reader->query("SELECT current_setting('transaction_isolation'), " "current_setting('transaction_read_only')", {}, 5s), "capture transaction properties"); require(settings.size() == 1 && settings[0].size() == 2 && take(settings[0][0], "capture isolation") == "repeatable read" && take(settings[0][1], "capture access") == "on", "capture did not own a repeatable read read-only snapshot"); auto publisher = connection(); published = take(publish_cellular_migration(*publisher, so, dst, sl, dl, sh, dh, frozen.trace, f.build, 5s), "publish real pair between snapshot head reads"); }}; const auto raced = take(load_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s, hook), "coherent census across a concurrent pair"); const auto committed = take(std::move(published), "pair was not published in snapshot hook"); sh = committed.source; dh = committed.destination; require(reader->transaction_idle() && raced.cuts.size() == 2 && before.cuts.size() == 2, "raced capture retained its transaction or lost coverage"); for (std::size_t i = 0; i < before.cuts.size(); ++i) require(raced.cuts[i].head == before.cuts[i].head && raced.cuts[i].manifest == before.cuts[i].manifest && std::ranges::equal(raced.census.regions()[i].rows(), before.census.regions()[i].rows()), "atomic pair split a captured cut or changed historical replay"); const auto after = take(load_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s), "snapshot after pair publication"); require(after.cuts[0].head == f.head.head && after.cuts[1].head == destination_head.head && after.cuts[0].head.sequence == before.cuts[0].head.sequence + 1 && after.cuts[1].head.sequence == before.cuts[1].head.sequence + 1, "later snapshot omitted one committed migration endpoint"); for (const auto scope : {CellularCensusScope::exact_genome, CellularCensusScope::descendants}) { const auto initial = take(before.census.query(genome, scope), "before pair lineage"); const auto captured = take(raced.census.query(genome, scope), "captured pair lineage"); const auto settled = take(after.census.query(genome, scope), "settled pair lineage"); require(initial == captured && initial.resident == settled.resident, "snapshot migration race lost/duplicated living count/body/reserve"); } } std::puts("consistent raw/cohort snapshots across atomic migration commits passed"); } void migration_publication(Fixture& f, const RegionKey destination_key, FilesystemObjectStore& destination_objects, LeaseGrant& destination_lease, CommittedCheckpoint& destination_head) { const std::array builds{f.build}; const auto wildlife = std::ranges::find_if( f.raw->lives(), [](const auto& life) { return life.account == LifeAccount{}; }); require(wildlife != f.raw->lives().end(), "paired publication needs real wildlife"); const auto genome = wildlife->identity.genome; const auto make_host = [&](RegionKey key) { auto objects = take( FilesystemObjectStore::open(environment("PGHOST"), key, max_cellular_population_bytes), "paired host objects"); CellularHostConfig config{key, 1, {}, f.build, {f.build}}; config.lease.commit_timeout = 300ms; return take( CellularRegionHost::create( config, [] { return std::expected, PgFailure>{connection()}; }, std::move(objects)), "paired host"); }; for (unsigned representation = 0; representation < 2; ++representation) { if (representation == 0) { f.head = take(materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s), "paired raw source"); destination_head = take(materialize_population_profile(*f.db, destination_objects, destination_lease, destination_head, f.id(), f.build, 5s), "paired raw destination"); } else { f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, {true, 1200, 1800}, 5s), "paired population source"); destination_head = take(migrate_cellular_to_population( *f.db, destination_objects, destination_lease, destination_head, f.id(), f.build, {true, 1200, 1800}, 5s), "paired population destination"); } std::optional outgoing_history, incoming_history; for (unsigned direction = 0; direction < 2; ++direction) { auto& sh = direction == 0 ? f.head : destination_head; auto& dh = direction == 0 ? destination_head : f.head; auto& so = direction == 0 ? *f.objects : destination_objects; auto& dst = direction == 0 ? destination_objects : *f.objects; auto& sl = direction == 0 ? f.lease : destination_lease; auto& dl = direction == 0 ? destination_lease : f.lease; const auto sk = direction == 0 ? f.key : destination_key; const auto dk = direction == 0 ? destination_key : f.key; take(release_region_lease(*f.db, sk, sl, 5s), "release paired source to host"); take(release_region_lease(*f.db, dk, dl, 5s), "release paired destination to host"); auto source_host = make_host(sk), destination_host = make_host(dk); require(source_host->step().ready && destination_host->step().ready, "paired hosts before preparation"); sl = take(source_host->authority(), "current source authority before pause"); dl = take(destination_host->authority(), "current destination authority before pause"); const auto source_before = sh, destination_before = dh; const auto s = take(decode_manifest(sh.manifest), "paired predecessor source"); const auto d = take(decode_manifest(dh.manifest), "paired predecessor destination"); const auto sb = take(so.get({s.payload_sha256, s.payload_bytes}), "paired source bytes"); const auto dbb = take(dst.get({d.payload_sha256, d.payload_bytes}), "paired destination bytes"); const CellularMigrationRequest request{ f.id(), {sk.region, sh.head.commit, f.id(), sh.head.digest}, {dk.region, dh.head.commit, f.id(), dh.head.digest}, genome}; ObjectStoreFaults source_faults, destination_faults; auto failing_source = take(FilesystemObjectStore::open(environment("PGHOST"), sk, max_cellular_population_bytes, &source_faults), "migration source sync fault store"); auto failing_destination = take( FilesystemObjectStore::open(environment("PGHOST"), dk, max_cellular_population_bytes, &destination_faults), "migration destination sync fault store"); source_faults.fail_directory_syncs = 1; const auto unsynced_preparation = prepare_cellular_migration_freeze( *f.db, *failing_source, dst, sl, sh, dh, request, 5s); require(!unsynced_preparation && unsynced_preparation.error().kind == PublicationError::upload && unsynced_preparation.error().object == ObjectError::io && source_faults.fail_directory_syncs == 0 && !take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s), "unsynced preparation pointer"), "failed capsule sync durably prepared a migration"); take(source_host->prepare_migration_freeze(dh, dst, request), "owning paired source freeze"); const auto frozen = take(take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s), "paired source evidence"), "missing paired freeze"); // Preparation alone cannot authorize either result. require(!publish_cellular_migration(*f.db, so, dst, sl, dl, sh, dh, frozen.trace, f.build, 5s), "unstaged migration published"); destination_faults.fail_directory_syncs = 1; const auto unsynced_stage = stage_cellular_migration(*f.db, so, *failing_destination, dl, sh, dh, frozen.trace, 5s); require(!unsynced_stage && unsynced_stage.error().kind == PublicationError::upload && unsynced_stage.error().object == ObjectError::io && destination_faults.fail_directory_syncs == 0 && !take(load_cellular_migration_stage(*f.db, dst, dk, dl, 5s), "unsynced destination stage pointer") && take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s), "source freeze after stage sync failure") .has_value(), "failed stage capsule sync activated a ghost or released source ownership"); take(destination_host->stage_migration(sh, so, frozen.trace), "owning paired stage"); for (unsigned endpoint = 0; endpoint < 2; ++endpoint) { auto& faults = endpoint == 0 ? source_faults : destination_faults; faults.fail_directory_syncs = 1; const auto unsynced_pair = publish_cellular_migration(*f.db, *failing_source, *failing_destination, sl, dl, sh, dh, frozen.trace, f.build, 5s); require( !unsynced_pair && unsynced_pair.error().kind == PublicationError::upload && unsynced_pair.error().object == ObjectError::io && faults.fail_directory_syncs == 0 && take(load_region_checkpoint(*f.db, sk, 5s), "unsynced source head").head == source_before.head && take(load_region_checkpoint(*f.db, dk, 5s), "unsynced destination head") .head == destination_before.head && take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s), "unsynced source freeze") .has_value() && take(load_cellular_migration_stage(*f.db, dst, dk, dl, 5s), "unsynced destination stage") .has_value(), "failed result sync published a half or released retained work"); } std::vector expected_source, expected_destination; if (representation == 0) { auto a = take(restore_cellular_payload(sb, s), "paired raw source oracle"); auto b = take(restore_cellular_payload(dbb, d), "paired raw destination oracle"); auto pair = take(replay_migration_trace(*a, *b, request, frozen.trace), "paired raw oracle"); expected_source = pair.source->snapshot(); expected_destination = pair.destination->snapshot(); } else { auto a = take(restore_population_payload(sb, s), "paired population source oracle"); auto b = take(restore_population_payload(dbb, d), "paired population destination oracle"); auto pair = take(replay_migration_trace(*a, *b, request, frozen.trace), "paired population oracle"); expected_source = pair.source->snapshot(); expected_destination = pair.destination->snapshot(); } // Fail specifically on the SECOND result insert, after the first and receipt. take(f.owner->query( "CREATE FUNCTION evolution.test_refuse_second_migration() RETURNS trigger " "LANGUAGE plpgsql AS $$ BEGIN IF NEW.commit_id='" + hexadecimal(request.destination.commit, false) + "'::uuid THEN RAISE EXCEPTION 'injected second insert' USING " "ERRCODE='EVL06'; END IF; RETURN NEW; END $$", {}, 5s), "second insert fault function"); take(f.owner->query( "CREATE TRIGGER test_refuse_second_migration BEFORE INSERT ON " "evolution.region_commits " "FOR EACH ROW EXECUTE FUNCTION evolution.test_refuse_second_migration()", {}, 5s), "second insert fault trigger"); const auto rejected = publish_cellular_migration(*f.db, so, dst, sl, dl, sh, dh, frozen.trace, f.build, 5s); take(f.owner->query( "DROP TRIGGER test_refuse_second_migration ON evolution.region_commits", {}, 5s), "remove second insert fault"); take(f.owner->query("DROP FUNCTION evolution.test_refuse_second_migration()", {}, 5s), "remove fault function"); require( !rejected && rejected.error().store.kind == LeaseStoreError::unsupported && take(load_region_checkpoint(*f.db, sk, 5s), "rollback source head").head == source_before.head && take(load_region_checkpoint(*f.db, dk, 5s), "rollback destination head").head == destination_before.head && take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s), "rollback source pause") .has_value() && take(load_cellular_migration_stage(*f.db, dst, dk, dl, 5s), "rollback destination pause") .has_value(), "failed second publication left half a migration or released a pause"); const auto prepared = rejected.error().prepared; take(f.db->query("BEGIN", {}, 5s), "false paired biology transaction"); auto false_pair = prepared; auto false_source = take(decode_manifest(false_pair.source_manifest), "false source manifest"); false_source.payload_sha256 = s.payload_sha256; false_source.payload_bytes = s.payload_bytes; false_pair.source_manifest = take(encode_manifest(false_source), "false source canonical bytes"); take(retry_cellular_migration_commit(*f.db, sl, dl, false_pair, 5s), "metadata-only false pair"); const auto false_recovery = verify_cellular_migration_checkpoint( *f.db, so, sl, take(load_region_checkpoint(*f.db, sk, 5s), "false paired head"), sb, 5s); const auto false_head = take(load_region_checkpoint(*f.db, sk, 5s), "false history cut"); const auto false_history = [&]() -> std::expected { if (representation == 0) { auto recovered = recover_cellular_commit( *f.db, so, sk, false_head.head.sequence, sl, builds, 5s); if (!recovered) return std::unexpected(recovered.error()); } else { auto recovered = recover_population_commit( *f.db, so, sk, false_head.head.sequence, sl, builds, 5s); if (!recovered) return std::unexpected(recovered.error()); } return {}; }(); take(f.db->query("ROLLBACK", {}, 5s), "rollback false paired biology fixture"); require(!false_recovery && false_recovery.error().kind == RecoveryError::history && !false_history && false_history.error().kind == RecoveryError::history, "paired recovery accepted a valid old object as the migrated biology"); const std::array generic{hexadecimal(prepared.source_manifest)}; require(!f.db->query("SELECT * FROM evolution.publish_region_checkpoint($1::bytea)", generic, 5s), "generic publisher admitted a migration half"); const auto child = ::fork(); require(child >= 0, "pre-transaction paired fork"); if (child == 0) { try { auto db = connection(); PublishHooks hooks{[] { ::kill(::getpid(), SIGKILL); }}; if (!publish_cellular_migration(*db, so, dst, sl, dl, sh, dh, frozen.trace, f.build, 5s, hooks)) ::_exit(52); } catch (...) { ::_exit(51); } ::_exit(52); } int status{}; require( ::waitpid(child, &status, 0) == child && WIFSIGNALED(status) && WTERMSIG(status) == SIGKILL && take(load_region_checkpoint(*f.db, sk, 5s), "killed pair source").head == source_before.head && take(load_region_checkpoint(*f.db, dk, 5s), "killed pair destination").head == destination_before.head, "pre-transaction paired death changed a head"); if (direction == 0 && std::getenv("EVO_MIGRATION_TCP_PORT")) migration_transport_commit(f, prepared, sk, dk, sl, dl, source_before.head, destination_before.head); if (direction == 0) { auto locker = connection(true), timed = connection(); take(locker->query("BEGIN", {}, 5s), "pair ambiguity transaction"); take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND " "region_id=" + std::to_string(sk.region) + " FOR UPDATE", {}, 5s), "hold paired reply"); const auto unknown = retry_cellular_migration_commit(*timed, sl, dl, prepared, 300ms); require(!unknown && unknown.error().kind == PublicationError::unknown && unknown.error().prepared.source_manifest == prepared.source_manifest && unknown.error().prepared.destination_manifest == prepared.destination_manifest, "ambiguous pair lost its exact manifests"); take(locker->query("ROLLBACK", {}, 5s), "release paired ambiguity"); } else { const auto committed_child = ::fork(); require(committed_child >= 0, "post-commit paired fork"); if (committed_child == 0) { try { auto db = connection(); if (!retry_cellular_migration_commit(*db, sl, dl, prepared, 5s)) ::_exit(53); ::kill(::getpid(), SIGKILL); } catch (...) { ::_exit(54); } ::_exit(55); } require(::waitpid(committed_child, &status, 0) == committed_child && WIFSIGNALED(status) && WTERMSIG(status) == SIGKILL, "paired publisher failed before post-commit kill"); } const auto committed = take(retry_cellular_migration_commit(*f.db, sl, dl, prepared, 5s), "resolve exact paired commit"); sh = committed.source; dh = committed.destination; require(!abort_cellular_migration_freeze(*f.db, sk, sl, request.request, 5s), "committed migration was aborted"); auto changed = prepared; auto altered = take(decode_manifest(changed.destination_manifest), "conflicting committed pair"); altered.payload_sha256.back() ^= 1; changed.destination_manifest = take(encode_manifest(altered), "conflicting pair bytes"); const auto conflict = retry_cellular_migration_commit(*f.db, sl, dl, changed, 5s); auto stale = sl; --stale.epoch; require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict && !retry_cellular_migration_commit(*f.db, stale, dl, prepared, 5s), "changed pair or stale source authority was accepted"); const auto sm = take(decode_manifest(sh.manifest), "unavailable source result"); const auto dm = take(decode_manifest(dh.manifest), "unavailable destination result"); // Outbound: deny current results. Return: deny retained counterpart // capsules. Both are required before either paused host may resume. const VerifiedObject unavailable_source = direction == 0 ? VerifiedObject{sm.payload_sha256, sm.payload_bytes} : VerifiedObject{d.payload_sha256, d.payload_bytes}; const VerifiedObject unavailable_destination = direction == 0 ? VerifiedObject{dm.payload_sha256, dm.payload_bytes} : VerifiedObject{s.payload_sha256, s.payload_bytes}; { UnreadableObject source_denial(so.directory() / hexadecimal(unavailable_source.sha256, false)); UnreadableObject destination_denial( dst.directory() / hexadecimal(unavailable_destination.sha256, false)); const auto denied_source = so.get(unavailable_source); const auto denied_destination = dst.get(unavailable_destination); require(!denied_source && denied_source.error() == ObjectError::io && !denied_destination && denied_destination.error() == ObjectError::io, "object read denial did not produce actual I/O failures"); const auto source_failures = source_host->status().store_failures; const auto destination_failures = destination_host->status().store_failures; const auto observed = [&] { if (direction == 0) return source_host->status().store_failures > source_failures && destination_host->status().store_failures > destination_failures; return source_host->status().phase == HostPhase::quarantined && destination_host->status().phase == HostPhase::quarantined; }; for (unsigned attempt = 0; attempt < 200 && !observed(); ++attempt) { source_host->step(); destination_host->step(); std::this_thread::sleep_for(10ms); } require( observed() && !source_host->status().ready && !destination_host->status().ready && !source_host->advance_unattended(1) && !destination_host->advance_unattended(1) && take(load_region_checkpoint(*f.db, sk, 5s), "unavailable source head") .head == sh.head && take(load_region_checkpoint(*f.db, dk, 5s), "unavailable destination head") .head == dh.head, "unreadable migration evidence resumed old ownership or changed a head"); } if (direction != 0) { require(!source_host->step().ready && !destination_host->step().ready, "restoring storage silently cleared quarantine"); source_host->drain(); destination_host->drain(); // The quarantined local authorities cannot release themselves. This // test coordinator explicitly retires their still-current DB leases. take(release_region_lease(*f.db, sk, sl, 5s), "retire quarantined source lease"); take(release_region_lease(*f.db, dk, dl, 5s), "retire quarantined destination lease"); source_host = make_host(sk); destination_host = make_host(dk); } for (unsigned attempt = 0; attempt < 200 && (!source_host->step().ready || !destination_host->step().ready); ++attempt) std::this_thread::sleep_for(25ms); require(source_host->status().ready && destination_host->status().ready && take(source_host->committed(), "adopted source").head == sh.head && take(destination_host->committed(), "adopted destination").head == dh.head, "pointer release resumed a stale owner instead of recovering the pair"); const auto old_source_epoch = sl.epoch, old_destination_epoch = dl.epoch; sl = take(source_host->authority(), "restored source object authority"); dl = take(destination_host->authority(), "restored destination object authority"); const bool expected_epochs = direction == 0 ? sl.epoch == old_source_epoch && dl.epoch == old_destination_epoch : sl.epoch > old_source_epoch && dl.epoch > old_destination_epoch; require(expected_epochs && take(retry_cellular_migration_commit(*f.db, sl, dl, prepared, 5s), "restored storage exact paired receipt") .source.head == sh.head, "storage recovery changed a committed pair or reused authority"); const auto check_bytes = [&](const CommittedCheckpoint& h, RegionKey key, const FilesystemObjectStore& objects, const LeaseGrant& lease) { require(take(load_region_checkpoint(*f.db, key, 5s), "current paired recovery head") .head == h.head, "paired recovery selected a different head"); if (representation == 0) return take(recover_cellular_checkpoint(*f.db, objects, key, lease, builds, 5s), "recover raw paired result") .state->snapshot(); return take(recover_population_checkpoint(*f.db, objects, key, lease, builds, 5s), "recover cohort paired result") .state->snapshot(); }; require(check_bytes(sh, sk, so, sl) == expected_source && check_bytes(dh, dk, dst, dl) == expected_destination, "committed biology/member/material differs from the exact pair"); auto source_history = take(load_cellular_census_history(*f.db, so, sk, sl, sh.head, sh.head.sequence, 1, builds, 5s), "paired source history"); auto destination_history = take(load_cellular_census_history(*f.db, dst, dk, dl, dh.head, dh.head.sequence, 1, builds, 5s), "paired destination history"); const auto census_oracle = [&](const auto& bytes, const CommittedCheckpoint& checkpoint) { const auto manifest = take(decode_manifest(checkpoint.manifest), "history pair oracle manifest"); if (representation == 0) return take( CellularCensus::observe(*take(restore_cellular_payload(bytes, manifest), "history raw pair oracle")), "history raw census oracle"); return take( CellularCensus::observe(*take(restore_population_payload(bytes, manifest), "history cohort pair oracle")), "history cohort census oracle"); }; require(source_history.entries.size() == 1 && destination_history.entries.size() == 1 && std::ranges::equal(source_history.entries[0].census.rows(), census_oracle(expected_source, sh).rows()) && std::ranges::equal(destination_history.entries[0].census.rows(), census_oracle(expected_destination, dh).rows()), "historical migration census differs from exact paired biology"); const auto& source_census = source_history.entries[0].census; const auto& destination_census = destination_history.entries[0].census; const auto& boundary = source_census.boundary(); std::array required_regions{sk.region, dk.region}; std::ranges::sort(required_regions); const auto census_set = take(CellularCensusSet::observe( {boundary.world, boundary.frame, boundary.tick, boundary.owner_format, boundary.rules, boundary.quantum_microseconds}, required_regions, std::array{source_census, destination_census}), "committed paired history census set"); // Identity and complete per-region rows survive real raw/cohort // publication, replay and repeated direction changes. require(census_set.regions().size() == 2 && census_set.regions()[0].boundary().region == required_regions[0] && census_set.regions()[1].boundary().region == required_regions[1], "paired history query lost declared region coverage"); for (const auto& row : destination_census.rows()) { const auto combined = take(census_set.query(row.genome, CellularCensusScope::descendants), "committed cross-region lineage query"); const auto local = take(destination_census.query(row.genome, CellularCensusScope::descendants), "committed destination lineage"); const auto other = source_census.query(row.genome, CellularCensusScope::descendants); for (std::size_t kind = 0; kind < local.resident.size(); ++kind) require(combined.resident[kind].count == local.resident[kind].count + (other ? other->resident[kind].count : 0), "committed set duplicated/dropped resident lineage stock"); } if (direction == 0) { outgoing_history = std::move(source_history.entries.front()); incoming_history = std::move(destination_history.entries.front()); } const auto capsule_path = so.directory() / hexadecimal(d.payload_sha256, false); const auto backup = capsule_path.string() + ".paired-test"; std::filesystem::rename(capsule_path, backup); const auto missing = verify_cellular_migration_checkpoint(*f.db, so, sl, sh, expected_source, 5s); const auto missing_history = load_cellular_census_history( *f.db, so, sk, sl, sh.head, sh.head.sequence, 1, builds, 5s); std::filesystem::rename(backup, capsule_path); require(!missing && missing.error().kind == RecoveryError::tail && !missing_history && missing_history.error().recovery.kind == RecoveryError::tail && !verify_cellular_migration_checkpoint(*f.db, so, sl, sh, sb, 5s), "pair recovery ignored missing counterpart evidence or false local biology"); source_host->drain(); destination_host->drain(); source_host = make_host(sk); destination_host = make_host(dk); require(source_host->step().ready && destination_host->step().ready, "replacement paired hosts failed complete recovery"); sl = take(source_host->authority(), "replacement source fence"); dl = take(destination_host->authority(), "replacement destination fence"); require(take(retry_cellular_migration_commit(*f.db, sl, dl, prepared, 5s), "new epoch exact paired retry") .source.head == sh.head, "fresh authority changed a committed retry"); source_host->drain(); destination_host->drain(); sl = take(acquire_region_lease(*f.db, sk, f.build, 60000ms, 5s), "return paired source authority"); dl = take(acquire_region_lease(*f.db, dk, f.build, 60000ms, 5s), "return paired destination authority"); } const auto outgoing = take(std::move(outgoing_history), "retained source historical cut"); const auto incoming = take(std::move(incoming_history), "retained destination historical cut"); const auto old_source = take(load_cellular_census_history(*f.db, *f.objects, f.key, f.lease, f.head.head, outgoing.committed.head.sequence, 1, builds, 5s), "source history after return"); const auto old_destination = take(load_cellular_census_history(*f.db, destination_objects, destination_key, destination_lease, destination_head.head, incoming.committed.head.sequence, 1, builds, 5s), "destination history after return"); require( old_source.entries[0].committed.head == outgoing.committed.head && old_destination.entries[0].committed.head == incoming.committed.head && take(decode_manifest(outgoing.committed.manifest), "historical source writer") .epoch < f.lease.epoch && take(decode_manifest(incoming.committed.manifest), "historical destination writer") .epoch < destination_lease.epoch && std::ranges::equal(old_source.entries[0].census.rows(), outgoing.census.rows()) && std::ranges::equal(old_destination.entries[0].census.rows(), incoming.census.rows()), "return migration rewrote earlier committed ownership observations"); migration_owner_partitions(f, destination_key, destination_objects, destination_lease, destination_head, genome); migration_snapshot_race(f, destination_key, destination_objects, destination_lease, destination_head, genome); } std::puts("migration object sync/read failures, quarantine and exact recovery passed"); // Race the actual two-head transaction against abort on independent connections. const auto prepare_pair = [&](bool reverse) { auto& sh = reverse ? destination_head : f.head; auto& dh = reverse ? f.head : destination_head; auto& so = reverse ? destination_objects : *f.objects; auto& dst = reverse ? *f.objects : destination_objects; auto& sl = reverse ? destination_lease : f.lease; auto& dl = reverse ? f.lease : destination_lease; const auto sk = reverse ? destination_key : f.key; const auto dk = reverse ? f.key : destination_key; CellularMigrationRequest request{f.id(), {sk.region, sh.head.commit, f.id(), sh.head.digest}, {dk.region, dh.head.commit, f.id(), dh.head.digest}, genome}; const auto freeze = take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s), "commit/abort source preparation"); take(stage_cellular_migration(*f.db, so, dst, dl, sh, dh, freeze.trace, 5s), "commit/abort destination stage"); return freeze.trace; }; const auto aborted_trace = prepare_pair(false); take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, aborted_trace.request.request, 5s), "abort wins before paired commit"); const auto after_abort = publish_cellular_migration( *f.db, *f.objects, destination_objects, f.lease, destination_lease, f.head, destination_head, aborted_trace, f.build, 5s); require(!after_abort && after_abort.error().store.kind == LeaseStoreError::commit_conflict, "paired publication reopened an aborted workflow"); for (unsigned race = 0; race < 4; ++race) { const auto trace = prepare_pair(false); auto committing = connection(), aborting = connection(); std::barrier start(2); std::optional> published; std::optional> aborted; std::thread commit_thread([&] { start.arrive_and_wait(); published = publish_cellular_migration(*committing, *f.objects, destination_objects, f.lease, destination_lease, f.head, destination_head, trace, f.build, 5s); }); std::thread abort_thread([&] { start.arrive_and_wait(); aborted = abort_cellular_migration_freeze(*aborting, f.key, f.lease, trace.request.request, 5s); }); commit_thread.join(); abort_thread.join(); require(published.has_value() && aborted.has_value() && (published->has_value() != aborted->has_value()), "commit/abort race had two outcomes or no definitive outcome"); if (published->has_value()) { require(aborted->error().kind == LeaseStoreError::commit_conflict, "losing abort was not refused"); f.head = (*published)->source; destination_head = (*published)->destination; const auto returning = prepare_pair(true); const auto returned = take(publish_cellular_migration(*f.db, destination_objects, *f.objects, destination_lease, f.lease, destination_head, f.head, returning, f.build, 5s), "return race winner's stock"); destination_head = returned.source; f.head = returned.destination; } else { require(published->error().store.kind == LeaseStoreError::commit_conflict, "losing paired publication was not refused"); } require( take(load_region_checkpoint(*f.db, f.key, 5s), "race source head").head == f.head.head && take(load_region_checkpoint(*f.db, destination_key, 5s), "race destination head") .head == destination_head.head && !take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s), "race source settled") .has_value() && !take(load_cellular_migration_stage(*f.db, destination_objects, destination_key, destination_lease, 5s), "race destination settled") .has_value(), "commit/abort race left half a head or a pending pause"); } recover(f); f.raw = restore(f.population->state()); } void migration_freezes(Fixture& f) { const RegionKey destination_key{f.key.world, 2}; auto destination_objects = take(FilesystemObjectStore::open(environment("PGHOST"), destination_key, max_cellular_population_bytes), "migration destination objects"); auto destination = opening(30001, true, 4, false, 2); auto destination_head = take(provision_cellular_region(*f.owner, *destination_objects, destination_key, 1, *destination, f.id(), f.build, 5s), "migration empty destination genesis"); auto destination_lease = take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s), "migration destination authority"); const auto unequal_cut = load_cellular_census_snapshot( *f.db, f.key.world, std::array{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()}, CellularCensusRegionRequest{destination_key.region, destination_lease, destination_objects.get()}}, std::array{f.build}, 5s); require(!unequal_cut && unequal_cut.error().kind == CellularSnapshotError::census && unequal_cut.error().census == CellularCensusSetError::boundary && f.db->transaction_idle(), "snapshot silently reconciled different current ticks/owner versions"); while (destination->tick() < f.raw->tick()) { const auto count = std::min(std::uint64_t{64}, f.raw->tick() - destination->tick()); CellularBatch inputs{{f.key.world, 2, 1, destination_head.head.commit, destination_head.head.digest, destination->tick(), destination->tick() + count}, {}}; inputs.ticks.resize(static_cast(count)); destination = take(replay_cellular_batch(*destination, inputs), "empty destination biology"); destination_head = take(publish_cellular_batch(*f.db, *destination_objects, destination_lease, destination_head, inputs, *destination, f.id(), f.build, 5s), "empty destination tick publication") .committed; } destination_head = take(migrate_cellular_boundary_profile(*f.db, *destination_objects, destination_lease, destination_head, f.id(), f.build, 5s), "destination accounting upgrade"); destination = take(destination->migration_upgrade_candidate(), "destination upgraded oracle"); const std::array builds{f.build}; const auto request_for = [&] { const auto member = std::ranges::find_if( f.raw->lives(), [](const auto& life) { return life.account == LifeAccount{}; }); require(member != f.raw->lives().end(), "freeze fixture lacks real wildlife"); return CellularMigrationRequest{ f.id(), {1, f.head.head.commit, f.id(), f.head.head.digest}, {2, destination_head.head.commit, f.id(), destination_head.head.digest}, member->identity.genome}; }; for (unsigned representation = 0; representation < 2; ++representation) { if (representation == 1) { f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, {true, 1200, 1800}, 5s), "freeze cohort source entry"); recover(f); destination_head = take(migrate_cellular_to_population( *f.db, *destination_objects, destination_lease, destination_head, f.id(), f.build, {true, 1200, 1800}, 5s), "freeze cohort destination entry"); } const auto source_before = f.head; const auto source_bytes = f.raw->snapshot(); const auto destination_before = destination_head; const auto request = request_for(); const auto prepare = [&](const CellularMigrationRequest& requested, std::chrono::milliseconds timeout = 5s) { return prepare_cellular_migration_freeze(*f.db, *f.objects, *destination_objects, f.lease, f.head, destination_head, requested, timeout); }; if (representation == 0) { Uuid account{}; account.back() = 1; take(f.db->query("BEGIN", {}, 5s), "pending player choice before freeze"); const auto control = take(claim_account_control(*f.db, f.key, f.lease, account, 2, 9900, 60000ms, 5s), "pending choice current control"); const auto input = encode_cellular_mutation_input( {account, 9900, f.head.head.end_tick + 1, {100, 1, 1, 1}}); take(reserve_cellular_mutation(*f.db, f.key, f.lease, account, control.generation, 9900, input, 5s), "reserve pre-freeze player choice"); const auto pending = prepare(request); take(f.db->query("ROLLBACK", {}, 5s), "rollback pre-freeze player choice"); require(!pending && std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18", "preparation stranded a player choice"); } else { take(f.db->query("BEGIN", {}, 5s), "prepare with admitted work transaction"); take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, f.head.head.end_tick + 20, 5s), "admitted work before freeze"); const auto pending = prepare(request); take(f.db->query("ROLLBACK", {}, 5s), "rollback pending-work preparation fixture"); require(!pending && std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18", "preparation skipped durable ecological work"); } auto frozen = take(prepare(request), "durable source freeze"); const CellularMigrationPrepared prepared{ f.key, take(encode_migration_trace(frozen.trace), "frozen canonical trace")}; auto malformed = prepared.trace; malformed.back() ^= 1; const std::array malformed_args{hexadecimal(f.lease.holder, false), std::to_string(f.lease.epoch), hexadecimal(malformed)}; const auto malformed_sql = f.db->query("SELECT * FROM " "evolution.prepare_cellular_migration_freeze(30201," "1,$1::uuid,$2::evolution.u64,$3::bytea)", malformed_args, 5s); require(!malformed_sql && std::string(malformed_sql.error().sqlstate.data(), 5) == "22023", "SQL accepted an invalid trace checksum before duplicate lookup"); require( !frozen.aborted && frozen.trace.request == request && take(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s), "replay frozen capsule"), "missing durable freeze") .trace == frozen.trace && take(load_region_checkpoint(*f.db, f.key, 5s), "frozen source head").head == source_before.head && take(load_region_checkpoint(*f.db, destination_key, 5s), "unowned destination head") .head == destination_before.head, "preparation changed either owner or lost its full replay capsule"); require( !take(retry_cellular_migration_freeze(*f.db, f.lease, prepared, 5s), "duplicate freeze") .aborted, "duplicate freeze lost its state"); const auto stage = [&](const CellularMigrationTrace& trace, std::chrono::milliseconds timeout = 5s) { return stage_cellular_migration(*f.db, *f.objects, *destination_objects, destination_lease, f.head, destination_head, trace, timeout); }; // The optimistic destination cut must still be exact at first staging. take(f.db->query("BEGIN", {}, 5s), "move destination before stage transaction"); if (representation == 0) take(migrate_cellular_to_population(*f.db, *destination_objects, destination_lease, destination_head, f.id(), f.build, {}, 5s), "temporarily move destination profile"); else take(materialize_population_profile(*f.db, *destination_objects, destination_lease, destination_head, f.id(), f.build, 5s), "temporarily move population destination"); const auto moved = stage(frozen.trace); take(f.db->query("ROLLBACK", {}, 5s), "rollback moved destination fixture"); require(!moved && moved.error().store.kind == LeaseStoreError::head_mismatch, "stage silently rebased a moved destination"); if (representation == 1) { take(f.db->query("BEGIN", {}, 5s), "stage pending interval transaction"); take(admit_population_interval(*f.db, destination_key, destination_lease, destination_head.head, destination_head.head.end_tick + 20, 5s), "destination admitted work before staging"); const auto pending = stage(frozen.trace); take(f.db->query("ROLLBACK", {}, 5s), "rollback destination interval fixture"); require(!pending && pending.error().store.kind == LeaseStoreError::commit_conflict, "stage stranded destination interval work"); } const auto staged = take(stage(frozen.trace), "durable destination staging"); const CellularMigrationStagePrepared stage_request{destination_key, prepared.trace, staged.ghost}; require( !staged.freeze.aborted && take(take(load_cellular_migration_stage(*f.db, *destination_objects, destination_key, destination_lease, 5s), "recover destination ghost"), "missing staged destination") .ghost == staged.ghost && take(load_region_checkpoint(*f.db, destination_key, 5s), "staged destination still owns original head") .head == destination_before.head && take(retry_cellular_migration_stage(*f.db, destination_lease, stage_request, 5s), "duplicate stage") .ghost == staged.ghost, "staging activated a ghost or lost exact replay"); auto changed_stage = stage_request; ++changed_stage.ghost.bytes; const auto changed = retry_cellular_migration_stage(*f.db, destination_lease, changed_stage, 5s); require(!changed && changed.error().store.kind == LeaseStoreError::commit_conflict, "stage accepted conflicting immutable ghost bytes"); auto reverse = frozen.trace; reverse.request.request = f.id(); std::swap(reverse.request.source, reverse.request.destination); const auto overlapping = retry_cellular_migration_freeze( *f.db, destination_lease, {destination_key, take(encode_migration_trace(reverse), "overlapping source trace")}, 5s); require(!overlapping && overlapping.error().store.kind == LeaseStoreError::commit_conflict, "staged destination simultaneously prepared an outgoing workflow"); take(f.db->query("BEGIN", {}, 5s), "staged destination life reservation transaction"); Uuid new_account{}; new_account.back() = 5; const auto staged_control = take(claim_account_control(*f.db, destination_key, destination_lease, new_account, 1, 9950, 60000ms, 5s), "staged destination reservation current control"); const auto staged_life = reserve_primitive_life(*f.db, destination_key, destination_lease, new_account, staged_control.generation, 1, 1, 5s); take(f.db->query("ROLLBACK", {}, 5s), "rollback staged life reservation fixture"); require(!staged_life && std::string(staged_life.error().cause.sqlstate.data(), 5) == "EVL18", "staged destination admitted a new player life reservation"); if (representation == 0) { CellularBatch inputs{{f.key.world, 2, 1, destination_head.head.commit, destination_head.head.digest, destination->tick(), destination->tick() + 1}, {{}}}; const auto advanced = take(replay_cellular_batch(*destination, inputs), "detached staged destination future"); const auto denied = publish_cellular_batch(*f.db, *destination_objects, destination_lease, destination_head, inputs, *advanced, f.id(), f.build, 5s); require(!denied && std::string(denied.error().store.cause.sqlstate.data(), 5) == "EVL18", "staged destination published an ordinary future"); } else { require(!admit_population_interval(*f.db, destination_key, destination_lease, destination_head.head, destination_head.head.end_tick + 20, 5s) && !materialize_population_profile(*f.db, *destination_objects, destination_lease, destination_head, f.id(), f.build, 5s), "staged destination progressed or changed profile"); } const auto ghost_path = destination_objects->directory() / hexadecimal(staged.ghost.sha256, false); const auto ghost_backup = ghost_path.string() + ".stage-test"; std::filesystem::rename(ghost_path, ghost_backup); const auto missing_ghost = load_cellular_migration_stage( *f.db, *destination_objects, destination_key, destination_lease, 5s); std::filesystem::rename(ghost_backup, ghost_path); require(!missing_ghost && missing_ghost.error().kind == RecoveryError::tail, "missing ghost fell back to the active destination"); char ghost_first{}; { std::fstream file(ghost_path, std::ios::binary | std::ios::in | std::ios::out); file.get(ghost_first); file.seekp(0); file.put(static_cast(static_cast(ghost_first) ^ 1U)); file.flush(); require(file.good(), "same-length ghost corruption fixture"); } const auto corrupt_ghost = load_cellular_migration_stage( *f.db, *destination_objects, destination_key, destination_lease, 5s); { std::fstream file(ghost_path, std::ios::binary | std::ios::in | std::ios::out); file.put(ghost_first); file.flush(); require(file.good(), "restore corrupt ghost fixture"); } require(!corrupt_ghost && corrupt_ghost.error().kind == RecoveryError::tail, "same-length corrupt ghost bypassed digest verification"); const auto source_manifest = take(decode_manifest(f.head.manifest), "stage source capsule"); const auto source_capsule_path = destination_objects->directory() / hexadecimal(source_manifest.payload_sha256, false); const auto source_capsule_backup = source_capsule_path.string() + ".stage-test"; std::filesystem::rename(source_capsule_path, source_capsule_backup); const auto missing_source = load_cellular_migration_stage( *f.db, *destination_objects, destination_key, destination_lease, 5s); std::filesystem::rename(source_capsule_backup, source_capsule_path); require(!missing_source && missing_source.error().kind == RecoveryError::tail, "destination recovery used the live source instead of its retained capsule"); const auto old_destination_lease = destination_lease; take(release_region_lease(*f.db, destination_key, destination_lease, 5s), "release staged destination owner"); destination_lease = take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s), "replacement staged destination owner"); require( take(load_cellular_migration_stage(*f.db, *destination_objects, destination_key, destination_lease, 5s), "new epoch ghost replay") .has_value() && !retry_cellular_migration_stage(*f.db, old_destination_lease, stage_request, 5s), "replacement activated the ghost or stale destination staged it"); auto conflict_trace = frozen.trace; ++conflict_trace.transferred.energy.value; auto conflict = retry_cellular_migration_freeze( *f.db, f.lease, {f.key, take(encode_migration_trace(conflict_trace), "conflict trace")}, 5s); require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict && !prepare(request_for()), "freeze accepted changed bytes or replaced pending request"); if (representation == 0) { const auto h = f.header(1); const auto inputs = take(unattended_population_inputs(*f.raw, h), "frozen raw inputs"); auto advanced = take(replay_cellular_batch(*f.raw, inputs), "frozen detached future"); auto denied = publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, inputs, *advanced, f.id(), f.build, 5s); require(!denied && std::string(denied.error().store.cause.sqlstate.data(), 5) == "EVL18" && !migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, {}, 5s), "raw source progressed or changed profile while frozen"); Uuid account{}; account.back() = 1; const auto input = encode_cellular_mutation_input( {account, 9901, f.head.head.end_tick + 1, {100, 1, 1, 1}}); take(f.db->query("BEGIN", {}, 5s), "current control while frozen fixture"); const auto control = take(claim_account_control(*f.db, f.key, f.lease, account, 2, 9901, 60000ms, 5s), "current frozen-source account control"); const auto choice = reserve_cellular_mutation(*f.db, f.key, f.lease, account, control.generation, 9901, input, 5s); take(f.db->query("ROLLBACK", {}, 5s), "rollback frozen control fixture"); require(!choice, "frozen raw owner admitted a new mutation reservation"); if (std::string(choice.error().cause.sqlstate.data(), 5) != "EVL18") throw std::runtime_error("frozen mutation guard SQLSTATE=" + std::string(choice.error().cause.sqlstate.data(), 5)); } else { require(!admit_population_interval(*f.db, f.key, f.lease, f.head.head, f.head.head.end_tick + 20, 5s) && !materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s), "population source admitted work or changed profile while frozen"); } const auto capsule_manifest = take(decode_manifest(destination_head.manifest), "capsule manifest"); const auto capsule_path = f.objects->directory() / hexadecimal(capsule_manifest.payload_sha256, false); const auto backup = capsule_path.string() + ".freeze-test"; std::filesystem::rename(capsule_path, backup); const auto missing = load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s); std::filesystem::rename(backup, capsule_path); require(!missing && missing.error().kind == RecoveryError::tail, "freeze recovery fell back to destination live state without the retained capsule"); char first{}; { std::fstream file(capsule_path, std::ios::binary | std::ios::in | std::ios::out); file.get(first); file.seekp(0); file.put(static_cast(static_cast(first) ^ 1U)); file.flush(); require(file.good(), "corrupt same-length capsule fixture"); } const auto corrupt = load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s); { std::fstream file(capsule_path, std::ios::binary | std::ios::in | std::ios::out); file.put(first); file.flush(); require(file.good(), "restore same-length capsule fixture"); } require(!corrupt && corrupt.error().kind == RecoveryError::tail, "corrupt capsule bypassed strong object binding"); const auto old_lease = f.lease; take(release_region_lease(*f.db, f.key, f.lease, 5s), "release frozen source owner"); f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "new frozen source owner"); require(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s), "recover freeze under new epoch") .has_value() && !abort_cellular_migration_freeze(*f.db, f.key, old_lease, request.request, 5s), "new epoch released a freeze or old epoch aborted it"); require( take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, request.request, 5s), "explicit source abort") .aborted && take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, request.request, 5s), "duplicate source abort") .aborted && take(retry_cellular_migration_freeze(*f.db, f.lease, prepared, 5s), "retry aborted preparation") .aborted && !take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s), "aborted pointer load") .has_value(), "abort was not durable or preparation retry reopened it"); require( !take(load_cellular_migration_stage(*f.db, *destination_objects, destination_key, destination_lease, 5s), "aborted destination pointer") .has_value() && take(retry_cellular_migration_stage(*f.db, destination_lease, stage_request, 5s), "retry aborted stage") .freeze.aborted, "source abort stranded the destination or stage retry reopened its pointer"); require(take(load_region_checkpoint(*f.db, f.key, 5s), "aborted source head").head == source_before.head && f.raw->snapshot() == source_bytes, "abort changed biological stock/time/history"); auto false_trace = frozen.trace; false_trace.request.request = f.id(); false_trace.request.source.commit = f.id(); false_trace.request.destination.commit = f.id(); ++false_trace.transferred.energy.value; take(retry_cellular_migration_freeze(*f.db, f.lease, {f.key, take(encode_migration_trace(false_trace), "canonical false inventory trace")}, 5s), "metadata-only false preparation"); const auto false_replay = load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s); require(!false_replay && false_replay.error().kind == RecoveryError::history, "freeze recovery trusted declared material without complete biological replay"); take( abort_cellular_migration_freeze(*f.db, f.key, f.lease, false_trace.request.request, 5s), "explicitly abort false preparation without moving stock"); // A genuine unknown reply retains exactly the chosen cut and group. auto locker = connection(true); take(locker->query("BEGIN", {}, 5s), "freeze ambiguity lock"); take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND " "region_id=1 FOR UPDATE", {}, 5s), "hold freeze reply"); auto timed = connection(); auto unknown = prepare_cellular_migration_freeze(*timed, *f.objects, *destination_objects, f.lease, f.head, destination_head, request_for(), 300ms); require(!unknown && unknown.error().kind == PublicationError::unknown && !unknown.error().prepared.trace.empty(), "freeze ambiguity lost its immutable request"); take(locker->query("ROLLBACK", {}, 5s), "release freeze ambiguity"); frozen = take(retry_cellular_migration_freeze(*f.db, f.lease, unknown.error().prepared, 5s), "reconcile unknown freeze"); take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, frozen.trace.request.request, 5s), "abort reconciled freeze"); const auto false_stage_freeze = take(prepare(request_for()), "false ghost preparation"); const auto original_destination = take(decode_manifest(destination_head.manifest), "false ghost existing object"); const CellularMigrationStagePrepared false_ghost{ destination_key, take(encode_migration_trace(false_stage_freeze.trace), "false ghost trace"), {original_destination.payload_sha256, original_destination.payload_bytes}}; take(retry_cellular_migration_stage(*f.db, destination_lease, false_ghost, 5s), "metadata-only false ghost stage"); const auto false_ghost_replay = load_cellular_migration_stage( *f.db, *destination_objects, destination_key, destination_lease, 5s); require(!false_ghost_replay && false_ghost_replay.error().kind == RecoveryError::history, "stage recovery accepted a real object with the wrong complete biology"); take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, false_stage_freeze.trace.request.request, 5s), "abort false ghost evidence"); const auto uncertain_stage_freeze = take(prepare(request_for()), "unknown stage preparation"); take(locker->query("BEGIN", {}, 5s), "stage ambiguity lock"); take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND " "region_id=2 FOR UPDATE", {}, 5s), "hold stage reply"); auto stage_timed = connection(); const auto stage_unknown = stage_cellular_migration( *stage_timed, *f.objects, *destination_objects, destination_lease, f.head, destination_head, uncertain_stage_freeze.trace, 300ms); require(!stage_unknown && stage_unknown.error().kind == PublicationError::unknown && !stage_unknown.error().prepared.trace.empty(), "stage ambiguity lost exact bytes"); take(locker->query("ROLLBACK", {}, 5s), "release stage ambiguity"); take(retry_cellular_migration_stage(*f.db, destination_lease, stage_unknown.error().prepared, 5s), "resolve unknown stage"); take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, uncertain_stage_freeze.trace.request.request, 5s), "abort reconciled stage"); for (unsigned race = 0; race < 4; ++race) { const auto race_freeze = take(prepare(request_for()), "stage race preparation"); auto stage_db = connection(), abort_db = connection(); std::barrier start(2); std::optional> staged_race; std::optional> aborted_race; std::thread staging([&] { start.arrive_and_wait(); staged_race = stage_cellular_migration(*stage_db, *f.objects, *destination_objects, destination_lease, f.head, destination_head, race_freeze.trace, 5s); }); std::thread aborting([&] { start.arrive_and_wait(); aborted_race = abort_cellular_migration_freeze( *abort_db, f.key, f.lease, race_freeze.trace.request.request, 5s); }); staging.join(); aborting.join(); require(aborted_race.has_value() && aborted_race->has_value() && staged_race.has_value() && (staged_race->has_value() || staged_race->error().store.kind == LeaseStoreError::commit_conflict) && !take(load_cellular_migration_stage(*f.db, *destination_objects, destination_key, destination_lease, 5s), "stage race final pointer") .has_value(), "stage/abort race left an active ghost or stranded pause"); } // Both sides of preparation's transaction boundary, with real process death. for (unsigned point = 0; point < 2; ++point) { const auto killed_request = request_for(); const auto child = ::fork(); require(child >= 0, "freeze publisher fork"); if (child == 0) { try { auto db = connection(); PublishHooks hooks; if (point == 0) hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); }; if (!prepare_cellular_migration_freeze(*db, *f.objects, *destination_objects, f.lease, f.head, destination_head, killed_request, 5s, hooks)) ::_exit(41); ::kill(::getpid(), SIGKILL); } catch (...) { ::_exit(42); } ::_exit(43); } int status{}; require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) && WTERMSIG(status) == SIGKILL && take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s), "recover killed preparation") .has_value() == (point == 1), "publisher death changed source ownership freeze"); if (point == 1) take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, killed_request.request, 5s), "abort killed preparation"); } for (unsigned point = 0; point < 2; ++point) { const auto killed = take(prepare(request_for()), "stage process-loss preparation"); const auto child = ::fork(); require(child >= 0, "stage publisher fork"); if (child == 0) { try { auto db = connection(); PublishHooks hooks; if (point == 0) hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); }; if (!stage_cellular_migration(*db, *f.objects, *destination_objects, destination_lease, f.head, destination_head, killed.trace, 5s, hooks)) ::_exit(44); ::kill(::getpid(), SIGKILL); } catch (...) { ::_exit(45); } ::_exit(46); } int status{}; require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) && WTERMSIG(status) == SIGKILL && take(load_cellular_migration_stage(*f.db, *destination_objects, destination_key, destination_lease, 5s), "recover killed stage") .has_value() == (point == 1), "stage process loss changed ownership or lost the durable pause"); take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, killed.trace.request.request, 5s), "abort killed stage"); } } // The owning host must retain the freeze through drain/recreation with entry off. take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture to frozen host"); const auto make_host = [&] { auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), f.key, max_cellular_population_bytes), "freeze host objects"); CellularHostConfig config{f.key, 1, {}, f.build, {f.build}}; config.lease.commit_timeout = 300ms; return take( CellularRegionHost::create( config, [] { return std::expected, PgFailure>{connection()}; }, std::move(objects)), "freeze host"); }; auto host = make_host(); require(host->step().ready, "source host before freeze"); const auto before = host->state()->snapshot(); const auto host_request = request_for(); take(host->prepare_migration_freeze(destination_head, *destination_objects, host_request), "host source freeze"); require(!host->status().ready && host->status().migration_frozen && !host->advance_unattended(64) && host->state()->snapshot() == before, "frozen host advanced or originated current output"); host->drain(); host = make_host(); require(!host->step().ready && host->status().migration_frozen && host->state()->snapshot() == before && !host->authority().has_value(), "replacement host implicitly aborted or ignored its freeze"); take(host->abort_migration_freeze(host_request.request), "replacement host explicit abort"); require(host->step().ready && !host->status().migration_frozen && host->state()->snapshot() == before, "host failed to resume the identical source after confirmed abort"); auto locker = connection(true); take(locker->query("BEGIN", {}, 5s), "host unknown freeze lock"); take( locker->query( "SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE", {}, 5s), "hold host freeze reply"); const auto uncertain_request = request_for(); const auto uncertain = host->prepare_migration_freeze(destination_head, *destination_objects, uncertain_request); require(!uncertain && !host->status().ready, "host originated during uncertain freeze"); take(locker->query("ROLLBACK", {}, 5s), "release host freeze reply"); for (unsigned attempt = 0; attempt < 200 && !host->status().migration_frozen; ++attempt) { host->step(); std::this_thread::sleep_for(25ms); } require(host->status().migration_frozen && !host->status().ready && host->state()->snapshot() == before, "host chose different biology after unknown preparation"); take(locker->query("BEGIN", {}, 5s), "host unknown abort lock"); take( locker->query( "SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE", {}, 5s), "hold host abort reply"); const auto unknown_abort = host->abort_migration_freeze(uncertain_request.request); require(!unknown_abort && !host->status().ready, "unconfirmed abort resumed the source"); take(locker->query("ROLLBACK", {}, 5s), "release host abort reply"); for (unsigned attempt = 0; attempt < 200 && !host->step().ready; ++attempt) std::this_thread::sleep_for(25ms); require(host->status().ready && !host->status().migration_frozen && host->state()->snapshot() == before, "host failed exact ambiguous-abort reconciliation"); host->drain(); f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "return freeze fixture authority"); const auto destination_host_freeze = take(prepare_cellular_migration_freeze(*f.db, *f.objects, *destination_objects, f.lease, f.head, destination_head, request_for(), 5s), "destination host source preparation"); take(release_region_lease(*f.db, destination_key, destination_lease, 5s), "release destination to owning host"); const auto make_destination_host = [&] { auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), destination_key, max_cellular_population_bytes), "destination host objects"); CellularHostConfig config{destination_key, 1, {}, f.build, {f.build}}; config.lease.commit_timeout = 300ms; return take( CellularRegionHost::create( config, [] { return std::expected, PgFailure>{connection()}; }, std::move(objects)), "destination staging host"); }; auto destination_host = make_destination_host(); require(destination_host->step().ready, "destination host before staging"); const auto destination_bytes = destination_host->state()->snapshot(); take(locker->query("BEGIN", {}, 5s), "host stage ambiguity transaction"); take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND " "region_id=1 FOR UPDATE", {}, 5s), "hold owning destination stage"); const auto host_stage_unknown = destination_host->stage_migration(f.head, *f.objects, destination_host_freeze.trace); require(!host_stage_unknown && !destination_host->status().ready && destination_host->state()->snapshot() == destination_bytes, "owning destination advanced during ambiguous staging"); take(locker->query("ROLLBACK", {}, 5s), "release owning stage reply"); for (unsigned attempt = 0; attempt < 200 && !destination_host->status().migration_frozen; ++attempt) { destination_host->step(); std::this_thread::sleep_for(25ms); } require(destination_host->status().migration_frozen && !destination_host->status().ready && !destination_host->advance_unattended(64), "owning destination activated its ghost"); destination_host->drain(); destination_host = make_destination_host(); require(!destination_host->step().ready && destination_host->status().migration_frozen && destination_host->state()->snapshot() == destination_bytes && !destination_host->authority().has_value(), "replacement destination lost staging or adopted the uncommitted ghost"); take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, destination_host_freeze.trace.request.request, 5s), "source-authorized abort releases destination host"); for (unsigned attempt = 0; attempt < 200 && !destination_host->step().ready; ++attempt) std::this_thread::sleep_for(25ms); require(destination_host->status().ready && !destination_host->status().migration_frozen && destination_host->state()->snapshot() == destination_bytes, "destination failed to resume its unchanged original state after source abort"); destination_host->drain(); destination_lease = take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s), "return staged destination fixture authority"); migration_publication(f, destination_key, *destination_objects, destination_lease, destination_head); f.head = take( materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s), "resume frozen population as raw owner"); f.raw = take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s), "source after frozen profile materialization") .state; take(release_region_lease(*f.db, destination_key, destination_lease, 5s), "release freeze destination"); } void host_ownership_and_admission(Fixture& f) { take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture host authority"); const auto make_host = [&](bool enabled) { auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), f.key, max_cellular_population_bytes), "host objects"); CellularHostConfig config{f.key, 1, {}, f.build, {f.build}}; config.lease.commit_timeout = 300ms; if (enabled) config.population_policy = CellularTierPolicy{true, 1200, 1800}; return take(CellularRegionHost::create( std::move(config), [] { return Postgres::connect({environment("PGHOST"), {}, environment("PGPORT"), environment("PGDATABASE"), environment("EVO_PG_RUNTIME_USER"), {}, {}}, 5s); }, std::move(objects)), "population host"); }; auto host = make_host(false); require(host->step().ready && !host->population_mode(), "raw host recovery"); require(!host->set_population_mode(true), "default host enabled population entry"); host->drain(); host = make_host(true); require(host->step().ready, "opt-in host recovery"); const auto before = host->state()->snapshot(); const auto tick = host->state()->tick(); take(host->set_population_mode(true), "host population entry"); require(host->population_mode() && host->state()->snapshot() == before, "host entry changed biology"); auto population = take(CellularPopulation::create( take(CellularLives::restore(before, f.key.world, f.key.region, 1, tick), "host reference restore"), {true, 1200, 1800}), "host reference population"); f.head = take(host->committed(), "host population head"); auto next = unattended(f); // uses the same deterministic frozen/wildlife input contract. // The fixture's old envelope may have a different membership history; biology is shared. auto expected = take(prepare_population_batch(*population, next.batch.biology), "host reference batch"); require(!take(host->advance_unattended(19), "collect bounded publication window").has_value() && host->state()->snapshot() == before, "small local debt was admitted prematurely"); const auto published = take(host->advance_unattended(65), "host population publication"); require(published.has_value(), "host skipped admitted interval"); require(host->population_mode() && host->state()->snapshot() == expected.state->state().snapshot(), "host population continuation differs"); const auto committed = take(host->committed(), "host continued head"); const auto continued = host->state()->snapshot(); const CellularBatchHeader recovered_header{f.key.world, f.key.region, 1, committed.head.commit, committed.head.digest, committed.head.end_tick, committed.head.end_tick + 64}; auto recovered_expected = take( prepare_population_batch( *expected.state, take(unattended_population_inputs(*host->state(), recovered_header), "recovered interval inputs")), "recovered interval reference"); take(admit_population_interval(*f.db, f.key, take(host->authority(), "host interval authority"), committed.head, committed.head.end_tick + 64, 5s), "admit before host loss"); host->drain(); host = make_host(false); require(host->step().ready && host->population_mode() && take(host->committed(), "host recovered head").head == committed.head && host->state()->snapshot() == continued, "restarted host failed to recover population head"); require(host->status().ecological_pending_ticks == 64 && !host->set_population_mode(false), "restarted host hid or skipped admitted work"); require(take(host->advance_unattended(0), "resume admitted work with zero new due time") .has_value() && host->state()->snapshot() == recovered_expected.state->state().snapshot() && host->status().ecological_pending_ticks == 0, "recovered interval did not settle exactly"); const auto recovered_head = take(host->committed(), "recovered interval head"); const auto recovered_bytes = host->state()->snapshot(); const CellularBatchHeader unknown_header{f.key.world, f.key.region, 1, recovered_head.head.commit, recovered_head.head.digest, recovered_head.head.end_tick, recovered_head.head.end_tick + 64}; auto unknown_expected = take( prepare_population_batch(*recovered_expected.state, take(unattended_population_inputs(*host->state(), unknown_header), "uncertain admission inputs")), "uncertain admission reference"); auto locker = connection(true); take(locker->query("BEGIN", {}, 5s), "host admission lock transaction"); take( locker->query( "SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE", {}, 5s), "hold host admission reply"); require(!host->advance_unattended(64) && !host->status().ready && host->state()->snapshot() == recovered_bytes, "uncertain admission allowed simulation/output"); take(locker->query("ROLLBACK", {}, 5s), "release host admission reply"); for (unsigned attempt = 0; attempt < 200 && !host->step().ready; ++attempt) std::this_thread::sleep_for(25ms); require(host->status().ready && host->status().ecological_pending_ticks == 64, "unknown admission failed exact reconciliation"); require(take(host->advance_unattended(0), "finish reconciled admission").has_value() && host->state()->snapshot() == unknown_expected.state->state().snapshot(), "admission retry chose different work"); const auto final_bytes = host->state()->snapshot(); Uuid account{}; account.back() = 1; // A returning player's operation completes the durable reverse transition before lookup. [[maybe_unused]] const auto queried = host->query_primitive_life(account, 1, 1); require(!host->population_mode() && host->state()->snapshot() == final_bytes && host->state()->tick() == tick + 192, "player operation did not materialize first"); host->drain(); } } // namespace void run_population_publication_tests() { Fixture fixture; funded_source(fixture); history_pages(fixture); snapshot_guards(fixture); conversions_and_retries(fixture); interval_contract(fixture); ambiguous_publication(fixture, false); ambiguous_publication(fixture, true); process_loss(fixture, 0); process_loss(fixture, 1); process_loss(fixture, 2); missing_and_corrupt_objects(fixture); isolation_and_reverse(fixture); boundary_upgrades(fixture); migration_freezes(fixture); host_ownership_and_admission(fixture); std::puts("population publication, exact recovery, process loss and fencing passed"); }