#include "evo/persistence/account_control_store.hpp" #include "evo/persistence/population_checkpoint_store.hpp" #include "evo/persistence/population_interval_store.hpp" #include "evo/persistence/sha256.hpp" #include "evo/server/cellular_region_host.hpp" #include "fixtures/cellular_ecology.hpp" #include #include #include #include #include #include #include #include #include void run_population_publication_tests(); namespace { using namespace evo; using namespace evo::server; using namespace evo::persistence; using namespace evo::fixture; using namespace std::chrono_literals; template T take(std::expected value, const char* where) { if (!value) { std::string message{where}; if constexpr (requires { value.error().cause.sqlstate; }) message += " SQLSTATE=" + std::string(value.error().cause.sqlstate.data(), 5); if constexpr (requires { value.error().store.cause.sqlstate; }) message += " SQLSTATE=" + std::string(value.error().store.cause.sqlstate.data(), 5); throw std::runtime_error(message); } if constexpr (!std::is_void_v) return std::move(*value); } template T take(std::optional value, const char* where) { if (!value.has_value()) throw std::runtime_error(where); return std::move(*value); } std::string environment(const char* name) { const auto* value = std::getenv(name); require(value != nullptr, "missing private test environment"); return value; } std::unique_ptr connection(bool owner = false) { return take(Postgres::connect({environment("PGHOST"), {}, environment("PGPORT"), environment("PGDATABASE"), environment(owner ? "PGUSER" : "EVO_PG_RUNTIME_USER"), {}, {}}, 5s), "population connection"); } Uuid identity(unsigned number) { Uuid id{}; id[0] = 0xDA; id[1] = 0x12; id[14] = static_cast(number >> 8U); id[15] = static_cast(number); return id; } std::string hexadecimal(std::span bytes, bool prefix = true) { std::string out = prefix ? "\\x" : ""; constexpr char digits[] = "0123456789abcdef"; for (const auto b : bytes) { out.push_back(digits[b >> 4U]); out.push_back(digits[b & 15U]); } return out; } struct Fixture { RegionKey key{30201, 1}; std::unique_ptr owner = connection(true), db = connection(); std::unique_ptr objects; LeaseGrant lease; Sha256Digest build{}; CommittedCheckpoint head; std::unique_ptr raw; std::unique_ptr population; unsigned next_id = 1; Uuid id() { return identity(next_id++); } CellularBatchHeader header(std::uint64_t count) const { return {key.world, key.region, 1, head.head.commit, head.head.digest, head.head.end_tick, head.head.end_tick + count}; } }; void funded_source(Fixture& f) { f.build.fill(0xDA); const std::array build{hexadecimal(f.build)}; take(f.owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES($1::bytea) " "ON CONFLICT DO NOTHING", build, 5s), "retain test build"); take(f.owner->query("INSERT INTO evolution.life_allocators VALUES(30201,100)", {}, 5s), "test allocator"); f.objects = take( FilesystemObjectStore::open(environment("PGHOST"), f.key, max_cellular_population_bytes), "population object namespace"); f.raw = opening(30001, true, 4, false); f.head = take(provision_cellular_region(*f.owner, *f.objects, f.key, 1, *f.raw, f.id(), f.build, 5s), "empty cellular genesis"); f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "population test lease"); CellularBatch birth{f.header(1), {{}}}; for (std::uint8_t index = 1; index <= 4; ++index) { Uuid account{}; account.back() = index; const auto control = take(claim_account_control(*f.db, f.key, f.lease, account, 1, index, 60000ms, 5s), "founder account control"); const auto reserved = take( reserve_primitive_life(*f.db, f.key, f.lease, account, control.generation, 1, 1, 5s), "funded founder reservation"); birth.ticks.front().births.push_back({account, 1, control.generation, reserved.identity}); } f.raw = take(replay_cellular_batch(*f.raw, birth), "funded founder replay"); f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, birth, *f.raw, f.id(), f.build, 5s), "funded founder publication") .committed; while (f.raw->next_bud_sequence() == 1 && f.raw->tick() < 20000) { CellularBatch batch{f.header(1), {}}; for (unsigned tick = 0; tick < 64; ++tick) { auto intents = take(wildlife_intents(*f.raw, f.raw->tick() + 1), "funded source wildlife"); for (const auto& life : f.raw->lives()) { if (life.account == LifeAccount{}) continue; CellularLifeIntent intent{life.identity.life, {}}; intent.intent.bud = (f.raw->tick() + life.identity.life) % 32 == 0; intents.push_back(intent); } std::ranges::sort(intents, {}, &CellularLifeIntent::life_id); take(f.raw->advance_tick(intents), "funded source biology"); batch.ticks.push_back({intents, {}, {}}); if (f.raw->next_bud_sequence() > 1) break; } batch.header.end_tick = f.raw->tick(); f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, batch, *f.raw, f.id(), f.build, 5s), "funded source batch") .committed; take(renew_region_lease(*f.db, f.key, f.lease, 60000ms, 5s), "source lease renewal"); } require(std::ranges::any_of(f.raw->lives(), [](const auto& life) { return life.account == LifeAccount{}; }), "durable fixture has no funded player-derived wildlife"); require(!migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, {true, 1200, 1800}, 5s), "population entry bypassed vulnerable controlled life"); const auto suspended_at = f.raw->tick() + cellular_linger_ticks + 1; while (f.raw->tick() < suspended_at) { auto h = f.header(std::min(std::uint64_t{64}, suspended_at - f.raw->tick())); auto prefix = take(unattended_population_inputs(*f.raw, h), "raw linger inputs"); f.raw = take(replay_cellular_batch(*f.raw, prefix), "raw linger biology"); f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, prefix, *f.raw, f.id(), f.build, 5s), "raw linger publication") .committed; std::vector endings; for (const auto& corpse : f.raw->corpses()) if (corpse.parent_life == 0) endings.push_back(corpse.life_id); if (!endings.empty()) take(end_cellular_lives(*f.db, f.key, f.lease, f.head.head.commit, endings, 5s), "raw linger endings"); } require(population_interval_safe(*f.raw), "population source retained a vulnerable controlled life"); } CellularPopulationExecution unattended(Fixture& f) { auto working = restore(f.population->state()); CellularBatch batch{f.header(64), {}}; for (unsigned tick = 0; tick < 64; ++tick) { auto intents = take(wildlife_intents(*working, working->tick() + 1), "unattended wildlife"); for (const auto& life : working->lives()) { if (life.account == LifeAccount{}) continue; CellularLifeIntent frozen{life.identity.life, {}}; frozen.intent.frozen = true; intents.push_back(frozen); } std::ranges::sort(intents, {}, &CellularLifeIntent::life_id); take(working->advance_tick(intents), "unattended fixture step"); batch.ticks.push_back({intents, {}, {}}); } return take(prepare_population_batch(*f.population, std::move(batch)), "population recorded interval"); } void recover(Fixture& f) { const std::array builds{f.build}; auto db = connection(); auto recovered = take(recover_population_checkpoint(*db, *f.objects, f.key, f.lease, builds, 5s), "population durable recovery"); f.head = std::move(recovered.committed); f.population = std::move(recovered.state); } void conversions_and_retries(Fixture& f) { const auto biology = f.raw->snapshot(); Uuid account{}; account.back() = 1; const auto input = encode_cellular_mutation_input({account, 9991, f.head.head.end_tick + 1, {100, 1, 1, 1}}); take(f.db->query("BEGIN", {}, 5s), "reservation guard transaction"); take(reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 9991, input, 5s), "reserve pending player choice"); const auto blocked = migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, {true, 1200, 1800}, 5s); require(!blocked && std::string(blocked.error().store.cause.sqlstate.data(), 5) == "EVL18", "population entry stranded a reserved player choice"); take(f.db->query("ROLLBACK", {}, 5s), "rollback guard fixture"); f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, {true, 1200, 1800}, 5s), "initial population transition"); recover(f); require(f.population->state().snapshot() == biology && f.population->members().empty(), "transition changed biology"); const auto refused = reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 9991, input, 5s); require(!refused && std::string(refused.error().cause.sqlstate.data(), 5) == "EVL06", "population profile admitted a player mutation reservation"); const auto wildlife = std::ranges::find_if(f.population->state().lives(), [](const auto& life) { return life.account == LifeAccount{}; }); require(wildlife != f.population->state().lives().end(), "missing durable descendant"); const auto genome = wildlife->identity.genome; auto conversion = take(prepare_population_conversion(*f.population, f.header(0), {genome, true}), "collapse sample"); auto bad = conversion.batch; ++bad.boundaries.front().sequence; require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, bad, *conversion.state, f.id(), f.build, 5s), "altered sample published"); const auto commit = f.id(); f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, conversion.batch, *conversion.state, commit, f.build, 5s), "durable collapse"); const PopulationPreparedCommit prepared{ f.key, commit, f.head.manifest, take(encode_population_batch(conversion.batch), "prepared trace")}; require(take(retry_population_prepared(*f.db, prepared, 5s), "idempotent retry").head == f.head.head, "retry advanced head"); require(take(resolve_population_prepared(*f.db, prepared, 5s), "read-only resolve").status == Resolution::committed, "commit unresolved"); auto future_batch = conversion.batch; future_batch.format_version = 2; auto future = prepared; future.trace = take(encode_population_batch(future_batch), "future trace fixture"); auto future_manifest = take(decode_manifest(future.manifest), "future manifest fixture"); future_manifest.batch.trace_bytes = future.trace.size(); future_manifest.batch.trace_sha256 = take(sha256(future.trace), "future trace digest"); future.manifest = take(encode_manifest(future_manifest), "future manifest encoding"); auto rejected_future = retry_population_prepared(*f.db, future, 5s); require(!rejected_future && rejected_future.error().kind == PublicationError::candidate, "bundle 9 prepared retry admitted population trace 2"); require(!resolve_population_prepared(*f.db, future, 5s), "bundle 9 prepared resolve admitted population trace 2"); auto rejected_sql = publish_population_checkpoint(*f.db, future.manifest, future.trace, 5s); require(!rejected_sql && rejected_sql.error().kind == LeaseStoreError::invalid, "schema 45 did not reject the unsupported trace format before commit lookup"); const auto current_manifest = take(decode_manifest(f.head.manifest), "upgrade source manifest"); const auto boundary_payload = take(cellular_boundary_upgrade_payload(conversion.state->snapshot(), current_manifest), "portable boundary upgrade source"); const auto boundary_manifest = take( cellular_boundary_upgrade_manifest( current_manifest, f.head.head.digest, f.id(), f.lease, f.build, boundary_payload.size(), take(sha256(boundary_payload), "boundary payload digest")), "portable boundary upgrade manifest"); const PopulationPreparedCommit boundary{ f.key, boundary_manifest.commit, take(encode_manifest(boundary_manifest), "boundary candidate encoding"), {}}; auto rejected_boundary = retry_population_prepared(*f.db, boundary, 5s); require(!rejected_boundary && rejected_boundary.error().kind == PublicationError::rejected && take(resolve_population_prepared(*f.db, boundary, 5s), "upgrade through wrong journal resolution") .status == Resolution::absent, "accounting upgrade bypassed its dedicated version journal"); auto rejected_upgrade = publish_population_profile_transition(*f.db, boundary.manifest, 5s); require(!rejected_upgrade, "representation publisher admitted accounting upgrade without dedicated evidence"); require(!publish_region_checkpoint(*f.db, boundary.manifest, {}, 5s), "generic publisher admitted boundary upgrade"); require(take(load_region_checkpoint(*f.db, f.key, 5s), "head after future rejection").head == f.head.head, "future trace changed the retained durable head"); auto changed = prepared; auto changed_manifest = take(decode_manifest(changed.manifest), "conflict manifest"); changed_manifest.payload_sha256[0] ^= 1; changed.manifest = take(encode_manifest(changed_manifest), "conflict encoding"); require(take(resolve_population_prepared(*f.db, changed, 5s), "conflicting resolve").status == Resolution::conflict, "changed commit was accepted"); require(!publish_region_checkpoint(*f.db, f.head.manifest, {}, 5s), "generic publisher bypassed profile gate"); const auto expected = conversion.state->snapshot(); recover(f); require(f.population->snapshot() == expected && !f.population->members().empty(), "committed sample not recovered"); require(f.population->state().snapshot() == biology, "collapse healed/aged/moved biology"); for (unsigned oscillation = 0; oscillation < 8; ++oscillation) { const bool collapse = oscillation % 2 != 0; auto next = take(prepare_population_conversion(*f.population, f.header(0), {genome, collapse}), "conversion oscillation"); f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, next.batch, *next.state, f.id(), f.build, 5s), "publish oscillation"); recover(f); require(f.population->state().snapshot() == biology, "durable conversion changed exact state"); } } void process_loss(Fixture& f, int point) { auto next = unattended(f); take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, next.batch.header.end_tick, 5s), "admit process-loss interval"); const auto predecessor = f.head.head; const auto expected = next.state->snapshot(); const auto commit = f.id(); int channel[2]{}; require(::pipe(channel) == 0, "process acknowledgement pipe"); const auto child = ::fork(); require(child >= 0, "population fork"); if (child == 0) { ::close(channel[0]); try { auto db = connection(); PublishHooks hooks; if (point == 0) hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); }; const auto published = publish_population_batch(*db, *f.objects, f.lease, f.head, next.batch, *next.state, commit, f.build, 5s, hooks); if (!published) ::_exit(20); if (point == 2) { const char ack = 'A'; if (::write(channel[1], &ack, 1) != 1) ::_exit(21); } ::kill(::getpid(), SIGKILL); } catch (...) { ::_exit(22); } ::_exit(23); } ::close(channel[1]); char ack{}; const auto acknowledged = ::read(channel[0], &ack, 1); ::close(channel[0]); int status{}; require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) && WTERMSIG(status) == SIGKILL, "child missed population crash boundary"); require((point == 2 && acknowledged == 1 && ack == 'A') || (point != 2 && acknowledged == 0), "unexpected permanent acknowledgement"); recover(f); const auto pending = take(load_population_interval(*f.db, f.key, f.lease, 5s), "recover process interval"); require(pending.has_value() == (point == 0), "process loss changed interval settlement"); if (point == 0) require(f.head.head == predecessor, "uncommitted process published a head"); else require(f.head.head.commit == commit && f.population->snapshot() == expected, "committed process loss changed state"); } void ambiguous_publication(Fixture& f, bool terminate) { auto next = unattended(f); take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, next.batch.header.end_tick, 5s), "admit ambiguous interval"); const auto expected = next.state->snapshot(); auto locker = connection(true); take(locker->query("BEGIN", {}, 5s), "population lease row transaction"); const std::array key{std::to_string(f.key.world), std::to_string(f.key.region)}; take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=$1::evolution.u64 " "AND region_id=$2::evolution.u64 FOR UPDATE", key, 5s), "hold population publication"); auto timed = connection(); auto unknown = publish_population_batch(*timed, *f.objects, f.lease, f.head, next.batch, *next.state, f.id(), f.build, 300ms); require(!unknown && unknown.error().kind == PublicationError::unknown, "population publication failed to retain ambiguous request"); if (terminate) { const std::array user{environment("EVO_PG_RUNTIME_USER")}; take(locker->query("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE " "usename=$1 AND wait_event_type='Lock'", user, 5s), "terminate uncommitted population writer"); } take(locker->query("ROLLBACK", {}, 5s), "release population publication"); auto resolver = connection(); bool resolved = false; for (unsigned attempt = 0; attempt < 200; ++attempt) { auto result = take(resolve_population_prepared(*resolver, unknown.error().prepared, 5s), "resolve ambiguous population publication"); if (result.status == (terminate ? Resolution::absent : Resolution::committed)) { resolved = true; break; } std::this_thread::sleep_for(25ms); } require(resolved, "ambiguous population publication did not resolve"); f.head = take(retry_population_prepared(*resolver, unknown.error().prepared, 5s), "retry exact population request"); recover(f); require(f.population->snapshot() == expected, "ambiguous publication changed retained sample"); } void interval_contract(Fixture& f) { require(!take(load_population_interval(*f.db, f.key, f.lease, 5s), "empty interval journal") .has_value(), "settled source retained pending work"); auto next = unattended(f); const auto source = f.head.head; const auto admitted = take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s), "durable interval admission"); require(take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s), "duplicate interval admission") == admitted, "admission retry changed boundary"); auto changed = admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 63, 5s); require(!changed && changed.error().kind == LeaseStoreError::commit_conflict, "source admitted two different intervals"); const auto expected = next.state->snapshot(); require(!f.population->members().empty(), "interval fixture lost its exact cohort"); const auto genome = f.population->members().front().genome; auto conversion = take(prepare_population_conversion(*f.population, f.header(0), {genome, false}), "conversion during admitted work"); require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, conversion.batch, *conversion.state, f.id(), f.build, 5s), "conversion skipped an admitted interval"); require( !materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s), "profile exit skipped an admitted interval"); auto shortened_header = f.header(63); auto shortened = take(prepare_population_batch( *f.population, take(unattended_population_inputs(f.population->state(), shortened_header), "shortened input")), "shortened interval"); require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, shortened.batch, *shortened.state, f.id(), f.build, 5s), "publication partially consumed an interval"); take(release_region_lease(*f.db, f.key, f.lease, 5s), "release interval owner"); const auto epoch = f.lease.epoch; f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "replace interval owner"); require(f.lease.epoch > epoch && take(take(load_population_interval(*f.db, f.key, f.lease, 5s), "new owner interval load"), "retained admission") == admitted, "replacement owner lost admitted work"); f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, next.batch, *next.state, f.id(), f.build, 5s), "settle exact admitted interval"); recover(f); require(f.population->snapshot() == expected && !take(load_population_interval(*f.db, f.key, f.lease, 5s), "settled interval load") .has_value(), "settlement lost work or retained spendable pending state"); require(take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s), "settled admission retry") == admitted && !take(load_population_interval(*f.db, f.key, f.lease, 5s), "settled retry pending") .has_value(), "old admission retry reopened settled work"); require(!admit_population_interval(*f.db, f.key, f.lease, f.head.head, f.head.head.end_tick + 65, 5s), "oversized interval admitted"); const auto denied = f.db->query( "SELECT * FROM evolution.publish_population_checkpoint_v44(''::bytea,''::bytea)", {}, 5s); require(!denied && std::string(denied.error().sqlstate.data(), 5) == "42501", "runtime bypassed interval guard"); } void isolation_and_reverse(Fixture& f) { auto interval = unattended(f); const auto old_lease = f.lease; take(release_region_lease(*f.db, f.key, f.lease, 5s), "release old authority"); auto new_db = connection(); f.lease = take(acquire_region_lease(*new_db, f.key, f.build, 60000ms, 5s), "new population authority"); require(f.lease.epoch > old_lease.epoch, "authority epoch did not advance"); const auto old_head = f.head.head; auto stale = publish_population_batch(*f.db, *f.objects, old_lease, f.head, interval.batch, *interval.state, f.id(), f.build, 5s); require(!stale && stale.error().store.kind == LeaseStoreError::fenced, "stale population writer published"); recover(f); require(f.head.head == old_head, "fenced writer changed head"); const auto expected = f.population->state().snapshot(); f.head = take( materialize_population_profile(*new_db, *f.objects, f.lease, f.head, f.id(), f.build, 5s), "return to cellular profile"); const std::array builds{f.build}; auto recovered = take(recover_cellular_checkpoint(*new_db, *f.objects, f.key, f.lease, builds, 5s), "cellular recovery after materialization"); require(recovered.state->snapshot() == expected && recovered.manifest.profile == CheckpointProfile::cellular_lives_v1, "reverse transition changed biology or time"); f.raw = std::move(recovered.state); } void missing_and_corrupt_objects(Fixture& f) { const auto manifest = take(decode_manifest(f.head.manifest), "object test manifest"); const auto path = f.objects->directory() / hexadecimal(manifest.payload_sha256, false); const auto backup = path.string() + ".population-test"; const std::array builds{f.build}; std::filesystem::rename(path, backup); auto missing = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s); std::filesystem::rename(backup, path); require(!missing && missing.error().kind == RecoveryError::missing_payload, "missing population payload did not fail closed"); std::filesystem::rename(path, backup); { std::ofstream corrupt(path, std::ios::binary); corrupt << 'X'; } auto corrupt = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s); std::filesystem::remove(path); std::filesystem::rename(backup, path); require(!corrupt && corrupt.error().kind == RecoveryError::corrupt_payload, "corrupt population payload did not fail closed"); recover(f); } void boundary_upgrades(Fixture& f) { const std::array builds{f.build}; const auto original = f.head; const auto expected_raw = take(f.raw->migration_upgrade_candidate(), "raw upgrade oracle")->snapshot(); // Rehearse the raw upgrade, recovery and ordinary trace-3 publication in one // transaction, then roll it back to retain a real bundle-9 source for profile 5. take(f.db->query("BEGIN", {}, 5s), "raw upgrade rollback rehearsal"); auto raw_head = take( migrate_cellular_boundary_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s), "raw accounting upgrade"); auto raw = take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s), "raw accounting recovery"); require(raw.state->snapshot() == expected_raw && raw.manifest.cellular_versions == cellular_v10_versions && raw_head.head.end_tick == original.head.end_tick, "raw accounting upgrade changed biology or time"); const CellularBatchHeader h{f.key.world, f.key.region, 1, raw_head.head.commit, raw_head.head.digest, raw_head.head.end_tick, raw_head.head.end_tick + 1}; const auto inputs = take(unattended_population_inputs(*raw.state, h), "upgraded raw inputs"); auto advanced = take(replay_cellular_batch(*raw.state, inputs), "upgraded raw oracle"); take(publish_cellular_batch(*f.db, *f.objects, f.lease, raw_head, inputs, *advanced, f.id(), f.build, 5s), "upgraded raw ordinary publication"); require(take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s), "upgraded raw ordinary recovery") .state->snapshot() == advanced->snapshot(), "raw accounting continuation differs"); take(f.db->query("ROLLBACK", {}, 5s), "roll back raw upgrade rehearsal"); require(take(load_region_checkpoint(*f.db, f.key, 5s), "raw rollback head").head == original.head, "rolled-back upgrade moved the durable head"); f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, {true, 1200, 1800}, 5s), "population accounting source entry"); recover(f); const auto member = std::ranges::find_if(f.population->state().lives(), [](const auto& life) { return life.account == LifeAccount{}; }); require(member != f.population->state().lives().end(), "accounting source has no wildlife"); auto collapsed = take( prepare_population_conversion(*f.population, f.header(0), {member->identity.genome, true}), "accounting source cohort"); f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, collapsed.batch, *collapsed.state, f.id(), f.build, 5s), "accounting source membership publication"); recover(f); require(!f.population->members().empty() && f.population->conversion_sequence() != 0, "accounting source lost cohort history"); auto interval = unattended(f); take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, interval.batch.header.end_tick, 5s), "upgrade pending interval"); auto pending = migrate_cellular_boundary_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s); require(!pending && std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18", "accounting upgrade discarded admitted work"); f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, interval.batch, *interval.state, f.id(), f.build, 5s), "settle pre-upgrade work"); recover(f); const auto before = f.head; const auto source = take(decode_manifest(before.manifest), "accounting source manifest"); const auto upgraded_population = take(f.population->migration_upgrade_candidate(), "cohort upgrade oracle"); const auto expected = upgraded_population->snapshot(); const auto altered = take(CellularPopulation::create(restore(upgraded_population->state()), {false, 1200, 1800}), "valid unequal upgrade target"); require(altered->state().current_total() == upgraded_population->state().current_total() && altered->snapshot() != expected, "unequal upgrade fixture must retain biological totals"); const auto altered_object = take(f.objects->put(altered->snapshot()), "unequal upgrade object"); const auto altered_manifest = take( cellular_boundary_upgrade_manifest(source, before.head.digest, f.id(), f.lease, f.build, altered_object.bytes, altered_object.sha256), "unequal upgrade manifest"); take(f.db->query("BEGIN", {}, 5s), "unequal upgrade recovery rehearsal"); take(publish_cellular_profile_migration( *f.db, take(encode_manifest(altered_manifest), "unequal upgrade encoding"), 5s), "metadata-only unequal upgrade publication"); const auto unequal = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s); take(f.db->query("ROLLBACK", {}, 5s), "undo unequal upgrade rehearsal"); require(!unequal && unequal.error().kind == RecoveryError::history && take(load_region_checkpoint(*f.db, f.key, 5s), "head after unequal upgrade").head == before.head, "upgrade recovery compared only aggregate stock or rewrote the retained source"); const auto child = ::fork(); require(child >= 0, "upgrade pre-transaction fork"); if (child == 0) { try { auto db = connection(); PublishHooks hooks; hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); }; [[maybe_unused]] const auto result = migrate_cellular_boundary_profile( *db, *f.objects, f.lease, before, identity(60000), f.build, 5s, hooks); } catch (...) { ::_exit(31); } ::_exit(32); } int status{}; require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) && WTERMSIG(status) == SIGKILL, "upgrade child missed pre-transaction kill"); recover(f); require(f.head.head == before.head, "unpublished upgrade became authoritative"); auto locker = connection(true); take(locker->query("BEGIN", {}, 5s), "accounting ambiguity lock transaction"); take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 " "AND region_id=1 FOR UPDATE", {}, 5s), "hold accounting reply"); auto timed = connection(); auto unknown = migrate_cellular_boundary_profile(*timed, *f.objects, f.lease, before, f.id(), f.build, 300ms); require(!unknown && unknown.error().kind == PublicationError::unknown && !unknown.error().prepared.manifest.empty(), "uncertain accounting upgrade lost its exact request"); take(locker->query("ROLLBACK", {}, 5s), "release accounting reply"); auto resolver = connection(); f.head = take(retry_cellular_profile_migration(*resolver, unknown.error().prepared, 5s), "retry exact accounting upgrade"); require(take(retry_cellular_profile_migration(*resolver, unknown.error().prepared, 5s), "duplicate accounting upgrade") .head == f.head.head, "duplicate accounting upgrade advanced head"); recover(f); require(f.population->snapshot() == expected && f.head.head.end_tick == before.head.end_tick && take(decode_manifest(f.head.manifest), "upgraded population manifest") .cellular_versions == cellular_v10_versions, "accounting upgrade changed biology, cohort history or time"); auto changed = unknown.error().prepared; auto changed_manifest = take(decode_manifest(changed.manifest), "changed accounting manifest"); changed_manifest.payload_sha256[0] ^= 1; changed.manifest = take(encode_manifest(changed_manifest), "changed accounting bytes"); auto conflict = retry_cellular_profile_migration(*resolver, changed, 5s); require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict, "accounting retry accepted changed bytes"); require( !migrate_cellular_boundary_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s), "accounting upgrade applied twice"); const auto path = f.objects->directory() / hexadecimal(source.payload_sha256, false); const auto backup = path.string() + ".upgrade-test"; std::filesystem::rename(path, backup); const auto missing = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s); std::filesystem::rename(backup, path); require(!missing && missing.error().kind == RecoveryError::tail, "accounting recovery trusted aggregate stock without its retained source"); missing_and_corrupt_objects(f); const auto stale = f.lease; take(release_region_lease(*f.db, f.key, f.lease, 5s), "release accounting writer"); f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "replace accounting writer"); auto stale_retry = retry_cellular_profile_migration(*f.db, unknown.error().prepared, 5s); require(!stale_retry && stale_retry.error().store.kind == LeaseStoreError::fenced && !recover_population_checkpoint(*f.db, *f.objects, f.key, stale, builds, 5s), "old accounting receipt granted current authority"); recover(f); require(f.population->snapshot() == expected, "new epoch changed accounting upgrade"); auto future_interval = unattended(f); auto old_trace = future_interval.batch; old_trace.format_version = 1; const auto old_bytes = take(encode_population_batch(old_trace), "wrong old trace fixture"); const auto upgraded_manifest = take(decode_manifest(f.head.manifest), "new accounting source"); auto wrong_trace_manifest = take(population_successor_manifest( upgraded_manifest, f.head.head.digest, f.id(), f.lease, old_trace.header.end_tick, f.build, future_interval.state->snapshot().size(), take(sha256(future_interval.state->snapshot()), "new payload SHA"), {old_bytes.size(), take(sha256(old_bytes), "old trace SHA"), 0, {}}), "wrong old trace manifest fixture"); const PopulationPreparedCommit wrong_trace{ f.key, wrong_trace_manifest.commit, take(encode_manifest(wrong_trace_manifest), "wrong old trace manifest"), old_bytes}; auto refused_old = retry_population_prepared(*f.db, wrong_trace, 5s); auto refused_old_sql = publish_population_checkpoint(*f.db, wrong_trace.manifest, old_bytes, 5s); require(!refused_old && refused_old.error().kind == PublicationError::candidate && !refused_old_sql && refused_old_sql.error().kind == LeaseStoreError::invalid && take(load_region_checkpoint(*f.db, f.key, 5s), "head after old trace").head == f.head.head, "bundle 10 reinterpreted retained population trace 1"); // Exercise the existing publication/process-loss matrix with batch 2 as well. process_loss(f, 0); process_loss(f, 1); process_loss(f, 2); isolation_and_reverse(f); require(f.raw->format_version() == cellular_lives_migration_format, "reverse transition downgraded accounting owner"); } void migration_freezes(Fixture& f) { const RegionKey destination_key{f.key.world, 2}; auto destination_objects = take(FilesystemObjectStore::open(environment("PGHOST"), destination_key, max_cellular_population_bytes), "migration destination objects"); auto destination = opening(30001, true, 4, false, 2); auto destination_head = take(provision_cellular_region(*f.owner, *destination_objects, destination_key, 1, *destination, f.id(), f.build, 5s), "migration empty destination genesis"); auto destination_lease = take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s), "migration destination authority"); while (destination->tick() < f.raw->tick()) { const auto count = std::min(std::uint64_t{64}, f.raw->tick() - destination->tick()); CellularBatch inputs{{f.key.world, 2, 1, destination_head.head.commit, destination_head.head.digest, destination->tick(), destination->tick() + count}, {}}; inputs.ticks.resize(static_cast(count)); destination = take(replay_cellular_batch(*destination, inputs), "empty destination biology"); destination_head = take(publish_cellular_batch(*f.db, *destination_objects, destination_lease, destination_head, inputs, *destination, f.id(), f.build, 5s), "empty destination tick publication") .committed; } destination_head = take(migrate_cellular_boundary_profile(*f.db, *destination_objects, destination_lease, destination_head, f.id(), f.build, 5s), "destination accounting upgrade"); destination = take(destination->migration_upgrade_candidate(), "destination upgraded oracle"); const std::array builds{f.build}; const auto request_for = [&] { const auto member = std::ranges::find_if( f.raw->lives(), [](const auto& life) { return life.account == LifeAccount{}; }); require(member != f.raw->lives().end(), "freeze fixture lacks real wildlife"); return CellularMigrationRequest{ f.id(), {1, f.head.head.commit, f.id(), f.head.head.digest}, {2, destination_head.head.commit, f.id(), destination_head.head.digest}, member->identity.genome}; }; for (unsigned representation = 0; representation < 2; ++representation) { if (representation == 1) { f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, {true, 1200, 1800}, 5s), "freeze cohort source entry"); recover(f); destination_head = take(migrate_cellular_to_population( *f.db, *destination_objects, destination_lease, destination_head, f.id(), f.build, {true, 1200, 1800}, 5s), "freeze cohort destination entry"); } const auto source_before = f.head; const auto source_bytes = f.raw->snapshot(); const auto destination_before = destination_head; const auto request = request_for(); const auto prepare = [&](const CellularMigrationRequest& requested, std::chrono::milliseconds timeout = 5s) { return prepare_cellular_migration_freeze(*f.db, *f.objects, *destination_objects, f.lease, f.head, destination_head, requested, timeout); }; if (representation == 0) { Uuid account{}; account.back() = 1; take(f.db->query("BEGIN", {}, 5s), "pending player choice before freeze"); const auto control = take(claim_account_control(*f.db, f.key, f.lease, account, 2, 9900, 60000ms, 5s), "pending choice current control"); const auto input = encode_cellular_mutation_input( {account, 9900, f.head.head.end_tick + 1, {100, 1, 1, 1}}); take(reserve_cellular_mutation(*f.db, f.key, f.lease, account, control.generation, 9900, input, 5s), "reserve pre-freeze player choice"); const auto pending = prepare(request); take(f.db->query("ROLLBACK", {}, 5s), "rollback pre-freeze player choice"); require(!pending && std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18", "preparation stranded a player choice"); } else { take(f.db->query("BEGIN", {}, 5s), "prepare with admitted work transaction"); take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, f.head.head.end_tick + 20, 5s), "admitted work before freeze"); const auto pending = prepare(request); take(f.db->query("ROLLBACK", {}, 5s), "rollback pending-work preparation fixture"); require(!pending && std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18", "preparation skipped durable ecological work"); } auto frozen = take(prepare(request), "durable source freeze"); const CellularMigrationPrepared prepared{ f.key, take(encode_migration_trace(frozen.trace), "frozen canonical trace")}; auto malformed = prepared.trace; malformed.back() ^= 1; const std::array malformed_args{hexadecimal(f.lease.holder, false), std::to_string(f.lease.epoch), hexadecimal(malformed)}; const auto malformed_sql = f.db->query("SELECT * FROM " "evolution.prepare_cellular_migration_freeze(30201," "1,$1::uuid,$2::evolution.u64,$3::bytea)", malformed_args, 5s); require(!malformed_sql && std::string(malformed_sql.error().sqlstate.data(), 5) == "22023", "SQL accepted an invalid trace checksum before duplicate lookup"); require( !frozen.aborted && frozen.trace.request == request && take(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s), "replay frozen capsule"), "missing durable freeze") .trace == frozen.trace && take(load_region_checkpoint(*f.db, f.key, 5s), "frozen source head").head == source_before.head && take(load_region_checkpoint(*f.db, destination_key, 5s), "unowned destination head") .head == destination_before.head, "preparation changed either owner or lost its full replay capsule"); require( !take(retry_cellular_migration_freeze(*f.db, f.lease, prepared, 5s), "duplicate freeze") .aborted, "duplicate freeze lost its state"); auto conflict_trace = frozen.trace; ++conflict_trace.transferred.energy.value; auto conflict = retry_cellular_migration_freeze( *f.db, f.lease, {f.key, take(encode_migration_trace(conflict_trace), "conflict trace")}, 5s); require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict && !prepare(request_for()), "freeze accepted changed bytes or replaced pending request"); if (representation == 0) { const auto h = f.header(1); const auto inputs = take(unattended_population_inputs(*f.raw, h), "frozen raw inputs"); auto advanced = take(replay_cellular_batch(*f.raw, inputs), "frozen detached future"); auto denied = publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, inputs, *advanced, f.id(), f.build, 5s); require(!denied && std::string(denied.error().store.cause.sqlstate.data(), 5) == "EVL18" && !migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, {}, 5s), "raw source progressed or changed profile while frozen"); Uuid account{}; account.back() = 1; const auto input = encode_cellular_mutation_input( {account, 9901, f.head.head.end_tick + 1, {100, 1, 1, 1}}); take(f.db->query("BEGIN", {}, 5s), "current control while frozen fixture"); const auto control = take(claim_account_control(*f.db, f.key, f.lease, account, 2, 9901, 60000ms, 5s), "current frozen-source account control"); const auto choice = reserve_cellular_mutation(*f.db, f.key, f.lease, account, control.generation, 9901, input, 5s); take(f.db->query("ROLLBACK", {}, 5s), "rollback frozen control fixture"); require(!choice, "frozen raw owner admitted a new mutation reservation"); if (std::string(choice.error().cause.sqlstate.data(), 5) != "EVL18") throw std::runtime_error("frozen mutation guard SQLSTATE=" + std::string(choice.error().cause.sqlstate.data(), 5)); } else { require(!admit_population_interval(*f.db, f.key, f.lease, f.head.head, f.head.head.end_tick + 20, 5s) && !materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s), "population source admitted work or changed profile while frozen"); } const auto capsule_manifest = take(decode_manifest(destination_head.manifest), "capsule manifest"); const auto capsule_path = f.objects->directory() / hexadecimal(capsule_manifest.payload_sha256, false); const auto backup = capsule_path.string() + ".freeze-test"; std::filesystem::rename(capsule_path, backup); const auto missing = load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s); std::filesystem::rename(backup, capsule_path); require(!missing && missing.error().kind == RecoveryError::tail, "freeze recovery fell back to destination live state without the retained capsule"); char first{}; { std::fstream file(capsule_path, std::ios::binary | std::ios::in | std::ios::out); file.get(first); file.seekp(0); file.put(static_cast(static_cast(first) ^ 1U)); file.flush(); require(file.good(), "corrupt same-length capsule fixture"); } const auto corrupt = load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s); { std::fstream file(capsule_path, std::ios::binary | std::ios::in | std::ios::out); file.put(first); file.flush(); require(file.good(), "restore same-length capsule fixture"); } require(!corrupt && corrupt.error().kind == RecoveryError::tail, "corrupt capsule bypassed strong object binding"); const auto old_lease = f.lease; take(release_region_lease(*f.db, f.key, f.lease, 5s), "release frozen source owner"); f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "new frozen source owner"); require(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s), "recover freeze under new epoch") .has_value() && !abort_cellular_migration_freeze(*f.db, f.key, old_lease, request.request, 5s), "new epoch released a freeze or old epoch aborted it"); require( take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, request.request, 5s), "explicit source abort") .aborted && take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, request.request, 5s), "duplicate source abort") .aborted && take(retry_cellular_migration_freeze(*f.db, f.lease, prepared, 5s), "retry aborted preparation") .aborted && !take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s), "aborted pointer load") .has_value(), "abort was not durable or preparation retry reopened it"); require(take(load_region_checkpoint(*f.db, f.key, 5s), "aborted source head").head == source_before.head && f.raw->snapshot() == source_bytes, "abort changed biological stock/time/history"); auto false_trace = frozen.trace; false_trace.request.request = f.id(); false_trace.request.source.commit = f.id(); false_trace.request.destination.commit = f.id(); ++false_trace.transferred.energy.value; take(retry_cellular_migration_freeze(*f.db, f.lease, {f.key, take(encode_migration_trace(false_trace), "canonical false inventory trace")}, 5s), "metadata-only false preparation"); const auto false_replay = load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s); require(!false_replay && false_replay.error().kind == RecoveryError::history, "freeze recovery trusted declared material without complete biological replay"); take( abort_cellular_migration_freeze(*f.db, f.key, f.lease, false_trace.request.request, 5s), "explicitly abort false preparation without moving stock"); // A genuine unknown reply retains exactly the chosen cut and group. auto locker = connection(true); take(locker->query("BEGIN", {}, 5s), "freeze ambiguity lock"); take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND " "region_id=1 FOR UPDATE", {}, 5s), "hold freeze reply"); auto timed = connection(); auto unknown = prepare_cellular_migration_freeze(*timed, *f.objects, *destination_objects, f.lease, f.head, destination_head, request_for(), 300ms); require(!unknown && unknown.error().kind == PublicationError::unknown && !unknown.error().prepared.trace.empty(), "freeze ambiguity lost its immutable request"); take(locker->query("ROLLBACK", {}, 5s), "release freeze ambiguity"); frozen = take(retry_cellular_migration_freeze(*f.db, f.lease, unknown.error().prepared, 5s), "reconcile unknown freeze"); take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, frozen.trace.request.request, 5s), "abort reconciled freeze"); // Both sides of preparation's transaction boundary, with real process death. for (unsigned point = 0; point < 2; ++point) { const auto killed_request = request_for(); const auto child = ::fork(); require(child >= 0, "freeze publisher fork"); if (child == 0) { try { auto db = connection(); PublishHooks hooks; if (point == 0) hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); }; if (!prepare_cellular_migration_freeze(*db, *f.objects, *destination_objects, f.lease, f.head, destination_head, killed_request, 5s, hooks)) ::_exit(41); ::kill(::getpid(), SIGKILL); } catch (...) { ::_exit(42); } ::_exit(43); } int status{}; require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) && WTERMSIG(status) == SIGKILL && take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s), "recover killed preparation") .has_value() == (point == 1), "publisher death changed source ownership freeze"); if (point == 1) take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, killed_request.request, 5s), "abort killed preparation"); } } // The owning host must retain the freeze through drain/recreation with entry off. take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture to frozen host"); const auto make_host = [&] { auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), f.key, max_cellular_population_bytes), "freeze host objects"); CellularHostConfig config{f.key, 1, {}, f.build, {f.build}}; config.lease.commit_timeout = 300ms; return take( CellularRegionHost::create( config, [] { return std::expected, PgFailure>{connection()}; }, std::move(objects)), "freeze host"); }; auto host = make_host(); require(host->step().ready, "source host before freeze"); const auto before = host->state()->snapshot(); const auto host_request = request_for(); take(host->prepare_migration_freeze(destination_head, *destination_objects, host_request), "host source freeze"); require(!host->status().ready && host->status().migration_frozen && !host->advance_unattended(64) && host->state()->snapshot() == before, "frozen host advanced or originated current output"); host->drain(); host = make_host(); require(!host->step().ready && host->status().migration_frozen && host->state()->snapshot() == before && !host->authority().has_value(), "replacement host implicitly aborted or ignored its freeze"); take(host->abort_migration_freeze(host_request.request), "replacement host explicit abort"); require(host->step().ready && !host->status().migration_frozen && host->state()->snapshot() == before, "host failed to resume the identical source after confirmed abort"); auto locker = connection(true); take(locker->query("BEGIN", {}, 5s), "host unknown freeze lock"); take( locker->query( "SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE", {}, 5s), "hold host freeze reply"); const auto uncertain_request = request_for(); const auto uncertain = host->prepare_migration_freeze(destination_head, *destination_objects, uncertain_request); require(!uncertain && !host->status().ready, "host originated during uncertain freeze"); take(locker->query("ROLLBACK", {}, 5s), "release host freeze reply"); for (unsigned attempt = 0; attempt < 200 && !host->status().migration_frozen; ++attempt) { host->step(); std::this_thread::sleep_for(25ms); } require(host->status().migration_frozen && !host->status().ready && host->state()->snapshot() == before, "host chose different biology after unknown preparation"); take(locker->query("BEGIN", {}, 5s), "host unknown abort lock"); take( locker->query( "SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE", {}, 5s), "hold host abort reply"); const auto unknown_abort = host->abort_migration_freeze(uncertain_request.request); require(!unknown_abort && !host->status().ready, "unconfirmed abort resumed the source"); take(locker->query("ROLLBACK", {}, 5s), "release host abort reply"); for (unsigned attempt = 0; attempt < 200 && !host->step().ready; ++attempt) std::this_thread::sleep_for(25ms); require(host->status().ready && !host->status().migration_frozen && host->state()->snapshot() == before, "host failed exact ambiguous-abort reconciliation"); host->drain(); f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "return freeze fixture authority"); f.head = take( materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s), "resume frozen population as raw owner"); f.raw = take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s), "source after frozen profile materialization") .state; take(release_region_lease(*f.db, destination_key, destination_lease, 5s), "release freeze destination"); } void host_ownership_and_admission(Fixture& f) { take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture host authority"); const auto make_host = [&](bool enabled) { auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), f.key, max_cellular_population_bytes), "host objects"); CellularHostConfig config{f.key, 1, {}, f.build, {f.build}}; config.lease.commit_timeout = 300ms; if (enabled) config.population_policy = CellularTierPolicy{true, 1200, 1800}; return take(CellularRegionHost::create( std::move(config), [] { return Postgres::connect({environment("PGHOST"), {}, environment("PGPORT"), environment("PGDATABASE"), environment("EVO_PG_RUNTIME_USER"), {}, {}}, 5s); }, std::move(objects)), "population host"); }; auto host = make_host(false); require(host->step().ready && !host->population_mode(), "raw host recovery"); require(!host->set_population_mode(true), "default host enabled population entry"); host->drain(); host = make_host(true); require(host->step().ready, "opt-in host recovery"); const auto before = host->state()->snapshot(); const auto tick = host->state()->tick(); take(host->set_population_mode(true), "host population entry"); require(host->population_mode() && host->state()->snapshot() == before, "host entry changed biology"); auto population = take(CellularPopulation::create( take(CellularLives::restore(before, f.key.world, f.key.region, 1, tick), "host reference restore"), {true, 1200, 1800}), "host reference population"); f.head = take(host->committed(), "host population head"); auto next = unattended(f); // uses the same deterministic frozen/wildlife input contract. // The fixture's old envelope may have a different membership history; biology is shared. auto expected = take(prepare_population_batch(*population, next.batch.biology), "host reference batch"); require(!take(host->advance_unattended(19), "collect bounded publication window").has_value() && host->state()->snapshot() == before, "small local debt was admitted prematurely"); const auto published = take(host->advance_unattended(65), "host population publication"); require(published.has_value(), "host skipped admitted interval"); require(host->population_mode() && host->state()->snapshot() == expected.state->state().snapshot(), "host population continuation differs"); const auto committed = take(host->committed(), "host continued head"); const auto continued = host->state()->snapshot(); const CellularBatchHeader recovered_header{f.key.world, f.key.region, 1, committed.head.commit, committed.head.digest, committed.head.end_tick, committed.head.end_tick + 64}; auto recovered_expected = take( prepare_population_batch( *expected.state, take(unattended_population_inputs(*host->state(), recovered_header), "recovered interval inputs")), "recovered interval reference"); take(admit_population_interval(*f.db, f.key, take(host->authority(), "host interval authority"), committed.head, committed.head.end_tick + 64, 5s), "admit before host loss"); host->drain(); host = make_host(false); require(host->step().ready && host->population_mode() && take(host->committed(), "host recovered head").head == committed.head && host->state()->snapshot() == continued, "restarted host failed to recover population head"); require(host->status().ecological_pending_ticks == 64 && !host->set_population_mode(false), "restarted host hid or skipped admitted work"); require(take(host->advance_unattended(0), "resume admitted work with zero new due time") .has_value() && host->state()->snapshot() == recovered_expected.state->state().snapshot() && host->status().ecological_pending_ticks == 0, "recovered interval did not settle exactly"); const auto recovered_head = take(host->committed(), "recovered interval head"); const auto recovered_bytes = host->state()->snapshot(); const CellularBatchHeader unknown_header{f.key.world, f.key.region, 1, recovered_head.head.commit, recovered_head.head.digest, recovered_head.head.end_tick, recovered_head.head.end_tick + 64}; auto unknown_expected = take( prepare_population_batch(*recovered_expected.state, take(unattended_population_inputs(*host->state(), unknown_header), "uncertain admission inputs")), "uncertain admission reference"); auto locker = connection(true); take(locker->query("BEGIN", {}, 5s), "host admission lock transaction"); take( locker->query( "SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE", {}, 5s), "hold host admission reply"); require(!host->advance_unattended(64) && !host->status().ready && host->state()->snapshot() == recovered_bytes, "uncertain admission allowed simulation/output"); take(locker->query("ROLLBACK", {}, 5s), "release host admission reply"); for (unsigned attempt = 0; attempt < 200 && !host->step().ready; ++attempt) std::this_thread::sleep_for(25ms); require(host->status().ready && host->status().ecological_pending_ticks == 64, "unknown admission failed exact reconciliation"); require(take(host->advance_unattended(0), "finish reconciled admission").has_value() && host->state()->snapshot() == unknown_expected.state->state().snapshot(), "admission retry chose different work"); const auto final_bytes = host->state()->snapshot(); Uuid account{}; account.back() = 1; // A returning player's operation completes the durable reverse transition before lookup. [[maybe_unused]] const auto queried = host->query_primitive_life(account, 1, 1); require(!host->population_mode() && host->state()->snapshot() == final_bytes && host->state()->tick() == tick + 192, "player operation did not materialize first"); host->drain(); } } // namespace void run_population_publication_tests() { Fixture fixture; funded_source(fixture); conversions_and_retries(fixture); interval_contract(fixture); ambiguous_publication(fixture, false); ambiguous_publication(fixture, true); process_loss(fixture, 0); process_loss(fixture, 1); process_loss(fixture, 2); missing_and_corrupt_objects(fixture); isolation_and_reverse(fixture); boundary_upgrades(fixture); migration_freezes(fixture); host_ownership_and_admission(fixture); std::puts("population publication, exact recovery, process loss and fencing passed"); }