Files
aevolution/tests/population_publication_tests.cpp
T
dingo 0f071f3f8b
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m21s
Foundation / checks (network) (push) Successful in 20m51s
Foundation / checks (client) (push) Successful in 22m2s
Foundation / checks (tsan) (push) Successful in 22m59s
Foundation / checks (postgres) (push) Successful in 53m40s
Foundation / checks (checks) (push) Successful in 1h7m19s
Foundation / postgres-tools (push) Successful in 29s
Foundation / server-image (push) Successful in 38s
Foundation / image (push) Successful in 4m31s
M12.4: atomically publish and recover biological migration pairs
2026-10-18 10:02:55 -04:00

1953 lines
111 KiB
C++

#include "evo/persistence/account_control_store.hpp"
#include "evo/persistence/population_checkpoint_store.hpp"
#include "evo/persistence/population_interval_store.hpp"
#include "evo/persistence/sha256.hpp"
#include "evo/server/cellular_region_host.hpp"
#include "fixtures/cellular_ecology.hpp"
#include <barrier>
#include <csignal>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <stdexcept>
#include <sys/wait.h>
#include <thread>
#include <type_traits>
#include <unistd.h>
void run_population_publication_tests();
namespace {
using namespace evo;
using namespace evo::server;
using namespace evo::persistence;
using namespace evo::fixture;
using namespace std::chrono_literals;
template <typename T, typename E> T take(std::expected<T, E> value, const char* where) {
if (!value) {
std::string message{where};
if constexpr (requires { value.error().cause.sqlstate; })
message += " SQLSTATE=" + std::string(value.error().cause.sqlstate.data(), 5);
if constexpr (requires { value.error().store.cause.sqlstate; })
message += " SQLSTATE=" + std::string(value.error().store.cause.sqlstate.data(), 5);
throw std::runtime_error(message);
}
if constexpr (!std::is_void_v<T>)
return std::move(*value);
}
template <typename T> T take(std::optional<T> value, const char* where) {
if (!value.has_value())
throw std::runtime_error(where);
return std::move(*value);
}
std::string environment(const char* name) {
const auto* value = std::getenv(name);
require(value != nullptr, "missing private test environment");
return value;
}
std::unique_ptr<Postgres> connection(bool owner = false) {
return take(Postgres::connect({environment("PGHOST"),
{},
environment("PGPORT"),
environment("PGDATABASE"),
environment(owner ? "PGUSER" : "EVO_PG_RUNTIME_USER"),
{},
{}},
5s),
"population connection");
}
Uuid identity(unsigned number) {
Uuid id{};
id[0] = 0xDA;
id[1] = 0x12;
id[14] = static_cast<std::uint8_t>(number >> 8U);
id[15] = static_cast<std::uint8_t>(number);
return id;
}
std::string hexadecimal(std::span<const std::uint8_t> bytes, bool prefix = true) {
std::string out = prefix ? "\\x" : "";
constexpr char digits[] = "0123456789abcdef";
for (const auto b : bytes) {
out.push_back(digits[b >> 4U]);
out.push_back(digits[b & 15U]);
}
return out;
}
struct Fixture {
RegionKey key{30201, 1};
std::unique_ptr<Postgres> owner = connection(true), db = connection();
std::unique_ptr<FilesystemObjectStore> objects;
LeaseGrant lease;
Sha256Digest build{};
CommittedCheckpoint head;
std::unique_ptr<CellularLives> raw;
std::unique_ptr<CellularPopulation> population;
unsigned next_id = 1;
Uuid id() {
return identity(next_id++);
}
CellularBatchHeader header(std::uint64_t count) const {
return {key.world,
key.region,
1,
head.head.commit,
head.head.digest,
head.head.end_tick,
head.head.end_tick + count};
}
};
void funded_source(Fixture& f) {
f.build.fill(0xDA);
const std::array<std::string, 1> build{hexadecimal(f.build)};
take(f.owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES($1::bytea) "
"ON CONFLICT DO NOTHING",
build, 5s),
"retain test build");
take(f.owner->query("INSERT INTO evolution.life_allocators VALUES(30201,100)", {}, 5s),
"test allocator");
f.objects = take(
FilesystemObjectStore::open(environment("PGHOST"), f.key, max_cellular_population_bytes),
"population object namespace");
f.raw = opening(30001, true, 4, false);
f.head =
take(provision_cellular_region(*f.owner, *f.objects, f.key, 1, *f.raw, f.id(), f.build, 5s),
"empty cellular genesis");
f.lease =
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "population test lease");
CellularBatch birth{f.header(1), {{}}};
for (std::uint8_t index = 1; index <= 4; ++index) {
Uuid account{};
account.back() = index;
const auto control =
take(claim_account_control(*f.db, f.key, f.lease, account, 1, index, 60000ms, 5s),
"founder account control");
const auto reserved = take(
reserve_primitive_life(*f.db, f.key, f.lease, account, control.generation, 1, 1, 5s),
"funded founder reservation");
birth.ticks.front().births.push_back({account, 1, control.generation, reserved.identity});
}
f.raw = take(replay_cellular_batch(*f.raw, birth), "funded founder replay");
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, birth, *f.raw, f.id(),
f.build, 5s),
"funded founder publication")
.committed;
while (f.raw->next_bud_sequence() == 1 && f.raw->tick() < 20000) {
CellularBatch batch{f.header(1), {}};
for (unsigned tick = 0; tick < 64; ++tick) {
auto intents =
take(wildlife_intents(*f.raw, f.raw->tick() + 1), "funded source wildlife");
for (const auto& life : f.raw->lives()) {
if (life.account == LifeAccount{})
continue;
CellularLifeIntent intent{life.identity.life, {}};
intent.intent.bud = (f.raw->tick() + life.identity.life) % 32 == 0;
intents.push_back(intent);
}
std::ranges::sort(intents, {}, &CellularLifeIntent::life_id);
take(f.raw->advance_tick(intents), "funded source biology");
batch.ticks.push_back({intents, {}, {}});
if (f.raw->next_bud_sequence() > 1)
break;
}
batch.header.end_tick = f.raw->tick();
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, batch, *f.raw,
f.id(), f.build, 5s),
"funded source batch")
.committed;
take(renew_region_lease(*f.db, f.key, f.lease, 60000ms, 5s), "source lease renewal");
}
require(std::ranges::any_of(f.raw->lives(),
[](const auto& life) { return life.account == LifeAccount{}; }),
"durable fixture has no funded player-derived wildlife");
require(!migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build,
{true, 1200, 1800}, 5s),
"population entry bypassed vulnerable controlled life");
const auto suspended_at = f.raw->tick() + cellular_linger_ticks + 1;
while (f.raw->tick() < suspended_at) {
auto h = f.header(std::min(std::uint64_t{64}, suspended_at - f.raw->tick()));
auto prefix = take(unattended_population_inputs(*f.raw, h), "raw linger inputs");
f.raw = take(replay_cellular_batch(*f.raw, prefix), "raw linger biology");
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, prefix, *f.raw,
f.id(), f.build, 5s),
"raw linger publication")
.committed;
std::vector<std::uint64_t> endings;
for (const auto& corpse : f.raw->corpses())
if (corpse.parent_life == 0)
endings.push_back(corpse.life_id);
if (!endings.empty())
take(end_cellular_lives(*f.db, f.key, f.lease, f.head.head.commit, endings, 5s),
"raw linger endings");
}
require(population_interval_safe(*f.raw),
"population source retained a vulnerable controlled life");
}
CellularPopulationExecution unattended(Fixture& f) {
auto working = restore(f.population->state());
CellularBatch batch{f.header(64), {}};
for (unsigned tick = 0; tick < 64; ++tick) {
auto intents = take(wildlife_intents(*working, working->tick() + 1), "unattended wildlife");
for (const auto& life : working->lives()) {
if (life.account == LifeAccount{})
continue;
CellularLifeIntent frozen{life.identity.life, {}};
frozen.intent.frozen = true;
intents.push_back(frozen);
}
std::ranges::sort(intents, {}, &CellularLifeIntent::life_id);
take(working->advance_tick(intents), "unattended fixture step");
batch.ticks.push_back({intents, {}, {}});
}
return take(prepare_population_batch(*f.population, std::move(batch)),
"population recorded interval");
}
void recover(Fixture& f) {
const std::array<Sha256Digest, 1> builds{f.build};
auto db = connection();
auto recovered =
take(recover_population_checkpoint(*db, *f.objects, f.key, f.lease, builds, 5s),
"population durable recovery");
f.head = std::move(recovered.committed);
f.population = std::move(recovered.state);
}
void conversions_and_retries(Fixture& f) {
const auto biology = f.raw->snapshot();
Uuid account{};
account.back() = 1;
const auto input =
encode_cellular_mutation_input({account, 9991, f.head.head.end_tick + 1, {100, 1, 1, 1}});
take(f.db->query("BEGIN", {}, 5s), "reservation guard transaction");
take(reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 9991, input, 5s),
"reserve pending player choice");
const auto blocked = migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
f.build, {true, 1200, 1800}, 5s);
require(!blocked && std::string(blocked.error().store.cause.sqlstate.data(), 5) == "EVL18",
"population entry stranded a reserved player choice");
take(f.db->query("ROLLBACK", {}, 5s), "rollback guard fixture");
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
f.build, {true, 1200, 1800}, 5s),
"initial population transition");
recover(f);
require(f.population->state().snapshot() == biology && f.population->members().empty(),
"transition changed biology");
const auto refused =
reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 9991, input, 5s);
require(!refused && std::string(refused.error().cause.sqlstate.data(), 5) == "EVL06",
"population profile admitted a player mutation reservation");
const auto wildlife = std::ranges::find_if(f.population->state().lives(), [](const auto& life) {
return life.account == LifeAccount{};
});
require(wildlife != f.population->state().lives().end(), "missing durable descendant");
const auto genome = wildlife->identity.genome;
auto conversion =
take(prepare_population_conversion(*f.population, f.header(0), {genome, true}),
"collapse sample");
auto bad = conversion.batch;
++bad.boundaries.front().sequence;
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, bad, *conversion.state,
f.id(), f.build, 5s),
"altered sample published");
const auto commit = f.id();
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, conversion.batch,
*conversion.state, commit, f.build, 5s),
"durable collapse");
const PopulationPreparedCommit prepared{
f.key, commit, f.head.manifest,
take(encode_population_batch(conversion.batch), "prepared trace")};
require(take(retry_population_prepared(*f.db, prepared, 5s), "idempotent retry").head ==
f.head.head,
"retry advanced head");
require(take(resolve_population_prepared(*f.db, prepared, 5s), "read-only resolve").status ==
Resolution::committed,
"commit unresolved");
auto future_batch = conversion.batch;
future_batch.format_version = 2;
auto future = prepared;
future.trace = take(encode_population_batch(future_batch), "future trace fixture");
auto future_manifest = take(decode_manifest(future.manifest), "future manifest fixture");
future_manifest.batch.trace_bytes = future.trace.size();
future_manifest.batch.trace_sha256 = take(sha256(future.trace), "future trace digest");
future.manifest = take(encode_manifest(future_manifest), "future manifest encoding");
auto rejected_future = retry_population_prepared(*f.db, future, 5s);
require(!rejected_future && rejected_future.error().kind == PublicationError::candidate,
"bundle 9 prepared retry admitted population trace 2");
require(!resolve_population_prepared(*f.db, future, 5s),
"bundle 9 prepared resolve admitted population trace 2");
auto rejected_sql = publish_population_checkpoint(*f.db, future.manifest, future.trace, 5s);
require(!rejected_sql && rejected_sql.error().kind == LeaseStoreError::invalid,
"schema 45 did not reject the unsupported trace format before commit lookup");
const auto current_manifest = take(decode_manifest(f.head.manifest), "upgrade source manifest");
const auto boundary_payload =
take(cellular_boundary_upgrade_payload(conversion.state->snapshot(), current_manifest),
"portable boundary upgrade source");
const auto boundary_manifest = take(
cellular_boundary_upgrade_manifest(
current_manifest, f.head.head.digest, f.id(), f.lease, f.build, boundary_payload.size(),
take(sha256(boundary_payload), "boundary payload digest")),
"portable boundary upgrade manifest");
const PopulationPreparedCommit boundary{
f.key,
boundary_manifest.commit,
take(encode_manifest(boundary_manifest), "boundary candidate encoding"),
{}};
auto rejected_boundary = retry_population_prepared(*f.db, boundary, 5s);
require(!rejected_boundary && rejected_boundary.error().kind == PublicationError::rejected &&
take(resolve_population_prepared(*f.db, boundary, 5s),
"upgrade through wrong journal resolution")
.status == Resolution::absent,
"accounting upgrade bypassed its dedicated version journal");
auto rejected_upgrade = publish_population_profile_transition(*f.db, boundary.manifest, 5s);
require(!rejected_upgrade,
"representation publisher admitted accounting upgrade without dedicated evidence");
require(!publish_region_checkpoint(*f.db, boundary.manifest, {}, 5s),
"generic publisher admitted boundary upgrade");
require(take(load_region_checkpoint(*f.db, f.key, 5s), "head after future rejection").head ==
f.head.head,
"future trace changed the retained durable head");
auto changed = prepared;
auto changed_manifest = take(decode_manifest(changed.manifest), "conflict manifest");
changed_manifest.payload_sha256[0] ^= 1;
changed.manifest = take(encode_manifest(changed_manifest), "conflict encoding");
require(take(resolve_population_prepared(*f.db, changed, 5s), "conflicting resolve").status ==
Resolution::conflict,
"changed commit was accepted");
require(!publish_region_checkpoint(*f.db, f.head.manifest, {}, 5s),
"generic publisher bypassed profile gate");
const auto expected = conversion.state->snapshot();
recover(f);
require(f.population->snapshot() == expected && !f.population->members().empty(),
"committed sample not recovered");
require(f.population->state().snapshot() == biology, "collapse healed/aged/moved biology");
for (unsigned oscillation = 0; oscillation < 8; ++oscillation) {
const bool collapse = oscillation % 2 != 0;
auto next =
take(prepare_population_conversion(*f.population, f.header(0), {genome, collapse}),
"conversion oscillation");
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, next.batch,
*next.state, f.id(), f.build, 5s),
"publish oscillation");
recover(f);
require(f.population->state().snapshot() == biology,
"durable conversion changed exact state");
}
}
void process_loss(Fixture& f, int point) {
auto next = unattended(f);
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, next.batch.header.end_tick,
5s),
"admit process-loss interval");
const auto predecessor = f.head.head;
const auto expected = next.state->snapshot();
const auto commit = f.id();
int channel[2]{};
require(::pipe(channel) == 0, "process acknowledgement pipe");
const auto child = ::fork();
require(child >= 0, "population fork");
if (child == 0) {
::close(channel[0]);
try {
auto db = connection();
PublishHooks hooks;
if (point == 0)
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
const auto published =
publish_population_batch(*db, *f.objects, f.lease, f.head, next.batch, *next.state,
commit, f.build, 5s, hooks);
if (!published)
::_exit(20);
if (point == 2) {
const char ack = 'A';
if (::write(channel[1], &ack, 1) != 1)
::_exit(21);
}
::kill(::getpid(), SIGKILL);
} catch (...) {
::_exit(22);
}
::_exit(23);
}
::close(channel[1]);
char ack{};
const auto acknowledged = ::read(channel[0], &ack, 1);
::close(channel[0]);
int status{};
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
WTERMSIG(status) == SIGKILL,
"child missed population crash boundary");
require((point == 2 && acknowledged == 1 && ack == 'A') || (point != 2 && acknowledged == 0),
"unexpected permanent acknowledgement");
recover(f);
const auto pending =
take(load_population_interval(*f.db, f.key, f.lease, 5s), "recover process interval");
require(pending.has_value() == (point == 0), "process loss changed interval settlement");
if (point == 0)
require(f.head.head == predecessor, "uncommitted process published a head");
else
require(f.head.head.commit == commit && f.population->snapshot() == expected,
"committed process loss changed state");
}
void ambiguous_publication(Fixture& f, bool terminate) {
auto next = unattended(f);
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, next.batch.header.end_tick,
5s),
"admit ambiguous interval");
const auto expected = next.state->snapshot();
auto locker = connection(true);
take(locker->query("BEGIN", {}, 5s), "population lease row transaction");
const std::array<std::string, 2> key{std::to_string(f.key.world), std::to_string(f.key.region)};
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=$1::evolution.u64 "
"AND region_id=$2::evolution.u64 FOR UPDATE",
key, 5s),
"hold population publication");
auto timed = connection();
auto unknown = publish_population_batch(*timed, *f.objects, f.lease, f.head, next.batch,
*next.state, f.id(), f.build, 300ms);
require(!unknown && unknown.error().kind == PublicationError::unknown,
"population publication failed to retain ambiguous request");
if (terminate) {
const std::array<std::string, 1> user{environment("EVO_PG_RUNTIME_USER")};
take(locker->query("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE "
"usename=$1 AND wait_event_type='Lock'",
user, 5s),
"terminate uncommitted population writer");
}
take(locker->query("ROLLBACK", {}, 5s), "release population publication");
auto resolver = connection();
bool resolved = false;
for (unsigned attempt = 0; attempt < 200; ++attempt) {
auto result = take(resolve_population_prepared(*resolver, unknown.error().prepared, 5s),
"resolve ambiguous population publication");
if (result.status == (terminate ? Resolution::absent : Resolution::committed)) {
resolved = true;
break;
}
std::this_thread::sleep_for(25ms);
}
require(resolved, "ambiguous population publication did not resolve");
f.head = take(retry_population_prepared(*resolver, unknown.error().prepared, 5s),
"retry exact population request");
recover(f);
require(f.population->snapshot() == expected, "ambiguous publication changed retained sample");
}
void interval_contract(Fixture& f) {
require(!take(load_population_interval(*f.db, f.key, f.lease, 5s), "empty interval journal")
.has_value(),
"settled source retained pending work");
auto next = unattended(f);
const auto source = f.head.head;
const auto admitted =
take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
"durable interval admission");
require(take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
"duplicate interval admission") == admitted,
"admission retry changed boundary");
auto changed =
admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 63, 5s);
require(!changed && changed.error().kind == LeaseStoreError::commit_conflict,
"source admitted two different intervals");
const auto expected = next.state->snapshot();
require(!f.population->members().empty(), "interval fixture lost its exact cohort");
const auto genome = f.population->members().front().genome;
auto conversion =
take(prepare_population_conversion(*f.population, f.header(0), {genome, false}),
"conversion during admitted work");
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, conversion.batch,
*conversion.state, f.id(), f.build, 5s),
"conversion skipped an admitted interval");
require(
!materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
"profile exit skipped an admitted interval");
auto shortened_header = f.header(63);
auto shortened =
take(prepare_population_batch(
*f.population,
take(unattended_population_inputs(f.population->state(), shortened_header),
"shortened input")),
"shortened interval");
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, shortened.batch,
*shortened.state, f.id(), f.build, 5s),
"publication partially consumed an interval");
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release interval owner");
const auto epoch = f.lease.epoch;
f.lease =
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "replace interval owner");
require(f.lease.epoch > epoch && take(take(load_population_interval(*f.db, f.key, f.lease, 5s),
"new owner interval load"),
"retained admission") == admitted,
"replacement owner lost admitted work");
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, next.batch,
*next.state, f.id(), f.build, 5s),
"settle exact admitted interval");
recover(f);
require(f.population->snapshot() == expected &&
!take(load_population_interval(*f.db, f.key, f.lease, 5s), "settled interval load")
.has_value(),
"settlement lost work or retained spendable pending state");
require(take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
"settled admission retry") == admitted &&
!take(load_population_interval(*f.db, f.key, f.lease, 5s), "settled retry pending")
.has_value(),
"old admission retry reopened settled work");
require(!admit_population_interval(*f.db, f.key, f.lease, f.head.head,
f.head.head.end_tick + 65, 5s),
"oversized interval admitted");
const auto denied = f.db->query(
"SELECT * FROM evolution.publish_population_checkpoint_v44(''::bytea,''::bytea)", {}, 5s);
require(!denied && std::string(denied.error().sqlstate.data(), 5) == "42501",
"runtime bypassed interval guard");
}
void isolation_and_reverse(Fixture& f) {
auto interval = unattended(f);
const auto old_lease = f.lease;
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release old authority");
auto new_db = connection();
f.lease = take(acquire_region_lease(*new_db, f.key, f.build, 60000ms, 5s),
"new population authority");
require(f.lease.epoch > old_lease.epoch, "authority epoch did not advance");
const auto old_head = f.head.head;
auto stale = publish_population_batch(*f.db, *f.objects, old_lease, f.head, interval.batch,
*interval.state, f.id(), f.build, 5s);
require(!stale && stale.error().store.kind == LeaseStoreError::fenced,
"stale population writer published");
recover(f);
require(f.head.head == old_head, "fenced writer changed head");
const auto expected = f.population->state().snapshot();
f.head = take(
materialize_population_profile(*new_db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
"return to cellular profile");
const std::array<Sha256Digest, 1> builds{f.build};
auto recovered =
take(recover_cellular_checkpoint(*new_db, *f.objects, f.key, f.lease, builds, 5s),
"cellular recovery after materialization");
require(recovered.state->snapshot() == expected &&
recovered.manifest.profile == CheckpointProfile::cellular_lives_v1,
"reverse transition changed biology or time");
f.raw = std::move(recovered.state);
}
void missing_and_corrupt_objects(Fixture& f) {
const auto manifest = take(decode_manifest(f.head.manifest), "object test manifest");
const auto path = f.objects->directory() / hexadecimal(manifest.payload_sha256, false);
const auto backup = path.string() + ".population-test";
const std::array<Sha256Digest, 1> builds{f.build};
std::filesystem::rename(path, backup);
auto missing = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
std::filesystem::rename(backup, path);
require(!missing && missing.error().kind == RecoveryError::missing_payload,
"missing population payload did not fail closed");
std::filesystem::rename(path, backup);
{
std::ofstream corrupt(path, std::ios::binary);
corrupt << 'X';
}
auto corrupt = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
std::filesystem::remove(path);
std::filesystem::rename(backup, path);
require(!corrupt && corrupt.error().kind == RecoveryError::corrupt_payload,
"corrupt population payload did not fail closed");
recover(f);
}
void boundary_upgrades(Fixture& f) {
const std::array<Sha256Digest, 1> builds{f.build};
const auto original = f.head;
const auto expected_raw =
take(f.raw->migration_upgrade_candidate(), "raw upgrade oracle")->snapshot();
// Rehearse the raw upgrade, recovery and ordinary trace-3 publication in one
// transaction, then roll it back to retain a real bundle-9 source for profile 5.
take(f.db->query("BEGIN", {}, 5s), "raw upgrade rollback rehearsal");
auto raw_head = take(
migrate_cellular_boundary_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
"raw accounting upgrade");
auto raw = take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s),
"raw accounting recovery");
require(raw.state->snapshot() == expected_raw &&
raw.manifest.cellular_versions == cellular_v10_versions &&
raw_head.head.end_tick == original.head.end_tick,
"raw accounting upgrade changed biology or time");
const CellularBatchHeader h{f.key.world,
f.key.region,
1,
raw_head.head.commit,
raw_head.head.digest,
raw_head.head.end_tick,
raw_head.head.end_tick + 1};
const auto inputs = take(unattended_population_inputs(*raw.state, h), "upgraded raw inputs");
auto advanced = take(replay_cellular_batch(*raw.state, inputs), "upgraded raw oracle");
take(publish_cellular_batch(*f.db, *f.objects, f.lease, raw_head, inputs, *advanced, f.id(),
f.build, 5s),
"upgraded raw ordinary publication");
require(take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s),
"upgraded raw ordinary recovery")
.state->snapshot() == advanced->snapshot(),
"raw accounting continuation differs");
take(f.db->query("ROLLBACK", {}, 5s), "roll back raw upgrade rehearsal");
require(take(load_region_checkpoint(*f.db, f.key, 5s), "raw rollback head").head ==
original.head,
"rolled-back upgrade moved the durable head");
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
f.build, {true, 1200, 1800}, 5s),
"population accounting source entry");
recover(f);
const auto member = std::ranges::find_if(f.population->state().lives(), [](const auto& life) {
return life.account == LifeAccount{};
});
require(member != f.population->state().lives().end(), "accounting source has no wildlife");
auto collapsed = take(
prepare_population_conversion(*f.population, f.header(0), {member->identity.genome, true}),
"accounting source cohort");
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, collapsed.batch,
*collapsed.state, f.id(), f.build, 5s),
"accounting source membership publication");
recover(f);
require(!f.population->members().empty() && f.population->conversion_sequence() != 0,
"accounting source lost cohort history");
auto interval = unattended(f);
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head,
interval.batch.header.end_tick, 5s),
"upgrade pending interval");
auto pending =
migrate_cellular_boundary_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s);
require(!pending && std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18",
"accounting upgrade discarded admitted work");
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, interval.batch,
*interval.state, f.id(), f.build, 5s),
"settle pre-upgrade work");
recover(f);
const auto before = f.head;
const auto source = take(decode_manifest(before.manifest), "accounting source manifest");
const auto upgraded_population =
take(f.population->migration_upgrade_candidate(), "cohort upgrade oracle");
const auto expected = upgraded_population->snapshot();
const auto altered =
take(CellularPopulation::create(restore(upgraded_population->state()), {false, 1200, 1800}),
"valid unequal upgrade target");
require(altered->state().current_total() == upgraded_population->state().current_total() &&
altered->snapshot() != expected,
"unequal upgrade fixture must retain biological totals");
const auto altered_object = take(f.objects->put(altered->snapshot()), "unequal upgrade object");
const auto altered_manifest = take(
cellular_boundary_upgrade_manifest(source, before.head.digest, f.id(), f.lease, f.build,
altered_object.bytes, altered_object.sha256),
"unequal upgrade manifest");
take(f.db->query("BEGIN", {}, 5s), "unequal upgrade recovery rehearsal");
take(publish_cellular_profile_migration(
*f.db, take(encode_manifest(altered_manifest), "unequal upgrade encoding"), 5s),
"metadata-only unequal upgrade publication");
const auto unequal =
recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
take(f.db->query("ROLLBACK", {}, 5s), "undo unequal upgrade rehearsal");
require(!unequal && unequal.error().kind == RecoveryError::history &&
take(load_region_checkpoint(*f.db, f.key, 5s), "head after unequal upgrade").head ==
before.head,
"upgrade recovery compared only aggregate stock or rewrote the retained source");
const auto child = ::fork();
require(child >= 0, "upgrade pre-transaction fork");
if (child == 0) {
try {
auto db = connection();
PublishHooks hooks;
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
[[maybe_unused]] const auto result = migrate_cellular_boundary_profile(
*db, *f.objects, f.lease, before, identity(60000), f.build, 5s, hooks);
} catch (...) {
::_exit(31);
}
::_exit(32);
}
int status{};
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
WTERMSIG(status) == SIGKILL,
"upgrade child missed pre-transaction kill");
recover(f);
require(f.head.head == before.head, "unpublished upgrade became authoritative");
auto locker = connection(true);
take(locker->query("BEGIN", {}, 5s), "accounting ambiguity lock transaction");
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 "
"AND region_id=1 FOR UPDATE",
{}, 5s),
"hold accounting reply");
auto timed = connection();
auto unknown = migrate_cellular_boundary_profile(*timed, *f.objects, f.lease, before, f.id(),
f.build, 300ms);
require(!unknown && unknown.error().kind == PublicationError::unknown &&
!unknown.error().prepared.manifest.empty(),
"uncertain accounting upgrade lost its exact request");
take(locker->query("ROLLBACK", {}, 5s), "release accounting reply");
auto resolver = connection();
f.head = take(retry_cellular_profile_migration(*resolver, unknown.error().prepared, 5s),
"retry exact accounting upgrade");
require(take(retry_cellular_profile_migration(*resolver, unknown.error().prepared, 5s),
"duplicate accounting upgrade")
.head == f.head.head,
"duplicate accounting upgrade advanced head");
recover(f);
require(f.population->snapshot() == expected && f.head.head.end_tick == before.head.end_tick &&
take(decode_manifest(f.head.manifest), "upgraded population manifest")
.cellular_versions == cellular_v10_versions,
"accounting upgrade changed biology, cohort history or time");
auto changed = unknown.error().prepared;
auto changed_manifest = take(decode_manifest(changed.manifest), "changed accounting manifest");
changed_manifest.payload_sha256[0] ^= 1;
changed.manifest = take(encode_manifest(changed_manifest), "changed accounting bytes");
auto conflict = retry_cellular_profile_migration(*resolver, changed, 5s);
require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict,
"accounting retry accepted changed bytes");
require(
!migrate_cellular_boundary_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
"accounting upgrade applied twice");
const auto path = f.objects->directory() / hexadecimal(source.payload_sha256, false);
const auto backup = path.string() + ".upgrade-test";
std::filesystem::rename(path, backup);
const auto missing =
recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
std::filesystem::rename(backup, path);
require(!missing && missing.error().kind == RecoveryError::tail,
"accounting recovery trusted aggregate stock without its retained source");
missing_and_corrupt_objects(f);
const auto stale = f.lease;
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release accounting writer");
f.lease =
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "replace accounting writer");
auto stale_retry = retry_cellular_profile_migration(*f.db, unknown.error().prepared, 5s);
require(!stale_retry && stale_retry.error().store.kind == LeaseStoreError::fenced &&
!recover_population_checkpoint(*f.db, *f.objects, f.key, stale, builds, 5s),
"old accounting receipt granted current authority");
recover(f);
require(f.population->snapshot() == expected, "new epoch changed accounting upgrade");
auto future_interval = unattended(f);
auto old_trace = future_interval.batch;
old_trace.format_version = 1;
const auto old_bytes = take(encode_population_batch(old_trace), "wrong old trace fixture");
const auto upgraded_manifest = take(decode_manifest(f.head.manifest), "new accounting source");
auto wrong_trace_manifest =
take(population_successor_manifest(
upgraded_manifest, f.head.head.digest, f.id(), f.lease, old_trace.header.end_tick,
f.build, future_interval.state->snapshot().size(),
take(sha256(future_interval.state->snapshot()), "new payload SHA"),
{old_bytes.size(), take(sha256(old_bytes), "old trace SHA"), 0, {}}),
"wrong old trace manifest fixture");
const PopulationPreparedCommit wrong_trace{
f.key, wrong_trace_manifest.commit,
take(encode_manifest(wrong_trace_manifest), "wrong old trace manifest"), old_bytes};
auto refused_old = retry_population_prepared(*f.db, wrong_trace, 5s);
auto refused_old_sql =
publish_population_checkpoint(*f.db, wrong_trace.manifest, old_bytes, 5s);
require(!refused_old && refused_old.error().kind == PublicationError::candidate &&
!refused_old_sql && refused_old_sql.error().kind == LeaseStoreError::invalid &&
take(load_region_checkpoint(*f.db, f.key, 5s), "head after old trace").head ==
f.head.head,
"bundle 10 reinterpreted retained population trace 1");
// Exercise the existing publication/process-loss matrix with batch 2 as well.
process_loss(f, 0);
process_loss(f, 1);
process_loss(f, 2);
isolation_and_reverse(f);
require(f.raw->format_version() == cellular_lives_migration_format,
"reverse transition downgraded accounting owner");
}
void migration_publication(Fixture& f, const RegionKey destination_key,
FilesystemObjectStore& destination_objects,
LeaseGrant& destination_lease, CommittedCheckpoint& destination_head) {
const std::array<Sha256Digest, 1> builds{f.build};
const auto wildlife = std::ranges::find_if(
f.raw->lives(), [](const auto& life) { return life.account == LifeAccount{}; });
require(wildlife != f.raw->lives().end(), "paired publication needs real wildlife");
const auto genome = wildlife->identity.genome;
const auto make_host = [&](RegionKey key) {
auto objects = take(
FilesystemObjectStore::open(environment("PGHOST"), key, max_cellular_population_bytes),
"paired host objects");
CellularHostConfig config{key, 1, {}, f.build, {f.build}};
config.lease.commit_timeout = 300ms;
return take(
CellularRegionHost::create(
config,
[] { return std::expected<std::unique_ptr<Postgres>, PgFailure>{connection()}; },
std::move(objects)),
"paired host");
};
for (unsigned representation = 0; representation < 2; ++representation) {
if (representation == 0) {
f.head = take(materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(),
f.build, 5s),
"paired raw source");
destination_head =
take(materialize_population_profile(*f.db, destination_objects, destination_lease,
destination_head, f.id(), f.build, 5s),
"paired raw destination");
} else {
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
f.build, {true, 1200, 1800}, 5s),
"paired population source");
destination_head = take(migrate_cellular_to_population(
*f.db, destination_objects, destination_lease,
destination_head, f.id(), f.build, {true, 1200, 1800}, 5s),
"paired population destination");
}
for (unsigned direction = 0; direction < 2; ++direction) {
auto& sh = direction == 0 ? f.head : destination_head;
auto& dh = direction == 0 ? destination_head : f.head;
auto& so = direction == 0 ? *f.objects : destination_objects;
auto& dst = direction == 0 ? destination_objects : *f.objects;
auto& sl = direction == 0 ? f.lease : destination_lease;
auto& dl = direction == 0 ? destination_lease : f.lease;
const auto sk = direction == 0 ? f.key : destination_key;
const auto dk = direction == 0 ? destination_key : f.key;
take(release_region_lease(*f.db, sk, sl, 5s), "release paired source to host");
take(release_region_lease(*f.db, dk, dl, 5s), "release paired destination to host");
auto source_host = make_host(sk), destination_host = make_host(dk);
require(source_host->step().ready && destination_host->step().ready,
"paired hosts before preparation");
sl = take(source_host->authority(), "current source authority before pause");
dl = take(destination_host->authority(), "current destination authority before pause");
const auto source_before = sh, destination_before = dh;
const auto s = take(decode_manifest(sh.manifest), "paired predecessor source");
const auto d = take(decode_manifest(dh.manifest), "paired predecessor destination");
const auto sb =
take(so.get({s.payload_sha256, s.payload_bytes}), "paired source bytes");
const auto dbb =
take(dst.get({d.payload_sha256, d.payload_bytes}), "paired destination bytes");
const CellularMigrationRequest request{
f.id(),
{sk.region, sh.head.commit, f.id(), sh.head.digest},
{dk.region, dh.head.commit, f.id(), dh.head.digest},
genome};
take(source_host->prepare_migration_freeze(dh, dst, request),
"owning paired source freeze");
const auto frozen = take(take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s),
"paired source evidence"),
"missing paired freeze");
// Preparation alone cannot authorize either result.
require(!publish_cellular_migration(*f.db, so, dst, sl, dl, sh, dh, frozen.trace,
f.build, 5s),
"unstaged migration published");
take(destination_host->stage_migration(sh, so, frozen.trace), "owning paired stage");
std::vector<std::uint8_t> expected_source, expected_destination;
if (representation == 0) {
auto a = take(restore_cellular_payload(sb, s), "paired raw source oracle");
auto b = take(restore_cellular_payload(dbb, d), "paired raw destination oracle");
auto pair = take(replay_migration_trace(*a, *b, request, frozen.trace),
"paired raw oracle");
expected_source = pair.source->snapshot();
expected_destination = pair.destination->snapshot();
} else {
auto a = take(restore_population_payload(sb, s), "paired population source oracle");
auto b = take(restore_population_payload(dbb, d),
"paired population destination oracle");
auto pair = take(replay_migration_trace(*a, *b, request, frozen.trace),
"paired population oracle");
expected_source = pair.source->snapshot();
expected_destination = pair.destination->snapshot();
}
// Fail specifically on the SECOND result insert, after the first and receipt.
take(f.owner->query(
"CREATE FUNCTION evolution.test_refuse_second_migration() RETURNS trigger "
"LANGUAGE plpgsql AS $$ BEGIN IF NEW.commit_id='" +
hexadecimal(request.destination.commit, false) +
"'::uuid THEN RAISE EXCEPTION 'injected second insert' USING "
"ERRCODE='EVL06'; END IF; RETURN NEW; END $$",
{}, 5s),
"second insert fault function");
take(f.owner->query(
"CREATE TRIGGER test_refuse_second_migration BEFORE INSERT ON "
"evolution.region_commits "
"FOR EACH ROW EXECUTE FUNCTION evolution.test_refuse_second_migration()",
{}, 5s),
"second insert fault trigger");
const auto rejected = publish_cellular_migration(*f.db, so, dst, sl, dl, sh, dh,
frozen.trace, f.build, 5s);
take(f.owner->query(
"DROP TRIGGER test_refuse_second_migration ON evolution.region_commits", {},
5s),
"remove second insert fault");
take(f.owner->query("DROP FUNCTION evolution.test_refuse_second_migration()", {}, 5s),
"remove fault function");
require(
!rejected && rejected.error().store.kind == LeaseStoreError::unsupported &&
take(load_region_checkpoint(*f.db, sk, 5s), "rollback source head").head ==
source_before.head &&
take(load_region_checkpoint(*f.db, dk, 5s), "rollback destination head").head ==
destination_before.head &&
take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s),
"rollback source pause")
.has_value() &&
take(load_cellular_migration_stage(*f.db, dst, dk, dl, 5s),
"rollback destination pause")
.has_value(),
"failed second publication left half a migration or released a pause");
const auto prepared = rejected.error().prepared;
take(f.db->query("BEGIN", {}, 5s), "false paired biology transaction");
auto false_pair = prepared;
auto false_source =
take(decode_manifest(false_pair.source_manifest), "false source manifest");
false_source.payload_sha256 = s.payload_sha256;
false_source.payload_bytes = s.payload_bytes;
false_pair.source_manifest =
take(encode_manifest(false_source), "false source canonical bytes");
take(retry_cellular_migration_commit(*f.db, sl, dl, false_pair, 5s),
"metadata-only false pair");
const auto false_recovery = verify_cellular_migration_checkpoint(
*f.db, so, sl, take(load_region_checkpoint(*f.db, sk, 5s), "false paired head"), sb,
5s);
take(f.db->query("ROLLBACK", {}, 5s), "rollback false paired biology fixture");
require(!false_recovery && false_recovery.error().kind == RecoveryError::history,
"paired recovery accepted a valid old object as the migrated biology");
const std::array<std::string, 1> generic{hexadecimal(prepared.source_manifest)};
require(!f.db->query("SELECT * FROM evolution.publish_region_checkpoint($1::bytea)",
generic, 5s),
"generic publisher admitted a migration half");
const auto child = ::fork();
require(child >= 0, "pre-transaction paired fork");
if (child == 0) {
try {
auto db = connection();
PublishHooks hooks{[] { ::kill(::getpid(), SIGKILL); }};
if (!publish_cellular_migration(*db, so, dst, sl, dl, sh, dh, frozen.trace,
f.build, 5s, hooks))
::_exit(52);
} catch (...) {
::_exit(51);
}
::_exit(52);
}
int status{};
require(
::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
WTERMSIG(status) == SIGKILL &&
take(load_region_checkpoint(*f.db, sk, 5s), "killed pair source").head ==
source_before.head &&
take(load_region_checkpoint(*f.db, dk, 5s), "killed pair destination").head ==
destination_before.head,
"pre-transaction paired death changed a head");
if (direction == 0) {
auto locker = connection(true), timed = connection();
take(locker->query("BEGIN", {}, 5s), "pair ambiguity transaction");
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND "
"region_id=" +
std::to_string(sk.region) + " FOR UPDATE",
{}, 5s),
"hold paired reply");
const auto unknown =
retry_cellular_migration_commit(*timed, sl, dl, prepared, 300ms);
require(!unknown && unknown.error().kind == PublicationError::unknown &&
unknown.error().prepared.source_manifest == prepared.source_manifest &&
unknown.error().prepared.destination_manifest ==
prepared.destination_manifest,
"ambiguous pair lost its exact manifests");
take(locker->query("ROLLBACK", {}, 5s), "release paired ambiguity");
} else {
const auto committed_child = ::fork();
require(committed_child >= 0, "post-commit paired fork");
if (committed_child == 0) {
try {
auto db = connection();
if (!retry_cellular_migration_commit(*db, sl, dl, prepared, 5s))
::_exit(53);
::kill(::getpid(), SIGKILL);
} catch (...) {
::_exit(54);
}
::_exit(55);
}
require(::waitpid(committed_child, &status, 0) == committed_child &&
WIFSIGNALED(status) && WTERMSIG(status) == SIGKILL,
"paired publisher failed before post-commit kill");
}
const auto committed =
take(retry_cellular_migration_commit(*f.db, sl, dl, prepared, 5s),
"resolve exact paired commit");
sh = committed.source;
dh = committed.destination;
require(!abort_cellular_migration_freeze(*f.db, sk, sl, request.request, 5s),
"committed migration was aborted");
auto changed = prepared;
auto altered =
take(decode_manifest(changed.destination_manifest), "conflicting committed pair");
altered.payload_sha256.back() ^= 1;
changed.destination_manifest = take(encode_manifest(altered), "conflicting pair bytes");
const auto conflict = retry_cellular_migration_commit(*f.db, sl, dl, changed, 5s);
auto stale = sl;
--stale.epoch;
require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict &&
!retry_cellular_migration_commit(*f.db, stale, dl, prepared, 5s),
"changed pair or stale source authority was accepted");
for (unsigned attempt = 0;
attempt < 200 && (!source_host->step().ready || !destination_host->step().ready);
++attempt)
std::this_thread::sleep_for(25ms);
require(source_host->status().ready && destination_host->status().ready &&
take(source_host->committed(), "adopted source").head == sh.head &&
take(destination_host->committed(), "adopted destination").head == dh.head,
"pointer release resumed a stale owner instead of recovering the pair");
const auto check_bytes = [&](const CommittedCheckpoint& h, RegionKey key,
const FilesystemObjectStore& objects,
const LeaseGrant& lease) {
require(take(load_region_checkpoint(*f.db, key, 5s), "current paired recovery head")
.head == h.head,
"paired recovery selected a different head");
if (representation == 0)
return take(recover_cellular_checkpoint(*f.db, objects, key, lease, builds, 5s),
"recover raw paired result")
.state->snapshot();
return take(recover_population_checkpoint(*f.db, objects, key, lease, builds, 5s),
"recover cohort paired result")
.state->snapshot();
};
require(check_bytes(sh, sk, so, sl) == expected_source &&
check_bytes(dh, dk, dst, dl) == expected_destination,
"committed biology/member/material differs from the exact pair");
const auto capsule_path = so.directory() / hexadecimal(d.payload_sha256, false);
const auto backup = capsule_path.string() + ".paired-test";
std::filesystem::rename(capsule_path, backup);
const auto missing =
verify_cellular_migration_checkpoint(*f.db, so, sl, sh, expected_source, 5s);
std::filesystem::rename(backup, capsule_path);
require(!missing && missing.error().kind == RecoveryError::tail &&
!verify_cellular_migration_checkpoint(*f.db, so, sl, sh, sb, 5s),
"pair recovery ignored missing counterpart evidence or false local biology");
source_host->drain();
destination_host->drain();
source_host = make_host(sk);
destination_host = make_host(dk);
require(source_host->step().ready && destination_host->step().ready,
"replacement paired hosts failed complete recovery");
sl = take(source_host->authority(), "replacement source fence");
dl = take(destination_host->authority(), "replacement destination fence");
require(take(retry_cellular_migration_commit(*f.db, sl, dl, prepared, 5s),
"new epoch exact paired retry")
.source.head == sh.head,
"fresh authority changed a committed retry");
source_host->drain();
destination_host->drain();
sl = take(acquire_region_lease(*f.db, sk, f.build, 60000ms, 5s),
"return paired source authority");
dl = take(acquire_region_lease(*f.db, dk, f.build, 60000ms, 5s),
"return paired destination authority");
}
}
// Race the actual two-head transaction against abort on independent connections.
const auto prepare_pair = [&](bool reverse) {
auto& sh = reverse ? destination_head : f.head;
auto& dh = reverse ? f.head : destination_head;
auto& so = reverse ? destination_objects : *f.objects;
auto& dst = reverse ? *f.objects : destination_objects;
auto& sl = reverse ? destination_lease : f.lease;
auto& dl = reverse ? f.lease : destination_lease;
const auto sk = reverse ? destination_key : f.key;
const auto dk = reverse ? f.key : destination_key;
CellularMigrationRequest request{f.id(),
{sk.region, sh.head.commit, f.id(), sh.head.digest},
{dk.region, dh.head.commit, f.id(), dh.head.digest},
genome};
const auto freeze =
take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s),
"commit/abort source preparation");
take(stage_cellular_migration(*f.db, so, dst, dl, sh, dh, freeze.trace, 5s),
"commit/abort destination stage");
return freeze.trace;
};
const auto aborted_trace = prepare_pair(false);
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, aborted_trace.request.request, 5s),
"abort wins before paired commit");
const auto after_abort = publish_cellular_migration(
*f.db, *f.objects, destination_objects, f.lease, destination_lease, f.head,
destination_head, aborted_trace, f.build, 5s);
require(!after_abort && after_abort.error().store.kind == LeaseStoreError::commit_conflict,
"paired publication reopened an aborted workflow");
for (unsigned race = 0; race < 4; ++race) {
const auto trace = prepare_pair(false);
auto committing = connection(), aborting = connection();
std::barrier start(2);
std::optional<std::expected<CellularMigrationCommitted, CellularMigrationCommitFailure>>
published;
std::optional<std::expected<CellularMigrationFreeze, LeaseStoreFailure>> aborted;
std::thread commit_thread([&] {
start.arrive_and_wait();
published = publish_cellular_migration(*committing, *f.objects, destination_objects,
f.lease, destination_lease, f.head,
destination_head, trace, f.build, 5s);
});
std::thread abort_thread([&] {
start.arrive_and_wait();
aborted = abort_cellular_migration_freeze(*aborting, f.key, f.lease,
trace.request.request, 5s);
});
commit_thread.join();
abort_thread.join();
require(published.has_value() && aborted.has_value() &&
(published->has_value() != aborted->has_value()),
"commit/abort race had two outcomes or no definitive outcome");
if (published->has_value()) {
require(aborted->error().kind == LeaseStoreError::commit_conflict,
"losing abort was not refused");
f.head = (*published)->source;
destination_head = (*published)->destination;
const auto returning = prepare_pair(true);
const auto returned =
take(publish_cellular_migration(*f.db, destination_objects, *f.objects,
destination_lease, f.lease, destination_head,
f.head, returning, f.build, 5s),
"return race winner's stock");
destination_head = returned.source;
f.head = returned.destination;
} else {
require(published->error().store.kind == LeaseStoreError::commit_conflict,
"losing paired publication was not refused");
}
require(
take(load_region_checkpoint(*f.db, f.key, 5s), "race source head").head ==
f.head.head &&
take(load_region_checkpoint(*f.db, destination_key, 5s), "race destination head")
.head == destination_head.head &&
!take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s),
"race source settled")
.has_value() &&
!take(load_cellular_migration_stage(*f.db, destination_objects, destination_key,
destination_lease, 5s),
"race destination settled")
.has_value(),
"commit/abort race left half a head or a pending pause");
}
recover(f);
f.raw = restore(f.population->state());
}
void migration_freezes(Fixture& f) {
const RegionKey destination_key{f.key.world, 2};
auto destination_objects =
take(FilesystemObjectStore::open(environment("PGHOST"), destination_key,
max_cellular_population_bytes),
"migration destination objects");
auto destination = opening(30001, true, 4, false, 2);
auto destination_head =
take(provision_cellular_region(*f.owner, *destination_objects, destination_key, 1,
*destination, f.id(), f.build, 5s),
"migration empty destination genesis");
auto destination_lease =
take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s),
"migration destination authority");
while (destination->tick() < f.raw->tick()) {
const auto count = std::min(std::uint64_t{64}, f.raw->tick() - destination->tick());
CellularBatch inputs{{f.key.world, 2, 1, destination_head.head.commit,
destination_head.head.digest, destination->tick(),
destination->tick() + count},
{}};
inputs.ticks.resize(static_cast<std::size_t>(count));
destination =
take(replay_cellular_batch(*destination, inputs), "empty destination biology");
destination_head = take(publish_cellular_batch(*f.db, *destination_objects,
destination_lease, destination_head, inputs,
*destination, f.id(), f.build, 5s),
"empty destination tick publication")
.committed;
}
destination_head =
take(migrate_cellular_boundary_profile(*f.db, *destination_objects, destination_lease,
destination_head, f.id(), f.build, 5s),
"destination accounting upgrade");
destination = take(destination->migration_upgrade_candidate(), "destination upgraded oracle");
const std::array<Sha256Digest, 1> builds{f.build};
const auto request_for = [&] {
const auto member = std::ranges::find_if(
f.raw->lives(), [](const auto& life) { return life.account == LifeAccount{}; });
require(member != f.raw->lives().end(), "freeze fixture lacks real wildlife");
return CellularMigrationRequest{
f.id(),
{1, f.head.head.commit, f.id(), f.head.head.digest},
{2, destination_head.head.commit, f.id(), destination_head.head.digest},
member->identity.genome};
};
for (unsigned representation = 0; representation < 2; ++representation) {
if (representation == 1) {
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
f.build, {true, 1200, 1800}, 5s),
"freeze cohort source entry");
recover(f);
destination_head = take(migrate_cellular_to_population(
*f.db, *destination_objects, destination_lease,
destination_head, f.id(), f.build, {true, 1200, 1800}, 5s),
"freeze cohort destination entry");
}
const auto source_before = f.head;
const auto source_bytes = f.raw->snapshot();
const auto destination_before = destination_head;
const auto request = request_for();
const auto prepare = [&](const CellularMigrationRequest& requested,
std::chrono::milliseconds timeout = 5s) {
return prepare_cellular_migration_freeze(*f.db, *f.objects, *destination_objects,
f.lease, f.head, destination_head, requested,
timeout);
};
if (representation == 0) {
Uuid account{};
account.back() = 1;
take(f.db->query("BEGIN", {}, 5s), "pending player choice before freeze");
const auto control =
take(claim_account_control(*f.db, f.key, f.lease, account, 2, 9900, 60000ms, 5s),
"pending choice current control");
const auto input = encode_cellular_mutation_input(
{account, 9900, f.head.head.end_tick + 1, {100, 1, 1, 1}});
take(reserve_cellular_mutation(*f.db, f.key, f.lease, account, control.generation, 9900,
input, 5s),
"reserve pre-freeze player choice");
const auto pending = prepare(request);
take(f.db->query("ROLLBACK", {}, 5s), "rollback pre-freeze player choice");
require(!pending &&
std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18",
"preparation stranded a player choice");
} else {
take(f.db->query("BEGIN", {}, 5s), "prepare with admitted work transaction");
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head,
f.head.head.end_tick + 20, 5s),
"admitted work before freeze");
const auto pending = prepare(request);
take(f.db->query("ROLLBACK", {}, 5s), "rollback pending-work preparation fixture");
require(!pending &&
std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18",
"preparation skipped durable ecological work");
}
auto frozen = take(prepare(request), "durable source freeze");
const CellularMigrationPrepared prepared{
f.key, take(encode_migration_trace(frozen.trace), "frozen canonical trace")};
auto malformed = prepared.trace;
malformed.back() ^= 1;
const std::array<std::string, 3> malformed_args{hexadecimal(f.lease.holder, false),
std::to_string(f.lease.epoch),
hexadecimal(malformed)};
const auto malformed_sql = f.db->query("SELECT * FROM "
"evolution.prepare_cellular_migration_freeze(30201,"
"1,$1::uuid,$2::evolution.u64,$3::bytea)",
malformed_args, 5s);
require(!malformed_sql && std::string(malformed_sql.error().sqlstate.data(), 5) == "22023",
"SQL accepted an invalid trace checksum before duplicate lookup");
require(
!frozen.aborted && frozen.trace.request == request &&
take(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s),
"replay frozen capsule"),
"missing durable freeze")
.trace == frozen.trace &&
take(load_region_checkpoint(*f.db, f.key, 5s), "frozen source head").head ==
source_before.head &&
take(load_region_checkpoint(*f.db, destination_key, 5s), "unowned destination head")
.head == destination_before.head,
"preparation changed either owner or lost its full replay capsule");
require(
!take(retry_cellular_migration_freeze(*f.db, f.lease, prepared, 5s), "duplicate freeze")
.aborted,
"duplicate freeze lost its state");
const auto stage = [&](const CellularMigrationTrace& trace,
std::chrono::milliseconds timeout = 5s) {
return stage_cellular_migration(*f.db, *f.objects, *destination_objects,
destination_lease, f.head, destination_head, trace,
timeout);
};
// The optimistic destination cut must still be exact at first staging.
take(f.db->query("BEGIN", {}, 5s), "move destination before stage transaction");
if (representation == 0)
take(migrate_cellular_to_population(*f.db, *destination_objects, destination_lease,
destination_head, f.id(), f.build, {}, 5s),
"temporarily move destination profile");
else
take(materialize_population_profile(*f.db, *destination_objects, destination_lease,
destination_head, f.id(), f.build, 5s),
"temporarily move population destination");
const auto moved = stage(frozen.trace);
take(f.db->query("ROLLBACK", {}, 5s), "rollback moved destination fixture");
require(!moved && moved.error().store.kind == LeaseStoreError::head_mismatch,
"stage silently rebased a moved destination");
if (representation == 1) {
take(f.db->query("BEGIN", {}, 5s), "stage pending interval transaction");
take(admit_population_interval(*f.db, destination_key, destination_lease,
destination_head.head,
destination_head.head.end_tick + 20, 5s),
"destination admitted work before staging");
const auto pending = stage(frozen.trace);
take(f.db->query("ROLLBACK", {}, 5s), "rollback destination interval fixture");
require(!pending && pending.error().store.kind == LeaseStoreError::commit_conflict,
"stage stranded destination interval work");
}
const auto staged = take(stage(frozen.trace), "durable destination staging");
const CellularMigrationStagePrepared stage_request{destination_key, prepared.trace,
staged.ghost};
require(
!staged.freeze.aborted &&
take(take(load_cellular_migration_stage(*f.db, *destination_objects,
destination_key, destination_lease, 5s),
"recover destination ghost"),
"missing staged destination")
.ghost == staged.ghost &&
take(load_region_checkpoint(*f.db, destination_key, 5s),
"staged destination still owns original head")
.head == destination_before.head &&
take(retry_cellular_migration_stage(*f.db, destination_lease, stage_request, 5s),
"duplicate stage")
.ghost == staged.ghost,
"staging activated a ghost or lost exact replay");
auto changed_stage = stage_request;
++changed_stage.ghost.bytes;
const auto changed =
retry_cellular_migration_stage(*f.db, destination_lease, changed_stage, 5s);
require(!changed && changed.error().store.kind == LeaseStoreError::commit_conflict,
"stage accepted conflicting immutable ghost bytes");
auto reverse = frozen.trace;
reverse.request.request = f.id();
std::swap(reverse.request.source, reverse.request.destination);
const auto overlapping = retry_cellular_migration_freeze(
*f.db, destination_lease,
{destination_key, take(encode_migration_trace(reverse), "overlapping source trace")},
5s);
require(!overlapping && overlapping.error().store.kind == LeaseStoreError::commit_conflict,
"staged destination simultaneously prepared an outgoing workflow");
take(f.db->query("BEGIN", {}, 5s), "staged destination life reservation transaction");
Uuid new_account{};
new_account.back() = 5;
const auto staged_control =
take(claim_account_control(*f.db, destination_key, destination_lease, new_account, 1,
9950, 60000ms, 5s),
"staged destination reservation current control");
const auto staged_life =
reserve_primitive_life(*f.db, destination_key, destination_lease, new_account,
staged_control.generation, 1, 1, 5s);
take(f.db->query("ROLLBACK", {}, 5s), "rollback staged life reservation fixture");
require(!staged_life &&
std::string(staged_life.error().cause.sqlstate.data(), 5) == "EVL18",
"staged destination admitted a new player life reservation");
if (representation == 0) {
CellularBatch inputs{{f.key.world, 2, 1, destination_head.head.commit,
destination_head.head.digest, destination->tick(),
destination->tick() + 1},
{{}}};
const auto advanced = take(replay_cellular_batch(*destination, inputs),
"detached staged destination future");
const auto denied =
publish_cellular_batch(*f.db, *destination_objects, destination_lease,
destination_head, inputs, *advanced, f.id(), f.build, 5s);
require(!denied &&
std::string(denied.error().store.cause.sqlstate.data(), 5) == "EVL18",
"staged destination published an ordinary future");
} else {
require(!admit_population_interval(*f.db, destination_key, destination_lease,
destination_head.head,
destination_head.head.end_tick + 20, 5s) &&
!materialize_population_profile(*f.db, *destination_objects,
destination_lease, destination_head, f.id(),
f.build, 5s),
"staged destination progressed or changed profile");
}
const auto ghost_path =
destination_objects->directory() / hexadecimal(staged.ghost.sha256, false);
const auto ghost_backup = ghost_path.string() + ".stage-test";
std::filesystem::rename(ghost_path, ghost_backup);
const auto missing_ghost = load_cellular_migration_stage(
*f.db, *destination_objects, destination_key, destination_lease, 5s);
std::filesystem::rename(ghost_backup, ghost_path);
require(!missing_ghost && missing_ghost.error().kind == RecoveryError::tail,
"missing ghost fell back to the active destination");
char ghost_first{};
{
std::fstream file(ghost_path, std::ios::binary | std::ios::in | std::ios::out);
file.get(ghost_first);
file.seekp(0);
file.put(static_cast<char>(static_cast<unsigned char>(ghost_first) ^ 1U));
file.flush();
require(file.good(), "same-length ghost corruption fixture");
}
const auto corrupt_ghost = load_cellular_migration_stage(
*f.db, *destination_objects, destination_key, destination_lease, 5s);
{
std::fstream file(ghost_path, std::ios::binary | std::ios::in | std::ios::out);
file.put(ghost_first);
file.flush();
require(file.good(), "restore corrupt ghost fixture");
}
require(!corrupt_ghost && corrupt_ghost.error().kind == RecoveryError::tail,
"same-length corrupt ghost bypassed digest verification");
const auto source_manifest = take(decode_manifest(f.head.manifest), "stage source capsule");
const auto source_capsule_path =
destination_objects->directory() / hexadecimal(source_manifest.payload_sha256, false);
const auto source_capsule_backup = source_capsule_path.string() + ".stage-test";
std::filesystem::rename(source_capsule_path, source_capsule_backup);
const auto missing_source = load_cellular_migration_stage(
*f.db, *destination_objects, destination_key, destination_lease, 5s);
std::filesystem::rename(source_capsule_backup, source_capsule_path);
require(!missing_source && missing_source.error().kind == RecoveryError::tail,
"destination recovery used the live source instead of its retained capsule");
const auto old_destination_lease = destination_lease;
take(release_region_lease(*f.db, destination_key, destination_lease, 5s),
"release staged destination owner");
destination_lease = take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s),
"replacement staged destination owner");
require(
take(load_cellular_migration_stage(*f.db, *destination_objects, destination_key,
destination_lease, 5s),
"new epoch ghost replay")
.has_value() &&
!retry_cellular_migration_stage(*f.db, old_destination_lease, stage_request, 5s),
"replacement activated the ghost or stale destination staged it");
auto conflict_trace = frozen.trace;
++conflict_trace.transferred.energy.value;
auto conflict = retry_cellular_migration_freeze(
*f.db, f.lease, {f.key, take(encode_migration_trace(conflict_trace), "conflict trace")},
5s);
require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict &&
!prepare(request_for()),
"freeze accepted changed bytes or replaced pending request");
if (representation == 0) {
const auto h = f.header(1);
const auto inputs = take(unattended_population_inputs(*f.raw, h), "frozen raw inputs");
auto advanced = take(replay_cellular_batch(*f.raw, inputs), "frozen detached future");
auto denied = publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, inputs,
*advanced, f.id(), f.build, 5s);
require(!denied &&
std::string(denied.error().store.cause.sqlstate.data(), 5) == "EVL18" &&
!migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
f.build, {}, 5s),
"raw source progressed or changed profile while frozen");
Uuid account{};
account.back() = 1;
const auto input = encode_cellular_mutation_input(
{account, 9901, f.head.head.end_tick + 1, {100, 1, 1, 1}});
take(f.db->query("BEGIN", {}, 5s), "current control while frozen fixture");
const auto control =
take(claim_account_control(*f.db, f.key, f.lease, account, 2, 9901, 60000ms, 5s),
"current frozen-source account control");
const auto choice = reserve_cellular_mutation(*f.db, f.key, f.lease, account,
control.generation, 9901, input, 5s);
take(f.db->query("ROLLBACK", {}, 5s), "rollback frozen control fixture");
require(!choice, "frozen raw owner admitted a new mutation reservation");
if (std::string(choice.error().cause.sqlstate.data(), 5) != "EVL18")
throw std::runtime_error("frozen mutation guard SQLSTATE=" +
std::string(choice.error().cause.sqlstate.data(), 5));
} else {
require(!admit_population_interval(*f.db, f.key, f.lease, f.head.head,
f.head.head.end_tick + 20, 5s) &&
!materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(),
f.build, 5s),
"population source admitted work or changed profile while frozen");
}
const auto capsule_manifest =
take(decode_manifest(destination_head.manifest), "capsule manifest");
const auto capsule_path =
f.objects->directory() / hexadecimal(capsule_manifest.payload_sha256, false);
const auto backup = capsule_path.string() + ".freeze-test";
std::filesystem::rename(capsule_path, backup);
const auto missing = load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s);
std::filesystem::rename(backup, capsule_path);
require(!missing && missing.error().kind == RecoveryError::tail,
"freeze recovery fell back to destination live state without the retained capsule");
char first{};
{
std::fstream file(capsule_path, std::ios::binary | std::ios::in | std::ios::out);
file.get(first);
file.seekp(0);
file.put(static_cast<char>(static_cast<unsigned char>(first) ^ 1U));
file.flush();
require(file.good(), "corrupt same-length capsule fixture");
}
const auto corrupt = load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s);
{
std::fstream file(capsule_path, std::ios::binary | std::ios::in | std::ios::out);
file.put(first);
file.flush();
require(file.good(), "restore same-length capsule fixture");
}
require(!corrupt && corrupt.error().kind == RecoveryError::tail,
"corrupt capsule bypassed strong object binding");
const auto old_lease = f.lease;
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release frozen source owner");
f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s),
"new frozen source owner");
require(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s),
"recover freeze under new epoch")
.has_value() &&
!abort_cellular_migration_freeze(*f.db, f.key, old_lease, request.request, 5s),
"new epoch released a freeze or old epoch aborted it");
require(
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, request.request, 5s),
"explicit source abort")
.aborted &&
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, request.request, 5s),
"duplicate source abort")
.aborted &&
take(retry_cellular_migration_freeze(*f.db, f.lease, prepared, 5s),
"retry aborted preparation")
.aborted &&
!take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s),
"aborted pointer load")
.has_value(),
"abort was not durable or preparation retry reopened it");
require(
!take(load_cellular_migration_stage(*f.db, *destination_objects, destination_key,
destination_lease, 5s),
"aborted destination pointer")
.has_value() &&
take(retry_cellular_migration_stage(*f.db, destination_lease, stage_request, 5s),
"retry aborted stage")
.freeze.aborted,
"source abort stranded the destination or stage retry reopened its pointer");
require(take(load_region_checkpoint(*f.db, f.key, 5s), "aborted source head").head ==
source_before.head &&
f.raw->snapshot() == source_bytes,
"abort changed biological stock/time/history");
auto false_trace = frozen.trace;
false_trace.request.request = f.id();
false_trace.request.source.commit = f.id();
false_trace.request.destination.commit = f.id();
++false_trace.transferred.energy.value;
take(retry_cellular_migration_freeze(*f.db, f.lease,
{f.key, take(encode_migration_trace(false_trace),
"canonical false inventory trace")},
5s),
"metadata-only false preparation");
const auto false_replay =
load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s);
require(!false_replay && false_replay.error().kind == RecoveryError::history,
"freeze recovery trusted declared material without complete biological replay");
take(
abort_cellular_migration_freeze(*f.db, f.key, f.lease, false_trace.request.request, 5s),
"explicitly abort false preparation without moving stock");
// A genuine unknown reply retains exactly the chosen cut and group.
auto locker = connection(true);
take(locker->query("BEGIN", {}, 5s), "freeze ambiguity lock");
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND "
"region_id=1 FOR UPDATE",
{}, 5s),
"hold freeze reply");
auto timed = connection();
auto unknown =
prepare_cellular_migration_freeze(*timed, *f.objects, *destination_objects, f.lease,
f.head, destination_head, request_for(), 300ms);
require(!unknown && unknown.error().kind == PublicationError::unknown &&
!unknown.error().prepared.trace.empty(),
"freeze ambiguity lost its immutable request");
take(locker->query("ROLLBACK", {}, 5s), "release freeze ambiguity");
frozen = take(retry_cellular_migration_freeze(*f.db, f.lease, unknown.error().prepared, 5s),
"reconcile unknown freeze");
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, frozen.trace.request.request,
5s),
"abort reconciled freeze");
const auto false_stage_freeze = take(prepare(request_for()), "false ghost preparation");
const auto original_destination =
take(decode_manifest(destination_head.manifest), "false ghost existing object");
const CellularMigrationStagePrepared false_ghost{
destination_key,
take(encode_migration_trace(false_stage_freeze.trace), "false ghost trace"),
{original_destination.payload_sha256, original_destination.payload_bytes}};
take(retry_cellular_migration_stage(*f.db, destination_lease, false_ghost, 5s),
"metadata-only false ghost stage");
const auto false_ghost_replay = load_cellular_migration_stage(
*f.db, *destination_objects, destination_key, destination_lease, 5s);
require(!false_ghost_replay && false_ghost_replay.error().kind == RecoveryError::history,
"stage recovery accepted a real object with the wrong complete biology");
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
false_stage_freeze.trace.request.request, 5s),
"abort false ghost evidence");
const auto uncertain_stage_freeze =
take(prepare(request_for()), "unknown stage preparation");
take(locker->query("BEGIN", {}, 5s), "stage ambiguity lock");
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND "
"region_id=2 FOR UPDATE",
{}, 5s),
"hold stage reply");
auto stage_timed = connection();
const auto stage_unknown = stage_cellular_migration(
*stage_timed, *f.objects, *destination_objects, destination_lease, f.head,
destination_head, uncertain_stage_freeze.trace, 300ms);
require(!stage_unknown && stage_unknown.error().kind == PublicationError::unknown &&
!stage_unknown.error().prepared.trace.empty(),
"stage ambiguity lost exact bytes");
take(locker->query("ROLLBACK", {}, 5s), "release stage ambiguity");
take(retry_cellular_migration_stage(*f.db, destination_lease,
stage_unknown.error().prepared, 5s),
"resolve unknown stage");
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
uncertain_stage_freeze.trace.request.request, 5s),
"abort reconciled stage");
for (unsigned race = 0; race < 4; ++race) {
const auto race_freeze = take(prepare(request_for()), "stage race preparation");
auto stage_db = connection(), abort_db = connection();
std::barrier start(2);
std::optional<std::expected<CellularMigrationStage, CellularMigrationStageFailure>>
staged_race;
std::optional<std::expected<CellularMigrationFreeze, LeaseStoreFailure>> aborted_race;
std::thread staging([&] {
start.arrive_and_wait();
staged_race = stage_cellular_migration(*stage_db, *f.objects, *destination_objects,
destination_lease, f.head, destination_head,
race_freeze.trace, 5s);
});
std::thread aborting([&] {
start.arrive_and_wait();
aborted_race = abort_cellular_migration_freeze(
*abort_db, f.key, f.lease, race_freeze.trace.request.request, 5s);
});
staging.join();
aborting.join();
require(aborted_race.has_value() && aborted_race->has_value() &&
staged_race.has_value() &&
(staged_race->has_value() ||
staged_race->error().store.kind == LeaseStoreError::commit_conflict) &&
!take(load_cellular_migration_stage(*f.db, *destination_objects,
destination_key, destination_lease, 5s),
"stage race final pointer")
.has_value(),
"stage/abort race left an active ghost or stranded pause");
}
// Both sides of preparation's transaction boundary, with real process death.
for (unsigned point = 0; point < 2; ++point) {
const auto killed_request = request_for();
const auto child = ::fork();
require(child >= 0, "freeze publisher fork");
if (child == 0) {
try {
auto db = connection();
PublishHooks hooks;
if (point == 0)
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
if (!prepare_cellular_migration_freeze(*db, *f.objects, *destination_objects,
f.lease, f.head, destination_head,
killed_request, 5s, hooks))
::_exit(41);
::kill(::getpid(), SIGKILL);
} catch (...) {
::_exit(42);
}
::_exit(43);
}
int status{};
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
WTERMSIG(status) == SIGKILL &&
take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s),
"recover killed preparation")
.has_value() == (point == 1),
"publisher death changed source ownership freeze");
if (point == 1)
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, killed_request.request,
5s),
"abort killed preparation");
}
for (unsigned point = 0; point < 2; ++point) {
const auto killed = take(prepare(request_for()), "stage process-loss preparation");
const auto child = ::fork();
require(child >= 0, "stage publisher fork");
if (child == 0) {
try {
auto db = connection();
PublishHooks hooks;
if (point == 0)
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
if (!stage_cellular_migration(*db, *f.objects, *destination_objects,
destination_lease, f.head, destination_head,
killed.trace, 5s, hooks))
::_exit(44);
::kill(::getpid(), SIGKILL);
} catch (...) {
::_exit(45);
}
::_exit(46);
}
int status{};
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
WTERMSIG(status) == SIGKILL &&
take(load_cellular_migration_stage(*f.db, *destination_objects,
destination_key, destination_lease, 5s),
"recover killed stage")
.has_value() == (point == 1),
"stage process loss changed ownership or lost the durable pause");
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
killed.trace.request.request, 5s),
"abort killed stage");
}
}
// The owning host must retain the freeze through drain/recreation with entry off.
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture to frozen host");
const auto make_host = [&] {
auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), f.key,
max_cellular_population_bytes),
"freeze host objects");
CellularHostConfig config{f.key, 1, {}, f.build, {f.build}};
config.lease.commit_timeout = 300ms;
return take(
CellularRegionHost::create(
config,
[] { return std::expected<std::unique_ptr<Postgres>, PgFailure>{connection()}; },
std::move(objects)),
"freeze host");
};
auto host = make_host();
require(host->step().ready, "source host before freeze");
const auto before = host->state()->snapshot();
const auto host_request = request_for();
take(host->prepare_migration_freeze(destination_head, *destination_objects, host_request),
"host source freeze");
require(!host->status().ready && host->status().migration_frozen &&
!host->advance_unattended(64) && host->state()->snapshot() == before,
"frozen host advanced or originated current output");
host->drain();
host = make_host();
require(!host->step().ready && host->status().migration_frozen &&
host->state()->snapshot() == before && !host->authority().has_value(),
"replacement host implicitly aborted or ignored its freeze");
take(host->abort_migration_freeze(host_request.request), "replacement host explicit abort");
require(host->step().ready && !host->status().migration_frozen &&
host->state()->snapshot() == before,
"host failed to resume the identical source after confirmed abort");
auto locker = connection(true);
take(locker->query("BEGIN", {}, 5s), "host unknown freeze lock");
take(
locker->query(
"SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE",
{}, 5s),
"hold host freeze reply");
const auto uncertain_request = request_for();
const auto uncertain =
host->prepare_migration_freeze(destination_head, *destination_objects, uncertain_request);
require(!uncertain && !host->status().ready, "host originated during uncertain freeze");
take(locker->query("ROLLBACK", {}, 5s), "release host freeze reply");
for (unsigned attempt = 0; attempt < 200 && !host->status().migration_frozen; ++attempt) {
host->step();
std::this_thread::sleep_for(25ms);
}
require(host->status().migration_frozen && !host->status().ready &&
host->state()->snapshot() == before,
"host chose different biology after unknown preparation");
take(locker->query("BEGIN", {}, 5s), "host unknown abort lock");
take(
locker->query(
"SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE",
{}, 5s),
"hold host abort reply");
const auto unknown_abort = host->abort_migration_freeze(uncertain_request.request);
require(!unknown_abort && !host->status().ready, "unconfirmed abort resumed the source");
take(locker->query("ROLLBACK", {}, 5s), "release host abort reply");
for (unsigned attempt = 0; attempt < 200 && !host->step().ready; ++attempt)
std::this_thread::sleep_for(25ms);
require(host->status().ready && !host->status().migration_frozen &&
host->state()->snapshot() == before,
"host failed exact ambiguous-abort reconciliation");
host->drain();
f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s),
"return freeze fixture authority");
const auto destination_host_freeze =
take(prepare_cellular_migration_freeze(*f.db, *f.objects, *destination_objects, f.lease,
f.head, destination_head, request_for(), 5s),
"destination host source preparation");
take(release_region_lease(*f.db, destination_key, destination_lease, 5s),
"release destination to owning host");
const auto make_destination_host = [&] {
auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), destination_key,
max_cellular_population_bytes),
"destination host objects");
CellularHostConfig config{destination_key, 1, {}, f.build, {f.build}};
config.lease.commit_timeout = 300ms;
return take(
CellularRegionHost::create(
config,
[] { return std::expected<std::unique_ptr<Postgres>, PgFailure>{connection()}; },
std::move(objects)),
"destination staging host");
};
auto destination_host = make_destination_host();
require(destination_host->step().ready, "destination host before staging");
const auto destination_bytes = destination_host->state()->snapshot();
take(locker->query("BEGIN", {}, 5s), "host stage ambiguity transaction");
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND "
"region_id=1 FOR UPDATE",
{}, 5s),
"hold owning destination stage");
const auto host_stage_unknown =
destination_host->stage_migration(f.head, *f.objects, destination_host_freeze.trace);
require(!host_stage_unknown && !destination_host->status().ready &&
destination_host->state()->snapshot() == destination_bytes,
"owning destination advanced during ambiguous staging");
take(locker->query("ROLLBACK", {}, 5s), "release owning stage reply");
for (unsigned attempt = 0; attempt < 200 && !destination_host->status().migration_frozen;
++attempt) {
destination_host->step();
std::this_thread::sleep_for(25ms);
}
require(destination_host->status().migration_frozen && !destination_host->status().ready &&
!destination_host->advance_unattended(64),
"owning destination activated its ghost");
destination_host->drain();
destination_host = make_destination_host();
require(!destination_host->step().ready && destination_host->status().migration_frozen &&
destination_host->state()->snapshot() == destination_bytes &&
!destination_host->authority().has_value(),
"replacement destination lost staging or adopted the uncommitted ghost");
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
destination_host_freeze.trace.request.request, 5s),
"source-authorized abort releases destination host");
for (unsigned attempt = 0; attempt < 200 && !destination_host->step().ready; ++attempt)
std::this_thread::sleep_for(25ms);
require(destination_host->status().ready && !destination_host->status().migration_frozen &&
destination_host->state()->snapshot() == destination_bytes,
"destination failed to resume its unchanged original state after source abort");
destination_host->drain();
destination_lease = take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s),
"return staged destination fixture authority");
migration_publication(f, destination_key, *destination_objects, destination_lease,
destination_head);
f.head = take(
materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
"resume frozen population as raw owner");
f.raw = take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s),
"source after frozen profile materialization")
.state;
take(release_region_lease(*f.db, destination_key, destination_lease, 5s),
"release freeze destination");
}
void host_ownership_and_admission(Fixture& f) {
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture host authority");
const auto make_host = [&](bool enabled) {
auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), f.key,
max_cellular_population_bytes),
"host objects");
CellularHostConfig config{f.key, 1, {}, f.build, {f.build}};
config.lease.commit_timeout = 300ms;
if (enabled)
config.population_policy = CellularTierPolicy{true, 1200, 1800};
return take(CellularRegionHost::create(
std::move(config),
[] {
return Postgres::connect({environment("PGHOST"),
{},
environment("PGPORT"),
environment("PGDATABASE"),
environment("EVO_PG_RUNTIME_USER"),
{},
{}},
5s);
},
std::move(objects)),
"population host");
};
auto host = make_host(false);
require(host->step().ready && !host->population_mode(), "raw host recovery");
require(!host->set_population_mode(true), "default host enabled population entry");
host->drain();
host = make_host(true);
require(host->step().ready, "opt-in host recovery");
const auto before = host->state()->snapshot();
const auto tick = host->state()->tick();
take(host->set_population_mode(true), "host population entry");
require(host->population_mode() && host->state()->snapshot() == before,
"host entry changed biology");
auto population =
take(CellularPopulation::create(
take(CellularLives::restore(before, f.key.world, f.key.region, 1, tick),
"host reference restore"),
{true, 1200, 1800}),
"host reference population");
f.head = take(host->committed(), "host population head");
auto next = unattended(f); // uses the same deterministic frozen/wildlife input contract.
// The fixture's old envelope may have a different membership history; biology is shared.
auto expected =
take(prepare_population_batch(*population, next.batch.biology), "host reference batch");
require(!take(host->advance_unattended(19), "collect bounded publication window").has_value() &&
host->state()->snapshot() == before,
"small local debt was admitted prematurely");
const auto published = take(host->advance_unattended(65), "host population publication");
require(published.has_value(), "host skipped admitted interval");
require(host->population_mode() &&
host->state()->snapshot() == expected.state->state().snapshot(),
"host population continuation differs");
const auto committed = take(host->committed(), "host continued head");
const auto continued = host->state()->snapshot();
const CellularBatchHeader recovered_header{f.key.world,
f.key.region,
1,
committed.head.commit,
committed.head.digest,
committed.head.end_tick,
committed.head.end_tick + 64};
auto recovered_expected = take(
prepare_population_batch(
*expected.state, take(unattended_population_inputs(*host->state(), recovered_header),
"recovered interval inputs")),
"recovered interval reference");
take(admit_population_interval(*f.db, f.key, take(host->authority(), "host interval authority"),
committed.head, committed.head.end_tick + 64, 5s),
"admit before host loss");
host->drain();
host = make_host(false);
require(host->step().ready && host->population_mode() &&
take(host->committed(), "host recovered head").head == committed.head &&
host->state()->snapshot() == continued,
"restarted host failed to recover population head");
require(host->status().ecological_pending_ticks == 64 && !host->set_population_mode(false),
"restarted host hid or skipped admitted work");
require(take(host->advance_unattended(0), "resume admitted work with zero new due time")
.has_value() &&
host->state()->snapshot() == recovered_expected.state->state().snapshot() &&
host->status().ecological_pending_ticks == 0,
"recovered interval did not settle exactly");
const auto recovered_head = take(host->committed(), "recovered interval head");
const auto recovered_bytes = host->state()->snapshot();
const CellularBatchHeader unknown_header{f.key.world,
f.key.region,
1,
recovered_head.head.commit,
recovered_head.head.digest,
recovered_head.head.end_tick,
recovered_head.head.end_tick + 64};
auto unknown_expected = take(
prepare_population_batch(*recovered_expected.state,
take(unattended_population_inputs(*host->state(), unknown_header),
"uncertain admission inputs")),
"uncertain admission reference");
auto locker = connection(true);
take(locker->query("BEGIN", {}, 5s), "host admission lock transaction");
take(
locker->query(
"SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE",
{}, 5s),
"hold host admission reply");
require(!host->advance_unattended(64) && !host->status().ready &&
host->state()->snapshot() == recovered_bytes,
"uncertain admission allowed simulation/output");
take(locker->query("ROLLBACK", {}, 5s), "release host admission reply");
for (unsigned attempt = 0; attempt < 200 && !host->step().ready; ++attempt)
std::this_thread::sleep_for(25ms);
require(host->status().ready && host->status().ecological_pending_ticks == 64,
"unknown admission failed exact reconciliation");
require(take(host->advance_unattended(0), "finish reconciled admission").has_value() &&
host->state()->snapshot() == unknown_expected.state->state().snapshot(),
"admission retry chose different work");
const auto final_bytes = host->state()->snapshot();
Uuid account{};
account.back() = 1;
// A returning player's operation completes the durable reverse transition before lookup.
[[maybe_unused]] const auto queried = host->query_primitive_life(account, 1, 1);
require(!host->population_mode() && host->state()->snapshot() == final_bytes &&
host->state()->tick() == tick + 192,
"player operation did not materialize first");
host->drain();
}
} // namespace
void run_population_publication_tests() {
Fixture fixture;
funded_source(fixture);
conversions_and_retries(fixture);
interval_contract(fixture);
ambiguous_publication(fixture, false);
ambiguous_publication(fixture, true);
process_loss(fixture, 0);
process_loss(fixture, 1);
process_loss(fixture, 2);
missing_and_corrupt_objects(fixture);
isolation_and_reverse(fixture);
boundary_upgrades(fixture);
migration_freezes(fixture);
host_ownership_and_admission(fixture);
std::puts("population publication, exact recovery, process loss and fencing passed");
}