Files
aevolution/tests/population_publication_tests.cpp
T
dingo 5e168d67c3
Foundation / checks (ecs) (push) Successful in 2m26s
Foundation / checks (deploy) (push) Successful in 4m12s
Foundation / checks (network) (push) Successful in 20m57s
Foundation / checks (client) (push) Successful in 22m9s
Foundation / checks (tsan) (push) Successful in 28m49s
Foundation / checks (postgres) (push) Successful in 1h5m2s
Foundation / checks (checks) (push) Successful in 1h17m42s
Foundation / postgres-tools (push) Successful in 1m10s
Foundation / server-image (push) Successful in 3m51s
Foundation / image (push) Successful in 4m54s
feat(lifecycle): qualify portable maturing wildlife transfers
2026-10-19 09:09:15 -04:00

3501 lines
205 KiB
C++

#include "evo/persistence/account_control_store.hpp"
#include "evo/persistence/cellular_census_snapshot.hpp"
#include "evo/persistence/cellular_history_store.hpp"
#include "evo/persistence/population_checkpoint_store.hpp"
#include "evo/persistence/population_interval_store.hpp"
#include "evo/persistence/sha256.hpp"
#include "evo/server/cellular_region_host.hpp"
#include "evo/server/lifecycle/cellular_census_set.hpp"
#include "fixtures/cellular_ecology.hpp"
#include "fixtures/postgres_relay.hpp"
#include <barrier>
#include <charconv>
#include <csignal>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <limits>
#include <stdexcept>
#include <sys/wait.h>
#include <thread>
#include <type_traits>
#include <unistd.h>
void run_population_publication_tests();
namespace {
using namespace evo;
using namespace evo::server;
using namespace evo::persistence;
using namespace evo::fixture;
using namespace std::chrono_literals;
template <typename T, typename E> T take(std::expected<T, E> value, const char* where) {
if (!value) {
std::string message{where};
if constexpr (requires { value.error().cause.sqlstate; })
message += " SQLSTATE=" + std::string(value.error().cause.sqlstate.data(), 5);
if constexpr (requires { value.error().store.cause.sqlstate; })
message += " SQLSTATE=" + std::string(value.error().store.cause.sqlstate.data(), 5);
throw std::runtime_error(message);
}
if constexpr (!std::is_void_v<T>)
return std::move(*value);
}
template <typename T> T take(std::optional<T> value, const char* where) {
if (!value.has_value())
throw std::runtime_error(where);
return std::move(*value);
}
std::string environment(const char* name) {
const auto* value = std::getenv(name);
require(value != nullptr, "missing private test environment");
return value;
}
std::unique_ptr<Postgres> connection(bool owner = false) {
return take(Postgres::connect({environment("PGHOST"),
{},
environment("PGPORT"),
environment("PGDATABASE"),
environment(owner ? "PGUSER" : "EVO_PG_RUNTIME_USER"),
{},
{}},
5s),
"population connection");
}
Uuid identity(unsigned number) {
Uuid id{};
id[0] = 0xDA;
id[1] = 0x12;
id[14] = static_cast<std::uint8_t>(number >> 8U);
id[15] = static_cast<std::uint8_t>(number);
return id;
}
std::string hexadecimal(std::span<const std::uint8_t> bytes, bool prefix = true) {
std::string out = prefix ? "\\x" : "";
constexpr char digits[] = "0123456789abcdef";
for (const auto b : bytes) {
out.push_back(digits[b >> 4U]);
out.push_back(digits[b & 15U]);
}
return out;
}
struct Fixture {
RegionKey key{30201, 1};
std::unique_ptr<Postgres> owner = connection(true), db = connection();
std::unique_ptr<FilesystemObjectStore> objects;
LeaseGrant lease;
Sha256Digest build{};
CommittedCheckpoint head;
std::unique_ptr<CellularLives> raw;
std::unique_ptr<CellularPopulation> population;
unsigned next_id = 1;
Uuid id() {
return identity(next_id++);
}
CellularBatchHeader header(std::uint64_t count) const {
return {key.world,
key.region,
1,
head.head.commit,
head.head.digest,
head.head.end_tick,
head.head.end_tick + count};
}
};
// Deny actual reads by the unprivileged fixture user, then restore permissions
// even if a runtime requirement throws. This never changes immutable bytes.
class UnreadableObject {
public:
explicit UnreadableObject(std::filesystem::path path)
: path_(std::move(path)), permissions_(std::filesystem::status(path_).permissions()) {
std::filesystem::permissions(path_, std::filesystem::perms::none);
}
~UnreadableObject() {
std::error_code error;
std::filesystem::permissions(path_, permissions_, error);
}
UnreadableObject(const UnreadableObject&) = delete;
UnreadableObject& operator=(const UnreadableObject&) = delete;
private:
std::filesystem::path path_;
std::filesystem::perms permissions_;
};
std::uint16_t migration_backend_port() {
const auto port = environment("EVO_MIGRATION_TCP_PORT");
unsigned value{};
const auto [end, error] = std::from_chars(port.data(), port.data() + port.size(), value);
require(error == std::errc{} && end == port.data() + port.size() && value > 0 && value <= 65535,
"invalid private migration TLS port");
return static_cast<std::uint16_t>(value);
}
PgOptions migration_tls_options(const test::PostgresRelay& relay) {
return {"localhost",
"127.0.0.1",
relay.port(),
environment("PGDATABASE"),
environment("EVO_PG_RUNTIME_USER"),
{},
environment("EVO_MIGRATION_CA")};
}
std::unique_ptr<Postgres> migration_tls_connection(const test::PostgresRelay& relay) {
auto db = take(Postgres::connect(migration_tls_options(relay), 2s),
"migration verify-full TLS connection");
take(db->query("SELECT 1", {}, 2s), "migration TLS warmup");
return db;
}
void migration_transport_commit(Fixture& f, const CellularMigrationCommitPrepared& prepared,
RegionKey sk, RegionKey dk, const LeaseGrant& sl,
const LeaseGrant& dl, const RegionHead& source_before,
const RegionHead& destination_before) {
{
test::PostgresRelay relay(migration_backend_port());
auto wrong = migration_tls_options(relay);
wrong.host = "untrusted.evolution.test";
require(!Postgres::connect(wrong, 1s), "relay bypassed TLS hostname verification");
}
{
test::PostgresRelay relay(migration_backend_port());
auto db = migration_tls_connection(relay);
relay.drop_requests(true);
const auto lost = retry_cellular_migration_commit(*db, sl, dl, prepared, 300ms);
require(
!lost && lost.error().kind == PublicationError::unknown &&
lost.error().prepared.source_manifest == prepared.source_manifest &&
lost.error().prepared.destination_manifest == prepared.destination_manifest &&
relay.dropped_requests() > 0 && relay.healthy() && !db->transaction_idle() &&
take(load_region_checkpoint(*f.db, sk, 5s), "lost request source head").head ==
source_before &&
take(load_region_checkpoint(*f.db, dk, 5s), "lost request destination head").head ==
destination_before,
"a dropped TLS request changed ownership or lost the exact pair");
}
{
test::PostgresRelay relay(migration_backend_port());
auto db = migration_tls_connection(relay);
relay.drop_responses(true);
const auto lost = retry_cellular_migration_commit(*db, sl, dl, prepared, 500ms);
require(
!lost && lost.error().kind == PublicationError::unknown &&
lost.error().prepared.source_manifest == prepared.source_manifest &&
lost.error().prepared.destination_manifest == prepared.destination_manifest &&
relay.dropped_responses() > 0 && relay.healthy() && !db->transaction_idle() &&
take(load_region_checkpoint(*f.db, sk, 5s), "lost reply source head").manifest ==
prepared.source_manifest &&
take(load_region_checkpoint(*f.db, dk, 5s), "lost reply destination head")
.manifest == prepared.destination_manifest,
"lost TLS receipt was treated as rollback or committed only one head");
}
std::puts("migration TLS request/reply loss and hostname verification passed");
}
void funded_source(Fixture& f) {
f.build.fill(0xDA);
const std::array<std::string, 1> build{hexadecimal(f.build)};
take(f.owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES($1::bytea) "
"ON CONFLICT DO NOTHING",
build, 5s),
"retain test build");
const std::array<std::string, 1> world{std::to_string(f.key.world)};
take(f.owner->query("INSERT INTO evolution.life_allocators VALUES($1::evolution.u64,100)",
world, 5s),
"test allocator");
f.objects = take(
FilesystemObjectStore::open(environment("PGHOST"), f.key, max_cellular_population_bytes),
"population object namespace");
f.raw = opening(f.key.world - 200, true, 4, false, f.key.region);
f.head =
take(provision_cellular_region(*f.owner, *f.objects, f.key, 1, *f.raw, f.id(), f.build, 5s),
"empty cellular genesis");
f.lease =
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "population test lease");
CellularBatch birth{f.header(1), {{}}};
for (std::uint8_t index = 1; index <= 4; ++index) {
Uuid account{};
account.back() = index;
const auto control =
take(claim_account_control(*f.db, f.key, f.lease, account, 1, index, 60000ms, 5s),
"founder account control");
const auto reserved = take(
reserve_primitive_life(*f.db, f.key, f.lease, account, control.generation, 1, 1, 5s),
"funded founder reservation");
birth.ticks.front().births.push_back({account, 1, control.generation, reserved.identity});
}
f.raw = take(replay_cellular_batch(*f.raw, birth), "funded founder replay");
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, birth, *f.raw, f.id(),
f.build, 5s),
"funded founder publication")
.committed;
while (f.raw->next_bud_sequence() == 1 && f.raw->tick() < 20000) {
CellularBatch batch{f.header(1), {}};
for (unsigned tick = 0; tick < 64; ++tick) {
auto intents =
take(wildlife_intents(*f.raw, f.raw->tick() + 1), "funded source wildlife");
for (const auto& life : f.raw->lives()) {
if (life.account == LifeAccount{})
continue;
CellularLifeIntent intent{life.identity.life, {}};
intent.intent.bud = (f.raw->tick() + life.identity.life) % 32 == 0;
intents.push_back(intent);
}
std::ranges::sort(intents, {}, &CellularLifeIntent::life_id);
take(f.raw->advance_tick(intents), "funded source biology");
batch.ticks.push_back({intents, {}, {}});
if (f.raw->next_bud_sequence() > 1)
break;
}
batch.header.end_tick = f.raw->tick();
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, batch, *f.raw,
f.id(), f.build, 5s),
"funded source batch")
.committed;
take(renew_region_lease(*f.db, f.key, f.lease, 60000ms, 5s), "source lease renewal");
}
require(std::ranges::any_of(f.raw->lives(),
[](const auto& life) { return life.account == LifeAccount{}; }),
"durable fixture has no funded player-derived wildlife");
require(!migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build,
{true, 1200, 1800}, 5s),
"population entry bypassed vulnerable controlled life");
const auto suspended_at = f.raw->tick() + cellular_linger_ticks + 1;
while (f.raw->tick() < suspended_at) {
auto h = f.header(std::min(std::uint64_t{64}, suspended_at - f.raw->tick()));
auto prefix = take(unattended_population_inputs(*f.raw, h), "raw linger inputs");
f.raw = take(replay_cellular_batch(*f.raw, prefix), "raw linger biology");
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, prefix, *f.raw,
f.id(), f.build, 5s),
"raw linger publication")
.committed;
std::vector<std::uint64_t> endings;
for (const auto& corpse : f.raw->corpses())
if (corpse.parent_life == 0)
endings.push_back(corpse.life_id);
if (!endings.empty())
take(end_cellular_lives(*f.db, f.key, f.lease, f.head.head.commit, endings, 5s),
"raw linger endings");
}
require(population_interval_safe(*f.raw),
"population source retained a vulnerable controlled life");
}
CellularPopulationExecution unattended(Fixture& f) {
auto working = restore(f.population->state());
CellularBatch batch{f.header(64), {}};
for (unsigned tick = 0; tick < 64; ++tick) {
auto intents = take(wildlife_intents(*working, working->tick() + 1), "unattended wildlife");
for (const auto& life : working->lives()) {
if (life.account == LifeAccount{})
continue;
CellularLifeIntent frozen{life.identity.life, {}};
frozen.intent.frozen = true;
intents.push_back(frozen);
}
std::ranges::sort(intents, {}, &CellularLifeIntent::life_id);
take(working->advance_tick(intents), "unattended fixture step");
batch.ticks.push_back({intents, {}, {}});
}
return take(prepare_population_batch(*f.population, std::move(batch)),
"population recorded interval");
}
void recover(Fixture& f) {
const std::array<Sha256Digest, 1> builds{f.build};
auto db = connection();
auto recovered =
take(recover_population_checkpoint(*db, *f.objects, f.key, f.lease, builds, 5s),
"population durable recovery");
f.head = std::move(recovered.committed);
f.population = std::move(recovered.state);
}
void conversions_and_retries(Fixture& f) {
const auto biology = f.raw->snapshot();
Uuid account{};
account.back() = 1;
const auto input =
encode_cellular_mutation_input({account, 9991, f.head.head.end_tick + 1, {100, 1, 1, 1}});
take(f.db->query("BEGIN", {}, 5s), "reservation guard transaction");
take(reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 9991, input, 5s),
"reserve pending player choice");
const auto blocked = migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
f.build, {true, 1200, 1800}, 5s);
require(!blocked && std::string(blocked.error().store.cause.sqlstate.data(), 5) == "EVL18",
"population entry stranded a reserved player choice");
take(f.db->query("ROLLBACK", {}, 5s), "rollback guard fixture");
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
f.build, {true, 1200, 1800}, 5s),
"initial population transition");
recover(f);
require(f.population->state().snapshot() == biology && f.population->members().empty(),
"transition changed biology");
const auto refused =
reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 9991, input, 5s);
require(!refused && std::string(refused.error().cause.sqlstate.data(), 5) == "EVL06",
"population profile admitted a player mutation reservation");
const auto wildlife = std::ranges::find_if(f.population->state().lives(), [](const auto& life) {
return life.account == LifeAccount{};
});
require(wildlife != f.population->state().lives().end(), "missing durable descendant");
const auto genome = wildlife->identity.genome;
auto conversion =
take(prepare_population_conversion(*f.population, f.header(0), {genome, true}),
"collapse sample");
auto bad = conversion.batch;
++bad.boundaries.front().sequence;
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, bad, *conversion.state,
f.id(), f.build, 5s),
"altered sample published");
const auto commit = f.id();
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, conversion.batch,
*conversion.state, commit, f.build, 5s),
"durable collapse");
const PopulationPreparedCommit prepared{
f.key, commit, f.head.manifest,
take(encode_population_batch(conversion.batch), "prepared trace")};
require(take(retry_population_prepared(*f.db, prepared, 5s), "idempotent retry").head ==
f.head.head,
"retry advanced head");
require(take(resolve_population_prepared(*f.db, prepared, 5s), "read-only resolve").status ==
Resolution::committed,
"commit unresolved");
auto future_batch = conversion.batch;
future_batch.format_version = 2;
auto future = prepared;
future.trace = take(encode_population_batch(future_batch), "future trace fixture");
auto future_manifest = take(decode_manifest(future.manifest), "future manifest fixture");
future_manifest.batch.trace_bytes = future.trace.size();
future_manifest.batch.trace_sha256 = take(sha256(future.trace), "future trace digest");
future.manifest = take(encode_manifest(future_manifest), "future manifest encoding");
auto rejected_future = retry_population_prepared(*f.db, future, 5s);
require(!rejected_future && rejected_future.error().kind == PublicationError::candidate,
"bundle 9 prepared retry admitted population trace 2");
require(!resolve_population_prepared(*f.db, future, 5s),
"bundle 9 prepared resolve admitted population trace 2");
auto rejected_sql = publish_population_checkpoint(*f.db, future.manifest, future.trace, 5s);
require(!rejected_sql && rejected_sql.error().kind == LeaseStoreError::invalid,
"schema 45 did not reject the unsupported trace format before commit lookup");
const auto current_manifest = take(decode_manifest(f.head.manifest), "upgrade source manifest");
const auto boundary_payload =
take(cellular_boundary_upgrade_payload(conversion.state->snapshot(), current_manifest),
"portable boundary upgrade source");
const auto boundary_manifest = take(
cellular_boundary_upgrade_manifest(
current_manifest, f.head.head.digest, f.id(), f.lease, f.build, boundary_payload.size(),
take(sha256(boundary_payload), "boundary payload digest")),
"portable boundary upgrade manifest");
const PopulationPreparedCommit boundary{
f.key,
boundary_manifest.commit,
take(encode_manifest(boundary_manifest), "boundary candidate encoding"),
{}};
auto rejected_boundary = retry_population_prepared(*f.db, boundary, 5s);
require(!rejected_boundary && rejected_boundary.error().kind == PublicationError::rejected &&
take(resolve_population_prepared(*f.db, boundary, 5s),
"upgrade through wrong journal resolution")
.status == Resolution::absent,
"accounting upgrade bypassed its dedicated version journal");
auto rejected_upgrade = publish_population_profile_transition(*f.db, boundary.manifest, 5s);
require(!rejected_upgrade,
"representation publisher admitted accounting upgrade without dedicated evidence");
require(!publish_region_checkpoint(*f.db, boundary.manifest, {}, 5s),
"generic publisher admitted boundary upgrade");
require(take(load_region_checkpoint(*f.db, f.key, 5s), "head after future rejection").head ==
f.head.head,
"future trace changed the retained durable head");
auto changed = prepared;
auto changed_manifest = take(decode_manifest(changed.manifest), "conflict manifest");
changed_manifest.payload_sha256[0] ^= 1;
changed.manifest = take(encode_manifest(changed_manifest), "conflict encoding");
require(take(resolve_population_prepared(*f.db, changed, 5s), "conflicting resolve").status ==
Resolution::conflict,
"changed commit was accepted");
require(!publish_region_checkpoint(*f.db, f.head.manifest, {}, 5s),
"generic publisher bypassed profile gate");
const auto expected = conversion.state->snapshot();
recover(f);
require(f.population->snapshot() == expected && !f.population->members().empty(),
"committed sample not recovered");
require(f.population->state().snapshot() == biology, "collapse healed/aged/moved biology");
for (unsigned oscillation = 0; oscillation < 8; ++oscillation) {
const bool collapse = oscillation % 2 != 0;
auto next =
take(prepare_population_conversion(*f.population, f.header(0), {genome, collapse}),
"conversion oscillation");
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, next.batch,
*next.state, f.id(), f.build, 5s),
"publish oscillation");
recover(f);
require(f.population->state().snapshot() == biology,
"durable conversion changed exact state");
}
}
void process_loss(Fixture& f, int point) {
auto next = unattended(f);
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, next.batch.header.end_tick,
5s),
"admit process-loss interval");
const auto predecessor = f.head.head;
const auto expected = next.state->snapshot();
const auto commit = f.id();
int channel[2]{};
require(::pipe(channel) == 0, "process acknowledgement pipe");
const auto child = ::fork();
require(child >= 0, "population fork");
if (child == 0) {
::close(channel[0]);
try {
auto db = connection();
PublishHooks hooks;
if (point == 0)
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
const auto published =
publish_population_batch(*db, *f.objects, f.lease, f.head, next.batch, *next.state,
commit, f.build, 5s, hooks);
if (!published)
::_exit(20);
if (point == 2) {
const char ack = 'A';
if (::write(channel[1], &ack, 1) != 1)
::_exit(21);
}
::kill(::getpid(), SIGKILL);
} catch (...) {
::_exit(22);
}
::_exit(23);
}
::close(channel[1]);
char ack{};
const auto acknowledged = ::read(channel[0], &ack, 1);
::close(channel[0]);
int status{};
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
WTERMSIG(status) == SIGKILL,
"child missed population crash boundary");
require((point == 2 && acknowledged == 1 && ack == 'A') || (point != 2 && acknowledged == 0),
"unexpected permanent acknowledgement");
recover(f);
const auto pending =
take(load_population_interval(*f.db, f.key, f.lease, 5s), "recover process interval");
require(pending.has_value() == (point == 0), "process loss changed interval settlement");
if (point == 0)
require(f.head.head == predecessor, "uncommitted process published a head");
else
require(f.head.head.commit == commit && f.population->snapshot() == expected,
"committed process loss changed state");
}
void ambiguous_publication(Fixture& f, bool terminate) {
auto next = unattended(f);
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, next.batch.header.end_tick,
5s),
"admit ambiguous interval");
const auto expected = next.state->snapshot();
auto locker = connection(true);
take(locker->query("BEGIN", {}, 5s), "population lease row transaction");
const std::array<std::string, 2> key{std::to_string(f.key.world), std::to_string(f.key.region)};
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=$1::evolution.u64 "
"AND region_id=$2::evolution.u64 FOR UPDATE",
key, 5s),
"hold population publication");
auto timed = connection();
auto unknown = publish_population_batch(*timed, *f.objects, f.lease, f.head, next.batch,
*next.state, f.id(), f.build, 300ms);
require(!unknown && unknown.error().kind == PublicationError::unknown,
"population publication failed to retain ambiguous request");
if (terminate) {
const std::array<std::string, 1> user{environment("EVO_PG_RUNTIME_USER")};
take(locker->query("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE "
"usename=$1 AND wait_event_type='Lock'",
user, 5s),
"terminate uncommitted population writer");
}
take(locker->query("ROLLBACK", {}, 5s), "release population publication");
auto resolver = connection();
bool resolved = false;
for (unsigned attempt = 0; attempt < 200; ++attempt) {
auto result = take(resolve_population_prepared(*resolver, unknown.error().prepared, 5s),
"resolve ambiguous population publication");
if (result.status == (terminate ? Resolution::absent : Resolution::committed)) {
resolved = true;
break;
}
std::this_thread::sleep_for(25ms);
}
require(resolved, "ambiguous population publication did not resolve");
f.head = take(retry_population_prepared(*resolver, unknown.error().prepared, 5s),
"retry exact population request");
recover(f);
require(f.population->snapshot() == expected, "ambiguous publication changed retained sample");
}
void interval_contract(Fixture& f) {
require(!take(load_population_interval(*f.db, f.key, f.lease, 5s), "empty interval journal")
.has_value(),
"settled source retained pending work");
auto next = unattended(f);
const auto source = f.head.head;
const auto admitted =
take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
"durable interval admission");
require(take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
"duplicate interval admission") == admitted,
"admission retry changed boundary");
auto changed =
admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 63, 5s);
require(!changed && changed.error().kind == LeaseStoreError::commit_conflict,
"source admitted two different intervals");
const auto expected = next.state->snapshot();
require(!f.population->members().empty(), "interval fixture lost its exact cohort");
const auto genome = f.population->members().front().genome;
auto conversion =
take(prepare_population_conversion(*f.population, f.header(0), {genome, false}),
"conversion during admitted work");
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, conversion.batch,
*conversion.state, f.id(), f.build, 5s),
"conversion skipped an admitted interval");
require(
!materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
"profile exit skipped an admitted interval");
auto shortened_header = f.header(63);
auto shortened =
take(prepare_population_batch(
*f.population,
take(unattended_population_inputs(f.population->state(), shortened_header),
"shortened input")),
"shortened interval");
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, shortened.batch,
*shortened.state, f.id(), f.build, 5s),
"publication partially consumed an interval");
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release interval owner");
const auto epoch = f.lease.epoch;
f.lease =
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "replace interval owner");
require(f.lease.epoch > epoch && take(take(load_population_interval(*f.db, f.key, f.lease, 5s),
"new owner interval load"),
"retained admission") == admitted,
"replacement owner lost admitted work");
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, next.batch,
*next.state, f.id(), f.build, 5s),
"settle exact admitted interval");
recover(f);
require(f.population->snapshot() == expected &&
!take(load_population_interval(*f.db, f.key, f.lease, 5s), "settled interval load")
.has_value(),
"settlement lost work or retained spendable pending state");
require(take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
"settled admission retry") == admitted &&
!take(load_population_interval(*f.db, f.key, f.lease, 5s), "settled retry pending")
.has_value(),
"old admission retry reopened settled work");
require(!admit_population_interval(*f.db, f.key, f.lease, f.head.head,
f.head.head.end_tick + 65, 5s),
"oversized interval admitted");
const auto denied = f.db->query(
"SELECT * FROM evolution.publish_population_checkpoint_v44(''::bytea,''::bytea)", {}, 5s);
require(!denied && std::string(denied.error().sqlstate.data(), 5) == "42501",
"runtime bypassed interval guard");
}
void isolation_and_reverse(Fixture& f) {
auto interval = unattended(f);
const auto old_lease = f.lease;
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release old authority");
auto new_db = connection();
f.lease = take(acquire_region_lease(*new_db, f.key, f.build, 60000ms, 5s),
"new population authority");
require(f.lease.epoch > old_lease.epoch, "authority epoch did not advance");
const auto old_head = f.head.head;
auto stale = publish_population_batch(*f.db, *f.objects, old_lease, f.head, interval.batch,
*interval.state, f.id(), f.build, 5s);
require(!stale && stale.error().store.kind == LeaseStoreError::fenced,
"stale population writer published");
recover(f);
require(f.head.head == old_head, "fenced writer changed head");
const auto expected = f.population->state().snapshot();
f.head = take(
materialize_population_profile(*new_db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
"return to cellular profile");
const std::array<Sha256Digest, 1> builds{f.build};
auto recovered =
take(recover_cellular_checkpoint(*new_db, *f.objects, f.key, f.lease, builds, 5s),
"cellular recovery after materialization");
require(recovered.state->snapshot() == expected &&
recovered.manifest.profile == CheckpointProfile::cellular_lives_v1,
"reverse transition changed biology or time");
f.raw = std::move(recovered.state);
}
void missing_and_corrupt_objects(Fixture& f) {
const auto manifest = take(decode_manifest(f.head.manifest), "object test manifest");
const auto path = f.objects->directory() / hexadecimal(manifest.payload_sha256, false);
const auto backup = path.string() + ".population-test";
const std::array<Sha256Digest, 1> builds{f.build};
std::filesystem::rename(path, backup);
auto missing = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
std::filesystem::rename(backup, path);
require(!missing && missing.error().kind == RecoveryError::missing_payload,
"missing population payload did not fail closed");
std::filesystem::rename(path, backup);
{
std::ofstream corrupt(path, std::ios::binary);
corrupt << 'X';
}
auto corrupt = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
std::filesystem::remove(path);
std::filesystem::rename(backup, path);
require(!corrupt && corrupt.error().kind == RecoveryError::corrupt_payload,
"corrupt population payload did not fail closed");
recover(f);
}
void history_pages(Fixture& f) {
const std::array<Sha256Digest, 1> builds{f.build};
const auto upper = f.head;
const auto biology = f.raw->snapshot();
const auto read = [&](const RegionHead& watermark, std::uint64_t first, std::size_t limit) {
return load_cellular_census_history(*f.db, *f.objects, f.key, f.lease, watermark, first,
limit, builds, 5s);
};
auto first = take(read(upper.head, 0, 2), "history first page");
require(first.entries.size() == 2 && first.next_sequence == 2 &&
first.entries.front().committed.head.sequence == 0 &&
first.entries.front().census.rows().empty(),
"history lost genesis or changed sequence pagination");
// Append two distinct no-tick cuts between requests; the old watermark stays pinned.
const auto population =
take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build,
{false, 1200, 1800}, 5s),
"history population cut");
f.head = take(
materialize_population_profile(*f.db, *f.objects, f.lease, population, f.id(), f.build, 5s),
"history raw cut");
std::uint64_t next = take(first.next_sequence, "history continuation cursor");
while (next <= upper.head.sequence) {
auto page = take(read(upper.head, next, 2), "pinned historical page");
require(page.watermark == upper.head && !page.entries.empty() && page.entries.size() <= 2,
"history changed its pinned upper cut");
for (const auto& entry : page.entries) {
const auto stored =
take(load_region_commit(*f.db, f.key, next, 5s), "history oracle cut");
const auto manifest = take(decode_manifest(stored.manifest), "history oracle manifest");
auto state =
take(restore_cellular_payload(
take(f.objects->get({manifest.payload_sha256, manifest.payload_bytes}),
"history oracle payload"),
manifest),
"history oracle biology");
const auto oracle = take(CellularCensus::observe(*state), "history oracle census");
require(entry.committed.head == stored.head &&
entry.committed.manifest == stored.manifest &&
std::ranges::equal(entry.census.rows(), oracle.rows()) &&
entry.census.boundary() == oracle.boundary(),
"history rebound old census to a newer or different committed cut");
++next;
}
require(
page.next_sequence ==
(next <= upper.head.sequence ? std::optional<std::uint64_t>{next} : std::nullopt),
"history cursor skipped or invented a commit");
}
const auto transitions = take(read(f.head.head, upper.head.sequence + 1, 8), "no-tick history");
require(transitions.entries.size() == 2 && !transitions.next_sequence &&
transitions.entries[0].committed.head == population.head &&
transitions.entries[1].committed.head == f.head.head &&
transitions.entries[0].census.boundary().population_format == 1 &&
transitions.entries[1].census.boundary().population_format == 0 &&
population.head.end_tick == upper.head.end_tick &&
f.head.head.end_tick == upper.head.end_tick,
"history collapsed distinct no-tick events or lost representation identity");
const auto repeat = take(read(upper.head, 0, 2), "repeat pinned history");
require(repeat.entries[0].committed.head == first.entries[0].committed.head &&
repeat.entries[1].committed.head == first.entries[1].committed.head &&
f.raw->snapshot() == biology &&
take(load_region_checkpoint(*f.db, f.key, 5s), "read-only history head").head ==
f.head.head,
"history read changed biology, head or event identity");
for (const auto limit :
{std::size_t{0}, max_cellular_history_page + 1, std::numeric_limits<std::size_t>::max()})
require(!read(upper.head, 0, limit) && read(upper.head, 0, limit).error().kind ==
CellularHistoryError::invalid_request,
"unbounded history request admitted");
require(!read(upper.head, upper.head.sequence + 1, 1), "history read beyond pinned cut");
auto wrong = upper.head;
wrong.digest.back() ^= 1;
require(!read(wrong, 0, 1) &&
read(wrong, 0, 1).error().kind == CellularHistoryError::cut_mismatch,
"history accepted a false watermark digest");
wrong = upper.head;
++wrong.end_tick;
require(!read(wrong, 0, 1), "history accepted a false watermark tick");
wrong = upper.head;
++wrong.frame;
require(!read(wrong, 0, 1), "history merged foreign frames");
require(!load_cellular_census_history(*f.db, *f.objects, {f.key.world, 999}, f.lease,
upper.head, 0, 1, builds, 5s) &&
!load_cellular_census_history(*f.db, *f.objects, f.key, f.lease, upper.head, 0, 1,
{}, 5s),
"foreign region or unretained build became an empty history");
const auto birth = take(load_region_commit(*f.db, f.key, 1, 5s), "historical birth cut");
const auto manifest = take(decode_manifest(birth.manifest), "historical birth manifest");
const auto path = f.objects->directory() / hexadecimal(manifest.payload_sha256, false);
const auto saved = path.string() + ".history-test";
const auto bytes = take(f.objects->get({manifest.payload_sha256, manifest.payload_bytes}),
"historical birth bytes");
std::filesystem::rename(path, saved);
const auto missing =
read(upper.head, 0, 2); // Genesis succeeded; second cut must fail the page.
std::filesystem::rename(saved, path);
require(!missing && missing.error().kind == CellularHistoryError::recovery,
"missing old payload returned a partial page or zero census");
{
std::ofstream file(path, std::ios::binary);
file << 'X';
require(file.good(), "corrupt old history payload fixture");
}
const auto corrupt = read(upper.head, 0, 2);
{
std::ofstream file(path, std::ios::binary);
file.write(reinterpret_cast<const char*>(bytes.data()),
static_cast<std::streamsize>(bytes.size()));
require(file.good(), "restore old history payload fixture");
}
require(!corrupt && corrupt.error().kind == CellularHistoryError::recovery,
"corrupt historical object returned a census");
const auto trace_path =
f.objects->directory() / hexadecimal(manifest.batch.trace_sha256, false);
const auto trace_saved = trace_path.string() + ".history-test";
std::filesystem::rename(trace_path, trace_saved);
const auto absent_trace = read(upper.head, 1, 1);
std::filesystem::rename(trace_saved, trace_path);
require(!absent_trace && absent_trace.error().recovery.kind == RecoveryError::tail,
"history trusted a payload without its replay evidence");
take(f.db->query("BEGIN", {}, 5s), "uncommitted history transaction");
const auto transactional = read(upper.head, 0, 1);
require(!f.db->transaction_idle() && !transactional &&
transactional.error().kind == CellularHistoryError::invalid_request,
"history admitted a caller-owned transaction");
const auto orphan =
take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build,
{false, 1200, 1800}, 5s),
"uncommitted history cut");
auto observer = connection();
const auto invisible = load_cellular_census_history(
*observer, *f.objects, f.key, f.lease, orphan.head, orphan.head.sequence, 1, builds, 5s);
take(f.db->query("ROLLBACK", {}, 5s), "rollback uncommitted history cut");
require(f.db->transaction_idle() && !invisible &&
take(load_region_checkpoint(*f.db, f.key, 5s), "orphan history head").head ==
f.head.head,
"uncommitted object/ghost became a durable history event");
auto old_lease = f.lease;
--old_lease.epoch;
const auto stale = load_cellular_census_history(*f.db, *f.objects, f.key, old_lease, upper.head,
1, 1, builds, 5s);
require(!stale && stale.error().recovery.store.kind == LeaseStoreError::fenced &&
take(read(upper.head, 0, 2), "current history reader").entries[1].committed.head ==
birth.head,
"historical evidence ignored a stale fence or rewrote original writers");
take(f.db->query("BEGIN READ ONLY", {}, 5s), "read-only history transaction");
const auto readonly = read(upper.head, 0, 1);
take(f.db->query("ROLLBACK", {}, 5s), "close read-only history transaction");
require(!readonly && readonly.error().kind == CellularHistoryError::invalid_request,
"history transaction requirement changed for read-only callers");
std::puts("committed census history pagination, replay, transaction and failure guards passed");
}
void maturing_census() {
Fixture f;
f.key = {30202, 1};
f.next_id = 40000;
funded_source(f);
const std::array<Sha256Digest, 1> builds{f.build};
const auto recover_raw = [&] {
auto recovered =
take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s),
"census raw recovery");
f.head = std::move(recovered.committed);
f.raw = std::move(recovered.state);
};
f.head = take(
migrate_cellular_boundary_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
"census boundary source");
recover_raw();
std::vector<CellularCensusHistoryEntry> expected;
expected.push_back({f.head, take(CellularCensus::observe(*f.raw), "old census oracle")});
auto reader = connection();
std::array<CellularCensusRegionRequest, 1> requests{{{1, f.lease, f.objects.get()}}};
const auto snapshot = [&](const CellularCensusSnapshotHooks& hooks = {}) {
return load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s,
hooks);
};
const auto verify_snapshot = [&](const RegisteredCellularCensusSnapshot& view,
const CellularCensusHistoryEntry& oracle) {
require(view.catalog.size() == 1 && view.snapshot.cuts.size() == 1 &&
view.snapshot.cuts.front().head == oracle.committed.head &&
view.snapshot.cuts.front().manifest == oracle.committed.manifest &&
view.snapshot.census.boundary().rules == oracle.census.boundary().rules,
"maturing snapshot rebound the captured cut");
for (const auto& row : oracle.census.rows())
for (const auto scope :
{CellularCensusScope::exact_genome, CellularCensusScope::descendants})
require(take(view.snapshot.census.query(row.genome, scope), "snapshot stock") ==
take(oracle.census.query(row.genome, scope), "oracle stock"),
"maturing snapshot lost or invented retained stock");
};
CellularCensusSnapshotHooks upgrade;
upgrade.after_head = [&](std::size_t index) {
require(index == 0, "unexpected census upgrade capture index");
f.head = take(migrate_cellular_maturation_profile(*f.db, *f.objects, f.lease, f.head,
f.id(), f.build, 5s),
"captured maturation");
recover_raw();
};
const auto old_view = take(snapshot(upgrade), "snapshot spanning maturation");
verify_snapshot(old_view, expected.front());
expected.push_back({f.head, take(CellularCensus::observe(*f.raw), "mature raw oracle")});
require(expected[0].committed.head.end_tick == expected[1].committed.head.end_tick &&
expected[0].census.boundary().rules == 8 &&
expected[1].census.boundary().rules == 9 &&
std::ranges::equal(expected[0].census.rows(), expected[1].census.rows()),
"no-tick maturation changed census stock or erased source rules");
verify_snapshot(take(snapshot(), "upgraded registered snapshot"), expected.back());
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
f.build, {true, 1200, 1800}, 5s),
"maturing population entry");
recover(f);
expected.push_back({f.head, take(CellularCensus::observe(*f.population), "population oracle")});
const auto wildlife = std::ranges::find_if(f.population->state().lives(), [](const auto& life) {
return life.account == LifeAccount{};
});
require(wildlife != f.population->state().lives().end(), "maturing census lost real juvenile");
const auto collapsed = take(prepare_population_conversion(*f.population, f.header(0),
{wildlife->identity.genome, true}),
"maturing census cohort");
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, collapsed.batch,
*collapsed.state, f.id(), f.build, 5s),
"maturing census collapse publication");
recover(f);
expected.push_back({f.head, take(CellularCensus::observe(*f.population), "cohort oracle")});
auto growing = unattended(f);
bool grew = false;
for (const auto& life : growing.state->state().lives()) {
const auto old = std::ranges::find(f.population->state().lives(), life.identity.life,
[](const auto& cell) { return cell.identity.life; });
grew |= old != f.population->state().lives().end() &&
life.growth.realized_per_thousand > old->growth.realized_per_thousand;
}
require(grew, "committed census never exercised funded juvenile growth");
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head,
growing.batch.header.end_tick, 5s),
"admit census growth interval");
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, growing.batch,
*growing.state, f.id(), f.build, 5s),
"publish census growth interval");
recover(f);
require(f.population->snapshot() == growing.state->snapshot(), "census growth replay differs");
expected.push_back(
{f.head, take(CellularCensus::observe(*f.population), "grown cohort oracle")});
const auto physical = f.population->state().snapshot();
CellularCensusSnapshotHooks materialize;
materialize.after_head = [&](std::size_t index) {
require(index == 0, "unexpected materialization capture index");
f.head = take(
materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
"captured maturing materialization");
recover_raw();
};
verify_snapshot(take(snapshot(materialize), "snapshot spanning materialization"),
expected.back());
require(f.raw->snapshot() == physical, "materialization changed observed growing biology");
expected.push_back({f.head, take(CellularCensus::observe(*f.raw), "grown raw oracle")});
verify_snapshot(take(snapshot(), "materialized registered snapshot"), expected.back());
require(expected[4].census.boundary().population_format == 2 &&
expected[5].census.boundary().population_format == 0 &&
expected[4].committed.head.end_tick == expected[5].committed.head.end_tick,
"same-tick materialization lost its distinct source representation");
const auto first = expected.front().committed.head.sequence;
const auto read = [&](const RegionHead& watermark, std::uint64_t start, std::size_t limit) {
return load_cellular_census_history(*f.db, *f.objects, f.key, f.lease, watermark, start,
limit, builds, 5s);
};
for (std::size_t offset = 0; offset < expected.size(); offset += 2) {
const auto page = take(read(f.head.head, first + offset, 2), "maturing history page");
require(page.watermark == f.head.head && page.entries.size() == 2 &&
page.next_sequence == (offset + 2 < expected.size()
? std::optional<std::uint64_t>{first + offset + 2}
: std::nullopt),
"maturing history skipped a cut or changed its watermark");
for (std::size_t index = 0; index < page.entries.size(); ++index) {
const auto& entry = page.entries[index];
const auto& oracle = expected[offset + index];
require(entry.committed.head == oracle.committed.head &&
entry.committed.manifest == oracle.committed.manifest &&
entry.census.boundary() == oracle.census.boundary() &&
std::ranges::equal(entry.census.rows(), oracle.census.rows()),
"committed growth census differs from exact retained oracle");
}
}
const auto pinned =
take(read(expected[1].committed.head, first, 8), "pinned pre-growth history");
require(pinned.entries.size() == 2 && !pinned.next_sequence &&
pinned.entries[0].census.boundary().rules == 8 &&
pinned.entries[1].census.boundary().rules == 9,
"later growth reinterpreted a pinned old rules cut");
for (const auto cut : {std::size_t{1}, expected.size() - 1}) {
const auto manifest =
take(decode_manifest(expected[cut].committed.manifest), "denied census manifest");
const UnreadableObject denied{f.objects->directory() /
hexadecimal(manifest.payload_sha256, false)};
require(!read(f.head.head, first + cut - 1, 2),
"missing growing object returned a partial history page");
if (cut == expected.size() - 1)
require(!snapshot(), "missing current growing object became a registered zero census");
}
require(!load_registered_cellular_census_snapshot(*reader, f.key.world, requests, {}, 5s) &&
!load_cellular_census_history(*f.db, *f.objects, f.key, f.lease, f.head.head, first,
2, {}, 5s),
"unretained growing build returned a census");
auto false_cut = f.head.head;
false_cut.digest.back() ^= 1;
require(!read(false_cut, first, 2), "growing history accepted a false watermark");
const auto stale = f.lease;
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release census authority");
f.lease =
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "replace census authority");
require(!snapshot() && !load_cellular_census_history(*f.db, *f.objects, f.key, stale,
f.head.head, first, 2, builds, 5s),
"growing census ignored a stale fence");
requests.front().lease = f.lease;
verify_snapshot(take(snapshot(), "fresh growing census authority"), expected.back());
require(take(load_region_checkpoint(*f.db, f.key, 5s), "read-only growing census head").head ==
f.head.head &&
f.raw->snapshot() == physical,
"census modified committed growth");
std::puts("maturing census history, registered capture races, stock and failure guards passed");
}
void registered_snapshot_coverage(Fixture& f) {
constexpr std::uint64_t seed = 31001, world = seed + 200;
const std::array<Sha256Digest, 1> builds{f.build};
auto reader = connection();
std::vector<std::unique_ptr<FilesystemObjectStore>> stores;
std::vector<CellularCensusRegionRequest> resources;
const auto enroll = [&](std::uint64_t region) {
const RegionKey key{world, region};
auto objects = take(
FilesystemObjectStore::open(environment("PGHOST"), key, max_cellular_population_bytes),
"registered empty region objects");
const auto state = opening(seed, true, 1, false, region);
take(provision_cellular_region(*f.owner, *objects, key, 1, *state, f.id(), f.build, 5s),
"registered empty region genesis");
const auto lease =
take(acquire_region_lease(*f.db, key, f.build, 60000ms, 5s), "registered region grant");
resources.push_back({region, lease, objects.get()});
stores.push_back(std::move(objects));
};
const auto read = [&](std::span<const CellularCensusRegionRequest> requests) {
return load_registered_cellular_census_snapshot(*reader, world, requests, builds, 5s);
};
const auto refused = [&](auto result, CellularCoverageError error) {
require(!result && result.error().kind == CellularSnapshotError::coverage &&
result.error().coverage.kind == error && reader->transaction_idle(),
"registered coverage omitted an owner or retained its transaction");
};
enroll(1);
const std::array first_request{resources.front()};
const auto before = take(read(first_request), "registered single-region census");
require(before.catalog == std::vector<RegisteredCellularRegion>{{1, 1}} &&
before.snapshot.cuts.size() == 1 && before.history_domain != Uuid{},
"registered snapshot lost catalog/cut evidence");
// Trusted owner metadata fault seam: change the namespace after head capture
// without changing biology. Real quarantined restore/fork is tested by the
// SQL and backup process qualifications, not by bypassing live authority here.
const auto domain_before =
take(f.owner->query("SELECT evolution.load_history_domain_v1()::text", {}, 5s),
"original domain");
const CellularCensusSnapshotHooks context_hook{[&](std::size_t) {
take(f.owner->query(
"WITH added AS (INSERT INTO evolution.history_domains(domain,parent,request,kind) "
"SELECT pg_catalog.gen_random_uuid(), domain, pg_catalog.gen_random_uuid(), "
"'restore' "
"FROM evolution.current_history_domain RETURNING domain) "
"UPDATE evolution.current_history_domain SET domain=(SELECT domain FROM added)",
{}, 5s),
"owner namespace change seam");
}};
const auto context_race = take(load_registered_cellular_census_snapshot(
*reader, world, first_request, builds, 5s, context_hook),
"context snapshot race");
const auto context_after = take(read(first_request), "next namespace snapshot");
require(context_race.history_domain == before.history_domain &&
context_after.history_domain != before.history_domain &&
context_race.snapshot.cuts[0].manifest == context_after.snapshot.cuts[0].manifest,
"namespace escaped the metadata snapshot or was inferred from unchanged cuts");
// Missing context must fail, never manufacture a namespace or a zero census.
take(f.owner->query("DELETE FROM evolution.current_history_domain", {}, 5s),
"owner missing context seam");
const auto absent_context = read(first_request);
require(!absent_context && absent_context.error().kind == CellularSnapshotError::context &&
reader->transaction_idle(),
"missing namespace was accepted");
require(domain_before.size() == 1 && domain_before[0].size() == 1, "original domain shape");
const std::array<std::string, 1> original_domain{
take(domain_before[0][0], "original domain value")};
take(f.owner->query("INSERT INTO evolution.current_history_domain VALUES(true,$1::uuid)",
original_domain, 5s),
"restore owner context seam");
const CellularCensusSnapshotHooks enrollment_hook{[&](std::size_t index) {
if (index == 0)
enroll(2); // Independent committed enrollment after the registry snapshot.
}};
const auto raced = take(load_registered_cellular_census_snapshot(*reader, world, first_request,
builds, 5s, enrollment_hook),
"registered snapshot concurrent enrollment");
require(raced.catalog == before.catalog &&
raced.snapshot.cuts[0].head == before.snapshot.cuts[0].head,
"concurrent enrollment changed an already selected registry cut");
refused(read(first_request), CellularCoverageError::regions);
auto extra = resources;
extra.push_back({999, resources.back().lease, stores.back().get()});
refused(read(extra), CellularCoverageError::regions);
auto wrong = resources;
++wrong.back().lease.frame;
refused(read(wrong), CellularCoverageError::frame);
wrong = resources;
wrong.back().region = 999;
refused(read(wrong), CellularCoverageError::regions);
wrong = resources;
wrong.back().region = wrong.front().region;
const auto duplicate = read(wrong);
require(!duplicate && duplicate.error().kind == CellularSnapshotError::invalid_request,
"registered snapshot accepted duplicate resources");
refused(
load_registered_cellular_census_snapshot(*reader, world + 1000, first_request, builds, 5s),
CellularCoverageError::unknown_world);
take(release_region_lease(*f.db, {world, 2}, resources[1].lease, 5s),
"release registered empty owner");
take(f.owner->query("UPDATE evolution.region_leases SET expires_at=clock_timestamp()-"
"interval '1 second' WHERE world_id=31201 AND region_id=1",
{}, 5s),
"expire registered empty owner");
const auto inactive = take(read(resources), "released and expired registry coverage");
require(inactive.catalog == std::vector<RegisteredCellularRegion>{{1, 1}, {2, 1}},
"registry hid released or expired holders");
// Genesis recovery has no fenced transition to read; this checks inclusion,
// not fresh ownership. No output grants authority to expired/released owners.
auto pending_store = take(FilesystemObjectStore::open(environment("PGHOST"), {world, 3},
max_cellular_population_bytes),
"pending registration objects");
const auto pending_state = opening(seed, true, 1, false, 3);
take(f.owner->query("BEGIN", {}, 5s), "pending registration transaction");
take(provision_cellular_region(*f.owner, *pending_store, {world, 3}, 1, *pending_state, f.id(),
f.build, 5s),
"pending registration genesis");
const auto isolated = read(resources);
take(f.owner->query("ROLLBACK", {}, 5s), "pending registration rollback");
require(take(isolated, "registered snapshot with pending enrollment").catalog ==
inactive.catalog,
"uncommitted enrollment entered a registered evidence cut");
take(reader->query("BEGIN READ ONLY", {}, 5s), "caller registered transaction");
const auto caller = read(resources);
take(reader->query("ROLLBACK", {}, 5s), "caller registered rollback");
require(!caller && caller.error().kind == CellularSnapshotError::invalid_request,
"registered snapshot took over a caller transaction");
for (std::uint64_t region = 3; region <= max_cellular_census_regions; ++region)
enroll(region);
const auto bounded = take(read(resources), "64 registered regions");
require(bounded.catalog.size() == max_cellular_census_regions &&
bounded.snapshot.cuts.size() == max_cellular_census_regions &&
bounded.catalog.back() == RegisteredCellularRegion{64, 1},
"registered coverage lost its last in-bound owner");
enroll(65);
refused(read(std::span{resources}.first(max_cellular_census_regions)),
CellularCoverageError::bounds);
const auto rows = take(reader->query("SELECT * FROM evolution.load_world_region_catalog_v1("
"31201::evolution.u64)",
{}, 5s),
"catalog overflow sentinel");
require(rows.size() == 65, "SQL registry silently truncated the overflow owner");
const RegionKey missing_key{world + 1, 1};
take(f.owner->query("INSERT INTO evolution.region_leases(world_id,region_id,frame_id) "
"VALUES(31202.00,1.0,1.00),(31202,18446744073709551615,1.000)",
{}, 5s),
"registered row with missing head fixture");
auto missing_store = take(FilesystemObjectStore::open(environment("PGHOST"), missing_key,
max_cellular_population_bytes),
"missing registered checkpoint store");
const std::array missing_resources{
CellularCensusRegionRequest{1, resources[0].lease, missing_store.get()},
CellularCensusRegionRequest{std::numeric_limits<std::uint64_t>::max(), resources[0].lease,
missing_store.get()}};
const auto missing = load_registered_cellular_census_snapshot(*reader, missing_key.world,
missing_resources, builds, 5s);
require(!missing && missing.error().kind == CellularSnapshotError::history &&
missing.error().region == 1 && reader->transaction_idle(),
"registered owner without a checkpoint disappeared from coverage");
const RegionKey unsupported_key{world + 2, 1};
const auto unsupported_manifest =
take(genesis_manifest(unsupported_key, 1, f.id(), 50000, f.build, 1, f.build),
"non-cellular registry manifest fixture");
const std::array<std::string, 1> manifest_parameter{
hexadecimal(take(encode_manifest(unsupported_manifest), "non-cellular fixture encoding"))};
take(f.owner->query("SELECT evolution.provision_region_checkpoint($1::bytea)",
manifest_parameter, 5s),
"non-cellular registered owner");
auto unsupported_store =
take(FilesystemObjectStore::open(environment("PGHOST"), unsupported_key,
max_cellular_population_bytes),
"unsupported registered store");
const std::array unsupported_resources{
CellularCensusRegionRequest{1, resources[0].lease, unsupported_store.get()}};
const auto unsupported = load_registered_cellular_census_snapshot(
*reader, unsupported_key.world, unsupported_resources, builds, 5s);
require(!unsupported && unsupported.error().kind == CellularSnapshotError::history &&
unsupported.error().history.recovery.kind == RecoveryError::unsupported &&
reader->transaction_idle(),
"unsupported registered owner was filtered out or reported as zero");
std::puts("registered world coverage, enrollment isolation and 64/65 bounds passed");
}
void snapshot_transport_faults(Fixture& f) {
if (!std::getenv("EVO_MIGRATION_TCP_PORT"))
return;
const std::array<Sha256Digest, 1> builds{f.build};
const std::array requests{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()}};
for (unsigned mode = 0; mode < 8; ++mode) {
test::PostgresRelay relay(migration_backend_port());
auto db = migration_tls_connection(relay);
const bool registered = mode >= 4;
const bool at_completion = mode % 4 >= 2;
const bool lost_request = mode % 2 == 0;
const auto partition = [&] {
if (lost_request)
relay.drop_requests(true);
else
relay.drop_responses(true);
};
if (!at_completion)
partition();
const CellularCensusSnapshotHooks hook{[&](std::size_t) {
if (at_completion)
partition();
}};
const auto failure = [](auto result) {
if (result)
throw std::runtime_error("lost capture I/O returned a census");
return result.error();
};
const auto lost = registered ? failure(load_registered_cellular_census_snapshot(
*db, f.key.world, requests, builds, 300ms, hook))
: failure(load_cellular_census_snapshot(
*db, f.key.world, requests, builds, 300ms, hook));
require(lost.kind == CellularSnapshotError::transaction &&
lost.transaction.kind == PgError::unknown && !db->transaction_idle() &&
relay.healthy() &&
(lost_request ? relay.dropped_requests() : relay.dropped_responses()) > 0 &&
take(load_region_checkpoint(*f.db, f.key, 5s), "head after lost capture I/O")
.head == f.head.head,
"lost snapshot BEGIN/COMMIT request/reply returned evidence or kept a session");
}
std::puts("snapshot TLS BEGIN/COMMIT request/reply loss fails closed");
}
void snapshot_guards(Fixture& f) {
const std::array<Sha256Digest, 1> builds{f.build};
const std::array requests{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()}};
const auto read = [&] {
return load_cellular_census_snapshot(*f.db, f.key.world, requests, builds, 5s);
};
const auto before = f.head;
const auto first = take(read(), "single-region captured snapshot");
const auto second = take(read(), "repeat captured snapshot");
require(first.cuts.size() == 1 && first.cuts[0].head == before.head &&
first.cuts[0].manifest == before.manifest &&
second.cuts[0].head == first.cuts[0].head &&
std::ranges::equal(
first.census.regions()[0].rows(),
take(CellularCensus::observe(*f.raw), "snapshot direct oracle").rows()) &&
f.db->transaction_idle(),
"captured snapshot lost its exact committed cut/biology or retained a transaction");
const auto invalid = [&](auto region_requests, std::uint64_t world = 30201,
std::chrono::milliseconds timeout = 5s) {
const auto result =
load_cellular_census_snapshot(*f.db, world, region_requests, builds, timeout);
require(!result && result.error().kind == CellularSnapshotError::invalid_request &&
f.db->transaction_idle(),
"invalid snapshot request started a transaction or returned a census");
};
invalid(std::vector<CellularCensusRegionRequest>{});
invalid(requests, 0);
invalid(requests, f.key.world, 0ms);
invalid(requests, f.key.world, 30001ms);
auto bad = requests;
bad[0].objects = nullptr;
invalid(bad);
bad = requests;
bad[0].region = 0;
invalid(bad);
bad = requests;
bad[0].lease.frame = 0;
invalid(bad);
invalid(std::array{requests[0], requests[0]});
auto higher = requests[0];
++higher.region;
invalid(std::array{higher, requests[0]});
invalid(std::vector<CellularCensusRegionRequest>(max_cellular_census_regions + 1, requests[0]));
higher.region = 999'999;
const auto missing = load_cellular_census_snapshot(*f.db, f.key.world,
std::array{requests[0], higher}, builds, 5s);
require(!missing && missing.error().kind == CellularSnapshotError::history &&
missing.error().region == higher.region && f.db->transaction_idle(),
"missing region became partial coverage or left capture open");
for (const auto* begin : {"BEGIN", "BEGIN READ ONLY"}) {
take(f.db->query(begin, {}, 5s), "caller snapshot transaction");
if (std::string_view{begin} == "BEGIN")
take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build,
{false, 1200, 1800}, 5s),
"pending caller snapshot publication");
const auto refused = read();
require(!refused && refused.error().kind == CellularSnapshotError::invalid_request &&
!f.db->transaction_idle(),
"snapshot took ownership of a caller transaction");
take(f.db->query("ROLLBACK", {}, 5s), "caller snapshot rollback");
}
const auto manifest = take(decode_manifest(before.manifest), "snapshot retained payload");
const auto path = f.objects->directory() / hexadecimal(manifest.payload_sha256, false);
const auto saved = path.string() + ".snapshot-test";
std::filesystem::rename(path, saved);
const auto missing_object = read();
std::filesystem::rename(saved, path);
require(!missing_object && missing_object.error().kind == CellularSnapshotError::history &&
f.db->transaction_idle(),
"missing replay payload became a snapshot or left capture open");
{
const UnreadableObject denied{path};
const auto unreadable = read();
require(!unreadable && unreadable.error().kind == CellularSnapshotError::history &&
f.db->transaction_idle(),
"unreadable captured payload became a census");
}
auto stale = requests;
--stale[0].lease.epoch;
const auto fenced = load_cellular_census_snapshot(*f.db, f.key.world, stale, builds, 5s);
require(!fenced && fenced.error().history.recovery.store.kind == LeaseStoreError::fenced,
"snapshot ignored stale evidence authority");
const std::array<Sha256Digest, 1> foreign_builds{};
require(!load_cellular_census_snapshot(*f.db, f.key.world, requests, foreign_builds, 5s),
"snapshot accepted unretained build evidence");
auto closed = connection();
CellularCensusSnapshotHooks close_hook{[&](std::size_t) { closed->disconnect(); }};
const auto lost =
load_cellular_census_snapshot(*closed, f.key.world, requests, builds, 5s, close_hook);
closed->disconnect();
const auto closed_query = closed->query("SELECT 1", {}, 5s);
require(!lost && lost.error().kind == CellularSnapshotError::transaction &&
!closed->transaction_idle() && !closed_query &&
closed_query.error().kind == PgError::closed,
"failed capture completion reused a dead session or returned a partial census");
auto throwing = connection();
bool threw = false;
bool hook_called = false;
try {
const CellularCensusSnapshotHooks hook{[&](std::size_t) {
hook_called = true;
throw std::runtime_error("capture hook test exception");
}};
const auto result =
load_cellular_census_snapshot(*throwing, f.key.world, requests, builds, 5s, hook);
require(!result, "throwing capture hook returned a snapshot");
} catch (const std::runtime_error&) {
threw = hook_called;
}
const auto after_throw = throwing->query("SELECT 1", {}, 5s);
require(threw && !after_throw && after_throw.error().kind == PgError::closed &&
!throwing->transaction_idle() &&
take(load_region_checkpoint(*f.db, f.key, 5s), "head after snapshot guards").head ==
before.head,
"capture exception retained its connection/transaction or changed a head");
std::puts("consistent census snapshot guards and connection cleanup passed");
snapshot_transport_faults(f);
}
void boundary_upgrades(Fixture& f, bool maturation = false) {
const auto migrate =
[maturation](Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
const CommittedCheckpoint& previous, Uuid commit, const Sha256Digest& build,
std::chrono::milliseconds timeout, const PublishHooks& hooks = {}) {
return (maturation ? migrate_cellular_maturation_profile
: migrate_cellular_boundary_profile)(db, objects, lease, previous,
commit, build, timeout, hooks);
};
const auto derive_manifest =
maturation ? cellular_maturation_upgrade_manifest : cellular_boundary_upgrade_manifest;
const auto target_bundle = maturation ? cellular_v11_versions : cellular_v10_versions;
const std::array<Sha256Digest, 1> builds{f.build};
const auto original = f.head;
if (maturation) {
const auto stale = f.lease;
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release pre-maturation authority");
f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s),
"new maturation authority");
const auto refused = migrate(*f.db, *f.objects, stale, f.head, f.id(), f.build, 5s);
require(!refused && refused.error().store.kind == LeaseStoreError::fenced &&
take(load_region_checkpoint(*f.db, f.key, 5s), "stale upgrade head").head ==
original.head,
"stale maturation writer changed head");
} else {
// A validly encoded target cannot skip the retained accounting predecessor.
const auto base = take(decode_manifest(f.head.manifest), "skipped upgrade source");
auto skipped = take(cellular_boundary_upgrade_manifest(base, f.head.head.digest, f.id(),
f.lease, f.build, base.payload_bytes,
base.payload_sha256),
"skipped target fixture");
skipped.cellular_versions = cellular_v11_versions;
require(!publish_cellular_profile_migration(
*f.db, take(encode_manifest(skipped), "skipped encode"), 5s) &&
take(load_region_checkpoint(*f.db, f.key, 5s), "skipped upgrade head").head ==
original.head,
"SQL maturation skipped bundle 10");
}
const auto expected_raw = take(maturation ? f.raw->maturation_upgrade_candidate()
: f.raw->migration_upgrade_candidate(),
"raw upgrade oracle")
->snapshot();
// Rehearse the raw upgrade, recovery and ordinary trace-3 publication in one
// transaction, then roll it back to retain a real bundle-9 source for profile 5.
take(f.db->query("BEGIN", {}, 5s), "raw upgrade rollback rehearsal");
auto raw_head = take(migrate(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
"raw accounting upgrade");
auto raw = take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s),
"raw accounting recovery");
require(raw.state->snapshot() == expected_raw &&
raw.manifest.cellular_versions == target_bundle &&
raw_head.head.end_tick == original.head.end_tick,
"raw accounting upgrade changed biology or time");
const CellularBatchHeader h{f.key.world,
f.key.region,
1,
raw_head.head.commit,
raw_head.head.digest,
raw_head.head.end_tick,
raw_head.head.end_tick + 1};
const auto inputs = take(unattended_population_inputs(*raw.state, h), "upgraded raw inputs");
auto advanced = take(replay_cellular_batch(*raw.state, inputs), "upgraded raw oracle");
take(publish_cellular_batch(*f.db, *f.objects, f.lease, raw_head, inputs, *advanced, f.id(),
f.build, 5s),
"upgraded raw ordinary publication");
require(take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s),
"upgraded raw ordinary recovery")
.state->snapshot() == advanced->snapshot(),
"raw accounting continuation differs");
take(f.db->query("ROLLBACK", {}, 5s), "roll back raw upgrade rehearsal");
require(take(load_region_checkpoint(*f.db, f.key, 5s), "raw rollback head").head ==
original.head,
"rolled-back upgrade moved the durable head");
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
f.build, {true, 1200, 1800}, 5s),
"population accounting source entry");
recover(f);
const auto member = std::ranges::find_if(f.population->state().lives(), [](const auto& life) {
return life.account == LifeAccount{};
});
require(member != f.population->state().lives().end(), "accounting source has no wildlife");
auto collapsed = take(
prepare_population_conversion(*f.population, f.header(0), {member->identity.genome, true}),
"accounting source cohort");
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, collapsed.batch,
*collapsed.state, f.id(), f.build, 5s),
"accounting source membership publication");
recover(f);
require(!f.population->members().empty() && f.population->conversion_sequence() != 0,
"accounting source lost cohort history");
auto interval = unattended(f);
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head,
interval.batch.header.end_tick, 5s),
"upgrade pending interval");
auto pending = migrate(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s);
require(!pending && std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18",
"accounting upgrade discarded admitted work");
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, interval.batch,
*interval.state, f.id(), f.build, 5s),
"settle pre-upgrade work");
recover(f);
const auto before = f.head;
const auto source = take(decode_manifest(before.manifest), "accounting source manifest");
const auto upgraded_population = take(maturation ? f.population->maturation_upgrade_candidate()
: f.population->migration_upgrade_candidate(),
"cohort upgrade oracle");
const auto expected = upgraded_population->snapshot();
const auto altered =
take(CellularPopulation::create(restore(upgraded_population->state()), {false, 1200, 1800}),
"valid unequal upgrade target");
require(altered->state().current_total() == upgraded_population->state().current_total() &&
altered->snapshot() != expected,
"unequal upgrade fixture must retain biological totals");
const auto altered_object = take(f.objects->put(altered->snapshot()), "unequal upgrade object");
const auto altered_manifest =
take(derive_manifest(source, before.head.digest, f.id(), f.lease, f.build,
altered_object.bytes, altered_object.sha256),
"unequal upgrade manifest");
take(f.db->query("BEGIN", {}, 5s), "unequal upgrade recovery rehearsal");
take(publish_cellular_profile_migration(
*f.db, take(encode_manifest(altered_manifest), "unequal upgrade encoding"), 5s),
"metadata-only unequal upgrade publication");
const auto unequal =
recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
take(f.db->query("ROLLBACK", {}, 5s), "undo unequal upgrade rehearsal");
require(!unequal && unequal.error().kind == RecoveryError::history &&
take(load_region_checkpoint(*f.db, f.key, 5s), "head after unequal upgrade").head ==
before.head,
"upgrade recovery compared only aggregate stock or rewrote the retained source");
const auto child = ::fork();
require(child >= 0, "upgrade pre-transaction fork");
if (child == 0) {
try {
auto db = connection();
PublishHooks hooks;
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
[[maybe_unused]] const auto result =
migrate(*db, *f.objects, f.lease, before, identity(60000), f.build, 5s, hooks);
} catch (...) {
::_exit(31);
}
::_exit(32);
}
int status{};
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
WTERMSIG(status) == SIGKILL,
"upgrade child missed pre-transaction kill");
recover(f);
require(f.head.head == before.head, "unpublished upgrade became authoritative");
auto locker = connection(true);
take(locker->query("BEGIN", {}, 5s), "accounting ambiguity lock transaction");
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 "
"AND region_id=1 FOR UPDATE",
{}, 5s),
"hold accounting reply");
auto timed = connection();
auto unknown = migrate(*timed, *f.objects, f.lease, before, f.id(), f.build, 300ms);
require(!unknown && unknown.error().kind == PublicationError::unknown &&
!unknown.error().prepared.manifest.empty(),
"uncertain accounting upgrade lost its exact request");
take(locker->query("ROLLBACK", {}, 5s), "release accounting reply");
auto resolver = connection();
f.head = take(retry_cellular_profile_migration(*resolver, unknown.error().prepared, 5s),
"retry exact accounting upgrade");
require(take(retry_cellular_profile_migration(*resolver, unknown.error().prepared, 5s),
"duplicate accounting upgrade")
.head == f.head.head,
"duplicate accounting upgrade advanced head");
recover(f);
require(f.population->snapshot() == expected && f.head.head.end_tick == before.head.end_tick &&
take(decode_manifest(f.head.manifest), "upgraded population manifest")
.cellular_versions == target_bundle,
"accounting upgrade changed biology, cohort history or time");
if (maturation) {
auto downgrade = take(decode_manifest(f.head.manifest), "downgrade fixture");
downgrade.predecessor_commit = downgrade.commit;
downgrade.predecessor_digest = f.head.head.digest;
downgrade.commit = f.id();
++downgrade.sequence;
downgrade.cellular_versions = cellular_v10_versions;
require(!publish_cellular_profile_migration(
*f.db, take(encode_manifest(downgrade), "downgrade encode"), 5s) &&
take(load_region_checkpoint(*f.db, f.key, 5s), "downgrade head").head ==
f.head.head,
"SQL downgraded maturation biology");
}
auto changed = unknown.error().prepared;
auto changed_manifest = take(decode_manifest(changed.manifest), "changed accounting manifest");
changed_manifest.payload_sha256[0] ^= 1;
changed.manifest = take(encode_manifest(changed_manifest), "changed accounting bytes");
auto conflict = retry_cellular_profile_migration(*resolver, changed, 5s);
require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict,
"accounting retry accepted changed bytes");
require(!migrate(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
"accounting upgrade applied twice");
const auto path = f.objects->directory() / hexadecimal(source.payload_sha256, false);
const auto backup = path.string() + ".upgrade-test";
std::filesystem::rename(path, backup);
const auto missing =
recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
std::filesystem::rename(backup, path);
require(!missing && missing.error().kind == RecoveryError::tail,
"accounting recovery trusted aggregate stock without its retained source");
missing_and_corrupt_objects(f);
const auto stale = f.lease;
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release accounting writer");
f.lease =
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "replace accounting writer");
auto stale_retry = retry_cellular_profile_migration(*f.db, unknown.error().prepared, 5s);
require(!stale_retry && stale_retry.error().store.kind == LeaseStoreError::fenced &&
!recover_population_checkpoint(*f.db, *f.objects, f.key, stale, builds, 5s),
"old accounting receipt granted current authority");
recover(f);
require(f.population->snapshot() == expected, "new epoch changed accounting upgrade");
auto future_interval = unattended(f);
auto old_trace = future_interval.batch;
old_trace.format_version = 1;
const auto old_bytes = take(encode_population_batch(old_trace), "wrong old trace fixture");
const auto upgraded_manifest = take(decode_manifest(f.head.manifest), "new accounting source");
auto wrong_trace_manifest =
take(population_successor_manifest(
upgraded_manifest, f.head.head.digest, f.id(), f.lease, old_trace.header.end_tick,
f.build, future_interval.state->snapshot().size(),
take(sha256(future_interval.state->snapshot()), "new payload SHA"),
{old_bytes.size(), take(sha256(old_bytes), "old trace SHA"), 0, {}}),
"wrong old trace manifest fixture");
const PopulationPreparedCommit wrong_trace{
f.key, wrong_trace_manifest.commit,
take(encode_manifest(wrong_trace_manifest), "wrong old trace manifest"), old_bytes};
auto refused_old = retry_population_prepared(*f.db, wrong_trace, 5s);
auto refused_old_sql =
publish_population_checkpoint(*f.db, wrong_trace.manifest, old_bytes, 5s);
require(!refused_old && refused_old.error().kind == PublicationError::candidate &&
!refused_old_sql && refused_old_sql.error().kind == LeaseStoreError::invalid &&
take(load_region_checkpoint(*f.db, f.key, 5s), "head after old trace").head ==
f.head.head,
"bundle 10 reinterpreted retained population trace 1");
if (maturation) {
auto growing = unattended(f);
bool grew = false;
for (const auto& life : growing.state->state().lives()) {
const auto old = std::ranges::find(f.population->state().lives(), life.identity.life,
[](const auto& cell) { return cell.identity.life; });
grew |= old != f.population->state().lives().end() &&
life.growth.realized_per_thousand > old->growth.realized_per_thousand;
}
require(grew, "durable maturation fixture never grew a real juvenile");
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head,
growing.batch.header.end_tick, 5s),
"admit funded growth interval");
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, growing.batch,
*growing.state, f.id(), f.build, 5s),
"funded maturation publication");
recover(f);
require(f.population->snapshot() == growing.state->snapshot(),
"funded growth recovery differs");
}
// Exercise the existing publication/process-loss matrix with batch 2 as well.
process_loss(f, 0);
process_loss(f, 1);
process_loss(f, 2);
isolation_and_reverse(f);
require(f.raw->format_version() == cellular_lives_migration_format,
"reverse transition downgraded accounting owner");
}
void migration_owner_partitions(Fixture& f, RegionKey destination_key,
FilesystemObjectStore& destination_objects,
LeaseGrant& destination_lease,
CommittedCheckpoint& destination_head, std::uint64_t genome) {
if (!std::getenv("EVO_MIGRATION_TCP_PORT"))
return;
const std::array<Sha256Digest, 1> builds{f.build};
for (unsigned lost_source = 0; lost_source < 2; ++lost_source) {
test::PostgresRelay source_relay(migration_backend_port()),
destination_relay(migration_backend_port());
const auto make_host = [&](RegionKey key, const test::PostgresRelay& relay) {
auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), key,
max_cellular_population_bytes),
"partition owner objects");
CellularHostConfig config{key, 1, {}, f.build, {f.build}};
config.lease = {3000ms, 400ms, 300ms, 150ms, 250ms};
const auto options = migration_tls_options(relay);
return take(CellularRegionHost::create(
config, [options] { return Postgres::connect(options, 500ms); },
std::move(objects)),
"partition owner host");
};
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release source to TLS owner");
take(release_region_lease(*f.db, destination_key, destination_lease, 5s),
"release destination to TLS owner");
auto source_host = make_host(f.key, source_relay),
destination_host = make_host(destination_key, destination_relay);
require(source_host->step().ready && destination_host->step().ready,
"TLS owners before freeze");
const auto sl = take(source_host->authority(), "TLS source fence"),
dl = take(destination_host->authority(), "TLS destination fence");
const auto source_bytes = source_host->state()->snapshot(),
destination_bytes = destination_host->state()->snapshot();
const auto source_before = f.head, destination_before = destination_head;
const CellularMigrationRequest request{
f.id(),
{f.key.region, f.head.head.commit, f.id(), f.head.head.digest},
{destination_key.region, destination_head.head.commit, f.id(),
destination_head.head.digest},
genome};
source_relay.drop_responses(true);
const auto unknown_prepare =
source_host->prepare_migration_freeze(destination_head, destination_objects, request);
require(!unknown_prepare &&
unknown_prepare.error().publication == PublicationError::unknown &&
!source_host->status().ready && source_relay.dropped_responses() > 0,
"lost preparation receipt did not pause source ownership");
const auto frozen =
take(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, sl, 5s),
"independent committed preparation observation"),
"missing preparation after TLS reply loss");
source_relay.drop_responses(false);
const auto reconcile_source_until = LeaseClock::now() + 2s;
while (!source_host->status().migration_frozen &&
LeaseClock::now() < reconcile_source_until) {
source_host->step();
destination_host->step();
std::this_thread::sleep_for(25ms);
}
require(source_host->status().migration_frozen && !source_host->status().ready,
"source lost the exact prepared request after TLS reconnect");
destination_relay.drop_responses(true);
const auto unknown_stage =
destination_host->stage_migration(f.head, *f.objects, frozen.trace);
require(!unknown_stage && unknown_stage.error().publication == PublicationError::unknown &&
!destination_host->status().ready &&
destination_relay.dropped_responses() > 0 &&
take(load_cellular_migration_stage(*f.db, destination_objects, destination_key,
dl, 5s),
"independent committed stage observation")
.has_value(),
"lost stage receipt activated the ghost or lost retained work");
destination_relay.drop_responses(false);
const auto reconcile_destination_until = LeaseClock::now() + 2s;
while (!destination_host->status().migration_frozen &&
LeaseClock::now() < reconcile_destination_until) {
source_host->step();
destination_host->step();
std::this_thread::sleep_for(25ms);
}
require(source_host->status().migration_frozen &&
destination_host->status().migration_frozen,
"TLS owners failed exact stage reconciliation");
auto& isolated = lost_source ? source_host : destination_host;
auto& healthy = lost_source ? destination_host : source_host;
auto& relay = lost_source ? source_relay : destination_relay;
const auto isolated_key = lost_source ? f.key : destination_key;
auto& replacement = lost_source ? f.lease : destination_lease;
const auto healthy_key = lost_source ? destination_key : f.key;
auto& healthy_lease = lost_source ? destination_lease : f.lease;
relay.drop_requests(true);
const auto expire_until = LeaseClock::now() + 3500ms;
while (LeaseClock::now() < expire_until) {
isolated->step();
healthy->step();
std::this_thread::sleep_for(25ms);
}
require(!isolated->status().ready && !isolated->authority().has_value() &&
!isolated->advance_unattended(64) && relay.dropped_requests() > 0 &&
healthy->status().phase == HostPhase::running &&
healthy->status().epoch == (lost_source ? dl.epoch : sl.epoch) &&
healthy->status().migration_frozen,
"isolated owner retained authority or healthy peer released a frozen ghost");
replacement = take(acquire_region_lease(*f.db, isolated_key, f.build, 60000ms, 5s),
"replacement after actual TLS partition expiry");
require(replacement.epoch > (lost_source ? sl.epoch : dl.epoch),
"replacement reused partitioned epoch");
if (lost_source)
require(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, replacement, 5s),
"replacement source retains freeze")
.has_value(),
"source lease expiry implicitly aborted");
else
require(take(load_cellular_migration_stage(*f.db, destination_objects, destination_key,
replacement, 5s),
"replacement destination retains stage")
.has_value(),
"destination lease expiry activated ghost");
const auto stale =
publish_cellular_migration(*f.db, *f.objects, destination_objects, sl, dl, f.head,
destination_head, frozen.trace, f.build, 5s);
require(!stale && stale.error().store.kind == LeaseStoreError::fenced &&
take(load_region_checkpoint(*f.db, f.key, 5s), "partition source head").head ==
source_before.head &&
take(load_region_checkpoint(*f.db, destination_key, 5s),
"partition destination head")
.head == destination_before.head,
"stale partitioned fence published a migration half");
healthy->step();
const auto current_source = lost_source ? replacement : sl;
if (lost_source) {
test::PostgresRelay abort_relay(migration_backend_port());
auto db = migration_tls_connection(abort_relay);
abort_relay.drop_responses(true);
const auto unknown =
abort_cellular_migration_freeze(*db, f.key, current_source, request.request, 300ms);
require(!unknown && unknown.error().kind == LeaseStoreError::unknown &&
abort_relay.dropped_responses() > 0 && abort_relay.healthy(),
"lost replacement abort receipt was not unknown");
take(abort_cellular_migration_freeze(*f.db, f.key, current_source, request.request, 5s),
"exact replacement abort reconciliation");
} else {
source_relay.drop_responses(true);
const auto unknown = source_host->abort_migration_freeze(request.request);
require(!unknown && !source_host->status().ready &&
source_relay.dropped_responses() > 0,
"lost owning-source abort receipt resumed the region");
source_relay.drop_responses(false);
}
require(!take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, current_source, 5s),
"independent abort source observation")
.has_value(),
"lost abort receipt reopened source stock");
const auto resume_until = LeaseClock::now() + 2s;
while (!healthy->step().ready && LeaseClock::now() < resume_until)
std::this_thread::sleep_for(25ms);
require(healthy->status().ready && healthy->state()->snapshot() ==
(lost_source ? destination_bytes : source_bytes),
"healthy owner resumed different stock after abort");
relay.drop_requests(false);
isolated->step();
require(!isolated->status().ready && !isolated->advance_unattended(64),
"healed stale owner resumed despite a replacement fence");
const auto isolated_manifest =
take(decode_manifest(lost_source ? f.head.manifest : destination_head.manifest),
"isolated head profile");
const auto& objects = lost_source ? *f.objects : destination_objects;
std::vector<std::uint8_t> recovered;
if (isolated_manifest.profile == CheckpointProfile::cellular_population_v1)
recovered = take(recover_population_checkpoint(*f.db, objects, isolated_key,
replacement, builds, 5s),
"replacement population replay")
.state->state()
.snapshot();
else
recovered = take(recover_cellular_checkpoint(*f.db, objects, isolated_key, replacement,
builds, 5s),
"replacement raw replay")
.state->snapshot();
require(recovered == (lost_source ? source_bytes : destination_bytes) &&
source_relay.healthy() && destination_relay.healthy(),
"replacement or relay lost exact stock");
source_host->drain();
destination_host->drain();
healthy_lease = take(acquire_region_lease(*f.db, healthy_key, f.build, 60000ms, 5s),
"return healthy TLS owner authority");
}
std::puts("migration independent TLS owner partitions, expiry, replacement and heal passed");
}
void migration_snapshot_race(Fixture& f, RegionKey destination_key,
FilesystemObjectStore& destination_objects,
const LeaseGrant& destination_lease,
CommittedCheckpoint& destination_head, std::uint64_t genome) {
const std::array<Sha256Digest, 1> builds{f.build};
const std::array requests{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()},
CellularCensusRegionRequest{destination_key.region, destination_lease,
&destination_objects}};
auto reader = connection();
for (const bool reverse : {false, true}) {
auto& sh = reverse ? destination_head : f.head;
auto& dh = reverse ? f.head : destination_head;
auto& so = reverse ? destination_objects : *f.objects;
auto& dst = reverse ? *f.objects : destination_objects;
const auto& sl = reverse ? destination_lease : f.lease;
const auto& dl = reverse ? f.lease : destination_lease;
const auto sk = reverse ? destination_key : f.key;
const auto dk = reverse ? f.key : destination_key;
const auto before_registered = take(
load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s),
"before snapshot migration race");
const auto& before = before_registered.snapshot;
CellularMigrationRequest request{f.id(),
{sk.region, sh.head.commit, f.id(), sh.head.digest},
{dk.region, dh.head.commit, f.id(), dh.head.digest},
genome};
const auto read = [&] {
return load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds,
5s);
};
const auto unchanged_stock = [&](const auto& snapshot) {
require(take(snapshot.snapshot.census.query(genome, CellularCensusScope::descendants),
"pending lineage") ==
take(before.census.query(genome, CellularCensusScope::descendants),
"before pending lineage"),
"pending evidence added stock");
};
const auto refused = [&](const auto& result, RecoveryError error) {
require(!result && result.error().kind == CellularSnapshotError::migration &&
result.error().migration.kind == error && reader->transaction_idle(),
"bad pending evidence returned a partial census or retained a transaction");
};
std::optional<CellularMigrationFreeze> preparation;
const CellularCensusSnapshotHooks prepare_hook{[&](std::size_t i) {
if (i == 0)
preparation =
take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s),
"capture prepare");
}};
const auto before_prepare =
take(load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds,
5s, prepare_hook),
"before concurrent prepare");
auto frozen = take(std::move(preparation), "preparation hook ran");
require(before_registered.migrations.empty() && before_prepare.migrations.empty(),
"concurrent preparation escaped its metadata view");
const auto prepared = take(read(), "prepared snapshot inventory");
require(prepared.migrations.size() == 1 && !prepared.migrations[0].staged &&
prepared.migrations[0].trace == frozen.trace,
"source-only inventory missing or treated as staged");
unchanged_stock(prepared);
const auto unretained = load_registered_cellular_census_snapshot(
*reader, f.key.world, requests, std::span<const Sha256Digest>{}, 5s);
refused(unretained, RecoveryError::unsupported);
const auto dm = take(decode_manifest(dh.manifest), "pending destination capsule");
{
UnreadableObject denied(so.directory() / hexadecimal(dm.payload_sha256, false));
refused(read(), RecoveryError::tail);
}
// Trusted owner fault seam: a pointer must not hide a foreign endpoint.
const std::array<std::string, 2> pointer{std::to_string(f.key.world),
std::to_string(sk.region)};
take(f.owner->query("UPDATE evolution.cellular_migration_pending SET region_id=99999 "
"WHERE world_id=$1::evolution.u64 AND region_id=$2::evolution.u64",
pointer, 5s),
"malformed pending pointer seam");
const auto malformed = read();
take(f.owner->query(
"UPDATE evolution.cellular_migration_pending SET region_id=$2::evolution.u64 "
"WHERE world_id=$1::evolution.u64 AND region_id=99999",
pointer, 5s),
"restore pending pointer seam");
refused(malformed, RecoveryError::history);
if (!reverse) {
const auto definition =
take(f.owner->query("SELECT "
"pg_get_functiondef('evolution.load_world_cellular_migrations_"
"v1(evolution.u64)'::regprocedure)",
{}, 5s),
"inventory definition");
const auto original = take(definition.at(0).at(0), "inventory definition value");
take(
f.owner->query(
"CREATE OR REPLACE FUNCTION evolution.load_world_cellular_migrations_v1(w "
"evolution.u64) "
"RETURNS TABLE(trace bytea,source_manifest bytea,destination_manifest bytea,"
"ghost_sha256 bytea,ghost_bytes evolution.u64,coherent boolean) "
"LANGUAGE sql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ "
"SELECT "
"NULL::bytea,NULL::bytea,NULL::bytea,NULL::bytea,NULL::evolution.u64,true "
"FROM pg_catalog.generate_series(1,65) $$",
{}, 5s),
"overflow response seam");
const auto overflow = read();
take(f.owner->query(original, {}, 5s), "restore bounded inventory function");
refused(overflow, RecoveryError::store);
require(overflow.error().migration.store.cause.kind == PgError::bounds,
"65th response was not rejected before parsing");
}
std::optional<CellularMigrationStage> stage;
const CellularCensusSnapshotHooks stage_hook{[&](std::size_t i) {
if (i == 0)
stage = take(stage_cellular_migration(*f.db, so, dst, dl, sh, dh, frozen.trace, 5s),
"capture stage");
}};
const auto before_stage = take(load_registered_cellular_census_snapshot(
*reader, f.key.world, requests, builds, 5s, stage_hook),
"before concurrent stage");
const auto staged = take(read(), "staged snapshot inventory");
require(before_stage.migrations.size() == 1 && !before_stage.migrations[0].staged &&
staged.migrations.size() == 1 && staged.migrations[0].staged,
"concurrent stage escaped the capture view");
unchanged_stock(staged);
const auto ghost = take(std::move(stage), "stage hook ran").ghost;
const auto ghost_path = dst.directory() / hexadecimal(ghost.sha256, false);
{
UnreadableObject denied(ghost_path);
refused(read(), RecoveryError::tail);
}
const CellularCensusSnapshotHooks abort_hook{[&](std::size_t i) {
if (i == 0)
take(abort_cellular_migration_freeze(*f.db, sk, sl, frozen.trace.request.request,
5s),
"capture abort");
}};
const auto before_abort = take(load_registered_cellular_census_snapshot(
*reader, f.key.world, requests, builds, 5s, abort_hook),
"before concurrent abort");
const auto after_abort = take(read(), "after abort inventory");
require(before_abort.migrations.size() == 1 && before_abort.migrations[0].staged &&
after_abort.migrations.empty(),
"abort escaped the selected inventory view");
unchanged_stock(before_abort);
unchanged_stock(after_abort);
request.request = f.id();
request.source.commit = f.id();
request.destination.commit = f.id();
frozen = take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s),
"snapshot race source freeze");
take(stage_cellular_migration(*f.db, so, dst, dl, sh, dh, frozen.trace, 5s),
"snapshot race destination stage");
std::optional<CellularMigrationCommitted> published;
const CellularCensusSnapshotHooks hook{[&](std::size_t index) {
if (index != 0)
return;
const auto settings =
take(reader->query("SELECT current_setting('transaction_isolation'), "
"current_setting('transaction_read_only')",
{}, 5s),
"capture transaction properties");
require(settings.size() == 1 && settings[0].size() == 2 &&
take(settings[0][0], "capture isolation") == "repeatable read" &&
take(settings[0][1], "capture access") == "on",
"capture did not own a repeatable read read-only snapshot");
auto publisher = connection();
published = take(publish_cellular_migration(*publisher, so, dst, sl, dl, sh, dh,
frozen.trace, f.build, 5s),
"publish real pair between snapshot head reads");
}};
const auto raced_registered = take(load_registered_cellular_census_snapshot(
*reader, f.key.world, requests, builds, 5s, hook),
"coherent census across a concurrent pair");
const auto& raced = raced_registered.snapshot;
const auto committed =
take(std::move(published), "pair was not published in snapshot hook");
sh = committed.source;
dh = committed.destination;
require(reader->transaction_idle() && raced.cuts.size() == 2 && before.cuts.size() == 2,
"raced capture retained its transaction or lost coverage");
for (std::size_t i = 0; i < before.cuts.size(); ++i)
require(raced.cuts[i].head == before.cuts[i].head &&
raced.cuts[i].manifest == before.cuts[i].manifest &&
std::ranges::equal(raced.census.regions()[i].rows(),
before.census.regions()[i].rows()),
"atomic pair split a captured cut or changed historical replay");
const auto after_registered = take(
load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s),
"snapshot after pair publication");
const auto& after = after_registered.snapshot;
require(raced_registered.migrations.size() == 1 && raced_registered.migrations[0].staged &&
raced_registered.migrations[0].trace == frozen.trace &&
after_registered.migrations.empty(),
"commit escaped the captured inventory or retained settled pointers");
require(before_registered.catalog == raced_registered.catalog &&
after_registered.catalog == before_registered.catalog &&
before_registered.catalog ==
std::vector<RegisteredCellularRegion>{
{f.key.region, f.lease.frame},
{destination_key.region, destination_lease.frame}},
"atomic migration changed or omitted registered owner coverage");
require(after.cuts[0].head == f.head.head && after.cuts[1].head == destination_head.head &&
after.cuts[0].head.sequence == before.cuts[0].head.sequence + 1 &&
after.cuts[1].head.sequence == before.cuts[1].head.sequence + 1,
"later snapshot omitted one committed migration endpoint");
for (const auto scope :
{CellularCensusScope::exact_genome, CellularCensusScope::descendants}) {
const auto initial = take(before.census.query(genome, scope), "before pair lineage");
const auto captured = take(raced.census.query(genome, scope), "captured pair lineage");
const auto settled = take(after.census.query(genome, scope), "settled pair lineage");
require(initial == captured && initial.resident == settled.resident,
"snapshot migration race lost/duplicated living count/body/reserve");
}
}
std::puts("consistent raw/cohort snapshots across atomic migration commits passed");
}
void migration_publication(Fixture& f, const RegionKey destination_key,
FilesystemObjectStore& destination_objects,
LeaseGrant& destination_lease, CommittedCheckpoint& destination_head) {
const std::array<Sha256Digest, 1> builds{f.build};
const auto wildlife = std::ranges::find_if(
f.raw->lives(), [](const auto& life) { return life.account == LifeAccount{}; });
require(wildlife != f.raw->lives().end(), "paired publication needs real wildlife");
const auto genome = wildlife->identity.genome;
const auto make_host = [&](RegionKey key) {
auto objects = take(
FilesystemObjectStore::open(environment("PGHOST"), key, max_cellular_population_bytes),
"paired host objects");
CellularHostConfig config{key, 1, {}, f.build, {f.build}};
config.lease.commit_timeout = 300ms;
return take(
CellularRegionHost::create(
config,
[] { return std::expected<std::unique_ptr<Postgres>, PgFailure>{connection()}; },
std::move(objects)),
"paired host");
};
for (unsigned representation = 0; representation < 2; ++representation) {
if (representation == 0) {
f.head = take(materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(),
f.build, 5s),
"paired raw source");
destination_head =
take(materialize_population_profile(*f.db, destination_objects, destination_lease,
destination_head, f.id(), f.build, 5s),
"paired raw destination");
} else {
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
f.build, {true, 1200, 1800}, 5s),
"paired population source");
destination_head = take(migrate_cellular_to_population(
*f.db, destination_objects, destination_lease,
destination_head, f.id(), f.build, {true, 1200, 1800}, 5s),
"paired population destination");
}
std::optional<CellularCensusHistoryEntry> outgoing_history, incoming_history;
for (unsigned direction = 0; direction < 2; ++direction) {
auto& sh = direction == 0 ? f.head : destination_head;
auto& dh = direction == 0 ? destination_head : f.head;
auto& so = direction == 0 ? *f.objects : destination_objects;
auto& dst = direction == 0 ? destination_objects : *f.objects;
auto& sl = direction == 0 ? f.lease : destination_lease;
auto& dl = direction == 0 ? destination_lease : f.lease;
const auto sk = direction == 0 ? f.key : destination_key;
const auto dk = direction == 0 ? destination_key : f.key;
take(release_region_lease(*f.db, sk, sl, 5s), "release paired source to host");
take(release_region_lease(*f.db, dk, dl, 5s), "release paired destination to host");
auto source_host = make_host(sk), destination_host = make_host(dk);
require(source_host->step().ready && destination_host->step().ready,
"paired hosts before preparation");
sl = take(source_host->authority(), "current source authority before pause");
dl = take(destination_host->authority(), "current destination authority before pause");
const auto source_before = sh, destination_before = dh;
const auto s = take(decode_manifest(sh.manifest), "paired predecessor source");
const auto d = take(decode_manifest(dh.manifest), "paired predecessor destination");
const auto sb =
take(so.get({s.payload_sha256, s.payload_bytes}), "paired source bytes");
const auto dbb =
take(dst.get({d.payload_sha256, d.payload_bytes}), "paired destination bytes");
const CellularMigrationRequest request{
f.id(),
{sk.region, sh.head.commit, f.id(), sh.head.digest},
{dk.region, dh.head.commit, f.id(), dh.head.digest},
genome};
ObjectStoreFaults source_faults, destination_faults;
auto failing_source =
take(FilesystemObjectStore::open(environment("PGHOST"), sk,
max_cellular_population_bytes, &source_faults),
"migration source sync fault store");
auto failing_destination = take(
FilesystemObjectStore::open(environment("PGHOST"), dk,
max_cellular_population_bytes, &destination_faults),
"migration destination sync fault store");
source_faults.fail_directory_syncs = 1;
const auto unsynced_preparation = prepare_cellular_migration_freeze(
*f.db, *failing_source, dst, sl, sh, dh, request, 5s);
require(!unsynced_preparation &&
unsynced_preparation.error().kind == PublicationError::upload &&
unsynced_preparation.error().object == ObjectError::io &&
source_faults.fail_directory_syncs == 0 &&
!take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s),
"unsynced preparation pointer"),
"failed capsule sync durably prepared a migration");
take(source_host->prepare_migration_freeze(dh, dst, request),
"owning paired source freeze");
const auto frozen = take(take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s),
"paired source evidence"),
"missing paired freeze");
// Preparation alone cannot authorize either result.
require(!publish_cellular_migration(*f.db, so, dst, sl, dl, sh, dh, frozen.trace,
f.build, 5s),
"unstaged migration published");
destination_faults.fail_directory_syncs = 1;
const auto unsynced_stage = stage_cellular_migration(*f.db, so, *failing_destination,
dl, sh, dh, frozen.trace, 5s);
require(!unsynced_stage && unsynced_stage.error().kind == PublicationError::upload &&
unsynced_stage.error().object == ObjectError::io &&
destination_faults.fail_directory_syncs == 0 &&
!take(load_cellular_migration_stage(*f.db, dst, dk, dl, 5s),
"unsynced destination stage pointer") &&
take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s),
"source freeze after stage sync failure")
.has_value(),
"failed stage capsule sync activated a ghost or released source ownership");
take(destination_host->stage_migration(sh, so, frozen.trace), "owning paired stage");
for (unsigned endpoint = 0; endpoint < 2; ++endpoint) {
auto& faults = endpoint == 0 ? source_faults : destination_faults;
faults.fail_directory_syncs = 1;
const auto unsynced_pair =
publish_cellular_migration(*f.db, *failing_source, *failing_destination, sl, dl,
sh, dh, frozen.trace, f.build, 5s);
require(
!unsynced_pair && unsynced_pair.error().kind == PublicationError::upload &&
unsynced_pair.error().object == ObjectError::io &&
faults.fail_directory_syncs == 0 &&
take(load_region_checkpoint(*f.db, sk, 5s), "unsynced source head").head ==
source_before.head &&
take(load_region_checkpoint(*f.db, dk, 5s), "unsynced destination head")
.head == destination_before.head &&
take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s),
"unsynced source freeze")
.has_value() &&
take(load_cellular_migration_stage(*f.db, dst, dk, dl, 5s),
"unsynced destination stage")
.has_value(),
"failed result sync published a half or released retained work");
}
std::vector<std::uint8_t> expected_source, expected_destination;
if (representation == 0) {
auto a = take(restore_cellular_payload(sb, s), "paired raw source oracle");
auto b = take(restore_cellular_payload(dbb, d), "paired raw destination oracle");
auto pair = take(replay_migration_trace(*a, *b, request, frozen.trace),
"paired raw oracle");
expected_source = pair.source->snapshot();
expected_destination = pair.destination->snapshot();
} else {
auto a = take(restore_population_payload(sb, s), "paired population source oracle");
auto b = take(restore_population_payload(dbb, d),
"paired population destination oracle");
auto pair = take(replay_migration_trace(*a, *b, request, frozen.trace),
"paired population oracle");
expected_source = pair.source->snapshot();
expected_destination = pair.destination->snapshot();
}
// Fail specifically on the SECOND result insert, after the first and receipt.
take(f.owner->query(
"CREATE FUNCTION evolution.test_refuse_second_migration() RETURNS trigger "
"LANGUAGE plpgsql AS $$ BEGIN IF NEW.commit_id='" +
hexadecimal(request.destination.commit, false) +
"'::uuid THEN RAISE EXCEPTION 'injected second insert' USING "
"ERRCODE='EVL06'; END IF; RETURN NEW; END $$",
{}, 5s),
"second insert fault function");
take(f.owner->query(
"CREATE TRIGGER test_refuse_second_migration BEFORE INSERT ON "
"evolution.region_commits "
"FOR EACH ROW EXECUTE FUNCTION evolution.test_refuse_second_migration()",
{}, 5s),
"second insert fault trigger");
const auto rejected = publish_cellular_migration(*f.db, so, dst, sl, dl, sh, dh,
frozen.trace, f.build, 5s);
take(f.owner->query(
"DROP TRIGGER test_refuse_second_migration ON evolution.region_commits", {},
5s),
"remove second insert fault");
take(f.owner->query("DROP FUNCTION evolution.test_refuse_second_migration()", {}, 5s),
"remove fault function");
require(
!rejected && rejected.error().store.kind == LeaseStoreError::unsupported &&
take(load_region_checkpoint(*f.db, sk, 5s), "rollback source head").head ==
source_before.head &&
take(load_region_checkpoint(*f.db, dk, 5s), "rollback destination head").head ==
destination_before.head &&
take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s),
"rollback source pause")
.has_value() &&
take(load_cellular_migration_stage(*f.db, dst, dk, dl, 5s),
"rollback destination pause")
.has_value(),
"failed second publication left half a migration or released a pause");
const auto prepared = rejected.error().prepared;
take(f.db->query("BEGIN", {}, 5s), "false paired biology transaction");
auto false_pair = prepared;
auto false_source =
take(decode_manifest(false_pair.source_manifest), "false source manifest");
false_source.payload_sha256 = s.payload_sha256;
false_source.payload_bytes = s.payload_bytes;
false_pair.source_manifest =
take(encode_manifest(false_source), "false source canonical bytes");
take(retry_cellular_migration_commit(*f.db, sl, dl, false_pair, 5s),
"metadata-only false pair");
const auto false_recovery = verify_cellular_migration_checkpoint(
*f.db, so, sl, take(load_region_checkpoint(*f.db, sk, 5s), "false paired head"), sb,
5s);
const auto false_head =
take(load_region_checkpoint(*f.db, sk, 5s), "false history cut");
const auto false_history = [&]() -> std::expected<void, RecoveryFailure> {
if (representation == 0) {
auto recovered = recover_cellular_commit(
*f.db, so, sk, false_head.head.sequence, sl, builds, 5s);
if (!recovered)
return std::unexpected(recovered.error());
} else {
auto recovered = recover_population_commit(
*f.db, so, sk, false_head.head.sequence, sl, builds, 5s);
if (!recovered)
return std::unexpected(recovered.error());
}
return {};
}();
take(f.db->query("ROLLBACK", {}, 5s), "rollback false paired biology fixture");
require(!false_recovery && false_recovery.error().kind == RecoveryError::history &&
!false_history && false_history.error().kind == RecoveryError::history,
"paired recovery accepted a valid old object as the migrated biology");
const std::array<std::string, 1> generic{hexadecimal(prepared.source_manifest)};
require(!f.db->query("SELECT * FROM evolution.publish_region_checkpoint($1::bytea)",
generic, 5s),
"generic publisher admitted a migration half");
const auto child = ::fork();
require(child >= 0, "pre-transaction paired fork");
if (child == 0) {
try {
auto db = connection();
PublishHooks hooks{[] { ::kill(::getpid(), SIGKILL); }};
if (!publish_cellular_migration(*db, so, dst, sl, dl, sh, dh, frozen.trace,
f.build, 5s, hooks))
::_exit(52);
} catch (...) {
::_exit(51);
}
::_exit(52);
}
int status{};
require(
::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
WTERMSIG(status) == SIGKILL &&
take(load_region_checkpoint(*f.db, sk, 5s), "killed pair source").head ==
source_before.head &&
take(load_region_checkpoint(*f.db, dk, 5s), "killed pair destination").head ==
destination_before.head,
"pre-transaction paired death changed a head");
if (direction == 0 && std::getenv("EVO_MIGRATION_TCP_PORT"))
migration_transport_commit(f, prepared, sk, dk, sl, dl, source_before.head,
destination_before.head);
if (direction == 0) {
auto locker = connection(true), timed = connection();
take(locker->query("BEGIN", {}, 5s), "pair ambiguity transaction");
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND "
"region_id=" +
std::to_string(sk.region) + " FOR UPDATE",
{}, 5s),
"hold paired reply");
const auto unknown =
retry_cellular_migration_commit(*timed, sl, dl, prepared, 300ms);
require(!unknown && unknown.error().kind == PublicationError::unknown &&
unknown.error().prepared.source_manifest == prepared.source_manifest &&
unknown.error().prepared.destination_manifest ==
prepared.destination_manifest,
"ambiguous pair lost its exact manifests");
take(locker->query("ROLLBACK", {}, 5s), "release paired ambiguity");
} else {
const auto committed_child = ::fork();
require(committed_child >= 0, "post-commit paired fork");
if (committed_child == 0) {
try {
auto db = connection();
if (!retry_cellular_migration_commit(*db, sl, dl, prepared, 5s))
::_exit(53);
::kill(::getpid(), SIGKILL);
} catch (...) {
::_exit(54);
}
::_exit(55);
}
require(::waitpid(committed_child, &status, 0) == committed_child &&
WIFSIGNALED(status) && WTERMSIG(status) == SIGKILL,
"paired publisher failed before post-commit kill");
}
const auto committed =
take(retry_cellular_migration_commit(*f.db, sl, dl, prepared, 5s),
"resolve exact paired commit");
sh = committed.source;
dh = committed.destination;
require(!abort_cellular_migration_freeze(*f.db, sk, sl, request.request, 5s),
"committed migration was aborted");
auto changed = prepared;
auto altered =
take(decode_manifest(changed.destination_manifest), "conflicting committed pair");
altered.payload_sha256.back() ^= 1;
changed.destination_manifest = take(encode_manifest(altered), "conflicting pair bytes");
const auto conflict = retry_cellular_migration_commit(*f.db, sl, dl, changed, 5s);
auto stale = sl;
--stale.epoch;
require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict &&
!retry_cellular_migration_commit(*f.db, stale, dl, prepared, 5s),
"changed pair or stale source authority was accepted");
const auto sm = take(decode_manifest(sh.manifest), "unavailable source result");
const auto dm = take(decode_manifest(dh.manifest), "unavailable destination result");
// Outbound: deny current results. Return: deny retained counterpart
// capsules. Both are required before either paused host may resume.
const VerifiedObject unavailable_source =
direction == 0 ? VerifiedObject{sm.payload_sha256, sm.payload_bytes}
: VerifiedObject{d.payload_sha256, d.payload_bytes};
const VerifiedObject unavailable_destination =
direction == 0 ? VerifiedObject{dm.payload_sha256, dm.payload_bytes}
: VerifiedObject{s.payload_sha256, s.payload_bytes};
{
UnreadableObject source_denial(so.directory() /
hexadecimal(unavailable_source.sha256, false));
UnreadableObject destination_denial(
dst.directory() / hexadecimal(unavailable_destination.sha256, false));
const auto denied_source = so.get(unavailable_source);
const auto denied_destination = dst.get(unavailable_destination);
require(!denied_source && denied_source.error() == ObjectError::io &&
!denied_destination && denied_destination.error() == ObjectError::io,
"object read denial did not produce actual I/O failures");
const auto source_failures = source_host->status().store_failures;
const auto destination_failures = destination_host->status().store_failures;
const auto observed = [&] {
if (direction == 0)
return source_host->status().store_failures > source_failures &&
destination_host->status().store_failures > destination_failures;
return source_host->status().phase == HostPhase::quarantined &&
destination_host->status().phase == HostPhase::quarantined;
};
for (unsigned attempt = 0; attempt < 200 && !observed(); ++attempt) {
source_host->step();
destination_host->step();
std::this_thread::sleep_for(10ms);
}
require(
observed() && !source_host->status().ready &&
!destination_host->status().ready && !source_host->advance_unattended(1) &&
!destination_host->advance_unattended(1) &&
take(load_region_checkpoint(*f.db, sk, 5s), "unavailable source head")
.head == sh.head &&
take(load_region_checkpoint(*f.db, dk, 5s), "unavailable destination head")
.head == dh.head,
"unreadable migration evidence resumed old ownership or changed a head");
}
if (direction != 0) {
require(!source_host->step().ready && !destination_host->step().ready,
"restoring storage silently cleared quarantine");
source_host->drain();
destination_host->drain();
// The quarantined local authorities cannot release themselves. This
// test coordinator explicitly retires their still-current DB leases.
take(release_region_lease(*f.db, sk, sl, 5s), "retire quarantined source lease");
take(release_region_lease(*f.db, dk, dl, 5s),
"retire quarantined destination lease");
source_host = make_host(sk);
destination_host = make_host(dk);
}
for (unsigned attempt = 0;
attempt < 200 && (!source_host->step().ready || !destination_host->step().ready);
++attempt)
std::this_thread::sleep_for(25ms);
require(source_host->status().ready && destination_host->status().ready &&
take(source_host->committed(), "adopted source").head == sh.head &&
take(destination_host->committed(), "adopted destination").head == dh.head,
"pointer release resumed a stale owner instead of recovering the pair");
const auto old_source_epoch = sl.epoch, old_destination_epoch = dl.epoch;
sl = take(source_host->authority(), "restored source object authority");
dl = take(destination_host->authority(), "restored destination object authority");
const bool expected_epochs =
direction == 0 ? sl.epoch == old_source_epoch && dl.epoch == old_destination_epoch
: sl.epoch > old_source_epoch && dl.epoch > old_destination_epoch;
require(expected_epochs &&
take(retry_cellular_migration_commit(*f.db, sl, dl, prepared, 5s),
"restored storage exact paired receipt")
.source.head == sh.head,
"storage recovery changed a committed pair or reused authority");
const auto check_bytes = [&](const CommittedCheckpoint& h, RegionKey key,
const FilesystemObjectStore& objects,
const LeaseGrant& lease) {
require(take(load_region_checkpoint(*f.db, key, 5s), "current paired recovery head")
.head == h.head,
"paired recovery selected a different head");
if (representation == 0)
return take(recover_cellular_checkpoint(*f.db, objects, key, lease, builds, 5s),
"recover raw paired result")
.state->snapshot();
return take(recover_population_checkpoint(*f.db, objects, key, lease, builds, 5s),
"recover cohort paired result")
.state->snapshot();
};
require(check_bytes(sh, sk, so, sl) == expected_source &&
check_bytes(dh, dk, dst, dl) == expected_destination,
"committed biology/member/material differs from the exact pair");
auto source_history =
take(load_cellular_census_history(*f.db, so, sk, sl, sh.head, sh.head.sequence, 1,
builds, 5s),
"paired source history");
auto destination_history =
take(load_cellular_census_history(*f.db, dst, dk, dl, dh.head, dh.head.sequence, 1,
builds, 5s),
"paired destination history");
const auto census_oracle = [&](const auto& bytes,
const CommittedCheckpoint& checkpoint) {
const auto manifest =
take(decode_manifest(checkpoint.manifest), "history pair oracle manifest");
if (representation == 0)
return take(
CellularCensus::observe(*take(restore_cellular_payload(bytes, manifest),
"history raw pair oracle")),
"history raw census oracle");
return take(
CellularCensus::observe(*take(restore_population_payload(bytes, manifest),
"history cohort pair oracle")),
"history cohort census oracle");
};
require(source_history.entries.size() == 1 && destination_history.entries.size() == 1 &&
std::ranges::equal(source_history.entries[0].census.rows(),
census_oracle(expected_source, sh).rows()) &&
std::ranges::equal(destination_history.entries[0].census.rows(),
census_oracle(expected_destination, dh).rows()),
"historical migration census differs from exact paired biology");
const auto& source_census = source_history.entries[0].census;
const auto& destination_census = destination_history.entries[0].census;
const auto& boundary = source_census.boundary();
std::array required_regions{sk.region, dk.region};
std::ranges::sort(required_regions);
const auto census_set =
take(CellularCensusSet::observe(
{boundary.world, boundary.frame, boundary.tick, boundary.owner_format,
boundary.rules, boundary.quantum_microseconds},
required_regions, std::array{source_census, destination_census}),
"committed paired history census set");
// Identity and complete per-region rows survive real raw/cohort
// publication, replay and repeated direction changes.
require(census_set.regions().size() == 2 &&
census_set.regions()[0].boundary().region == required_regions[0] &&
census_set.regions()[1].boundary().region == required_regions[1],
"paired history query lost declared region coverage");
for (const auto& row : destination_census.rows()) {
const auto combined =
take(census_set.query(row.genome, CellularCensusScope::descendants),
"committed cross-region lineage query");
const auto local =
take(destination_census.query(row.genome, CellularCensusScope::descendants),
"committed destination lineage");
const auto other =
source_census.query(row.genome, CellularCensusScope::descendants);
for (std::size_t kind = 0; kind < local.resident.size(); ++kind)
require(combined.resident[kind].count ==
local.resident[kind].count +
(other ? other->resident[kind].count : 0),
"committed set duplicated/dropped resident lineage stock");
}
if (direction == 0) {
outgoing_history = std::move(source_history.entries.front());
incoming_history = std::move(destination_history.entries.front());
}
const auto capsule_path = so.directory() / hexadecimal(d.payload_sha256, false);
const auto backup = capsule_path.string() + ".paired-test";
std::filesystem::rename(capsule_path, backup);
const auto missing =
verify_cellular_migration_checkpoint(*f.db, so, sl, sh, expected_source, 5s);
const auto missing_history = load_cellular_census_history(
*f.db, so, sk, sl, sh.head, sh.head.sequence, 1, builds, 5s);
std::filesystem::rename(backup, capsule_path);
require(!missing && missing.error().kind == RecoveryError::tail && !missing_history &&
missing_history.error().recovery.kind == RecoveryError::tail &&
!verify_cellular_migration_checkpoint(*f.db, so, sl, sh, sb, 5s),
"pair recovery ignored missing counterpart evidence or false local biology");
source_host->drain();
destination_host->drain();
source_host = make_host(sk);
destination_host = make_host(dk);
require(source_host->step().ready && destination_host->step().ready,
"replacement paired hosts failed complete recovery");
sl = take(source_host->authority(), "replacement source fence");
dl = take(destination_host->authority(), "replacement destination fence");
require(take(retry_cellular_migration_commit(*f.db, sl, dl, prepared, 5s),
"new epoch exact paired retry")
.source.head == sh.head,
"fresh authority changed a committed retry");
source_host->drain();
destination_host->drain();
sl = take(acquire_region_lease(*f.db, sk, f.build, 60000ms, 5s),
"return paired source authority");
dl = take(acquire_region_lease(*f.db, dk, f.build, 60000ms, 5s),
"return paired destination authority");
}
const auto outgoing = take(std::move(outgoing_history), "retained source historical cut");
const auto incoming =
take(std::move(incoming_history), "retained destination historical cut");
const auto old_source =
take(load_cellular_census_history(*f.db, *f.objects, f.key, f.lease, f.head.head,
outgoing.committed.head.sequence, 1, builds, 5s),
"source history after return");
const auto old_destination =
take(load_cellular_census_history(*f.db, destination_objects, destination_key,
destination_lease, destination_head.head,
incoming.committed.head.sequence, 1, builds, 5s),
"destination history after return");
require(
old_source.entries[0].committed.head == outgoing.committed.head &&
old_destination.entries[0].committed.head == incoming.committed.head &&
take(decode_manifest(outgoing.committed.manifest), "historical source writer")
.epoch < f.lease.epoch &&
take(decode_manifest(incoming.committed.manifest), "historical destination writer")
.epoch < destination_lease.epoch &&
std::ranges::equal(old_source.entries[0].census.rows(), outgoing.census.rows()) &&
std::ranges::equal(old_destination.entries[0].census.rows(),
incoming.census.rows()),
"return migration rewrote earlier committed ownership observations");
migration_owner_partitions(f, destination_key, destination_objects, destination_lease,
destination_head, genome);
migration_snapshot_race(f, destination_key, destination_objects, destination_lease,
destination_head, genome);
}
std::puts("migration object sync/read failures, quarantine and exact recovery passed");
// Race the actual two-head transaction against abort on independent connections.
const auto prepare_pair = [&](bool reverse) {
auto& sh = reverse ? destination_head : f.head;
auto& dh = reverse ? f.head : destination_head;
auto& so = reverse ? destination_objects : *f.objects;
auto& dst = reverse ? *f.objects : destination_objects;
auto& sl = reverse ? destination_lease : f.lease;
auto& dl = reverse ? f.lease : destination_lease;
const auto sk = reverse ? destination_key : f.key;
const auto dk = reverse ? f.key : destination_key;
CellularMigrationRequest request{f.id(),
{sk.region, sh.head.commit, f.id(), sh.head.digest},
{dk.region, dh.head.commit, f.id(), dh.head.digest},
genome};
const auto freeze =
take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s),
"commit/abort source preparation");
take(stage_cellular_migration(*f.db, so, dst, dl, sh, dh, freeze.trace, 5s),
"commit/abort destination stage");
return freeze.trace;
};
const auto aborted_trace = prepare_pair(false);
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, aborted_trace.request.request, 5s),
"abort wins before paired commit");
const auto after_abort = publish_cellular_migration(
*f.db, *f.objects, destination_objects, f.lease, destination_lease, f.head,
destination_head, aborted_trace, f.build, 5s);
require(!after_abort && after_abort.error().store.kind == LeaseStoreError::commit_conflict,
"paired publication reopened an aborted workflow");
for (unsigned race = 0; race < 4; ++race) {
const auto trace = prepare_pair(false);
auto committing = connection(), aborting = connection();
std::barrier start(2);
std::optional<std::expected<CellularMigrationCommitted, CellularMigrationCommitFailure>>
published;
std::optional<std::expected<CellularMigrationFreeze, LeaseStoreFailure>> aborted;
std::thread commit_thread([&] {
start.arrive_and_wait();
published = publish_cellular_migration(*committing, *f.objects, destination_objects,
f.lease, destination_lease, f.head,
destination_head, trace, f.build, 5s);
});
std::thread abort_thread([&] {
start.arrive_and_wait();
aborted = abort_cellular_migration_freeze(*aborting, f.key, f.lease,
trace.request.request, 5s);
});
commit_thread.join();
abort_thread.join();
require(published.has_value() && aborted.has_value() &&
(published->has_value() != aborted->has_value()),
"commit/abort race had two outcomes or no definitive outcome");
if (published->has_value()) {
require(aborted->error().kind == LeaseStoreError::commit_conflict,
"losing abort was not refused");
f.head = (*published)->source;
destination_head = (*published)->destination;
const auto returning = prepare_pair(true);
const auto returned =
take(publish_cellular_migration(*f.db, destination_objects, *f.objects,
destination_lease, f.lease, destination_head,
f.head, returning, f.build, 5s),
"return race winner's stock");
destination_head = returned.source;
f.head = returned.destination;
} else {
require(published->error().store.kind == LeaseStoreError::commit_conflict,
"losing paired publication was not refused");
}
require(
take(load_region_checkpoint(*f.db, f.key, 5s), "race source head").head ==
f.head.head &&
take(load_region_checkpoint(*f.db, destination_key, 5s), "race destination head")
.head == destination_head.head &&
!take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s),
"race source settled")
.has_value() &&
!take(load_cellular_migration_stage(*f.db, destination_objects, destination_key,
destination_lease, 5s),
"race destination settled")
.has_value(),
"commit/abort race left half a head or a pending pause");
}
recover(f);
f.raw = restore(f.population->state());
}
void migration_freezes(Fixture& f) {
const RegionKey destination_key{f.key.world, 2};
auto destination_objects =
take(FilesystemObjectStore::open(environment("PGHOST"), destination_key,
max_cellular_population_bytes),
"migration destination objects");
auto destination = opening(30001, true, 4, false, 2);
auto destination_head =
take(provision_cellular_region(*f.owner, *destination_objects, destination_key, 1,
*destination, f.id(), f.build, 5s),
"migration empty destination genesis");
auto destination_lease =
take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s),
"migration destination authority");
const auto unequal_cut = load_cellular_census_snapshot(
*f.db, f.key.world,
std::array{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()},
CellularCensusRegionRequest{destination_key.region, destination_lease,
destination_objects.get()}},
std::array<Sha256Digest, 1>{f.build}, 5s);
require(!unequal_cut && unequal_cut.error().kind == CellularSnapshotError::census &&
unequal_cut.error().census == CellularCensusSetError::boundary &&
f.db->transaction_idle(),
"snapshot silently reconciled different current ticks/owner versions");
while (destination->tick() < f.raw->tick()) {
const auto count = std::min(std::uint64_t{64}, f.raw->tick() - destination->tick());
CellularBatch inputs{{f.key.world, 2, 1, destination_head.head.commit,
destination_head.head.digest, destination->tick(),
destination->tick() + count},
{}};
inputs.ticks.resize(static_cast<std::size_t>(count));
destination =
take(replay_cellular_batch(*destination, inputs), "empty destination biology");
destination_head = take(publish_cellular_batch(*f.db, *destination_objects,
destination_lease, destination_head, inputs,
*destination, f.id(), f.build, 5s),
"empty destination tick publication")
.committed;
}
destination_head =
take(migrate_cellular_boundary_profile(*f.db, *destination_objects, destination_lease,
destination_head, f.id(), f.build, 5s),
"destination accounting upgrade");
destination = take(destination->migration_upgrade_candidate(), "destination upgraded oracle");
const std::array<Sha256Digest, 1> builds{f.build};
const auto request_for = [&] {
const auto member = std::ranges::find_if(
f.raw->lives(), [](const auto& life) { return life.account == LifeAccount{}; });
require(member != f.raw->lives().end(), "freeze fixture lacks real wildlife");
return CellularMigrationRequest{
f.id(),
{1, f.head.head.commit, f.id(), f.head.head.digest},
{2, destination_head.head.commit, f.id(), destination_head.head.digest},
member->identity.genome};
};
for (unsigned representation = 0; representation < 2; ++representation) {
if (representation == 1) {
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
f.build, {true, 1200, 1800}, 5s),
"freeze cohort source entry");
recover(f);
destination_head = take(migrate_cellular_to_population(
*f.db, *destination_objects, destination_lease,
destination_head, f.id(), f.build, {true, 1200, 1800}, 5s),
"freeze cohort destination entry");
}
const auto source_before = f.head;
const auto source_bytes = f.raw->snapshot();
const auto destination_before = destination_head;
const auto request = request_for();
const auto prepare = [&](const CellularMigrationRequest& requested,
std::chrono::milliseconds timeout = 5s) {
return prepare_cellular_migration_freeze(*f.db, *f.objects, *destination_objects,
f.lease, f.head, destination_head, requested,
timeout);
};
if (representation == 0) {
Uuid account{};
account.back() = 1;
take(f.db->query("BEGIN", {}, 5s), "pending player choice before freeze");
const auto control =
take(claim_account_control(*f.db, f.key, f.lease, account, 2, 9900, 60000ms, 5s),
"pending choice current control");
const auto input = encode_cellular_mutation_input(
{account, 9900, f.head.head.end_tick + 1, {100, 1, 1, 1}});
take(reserve_cellular_mutation(*f.db, f.key, f.lease, account, control.generation, 9900,
input, 5s),
"reserve pre-freeze player choice");
const auto pending = prepare(request);
take(f.db->query("ROLLBACK", {}, 5s), "rollback pre-freeze player choice");
require(!pending &&
std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18",
"preparation stranded a player choice");
} else {
take(f.db->query("BEGIN", {}, 5s), "prepare with admitted work transaction");
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head,
f.head.head.end_tick + 20, 5s),
"admitted work before freeze");
const auto pending = prepare(request);
take(f.db->query("ROLLBACK", {}, 5s), "rollback pending-work preparation fixture");
require(!pending &&
std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18",
"preparation skipped durable ecological work");
}
auto frozen = take(prepare(request), "durable source freeze");
const CellularMigrationPrepared prepared{
f.key, take(encode_migration_trace(frozen.trace), "frozen canonical trace")};
auto unsupported = frozen.trace;
unsupported.format_version = 2;
const auto unsupported_bytes =
take(encode_migration_trace(unsupported), "portable rules-9 trace");
const auto refused_freeze =
retry_cellular_migration_freeze(*f.db, f.lease, {f.key, unsupported_bytes}, 5s);
const auto refused_stage = retry_cellular_migration_stage(
*f.db, destination_lease, {destination_key, unsupported_bytes, {f.build, 1}}, 5s);
require(!refused_freeze && refused_freeze.error().kind == PublicationError::candidate &&
!refused_stage && refused_stage.error().kind == PublicationError::candidate &&
!stage_cellular_migration(*f.db, *f.objects, *destination_objects,
destination_lease, f.head, destination_head,
unsupported, 5s),
"portable rules-9 trace expanded durable migration admission");
const std::array<std::string, 3> unsupported_args{hexadecimal(f.lease.holder, false),
std::to_string(f.lease.epoch),
hexadecimal(unsupported_bytes)};
const auto refused_sql =
f.db->query("SELECT * FROM evolution.prepare_cellular_migration_freeze(30201,"
"1,$1::uuid,$2::evolution.u64,$3::bytea)",
unsupported_args, 5s);
require(!refused_sql && std::string(refused_sql.error().sqlstate.data(), 5) == "22023",
"SQL admitted portable trace 2 before duplicate lookup");
auto malformed = prepared.trace;
malformed.back() ^= 1;
const std::array<std::string, 3> malformed_args{hexadecimal(f.lease.holder, false),
std::to_string(f.lease.epoch),
hexadecimal(malformed)};
const auto malformed_sql = f.db->query("SELECT * FROM "
"evolution.prepare_cellular_migration_freeze(30201,"
"1,$1::uuid,$2::evolution.u64,$3::bytea)",
malformed_args, 5s);
require(!malformed_sql && std::string(malformed_sql.error().sqlstate.data(), 5) == "22023",
"SQL accepted an invalid trace checksum before duplicate lookup");
require(
!frozen.aborted && frozen.trace.request == request &&
take(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s),
"replay frozen capsule"),
"missing durable freeze")
.trace == frozen.trace &&
take(load_region_checkpoint(*f.db, f.key, 5s), "frozen source head").head ==
source_before.head &&
take(load_region_checkpoint(*f.db, destination_key, 5s), "unowned destination head")
.head == destination_before.head,
"preparation changed either owner or lost its full replay capsule");
require(
!take(retry_cellular_migration_freeze(*f.db, f.lease, prepared, 5s), "duplicate freeze")
.aborted,
"duplicate freeze lost its state");
const auto stage = [&](const CellularMigrationTrace& trace,
std::chrono::milliseconds timeout = 5s) {
return stage_cellular_migration(*f.db, *f.objects, *destination_objects,
destination_lease, f.head, destination_head, trace,
timeout);
};
// The optimistic destination cut must still be exact at first staging.
take(f.db->query("BEGIN", {}, 5s), "move destination before stage transaction");
if (representation == 0)
take(migrate_cellular_to_population(*f.db, *destination_objects, destination_lease,
destination_head, f.id(), f.build, {}, 5s),
"temporarily move destination profile");
else
take(materialize_population_profile(*f.db, *destination_objects, destination_lease,
destination_head, f.id(), f.build, 5s),
"temporarily move population destination");
const auto moved = stage(frozen.trace);
take(f.db->query("ROLLBACK", {}, 5s), "rollback moved destination fixture");
require(!moved && moved.error().store.kind == LeaseStoreError::head_mismatch,
"stage silently rebased a moved destination");
if (representation == 1) {
take(f.db->query("BEGIN", {}, 5s), "stage pending interval transaction");
take(admit_population_interval(*f.db, destination_key, destination_lease,
destination_head.head,
destination_head.head.end_tick + 20, 5s),
"destination admitted work before staging");
const auto pending = stage(frozen.trace);
take(f.db->query("ROLLBACK", {}, 5s), "rollback destination interval fixture");
require(!pending && pending.error().store.kind == LeaseStoreError::commit_conflict,
"stage stranded destination interval work");
}
const auto staged = take(stage(frozen.trace), "durable destination staging");
const CellularMigrationStagePrepared stage_request{destination_key, prepared.trace,
staged.ghost};
require(
!staged.freeze.aborted &&
take(take(load_cellular_migration_stage(*f.db, *destination_objects,
destination_key, destination_lease, 5s),
"recover destination ghost"),
"missing staged destination")
.ghost == staged.ghost &&
take(load_region_checkpoint(*f.db, destination_key, 5s),
"staged destination still owns original head")
.head == destination_before.head &&
take(retry_cellular_migration_stage(*f.db, destination_lease, stage_request, 5s),
"duplicate stage")
.ghost == staged.ghost,
"staging activated a ghost or lost exact replay");
auto changed_stage = stage_request;
++changed_stage.ghost.bytes;
const auto changed =
retry_cellular_migration_stage(*f.db, destination_lease, changed_stage, 5s);
require(!changed && changed.error().store.kind == LeaseStoreError::commit_conflict,
"stage accepted conflicting immutable ghost bytes");
auto reverse = frozen.trace;
reverse.request.request = f.id();
std::swap(reverse.request.source, reverse.request.destination);
const auto overlapping = retry_cellular_migration_freeze(
*f.db, destination_lease,
{destination_key, take(encode_migration_trace(reverse), "overlapping source trace")},
5s);
require(!overlapping && overlapping.error().store.kind == LeaseStoreError::commit_conflict,
"staged destination simultaneously prepared an outgoing workflow");
take(f.db->query("BEGIN", {}, 5s), "staged destination life reservation transaction");
Uuid new_account{};
new_account.back() = 5;
const auto staged_control =
take(claim_account_control(*f.db, destination_key, destination_lease, new_account, 1,
9950, 60000ms, 5s),
"staged destination reservation current control");
const auto staged_life =
reserve_primitive_life(*f.db, destination_key, destination_lease, new_account,
staged_control.generation, 1, 1, 5s);
take(f.db->query("ROLLBACK", {}, 5s), "rollback staged life reservation fixture");
require(!staged_life &&
std::string(staged_life.error().cause.sqlstate.data(), 5) == "EVL18",
"staged destination admitted a new player life reservation");
if (representation == 0) {
CellularBatch inputs{{f.key.world, 2, 1, destination_head.head.commit,
destination_head.head.digest, destination->tick(),
destination->tick() + 1},
{{}}};
const auto advanced = take(replay_cellular_batch(*destination, inputs),
"detached staged destination future");
const auto denied =
publish_cellular_batch(*f.db, *destination_objects, destination_lease,
destination_head, inputs, *advanced, f.id(), f.build, 5s);
require(!denied &&
std::string(denied.error().store.cause.sqlstate.data(), 5) == "EVL18",
"staged destination published an ordinary future");
} else {
require(!admit_population_interval(*f.db, destination_key, destination_lease,
destination_head.head,
destination_head.head.end_tick + 20, 5s) &&
!materialize_population_profile(*f.db, *destination_objects,
destination_lease, destination_head, f.id(),
f.build, 5s),
"staged destination progressed or changed profile");
}
const auto ghost_path =
destination_objects->directory() / hexadecimal(staged.ghost.sha256, false);
const auto ghost_backup = ghost_path.string() + ".stage-test";
std::filesystem::rename(ghost_path, ghost_backup);
const auto missing_ghost = load_cellular_migration_stage(
*f.db, *destination_objects, destination_key, destination_lease, 5s);
std::filesystem::rename(ghost_backup, ghost_path);
require(!missing_ghost && missing_ghost.error().kind == RecoveryError::tail,
"missing ghost fell back to the active destination");
char ghost_first{};
{
std::fstream file(ghost_path, std::ios::binary | std::ios::in | std::ios::out);
file.get(ghost_first);
file.seekp(0);
file.put(static_cast<char>(static_cast<unsigned char>(ghost_first) ^ 1U));
file.flush();
require(file.good(), "same-length ghost corruption fixture");
}
const auto corrupt_ghost = load_cellular_migration_stage(
*f.db, *destination_objects, destination_key, destination_lease, 5s);
{
std::fstream file(ghost_path, std::ios::binary | std::ios::in | std::ios::out);
file.put(ghost_first);
file.flush();
require(file.good(), "restore corrupt ghost fixture");
}
require(!corrupt_ghost && corrupt_ghost.error().kind == RecoveryError::tail,
"same-length corrupt ghost bypassed digest verification");
const auto source_manifest = take(decode_manifest(f.head.manifest), "stage source capsule");
const auto source_capsule_path =
destination_objects->directory() / hexadecimal(source_manifest.payload_sha256, false);
const auto source_capsule_backup = source_capsule_path.string() + ".stage-test";
std::filesystem::rename(source_capsule_path, source_capsule_backup);
const auto missing_source = load_cellular_migration_stage(
*f.db, *destination_objects, destination_key, destination_lease, 5s);
std::filesystem::rename(source_capsule_backup, source_capsule_path);
require(!missing_source && missing_source.error().kind == RecoveryError::tail,
"destination recovery used the live source instead of its retained capsule");
const auto old_destination_lease = destination_lease;
take(release_region_lease(*f.db, destination_key, destination_lease, 5s),
"release staged destination owner");
destination_lease = take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s),
"replacement staged destination owner");
require(
take(load_cellular_migration_stage(*f.db, *destination_objects, destination_key,
destination_lease, 5s),
"new epoch ghost replay")
.has_value() &&
!retry_cellular_migration_stage(*f.db, old_destination_lease, stage_request, 5s),
"replacement activated the ghost or stale destination staged it");
auto conflict_trace = frozen.trace;
++conflict_trace.transferred.energy.value;
auto conflict = retry_cellular_migration_freeze(
*f.db, f.lease, {f.key, take(encode_migration_trace(conflict_trace), "conflict trace")},
5s);
require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict &&
!prepare(request_for()),
"freeze accepted changed bytes or replaced pending request");
if (representation == 0) {
const auto h = f.header(1);
const auto inputs = take(unattended_population_inputs(*f.raw, h), "frozen raw inputs");
auto advanced = take(replay_cellular_batch(*f.raw, inputs), "frozen detached future");
auto denied = publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, inputs,
*advanced, f.id(), f.build, 5s);
require(!denied &&
std::string(denied.error().store.cause.sqlstate.data(), 5) == "EVL18" &&
!migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
f.build, {}, 5s),
"raw source progressed or changed profile while frozen");
Uuid account{};
account.back() = 1;
const auto input = encode_cellular_mutation_input(
{account, 9901, f.head.head.end_tick + 1, {100, 1, 1, 1}});
take(f.db->query("BEGIN", {}, 5s), "current control while frozen fixture");
const auto control =
take(claim_account_control(*f.db, f.key, f.lease, account, 2, 9901, 60000ms, 5s),
"current frozen-source account control");
const auto choice = reserve_cellular_mutation(*f.db, f.key, f.lease, account,
control.generation, 9901, input, 5s);
take(f.db->query("ROLLBACK", {}, 5s), "rollback frozen control fixture");
require(!choice, "frozen raw owner admitted a new mutation reservation");
if (std::string(choice.error().cause.sqlstate.data(), 5) != "EVL18")
throw std::runtime_error("frozen mutation guard SQLSTATE=" +
std::string(choice.error().cause.sqlstate.data(), 5));
} else {
require(!admit_population_interval(*f.db, f.key, f.lease, f.head.head,
f.head.head.end_tick + 20, 5s) &&
!materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(),
f.build, 5s),
"population source admitted work or changed profile while frozen");
}
const auto capsule_manifest =
take(decode_manifest(destination_head.manifest), "capsule manifest");
const auto denied_upgrade_id = f.id();
const auto denied_upgrade = migrate_cellular_maturation_profile(
*f.db, *f.objects, f.lease, f.head, denied_upgrade_id, f.build, 5s);
require(
!denied_upgrade &&
std::string(denied_upgrade.error().store.cause.sqlstate.data(), 5) == "EVL18" &&
!take(load_cellular_profile_migration(*f.db, f.key, f.lease, denied_upgrade_id, 5s),
"frozen maturation journal")
.has_value() &&
take(load_region_checkpoint(*f.db, f.key, 5s), "frozen maturation head").head ==
f.head.head,
"maturation escaped a frozen handoff or retained partial upgrade evidence");
const auto capsule_path =
f.objects->directory() / hexadecimal(capsule_manifest.payload_sha256, false);
const auto backup = capsule_path.string() + ".freeze-test";
std::filesystem::rename(capsule_path, backup);
const auto missing = load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s);
std::filesystem::rename(backup, capsule_path);
require(!missing && missing.error().kind == RecoveryError::tail,
"freeze recovery fell back to destination live state without the retained capsule");
char first{};
{
std::fstream file(capsule_path, std::ios::binary | std::ios::in | std::ios::out);
file.get(first);
file.seekp(0);
file.put(static_cast<char>(static_cast<unsigned char>(first) ^ 1U));
file.flush();
require(file.good(), "corrupt same-length capsule fixture");
}
const auto corrupt = load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s);
{
std::fstream file(capsule_path, std::ios::binary | std::ios::in | std::ios::out);
file.put(first);
file.flush();
require(file.good(), "restore same-length capsule fixture");
}
require(!corrupt && corrupt.error().kind == RecoveryError::tail,
"corrupt capsule bypassed strong object binding");
const auto old_lease = f.lease;
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release frozen source owner");
f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s),
"new frozen source owner");
require(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s),
"recover freeze under new epoch")
.has_value() &&
!abort_cellular_migration_freeze(*f.db, f.key, old_lease, request.request, 5s),
"new epoch released a freeze or old epoch aborted it");
require(
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, request.request, 5s),
"explicit source abort")
.aborted &&
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, request.request, 5s),
"duplicate source abort")
.aborted &&
take(retry_cellular_migration_freeze(*f.db, f.lease, prepared, 5s),
"retry aborted preparation")
.aborted &&
!take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s),
"aborted pointer load")
.has_value(),
"abort was not durable or preparation retry reopened it");
require(
!take(load_cellular_migration_stage(*f.db, *destination_objects, destination_key,
destination_lease, 5s),
"aborted destination pointer")
.has_value() &&
take(retry_cellular_migration_stage(*f.db, destination_lease, stage_request, 5s),
"retry aborted stage")
.freeze.aborted,
"source abort stranded the destination or stage retry reopened its pointer");
require(take(load_region_checkpoint(*f.db, f.key, 5s), "aborted source head").head ==
source_before.head &&
f.raw->snapshot() == source_bytes,
"abort changed biological stock/time/history");
auto false_trace = frozen.trace;
false_trace.request.request = f.id();
false_trace.request.source.commit = f.id();
false_trace.request.destination.commit = f.id();
++false_trace.transferred.energy.value;
take(retry_cellular_migration_freeze(*f.db, f.lease,
{f.key, take(encode_migration_trace(false_trace),
"canonical false inventory trace")},
5s),
"metadata-only false preparation");
const auto false_replay =
load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s);
require(!false_replay && false_replay.error().kind == RecoveryError::history,
"freeze recovery trusted declared material without complete biological replay");
const auto refuse_false_inventory = [&] {
const std::array resources{
CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()},
CellularCensusRegionRequest{destination_key.region, destination_lease,
destination_objects.get()}};
const auto result =
load_registered_cellular_census_snapshot(*f.db, f.key.world, resources, builds, 5s);
require(!result && result.error().kind == CellularSnapshotError::migration &&
result.error().migration.kind == RecoveryError::history &&
f.db->transaction_idle(),
"registered census accepted semantically false pending biology");
};
refuse_false_inventory();
take(
abort_cellular_migration_freeze(*f.db, f.key, f.lease, false_trace.request.request, 5s),
"explicitly abort false preparation without moving stock");
// A genuine unknown reply retains exactly the chosen cut and group.
auto locker = connection(true);
take(locker->query("BEGIN", {}, 5s), "freeze ambiguity lock");
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND "
"region_id=1 FOR UPDATE",
{}, 5s),
"hold freeze reply");
auto timed = connection();
auto unknown =
prepare_cellular_migration_freeze(*timed, *f.objects, *destination_objects, f.lease,
f.head, destination_head, request_for(), 300ms);
require(!unknown && unknown.error().kind == PublicationError::unknown &&
!unknown.error().prepared.trace.empty(),
"freeze ambiguity lost its immutable request");
take(locker->query("ROLLBACK", {}, 5s), "release freeze ambiguity");
frozen = take(retry_cellular_migration_freeze(*f.db, f.lease, unknown.error().prepared, 5s),
"reconcile unknown freeze");
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, frozen.trace.request.request,
5s),
"abort reconciled freeze");
const auto false_stage_freeze = take(prepare(request_for()), "false ghost preparation");
const auto original_destination =
take(decode_manifest(destination_head.manifest), "false ghost existing object");
const CellularMigrationStagePrepared false_ghost{
destination_key,
take(encode_migration_trace(false_stage_freeze.trace), "false ghost trace"),
{original_destination.payload_sha256, original_destination.payload_bytes}};
take(retry_cellular_migration_stage(*f.db, destination_lease, false_ghost, 5s),
"metadata-only false ghost stage");
const auto false_ghost_replay = load_cellular_migration_stage(
*f.db, *destination_objects, destination_key, destination_lease, 5s);
require(!false_ghost_replay && false_ghost_replay.error().kind == RecoveryError::history,
"stage recovery accepted a real object with the wrong complete biology");
refuse_false_inventory();
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
false_stage_freeze.trace.request.request, 5s),
"abort false ghost evidence");
const auto uncertain_stage_freeze =
take(prepare(request_for()), "unknown stage preparation");
take(locker->query("BEGIN", {}, 5s), "stage ambiguity lock");
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND "
"region_id=2 FOR UPDATE",
{}, 5s),
"hold stage reply");
auto stage_timed = connection();
const auto stage_unknown = stage_cellular_migration(
*stage_timed, *f.objects, *destination_objects, destination_lease, f.head,
destination_head, uncertain_stage_freeze.trace, 300ms);
require(!stage_unknown && stage_unknown.error().kind == PublicationError::unknown &&
!stage_unknown.error().prepared.trace.empty(),
"stage ambiguity lost exact bytes");
take(locker->query("ROLLBACK", {}, 5s), "release stage ambiguity");
take(retry_cellular_migration_stage(*f.db, destination_lease,
stage_unknown.error().prepared, 5s),
"resolve unknown stage");
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
uncertain_stage_freeze.trace.request.request, 5s),
"abort reconciled stage");
for (unsigned race = 0; race < 4; ++race) {
const auto race_freeze = take(prepare(request_for()), "stage race preparation");
auto stage_db = connection(), abort_db = connection();
std::barrier start(2);
std::optional<std::expected<CellularMigrationStage, CellularMigrationStageFailure>>
staged_race;
std::optional<std::expected<CellularMigrationFreeze, LeaseStoreFailure>> aborted_race;
std::thread staging([&] {
start.arrive_and_wait();
staged_race = stage_cellular_migration(*stage_db, *f.objects, *destination_objects,
destination_lease, f.head, destination_head,
race_freeze.trace, 5s);
});
std::thread aborting([&] {
start.arrive_and_wait();
aborted_race = abort_cellular_migration_freeze(
*abort_db, f.key, f.lease, race_freeze.trace.request.request, 5s);
});
staging.join();
aborting.join();
require(aborted_race.has_value() && aborted_race->has_value() &&
staged_race.has_value() &&
(staged_race->has_value() ||
staged_race->error().store.kind == LeaseStoreError::commit_conflict) &&
!take(load_cellular_migration_stage(*f.db, *destination_objects,
destination_key, destination_lease, 5s),
"stage race final pointer")
.has_value(),
"stage/abort race left an active ghost or stranded pause");
}
// Both sides of preparation's transaction boundary, with real process death.
for (unsigned point = 0; point < 2; ++point) {
const auto killed_request = request_for();
const auto child = ::fork();
require(child >= 0, "freeze publisher fork");
if (child == 0) {
try {
auto db = connection();
PublishHooks hooks;
if (point == 0)
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
if (!prepare_cellular_migration_freeze(*db, *f.objects, *destination_objects,
f.lease, f.head, destination_head,
killed_request, 5s, hooks))
::_exit(41);
::kill(::getpid(), SIGKILL);
} catch (...) {
::_exit(42);
}
::_exit(43);
}
int status{};
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
WTERMSIG(status) == SIGKILL &&
take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s),
"recover killed preparation")
.has_value() == (point == 1),
"publisher death changed source ownership freeze");
if (point == 1)
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, killed_request.request,
5s),
"abort killed preparation");
}
for (unsigned point = 0; point < 2; ++point) {
const auto killed = take(prepare(request_for()), "stage process-loss preparation");
const auto child = ::fork();
require(child >= 0, "stage publisher fork");
if (child == 0) {
try {
auto db = connection();
PublishHooks hooks;
if (point == 0)
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
if (!stage_cellular_migration(*db, *f.objects, *destination_objects,
destination_lease, f.head, destination_head,
killed.trace, 5s, hooks))
::_exit(44);
::kill(::getpid(), SIGKILL);
} catch (...) {
::_exit(45);
}
::_exit(46);
}
int status{};
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
WTERMSIG(status) == SIGKILL &&
take(load_cellular_migration_stage(*f.db, *destination_objects,
destination_key, destination_lease, 5s),
"recover killed stage")
.has_value() == (point == 1),
"stage process loss changed ownership or lost the durable pause");
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
killed.trace.request.request, 5s),
"abort killed stage");
}
}
// The owning host must retain the freeze through drain/recreation with entry off.
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture to frozen host");
const auto make_host = [&] {
auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), f.key,
max_cellular_population_bytes),
"freeze host objects");
CellularHostConfig config{f.key, 1, {}, f.build, {f.build}};
config.lease.commit_timeout = 300ms;
return take(
CellularRegionHost::create(
config,
[] { return std::expected<std::unique_ptr<Postgres>, PgFailure>{connection()}; },
std::move(objects)),
"freeze host");
};
auto host = make_host();
require(host->step().ready, "source host before freeze");
const auto before = host->state()->snapshot();
const auto host_request = request_for();
take(host->prepare_migration_freeze(destination_head, *destination_objects, host_request),
"host source freeze");
require(!host->status().ready && host->status().migration_frozen &&
!host->advance_unattended(64) && host->state()->snapshot() == before,
"frozen host advanced or originated current output");
host->drain();
host = make_host();
require(!host->step().ready && host->status().migration_frozen &&
host->state()->snapshot() == before && !host->authority().has_value(),
"replacement host implicitly aborted or ignored its freeze");
take(host->abort_migration_freeze(host_request.request), "replacement host explicit abort");
require(host->step().ready && !host->status().migration_frozen &&
host->state()->snapshot() == before,
"host failed to resume the identical source after confirmed abort");
auto locker = connection(true);
take(locker->query("BEGIN", {}, 5s), "host unknown freeze lock");
take(
locker->query(
"SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE",
{}, 5s),
"hold host freeze reply");
const auto uncertain_request = request_for();
const auto uncertain =
host->prepare_migration_freeze(destination_head, *destination_objects, uncertain_request);
require(!uncertain && !host->status().ready, "host originated during uncertain freeze");
take(locker->query("ROLLBACK", {}, 5s), "release host freeze reply");
for (unsigned attempt = 0; attempt < 200 && !host->status().migration_frozen; ++attempt) {
host->step();
std::this_thread::sleep_for(25ms);
}
require(host->status().migration_frozen && !host->status().ready &&
host->state()->snapshot() == before,
"host chose different biology after unknown preparation");
take(locker->query("BEGIN", {}, 5s), "host unknown abort lock");
take(
locker->query(
"SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE",
{}, 5s),
"hold host abort reply");
const auto unknown_abort = host->abort_migration_freeze(uncertain_request.request);
require(!unknown_abort && !host->status().ready, "unconfirmed abort resumed the source");
take(locker->query("ROLLBACK", {}, 5s), "release host abort reply");
for (unsigned attempt = 0; attempt < 200 && !host->step().ready; ++attempt)
std::this_thread::sleep_for(25ms);
require(host->status().ready && !host->status().migration_frozen &&
host->state()->snapshot() == before,
"host failed exact ambiguous-abort reconciliation");
host->drain();
f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s),
"return freeze fixture authority");
const auto destination_host_freeze =
take(prepare_cellular_migration_freeze(*f.db, *f.objects, *destination_objects, f.lease,
f.head, destination_head, request_for(), 5s),
"destination host source preparation");
take(release_region_lease(*f.db, destination_key, destination_lease, 5s),
"release destination to owning host");
const auto make_destination_host = [&] {
auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), destination_key,
max_cellular_population_bytes),
"destination host objects");
CellularHostConfig config{destination_key, 1, {}, f.build, {f.build}};
config.lease.commit_timeout = 300ms;
return take(
CellularRegionHost::create(
config,
[] { return std::expected<std::unique_ptr<Postgres>, PgFailure>{connection()}; },
std::move(objects)),
"destination staging host");
};
auto destination_host = make_destination_host();
require(destination_host->step().ready, "destination host before staging");
const auto destination_bytes = destination_host->state()->snapshot();
take(locker->query("BEGIN", {}, 5s), "host stage ambiguity transaction");
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND "
"region_id=1 FOR UPDATE",
{}, 5s),
"hold owning destination stage");
const auto host_stage_unknown =
destination_host->stage_migration(f.head, *f.objects, destination_host_freeze.trace);
require(!host_stage_unknown && !destination_host->status().ready &&
destination_host->state()->snapshot() == destination_bytes,
"owning destination advanced during ambiguous staging");
take(locker->query("ROLLBACK", {}, 5s), "release owning stage reply");
for (unsigned attempt = 0; attempt < 200 && !destination_host->status().migration_frozen;
++attempt) {
destination_host->step();
std::this_thread::sleep_for(25ms);
}
require(destination_host->status().migration_frozen && !destination_host->status().ready &&
!destination_host->advance_unattended(64),
"owning destination activated its ghost");
destination_host->drain();
destination_host = make_destination_host();
require(!destination_host->step().ready && destination_host->status().migration_frozen &&
destination_host->state()->snapshot() == destination_bytes &&
!destination_host->authority().has_value(),
"replacement destination lost staging or adopted the uncommitted ghost");
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
destination_host_freeze.trace.request.request, 5s),
"source-authorized abort releases destination host");
for (unsigned attempt = 0; attempt < 200 && !destination_host->step().ready; ++attempt)
std::this_thread::sleep_for(25ms);
require(destination_host->status().ready && !destination_host->status().migration_frozen &&
destination_host->state()->snapshot() == destination_bytes,
"destination failed to resume its unchanged original state after source abort");
destination_host->drain();
destination_lease = take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s),
"return staged destination fixture authority");
migration_publication(f, destination_key, *destination_objects, destination_lease,
destination_head);
f.head = take(
materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
"resume frozen population as raw owner");
f.raw = take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s),
"source after frozen profile materialization")
.state;
take(release_region_lease(*f.db, destination_key, destination_lease, 5s),
"release freeze destination");
}
void host_ownership_and_admission(Fixture& f) {
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture host authority");
const auto make_host = [&](bool enabled) {
auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), f.key,
max_cellular_population_bytes),
"host objects");
CellularHostConfig config{f.key, 1, {}, f.build, {f.build}};
config.lease.commit_timeout = 300ms;
if (enabled)
config.population_policy = CellularTierPolicy{true, 1200, 1800};
return take(CellularRegionHost::create(
std::move(config),
[] {
return Postgres::connect({environment("PGHOST"),
{},
environment("PGPORT"),
environment("PGDATABASE"),
environment("EVO_PG_RUNTIME_USER"),
{},
{}},
5s);
},
std::move(objects)),
"population host");
};
auto host = make_host(false);
require(host->step().ready && !host->population_mode(), "raw host recovery");
require(!host->set_population_mode(true), "default host enabled population entry");
host->drain();
host = make_host(true);
require(host->step().ready, "opt-in host recovery");
const auto before = host->state()->snapshot();
const auto tick = host->state()->tick();
take(host->set_population_mode(true), "host population entry");
require(host->population_mode() && host->state()->snapshot() == before,
"host entry changed biology");
auto population =
take(CellularPopulation::create(
take(CellularLives::restore(before, f.key.world, f.key.region, 1, tick),
"host reference restore"),
{true, 1200, 1800}),
"host reference population");
f.head = take(host->committed(), "host population head");
auto next = unattended(f); // uses the same deterministic frozen/wildlife input contract.
// The fixture's old envelope may have a different membership history; biology is shared.
auto expected =
take(prepare_population_batch(*population, next.batch.biology), "host reference batch");
require(!take(host->advance_unattended(19), "collect bounded publication window").has_value() &&
host->state()->snapshot() == before,
"small local debt was admitted prematurely");
const auto published = take(host->advance_unattended(65), "host population publication");
require(published.has_value(), "host skipped admitted interval");
require(host->population_mode() &&
host->state()->snapshot() == expected.state->state().snapshot(),
"host population continuation differs");
const auto committed = take(host->committed(), "host continued head");
const auto continued = host->state()->snapshot();
const CellularBatchHeader recovered_header{f.key.world,
f.key.region,
1,
committed.head.commit,
committed.head.digest,
committed.head.end_tick,
committed.head.end_tick + 64};
auto recovered_expected = take(
prepare_population_batch(
*expected.state, take(unattended_population_inputs(*host->state(), recovered_header),
"recovered interval inputs")),
"recovered interval reference");
take(admit_population_interval(*f.db, f.key, take(host->authority(), "host interval authority"),
committed.head, committed.head.end_tick + 64, 5s),
"admit before host loss");
host->drain();
host = make_host(false);
require(host->step().ready && host->population_mode() &&
take(host->committed(), "host recovered head").head == committed.head &&
host->state()->snapshot() == continued,
"restarted host failed to recover population head");
require(host->status().ecological_pending_ticks == 64 && !host->set_population_mode(false),
"restarted host hid or skipped admitted work");
require(take(host->advance_unattended(0), "resume admitted work with zero new due time")
.has_value() &&
host->state()->snapshot() == recovered_expected.state->state().snapshot() &&
host->status().ecological_pending_ticks == 0,
"recovered interval did not settle exactly");
const auto recovered_head = take(host->committed(), "recovered interval head");
const auto recovered_bytes = host->state()->snapshot();
const CellularBatchHeader unknown_header{f.key.world,
f.key.region,
1,
recovered_head.head.commit,
recovered_head.head.digest,
recovered_head.head.end_tick,
recovered_head.head.end_tick + 64};
auto unknown_expected = take(
prepare_population_batch(*recovered_expected.state,
take(unattended_population_inputs(*host->state(), unknown_header),
"uncertain admission inputs")),
"uncertain admission reference");
auto locker = connection(true);
take(locker->query("BEGIN", {}, 5s), "host admission lock transaction");
take(
locker->query(
"SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE",
{}, 5s),
"hold host admission reply");
require(!host->advance_unattended(64) && !host->status().ready &&
host->state()->snapshot() == recovered_bytes,
"uncertain admission allowed simulation/output");
take(locker->query("ROLLBACK", {}, 5s), "release host admission reply");
for (unsigned attempt = 0; attempt < 200 && !host->step().ready; ++attempt)
std::this_thread::sleep_for(25ms);
require(host->status().ready && host->status().ecological_pending_ticks == 64,
"unknown admission failed exact reconciliation");
require(take(host->advance_unattended(0), "finish reconciled admission").has_value() &&
host->state()->snapshot() == unknown_expected.state->state().snapshot(),
"admission retry chose different work");
const auto final_bytes = host->state()->snapshot();
const auto final_authority = take(host->authority(), "final host authority");
Uuid account{};
account.back() = 1;
// A returning player's operation completes the durable reverse transition before lookup.
[[maybe_unused]] const auto queried = host->query_primitive_life(account, 1, 1);
require(!host->population_mode() && host->state()->snapshot() == final_bytes &&
host->state()->tick() == tick + 192,
"player operation did not materialize first");
host->drain();
// drain is best effort after the deliberate unknown-admission connection
// loss. Confirm the fixture handoff before another test acquires authority.
const auto released = release_region_lease(*f.db, f.key, final_authority, 5s);
require(released.has_value() || released.error().kind == LeaseStoreError::fenced,
"final host authority release failed");
}
} // namespace
void run_population_publication_tests() {
Fixture fixture;
funded_source(fixture);
history_pages(fixture);
snapshot_guards(fixture);
registered_snapshot_coverage(fixture);
conversions_and_retries(fixture);
interval_contract(fixture);
ambiguous_publication(fixture, false);
ambiguous_publication(fixture, true);
process_loss(fixture, 0);
process_loss(fixture, 1);
process_loss(fixture, 2);
missing_and_corrupt_objects(fixture);
isolation_and_reverse(fixture);
boundary_upgrades(fixture);
migration_freezes(fixture);
host_ownership_and_admission(fixture);
fixture.lease = take(acquire_region_lease(*fixture.db, fixture.key, fixture.build, 60000ms, 5s),
"maturation replacement authority");
const std::array<Sha256Digest, 1> retained{fixture.build};
auto maturation_source =
take(recover_cellular_checkpoint(*fixture.db, *fixture.objects, fixture.key, fixture.lease,
retained, 5s),
"maturation source recovery");
fixture.head = std::move(maturation_source.committed);
fixture.raw = std::move(maturation_source.state);
boundary_upgrades(fixture, true);
host_ownership_and_admission(fixture);
maturing_census();
std::puts("population publication, exact recovery, process loss and fencing passed");
}