Files
aevolution/tests/population_publication_tests.cpp
T
dingo 2416735e2b
Foundation / checks (deploy) (push) Successful in 4s
Foundation / checks (ecs) (push) Successful in 2m38s
Foundation / checks (network) (push) Successful in 22m52s
Foundation / checks (client) (push) Successful in 23m48s
Foundation / checks (tsan) (push) Successful in 23m38s
Foundation / checks (postgres) (push) Successful in 53m31s
Foundation / checks (checks) (push) Successful in 1h9m6s
Foundation / postgres-tools (push) Successful in 2m0s
Foundation / server-image (push) Successful in 4m2s
Foundation / image (push) Successful in 4m40s
M12.4: define explicit boundary-accounting checkpoint upgrades
2026-10-17 22:33:19 -04:00

689 lines
35 KiB
C++

#include "evo/persistence/account_control_store.hpp"
#include "evo/persistence/population_checkpoint_store.hpp"
#include "evo/persistence/population_interval_store.hpp"
#include "evo/persistence/sha256.hpp"
#include "evo/server/cellular_region_host.hpp"
#include "fixtures/cellular_ecology.hpp"
#include <csignal>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <stdexcept>
#include <sys/wait.h>
#include <thread>
#include <type_traits>
#include <unistd.h>
void run_population_publication_tests();
namespace {
using namespace evo;
using namespace evo::server;
using namespace evo::persistence;
using namespace evo::fixture;
using namespace std::chrono_literals;
template <typename T, typename E> T take(std::expected<T, E> value, const char* where) {
if (!value) {
std::string message{where};
if constexpr (requires { value.error().cause.sqlstate; })
message += " SQLSTATE=" + std::string(value.error().cause.sqlstate.data(), 5);
if constexpr (requires { value.error().store.cause.sqlstate; })
message += " SQLSTATE=" + std::string(value.error().store.cause.sqlstate.data(), 5);
throw std::runtime_error(message);
}
if constexpr (!std::is_void_v<T>)
return std::move(*value);
}
template <typename T> T take(std::optional<T> value, const char* where) {
if (!value.has_value())
throw std::runtime_error(where);
return std::move(*value);
}
std::string environment(const char* name) {
const auto* value = std::getenv(name);
require(value != nullptr, "missing private test environment");
return value;
}
std::unique_ptr<Postgres> connection(bool owner = false) {
return take(Postgres::connect({environment("PGHOST"),
{},
environment("PGPORT"),
environment("PGDATABASE"),
environment(owner ? "PGUSER" : "EVO_PG_RUNTIME_USER"),
{},
{}},
5s),
"population connection");
}
Uuid identity(unsigned number) {
Uuid id{};
id[0] = 0xDA;
id[1] = 0x12;
id[14] = static_cast<std::uint8_t>(number >> 8U);
id[15] = static_cast<std::uint8_t>(number);
return id;
}
std::string hexadecimal(std::span<const std::uint8_t> bytes, bool prefix = true) {
std::string out = prefix ? "\\x" : "";
constexpr char digits[] = "0123456789abcdef";
for (const auto b : bytes) {
out.push_back(digits[b >> 4U]);
out.push_back(digits[b & 15U]);
}
return out;
}
struct Fixture {
RegionKey key{30201, 1};
std::unique_ptr<Postgres> owner = connection(true), db = connection();
std::unique_ptr<FilesystemObjectStore> objects;
LeaseGrant lease;
Sha256Digest build{};
CommittedCheckpoint head;
std::unique_ptr<CellularLives> raw;
std::unique_ptr<CellularPopulation> population;
unsigned next_id = 1;
Uuid id() {
return identity(next_id++);
}
CellularBatchHeader header(std::uint64_t count) const {
return {key.world,
key.region,
1,
head.head.commit,
head.head.digest,
head.head.end_tick,
head.head.end_tick + count};
}
};
void funded_source(Fixture& f) {
f.build.fill(0xDA);
const std::array<std::string, 1> build{hexadecimal(f.build)};
take(f.owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES($1::bytea) "
"ON CONFLICT DO NOTHING",
build, 5s),
"retain test build");
take(f.owner->query("INSERT INTO evolution.life_allocators VALUES(30201,100)", {}, 5s),
"test allocator");
f.objects = take(
FilesystemObjectStore::open(environment("PGHOST"), f.key, max_cellular_population_bytes),
"population object namespace");
f.raw = opening(30001, true, 4, false);
f.head =
take(provision_cellular_region(*f.owner, *f.objects, f.key, 1, *f.raw, f.id(), f.build, 5s),
"empty cellular genesis");
f.lease =
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "population test lease");
CellularBatch birth{f.header(1), {{}}};
for (std::uint8_t index = 1; index <= 4; ++index) {
Uuid account{};
account.back() = index;
const auto control =
take(claim_account_control(*f.db, f.key, f.lease, account, 1, index, 60000ms, 5s),
"founder account control");
const auto reserved = take(
reserve_primitive_life(*f.db, f.key, f.lease, account, control.generation, 1, 1, 5s),
"funded founder reservation");
birth.ticks.front().births.push_back({account, 1, control.generation, reserved.identity});
}
f.raw = take(replay_cellular_batch(*f.raw, birth), "funded founder replay");
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, birth, *f.raw, f.id(),
f.build, 5s),
"funded founder publication")
.committed;
while (f.raw->next_bud_sequence() == 1 && f.raw->tick() < 20000) {
CellularBatch batch{f.header(1), {}};
for (unsigned tick = 0; tick < 64; ++tick) {
auto intents =
take(wildlife_intents(*f.raw, f.raw->tick() + 1), "funded source wildlife");
for (const auto& life : f.raw->lives()) {
if (life.account == LifeAccount{})
continue;
CellularLifeIntent intent{life.identity.life, {}};
intent.intent.bud = (f.raw->tick() + life.identity.life) % 32 == 0;
intents.push_back(intent);
}
std::ranges::sort(intents, {}, &CellularLifeIntent::life_id);
take(f.raw->advance_tick(intents), "funded source biology");
batch.ticks.push_back({intents, {}, {}});
if (f.raw->next_bud_sequence() > 1)
break;
}
batch.header.end_tick = f.raw->tick();
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, batch, *f.raw,
f.id(), f.build, 5s),
"funded source batch")
.committed;
take(renew_region_lease(*f.db, f.key, f.lease, 60000ms, 5s), "source lease renewal");
}
require(std::ranges::any_of(f.raw->lives(),
[](const auto& life) { return life.account == LifeAccount{}; }),
"durable fixture has no funded player-derived wildlife");
require(!migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build,
{true, 1200, 1800}, 5s),
"population entry bypassed vulnerable controlled life");
const auto suspended_at = f.raw->tick() + cellular_linger_ticks + 1;
while (f.raw->tick() < suspended_at) {
auto h = f.header(std::min(std::uint64_t{64}, suspended_at - f.raw->tick()));
auto prefix = take(unattended_population_inputs(*f.raw, h), "raw linger inputs");
f.raw = take(replay_cellular_batch(*f.raw, prefix), "raw linger biology");
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, prefix, *f.raw,
f.id(), f.build, 5s),
"raw linger publication")
.committed;
std::vector<std::uint64_t> endings;
for (const auto& corpse : f.raw->corpses())
if (corpse.parent_life == 0)
endings.push_back(corpse.life_id);
if (!endings.empty())
take(end_cellular_lives(*f.db, f.key, f.lease, f.head.head.commit, endings, 5s),
"raw linger endings");
}
require(population_interval_safe(*f.raw),
"population source retained a vulnerable controlled life");
}
CellularPopulationExecution unattended(Fixture& f) {
auto working = restore(f.population->state());
CellularBatch batch{f.header(64), {}};
for (unsigned tick = 0; tick < 64; ++tick) {
auto intents = take(wildlife_intents(*working, working->tick() + 1), "unattended wildlife");
for (const auto& life : working->lives()) {
if (life.account == LifeAccount{})
continue;
CellularLifeIntent frozen{life.identity.life, {}};
frozen.intent.frozen = true;
intents.push_back(frozen);
}
std::ranges::sort(intents, {}, &CellularLifeIntent::life_id);
take(working->advance_tick(intents), "unattended fixture step");
batch.ticks.push_back({intents, {}, {}});
}
return take(prepare_population_batch(*f.population, std::move(batch)),
"population recorded interval");
}
void recover(Fixture& f) {
const std::array<Sha256Digest, 1> builds{f.build};
auto db = connection();
auto recovered =
take(recover_population_checkpoint(*db, *f.objects, f.key, f.lease, builds, 5s),
"population durable recovery");
f.head = std::move(recovered.committed);
f.population = std::move(recovered.state);
}
void conversions_and_retries(Fixture& f) {
const auto biology = f.raw->snapshot();
Uuid account{};
account.back() = 1;
const auto input =
encode_cellular_mutation_input({account, 9991, f.head.head.end_tick + 1, {100, 1, 1, 1}});
take(f.db->query("BEGIN", {}, 5s), "reservation guard transaction");
take(reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 9991, input, 5s),
"reserve pending player choice");
const auto blocked = migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
f.build, {true, 1200, 1800}, 5s);
require(!blocked && std::string(blocked.error().store.cause.sqlstate.data(), 5) == "EVL18",
"population entry stranded a reserved player choice");
take(f.db->query("ROLLBACK", {}, 5s), "rollback guard fixture");
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
f.build, {true, 1200, 1800}, 5s),
"initial population transition");
recover(f);
require(f.population->state().snapshot() == biology && f.population->members().empty(),
"transition changed biology");
const auto refused =
reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 9991, input, 5s);
require(!refused && std::string(refused.error().cause.sqlstate.data(), 5) == "EVL06",
"population profile admitted a player mutation reservation");
const auto wildlife = std::ranges::find_if(f.population->state().lives(), [](const auto& life) {
return life.account == LifeAccount{};
});
require(wildlife != f.population->state().lives().end(), "missing durable descendant");
const auto genome = wildlife->identity.genome;
auto conversion =
take(prepare_population_conversion(*f.population, f.header(0), {genome, true}),
"collapse sample");
auto bad = conversion.batch;
++bad.boundaries.front().sequence;
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, bad, *conversion.state,
f.id(), f.build, 5s),
"altered sample published");
const auto commit = f.id();
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, conversion.batch,
*conversion.state, commit, f.build, 5s),
"durable collapse");
const PopulationPreparedCommit prepared{
f.key, commit, f.head.manifest,
take(encode_population_batch(conversion.batch), "prepared trace")};
require(take(retry_population_prepared(*f.db, prepared, 5s), "idempotent retry").head ==
f.head.head,
"retry advanced head");
require(take(resolve_population_prepared(*f.db, prepared, 5s), "read-only resolve").status ==
Resolution::committed,
"commit unresolved");
auto future_batch = conversion.batch;
future_batch.format_version = 2;
auto future = prepared;
future.trace = take(encode_population_batch(future_batch), "future trace fixture");
auto future_manifest = take(decode_manifest(future.manifest), "future manifest fixture");
future_manifest.batch.trace_bytes = future.trace.size();
future_manifest.batch.trace_sha256 = take(sha256(future.trace), "future trace digest");
future.manifest = take(encode_manifest(future_manifest), "future manifest encoding");
auto rejected_future = retry_population_prepared(*f.db, future, 5s);
require(!rejected_future && rejected_future.error().kind == PublicationError::candidate,
"bundle 9 prepared retry admitted population trace 2");
require(!resolve_population_prepared(*f.db, future, 5s),
"bundle 9 prepared resolve admitted population trace 2");
auto rejected_sql = publish_population_checkpoint(*f.db, future.manifest, future.trace, 5s);
require(!rejected_sql && rejected_sql.error().kind == LeaseStoreError::invalid,
"schema 45 did not reject the unsupported trace format before commit lookup");
const auto current_manifest = take(decode_manifest(f.head.manifest), "upgrade source manifest");
const auto boundary_payload =
take(cellular_boundary_upgrade_payload(conversion.state->snapshot(), current_manifest),
"portable boundary upgrade source");
const auto boundary_manifest = take(
cellular_boundary_upgrade_manifest(
current_manifest, f.head.head.digest, f.id(), f.lease, f.build, boundary_payload.size(),
take(sha256(boundary_payload), "boundary payload digest")),
"portable boundary upgrade manifest");
const PopulationPreparedCommit boundary{
f.key,
boundary_manifest.commit,
take(encode_manifest(boundary_manifest), "boundary candidate encoding"),
{}};
auto rejected_boundary = retry_population_prepared(*f.db, boundary, 5s);
require(!rejected_boundary && rejected_boundary.error().kind == PublicationError::candidate &&
!resolve_population_prepared(*f.db, boundary, 5s),
"portable bundle 10 bypassed prepared publication admission");
auto rejected_upgrade = publish_population_profile_transition(*f.db, boundary.manifest, 5s);
require(!rejected_upgrade && rejected_upgrade.error().kind == LeaseStoreError::invalid,
"schema 45 admitted portable accounting upgrade without dedicated evidence");
require(!publish_region_checkpoint(*f.db, boundary.manifest, {}, 5s),
"generic publisher admitted boundary upgrade");
require(take(load_region_checkpoint(*f.db, f.key, 5s), "head after future rejection").head ==
f.head.head,
"future trace changed the retained durable head");
auto changed = prepared;
auto changed_manifest = take(decode_manifest(changed.manifest), "conflict manifest");
changed_manifest.payload_sha256[0] ^= 1;
changed.manifest = take(encode_manifest(changed_manifest), "conflict encoding");
require(take(resolve_population_prepared(*f.db, changed, 5s), "conflicting resolve").status ==
Resolution::conflict,
"changed commit was accepted");
require(!publish_region_checkpoint(*f.db, f.head.manifest, {}, 5s),
"generic publisher bypassed profile gate");
const auto expected = conversion.state->snapshot();
recover(f);
require(f.population->snapshot() == expected && !f.population->members().empty(),
"committed sample not recovered");
require(f.population->state().snapshot() == biology, "collapse healed/aged/moved biology");
for (unsigned oscillation = 0; oscillation < 8; ++oscillation) {
const bool collapse = oscillation % 2 != 0;
auto next =
take(prepare_population_conversion(*f.population, f.header(0), {genome, collapse}),
"conversion oscillation");
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, next.batch,
*next.state, f.id(), f.build, 5s),
"publish oscillation");
recover(f);
require(f.population->state().snapshot() == biology,
"durable conversion changed exact state");
}
}
void process_loss(Fixture& f, int point) {
auto next = unattended(f);
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, next.batch.header.end_tick,
5s),
"admit process-loss interval");
const auto predecessor = f.head.head;
const auto expected = next.state->snapshot();
const auto commit = f.id();
int channel[2]{};
require(::pipe(channel) == 0, "process acknowledgement pipe");
const auto child = ::fork();
require(child >= 0, "population fork");
if (child == 0) {
::close(channel[0]);
try {
auto db = connection();
PublishHooks hooks;
if (point == 0)
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
const auto published =
publish_population_batch(*db, *f.objects, f.lease, f.head, next.batch, *next.state,
commit, f.build, 5s, hooks);
if (!published)
::_exit(20);
if (point == 2) {
const char ack = 'A';
if (::write(channel[1], &ack, 1) != 1)
::_exit(21);
}
::kill(::getpid(), SIGKILL);
} catch (...) {
::_exit(22);
}
::_exit(23);
}
::close(channel[1]);
char ack{};
const auto acknowledged = ::read(channel[0], &ack, 1);
::close(channel[0]);
int status{};
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
WTERMSIG(status) == SIGKILL,
"child missed population crash boundary");
require((point == 2 && acknowledged == 1 && ack == 'A') || (point != 2 && acknowledged == 0),
"unexpected permanent acknowledgement");
recover(f);
const auto pending =
take(load_population_interval(*f.db, f.key, f.lease, 5s), "recover process interval");
require(pending.has_value() == (point == 0), "process loss changed interval settlement");
if (point == 0)
require(f.head.head == predecessor, "uncommitted process published a head");
else
require(f.head.head.commit == commit && f.population->snapshot() == expected,
"committed process loss changed state");
}
void ambiguous_publication(Fixture& f, bool terminate) {
auto next = unattended(f);
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, next.batch.header.end_tick,
5s),
"admit ambiguous interval");
const auto expected = next.state->snapshot();
auto locker = connection(true);
take(locker->query("BEGIN", {}, 5s), "population lease row transaction");
const std::array<std::string, 2> key{std::to_string(f.key.world), std::to_string(f.key.region)};
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=$1::evolution.u64 "
"AND region_id=$2::evolution.u64 FOR UPDATE",
key, 5s),
"hold population publication");
auto timed = connection();
auto unknown = publish_population_batch(*timed, *f.objects, f.lease, f.head, next.batch,
*next.state, f.id(), f.build, 300ms);
require(!unknown && unknown.error().kind == PublicationError::unknown,
"population publication failed to retain ambiguous request");
if (terminate) {
const std::array<std::string, 1> user{environment("EVO_PG_RUNTIME_USER")};
take(locker->query("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE "
"usename=$1 AND wait_event_type='Lock'",
user, 5s),
"terminate uncommitted population writer");
}
take(locker->query("ROLLBACK", {}, 5s), "release population publication");
auto resolver = connection();
bool resolved = false;
for (unsigned attempt = 0; attempt < 200; ++attempt) {
auto result = take(resolve_population_prepared(*resolver, unknown.error().prepared, 5s),
"resolve ambiguous population publication");
if (result.status == (terminate ? Resolution::absent : Resolution::committed)) {
resolved = true;
break;
}
std::this_thread::sleep_for(25ms);
}
require(resolved, "ambiguous population publication did not resolve");
f.head = take(retry_population_prepared(*resolver, unknown.error().prepared, 5s),
"retry exact population request");
recover(f);
require(f.population->snapshot() == expected, "ambiguous publication changed retained sample");
}
void interval_contract(Fixture& f) {
require(!take(load_population_interval(*f.db, f.key, f.lease, 5s), "empty interval journal")
.has_value(),
"settled source retained pending work");
auto next = unattended(f);
const auto source = f.head.head;
const auto admitted =
take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
"durable interval admission");
require(take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
"duplicate interval admission") == admitted,
"admission retry changed boundary");
auto changed =
admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 63, 5s);
require(!changed && changed.error().kind == LeaseStoreError::commit_conflict,
"source admitted two different intervals");
const auto expected = next.state->snapshot();
require(!f.population->members().empty(), "interval fixture lost its exact cohort");
const auto genome = f.population->members().front().genome;
auto conversion =
take(prepare_population_conversion(*f.population, f.header(0), {genome, false}),
"conversion during admitted work");
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, conversion.batch,
*conversion.state, f.id(), f.build, 5s),
"conversion skipped an admitted interval");
require(
!materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
"profile exit skipped an admitted interval");
auto shortened_header = f.header(63);
auto shortened =
take(prepare_population_batch(
*f.population,
take(unattended_population_inputs(f.population->state(), shortened_header),
"shortened input")),
"shortened interval");
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, shortened.batch,
*shortened.state, f.id(), f.build, 5s),
"publication partially consumed an interval");
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release interval owner");
const auto epoch = f.lease.epoch;
f.lease =
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "replace interval owner");
require(f.lease.epoch > epoch && take(take(load_population_interval(*f.db, f.key, f.lease, 5s),
"new owner interval load"),
"retained admission") == admitted,
"replacement owner lost admitted work");
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, next.batch,
*next.state, f.id(), f.build, 5s),
"settle exact admitted interval");
recover(f);
require(f.population->snapshot() == expected &&
!take(load_population_interval(*f.db, f.key, f.lease, 5s), "settled interval load")
.has_value(),
"settlement lost work or retained spendable pending state");
require(take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
"settled admission retry") == admitted &&
!take(load_population_interval(*f.db, f.key, f.lease, 5s), "settled retry pending")
.has_value(),
"old admission retry reopened settled work");
require(!admit_population_interval(*f.db, f.key, f.lease, f.head.head,
f.head.head.end_tick + 65, 5s),
"oversized interval admitted");
const auto denied = f.db->query(
"SELECT * FROM evolution.publish_population_checkpoint_v44(''::bytea,''::bytea)", {}, 5s);
require(!denied && std::string(denied.error().sqlstate.data(), 5) == "42501",
"runtime bypassed interval guard");
}
void isolation_and_reverse(Fixture& f) {
auto interval = unattended(f);
const auto old_lease = f.lease;
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release old authority");
auto new_db = connection();
f.lease = take(acquire_region_lease(*new_db, f.key, f.build, 60000ms, 5s),
"new population authority");
require(f.lease.epoch > old_lease.epoch, "authority epoch did not advance");
const auto old_head = f.head.head;
auto stale = publish_population_batch(*f.db, *f.objects, old_lease, f.head, interval.batch,
*interval.state, f.id(), f.build, 5s);
require(!stale && stale.error().store.kind == LeaseStoreError::fenced,
"stale population writer published");
recover(f);
require(f.head.head == old_head, "fenced writer changed head");
const auto expected = f.population->state().snapshot();
f.head = take(
materialize_population_profile(*new_db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
"return to cellular profile");
const std::array<Sha256Digest, 1> builds{f.build};
auto recovered =
take(recover_cellular_checkpoint(*new_db, *f.objects, f.key, f.lease, builds, 5s),
"cellular recovery after materialization");
require(recovered.state->snapshot() == expected &&
recovered.manifest.profile == CheckpointProfile::cellular_lives_v1,
"reverse transition changed biology or time");
f.raw = std::move(recovered.state);
}
void missing_and_corrupt_objects(Fixture& f) {
const auto manifest = take(decode_manifest(f.head.manifest), "object test manifest");
const auto path = f.objects->directory() / hexadecimal(manifest.payload_sha256, false);
const auto backup = path.string() + ".population-test";
const std::array<Sha256Digest, 1> builds{f.build};
std::filesystem::rename(path, backup);
auto missing = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
std::filesystem::rename(backup, path);
require(!missing && missing.error().kind == RecoveryError::missing_payload,
"missing population payload did not fail closed");
std::filesystem::rename(path, backup);
{
std::ofstream corrupt(path, std::ios::binary);
corrupt << 'X';
}
auto corrupt = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
std::filesystem::remove(path);
std::filesystem::rename(backup, path);
require(!corrupt && corrupt.error().kind == RecoveryError::corrupt_payload,
"corrupt population payload did not fail closed");
recover(f);
}
void host_ownership_and_admission(Fixture& f) {
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture host authority");
const auto make_host = [&](bool enabled) {
auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), f.key,
max_cellular_population_bytes),
"host objects");
CellularHostConfig config{f.key, 1, {}, f.build, {f.build}};
config.lease.commit_timeout = 300ms;
if (enabled)
config.population_policy = CellularTierPolicy{true, 1200, 1800};
return take(CellularRegionHost::create(
std::move(config),
[] {
return Postgres::connect({environment("PGHOST"),
{},
environment("PGPORT"),
environment("PGDATABASE"),
environment("EVO_PG_RUNTIME_USER"),
{},
{}},
5s);
},
std::move(objects)),
"population host");
};
auto host = make_host(false);
require(host->step().ready && !host->population_mode(), "raw host recovery");
require(!host->set_population_mode(true), "default host enabled population entry");
host->drain();
host = make_host(true);
require(host->step().ready, "opt-in host recovery");
const auto before = host->state()->snapshot();
const auto tick = host->state()->tick();
take(host->set_population_mode(true), "host population entry");
require(host->population_mode() && host->state()->snapshot() == before,
"host entry changed biology");
auto population =
take(CellularPopulation::create(
take(CellularLives::restore(before, f.key.world, f.key.region, 1, tick),
"host reference restore"),
{true, 1200, 1800}),
"host reference population");
f.head = take(host->committed(), "host population head");
auto next = unattended(f); // uses the same deterministic frozen/wildlife input contract.
// The fixture's old envelope may have a different membership history; biology is shared.
auto expected =
take(prepare_population_batch(*population, next.batch.biology), "host reference batch");
require(!take(host->advance_unattended(19), "collect bounded publication window").has_value() &&
host->state()->snapshot() == before,
"small local debt was admitted prematurely");
const auto published = take(host->advance_unattended(65), "host population publication");
require(published.has_value(), "host skipped admitted interval");
require(host->population_mode() &&
host->state()->snapshot() == expected.state->state().snapshot(),
"host population continuation differs");
const auto committed = take(host->committed(), "host continued head");
const auto continued = host->state()->snapshot();
const CellularBatchHeader recovered_header{f.key.world,
f.key.region,
1,
committed.head.commit,
committed.head.digest,
committed.head.end_tick,
committed.head.end_tick + 64};
auto recovered_expected = take(
prepare_population_batch(
*expected.state, take(unattended_population_inputs(*host->state(), recovered_header),
"recovered interval inputs")),
"recovered interval reference");
take(admit_population_interval(*f.db, f.key, take(host->authority(), "host interval authority"),
committed.head, committed.head.end_tick + 64, 5s),
"admit before host loss");
host->drain();
host = make_host(false);
require(host->step().ready && host->population_mode() &&
take(host->committed(), "host recovered head").head == committed.head &&
host->state()->snapshot() == continued,
"restarted host failed to recover population head");
require(host->status().ecological_pending_ticks == 64 && !host->set_population_mode(false),
"restarted host hid or skipped admitted work");
require(take(host->advance_unattended(0), "resume admitted work with zero new due time")
.has_value() &&
host->state()->snapshot() == recovered_expected.state->state().snapshot() &&
host->status().ecological_pending_ticks == 0,
"recovered interval did not settle exactly");
const auto recovered_head = take(host->committed(), "recovered interval head");
const auto recovered_bytes = host->state()->snapshot();
const CellularBatchHeader unknown_header{f.key.world,
f.key.region,
1,
recovered_head.head.commit,
recovered_head.head.digest,
recovered_head.head.end_tick,
recovered_head.head.end_tick + 64};
auto unknown_expected = take(
prepare_population_batch(*recovered_expected.state,
take(unattended_population_inputs(*host->state(), unknown_header),
"uncertain admission inputs")),
"uncertain admission reference");
auto locker = connection(true);
take(locker->query("BEGIN", {}, 5s), "host admission lock transaction");
take(
locker->query(
"SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE",
{}, 5s),
"hold host admission reply");
require(!host->advance_unattended(64) && !host->status().ready &&
host->state()->snapshot() == recovered_bytes,
"uncertain admission allowed simulation/output");
take(locker->query("ROLLBACK", {}, 5s), "release host admission reply");
for (unsigned attempt = 0; attempt < 200 && !host->step().ready; ++attempt)
std::this_thread::sleep_for(25ms);
require(host->status().ready && host->status().ecological_pending_ticks == 64,
"unknown admission failed exact reconciliation");
require(take(host->advance_unattended(0), "finish reconciled admission").has_value() &&
host->state()->snapshot() == unknown_expected.state->state().snapshot(),
"admission retry chose different work");
const auto final_bytes = host->state()->snapshot();
Uuid account{};
account.back() = 1;
// A returning player's operation completes the durable reverse transition before lookup.
[[maybe_unused]] const auto queried = host->query_primitive_life(account, 1, 1);
require(!host->population_mode() && host->state()->snapshot() == final_bytes &&
host->state()->tick() == tick + 192,
"player operation did not materialize first");
host->drain();
}
} // namespace
void run_population_publication_tests() {
Fixture fixture;
funded_source(fixture);
conversions_and_retries(fixture);
interval_contract(fixture);
ambiguous_publication(fixture, false);
ambiguous_publication(fixture, true);
process_loss(fixture, 0);
process_loss(fixture, 1);
process_loss(fixture, 2);
missing_and_corrupt_objects(fixture);
isolation_and_reverse(fixture);
host_ownership_and_admission(fixture);
std::puts("population publication, exact recovery, process loss and fencing passed");
}