Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m28s
Foundation / checks (tsan) (push) Failing after 18m22s
Foundation / checks (network) (push) Successful in 20m38s
Foundation / checks (client) (push) Failing after 21m6s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (postgres) (push) Canceled after 46m54s
Foundation / checks (checks) (push) Canceled after 47m3s
647 lines
33 KiB
C++
647 lines
33 KiB
C++
#include "evo/persistence/account_control_store.hpp"
|
|
#include "evo/persistence/population_checkpoint_store.hpp"
|
|
#include "evo/persistence/population_interval_store.hpp"
|
|
#include "evo/persistence/sha256.hpp"
|
|
#include "evo/server/cellular_region_host.hpp"
|
|
#include "fixtures/cellular_ecology.hpp"
|
|
|
|
#include <csignal>
|
|
#include <cstdlib>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <stdexcept>
|
|
#include <sys/wait.h>
|
|
#include <thread>
|
|
#include <type_traits>
|
|
#include <unistd.h>
|
|
|
|
void run_population_publication_tests();
|
|
namespace {
|
|
using namespace evo;
|
|
using namespace evo::server;
|
|
using namespace evo::persistence;
|
|
using namespace evo::fixture;
|
|
using namespace std::chrono_literals;
|
|
template <typename T, typename E> T take(std::expected<T, E> value, const char* where) {
|
|
if (!value) {
|
|
std::string message{where};
|
|
if constexpr (requires { value.error().cause.sqlstate; })
|
|
message += " SQLSTATE=" + std::string(value.error().cause.sqlstate.data(), 5);
|
|
if constexpr (requires { value.error().store.cause.sqlstate; })
|
|
message += " SQLSTATE=" + std::string(value.error().store.cause.sqlstate.data(), 5);
|
|
throw std::runtime_error(message);
|
|
}
|
|
if constexpr (!std::is_void_v<T>)
|
|
return std::move(*value);
|
|
}
|
|
template <typename T> T take(std::optional<T> value, const char* where) {
|
|
if (!value.has_value())
|
|
throw std::runtime_error(where);
|
|
return std::move(*value);
|
|
}
|
|
std::string environment(const char* name) {
|
|
const auto* value = std::getenv(name);
|
|
require(value != nullptr, "missing private test environment");
|
|
return value;
|
|
}
|
|
std::unique_ptr<Postgres> connection(bool owner = false) {
|
|
return take(Postgres::connect({environment("PGHOST"),
|
|
{},
|
|
environment("PGPORT"),
|
|
environment("PGDATABASE"),
|
|
environment(owner ? "PGUSER" : "EVO_PG_RUNTIME_USER"),
|
|
{},
|
|
{}},
|
|
5s),
|
|
"population connection");
|
|
}
|
|
Uuid identity(unsigned number) {
|
|
Uuid id{};
|
|
id[0] = 0xDA;
|
|
id[1] = 0x12;
|
|
id[14] = static_cast<std::uint8_t>(number >> 8U);
|
|
id[15] = static_cast<std::uint8_t>(number);
|
|
return id;
|
|
}
|
|
std::string hexadecimal(std::span<const std::uint8_t> bytes, bool prefix = true) {
|
|
std::string out = prefix ? "\\x" : "";
|
|
constexpr char digits[] = "0123456789abcdef";
|
|
for (const auto b : bytes) {
|
|
out.push_back(digits[b >> 4U]);
|
|
out.push_back(digits[b & 15U]);
|
|
}
|
|
return out;
|
|
}
|
|
struct Fixture {
|
|
RegionKey key{30201, 1};
|
|
std::unique_ptr<Postgres> owner = connection(true), db = connection();
|
|
std::unique_ptr<FilesystemObjectStore> objects;
|
|
LeaseGrant lease;
|
|
Sha256Digest build{};
|
|
CommittedCheckpoint head;
|
|
std::unique_ptr<CellularLives> raw;
|
|
std::unique_ptr<CellularPopulation> population;
|
|
unsigned next_id = 1;
|
|
Uuid id() {
|
|
return identity(next_id++);
|
|
}
|
|
CellularBatchHeader header(std::uint64_t count) const {
|
|
return {key.world,
|
|
key.region,
|
|
1,
|
|
head.head.commit,
|
|
head.head.digest,
|
|
head.head.end_tick,
|
|
head.head.end_tick + count};
|
|
}
|
|
};
|
|
void funded_source(Fixture& f) {
|
|
f.build.fill(0xDA);
|
|
const std::array<std::string, 1> build{hexadecimal(f.build)};
|
|
take(f.owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES($1::bytea) "
|
|
"ON CONFLICT DO NOTHING",
|
|
build, 5s),
|
|
"retain test build");
|
|
take(f.owner->query("INSERT INTO evolution.life_allocators VALUES(30201,100)", {}, 5s),
|
|
"test allocator");
|
|
f.objects = take(
|
|
FilesystemObjectStore::open(environment("PGHOST"), f.key, max_cellular_population_bytes),
|
|
"population object namespace");
|
|
f.raw = opening(30001, true, 4, false);
|
|
f.head =
|
|
take(provision_cellular_region(*f.owner, *f.objects, f.key, 1, *f.raw, f.id(), f.build, 5s),
|
|
"empty cellular genesis");
|
|
f.lease =
|
|
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "population test lease");
|
|
CellularBatch birth{f.header(1), {{}}};
|
|
for (std::uint8_t index = 1; index <= 4; ++index) {
|
|
Uuid account{};
|
|
account.back() = index;
|
|
const auto control =
|
|
take(claim_account_control(*f.db, f.key, f.lease, account, 1, index, 60000ms, 5s),
|
|
"founder account control");
|
|
const auto reserved = take(
|
|
reserve_primitive_life(*f.db, f.key, f.lease, account, control.generation, 1, 1, 5s),
|
|
"funded founder reservation");
|
|
birth.ticks.front().births.push_back({account, 1, control.generation, reserved.identity});
|
|
}
|
|
f.raw = take(replay_cellular_batch(*f.raw, birth), "funded founder replay");
|
|
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, birth, *f.raw, f.id(),
|
|
f.build, 5s),
|
|
"funded founder publication")
|
|
.committed;
|
|
while (f.raw->next_bud_sequence() == 1 && f.raw->tick() < 20000) {
|
|
CellularBatch batch{f.header(1), {}};
|
|
for (unsigned tick = 0; tick < 64; ++tick) {
|
|
auto intents =
|
|
take(wildlife_intents(*f.raw, f.raw->tick() + 1), "funded source wildlife");
|
|
for (const auto& life : f.raw->lives()) {
|
|
if (life.account == LifeAccount{})
|
|
continue;
|
|
CellularLifeIntent intent{life.identity.life, {}};
|
|
intent.intent.bud = (f.raw->tick() + life.identity.life) % 32 == 0;
|
|
intents.push_back(intent);
|
|
}
|
|
std::ranges::sort(intents, {}, &CellularLifeIntent::life_id);
|
|
take(f.raw->advance_tick(intents), "funded source biology");
|
|
batch.ticks.push_back({intents, {}, {}});
|
|
if (f.raw->next_bud_sequence() > 1)
|
|
break;
|
|
}
|
|
batch.header.end_tick = f.raw->tick();
|
|
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, batch, *f.raw,
|
|
f.id(), f.build, 5s),
|
|
"funded source batch")
|
|
.committed;
|
|
take(renew_region_lease(*f.db, f.key, f.lease, 60000ms, 5s), "source lease renewal");
|
|
}
|
|
require(std::ranges::any_of(f.raw->lives(),
|
|
[](const auto& life) { return life.account == LifeAccount{}; }),
|
|
"durable fixture has no funded player-derived wildlife");
|
|
require(!migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build,
|
|
{true, 1200, 1800}, 5s),
|
|
"population entry bypassed vulnerable controlled life");
|
|
const auto suspended_at = f.raw->tick() + cellular_linger_ticks + 1;
|
|
while (f.raw->tick() < suspended_at) {
|
|
auto h = f.header(std::min(std::uint64_t{64}, suspended_at - f.raw->tick()));
|
|
auto prefix = take(unattended_population_inputs(*f.raw, h), "raw linger inputs");
|
|
f.raw = take(replay_cellular_batch(*f.raw, prefix), "raw linger biology");
|
|
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, prefix, *f.raw,
|
|
f.id(), f.build, 5s),
|
|
"raw linger publication")
|
|
.committed;
|
|
std::vector<std::uint64_t> endings;
|
|
for (const auto& corpse : f.raw->corpses())
|
|
if (corpse.parent_life == 0)
|
|
endings.push_back(corpse.life_id);
|
|
if (!endings.empty())
|
|
take(end_cellular_lives(*f.db, f.key, f.lease, f.head.head.commit, endings, 5s),
|
|
"raw linger endings");
|
|
}
|
|
require(population_interval_safe(*f.raw),
|
|
"population source retained a vulnerable controlled life");
|
|
}
|
|
CellularPopulationExecution unattended(Fixture& f) {
|
|
auto working = restore(f.population->state());
|
|
CellularBatch batch{f.header(64), {}};
|
|
for (unsigned tick = 0; tick < 64; ++tick) {
|
|
auto intents = take(wildlife_intents(*working, working->tick() + 1), "unattended wildlife");
|
|
for (const auto& life : working->lives()) {
|
|
if (life.account == LifeAccount{})
|
|
continue;
|
|
CellularLifeIntent frozen{life.identity.life, {}};
|
|
frozen.intent.frozen = true;
|
|
intents.push_back(frozen);
|
|
}
|
|
std::ranges::sort(intents, {}, &CellularLifeIntent::life_id);
|
|
take(working->advance_tick(intents), "unattended fixture step");
|
|
batch.ticks.push_back({intents, {}, {}});
|
|
}
|
|
return take(prepare_population_batch(*f.population, std::move(batch)),
|
|
"population recorded interval");
|
|
}
|
|
void recover(Fixture& f) {
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
auto db = connection();
|
|
auto recovered =
|
|
take(recover_population_checkpoint(*db, *f.objects, f.key, f.lease, builds, 5s),
|
|
"population durable recovery");
|
|
f.head = std::move(recovered.committed);
|
|
f.population = std::move(recovered.state);
|
|
}
|
|
void conversions_and_retries(Fixture& f) {
|
|
const auto biology = f.raw->snapshot();
|
|
Uuid account{};
|
|
account.back() = 1;
|
|
const auto input =
|
|
encode_cellular_mutation_input({account, 9991, f.head.head.end_tick + 1, {100, 1, 1, 1}});
|
|
take(f.db->query("BEGIN", {}, 5s), "reservation guard transaction");
|
|
take(reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 9991, input, 5s),
|
|
"reserve pending player choice");
|
|
const auto blocked = migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
|
|
f.build, {true, 1200, 1800}, 5s);
|
|
require(!blocked && std::string(blocked.error().store.cause.sqlstate.data(), 5) == "EVL18",
|
|
"population entry stranded a reserved player choice");
|
|
take(f.db->query("ROLLBACK", {}, 5s), "rollback guard fixture");
|
|
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
|
|
f.build, {true, 1200, 1800}, 5s),
|
|
"initial population transition");
|
|
recover(f);
|
|
require(f.population->state().snapshot() == biology && f.population->members().empty(),
|
|
"transition changed biology");
|
|
const auto refused =
|
|
reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 9991, input, 5s);
|
|
require(!refused && std::string(refused.error().cause.sqlstate.data(), 5) == "EVL06",
|
|
"population profile admitted a player mutation reservation");
|
|
const auto wildlife = std::ranges::find_if(f.population->state().lives(), [](const auto& life) {
|
|
return life.account == LifeAccount{};
|
|
});
|
|
require(wildlife != f.population->state().lives().end(), "missing durable descendant");
|
|
const auto genome = wildlife->identity.genome;
|
|
auto conversion =
|
|
take(prepare_population_conversion(*f.population, f.header(0), {genome, true}),
|
|
"collapse sample");
|
|
auto bad = conversion.batch;
|
|
++bad.boundaries.front().sequence;
|
|
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, bad, *conversion.state,
|
|
f.id(), f.build, 5s),
|
|
"altered sample published");
|
|
const auto commit = f.id();
|
|
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, conversion.batch,
|
|
*conversion.state, commit, f.build, 5s),
|
|
"durable collapse");
|
|
const PopulationPreparedCommit prepared{
|
|
f.key, commit, f.head.manifest,
|
|
take(encode_population_batch(conversion.batch), "prepared trace")};
|
|
require(take(retry_population_prepared(*f.db, prepared, 5s), "idempotent retry").head ==
|
|
f.head.head,
|
|
"retry advanced head");
|
|
require(take(resolve_population_prepared(*f.db, prepared, 5s), "read-only resolve").status ==
|
|
Resolution::committed,
|
|
"commit unresolved");
|
|
auto changed = prepared;
|
|
auto changed_manifest = take(decode_manifest(changed.manifest), "conflict manifest");
|
|
changed_manifest.payload_sha256[0] ^= 1;
|
|
changed.manifest = take(encode_manifest(changed_manifest), "conflict encoding");
|
|
require(take(resolve_population_prepared(*f.db, changed, 5s), "conflicting resolve").status ==
|
|
Resolution::conflict,
|
|
"changed commit was accepted");
|
|
require(!publish_region_checkpoint(*f.db, f.head.manifest, {}, 5s),
|
|
"generic publisher bypassed profile gate");
|
|
const auto expected = conversion.state->snapshot();
|
|
recover(f);
|
|
require(f.population->snapshot() == expected && !f.population->members().empty(),
|
|
"committed sample not recovered");
|
|
require(f.population->state().snapshot() == biology, "collapse healed/aged/moved biology");
|
|
for (unsigned oscillation = 0; oscillation < 8; ++oscillation) {
|
|
const bool collapse = oscillation % 2 != 0;
|
|
auto next =
|
|
take(prepare_population_conversion(*f.population, f.header(0), {genome, collapse}),
|
|
"conversion oscillation");
|
|
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, next.batch,
|
|
*next.state, f.id(), f.build, 5s),
|
|
"publish oscillation");
|
|
recover(f);
|
|
require(f.population->state().snapshot() == biology,
|
|
"durable conversion changed exact state");
|
|
}
|
|
}
|
|
void process_loss(Fixture& f, int point) {
|
|
auto next = unattended(f);
|
|
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, next.batch.header.end_tick,
|
|
5s),
|
|
"admit process-loss interval");
|
|
const auto predecessor = f.head.head;
|
|
const auto expected = next.state->snapshot();
|
|
const auto commit = f.id();
|
|
int channel[2]{};
|
|
require(::pipe(channel) == 0, "process acknowledgement pipe");
|
|
const auto child = ::fork();
|
|
require(child >= 0, "population fork");
|
|
if (child == 0) {
|
|
::close(channel[0]);
|
|
try {
|
|
auto db = connection();
|
|
PublishHooks hooks;
|
|
if (point == 0)
|
|
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
|
|
const auto published =
|
|
publish_population_batch(*db, *f.objects, f.lease, f.head, next.batch, *next.state,
|
|
commit, f.build, 5s, hooks);
|
|
if (!published)
|
|
::_exit(20);
|
|
if (point == 2) {
|
|
const char ack = 'A';
|
|
if (::write(channel[1], &ack, 1) != 1)
|
|
::_exit(21);
|
|
}
|
|
::kill(::getpid(), SIGKILL);
|
|
} catch (...) {
|
|
::_exit(22);
|
|
}
|
|
::_exit(23);
|
|
}
|
|
::close(channel[1]);
|
|
char ack{};
|
|
const auto acknowledged = ::read(channel[0], &ack, 1);
|
|
::close(channel[0]);
|
|
int status{};
|
|
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
|
|
WTERMSIG(status) == SIGKILL,
|
|
"child missed population crash boundary");
|
|
require((point == 2 && acknowledged == 1 && ack == 'A') || (point != 2 && acknowledged == 0),
|
|
"unexpected permanent acknowledgement");
|
|
recover(f);
|
|
const auto pending =
|
|
take(load_population_interval(*f.db, f.key, f.lease, 5s), "recover process interval");
|
|
require(pending.has_value() == (point == 0), "process loss changed interval settlement");
|
|
if (point == 0)
|
|
require(f.head.head == predecessor, "uncommitted process published a head");
|
|
else
|
|
require(f.head.head.commit == commit && f.population->snapshot() == expected,
|
|
"committed process loss changed state");
|
|
}
|
|
void ambiguous_publication(Fixture& f, bool terminate) {
|
|
auto next = unattended(f);
|
|
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, next.batch.header.end_tick,
|
|
5s),
|
|
"admit ambiguous interval");
|
|
const auto expected = next.state->snapshot();
|
|
auto locker = connection(true);
|
|
take(locker->query("BEGIN", {}, 5s), "population lease row transaction");
|
|
const std::array<std::string, 2> key{std::to_string(f.key.world), std::to_string(f.key.region)};
|
|
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=$1::evolution.u64 "
|
|
"AND region_id=$2::evolution.u64 FOR UPDATE",
|
|
key, 5s),
|
|
"hold population publication");
|
|
auto timed = connection();
|
|
auto unknown = publish_population_batch(*timed, *f.objects, f.lease, f.head, next.batch,
|
|
*next.state, f.id(), f.build, 300ms);
|
|
require(!unknown && unknown.error().kind == PublicationError::unknown,
|
|
"population publication failed to retain ambiguous request");
|
|
if (terminate) {
|
|
const std::array<std::string, 1> user{environment("EVO_PG_RUNTIME_USER")};
|
|
take(locker->query("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE "
|
|
"usename=$1 AND wait_event_type='Lock'",
|
|
user, 5s),
|
|
"terminate uncommitted population writer");
|
|
}
|
|
take(locker->query("ROLLBACK", {}, 5s), "release population publication");
|
|
auto resolver = connection();
|
|
bool resolved = false;
|
|
for (unsigned attempt = 0; attempt < 200; ++attempt) {
|
|
auto result = take(resolve_population_prepared(*resolver, unknown.error().prepared, 5s),
|
|
"resolve ambiguous population publication");
|
|
if (result.status == (terminate ? Resolution::absent : Resolution::committed)) {
|
|
resolved = true;
|
|
break;
|
|
}
|
|
std::this_thread::sleep_for(25ms);
|
|
}
|
|
require(resolved, "ambiguous population publication did not resolve");
|
|
f.head = take(retry_population_prepared(*resolver, unknown.error().prepared, 5s),
|
|
"retry exact population request");
|
|
recover(f);
|
|
require(f.population->snapshot() == expected, "ambiguous publication changed retained sample");
|
|
}
|
|
void interval_contract(Fixture& f) {
|
|
require(!take(load_population_interval(*f.db, f.key, f.lease, 5s), "empty interval journal")
|
|
.has_value(),
|
|
"settled source retained pending work");
|
|
auto next = unattended(f);
|
|
const auto source = f.head.head;
|
|
const auto admitted =
|
|
take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
|
|
"durable interval admission");
|
|
require(take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
|
|
"duplicate interval admission") == admitted,
|
|
"admission retry changed boundary");
|
|
auto changed =
|
|
admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 63, 5s);
|
|
require(!changed && changed.error().kind == LeaseStoreError::commit_conflict,
|
|
"source admitted two different intervals");
|
|
const auto expected = next.state->snapshot();
|
|
require(!f.population->members().empty(), "interval fixture lost its exact cohort");
|
|
const auto genome = f.population->members().front().genome;
|
|
auto conversion =
|
|
take(prepare_population_conversion(*f.population, f.header(0), {genome, false}),
|
|
"conversion during admitted work");
|
|
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, conversion.batch,
|
|
*conversion.state, f.id(), f.build, 5s),
|
|
"conversion skipped an admitted interval");
|
|
require(
|
|
!materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
|
|
"profile exit skipped an admitted interval");
|
|
auto shortened_header = f.header(63);
|
|
auto shortened =
|
|
take(prepare_population_batch(
|
|
*f.population,
|
|
take(unattended_population_inputs(f.population->state(), shortened_header),
|
|
"shortened input")),
|
|
"shortened interval");
|
|
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, shortened.batch,
|
|
*shortened.state, f.id(), f.build, 5s),
|
|
"publication partially consumed an interval");
|
|
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release interval owner");
|
|
const auto epoch = f.lease.epoch;
|
|
f.lease =
|
|
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "replace interval owner");
|
|
require(f.lease.epoch > epoch && take(take(load_population_interval(*f.db, f.key, f.lease, 5s),
|
|
"new owner interval load"),
|
|
"retained admission") == admitted,
|
|
"replacement owner lost admitted work");
|
|
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, next.batch,
|
|
*next.state, f.id(), f.build, 5s),
|
|
"settle exact admitted interval");
|
|
recover(f);
|
|
require(f.population->snapshot() == expected &&
|
|
!take(load_population_interval(*f.db, f.key, f.lease, 5s), "settled interval load")
|
|
.has_value(),
|
|
"settlement lost work or retained spendable pending state");
|
|
require(take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
|
|
"settled admission retry") == admitted &&
|
|
!take(load_population_interval(*f.db, f.key, f.lease, 5s), "settled retry pending")
|
|
.has_value(),
|
|
"old admission retry reopened settled work");
|
|
require(!admit_population_interval(*f.db, f.key, f.lease, f.head.head,
|
|
f.head.head.end_tick + 65, 5s),
|
|
"oversized interval admitted");
|
|
const auto denied = f.db->query(
|
|
"SELECT * FROM evolution.publish_population_checkpoint_v44(''::bytea,''::bytea)", {}, 5s);
|
|
require(!denied && std::string(denied.error().sqlstate.data(), 5) == "42501",
|
|
"runtime bypassed interval guard");
|
|
}
|
|
void isolation_and_reverse(Fixture& f) {
|
|
auto interval = unattended(f);
|
|
const auto old_lease = f.lease;
|
|
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release old authority");
|
|
auto new_db = connection();
|
|
f.lease = take(acquire_region_lease(*new_db, f.key, f.build, 60000ms, 5s),
|
|
"new population authority");
|
|
require(f.lease.epoch > old_lease.epoch, "authority epoch did not advance");
|
|
const auto old_head = f.head.head;
|
|
auto stale = publish_population_batch(*f.db, *f.objects, old_lease, f.head, interval.batch,
|
|
*interval.state, f.id(), f.build, 5s);
|
|
require(!stale && stale.error().store.kind == LeaseStoreError::fenced,
|
|
"stale population writer published");
|
|
recover(f);
|
|
require(f.head.head == old_head, "fenced writer changed head");
|
|
const auto expected = f.population->state().snapshot();
|
|
f.head = take(
|
|
materialize_population_profile(*new_db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
|
|
"return to cellular profile");
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
auto recovered =
|
|
take(recover_cellular_checkpoint(*new_db, *f.objects, f.key, f.lease, builds, 5s),
|
|
"cellular recovery after materialization");
|
|
require(recovered.state->snapshot() == expected &&
|
|
recovered.manifest.profile == CheckpointProfile::cellular_lives_v1,
|
|
"reverse transition changed biology or time");
|
|
f.raw = std::move(recovered.state);
|
|
}
|
|
void missing_and_corrupt_objects(Fixture& f) {
|
|
const auto manifest = take(decode_manifest(f.head.manifest), "object test manifest");
|
|
const auto path = f.objects->directory() / hexadecimal(manifest.payload_sha256, false);
|
|
const auto backup = path.string() + ".population-test";
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
std::filesystem::rename(path, backup);
|
|
auto missing = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
|
|
std::filesystem::rename(backup, path);
|
|
require(!missing && missing.error().kind == RecoveryError::missing_payload,
|
|
"missing population payload did not fail closed");
|
|
std::filesystem::rename(path, backup);
|
|
{
|
|
std::ofstream corrupt(path, std::ios::binary);
|
|
corrupt << 'X';
|
|
}
|
|
auto corrupt = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
|
|
std::filesystem::remove(path);
|
|
std::filesystem::rename(backup, path);
|
|
require(!corrupt && corrupt.error().kind == RecoveryError::corrupt_payload,
|
|
"corrupt population payload did not fail closed");
|
|
recover(f);
|
|
}
|
|
void host_ownership_and_admission(Fixture& f) {
|
|
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture host authority");
|
|
const auto make_host = [&](bool enabled) {
|
|
auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), f.key,
|
|
max_cellular_population_bytes),
|
|
"host objects");
|
|
CellularHostConfig config{f.key, 1, {}, f.build, {f.build}};
|
|
config.lease.commit_timeout = 300ms;
|
|
if (enabled)
|
|
config.population_policy = CellularTierPolicy{true, 1200, 1800};
|
|
return take(CellularRegionHost::create(
|
|
std::move(config),
|
|
[] {
|
|
return Postgres::connect({environment("PGHOST"),
|
|
{},
|
|
environment("PGPORT"),
|
|
environment("PGDATABASE"),
|
|
environment("EVO_PG_RUNTIME_USER"),
|
|
{},
|
|
{}},
|
|
5s);
|
|
},
|
|
std::move(objects)),
|
|
"population host");
|
|
};
|
|
auto host = make_host(false);
|
|
require(host->step().ready && !host->population_mode(), "raw host recovery");
|
|
require(!host->set_population_mode(true), "default host enabled population entry");
|
|
host->drain();
|
|
host = make_host(true);
|
|
require(host->step().ready, "opt-in host recovery");
|
|
const auto before = host->state()->snapshot();
|
|
const auto tick = host->state()->tick();
|
|
take(host->set_population_mode(true), "host population entry");
|
|
require(host->population_mode() && host->state()->snapshot() == before,
|
|
"host entry changed biology");
|
|
auto population =
|
|
take(CellularPopulation::create(
|
|
take(CellularLives::restore(before, f.key.world, f.key.region, 1, tick),
|
|
"host reference restore"),
|
|
{true, 1200, 1800}),
|
|
"host reference population");
|
|
f.head = take(host->committed(), "host population head");
|
|
auto next = unattended(f); // uses the same deterministic frozen/wildlife input contract.
|
|
// The fixture's old envelope may have a different membership history; biology is shared.
|
|
auto expected =
|
|
take(prepare_population_batch(*population, next.batch.biology), "host reference batch");
|
|
require(!take(host->advance_unattended(19), "collect bounded publication window").has_value() &&
|
|
host->state()->snapshot() == before,
|
|
"small local debt was admitted prematurely");
|
|
const auto published = take(host->advance_unattended(65), "host population publication");
|
|
require(published.has_value(), "host skipped admitted interval");
|
|
require(host->population_mode() &&
|
|
host->state()->snapshot() == expected.state->state().snapshot(),
|
|
"host population continuation differs");
|
|
const auto committed = take(host->committed(), "host continued head");
|
|
const auto continued = host->state()->snapshot();
|
|
const CellularBatchHeader recovered_header{f.key.world,
|
|
f.key.region,
|
|
1,
|
|
committed.head.commit,
|
|
committed.head.digest,
|
|
committed.head.end_tick,
|
|
committed.head.end_tick + 64};
|
|
auto recovered_expected = take(
|
|
prepare_population_batch(
|
|
*expected.state, take(unattended_population_inputs(*host->state(), recovered_header),
|
|
"recovered interval inputs")),
|
|
"recovered interval reference");
|
|
take(admit_population_interval(*f.db, f.key, take(host->authority(), "host interval authority"),
|
|
committed.head, committed.head.end_tick + 64, 5s),
|
|
"admit before host loss");
|
|
host->drain();
|
|
host = make_host(false);
|
|
require(host->step().ready && host->population_mode() &&
|
|
take(host->committed(), "host recovered head").head == committed.head &&
|
|
host->state()->snapshot() == continued,
|
|
"restarted host failed to recover population head");
|
|
require(host->status().ecological_pending_ticks == 64 && !host->set_population_mode(false),
|
|
"restarted host hid or skipped admitted work");
|
|
require(take(host->advance_unattended(0), "resume admitted work with zero new due time")
|
|
.has_value() &&
|
|
host->state()->snapshot() == recovered_expected.state->state().snapshot() &&
|
|
host->status().ecological_pending_ticks == 0,
|
|
"recovered interval did not settle exactly");
|
|
const auto recovered_head = take(host->committed(), "recovered interval head");
|
|
const auto recovered_bytes = host->state()->snapshot();
|
|
const CellularBatchHeader unknown_header{f.key.world,
|
|
f.key.region,
|
|
1,
|
|
recovered_head.head.commit,
|
|
recovered_head.head.digest,
|
|
recovered_head.head.end_tick,
|
|
recovered_head.head.end_tick + 64};
|
|
auto unknown_expected = take(
|
|
prepare_population_batch(*recovered_expected.state,
|
|
take(unattended_population_inputs(*host->state(), unknown_header),
|
|
"uncertain admission inputs")),
|
|
"uncertain admission reference");
|
|
auto locker = connection(true);
|
|
take(locker->query("BEGIN", {}, 5s), "host admission lock transaction");
|
|
take(
|
|
locker->query(
|
|
"SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE",
|
|
{}, 5s),
|
|
"hold host admission reply");
|
|
require(!host->advance_unattended(64) && !host->status().ready &&
|
|
host->state()->snapshot() == recovered_bytes,
|
|
"uncertain admission allowed simulation/output");
|
|
take(locker->query("ROLLBACK", {}, 5s), "release host admission reply");
|
|
for (unsigned attempt = 0; attempt < 200 && !host->step().ready; ++attempt)
|
|
std::this_thread::sleep_for(25ms);
|
|
require(host->status().ready && host->status().ecological_pending_ticks == 64,
|
|
"unknown admission failed exact reconciliation");
|
|
require(take(host->advance_unattended(0), "finish reconciled admission").has_value() &&
|
|
host->state()->snapshot() == unknown_expected.state->state().snapshot(),
|
|
"admission retry chose different work");
|
|
const auto final_bytes = host->state()->snapshot();
|
|
Uuid account{};
|
|
account.back() = 1;
|
|
// A returning player's operation completes the durable reverse transition before lookup.
|
|
[[maybe_unused]] const auto queried = host->query_primitive_life(account, 1, 1);
|
|
require(!host->population_mode() && host->state()->snapshot() == final_bytes &&
|
|
host->state()->tick() == tick + 192,
|
|
"player operation did not materialize first");
|
|
host->drain();
|
|
}
|
|
} // namespace
|
|
void run_population_publication_tests() {
|
|
Fixture fixture;
|
|
funded_source(fixture);
|
|
conversions_and_retries(fixture);
|
|
interval_contract(fixture);
|
|
ambiguous_publication(fixture, false);
|
|
ambiguous_publication(fixture, true);
|
|
process_loss(fixture, 0);
|
|
process_loss(fixture, 1);
|
|
process_loss(fixture, 2);
|
|
missing_and_corrupt_objects(fixture);
|
|
isolation_and_reverse(fixture);
|
|
host_ownership_and_admission(fixture);
|
|
std::puts("population publication, exact recovery, process loss and fencing passed");
|
|
}
|