Foundation / checks (ecs) (push) Successful in 2m41s
Foundation / checks (tsan) (push) Failing after 18m12s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (network) (push) Canceled after 20m41s
Foundation / checks (postgres) (push) Canceled after 20m43s
Foundation / checks (checks) (push) Canceled after 20m49s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (client) (push) Failing after 17m4s
Foundation / postgres-tools (push) Canceled after 0s
487 lines
23 KiB
C++
487 lines
23 KiB
C++
#include "evo/persistence/account_control_store.hpp"
|
|
#include "evo/persistence/population_checkpoint_store.hpp"
|
|
#include "evo/persistence/sha256.hpp"
|
|
#include "evo/server/cellular_region_host.hpp"
|
|
#include "fixtures/cellular_ecology.hpp"
|
|
|
|
#include <csignal>
|
|
#include <cstdlib>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <stdexcept>
|
|
#include <sys/wait.h>
|
|
#include <thread>
|
|
#include <type_traits>
|
|
#include <unistd.h>
|
|
|
|
void run_population_publication_tests();
|
|
namespace {
|
|
using namespace evo;
|
|
using namespace evo::server;
|
|
using namespace evo::persistence;
|
|
using namespace evo::fixture;
|
|
using namespace std::chrono_literals;
|
|
template <typename T, typename E> T take(std::expected<T, E> value, const char* where) {
|
|
if (!value) {
|
|
std::string message{where};
|
|
if constexpr (requires { value.error().cause.sqlstate; })
|
|
message += " SQLSTATE=" + std::string(value.error().cause.sqlstate.data(), 5);
|
|
if constexpr (requires { value.error().store.cause.sqlstate; })
|
|
message += " SQLSTATE=" + std::string(value.error().store.cause.sqlstate.data(), 5);
|
|
throw std::runtime_error(message);
|
|
}
|
|
if constexpr (!std::is_void_v<T>)
|
|
return std::move(*value);
|
|
}
|
|
template <typename T> T take(std::optional<T> value, const char* where) {
|
|
if (!value.has_value())
|
|
throw std::runtime_error(where);
|
|
return std::move(*value);
|
|
}
|
|
std::string environment(const char* name) {
|
|
const auto* value = std::getenv(name);
|
|
require(value != nullptr, "missing private test environment");
|
|
return value;
|
|
}
|
|
std::unique_ptr<Postgres> connection(bool owner = false) {
|
|
return take(Postgres::connect({environment("PGHOST"),
|
|
{},
|
|
environment("PGPORT"),
|
|
environment("PGDATABASE"),
|
|
environment(owner ? "PGUSER" : "EVO_PG_RUNTIME_USER"),
|
|
{},
|
|
{}},
|
|
5s),
|
|
"population connection");
|
|
}
|
|
Uuid identity(unsigned number) {
|
|
Uuid id{};
|
|
id[0] = 0xDA;
|
|
id[1] = 0x12;
|
|
id[14] = static_cast<std::uint8_t>(number >> 8U);
|
|
id[15] = static_cast<std::uint8_t>(number);
|
|
return id;
|
|
}
|
|
std::string hexadecimal(std::span<const std::uint8_t> bytes, bool prefix = true) {
|
|
std::string out = prefix ? "\\x" : "";
|
|
constexpr char digits[] = "0123456789abcdef";
|
|
for (const auto b : bytes) {
|
|
out.push_back(digits[b >> 4U]);
|
|
out.push_back(digits[b & 15U]);
|
|
}
|
|
return out;
|
|
}
|
|
struct Fixture {
|
|
RegionKey key{30201, 1};
|
|
std::unique_ptr<Postgres> owner = connection(true), db = connection();
|
|
std::unique_ptr<FilesystemObjectStore> objects;
|
|
LeaseGrant lease;
|
|
Sha256Digest build{};
|
|
CommittedCheckpoint head;
|
|
std::unique_ptr<CellularLives> raw;
|
|
std::unique_ptr<CellularPopulation> population;
|
|
unsigned next_id = 1;
|
|
Uuid id() {
|
|
return identity(next_id++);
|
|
}
|
|
CellularBatchHeader header(std::uint64_t count) const {
|
|
return {key.world,
|
|
key.region,
|
|
1,
|
|
head.head.commit,
|
|
head.head.digest,
|
|
head.head.end_tick,
|
|
head.head.end_tick + count};
|
|
}
|
|
};
|
|
void funded_source(Fixture& f) {
|
|
f.build.fill(0xDA);
|
|
const std::array<std::string, 1> build{hexadecimal(f.build)};
|
|
take(f.owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES($1::bytea) "
|
|
"ON CONFLICT DO NOTHING",
|
|
build, 5s),
|
|
"retain test build");
|
|
take(f.owner->query("INSERT INTO evolution.life_allocators VALUES(30201,100)", {}, 5s),
|
|
"test allocator");
|
|
f.objects = take(
|
|
FilesystemObjectStore::open(environment("PGHOST"), f.key, max_cellular_population_bytes),
|
|
"population object namespace");
|
|
f.raw = opening(30001, true, 4, false);
|
|
f.head =
|
|
take(provision_cellular_region(*f.owner, *f.objects, f.key, 1, *f.raw, f.id(), f.build, 5s),
|
|
"empty cellular genesis");
|
|
f.lease =
|
|
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "population test lease");
|
|
CellularBatch birth{f.header(1), {{}}};
|
|
for (std::uint8_t index = 1; index <= 4; ++index) {
|
|
Uuid account{};
|
|
account.back() = index;
|
|
const auto control =
|
|
take(claim_account_control(*f.db, f.key, f.lease, account, 1, index, 60000ms, 5s),
|
|
"founder account control");
|
|
const auto reserved = take(
|
|
reserve_primitive_life(*f.db, f.key, f.lease, account, control.generation, 1, 1, 5s),
|
|
"funded founder reservation");
|
|
birth.ticks.front().births.push_back({account, 1, control.generation, reserved.identity});
|
|
}
|
|
f.raw = take(replay_cellular_batch(*f.raw, birth), "funded founder replay");
|
|
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, birth, *f.raw, f.id(),
|
|
f.build, 5s),
|
|
"funded founder publication")
|
|
.committed;
|
|
while (f.raw->next_bud_sequence() == 1 && f.raw->tick() < 20000) {
|
|
CellularBatch batch{f.header(1), {}};
|
|
for (unsigned tick = 0; tick < 64; ++tick) {
|
|
auto intents =
|
|
take(wildlife_intents(*f.raw, f.raw->tick() + 1), "funded source wildlife");
|
|
for (const auto& life : f.raw->lives()) {
|
|
if (life.account == LifeAccount{})
|
|
continue;
|
|
CellularLifeIntent intent{life.identity.life, {}};
|
|
intent.intent.bud = (f.raw->tick() + life.identity.life) % 32 == 0;
|
|
intents.push_back(intent);
|
|
}
|
|
std::ranges::sort(intents, {}, &CellularLifeIntent::life_id);
|
|
take(f.raw->advance_tick(intents), "funded source biology");
|
|
batch.ticks.push_back({intents, {}, {}});
|
|
if (f.raw->next_bud_sequence() > 1)
|
|
break;
|
|
}
|
|
batch.header.end_tick = f.raw->tick();
|
|
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, batch, *f.raw,
|
|
f.id(), f.build, 5s),
|
|
"funded source batch")
|
|
.committed;
|
|
take(renew_region_lease(*f.db, f.key, f.lease, 60000ms, 5s), "source lease renewal");
|
|
}
|
|
require(std::ranges::any_of(f.raw->lives(),
|
|
[](const auto& life) { return life.account == LifeAccount{}; }),
|
|
"durable fixture has no funded player-derived wildlife");
|
|
}
|
|
CellularPopulationExecution unattended(Fixture& f) {
|
|
auto working = restore(f.population->state());
|
|
CellularBatch batch{f.header(64), {}};
|
|
for (unsigned tick = 0; tick < 64; ++tick) {
|
|
auto intents = take(wildlife_intents(*working, working->tick() + 1), "unattended wildlife");
|
|
for (const auto& life : working->lives()) {
|
|
if (life.account == LifeAccount{})
|
|
continue;
|
|
CellularLifeIntent frozen{life.identity.life, {}};
|
|
frozen.intent.frozen = true;
|
|
intents.push_back(frozen);
|
|
}
|
|
std::ranges::sort(intents, {}, &CellularLifeIntent::life_id);
|
|
take(working->advance_tick(intents), "unattended fixture step");
|
|
batch.ticks.push_back({intents, {}, {}});
|
|
}
|
|
return take(prepare_population_batch(*f.population, std::move(batch)),
|
|
"population recorded interval");
|
|
}
|
|
void recover(Fixture& f) {
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
auto db = connection();
|
|
auto recovered =
|
|
take(recover_population_checkpoint(*db, *f.objects, f.key, f.lease, builds, 5s),
|
|
"population durable recovery");
|
|
f.head = std::move(recovered.committed);
|
|
f.population = std::move(recovered.state);
|
|
}
|
|
void conversions_and_retries(Fixture& f) {
|
|
const auto biology = f.raw->snapshot();
|
|
Uuid account{};
|
|
account.back() = 1;
|
|
const auto input =
|
|
encode_cellular_mutation_input({account, 9991, f.head.head.end_tick + 1, {100, 1, 1, 1}});
|
|
take(f.db->query("BEGIN", {}, 5s), "reservation guard transaction");
|
|
take(reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 9991, input, 5s),
|
|
"reserve pending player choice");
|
|
const auto blocked = migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
|
|
f.build, {true, 1200, 1800}, 5s);
|
|
require(!blocked && std::string(blocked.error().store.cause.sqlstate.data(), 5) == "EVL18",
|
|
"population entry stranded a reserved player choice");
|
|
take(f.db->query("ROLLBACK", {}, 5s), "rollback guard fixture");
|
|
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
|
|
f.build, {true, 1200, 1800}, 5s),
|
|
"initial population transition");
|
|
recover(f);
|
|
require(f.population->state().snapshot() == biology && f.population->members().empty(),
|
|
"transition changed biology");
|
|
const auto refused =
|
|
reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 9991, input, 5s);
|
|
require(!refused && std::string(refused.error().cause.sqlstate.data(), 5) == "EVL06",
|
|
"population profile admitted a player mutation reservation");
|
|
const auto wildlife = std::ranges::find_if(f.population->state().lives(), [](const auto& life) {
|
|
return life.account == LifeAccount{};
|
|
});
|
|
require(wildlife != f.population->state().lives().end(), "missing durable descendant");
|
|
const auto genome = wildlife->identity.genome;
|
|
auto conversion =
|
|
take(prepare_population_conversion(*f.population, f.header(0), {genome, true}),
|
|
"collapse sample");
|
|
auto bad = conversion.batch;
|
|
++bad.boundaries.front().sequence;
|
|
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, bad, *conversion.state,
|
|
f.id(), f.build, 5s),
|
|
"altered sample published");
|
|
const auto commit = f.id();
|
|
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, conversion.batch,
|
|
*conversion.state, commit, f.build, 5s),
|
|
"durable collapse");
|
|
const PopulationPreparedCommit prepared{
|
|
f.key, commit, f.head.manifest,
|
|
take(encode_population_batch(conversion.batch), "prepared trace")};
|
|
require(take(retry_population_prepared(*f.db, prepared, 5s), "idempotent retry").head ==
|
|
f.head.head,
|
|
"retry advanced head");
|
|
require(take(resolve_population_prepared(*f.db, prepared, 5s), "read-only resolve").status ==
|
|
Resolution::committed,
|
|
"commit unresolved");
|
|
auto changed = prepared;
|
|
auto changed_manifest = take(decode_manifest(changed.manifest), "conflict manifest");
|
|
changed_manifest.payload_sha256[0] ^= 1;
|
|
changed.manifest = take(encode_manifest(changed_manifest), "conflict encoding");
|
|
require(take(resolve_population_prepared(*f.db, changed, 5s), "conflicting resolve").status ==
|
|
Resolution::conflict,
|
|
"changed commit was accepted");
|
|
require(!publish_region_checkpoint(*f.db, f.head.manifest, {}, 5s),
|
|
"generic publisher bypassed profile gate");
|
|
const auto expected = conversion.state->snapshot();
|
|
recover(f);
|
|
require(f.population->snapshot() == expected && !f.population->members().empty(),
|
|
"committed sample not recovered");
|
|
require(f.population->state().snapshot() == biology, "collapse healed/aged/moved biology");
|
|
for (unsigned oscillation = 0; oscillation < 8; ++oscillation) {
|
|
const bool collapse = oscillation % 2 != 0;
|
|
auto next =
|
|
take(prepare_population_conversion(*f.population, f.header(0), {genome, collapse}),
|
|
"conversion oscillation");
|
|
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, next.batch,
|
|
*next.state, f.id(), f.build, 5s),
|
|
"publish oscillation");
|
|
recover(f);
|
|
require(f.population->state().snapshot() == biology,
|
|
"durable conversion changed exact state");
|
|
}
|
|
}
|
|
void process_loss(Fixture& f, int point) {
|
|
auto next = unattended(f);
|
|
const auto predecessor = f.head.head;
|
|
const auto expected = next.state->snapshot();
|
|
const auto commit = f.id();
|
|
int channel[2]{};
|
|
require(::pipe(channel) == 0, "process acknowledgement pipe");
|
|
const auto child = ::fork();
|
|
require(child >= 0, "population fork");
|
|
if (child == 0) {
|
|
::close(channel[0]);
|
|
try {
|
|
auto db = connection();
|
|
PublishHooks hooks;
|
|
if (point == 0)
|
|
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
|
|
const auto published =
|
|
publish_population_batch(*db, *f.objects, f.lease, f.head, next.batch, *next.state,
|
|
commit, f.build, 5s, hooks);
|
|
if (!published)
|
|
::_exit(20);
|
|
if (point == 2) {
|
|
const char ack = 'A';
|
|
if (::write(channel[1], &ack, 1) != 1)
|
|
::_exit(21);
|
|
}
|
|
::kill(::getpid(), SIGKILL);
|
|
} catch (...) {
|
|
::_exit(22);
|
|
}
|
|
::_exit(23);
|
|
}
|
|
::close(channel[1]);
|
|
char ack{};
|
|
const auto acknowledged = ::read(channel[0], &ack, 1);
|
|
::close(channel[0]);
|
|
int status{};
|
|
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
|
|
WTERMSIG(status) == SIGKILL,
|
|
"child missed population crash boundary");
|
|
require((point == 2 && acknowledged == 1 && ack == 'A') || (point != 2 && acknowledged == 0),
|
|
"unexpected permanent acknowledgement");
|
|
recover(f);
|
|
if (point == 0)
|
|
require(f.head.head == predecessor, "uncommitted process published a head");
|
|
else
|
|
require(f.head.head.commit == commit && f.population->snapshot() == expected,
|
|
"committed process loss changed state");
|
|
}
|
|
void ambiguous_publication(Fixture& f, bool terminate) {
|
|
auto next = unattended(f);
|
|
const auto expected = next.state->snapshot();
|
|
auto locker = connection(true);
|
|
take(locker->query("BEGIN", {}, 5s), "population lease row transaction");
|
|
const std::array<std::string, 2> key{std::to_string(f.key.world), std::to_string(f.key.region)};
|
|
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=$1::evolution.u64 "
|
|
"AND region_id=$2::evolution.u64 FOR UPDATE",
|
|
key, 5s),
|
|
"hold population publication");
|
|
auto timed = connection();
|
|
auto unknown = publish_population_batch(*timed, *f.objects, f.lease, f.head, next.batch,
|
|
*next.state, f.id(), f.build, 300ms);
|
|
require(!unknown && unknown.error().kind == PublicationError::unknown,
|
|
"population publication failed to retain ambiguous request");
|
|
if (terminate) {
|
|
const std::array<std::string, 1> user{environment("EVO_PG_RUNTIME_USER")};
|
|
take(locker->query("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE "
|
|
"usename=$1 AND wait_event_type='Lock'",
|
|
user, 5s),
|
|
"terminate uncommitted population writer");
|
|
}
|
|
take(locker->query("ROLLBACK", {}, 5s), "release population publication");
|
|
auto resolver = connection();
|
|
bool resolved = false;
|
|
for (unsigned attempt = 0; attempt < 200; ++attempt) {
|
|
auto result = take(resolve_population_prepared(*resolver, unknown.error().prepared, 5s),
|
|
"resolve ambiguous population publication");
|
|
if (result.status == (terminate ? Resolution::absent : Resolution::committed)) {
|
|
resolved = true;
|
|
break;
|
|
}
|
|
std::this_thread::sleep_for(25ms);
|
|
}
|
|
require(resolved, "ambiguous population publication did not resolve");
|
|
f.head = take(retry_population_prepared(*resolver, unknown.error().prepared, 5s),
|
|
"retry exact population request");
|
|
recover(f);
|
|
require(f.population->snapshot() == expected, "ambiguous publication changed retained sample");
|
|
}
|
|
void isolation_and_reverse(Fixture& f) {
|
|
auto interval = unattended(f);
|
|
const auto old_lease = f.lease;
|
|
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release old authority");
|
|
auto new_db = connection();
|
|
f.lease = take(acquire_region_lease(*new_db, f.key, f.build, 60000ms, 5s),
|
|
"new population authority");
|
|
require(f.lease.epoch > old_lease.epoch, "authority epoch did not advance");
|
|
const auto old_head = f.head.head;
|
|
auto stale = publish_population_batch(*f.db, *f.objects, old_lease, f.head, interval.batch,
|
|
*interval.state, f.id(), f.build, 5s);
|
|
require(!stale && stale.error().store.kind == LeaseStoreError::fenced,
|
|
"stale population writer published");
|
|
recover(f);
|
|
require(f.head.head == old_head, "fenced writer changed head");
|
|
const auto expected = f.population->state().snapshot();
|
|
f.head = take(
|
|
materialize_population_profile(*new_db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
|
|
"return to cellular profile");
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
auto recovered =
|
|
take(recover_cellular_checkpoint(*new_db, *f.objects, f.key, f.lease, builds, 5s),
|
|
"cellular recovery after materialization");
|
|
require(recovered.state->snapshot() == expected &&
|
|
recovered.manifest.profile == CheckpointProfile::cellular_lives_v1,
|
|
"reverse transition changed biology or time");
|
|
f.raw = std::move(recovered.state);
|
|
}
|
|
void missing_and_corrupt_objects(Fixture& f) {
|
|
const auto manifest = take(decode_manifest(f.head.manifest), "object test manifest");
|
|
const auto path = f.objects->directory() / hexadecimal(manifest.payload_sha256, false);
|
|
const auto backup = path.string() + ".population-test";
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
std::filesystem::rename(path, backup);
|
|
auto missing = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
|
|
std::filesystem::rename(backup, path);
|
|
require(!missing && missing.error().kind == RecoveryError::missing_payload,
|
|
"missing population payload did not fail closed");
|
|
std::filesystem::rename(path, backup);
|
|
{
|
|
std::ofstream corrupt(path, std::ios::binary);
|
|
corrupt << 'X';
|
|
}
|
|
auto corrupt = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
|
|
std::filesystem::remove(path);
|
|
std::filesystem::rename(backup, path);
|
|
require(!corrupt && corrupt.error().kind == RecoveryError::corrupt_payload,
|
|
"corrupt population payload did not fail closed");
|
|
recover(f);
|
|
}
|
|
void host_ownership_and_admission(Fixture& f) {
|
|
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture host authority");
|
|
const auto make_host = [&](bool enabled) {
|
|
auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), f.key,
|
|
max_cellular_population_bytes),
|
|
"host objects");
|
|
CellularHostConfig config{f.key, 1, {}, f.build, {f.build}};
|
|
if (enabled)
|
|
config.population_policy = CellularTierPolicy{true, 1200, 1800};
|
|
return take(CellularRegionHost::create(
|
|
std::move(config),
|
|
[] {
|
|
return Postgres::connect({environment("PGHOST"),
|
|
{},
|
|
environment("PGPORT"),
|
|
environment("PGDATABASE"),
|
|
environment("EVO_PG_RUNTIME_USER"),
|
|
{},
|
|
{}},
|
|
5s);
|
|
},
|
|
std::move(objects)),
|
|
"population host");
|
|
};
|
|
auto host = make_host(false);
|
|
require(host->step().ready && !host->population_mode(), "raw host recovery");
|
|
require(!host->set_population_mode(true), "default host enabled population entry");
|
|
host->drain();
|
|
host = make_host(true);
|
|
require(host->step().ready, "opt-in host recovery");
|
|
const auto before = host->state()->snapshot();
|
|
const auto tick = host->state()->tick();
|
|
take(host->set_population_mode(true), "host population entry");
|
|
require(host->population_mode() && host->state()->snapshot() == before,
|
|
"host entry changed biology");
|
|
auto population =
|
|
take(CellularPopulation::create(
|
|
take(CellularLives::restore(before, f.key.world, f.key.region, 1, tick),
|
|
"host reference restore"),
|
|
{true, 1200, 1800}),
|
|
"host reference population");
|
|
f.head = take(host->committed(), "host population head");
|
|
auto next = unattended(f); // uses the same deterministic frozen/wildlife input contract.
|
|
// The fixture's old envelope may have a different membership history; biology is shared.
|
|
auto expected =
|
|
take(prepare_population_batch(*population, next.batch.biology), "host reference batch");
|
|
take(host->publish(next.batch.biology), "host population publication");
|
|
require(host->population_mode() &&
|
|
host->state()->snapshot() == expected.state->state().snapshot(),
|
|
"host population continuation differs");
|
|
const auto committed = take(host->committed(), "host continued head");
|
|
const auto continued = host->state()->snapshot();
|
|
host->drain();
|
|
host = make_host(false);
|
|
require(host->step().ready && host->population_mode() &&
|
|
take(host->committed(), "host recovered head").head == committed.head &&
|
|
host->state()->snapshot() == continued,
|
|
"restarted host failed to recover population head");
|
|
Uuid account{};
|
|
account.back() = 1;
|
|
// A returning player's operation completes the durable reverse transition before lookup.
|
|
[[maybe_unused]] const auto queried = host->query_primitive_life(account, 1, 1);
|
|
require(!host->population_mode() && host->state()->snapshot() == continued &&
|
|
host->state()->tick() == tick + 64,
|
|
"player operation did not materialize first");
|
|
host->drain();
|
|
}
|
|
} // namespace
|
|
void run_population_publication_tests() {
|
|
Fixture fixture;
|
|
funded_source(fixture);
|
|
conversions_and_retries(fixture);
|
|
ambiguous_publication(fixture, false);
|
|
ambiguous_publication(fixture, true);
|
|
process_loss(fixture, 0);
|
|
process_loss(fixture, 1);
|
|
process_loss(fixture, 2);
|
|
missing_and_corrupt_objects(fixture);
|
|
isolation_and_reverse(fixture);
|
|
host_ownership_and_admission(fixture);
|
|
std::puts("population publication, exact recovery, process loss and fencing passed");
|
|
}
|