Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m21s
Foundation / checks (network) (push) Successful in 20m51s
Foundation / checks (client) (push) Successful in 22m2s
Foundation / checks (tsan) (push) Successful in 22m59s
Foundation / checks (postgres) (push) Successful in 53m40s
Foundation / checks (checks) (push) Successful in 1h7m19s
Foundation / postgres-tools (push) Successful in 29s
Foundation / server-image (push) Successful in 38s
Foundation / image (push) Successful in 4m31s
1953 lines
111 KiB
C++
1953 lines
111 KiB
C++
#include "evo/persistence/account_control_store.hpp"
|
|
#include "evo/persistence/population_checkpoint_store.hpp"
|
|
#include "evo/persistence/population_interval_store.hpp"
|
|
#include "evo/persistence/sha256.hpp"
|
|
#include "evo/server/cellular_region_host.hpp"
|
|
#include "fixtures/cellular_ecology.hpp"
|
|
|
|
#include <barrier>
|
|
#include <csignal>
|
|
#include <cstdlib>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <stdexcept>
|
|
#include <sys/wait.h>
|
|
#include <thread>
|
|
#include <type_traits>
|
|
#include <unistd.h>
|
|
|
|
void run_population_publication_tests();
|
|
namespace {
|
|
using namespace evo;
|
|
using namespace evo::server;
|
|
using namespace evo::persistence;
|
|
using namespace evo::fixture;
|
|
using namespace std::chrono_literals;
|
|
template <typename T, typename E> T take(std::expected<T, E> value, const char* where) {
|
|
if (!value) {
|
|
std::string message{where};
|
|
if constexpr (requires { value.error().cause.sqlstate; })
|
|
message += " SQLSTATE=" + std::string(value.error().cause.sqlstate.data(), 5);
|
|
if constexpr (requires { value.error().store.cause.sqlstate; })
|
|
message += " SQLSTATE=" + std::string(value.error().store.cause.sqlstate.data(), 5);
|
|
throw std::runtime_error(message);
|
|
}
|
|
if constexpr (!std::is_void_v<T>)
|
|
return std::move(*value);
|
|
}
|
|
template <typename T> T take(std::optional<T> value, const char* where) {
|
|
if (!value.has_value())
|
|
throw std::runtime_error(where);
|
|
return std::move(*value);
|
|
}
|
|
std::string environment(const char* name) {
|
|
const auto* value = std::getenv(name);
|
|
require(value != nullptr, "missing private test environment");
|
|
return value;
|
|
}
|
|
std::unique_ptr<Postgres> connection(bool owner = false) {
|
|
return take(Postgres::connect({environment("PGHOST"),
|
|
{},
|
|
environment("PGPORT"),
|
|
environment("PGDATABASE"),
|
|
environment(owner ? "PGUSER" : "EVO_PG_RUNTIME_USER"),
|
|
{},
|
|
{}},
|
|
5s),
|
|
"population connection");
|
|
}
|
|
Uuid identity(unsigned number) {
|
|
Uuid id{};
|
|
id[0] = 0xDA;
|
|
id[1] = 0x12;
|
|
id[14] = static_cast<std::uint8_t>(number >> 8U);
|
|
id[15] = static_cast<std::uint8_t>(number);
|
|
return id;
|
|
}
|
|
std::string hexadecimal(std::span<const std::uint8_t> bytes, bool prefix = true) {
|
|
std::string out = prefix ? "\\x" : "";
|
|
constexpr char digits[] = "0123456789abcdef";
|
|
for (const auto b : bytes) {
|
|
out.push_back(digits[b >> 4U]);
|
|
out.push_back(digits[b & 15U]);
|
|
}
|
|
return out;
|
|
}
|
|
struct Fixture {
|
|
RegionKey key{30201, 1};
|
|
std::unique_ptr<Postgres> owner = connection(true), db = connection();
|
|
std::unique_ptr<FilesystemObjectStore> objects;
|
|
LeaseGrant lease;
|
|
Sha256Digest build{};
|
|
CommittedCheckpoint head;
|
|
std::unique_ptr<CellularLives> raw;
|
|
std::unique_ptr<CellularPopulation> population;
|
|
unsigned next_id = 1;
|
|
Uuid id() {
|
|
return identity(next_id++);
|
|
}
|
|
CellularBatchHeader header(std::uint64_t count) const {
|
|
return {key.world,
|
|
key.region,
|
|
1,
|
|
head.head.commit,
|
|
head.head.digest,
|
|
head.head.end_tick,
|
|
head.head.end_tick + count};
|
|
}
|
|
};
|
|
void funded_source(Fixture& f) {
|
|
f.build.fill(0xDA);
|
|
const std::array<std::string, 1> build{hexadecimal(f.build)};
|
|
take(f.owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES($1::bytea) "
|
|
"ON CONFLICT DO NOTHING",
|
|
build, 5s),
|
|
"retain test build");
|
|
take(f.owner->query("INSERT INTO evolution.life_allocators VALUES(30201,100)", {}, 5s),
|
|
"test allocator");
|
|
f.objects = take(
|
|
FilesystemObjectStore::open(environment("PGHOST"), f.key, max_cellular_population_bytes),
|
|
"population object namespace");
|
|
f.raw = opening(30001, true, 4, false);
|
|
f.head =
|
|
take(provision_cellular_region(*f.owner, *f.objects, f.key, 1, *f.raw, f.id(), f.build, 5s),
|
|
"empty cellular genesis");
|
|
f.lease =
|
|
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "population test lease");
|
|
CellularBatch birth{f.header(1), {{}}};
|
|
for (std::uint8_t index = 1; index <= 4; ++index) {
|
|
Uuid account{};
|
|
account.back() = index;
|
|
const auto control =
|
|
take(claim_account_control(*f.db, f.key, f.lease, account, 1, index, 60000ms, 5s),
|
|
"founder account control");
|
|
const auto reserved = take(
|
|
reserve_primitive_life(*f.db, f.key, f.lease, account, control.generation, 1, 1, 5s),
|
|
"funded founder reservation");
|
|
birth.ticks.front().births.push_back({account, 1, control.generation, reserved.identity});
|
|
}
|
|
f.raw = take(replay_cellular_batch(*f.raw, birth), "funded founder replay");
|
|
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, birth, *f.raw, f.id(),
|
|
f.build, 5s),
|
|
"funded founder publication")
|
|
.committed;
|
|
while (f.raw->next_bud_sequence() == 1 && f.raw->tick() < 20000) {
|
|
CellularBatch batch{f.header(1), {}};
|
|
for (unsigned tick = 0; tick < 64; ++tick) {
|
|
auto intents =
|
|
take(wildlife_intents(*f.raw, f.raw->tick() + 1), "funded source wildlife");
|
|
for (const auto& life : f.raw->lives()) {
|
|
if (life.account == LifeAccount{})
|
|
continue;
|
|
CellularLifeIntent intent{life.identity.life, {}};
|
|
intent.intent.bud = (f.raw->tick() + life.identity.life) % 32 == 0;
|
|
intents.push_back(intent);
|
|
}
|
|
std::ranges::sort(intents, {}, &CellularLifeIntent::life_id);
|
|
take(f.raw->advance_tick(intents), "funded source biology");
|
|
batch.ticks.push_back({intents, {}, {}});
|
|
if (f.raw->next_bud_sequence() > 1)
|
|
break;
|
|
}
|
|
batch.header.end_tick = f.raw->tick();
|
|
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, batch, *f.raw,
|
|
f.id(), f.build, 5s),
|
|
"funded source batch")
|
|
.committed;
|
|
take(renew_region_lease(*f.db, f.key, f.lease, 60000ms, 5s), "source lease renewal");
|
|
}
|
|
require(std::ranges::any_of(f.raw->lives(),
|
|
[](const auto& life) { return life.account == LifeAccount{}; }),
|
|
"durable fixture has no funded player-derived wildlife");
|
|
require(!migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build,
|
|
{true, 1200, 1800}, 5s),
|
|
"population entry bypassed vulnerable controlled life");
|
|
const auto suspended_at = f.raw->tick() + cellular_linger_ticks + 1;
|
|
while (f.raw->tick() < suspended_at) {
|
|
auto h = f.header(std::min(std::uint64_t{64}, suspended_at - f.raw->tick()));
|
|
auto prefix = take(unattended_population_inputs(*f.raw, h), "raw linger inputs");
|
|
f.raw = take(replay_cellular_batch(*f.raw, prefix), "raw linger biology");
|
|
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, prefix, *f.raw,
|
|
f.id(), f.build, 5s),
|
|
"raw linger publication")
|
|
.committed;
|
|
std::vector<std::uint64_t> endings;
|
|
for (const auto& corpse : f.raw->corpses())
|
|
if (corpse.parent_life == 0)
|
|
endings.push_back(corpse.life_id);
|
|
if (!endings.empty())
|
|
take(end_cellular_lives(*f.db, f.key, f.lease, f.head.head.commit, endings, 5s),
|
|
"raw linger endings");
|
|
}
|
|
require(population_interval_safe(*f.raw),
|
|
"population source retained a vulnerable controlled life");
|
|
}
|
|
CellularPopulationExecution unattended(Fixture& f) {
|
|
auto working = restore(f.population->state());
|
|
CellularBatch batch{f.header(64), {}};
|
|
for (unsigned tick = 0; tick < 64; ++tick) {
|
|
auto intents = take(wildlife_intents(*working, working->tick() + 1), "unattended wildlife");
|
|
for (const auto& life : working->lives()) {
|
|
if (life.account == LifeAccount{})
|
|
continue;
|
|
CellularLifeIntent frozen{life.identity.life, {}};
|
|
frozen.intent.frozen = true;
|
|
intents.push_back(frozen);
|
|
}
|
|
std::ranges::sort(intents, {}, &CellularLifeIntent::life_id);
|
|
take(working->advance_tick(intents), "unattended fixture step");
|
|
batch.ticks.push_back({intents, {}, {}});
|
|
}
|
|
return take(prepare_population_batch(*f.population, std::move(batch)),
|
|
"population recorded interval");
|
|
}
|
|
void recover(Fixture& f) {
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
auto db = connection();
|
|
auto recovered =
|
|
take(recover_population_checkpoint(*db, *f.objects, f.key, f.lease, builds, 5s),
|
|
"population durable recovery");
|
|
f.head = std::move(recovered.committed);
|
|
f.population = std::move(recovered.state);
|
|
}
|
|
void conversions_and_retries(Fixture& f) {
|
|
const auto biology = f.raw->snapshot();
|
|
Uuid account{};
|
|
account.back() = 1;
|
|
const auto input =
|
|
encode_cellular_mutation_input({account, 9991, f.head.head.end_tick + 1, {100, 1, 1, 1}});
|
|
take(f.db->query("BEGIN", {}, 5s), "reservation guard transaction");
|
|
take(reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 9991, input, 5s),
|
|
"reserve pending player choice");
|
|
const auto blocked = migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
|
|
f.build, {true, 1200, 1800}, 5s);
|
|
require(!blocked && std::string(blocked.error().store.cause.sqlstate.data(), 5) == "EVL18",
|
|
"population entry stranded a reserved player choice");
|
|
take(f.db->query("ROLLBACK", {}, 5s), "rollback guard fixture");
|
|
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
|
|
f.build, {true, 1200, 1800}, 5s),
|
|
"initial population transition");
|
|
recover(f);
|
|
require(f.population->state().snapshot() == biology && f.population->members().empty(),
|
|
"transition changed biology");
|
|
const auto refused =
|
|
reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 9991, input, 5s);
|
|
require(!refused && std::string(refused.error().cause.sqlstate.data(), 5) == "EVL06",
|
|
"population profile admitted a player mutation reservation");
|
|
const auto wildlife = std::ranges::find_if(f.population->state().lives(), [](const auto& life) {
|
|
return life.account == LifeAccount{};
|
|
});
|
|
require(wildlife != f.population->state().lives().end(), "missing durable descendant");
|
|
const auto genome = wildlife->identity.genome;
|
|
auto conversion =
|
|
take(prepare_population_conversion(*f.population, f.header(0), {genome, true}),
|
|
"collapse sample");
|
|
auto bad = conversion.batch;
|
|
++bad.boundaries.front().sequence;
|
|
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, bad, *conversion.state,
|
|
f.id(), f.build, 5s),
|
|
"altered sample published");
|
|
const auto commit = f.id();
|
|
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, conversion.batch,
|
|
*conversion.state, commit, f.build, 5s),
|
|
"durable collapse");
|
|
const PopulationPreparedCommit prepared{
|
|
f.key, commit, f.head.manifest,
|
|
take(encode_population_batch(conversion.batch), "prepared trace")};
|
|
require(take(retry_population_prepared(*f.db, prepared, 5s), "idempotent retry").head ==
|
|
f.head.head,
|
|
"retry advanced head");
|
|
require(take(resolve_population_prepared(*f.db, prepared, 5s), "read-only resolve").status ==
|
|
Resolution::committed,
|
|
"commit unresolved");
|
|
auto future_batch = conversion.batch;
|
|
future_batch.format_version = 2;
|
|
auto future = prepared;
|
|
future.trace = take(encode_population_batch(future_batch), "future trace fixture");
|
|
auto future_manifest = take(decode_manifest(future.manifest), "future manifest fixture");
|
|
future_manifest.batch.trace_bytes = future.trace.size();
|
|
future_manifest.batch.trace_sha256 = take(sha256(future.trace), "future trace digest");
|
|
future.manifest = take(encode_manifest(future_manifest), "future manifest encoding");
|
|
auto rejected_future = retry_population_prepared(*f.db, future, 5s);
|
|
require(!rejected_future && rejected_future.error().kind == PublicationError::candidate,
|
|
"bundle 9 prepared retry admitted population trace 2");
|
|
require(!resolve_population_prepared(*f.db, future, 5s),
|
|
"bundle 9 prepared resolve admitted population trace 2");
|
|
auto rejected_sql = publish_population_checkpoint(*f.db, future.manifest, future.trace, 5s);
|
|
require(!rejected_sql && rejected_sql.error().kind == LeaseStoreError::invalid,
|
|
"schema 45 did not reject the unsupported trace format before commit lookup");
|
|
const auto current_manifest = take(decode_manifest(f.head.manifest), "upgrade source manifest");
|
|
const auto boundary_payload =
|
|
take(cellular_boundary_upgrade_payload(conversion.state->snapshot(), current_manifest),
|
|
"portable boundary upgrade source");
|
|
const auto boundary_manifest = take(
|
|
cellular_boundary_upgrade_manifest(
|
|
current_manifest, f.head.head.digest, f.id(), f.lease, f.build, boundary_payload.size(),
|
|
take(sha256(boundary_payload), "boundary payload digest")),
|
|
"portable boundary upgrade manifest");
|
|
const PopulationPreparedCommit boundary{
|
|
f.key,
|
|
boundary_manifest.commit,
|
|
take(encode_manifest(boundary_manifest), "boundary candidate encoding"),
|
|
{}};
|
|
auto rejected_boundary = retry_population_prepared(*f.db, boundary, 5s);
|
|
require(!rejected_boundary && rejected_boundary.error().kind == PublicationError::rejected &&
|
|
take(resolve_population_prepared(*f.db, boundary, 5s),
|
|
"upgrade through wrong journal resolution")
|
|
.status == Resolution::absent,
|
|
"accounting upgrade bypassed its dedicated version journal");
|
|
auto rejected_upgrade = publish_population_profile_transition(*f.db, boundary.manifest, 5s);
|
|
require(!rejected_upgrade,
|
|
"representation publisher admitted accounting upgrade without dedicated evidence");
|
|
require(!publish_region_checkpoint(*f.db, boundary.manifest, {}, 5s),
|
|
"generic publisher admitted boundary upgrade");
|
|
require(take(load_region_checkpoint(*f.db, f.key, 5s), "head after future rejection").head ==
|
|
f.head.head,
|
|
"future trace changed the retained durable head");
|
|
auto changed = prepared;
|
|
auto changed_manifest = take(decode_manifest(changed.manifest), "conflict manifest");
|
|
changed_manifest.payload_sha256[0] ^= 1;
|
|
changed.manifest = take(encode_manifest(changed_manifest), "conflict encoding");
|
|
require(take(resolve_population_prepared(*f.db, changed, 5s), "conflicting resolve").status ==
|
|
Resolution::conflict,
|
|
"changed commit was accepted");
|
|
require(!publish_region_checkpoint(*f.db, f.head.manifest, {}, 5s),
|
|
"generic publisher bypassed profile gate");
|
|
const auto expected = conversion.state->snapshot();
|
|
recover(f);
|
|
require(f.population->snapshot() == expected && !f.population->members().empty(),
|
|
"committed sample not recovered");
|
|
require(f.population->state().snapshot() == biology, "collapse healed/aged/moved biology");
|
|
for (unsigned oscillation = 0; oscillation < 8; ++oscillation) {
|
|
const bool collapse = oscillation % 2 != 0;
|
|
auto next =
|
|
take(prepare_population_conversion(*f.population, f.header(0), {genome, collapse}),
|
|
"conversion oscillation");
|
|
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, next.batch,
|
|
*next.state, f.id(), f.build, 5s),
|
|
"publish oscillation");
|
|
recover(f);
|
|
require(f.population->state().snapshot() == biology,
|
|
"durable conversion changed exact state");
|
|
}
|
|
}
|
|
void process_loss(Fixture& f, int point) {
|
|
auto next = unattended(f);
|
|
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, next.batch.header.end_tick,
|
|
5s),
|
|
"admit process-loss interval");
|
|
const auto predecessor = f.head.head;
|
|
const auto expected = next.state->snapshot();
|
|
const auto commit = f.id();
|
|
int channel[2]{};
|
|
require(::pipe(channel) == 0, "process acknowledgement pipe");
|
|
const auto child = ::fork();
|
|
require(child >= 0, "population fork");
|
|
if (child == 0) {
|
|
::close(channel[0]);
|
|
try {
|
|
auto db = connection();
|
|
PublishHooks hooks;
|
|
if (point == 0)
|
|
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
|
|
const auto published =
|
|
publish_population_batch(*db, *f.objects, f.lease, f.head, next.batch, *next.state,
|
|
commit, f.build, 5s, hooks);
|
|
if (!published)
|
|
::_exit(20);
|
|
if (point == 2) {
|
|
const char ack = 'A';
|
|
if (::write(channel[1], &ack, 1) != 1)
|
|
::_exit(21);
|
|
}
|
|
::kill(::getpid(), SIGKILL);
|
|
} catch (...) {
|
|
::_exit(22);
|
|
}
|
|
::_exit(23);
|
|
}
|
|
::close(channel[1]);
|
|
char ack{};
|
|
const auto acknowledged = ::read(channel[0], &ack, 1);
|
|
::close(channel[0]);
|
|
int status{};
|
|
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
|
|
WTERMSIG(status) == SIGKILL,
|
|
"child missed population crash boundary");
|
|
require((point == 2 && acknowledged == 1 && ack == 'A') || (point != 2 && acknowledged == 0),
|
|
"unexpected permanent acknowledgement");
|
|
recover(f);
|
|
const auto pending =
|
|
take(load_population_interval(*f.db, f.key, f.lease, 5s), "recover process interval");
|
|
require(pending.has_value() == (point == 0), "process loss changed interval settlement");
|
|
if (point == 0)
|
|
require(f.head.head == predecessor, "uncommitted process published a head");
|
|
else
|
|
require(f.head.head.commit == commit && f.population->snapshot() == expected,
|
|
"committed process loss changed state");
|
|
}
|
|
void ambiguous_publication(Fixture& f, bool terminate) {
|
|
auto next = unattended(f);
|
|
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, next.batch.header.end_tick,
|
|
5s),
|
|
"admit ambiguous interval");
|
|
const auto expected = next.state->snapshot();
|
|
auto locker = connection(true);
|
|
take(locker->query("BEGIN", {}, 5s), "population lease row transaction");
|
|
const std::array<std::string, 2> key{std::to_string(f.key.world), std::to_string(f.key.region)};
|
|
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=$1::evolution.u64 "
|
|
"AND region_id=$2::evolution.u64 FOR UPDATE",
|
|
key, 5s),
|
|
"hold population publication");
|
|
auto timed = connection();
|
|
auto unknown = publish_population_batch(*timed, *f.objects, f.lease, f.head, next.batch,
|
|
*next.state, f.id(), f.build, 300ms);
|
|
require(!unknown && unknown.error().kind == PublicationError::unknown,
|
|
"population publication failed to retain ambiguous request");
|
|
if (terminate) {
|
|
const std::array<std::string, 1> user{environment("EVO_PG_RUNTIME_USER")};
|
|
take(locker->query("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE "
|
|
"usename=$1 AND wait_event_type='Lock'",
|
|
user, 5s),
|
|
"terminate uncommitted population writer");
|
|
}
|
|
take(locker->query("ROLLBACK", {}, 5s), "release population publication");
|
|
auto resolver = connection();
|
|
bool resolved = false;
|
|
for (unsigned attempt = 0; attempt < 200; ++attempt) {
|
|
auto result = take(resolve_population_prepared(*resolver, unknown.error().prepared, 5s),
|
|
"resolve ambiguous population publication");
|
|
if (result.status == (terminate ? Resolution::absent : Resolution::committed)) {
|
|
resolved = true;
|
|
break;
|
|
}
|
|
std::this_thread::sleep_for(25ms);
|
|
}
|
|
require(resolved, "ambiguous population publication did not resolve");
|
|
f.head = take(retry_population_prepared(*resolver, unknown.error().prepared, 5s),
|
|
"retry exact population request");
|
|
recover(f);
|
|
require(f.population->snapshot() == expected, "ambiguous publication changed retained sample");
|
|
}
|
|
void interval_contract(Fixture& f) {
|
|
require(!take(load_population_interval(*f.db, f.key, f.lease, 5s), "empty interval journal")
|
|
.has_value(),
|
|
"settled source retained pending work");
|
|
auto next = unattended(f);
|
|
const auto source = f.head.head;
|
|
const auto admitted =
|
|
take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
|
|
"durable interval admission");
|
|
require(take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
|
|
"duplicate interval admission") == admitted,
|
|
"admission retry changed boundary");
|
|
auto changed =
|
|
admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 63, 5s);
|
|
require(!changed && changed.error().kind == LeaseStoreError::commit_conflict,
|
|
"source admitted two different intervals");
|
|
const auto expected = next.state->snapshot();
|
|
require(!f.population->members().empty(), "interval fixture lost its exact cohort");
|
|
const auto genome = f.population->members().front().genome;
|
|
auto conversion =
|
|
take(prepare_population_conversion(*f.population, f.header(0), {genome, false}),
|
|
"conversion during admitted work");
|
|
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, conversion.batch,
|
|
*conversion.state, f.id(), f.build, 5s),
|
|
"conversion skipped an admitted interval");
|
|
require(
|
|
!materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
|
|
"profile exit skipped an admitted interval");
|
|
auto shortened_header = f.header(63);
|
|
auto shortened =
|
|
take(prepare_population_batch(
|
|
*f.population,
|
|
take(unattended_population_inputs(f.population->state(), shortened_header),
|
|
"shortened input")),
|
|
"shortened interval");
|
|
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, shortened.batch,
|
|
*shortened.state, f.id(), f.build, 5s),
|
|
"publication partially consumed an interval");
|
|
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release interval owner");
|
|
const auto epoch = f.lease.epoch;
|
|
f.lease =
|
|
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "replace interval owner");
|
|
require(f.lease.epoch > epoch && take(take(load_population_interval(*f.db, f.key, f.lease, 5s),
|
|
"new owner interval load"),
|
|
"retained admission") == admitted,
|
|
"replacement owner lost admitted work");
|
|
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, next.batch,
|
|
*next.state, f.id(), f.build, 5s),
|
|
"settle exact admitted interval");
|
|
recover(f);
|
|
require(f.population->snapshot() == expected &&
|
|
!take(load_population_interval(*f.db, f.key, f.lease, 5s), "settled interval load")
|
|
.has_value(),
|
|
"settlement lost work or retained spendable pending state");
|
|
require(take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
|
|
"settled admission retry") == admitted &&
|
|
!take(load_population_interval(*f.db, f.key, f.lease, 5s), "settled retry pending")
|
|
.has_value(),
|
|
"old admission retry reopened settled work");
|
|
require(!admit_population_interval(*f.db, f.key, f.lease, f.head.head,
|
|
f.head.head.end_tick + 65, 5s),
|
|
"oversized interval admitted");
|
|
const auto denied = f.db->query(
|
|
"SELECT * FROM evolution.publish_population_checkpoint_v44(''::bytea,''::bytea)", {}, 5s);
|
|
require(!denied && std::string(denied.error().sqlstate.data(), 5) == "42501",
|
|
"runtime bypassed interval guard");
|
|
}
|
|
void isolation_and_reverse(Fixture& f) {
|
|
auto interval = unattended(f);
|
|
const auto old_lease = f.lease;
|
|
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release old authority");
|
|
auto new_db = connection();
|
|
f.lease = take(acquire_region_lease(*new_db, f.key, f.build, 60000ms, 5s),
|
|
"new population authority");
|
|
require(f.lease.epoch > old_lease.epoch, "authority epoch did not advance");
|
|
const auto old_head = f.head.head;
|
|
auto stale = publish_population_batch(*f.db, *f.objects, old_lease, f.head, interval.batch,
|
|
*interval.state, f.id(), f.build, 5s);
|
|
require(!stale && stale.error().store.kind == LeaseStoreError::fenced,
|
|
"stale population writer published");
|
|
recover(f);
|
|
require(f.head.head == old_head, "fenced writer changed head");
|
|
const auto expected = f.population->state().snapshot();
|
|
f.head = take(
|
|
materialize_population_profile(*new_db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
|
|
"return to cellular profile");
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
auto recovered =
|
|
take(recover_cellular_checkpoint(*new_db, *f.objects, f.key, f.lease, builds, 5s),
|
|
"cellular recovery after materialization");
|
|
require(recovered.state->snapshot() == expected &&
|
|
recovered.manifest.profile == CheckpointProfile::cellular_lives_v1,
|
|
"reverse transition changed biology or time");
|
|
f.raw = std::move(recovered.state);
|
|
}
|
|
void missing_and_corrupt_objects(Fixture& f) {
|
|
const auto manifest = take(decode_manifest(f.head.manifest), "object test manifest");
|
|
const auto path = f.objects->directory() / hexadecimal(manifest.payload_sha256, false);
|
|
const auto backup = path.string() + ".population-test";
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
std::filesystem::rename(path, backup);
|
|
auto missing = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
|
|
std::filesystem::rename(backup, path);
|
|
require(!missing && missing.error().kind == RecoveryError::missing_payload,
|
|
"missing population payload did not fail closed");
|
|
std::filesystem::rename(path, backup);
|
|
{
|
|
std::ofstream corrupt(path, std::ios::binary);
|
|
corrupt << 'X';
|
|
}
|
|
auto corrupt = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
|
|
std::filesystem::remove(path);
|
|
std::filesystem::rename(backup, path);
|
|
require(!corrupt && corrupt.error().kind == RecoveryError::corrupt_payload,
|
|
"corrupt population payload did not fail closed");
|
|
recover(f);
|
|
}
|
|
void boundary_upgrades(Fixture& f) {
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
const auto original = f.head;
|
|
const auto expected_raw =
|
|
take(f.raw->migration_upgrade_candidate(), "raw upgrade oracle")->snapshot();
|
|
// Rehearse the raw upgrade, recovery and ordinary trace-3 publication in one
|
|
// transaction, then roll it back to retain a real bundle-9 source for profile 5.
|
|
take(f.db->query("BEGIN", {}, 5s), "raw upgrade rollback rehearsal");
|
|
auto raw_head = take(
|
|
migrate_cellular_boundary_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
|
|
"raw accounting upgrade");
|
|
auto raw = take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s),
|
|
"raw accounting recovery");
|
|
require(raw.state->snapshot() == expected_raw &&
|
|
raw.manifest.cellular_versions == cellular_v10_versions &&
|
|
raw_head.head.end_tick == original.head.end_tick,
|
|
"raw accounting upgrade changed biology or time");
|
|
const CellularBatchHeader h{f.key.world,
|
|
f.key.region,
|
|
1,
|
|
raw_head.head.commit,
|
|
raw_head.head.digest,
|
|
raw_head.head.end_tick,
|
|
raw_head.head.end_tick + 1};
|
|
const auto inputs = take(unattended_population_inputs(*raw.state, h), "upgraded raw inputs");
|
|
auto advanced = take(replay_cellular_batch(*raw.state, inputs), "upgraded raw oracle");
|
|
take(publish_cellular_batch(*f.db, *f.objects, f.lease, raw_head, inputs, *advanced, f.id(),
|
|
f.build, 5s),
|
|
"upgraded raw ordinary publication");
|
|
require(take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s),
|
|
"upgraded raw ordinary recovery")
|
|
.state->snapshot() == advanced->snapshot(),
|
|
"raw accounting continuation differs");
|
|
take(f.db->query("ROLLBACK", {}, 5s), "roll back raw upgrade rehearsal");
|
|
require(take(load_region_checkpoint(*f.db, f.key, 5s), "raw rollback head").head ==
|
|
original.head,
|
|
"rolled-back upgrade moved the durable head");
|
|
|
|
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
|
|
f.build, {true, 1200, 1800}, 5s),
|
|
"population accounting source entry");
|
|
recover(f);
|
|
const auto member = std::ranges::find_if(f.population->state().lives(), [](const auto& life) {
|
|
return life.account == LifeAccount{};
|
|
});
|
|
require(member != f.population->state().lives().end(), "accounting source has no wildlife");
|
|
auto collapsed = take(
|
|
prepare_population_conversion(*f.population, f.header(0), {member->identity.genome, true}),
|
|
"accounting source cohort");
|
|
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, collapsed.batch,
|
|
*collapsed.state, f.id(), f.build, 5s),
|
|
"accounting source membership publication");
|
|
recover(f);
|
|
require(!f.population->members().empty() && f.population->conversion_sequence() != 0,
|
|
"accounting source lost cohort history");
|
|
auto interval = unattended(f);
|
|
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head,
|
|
interval.batch.header.end_tick, 5s),
|
|
"upgrade pending interval");
|
|
auto pending =
|
|
migrate_cellular_boundary_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s);
|
|
require(!pending && std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18",
|
|
"accounting upgrade discarded admitted work");
|
|
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, interval.batch,
|
|
*interval.state, f.id(), f.build, 5s),
|
|
"settle pre-upgrade work");
|
|
recover(f);
|
|
const auto before = f.head;
|
|
const auto source = take(decode_manifest(before.manifest), "accounting source manifest");
|
|
const auto upgraded_population =
|
|
take(f.population->migration_upgrade_candidate(), "cohort upgrade oracle");
|
|
const auto expected = upgraded_population->snapshot();
|
|
const auto altered =
|
|
take(CellularPopulation::create(restore(upgraded_population->state()), {false, 1200, 1800}),
|
|
"valid unequal upgrade target");
|
|
require(altered->state().current_total() == upgraded_population->state().current_total() &&
|
|
altered->snapshot() != expected,
|
|
"unequal upgrade fixture must retain biological totals");
|
|
const auto altered_object = take(f.objects->put(altered->snapshot()), "unequal upgrade object");
|
|
const auto altered_manifest = take(
|
|
cellular_boundary_upgrade_manifest(source, before.head.digest, f.id(), f.lease, f.build,
|
|
altered_object.bytes, altered_object.sha256),
|
|
"unequal upgrade manifest");
|
|
take(f.db->query("BEGIN", {}, 5s), "unequal upgrade recovery rehearsal");
|
|
take(publish_cellular_profile_migration(
|
|
*f.db, take(encode_manifest(altered_manifest), "unequal upgrade encoding"), 5s),
|
|
"metadata-only unequal upgrade publication");
|
|
const auto unequal =
|
|
recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
|
|
take(f.db->query("ROLLBACK", {}, 5s), "undo unequal upgrade rehearsal");
|
|
require(!unequal && unequal.error().kind == RecoveryError::history &&
|
|
take(load_region_checkpoint(*f.db, f.key, 5s), "head after unequal upgrade").head ==
|
|
before.head,
|
|
"upgrade recovery compared only aggregate stock or rewrote the retained source");
|
|
const auto child = ::fork();
|
|
require(child >= 0, "upgrade pre-transaction fork");
|
|
if (child == 0) {
|
|
try {
|
|
auto db = connection();
|
|
PublishHooks hooks;
|
|
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
|
|
[[maybe_unused]] const auto result = migrate_cellular_boundary_profile(
|
|
*db, *f.objects, f.lease, before, identity(60000), f.build, 5s, hooks);
|
|
} catch (...) {
|
|
::_exit(31);
|
|
}
|
|
::_exit(32);
|
|
}
|
|
int status{};
|
|
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
|
|
WTERMSIG(status) == SIGKILL,
|
|
"upgrade child missed pre-transaction kill");
|
|
recover(f);
|
|
require(f.head.head == before.head, "unpublished upgrade became authoritative");
|
|
|
|
auto locker = connection(true);
|
|
take(locker->query("BEGIN", {}, 5s), "accounting ambiguity lock transaction");
|
|
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 "
|
|
"AND region_id=1 FOR UPDATE",
|
|
{}, 5s),
|
|
"hold accounting reply");
|
|
auto timed = connection();
|
|
auto unknown = migrate_cellular_boundary_profile(*timed, *f.objects, f.lease, before, f.id(),
|
|
f.build, 300ms);
|
|
require(!unknown && unknown.error().kind == PublicationError::unknown &&
|
|
!unknown.error().prepared.manifest.empty(),
|
|
"uncertain accounting upgrade lost its exact request");
|
|
take(locker->query("ROLLBACK", {}, 5s), "release accounting reply");
|
|
auto resolver = connection();
|
|
f.head = take(retry_cellular_profile_migration(*resolver, unknown.error().prepared, 5s),
|
|
"retry exact accounting upgrade");
|
|
require(take(retry_cellular_profile_migration(*resolver, unknown.error().prepared, 5s),
|
|
"duplicate accounting upgrade")
|
|
.head == f.head.head,
|
|
"duplicate accounting upgrade advanced head");
|
|
recover(f);
|
|
require(f.population->snapshot() == expected && f.head.head.end_tick == before.head.end_tick &&
|
|
take(decode_manifest(f.head.manifest), "upgraded population manifest")
|
|
.cellular_versions == cellular_v10_versions,
|
|
"accounting upgrade changed biology, cohort history or time");
|
|
auto changed = unknown.error().prepared;
|
|
auto changed_manifest = take(decode_manifest(changed.manifest), "changed accounting manifest");
|
|
changed_manifest.payload_sha256[0] ^= 1;
|
|
changed.manifest = take(encode_manifest(changed_manifest), "changed accounting bytes");
|
|
auto conflict = retry_cellular_profile_migration(*resolver, changed, 5s);
|
|
require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict,
|
|
"accounting retry accepted changed bytes");
|
|
require(
|
|
!migrate_cellular_boundary_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
|
|
"accounting upgrade applied twice");
|
|
const auto path = f.objects->directory() / hexadecimal(source.payload_sha256, false);
|
|
const auto backup = path.string() + ".upgrade-test";
|
|
std::filesystem::rename(path, backup);
|
|
const auto missing =
|
|
recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
|
|
std::filesystem::rename(backup, path);
|
|
require(!missing && missing.error().kind == RecoveryError::tail,
|
|
"accounting recovery trusted aggregate stock without its retained source");
|
|
missing_and_corrupt_objects(f);
|
|
const auto stale = f.lease;
|
|
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release accounting writer");
|
|
f.lease =
|
|
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "replace accounting writer");
|
|
auto stale_retry = retry_cellular_profile_migration(*f.db, unknown.error().prepared, 5s);
|
|
require(!stale_retry && stale_retry.error().store.kind == LeaseStoreError::fenced &&
|
|
!recover_population_checkpoint(*f.db, *f.objects, f.key, stale, builds, 5s),
|
|
"old accounting receipt granted current authority");
|
|
recover(f);
|
|
require(f.population->snapshot() == expected, "new epoch changed accounting upgrade");
|
|
auto future_interval = unattended(f);
|
|
auto old_trace = future_interval.batch;
|
|
old_trace.format_version = 1;
|
|
const auto old_bytes = take(encode_population_batch(old_trace), "wrong old trace fixture");
|
|
const auto upgraded_manifest = take(decode_manifest(f.head.manifest), "new accounting source");
|
|
auto wrong_trace_manifest =
|
|
take(population_successor_manifest(
|
|
upgraded_manifest, f.head.head.digest, f.id(), f.lease, old_trace.header.end_tick,
|
|
f.build, future_interval.state->snapshot().size(),
|
|
take(sha256(future_interval.state->snapshot()), "new payload SHA"),
|
|
{old_bytes.size(), take(sha256(old_bytes), "old trace SHA"), 0, {}}),
|
|
"wrong old trace manifest fixture");
|
|
const PopulationPreparedCommit wrong_trace{
|
|
f.key, wrong_trace_manifest.commit,
|
|
take(encode_manifest(wrong_trace_manifest), "wrong old trace manifest"), old_bytes};
|
|
auto refused_old = retry_population_prepared(*f.db, wrong_trace, 5s);
|
|
auto refused_old_sql =
|
|
publish_population_checkpoint(*f.db, wrong_trace.manifest, old_bytes, 5s);
|
|
require(!refused_old && refused_old.error().kind == PublicationError::candidate &&
|
|
!refused_old_sql && refused_old_sql.error().kind == LeaseStoreError::invalid &&
|
|
take(load_region_checkpoint(*f.db, f.key, 5s), "head after old trace").head ==
|
|
f.head.head,
|
|
"bundle 10 reinterpreted retained population trace 1");
|
|
// Exercise the existing publication/process-loss matrix with batch 2 as well.
|
|
process_loss(f, 0);
|
|
process_loss(f, 1);
|
|
process_loss(f, 2);
|
|
isolation_and_reverse(f);
|
|
require(f.raw->format_version() == cellular_lives_migration_format,
|
|
"reverse transition downgraded accounting owner");
|
|
}
|
|
void migration_publication(Fixture& f, const RegionKey destination_key,
|
|
FilesystemObjectStore& destination_objects,
|
|
LeaseGrant& destination_lease, CommittedCheckpoint& destination_head) {
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
const auto wildlife = std::ranges::find_if(
|
|
f.raw->lives(), [](const auto& life) { return life.account == LifeAccount{}; });
|
|
require(wildlife != f.raw->lives().end(), "paired publication needs real wildlife");
|
|
const auto genome = wildlife->identity.genome;
|
|
const auto make_host = [&](RegionKey key) {
|
|
auto objects = take(
|
|
FilesystemObjectStore::open(environment("PGHOST"), key, max_cellular_population_bytes),
|
|
"paired host objects");
|
|
CellularHostConfig config{key, 1, {}, f.build, {f.build}};
|
|
config.lease.commit_timeout = 300ms;
|
|
return take(
|
|
CellularRegionHost::create(
|
|
config,
|
|
[] { return std::expected<std::unique_ptr<Postgres>, PgFailure>{connection()}; },
|
|
std::move(objects)),
|
|
"paired host");
|
|
};
|
|
for (unsigned representation = 0; representation < 2; ++representation) {
|
|
if (representation == 0) {
|
|
f.head = take(materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(),
|
|
f.build, 5s),
|
|
"paired raw source");
|
|
destination_head =
|
|
take(materialize_population_profile(*f.db, destination_objects, destination_lease,
|
|
destination_head, f.id(), f.build, 5s),
|
|
"paired raw destination");
|
|
} else {
|
|
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
|
|
f.build, {true, 1200, 1800}, 5s),
|
|
"paired population source");
|
|
destination_head = take(migrate_cellular_to_population(
|
|
*f.db, destination_objects, destination_lease,
|
|
destination_head, f.id(), f.build, {true, 1200, 1800}, 5s),
|
|
"paired population destination");
|
|
}
|
|
for (unsigned direction = 0; direction < 2; ++direction) {
|
|
auto& sh = direction == 0 ? f.head : destination_head;
|
|
auto& dh = direction == 0 ? destination_head : f.head;
|
|
auto& so = direction == 0 ? *f.objects : destination_objects;
|
|
auto& dst = direction == 0 ? destination_objects : *f.objects;
|
|
auto& sl = direction == 0 ? f.lease : destination_lease;
|
|
auto& dl = direction == 0 ? destination_lease : f.lease;
|
|
const auto sk = direction == 0 ? f.key : destination_key;
|
|
const auto dk = direction == 0 ? destination_key : f.key;
|
|
take(release_region_lease(*f.db, sk, sl, 5s), "release paired source to host");
|
|
take(release_region_lease(*f.db, dk, dl, 5s), "release paired destination to host");
|
|
auto source_host = make_host(sk), destination_host = make_host(dk);
|
|
require(source_host->step().ready && destination_host->step().ready,
|
|
"paired hosts before preparation");
|
|
sl = take(source_host->authority(), "current source authority before pause");
|
|
dl = take(destination_host->authority(), "current destination authority before pause");
|
|
const auto source_before = sh, destination_before = dh;
|
|
const auto s = take(decode_manifest(sh.manifest), "paired predecessor source");
|
|
const auto d = take(decode_manifest(dh.manifest), "paired predecessor destination");
|
|
const auto sb =
|
|
take(so.get({s.payload_sha256, s.payload_bytes}), "paired source bytes");
|
|
const auto dbb =
|
|
take(dst.get({d.payload_sha256, d.payload_bytes}), "paired destination bytes");
|
|
const CellularMigrationRequest request{
|
|
f.id(),
|
|
{sk.region, sh.head.commit, f.id(), sh.head.digest},
|
|
{dk.region, dh.head.commit, f.id(), dh.head.digest},
|
|
genome};
|
|
take(source_host->prepare_migration_freeze(dh, dst, request),
|
|
"owning paired source freeze");
|
|
const auto frozen = take(take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s),
|
|
"paired source evidence"),
|
|
"missing paired freeze");
|
|
// Preparation alone cannot authorize either result.
|
|
require(!publish_cellular_migration(*f.db, so, dst, sl, dl, sh, dh, frozen.trace,
|
|
f.build, 5s),
|
|
"unstaged migration published");
|
|
take(destination_host->stage_migration(sh, so, frozen.trace), "owning paired stage");
|
|
std::vector<std::uint8_t> expected_source, expected_destination;
|
|
if (representation == 0) {
|
|
auto a = take(restore_cellular_payload(sb, s), "paired raw source oracle");
|
|
auto b = take(restore_cellular_payload(dbb, d), "paired raw destination oracle");
|
|
auto pair = take(replay_migration_trace(*a, *b, request, frozen.trace),
|
|
"paired raw oracle");
|
|
expected_source = pair.source->snapshot();
|
|
expected_destination = pair.destination->snapshot();
|
|
} else {
|
|
auto a = take(restore_population_payload(sb, s), "paired population source oracle");
|
|
auto b = take(restore_population_payload(dbb, d),
|
|
"paired population destination oracle");
|
|
auto pair = take(replay_migration_trace(*a, *b, request, frozen.trace),
|
|
"paired population oracle");
|
|
expected_source = pair.source->snapshot();
|
|
expected_destination = pair.destination->snapshot();
|
|
}
|
|
// Fail specifically on the SECOND result insert, after the first and receipt.
|
|
take(f.owner->query(
|
|
"CREATE FUNCTION evolution.test_refuse_second_migration() RETURNS trigger "
|
|
"LANGUAGE plpgsql AS $$ BEGIN IF NEW.commit_id='" +
|
|
hexadecimal(request.destination.commit, false) +
|
|
"'::uuid THEN RAISE EXCEPTION 'injected second insert' USING "
|
|
"ERRCODE='EVL06'; END IF; RETURN NEW; END $$",
|
|
{}, 5s),
|
|
"second insert fault function");
|
|
take(f.owner->query(
|
|
"CREATE TRIGGER test_refuse_second_migration BEFORE INSERT ON "
|
|
"evolution.region_commits "
|
|
"FOR EACH ROW EXECUTE FUNCTION evolution.test_refuse_second_migration()",
|
|
{}, 5s),
|
|
"second insert fault trigger");
|
|
const auto rejected = publish_cellular_migration(*f.db, so, dst, sl, dl, sh, dh,
|
|
frozen.trace, f.build, 5s);
|
|
take(f.owner->query(
|
|
"DROP TRIGGER test_refuse_second_migration ON evolution.region_commits", {},
|
|
5s),
|
|
"remove second insert fault");
|
|
take(f.owner->query("DROP FUNCTION evolution.test_refuse_second_migration()", {}, 5s),
|
|
"remove fault function");
|
|
require(
|
|
!rejected && rejected.error().store.kind == LeaseStoreError::unsupported &&
|
|
take(load_region_checkpoint(*f.db, sk, 5s), "rollback source head").head ==
|
|
source_before.head &&
|
|
take(load_region_checkpoint(*f.db, dk, 5s), "rollback destination head").head ==
|
|
destination_before.head &&
|
|
take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s),
|
|
"rollback source pause")
|
|
.has_value() &&
|
|
take(load_cellular_migration_stage(*f.db, dst, dk, dl, 5s),
|
|
"rollback destination pause")
|
|
.has_value(),
|
|
"failed second publication left half a migration or released a pause");
|
|
const auto prepared = rejected.error().prepared;
|
|
take(f.db->query("BEGIN", {}, 5s), "false paired biology transaction");
|
|
auto false_pair = prepared;
|
|
auto false_source =
|
|
take(decode_manifest(false_pair.source_manifest), "false source manifest");
|
|
false_source.payload_sha256 = s.payload_sha256;
|
|
false_source.payload_bytes = s.payload_bytes;
|
|
false_pair.source_manifest =
|
|
take(encode_manifest(false_source), "false source canonical bytes");
|
|
take(retry_cellular_migration_commit(*f.db, sl, dl, false_pair, 5s),
|
|
"metadata-only false pair");
|
|
const auto false_recovery = verify_cellular_migration_checkpoint(
|
|
*f.db, so, sl, take(load_region_checkpoint(*f.db, sk, 5s), "false paired head"), sb,
|
|
5s);
|
|
take(f.db->query("ROLLBACK", {}, 5s), "rollback false paired biology fixture");
|
|
require(!false_recovery && false_recovery.error().kind == RecoveryError::history,
|
|
"paired recovery accepted a valid old object as the migrated biology");
|
|
const std::array<std::string, 1> generic{hexadecimal(prepared.source_manifest)};
|
|
require(!f.db->query("SELECT * FROM evolution.publish_region_checkpoint($1::bytea)",
|
|
generic, 5s),
|
|
"generic publisher admitted a migration half");
|
|
const auto child = ::fork();
|
|
require(child >= 0, "pre-transaction paired fork");
|
|
if (child == 0) {
|
|
try {
|
|
auto db = connection();
|
|
PublishHooks hooks{[] { ::kill(::getpid(), SIGKILL); }};
|
|
if (!publish_cellular_migration(*db, so, dst, sl, dl, sh, dh, frozen.trace,
|
|
f.build, 5s, hooks))
|
|
::_exit(52);
|
|
} catch (...) {
|
|
::_exit(51);
|
|
}
|
|
::_exit(52);
|
|
}
|
|
int status{};
|
|
require(
|
|
::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
|
|
WTERMSIG(status) == SIGKILL &&
|
|
take(load_region_checkpoint(*f.db, sk, 5s), "killed pair source").head ==
|
|
source_before.head &&
|
|
take(load_region_checkpoint(*f.db, dk, 5s), "killed pair destination").head ==
|
|
destination_before.head,
|
|
"pre-transaction paired death changed a head");
|
|
if (direction == 0) {
|
|
auto locker = connection(true), timed = connection();
|
|
take(locker->query("BEGIN", {}, 5s), "pair ambiguity transaction");
|
|
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND "
|
|
"region_id=" +
|
|
std::to_string(sk.region) + " FOR UPDATE",
|
|
{}, 5s),
|
|
"hold paired reply");
|
|
const auto unknown =
|
|
retry_cellular_migration_commit(*timed, sl, dl, prepared, 300ms);
|
|
require(!unknown && unknown.error().kind == PublicationError::unknown &&
|
|
unknown.error().prepared.source_manifest == prepared.source_manifest &&
|
|
unknown.error().prepared.destination_manifest ==
|
|
prepared.destination_manifest,
|
|
"ambiguous pair lost its exact manifests");
|
|
take(locker->query("ROLLBACK", {}, 5s), "release paired ambiguity");
|
|
} else {
|
|
const auto committed_child = ::fork();
|
|
require(committed_child >= 0, "post-commit paired fork");
|
|
if (committed_child == 0) {
|
|
try {
|
|
auto db = connection();
|
|
if (!retry_cellular_migration_commit(*db, sl, dl, prepared, 5s))
|
|
::_exit(53);
|
|
::kill(::getpid(), SIGKILL);
|
|
} catch (...) {
|
|
::_exit(54);
|
|
}
|
|
::_exit(55);
|
|
}
|
|
require(::waitpid(committed_child, &status, 0) == committed_child &&
|
|
WIFSIGNALED(status) && WTERMSIG(status) == SIGKILL,
|
|
"paired publisher failed before post-commit kill");
|
|
}
|
|
const auto committed =
|
|
take(retry_cellular_migration_commit(*f.db, sl, dl, prepared, 5s),
|
|
"resolve exact paired commit");
|
|
sh = committed.source;
|
|
dh = committed.destination;
|
|
require(!abort_cellular_migration_freeze(*f.db, sk, sl, request.request, 5s),
|
|
"committed migration was aborted");
|
|
auto changed = prepared;
|
|
auto altered =
|
|
take(decode_manifest(changed.destination_manifest), "conflicting committed pair");
|
|
altered.payload_sha256.back() ^= 1;
|
|
changed.destination_manifest = take(encode_manifest(altered), "conflicting pair bytes");
|
|
const auto conflict = retry_cellular_migration_commit(*f.db, sl, dl, changed, 5s);
|
|
auto stale = sl;
|
|
--stale.epoch;
|
|
require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict &&
|
|
!retry_cellular_migration_commit(*f.db, stale, dl, prepared, 5s),
|
|
"changed pair or stale source authority was accepted");
|
|
for (unsigned attempt = 0;
|
|
attempt < 200 && (!source_host->step().ready || !destination_host->step().ready);
|
|
++attempt)
|
|
std::this_thread::sleep_for(25ms);
|
|
require(source_host->status().ready && destination_host->status().ready &&
|
|
take(source_host->committed(), "adopted source").head == sh.head &&
|
|
take(destination_host->committed(), "adopted destination").head == dh.head,
|
|
"pointer release resumed a stale owner instead of recovering the pair");
|
|
const auto check_bytes = [&](const CommittedCheckpoint& h, RegionKey key,
|
|
const FilesystemObjectStore& objects,
|
|
const LeaseGrant& lease) {
|
|
require(take(load_region_checkpoint(*f.db, key, 5s), "current paired recovery head")
|
|
.head == h.head,
|
|
"paired recovery selected a different head");
|
|
if (representation == 0)
|
|
return take(recover_cellular_checkpoint(*f.db, objects, key, lease, builds, 5s),
|
|
"recover raw paired result")
|
|
.state->snapshot();
|
|
return take(recover_population_checkpoint(*f.db, objects, key, lease, builds, 5s),
|
|
"recover cohort paired result")
|
|
.state->snapshot();
|
|
};
|
|
require(check_bytes(sh, sk, so, sl) == expected_source &&
|
|
check_bytes(dh, dk, dst, dl) == expected_destination,
|
|
"committed biology/member/material differs from the exact pair");
|
|
const auto capsule_path = so.directory() / hexadecimal(d.payload_sha256, false);
|
|
const auto backup = capsule_path.string() + ".paired-test";
|
|
std::filesystem::rename(capsule_path, backup);
|
|
const auto missing =
|
|
verify_cellular_migration_checkpoint(*f.db, so, sl, sh, expected_source, 5s);
|
|
std::filesystem::rename(backup, capsule_path);
|
|
require(!missing && missing.error().kind == RecoveryError::tail &&
|
|
!verify_cellular_migration_checkpoint(*f.db, so, sl, sh, sb, 5s),
|
|
"pair recovery ignored missing counterpart evidence or false local biology");
|
|
source_host->drain();
|
|
destination_host->drain();
|
|
source_host = make_host(sk);
|
|
destination_host = make_host(dk);
|
|
require(source_host->step().ready && destination_host->step().ready,
|
|
"replacement paired hosts failed complete recovery");
|
|
sl = take(source_host->authority(), "replacement source fence");
|
|
dl = take(destination_host->authority(), "replacement destination fence");
|
|
require(take(retry_cellular_migration_commit(*f.db, sl, dl, prepared, 5s),
|
|
"new epoch exact paired retry")
|
|
.source.head == sh.head,
|
|
"fresh authority changed a committed retry");
|
|
source_host->drain();
|
|
destination_host->drain();
|
|
sl = take(acquire_region_lease(*f.db, sk, f.build, 60000ms, 5s),
|
|
"return paired source authority");
|
|
dl = take(acquire_region_lease(*f.db, dk, f.build, 60000ms, 5s),
|
|
"return paired destination authority");
|
|
}
|
|
}
|
|
// Race the actual two-head transaction against abort on independent connections.
|
|
const auto prepare_pair = [&](bool reverse) {
|
|
auto& sh = reverse ? destination_head : f.head;
|
|
auto& dh = reverse ? f.head : destination_head;
|
|
auto& so = reverse ? destination_objects : *f.objects;
|
|
auto& dst = reverse ? *f.objects : destination_objects;
|
|
auto& sl = reverse ? destination_lease : f.lease;
|
|
auto& dl = reverse ? f.lease : destination_lease;
|
|
const auto sk = reverse ? destination_key : f.key;
|
|
const auto dk = reverse ? f.key : destination_key;
|
|
CellularMigrationRequest request{f.id(),
|
|
{sk.region, sh.head.commit, f.id(), sh.head.digest},
|
|
{dk.region, dh.head.commit, f.id(), dh.head.digest},
|
|
genome};
|
|
const auto freeze =
|
|
take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s),
|
|
"commit/abort source preparation");
|
|
take(stage_cellular_migration(*f.db, so, dst, dl, sh, dh, freeze.trace, 5s),
|
|
"commit/abort destination stage");
|
|
return freeze.trace;
|
|
};
|
|
const auto aborted_trace = prepare_pair(false);
|
|
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, aborted_trace.request.request, 5s),
|
|
"abort wins before paired commit");
|
|
const auto after_abort = publish_cellular_migration(
|
|
*f.db, *f.objects, destination_objects, f.lease, destination_lease, f.head,
|
|
destination_head, aborted_trace, f.build, 5s);
|
|
require(!after_abort && after_abort.error().store.kind == LeaseStoreError::commit_conflict,
|
|
"paired publication reopened an aborted workflow");
|
|
for (unsigned race = 0; race < 4; ++race) {
|
|
const auto trace = prepare_pair(false);
|
|
auto committing = connection(), aborting = connection();
|
|
std::barrier start(2);
|
|
std::optional<std::expected<CellularMigrationCommitted, CellularMigrationCommitFailure>>
|
|
published;
|
|
std::optional<std::expected<CellularMigrationFreeze, LeaseStoreFailure>> aborted;
|
|
std::thread commit_thread([&] {
|
|
start.arrive_and_wait();
|
|
published = publish_cellular_migration(*committing, *f.objects, destination_objects,
|
|
f.lease, destination_lease, f.head,
|
|
destination_head, trace, f.build, 5s);
|
|
});
|
|
std::thread abort_thread([&] {
|
|
start.arrive_and_wait();
|
|
aborted = abort_cellular_migration_freeze(*aborting, f.key, f.lease,
|
|
trace.request.request, 5s);
|
|
});
|
|
commit_thread.join();
|
|
abort_thread.join();
|
|
require(published.has_value() && aborted.has_value() &&
|
|
(published->has_value() != aborted->has_value()),
|
|
"commit/abort race had two outcomes or no definitive outcome");
|
|
if (published->has_value()) {
|
|
require(aborted->error().kind == LeaseStoreError::commit_conflict,
|
|
"losing abort was not refused");
|
|
f.head = (*published)->source;
|
|
destination_head = (*published)->destination;
|
|
const auto returning = prepare_pair(true);
|
|
const auto returned =
|
|
take(publish_cellular_migration(*f.db, destination_objects, *f.objects,
|
|
destination_lease, f.lease, destination_head,
|
|
f.head, returning, f.build, 5s),
|
|
"return race winner's stock");
|
|
destination_head = returned.source;
|
|
f.head = returned.destination;
|
|
} else {
|
|
require(published->error().store.kind == LeaseStoreError::commit_conflict,
|
|
"losing paired publication was not refused");
|
|
}
|
|
require(
|
|
take(load_region_checkpoint(*f.db, f.key, 5s), "race source head").head ==
|
|
f.head.head &&
|
|
take(load_region_checkpoint(*f.db, destination_key, 5s), "race destination head")
|
|
.head == destination_head.head &&
|
|
!take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s),
|
|
"race source settled")
|
|
.has_value() &&
|
|
!take(load_cellular_migration_stage(*f.db, destination_objects, destination_key,
|
|
destination_lease, 5s),
|
|
"race destination settled")
|
|
.has_value(),
|
|
"commit/abort race left half a head or a pending pause");
|
|
}
|
|
recover(f);
|
|
f.raw = restore(f.population->state());
|
|
}
|
|
|
|
void migration_freezes(Fixture& f) {
|
|
const RegionKey destination_key{f.key.world, 2};
|
|
auto destination_objects =
|
|
take(FilesystemObjectStore::open(environment("PGHOST"), destination_key,
|
|
max_cellular_population_bytes),
|
|
"migration destination objects");
|
|
auto destination = opening(30001, true, 4, false, 2);
|
|
auto destination_head =
|
|
take(provision_cellular_region(*f.owner, *destination_objects, destination_key, 1,
|
|
*destination, f.id(), f.build, 5s),
|
|
"migration empty destination genesis");
|
|
auto destination_lease =
|
|
take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s),
|
|
"migration destination authority");
|
|
while (destination->tick() < f.raw->tick()) {
|
|
const auto count = std::min(std::uint64_t{64}, f.raw->tick() - destination->tick());
|
|
CellularBatch inputs{{f.key.world, 2, 1, destination_head.head.commit,
|
|
destination_head.head.digest, destination->tick(),
|
|
destination->tick() + count},
|
|
{}};
|
|
inputs.ticks.resize(static_cast<std::size_t>(count));
|
|
destination =
|
|
take(replay_cellular_batch(*destination, inputs), "empty destination biology");
|
|
destination_head = take(publish_cellular_batch(*f.db, *destination_objects,
|
|
destination_lease, destination_head, inputs,
|
|
*destination, f.id(), f.build, 5s),
|
|
"empty destination tick publication")
|
|
.committed;
|
|
}
|
|
destination_head =
|
|
take(migrate_cellular_boundary_profile(*f.db, *destination_objects, destination_lease,
|
|
destination_head, f.id(), f.build, 5s),
|
|
"destination accounting upgrade");
|
|
destination = take(destination->migration_upgrade_candidate(), "destination upgraded oracle");
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
const auto request_for = [&] {
|
|
const auto member = std::ranges::find_if(
|
|
f.raw->lives(), [](const auto& life) { return life.account == LifeAccount{}; });
|
|
require(member != f.raw->lives().end(), "freeze fixture lacks real wildlife");
|
|
return CellularMigrationRequest{
|
|
f.id(),
|
|
{1, f.head.head.commit, f.id(), f.head.head.digest},
|
|
{2, destination_head.head.commit, f.id(), destination_head.head.digest},
|
|
member->identity.genome};
|
|
};
|
|
for (unsigned representation = 0; representation < 2; ++representation) {
|
|
if (representation == 1) {
|
|
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
|
|
f.build, {true, 1200, 1800}, 5s),
|
|
"freeze cohort source entry");
|
|
recover(f);
|
|
destination_head = take(migrate_cellular_to_population(
|
|
*f.db, *destination_objects, destination_lease,
|
|
destination_head, f.id(), f.build, {true, 1200, 1800}, 5s),
|
|
"freeze cohort destination entry");
|
|
}
|
|
const auto source_before = f.head;
|
|
const auto source_bytes = f.raw->snapshot();
|
|
const auto destination_before = destination_head;
|
|
const auto request = request_for();
|
|
const auto prepare = [&](const CellularMigrationRequest& requested,
|
|
std::chrono::milliseconds timeout = 5s) {
|
|
return prepare_cellular_migration_freeze(*f.db, *f.objects, *destination_objects,
|
|
f.lease, f.head, destination_head, requested,
|
|
timeout);
|
|
};
|
|
if (representation == 0) {
|
|
Uuid account{};
|
|
account.back() = 1;
|
|
take(f.db->query("BEGIN", {}, 5s), "pending player choice before freeze");
|
|
const auto control =
|
|
take(claim_account_control(*f.db, f.key, f.lease, account, 2, 9900, 60000ms, 5s),
|
|
"pending choice current control");
|
|
const auto input = encode_cellular_mutation_input(
|
|
{account, 9900, f.head.head.end_tick + 1, {100, 1, 1, 1}});
|
|
take(reserve_cellular_mutation(*f.db, f.key, f.lease, account, control.generation, 9900,
|
|
input, 5s),
|
|
"reserve pre-freeze player choice");
|
|
const auto pending = prepare(request);
|
|
take(f.db->query("ROLLBACK", {}, 5s), "rollback pre-freeze player choice");
|
|
require(!pending &&
|
|
std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18",
|
|
"preparation stranded a player choice");
|
|
} else {
|
|
take(f.db->query("BEGIN", {}, 5s), "prepare with admitted work transaction");
|
|
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head,
|
|
f.head.head.end_tick + 20, 5s),
|
|
"admitted work before freeze");
|
|
const auto pending = prepare(request);
|
|
take(f.db->query("ROLLBACK", {}, 5s), "rollback pending-work preparation fixture");
|
|
require(!pending &&
|
|
std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18",
|
|
"preparation skipped durable ecological work");
|
|
}
|
|
auto frozen = take(prepare(request), "durable source freeze");
|
|
const CellularMigrationPrepared prepared{
|
|
f.key, take(encode_migration_trace(frozen.trace), "frozen canonical trace")};
|
|
auto malformed = prepared.trace;
|
|
malformed.back() ^= 1;
|
|
const std::array<std::string, 3> malformed_args{hexadecimal(f.lease.holder, false),
|
|
std::to_string(f.lease.epoch),
|
|
hexadecimal(malformed)};
|
|
const auto malformed_sql = f.db->query("SELECT * FROM "
|
|
"evolution.prepare_cellular_migration_freeze(30201,"
|
|
"1,$1::uuid,$2::evolution.u64,$3::bytea)",
|
|
malformed_args, 5s);
|
|
require(!malformed_sql && std::string(malformed_sql.error().sqlstate.data(), 5) == "22023",
|
|
"SQL accepted an invalid trace checksum before duplicate lookup");
|
|
require(
|
|
!frozen.aborted && frozen.trace.request == request &&
|
|
take(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s),
|
|
"replay frozen capsule"),
|
|
"missing durable freeze")
|
|
.trace == frozen.trace &&
|
|
take(load_region_checkpoint(*f.db, f.key, 5s), "frozen source head").head ==
|
|
source_before.head &&
|
|
take(load_region_checkpoint(*f.db, destination_key, 5s), "unowned destination head")
|
|
.head == destination_before.head,
|
|
"preparation changed either owner or lost its full replay capsule");
|
|
require(
|
|
!take(retry_cellular_migration_freeze(*f.db, f.lease, prepared, 5s), "duplicate freeze")
|
|
.aborted,
|
|
"duplicate freeze lost its state");
|
|
const auto stage = [&](const CellularMigrationTrace& trace,
|
|
std::chrono::milliseconds timeout = 5s) {
|
|
return stage_cellular_migration(*f.db, *f.objects, *destination_objects,
|
|
destination_lease, f.head, destination_head, trace,
|
|
timeout);
|
|
};
|
|
// The optimistic destination cut must still be exact at first staging.
|
|
take(f.db->query("BEGIN", {}, 5s), "move destination before stage transaction");
|
|
if (representation == 0)
|
|
take(migrate_cellular_to_population(*f.db, *destination_objects, destination_lease,
|
|
destination_head, f.id(), f.build, {}, 5s),
|
|
"temporarily move destination profile");
|
|
else
|
|
take(materialize_population_profile(*f.db, *destination_objects, destination_lease,
|
|
destination_head, f.id(), f.build, 5s),
|
|
"temporarily move population destination");
|
|
const auto moved = stage(frozen.trace);
|
|
take(f.db->query("ROLLBACK", {}, 5s), "rollback moved destination fixture");
|
|
require(!moved && moved.error().store.kind == LeaseStoreError::head_mismatch,
|
|
"stage silently rebased a moved destination");
|
|
if (representation == 1) {
|
|
take(f.db->query("BEGIN", {}, 5s), "stage pending interval transaction");
|
|
take(admit_population_interval(*f.db, destination_key, destination_lease,
|
|
destination_head.head,
|
|
destination_head.head.end_tick + 20, 5s),
|
|
"destination admitted work before staging");
|
|
const auto pending = stage(frozen.trace);
|
|
take(f.db->query("ROLLBACK", {}, 5s), "rollback destination interval fixture");
|
|
require(!pending && pending.error().store.kind == LeaseStoreError::commit_conflict,
|
|
"stage stranded destination interval work");
|
|
}
|
|
const auto staged = take(stage(frozen.trace), "durable destination staging");
|
|
const CellularMigrationStagePrepared stage_request{destination_key, prepared.trace,
|
|
staged.ghost};
|
|
require(
|
|
!staged.freeze.aborted &&
|
|
take(take(load_cellular_migration_stage(*f.db, *destination_objects,
|
|
destination_key, destination_lease, 5s),
|
|
"recover destination ghost"),
|
|
"missing staged destination")
|
|
.ghost == staged.ghost &&
|
|
take(load_region_checkpoint(*f.db, destination_key, 5s),
|
|
"staged destination still owns original head")
|
|
.head == destination_before.head &&
|
|
take(retry_cellular_migration_stage(*f.db, destination_lease, stage_request, 5s),
|
|
"duplicate stage")
|
|
.ghost == staged.ghost,
|
|
"staging activated a ghost or lost exact replay");
|
|
auto changed_stage = stage_request;
|
|
++changed_stage.ghost.bytes;
|
|
const auto changed =
|
|
retry_cellular_migration_stage(*f.db, destination_lease, changed_stage, 5s);
|
|
require(!changed && changed.error().store.kind == LeaseStoreError::commit_conflict,
|
|
"stage accepted conflicting immutable ghost bytes");
|
|
auto reverse = frozen.trace;
|
|
reverse.request.request = f.id();
|
|
std::swap(reverse.request.source, reverse.request.destination);
|
|
const auto overlapping = retry_cellular_migration_freeze(
|
|
*f.db, destination_lease,
|
|
{destination_key, take(encode_migration_trace(reverse), "overlapping source trace")},
|
|
5s);
|
|
require(!overlapping && overlapping.error().store.kind == LeaseStoreError::commit_conflict,
|
|
"staged destination simultaneously prepared an outgoing workflow");
|
|
take(f.db->query("BEGIN", {}, 5s), "staged destination life reservation transaction");
|
|
Uuid new_account{};
|
|
new_account.back() = 5;
|
|
const auto staged_control =
|
|
take(claim_account_control(*f.db, destination_key, destination_lease, new_account, 1,
|
|
9950, 60000ms, 5s),
|
|
"staged destination reservation current control");
|
|
const auto staged_life =
|
|
reserve_primitive_life(*f.db, destination_key, destination_lease, new_account,
|
|
staged_control.generation, 1, 1, 5s);
|
|
take(f.db->query("ROLLBACK", {}, 5s), "rollback staged life reservation fixture");
|
|
require(!staged_life &&
|
|
std::string(staged_life.error().cause.sqlstate.data(), 5) == "EVL18",
|
|
"staged destination admitted a new player life reservation");
|
|
if (representation == 0) {
|
|
CellularBatch inputs{{f.key.world, 2, 1, destination_head.head.commit,
|
|
destination_head.head.digest, destination->tick(),
|
|
destination->tick() + 1},
|
|
{{}}};
|
|
const auto advanced = take(replay_cellular_batch(*destination, inputs),
|
|
"detached staged destination future");
|
|
const auto denied =
|
|
publish_cellular_batch(*f.db, *destination_objects, destination_lease,
|
|
destination_head, inputs, *advanced, f.id(), f.build, 5s);
|
|
require(!denied &&
|
|
std::string(denied.error().store.cause.sqlstate.data(), 5) == "EVL18",
|
|
"staged destination published an ordinary future");
|
|
} else {
|
|
require(!admit_population_interval(*f.db, destination_key, destination_lease,
|
|
destination_head.head,
|
|
destination_head.head.end_tick + 20, 5s) &&
|
|
!materialize_population_profile(*f.db, *destination_objects,
|
|
destination_lease, destination_head, f.id(),
|
|
f.build, 5s),
|
|
"staged destination progressed or changed profile");
|
|
}
|
|
const auto ghost_path =
|
|
destination_objects->directory() / hexadecimal(staged.ghost.sha256, false);
|
|
const auto ghost_backup = ghost_path.string() + ".stage-test";
|
|
std::filesystem::rename(ghost_path, ghost_backup);
|
|
const auto missing_ghost = load_cellular_migration_stage(
|
|
*f.db, *destination_objects, destination_key, destination_lease, 5s);
|
|
std::filesystem::rename(ghost_backup, ghost_path);
|
|
require(!missing_ghost && missing_ghost.error().kind == RecoveryError::tail,
|
|
"missing ghost fell back to the active destination");
|
|
char ghost_first{};
|
|
{
|
|
std::fstream file(ghost_path, std::ios::binary | std::ios::in | std::ios::out);
|
|
file.get(ghost_first);
|
|
file.seekp(0);
|
|
file.put(static_cast<char>(static_cast<unsigned char>(ghost_first) ^ 1U));
|
|
file.flush();
|
|
require(file.good(), "same-length ghost corruption fixture");
|
|
}
|
|
const auto corrupt_ghost = load_cellular_migration_stage(
|
|
*f.db, *destination_objects, destination_key, destination_lease, 5s);
|
|
{
|
|
std::fstream file(ghost_path, std::ios::binary | std::ios::in | std::ios::out);
|
|
file.put(ghost_first);
|
|
file.flush();
|
|
require(file.good(), "restore corrupt ghost fixture");
|
|
}
|
|
require(!corrupt_ghost && corrupt_ghost.error().kind == RecoveryError::tail,
|
|
"same-length corrupt ghost bypassed digest verification");
|
|
const auto source_manifest = take(decode_manifest(f.head.manifest), "stage source capsule");
|
|
const auto source_capsule_path =
|
|
destination_objects->directory() / hexadecimal(source_manifest.payload_sha256, false);
|
|
const auto source_capsule_backup = source_capsule_path.string() + ".stage-test";
|
|
std::filesystem::rename(source_capsule_path, source_capsule_backup);
|
|
const auto missing_source = load_cellular_migration_stage(
|
|
*f.db, *destination_objects, destination_key, destination_lease, 5s);
|
|
std::filesystem::rename(source_capsule_backup, source_capsule_path);
|
|
require(!missing_source && missing_source.error().kind == RecoveryError::tail,
|
|
"destination recovery used the live source instead of its retained capsule");
|
|
const auto old_destination_lease = destination_lease;
|
|
take(release_region_lease(*f.db, destination_key, destination_lease, 5s),
|
|
"release staged destination owner");
|
|
destination_lease = take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s),
|
|
"replacement staged destination owner");
|
|
require(
|
|
take(load_cellular_migration_stage(*f.db, *destination_objects, destination_key,
|
|
destination_lease, 5s),
|
|
"new epoch ghost replay")
|
|
.has_value() &&
|
|
!retry_cellular_migration_stage(*f.db, old_destination_lease, stage_request, 5s),
|
|
"replacement activated the ghost or stale destination staged it");
|
|
auto conflict_trace = frozen.trace;
|
|
++conflict_trace.transferred.energy.value;
|
|
auto conflict = retry_cellular_migration_freeze(
|
|
*f.db, f.lease, {f.key, take(encode_migration_trace(conflict_trace), "conflict trace")},
|
|
5s);
|
|
require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict &&
|
|
!prepare(request_for()),
|
|
"freeze accepted changed bytes or replaced pending request");
|
|
if (representation == 0) {
|
|
const auto h = f.header(1);
|
|
const auto inputs = take(unattended_population_inputs(*f.raw, h), "frozen raw inputs");
|
|
auto advanced = take(replay_cellular_batch(*f.raw, inputs), "frozen detached future");
|
|
auto denied = publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, inputs,
|
|
*advanced, f.id(), f.build, 5s);
|
|
require(!denied &&
|
|
std::string(denied.error().store.cause.sqlstate.data(), 5) == "EVL18" &&
|
|
!migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
|
|
f.build, {}, 5s),
|
|
"raw source progressed or changed profile while frozen");
|
|
Uuid account{};
|
|
account.back() = 1;
|
|
const auto input = encode_cellular_mutation_input(
|
|
{account, 9901, f.head.head.end_tick + 1, {100, 1, 1, 1}});
|
|
take(f.db->query("BEGIN", {}, 5s), "current control while frozen fixture");
|
|
const auto control =
|
|
take(claim_account_control(*f.db, f.key, f.lease, account, 2, 9901, 60000ms, 5s),
|
|
"current frozen-source account control");
|
|
const auto choice = reserve_cellular_mutation(*f.db, f.key, f.lease, account,
|
|
control.generation, 9901, input, 5s);
|
|
take(f.db->query("ROLLBACK", {}, 5s), "rollback frozen control fixture");
|
|
require(!choice, "frozen raw owner admitted a new mutation reservation");
|
|
if (std::string(choice.error().cause.sqlstate.data(), 5) != "EVL18")
|
|
throw std::runtime_error("frozen mutation guard SQLSTATE=" +
|
|
std::string(choice.error().cause.sqlstate.data(), 5));
|
|
} else {
|
|
require(!admit_population_interval(*f.db, f.key, f.lease, f.head.head,
|
|
f.head.head.end_tick + 20, 5s) &&
|
|
!materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(),
|
|
f.build, 5s),
|
|
"population source admitted work or changed profile while frozen");
|
|
}
|
|
const auto capsule_manifest =
|
|
take(decode_manifest(destination_head.manifest), "capsule manifest");
|
|
const auto capsule_path =
|
|
f.objects->directory() / hexadecimal(capsule_manifest.payload_sha256, false);
|
|
const auto backup = capsule_path.string() + ".freeze-test";
|
|
std::filesystem::rename(capsule_path, backup);
|
|
const auto missing = load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s);
|
|
std::filesystem::rename(backup, capsule_path);
|
|
require(!missing && missing.error().kind == RecoveryError::tail,
|
|
"freeze recovery fell back to destination live state without the retained capsule");
|
|
char first{};
|
|
{
|
|
std::fstream file(capsule_path, std::ios::binary | std::ios::in | std::ios::out);
|
|
file.get(first);
|
|
file.seekp(0);
|
|
file.put(static_cast<char>(static_cast<unsigned char>(first) ^ 1U));
|
|
file.flush();
|
|
require(file.good(), "corrupt same-length capsule fixture");
|
|
}
|
|
const auto corrupt = load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s);
|
|
{
|
|
std::fstream file(capsule_path, std::ios::binary | std::ios::in | std::ios::out);
|
|
file.put(first);
|
|
file.flush();
|
|
require(file.good(), "restore same-length capsule fixture");
|
|
}
|
|
require(!corrupt && corrupt.error().kind == RecoveryError::tail,
|
|
"corrupt capsule bypassed strong object binding");
|
|
const auto old_lease = f.lease;
|
|
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release frozen source owner");
|
|
f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s),
|
|
"new frozen source owner");
|
|
require(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s),
|
|
"recover freeze under new epoch")
|
|
.has_value() &&
|
|
!abort_cellular_migration_freeze(*f.db, f.key, old_lease, request.request, 5s),
|
|
"new epoch released a freeze or old epoch aborted it");
|
|
require(
|
|
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, request.request, 5s),
|
|
"explicit source abort")
|
|
.aborted &&
|
|
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, request.request, 5s),
|
|
"duplicate source abort")
|
|
.aborted &&
|
|
take(retry_cellular_migration_freeze(*f.db, f.lease, prepared, 5s),
|
|
"retry aborted preparation")
|
|
.aborted &&
|
|
!take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s),
|
|
"aborted pointer load")
|
|
.has_value(),
|
|
"abort was not durable or preparation retry reopened it");
|
|
require(
|
|
!take(load_cellular_migration_stage(*f.db, *destination_objects, destination_key,
|
|
destination_lease, 5s),
|
|
"aborted destination pointer")
|
|
.has_value() &&
|
|
take(retry_cellular_migration_stage(*f.db, destination_lease, stage_request, 5s),
|
|
"retry aborted stage")
|
|
.freeze.aborted,
|
|
"source abort stranded the destination or stage retry reopened its pointer");
|
|
require(take(load_region_checkpoint(*f.db, f.key, 5s), "aborted source head").head ==
|
|
source_before.head &&
|
|
f.raw->snapshot() == source_bytes,
|
|
"abort changed biological stock/time/history");
|
|
|
|
auto false_trace = frozen.trace;
|
|
false_trace.request.request = f.id();
|
|
false_trace.request.source.commit = f.id();
|
|
false_trace.request.destination.commit = f.id();
|
|
++false_trace.transferred.energy.value;
|
|
take(retry_cellular_migration_freeze(*f.db, f.lease,
|
|
{f.key, take(encode_migration_trace(false_trace),
|
|
"canonical false inventory trace")},
|
|
5s),
|
|
"metadata-only false preparation");
|
|
const auto false_replay =
|
|
load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s);
|
|
require(!false_replay && false_replay.error().kind == RecoveryError::history,
|
|
"freeze recovery trusted declared material without complete biological replay");
|
|
take(
|
|
abort_cellular_migration_freeze(*f.db, f.key, f.lease, false_trace.request.request, 5s),
|
|
"explicitly abort false preparation without moving stock");
|
|
|
|
// A genuine unknown reply retains exactly the chosen cut and group.
|
|
auto locker = connection(true);
|
|
take(locker->query("BEGIN", {}, 5s), "freeze ambiguity lock");
|
|
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND "
|
|
"region_id=1 FOR UPDATE",
|
|
{}, 5s),
|
|
"hold freeze reply");
|
|
auto timed = connection();
|
|
auto unknown =
|
|
prepare_cellular_migration_freeze(*timed, *f.objects, *destination_objects, f.lease,
|
|
f.head, destination_head, request_for(), 300ms);
|
|
require(!unknown && unknown.error().kind == PublicationError::unknown &&
|
|
!unknown.error().prepared.trace.empty(),
|
|
"freeze ambiguity lost its immutable request");
|
|
take(locker->query("ROLLBACK", {}, 5s), "release freeze ambiguity");
|
|
frozen = take(retry_cellular_migration_freeze(*f.db, f.lease, unknown.error().prepared, 5s),
|
|
"reconcile unknown freeze");
|
|
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, frozen.trace.request.request,
|
|
5s),
|
|
"abort reconciled freeze");
|
|
|
|
const auto false_stage_freeze = take(prepare(request_for()), "false ghost preparation");
|
|
const auto original_destination =
|
|
take(decode_manifest(destination_head.manifest), "false ghost existing object");
|
|
const CellularMigrationStagePrepared false_ghost{
|
|
destination_key,
|
|
take(encode_migration_trace(false_stage_freeze.trace), "false ghost trace"),
|
|
{original_destination.payload_sha256, original_destination.payload_bytes}};
|
|
take(retry_cellular_migration_stage(*f.db, destination_lease, false_ghost, 5s),
|
|
"metadata-only false ghost stage");
|
|
const auto false_ghost_replay = load_cellular_migration_stage(
|
|
*f.db, *destination_objects, destination_key, destination_lease, 5s);
|
|
require(!false_ghost_replay && false_ghost_replay.error().kind == RecoveryError::history,
|
|
"stage recovery accepted a real object with the wrong complete biology");
|
|
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
|
|
false_stage_freeze.trace.request.request, 5s),
|
|
"abort false ghost evidence");
|
|
|
|
const auto uncertain_stage_freeze =
|
|
take(prepare(request_for()), "unknown stage preparation");
|
|
take(locker->query("BEGIN", {}, 5s), "stage ambiguity lock");
|
|
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND "
|
|
"region_id=2 FOR UPDATE",
|
|
{}, 5s),
|
|
"hold stage reply");
|
|
auto stage_timed = connection();
|
|
const auto stage_unknown = stage_cellular_migration(
|
|
*stage_timed, *f.objects, *destination_objects, destination_lease, f.head,
|
|
destination_head, uncertain_stage_freeze.trace, 300ms);
|
|
require(!stage_unknown && stage_unknown.error().kind == PublicationError::unknown &&
|
|
!stage_unknown.error().prepared.trace.empty(),
|
|
"stage ambiguity lost exact bytes");
|
|
take(locker->query("ROLLBACK", {}, 5s), "release stage ambiguity");
|
|
take(retry_cellular_migration_stage(*f.db, destination_lease,
|
|
stage_unknown.error().prepared, 5s),
|
|
"resolve unknown stage");
|
|
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
|
|
uncertain_stage_freeze.trace.request.request, 5s),
|
|
"abort reconciled stage");
|
|
|
|
for (unsigned race = 0; race < 4; ++race) {
|
|
const auto race_freeze = take(prepare(request_for()), "stage race preparation");
|
|
auto stage_db = connection(), abort_db = connection();
|
|
std::barrier start(2);
|
|
std::optional<std::expected<CellularMigrationStage, CellularMigrationStageFailure>>
|
|
staged_race;
|
|
std::optional<std::expected<CellularMigrationFreeze, LeaseStoreFailure>> aborted_race;
|
|
std::thread staging([&] {
|
|
start.arrive_and_wait();
|
|
staged_race = stage_cellular_migration(*stage_db, *f.objects, *destination_objects,
|
|
destination_lease, f.head, destination_head,
|
|
race_freeze.trace, 5s);
|
|
});
|
|
std::thread aborting([&] {
|
|
start.arrive_and_wait();
|
|
aborted_race = abort_cellular_migration_freeze(
|
|
*abort_db, f.key, f.lease, race_freeze.trace.request.request, 5s);
|
|
});
|
|
staging.join();
|
|
aborting.join();
|
|
require(aborted_race.has_value() && aborted_race->has_value() &&
|
|
staged_race.has_value() &&
|
|
(staged_race->has_value() ||
|
|
staged_race->error().store.kind == LeaseStoreError::commit_conflict) &&
|
|
!take(load_cellular_migration_stage(*f.db, *destination_objects,
|
|
destination_key, destination_lease, 5s),
|
|
"stage race final pointer")
|
|
.has_value(),
|
|
"stage/abort race left an active ghost or stranded pause");
|
|
}
|
|
|
|
// Both sides of preparation's transaction boundary, with real process death.
|
|
for (unsigned point = 0; point < 2; ++point) {
|
|
const auto killed_request = request_for();
|
|
const auto child = ::fork();
|
|
require(child >= 0, "freeze publisher fork");
|
|
if (child == 0) {
|
|
try {
|
|
auto db = connection();
|
|
PublishHooks hooks;
|
|
if (point == 0)
|
|
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
|
|
if (!prepare_cellular_migration_freeze(*db, *f.objects, *destination_objects,
|
|
f.lease, f.head, destination_head,
|
|
killed_request, 5s, hooks))
|
|
::_exit(41);
|
|
::kill(::getpid(), SIGKILL);
|
|
} catch (...) {
|
|
::_exit(42);
|
|
}
|
|
::_exit(43);
|
|
}
|
|
int status{};
|
|
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
|
|
WTERMSIG(status) == SIGKILL &&
|
|
take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s),
|
|
"recover killed preparation")
|
|
.has_value() == (point == 1),
|
|
"publisher death changed source ownership freeze");
|
|
if (point == 1)
|
|
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, killed_request.request,
|
|
5s),
|
|
"abort killed preparation");
|
|
}
|
|
for (unsigned point = 0; point < 2; ++point) {
|
|
const auto killed = take(prepare(request_for()), "stage process-loss preparation");
|
|
const auto child = ::fork();
|
|
require(child >= 0, "stage publisher fork");
|
|
if (child == 0) {
|
|
try {
|
|
auto db = connection();
|
|
PublishHooks hooks;
|
|
if (point == 0)
|
|
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
|
|
if (!stage_cellular_migration(*db, *f.objects, *destination_objects,
|
|
destination_lease, f.head, destination_head,
|
|
killed.trace, 5s, hooks))
|
|
::_exit(44);
|
|
::kill(::getpid(), SIGKILL);
|
|
} catch (...) {
|
|
::_exit(45);
|
|
}
|
|
::_exit(46);
|
|
}
|
|
int status{};
|
|
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
|
|
WTERMSIG(status) == SIGKILL &&
|
|
take(load_cellular_migration_stage(*f.db, *destination_objects,
|
|
destination_key, destination_lease, 5s),
|
|
"recover killed stage")
|
|
.has_value() == (point == 1),
|
|
"stage process loss changed ownership or lost the durable pause");
|
|
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
|
|
killed.trace.request.request, 5s),
|
|
"abort killed stage");
|
|
}
|
|
}
|
|
// The owning host must retain the freeze through drain/recreation with entry off.
|
|
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture to frozen host");
|
|
const auto make_host = [&] {
|
|
auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), f.key,
|
|
max_cellular_population_bytes),
|
|
"freeze host objects");
|
|
CellularHostConfig config{f.key, 1, {}, f.build, {f.build}};
|
|
config.lease.commit_timeout = 300ms;
|
|
return take(
|
|
CellularRegionHost::create(
|
|
config,
|
|
[] { return std::expected<std::unique_ptr<Postgres>, PgFailure>{connection()}; },
|
|
std::move(objects)),
|
|
"freeze host");
|
|
};
|
|
auto host = make_host();
|
|
require(host->step().ready, "source host before freeze");
|
|
const auto before = host->state()->snapshot();
|
|
const auto host_request = request_for();
|
|
take(host->prepare_migration_freeze(destination_head, *destination_objects, host_request),
|
|
"host source freeze");
|
|
require(!host->status().ready && host->status().migration_frozen &&
|
|
!host->advance_unattended(64) && host->state()->snapshot() == before,
|
|
"frozen host advanced or originated current output");
|
|
host->drain();
|
|
host = make_host();
|
|
require(!host->step().ready && host->status().migration_frozen &&
|
|
host->state()->snapshot() == before && !host->authority().has_value(),
|
|
"replacement host implicitly aborted or ignored its freeze");
|
|
take(host->abort_migration_freeze(host_request.request), "replacement host explicit abort");
|
|
require(host->step().ready && !host->status().migration_frozen &&
|
|
host->state()->snapshot() == before,
|
|
"host failed to resume the identical source after confirmed abort");
|
|
auto locker = connection(true);
|
|
take(locker->query("BEGIN", {}, 5s), "host unknown freeze lock");
|
|
take(
|
|
locker->query(
|
|
"SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE",
|
|
{}, 5s),
|
|
"hold host freeze reply");
|
|
const auto uncertain_request = request_for();
|
|
const auto uncertain =
|
|
host->prepare_migration_freeze(destination_head, *destination_objects, uncertain_request);
|
|
require(!uncertain && !host->status().ready, "host originated during uncertain freeze");
|
|
take(locker->query("ROLLBACK", {}, 5s), "release host freeze reply");
|
|
for (unsigned attempt = 0; attempt < 200 && !host->status().migration_frozen; ++attempt) {
|
|
host->step();
|
|
std::this_thread::sleep_for(25ms);
|
|
}
|
|
require(host->status().migration_frozen && !host->status().ready &&
|
|
host->state()->snapshot() == before,
|
|
"host chose different biology after unknown preparation");
|
|
take(locker->query("BEGIN", {}, 5s), "host unknown abort lock");
|
|
take(
|
|
locker->query(
|
|
"SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE",
|
|
{}, 5s),
|
|
"hold host abort reply");
|
|
const auto unknown_abort = host->abort_migration_freeze(uncertain_request.request);
|
|
require(!unknown_abort && !host->status().ready, "unconfirmed abort resumed the source");
|
|
take(locker->query("ROLLBACK", {}, 5s), "release host abort reply");
|
|
for (unsigned attempt = 0; attempt < 200 && !host->step().ready; ++attempt)
|
|
std::this_thread::sleep_for(25ms);
|
|
require(host->status().ready && !host->status().migration_frozen &&
|
|
host->state()->snapshot() == before,
|
|
"host failed exact ambiguous-abort reconciliation");
|
|
host->drain();
|
|
f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s),
|
|
"return freeze fixture authority");
|
|
const auto destination_host_freeze =
|
|
take(prepare_cellular_migration_freeze(*f.db, *f.objects, *destination_objects, f.lease,
|
|
f.head, destination_head, request_for(), 5s),
|
|
"destination host source preparation");
|
|
take(release_region_lease(*f.db, destination_key, destination_lease, 5s),
|
|
"release destination to owning host");
|
|
const auto make_destination_host = [&] {
|
|
auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), destination_key,
|
|
max_cellular_population_bytes),
|
|
"destination host objects");
|
|
CellularHostConfig config{destination_key, 1, {}, f.build, {f.build}};
|
|
config.lease.commit_timeout = 300ms;
|
|
return take(
|
|
CellularRegionHost::create(
|
|
config,
|
|
[] { return std::expected<std::unique_ptr<Postgres>, PgFailure>{connection()}; },
|
|
std::move(objects)),
|
|
"destination staging host");
|
|
};
|
|
auto destination_host = make_destination_host();
|
|
require(destination_host->step().ready, "destination host before staging");
|
|
const auto destination_bytes = destination_host->state()->snapshot();
|
|
take(locker->query("BEGIN", {}, 5s), "host stage ambiguity transaction");
|
|
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND "
|
|
"region_id=1 FOR UPDATE",
|
|
{}, 5s),
|
|
"hold owning destination stage");
|
|
const auto host_stage_unknown =
|
|
destination_host->stage_migration(f.head, *f.objects, destination_host_freeze.trace);
|
|
require(!host_stage_unknown && !destination_host->status().ready &&
|
|
destination_host->state()->snapshot() == destination_bytes,
|
|
"owning destination advanced during ambiguous staging");
|
|
take(locker->query("ROLLBACK", {}, 5s), "release owning stage reply");
|
|
for (unsigned attempt = 0; attempt < 200 && !destination_host->status().migration_frozen;
|
|
++attempt) {
|
|
destination_host->step();
|
|
std::this_thread::sleep_for(25ms);
|
|
}
|
|
require(destination_host->status().migration_frozen && !destination_host->status().ready &&
|
|
!destination_host->advance_unattended(64),
|
|
"owning destination activated its ghost");
|
|
destination_host->drain();
|
|
destination_host = make_destination_host();
|
|
require(!destination_host->step().ready && destination_host->status().migration_frozen &&
|
|
destination_host->state()->snapshot() == destination_bytes &&
|
|
!destination_host->authority().has_value(),
|
|
"replacement destination lost staging or adopted the uncommitted ghost");
|
|
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
|
|
destination_host_freeze.trace.request.request, 5s),
|
|
"source-authorized abort releases destination host");
|
|
for (unsigned attempt = 0; attempt < 200 && !destination_host->step().ready; ++attempt)
|
|
std::this_thread::sleep_for(25ms);
|
|
require(destination_host->status().ready && !destination_host->status().migration_frozen &&
|
|
destination_host->state()->snapshot() == destination_bytes,
|
|
"destination failed to resume its unchanged original state after source abort");
|
|
destination_host->drain();
|
|
destination_lease = take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s),
|
|
"return staged destination fixture authority");
|
|
migration_publication(f, destination_key, *destination_objects, destination_lease,
|
|
destination_head);
|
|
f.head = take(
|
|
materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
|
|
"resume frozen population as raw owner");
|
|
f.raw = take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s),
|
|
"source after frozen profile materialization")
|
|
.state;
|
|
take(release_region_lease(*f.db, destination_key, destination_lease, 5s),
|
|
"release freeze destination");
|
|
}
|
|
|
|
void host_ownership_and_admission(Fixture& f) {
|
|
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture host authority");
|
|
const auto make_host = [&](bool enabled) {
|
|
auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), f.key,
|
|
max_cellular_population_bytes),
|
|
"host objects");
|
|
CellularHostConfig config{f.key, 1, {}, f.build, {f.build}};
|
|
config.lease.commit_timeout = 300ms;
|
|
if (enabled)
|
|
config.population_policy = CellularTierPolicy{true, 1200, 1800};
|
|
return take(CellularRegionHost::create(
|
|
std::move(config),
|
|
[] {
|
|
return Postgres::connect({environment("PGHOST"),
|
|
{},
|
|
environment("PGPORT"),
|
|
environment("PGDATABASE"),
|
|
environment("EVO_PG_RUNTIME_USER"),
|
|
{},
|
|
{}},
|
|
5s);
|
|
},
|
|
std::move(objects)),
|
|
"population host");
|
|
};
|
|
auto host = make_host(false);
|
|
require(host->step().ready && !host->population_mode(), "raw host recovery");
|
|
require(!host->set_population_mode(true), "default host enabled population entry");
|
|
host->drain();
|
|
host = make_host(true);
|
|
require(host->step().ready, "opt-in host recovery");
|
|
const auto before = host->state()->snapshot();
|
|
const auto tick = host->state()->tick();
|
|
take(host->set_population_mode(true), "host population entry");
|
|
require(host->population_mode() && host->state()->snapshot() == before,
|
|
"host entry changed biology");
|
|
auto population =
|
|
take(CellularPopulation::create(
|
|
take(CellularLives::restore(before, f.key.world, f.key.region, 1, tick),
|
|
"host reference restore"),
|
|
{true, 1200, 1800}),
|
|
"host reference population");
|
|
f.head = take(host->committed(), "host population head");
|
|
auto next = unattended(f); // uses the same deterministic frozen/wildlife input contract.
|
|
// The fixture's old envelope may have a different membership history; biology is shared.
|
|
auto expected =
|
|
take(prepare_population_batch(*population, next.batch.biology), "host reference batch");
|
|
require(!take(host->advance_unattended(19), "collect bounded publication window").has_value() &&
|
|
host->state()->snapshot() == before,
|
|
"small local debt was admitted prematurely");
|
|
const auto published = take(host->advance_unattended(65), "host population publication");
|
|
require(published.has_value(), "host skipped admitted interval");
|
|
require(host->population_mode() &&
|
|
host->state()->snapshot() == expected.state->state().snapshot(),
|
|
"host population continuation differs");
|
|
const auto committed = take(host->committed(), "host continued head");
|
|
const auto continued = host->state()->snapshot();
|
|
const CellularBatchHeader recovered_header{f.key.world,
|
|
f.key.region,
|
|
1,
|
|
committed.head.commit,
|
|
committed.head.digest,
|
|
committed.head.end_tick,
|
|
committed.head.end_tick + 64};
|
|
auto recovered_expected = take(
|
|
prepare_population_batch(
|
|
*expected.state, take(unattended_population_inputs(*host->state(), recovered_header),
|
|
"recovered interval inputs")),
|
|
"recovered interval reference");
|
|
take(admit_population_interval(*f.db, f.key, take(host->authority(), "host interval authority"),
|
|
committed.head, committed.head.end_tick + 64, 5s),
|
|
"admit before host loss");
|
|
host->drain();
|
|
host = make_host(false);
|
|
require(host->step().ready && host->population_mode() &&
|
|
take(host->committed(), "host recovered head").head == committed.head &&
|
|
host->state()->snapshot() == continued,
|
|
"restarted host failed to recover population head");
|
|
require(host->status().ecological_pending_ticks == 64 && !host->set_population_mode(false),
|
|
"restarted host hid or skipped admitted work");
|
|
require(take(host->advance_unattended(0), "resume admitted work with zero new due time")
|
|
.has_value() &&
|
|
host->state()->snapshot() == recovered_expected.state->state().snapshot() &&
|
|
host->status().ecological_pending_ticks == 0,
|
|
"recovered interval did not settle exactly");
|
|
const auto recovered_head = take(host->committed(), "recovered interval head");
|
|
const auto recovered_bytes = host->state()->snapshot();
|
|
const CellularBatchHeader unknown_header{f.key.world,
|
|
f.key.region,
|
|
1,
|
|
recovered_head.head.commit,
|
|
recovered_head.head.digest,
|
|
recovered_head.head.end_tick,
|
|
recovered_head.head.end_tick + 64};
|
|
auto unknown_expected = take(
|
|
prepare_population_batch(*recovered_expected.state,
|
|
take(unattended_population_inputs(*host->state(), unknown_header),
|
|
"uncertain admission inputs")),
|
|
"uncertain admission reference");
|
|
auto locker = connection(true);
|
|
take(locker->query("BEGIN", {}, 5s), "host admission lock transaction");
|
|
take(
|
|
locker->query(
|
|
"SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE",
|
|
{}, 5s),
|
|
"hold host admission reply");
|
|
require(!host->advance_unattended(64) && !host->status().ready &&
|
|
host->state()->snapshot() == recovered_bytes,
|
|
"uncertain admission allowed simulation/output");
|
|
take(locker->query("ROLLBACK", {}, 5s), "release host admission reply");
|
|
for (unsigned attempt = 0; attempt < 200 && !host->step().ready; ++attempt)
|
|
std::this_thread::sleep_for(25ms);
|
|
require(host->status().ready && host->status().ecological_pending_ticks == 64,
|
|
"unknown admission failed exact reconciliation");
|
|
require(take(host->advance_unattended(0), "finish reconciled admission").has_value() &&
|
|
host->state()->snapshot() == unknown_expected.state->state().snapshot(),
|
|
"admission retry chose different work");
|
|
const auto final_bytes = host->state()->snapshot();
|
|
Uuid account{};
|
|
account.back() = 1;
|
|
// A returning player's operation completes the durable reverse transition before lookup.
|
|
[[maybe_unused]] const auto queried = host->query_primitive_life(account, 1, 1);
|
|
require(!host->population_mode() && host->state()->snapshot() == final_bytes &&
|
|
host->state()->tick() == tick + 192,
|
|
"player operation did not materialize first");
|
|
host->drain();
|
|
}
|
|
} // namespace
|
|
void run_population_publication_tests() {
|
|
Fixture fixture;
|
|
funded_source(fixture);
|
|
conversions_and_retries(fixture);
|
|
interval_contract(fixture);
|
|
ambiguous_publication(fixture, false);
|
|
ambiguous_publication(fixture, true);
|
|
process_loss(fixture, 0);
|
|
process_loss(fixture, 1);
|
|
process_loss(fixture, 2);
|
|
missing_and_corrupt_objects(fixture);
|
|
isolation_and_reverse(fixture);
|
|
boundary_upgrades(fixture);
|
|
migration_freezes(fixture);
|
|
host_ownership_and_admission(fixture);
|
|
std::puts("population publication, exact recovery, process loss and fencing passed");
|
|
}
|