Foundation / checks (deploy) (push) Successful in 6s
Foundation / checks (ecs) (push) Successful in 2m24s
Foundation / checks (network) (push) Successful in 20m40s
Foundation / checks (client) (push) Successful in 21m57s
Foundation / checks (tsan) (push) Successful in 26m32s
Foundation / checks (postgres) (push) Successful in 1h3m17s
Foundation / checks (checks) (push) Successful in 1h14m29s
Foundation / postgres-tools (push) Successful in 34s
Foundation / server-image (push) Successful in 4m6s
Foundation / image (push) Successful in 4m52s
3305 lines
192 KiB
C++
3305 lines
192 KiB
C++
#include "evo/persistence/account_control_store.hpp"
|
|
#include "evo/persistence/cellular_census_snapshot.hpp"
|
|
#include "evo/persistence/cellular_history_store.hpp"
|
|
#include "evo/persistence/population_checkpoint_store.hpp"
|
|
#include "evo/persistence/population_interval_store.hpp"
|
|
#include "evo/persistence/sha256.hpp"
|
|
#include "evo/server/cellular_region_host.hpp"
|
|
#include "evo/server/lifecycle/cellular_census_set.hpp"
|
|
#include "fixtures/cellular_ecology.hpp"
|
|
#include "fixtures/postgres_relay.hpp"
|
|
|
|
#include <barrier>
|
|
#include <charconv>
|
|
#include <csignal>
|
|
#include <cstdlib>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <limits>
|
|
#include <stdexcept>
|
|
#include <sys/wait.h>
|
|
#include <thread>
|
|
#include <type_traits>
|
|
#include <unistd.h>
|
|
|
|
void run_population_publication_tests();
|
|
namespace {
|
|
using namespace evo;
|
|
using namespace evo::server;
|
|
using namespace evo::persistence;
|
|
using namespace evo::fixture;
|
|
using namespace std::chrono_literals;
|
|
template <typename T, typename E> T take(std::expected<T, E> value, const char* where) {
|
|
if (!value) {
|
|
std::string message{where};
|
|
if constexpr (requires { value.error().cause.sqlstate; })
|
|
message += " SQLSTATE=" + std::string(value.error().cause.sqlstate.data(), 5);
|
|
if constexpr (requires { value.error().store.cause.sqlstate; })
|
|
message += " SQLSTATE=" + std::string(value.error().store.cause.sqlstate.data(), 5);
|
|
throw std::runtime_error(message);
|
|
}
|
|
if constexpr (!std::is_void_v<T>)
|
|
return std::move(*value);
|
|
}
|
|
template <typename T> T take(std::optional<T> value, const char* where) {
|
|
if (!value.has_value())
|
|
throw std::runtime_error(where);
|
|
return std::move(*value);
|
|
}
|
|
std::string environment(const char* name) {
|
|
const auto* value = std::getenv(name);
|
|
require(value != nullptr, "missing private test environment");
|
|
return value;
|
|
}
|
|
std::unique_ptr<Postgres> connection(bool owner = false) {
|
|
return take(Postgres::connect({environment("PGHOST"),
|
|
{},
|
|
environment("PGPORT"),
|
|
environment("PGDATABASE"),
|
|
environment(owner ? "PGUSER" : "EVO_PG_RUNTIME_USER"),
|
|
{},
|
|
{}},
|
|
5s),
|
|
"population connection");
|
|
}
|
|
Uuid identity(unsigned number) {
|
|
Uuid id{};
|
|
id[0] = 0xDA;
|
|
id[1] = 0x12;
|
|
id[14] = static_cast<std::uint8_t>(number >> 8U);
|
|
id[15] = static_cast<std::uint8_t>(number);
|
|
return id;
|
|
}
|
|
std::string hexadecimal(std::span<const std::uint8_t> bytes, bool prefix = true) {
|
|
std::string out = prefix ? "\\x" : "";
|
|
constexpr char digits[] = "0123456789abcdef";
|
|
for (const auto b : bytes) {
|
|
out.push_back(digits[b >> 4U]);
|
|
out.push_back(digits[b & 15U]);
|
|
}
|
|
return out;
|
|
}
|
|
struct Fixture {
|
|
RegionKey key{30201, 1};
|
|
std::unique_ptr<Postgres> owner = connection(true), db = connection();
|
|
std::unique_ptr<FilesystemObjectStore> objects;
|
|
LeaseGrant lease;
|
|
Sha256Digest build{};
|
|
CommittedCheckpoint head;
|
|
std::unique_ptr<CellularLives> raw;
|
|
std::unique_ptr<CellularPopulation> population;
|
|
unsigned next_id = 1;
|
|
Uuid id() {
|
|
return identity(next_id++);
|
|
}
|
|
CellularBatchHeader header(std::uint64_t count) const {
|
|
return {key.world,
|
|
key.region,
|
|
1,
|
|
head.head.commit,
|
|
head.head.digest,
|
|
head.head.end_tick,
|
|
head.head.end_tick + count};
|
|
}
|
|
};
|
|
// Deny actual reads by the unprivileged fixture user, then restore permissions
|
|
// even if a runtime requirement throws. This never changes immutable bytes.
|
|
class UnreadableObject {
|
|
public:
|
|
explicit UnreadableObject(std::filesystem::path path)
|
|
: path_(std::move(path)), permissions_(std::filesystem::status(path_).permissions()) {
|
|
std::filesystem::permissions(path_, std::filesystem::perms::none);
|
|
}
|
|
~UnreadableObject() {
|
|
std::error_code error;
|
|
std::filesystem::permissions(path_, permissions_, error);
|
|
}
|
|
UnreadableObject(const UnreadableObject&) = delete;
|
|
UnreadableObject& operator=(const UnreadableObject&) = delete;
|
|
|
|
private:
|
|
std::filesystem::path path_;
|
|
std::filesystem::perms permissions_;
|
|
};
|
|
std::uint16_t migration_backend_port() {
|
|
const auto port = environment("EVO_MIGRATION_TCP_PORT");
|
|
unsigned value{};
|
|
const auto [end, error] = std::from_chars(port.data(), port.data() + port.size(), value);
|
|
require(error == std::errc{} && end == port.data() + port.size() && value > 0 && value <= 65535,
|
|
"invalid private migration TLS port");
|
|
return static_cast<std::uint16_t>(value);
|
|
}
|
|
PgOptions migration_tls_options(const test::PostgresRelay& relay) {
|
|
return {"localhost",
|
|
"127.0.0.1",
|
|
relay.port(),
|
|
environment("PGDATABASE"),
|
|
environment("EVO_PG_RUNTIME_USER"),
|
|
{},
|
|
environment("EVO_MIGRATION_CA")};
|
|
}
|
|
std::unique_ptr<Postgres> migration_tls_connection(const test::PostgresRelay& relay) {
|
|
auto db = take(Postgres::connect(migration_tls_options(relay), 2s),
|
|
"migration verify-full TLS connection");
|
|
take(db->query("SELECT 1", {}, 2s), "migration TLS warmup");
|
|
return db;
|
|
}
|
|
void migration_transport_commit(Fixture& f, const CellularMigrationCommitPrepared& prepared,
|
|
RegionKey sk, RegionKey dk, const LeaseGrant& sl,
|
|
const LeaseGrant& dl, const RegionHead& source_before,
|
|
const RegionHead& destination_before) {
|
|
{
|
|
test::PostgresRelay relay(migration_backend_port());
|
|
auto wrong = migration_tls_options(relay);
|
|
wrong.host = "untrusted.evolution.test";
|
|
require(!Postgres::connect(wrong, 1s), "relay bypassed TLS hostname verification");
|
|
}
|
|
{
|
|
test::PostgresRelay relay(migration_backend_port());
|
|
auto db = migration_tls_connection(relay);
|
|
relay.drop_requests(true);
|
|
const auto lost = retry_cellular_migration_commit(*db, sl, dl, prepared, 300ms);
|
|
require(
|
|
!lost && lost.error().kind == PublicationError::unknown &&
|
|
lost.error().prepared.source_manifest == prepared.source_manifest &&
|
|
lost.error().prepared.destination_manifest == prepared.destination_manifest &&
|
|
relay.dropped_requests() > 0 && relay.healthy() && !db->transaction_idle() &&
|
|
take(load_region_checkpoint(*f.db, sk, 5s), "lost request source head").head ==
|
|
source_before &&
|
|
take(load_region_checkpoint(*f.db, dk, 5s), "lost request destination head").head ==
|
|
destination_before,
|
|
"a dropped TLS request changed ownership or lost the exact pair");
|
|
}
|
|
{
|
|
test::PostgresRelay relay(migration_backend_port());
|
|
auto db = migration_tls_connection(relay);
|
|
relay.drop_responses(true);
|
|
const auto lost = retry_cellular_migration_commit(*db, sl, dl, prepared, 500ms);
|
|
require(
|
|
!lost && lost.error().kind == PublicationError::unknown &&
|
|
lost.error().prepared.source_manifest == prepared.source_manifest &&
|
|
lost.error().prepared.destination_manifest == prepared.destination_manifest &&
|
|
relay.dropped_responses() > 0 && relay.healthy() && !db->transaction_idle() &&
|
|
take(load_region_checkpoint(*f.db, sk, 5s), "lost reply source head").manifest ==
|
|
prepared.source_manifest &&
|
|
take(load_region_checkpoint(*f.db, dk, 5s), "lost reply destination head")
|
|
.manifest == prepared.destination_manifest,
|
|
"lost TLS receipt was treated as rollback or committed only one head");
|
|
}
|
|
std::puts("migration TLS request/reply loss and hostname verification passed");
|
|
}
|
|
void funded_source(Fixture& f) {
|
|
f.build.fill(0xDA);
|
|
const std::array<std::string, 1> build{hexadecimal(f.build)};
|
|
take(f.owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES($1::bytea) "
|
|
"ON CONFLICT DO NOTHING",
|
|
build, 5s),
|
|
"retain test build");
|
|
take(f.owner->query("INSERT INTO evolution.life_allocators VALUES(30201,100)", {}, 5s),
|
|
"test allocator");
|
|
f.objects = take(
|
|
FilesystemObjectStore::open(environment("PGHOST"), f.key, max_cellular_population_bytes),
|
|
"population object namespace");
|
|
f.raw = opening(30001, true, 4, false);
|
|
f.head =
|
|
take(provision_cellular_region(*f.owner, *f.objects, f.key, 1, *f.raw, f.id(), f.build, 5s),
|
|
"empty cellular genesis");
|
|
f.lease =
|
|
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "population test lease");
|
|
CellularBatch birth{f.header(1), {{}}};
|
|
for (std::uint8_t index = 1; index <= 4; ++index) {
|
|
Uuid account{};
|
|
account.back() = index;
|
|
const auto control =
|
|
take(claim_account_control(*f.db, f.key, f.lease, account, 1, index, 60000ms, 5s),
|
|
"founder account control");
|
|
const auto reserved = take(
|
|
reserve_primitive_life(*f.db, f.key, f.lease, account, control.generation, 1, 1, 5s),
|
|
"funded founder reservation");
|
|
birth.ticks.front().births.push_back({account, 1, control.generation, reserved.identity});
|
|
}
|
|
f.raw = take(replay_cellular_batch(*f.raw, birth), "funded founder replay");
|
|
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, birth, *f.raw, f.id(),
|
|
f.build, 5s),
|
|
"funded founder publication")
|
|
.committed;
|
|
while (f.raw->next_bud_sequence() == 1 && f.raw->tick() < 20000) {
|
|
CellularBatch batch{f.header(1), {}};
|
|
for (unsigned tick = 0; tick < 64; ++tick) {
|
|
auto intents =
|
|
take(wildlife_intents(*f.raw, f.raw->tick() + 1), "funded source wildlife");
|
|
for (const auto& life : f.raw->lives()) {
|
|
if (life.account == LifeAccount{})
|
|
continue;
|
|
CellularLifeIntent intent{life.identity.life, {}};
|
|
intent.intent.bud = (f.raw->tick() + life.identity.life) % 32 == 0;
|
|
intents.push_back(intent);
|
|
}
|
|
std::ranges::sort(intents, {}, &CellularLifeIntent::life_id);
|
|
take(f.raw->advance_tick(intents), "funded source biology");
|
|
batch.ticks.push_back({intents, {}, {}});
|
|
if (f.raw->next_bud_sequence() > 1)
|
|
break;
|
|
}
|
|
batch.header.end_tick = f.raw->tick();
|
|
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, batch, *f.raw,
|
|
f.id(), f.build, 5s),
|
|
"funded source batch")
|
|
.committed;
|
|
take(renew_region_lease(*f.db, f.key, f.lease, 60000ms, 5s), "source lease renewal");
|
|
}
|
|
require(std::ranges::any_of(f.raw->lives(),
|
|
[](const auto& life) { return life.account == LifeAccount{}; }),
|
|
"durable fixture has no funded player-derived wildlife");
|
|
require(!migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build,
|
|
{true, 1200, 1800}, 5s),
|
|
"population entry bypassed vulnerable controlled life");
|
|
const auto suspended_at = f.raw->tick() + cellular_linger_ticks + 1;
|
|
while (f.raw->tick() < suspended_at) {
|
|
auto h = f.header(std::min(std::uint64_t{64}, suspended_at - f.raw->tick()));
|
|
auto prefix = take(unattended_population_inputs(*f.raw, h), "raw linger inputs");
|
|
f.raw = take(replay_cellular_batch(*f.raw, prefix), "raw linger biology");
|
|
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, prefix, *f.raw,
|
|
f.id(), f.build, 5s),
|
|
"raw linger publication")
|
|
.committed;
|
|
std::vector<std::uint64_t> endings;
|
|
for (const auto& corpse : f.raw->corpses())
|
|
if (corpse.parent_life == 0)
|
|
endings.push_back(corpse.life_id);
|
|
if (!endings.empty())
|
|
take(end_cellular_lives(*f.db, f.key, f.lease, f.head.head.commit, endings, 5s),
|
|
"raw linger endings");
|
|
}
|
|
require(population_interval_safe(*f.raw),
|
|
"population source retained a vulnerable controlled life");
|
|
}
|
|
CellularPopulationExecution unattended(Fixture& f) {
|
|
auto working = restore(f.population->state());
|
|
CellularBatch batch{f.header(64), {}};
|
|
for (unsigned tick = 0; tick < 64; ++tick) {
|
|
auto intents = take(wildlife_intents(*working, working->tick() + 1), "unattended wildlife");
|
|
for (const auto& life : working->lives()) {
|
|
if (life.account == LifeAccount{})
|
|
continue;
|
|
CellularLifeIntent frozen{life.identity.life, {}};
|
|
frozen.intent.frozen = true;
|
|
intents.push_back(frozen);
|
|
}
|
|
std::ranges::sort(intents, {}, &CellularLifeIntent::life_id);
|
|
take(working->advance_tick(intents), "unattended fixture step");
|
|
batch.ticks.push_back({intents, {}, {}});
|
|
}
|
|
return take(prepare_population_batch(*f.population, std::move(batch)),
|
|
"population recorded interval");
|
|
}
|
|
void recover(Fixture& f) {
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
auto db = connection();
|
|
auto recovered =
|
|
take(recover_population_checkpoint(*db, *f.objects, f.key, f.lease, builds, 5s),
|
|
"population durable recovery");
|
|
f.head = std::move(recovered.committed);
|
|
f.population = std::move(recovered.state);
|
|
}
|
|
void conversions_and_retries(Fixture& f) {
|
|
const auto biology = f.raw->snapshot();
|
|
Uuid account{};
|
|
account.back() = 1;
|
|
const auto input =
|
|
encode_cellular_mutation_input({account, 9991, f.head.head.end_tick + 1, {100, 1, 1, 1}});
|
|
take(f.db->query("BEGIN", {}, 5s), "reservation guard transaction");
|
|
take(reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 9991, input, 5s),
|
|
"reserve pending player choice");
|
|
const auto blocked = migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
|
|
f.build, {true, 1200, 1800}, 5s);
|
|
require(!blocked && std::string(blocked.error().store.cause.sqlstate.data(), 5) == "EVL18",
|
|
"population entry stranded a reserved player choice");
|
|
take(f.db->query("ROLLBACK", {}, 5s), "rollback guard fixture");
|
|
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
|
|
f.build, {true, 1200, 1800}, 5s),
|
|
"initial population transition");
|
|
recover(f);
|
|
require(f.population->state().snapshot() == biology && f.population->members().empty(),
|
|
"transition changed biology");
|
|
const auto refused =
|
|
reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 9991, input, 5s);
|
|
require(!refused && std::string(refused.error().cause.sqlstate.data(), 5) == "EVL06",
|
|
"population profile admitted a player mutation reservation");
|
|
const auto wildlife = std::ranges::find_if(f.population->state().lives(), [](const auto& life) {
|
|
return life.account == LifeAccount{};
|
|
});
|
|
require(wildlife != f.population->state().lives().end(), "missing durable descendant");
|
|
const auto genome = wildlife->identity.genome;
|
|
auto conversion =
|
|
take(prepare_population_conversion(*f.population, f.header(0), {genome, true}),
|
|
"collapse sample");
|
|
auto bad = conversion.batch;
|
|
++bad.boundaries.front().sequence;
|
|
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, bad, *conversion.state,
|
|
f.id(), f.build, 5s),
|
|
"altered sample published");
|
|
const auto commit = f.id();
|
|
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, conversion.batch,
|
|
*conversion.state, commit, f.build, 5s),
|
|
"durable collapse");
|
|
const PopulationPreparedCommit prepared{
|
|
f.key, commit, f.head.manifest,
|
|
take(encode_population_batch(conversion.batch), "prepared trace")};
|
|
require(take(retry_population_prepared(*f.db, prepared, 5s), "idempotent retry").head ==
|
|
f.head.head,
|
|
"retry advanced head");
|
|
require(take(resolve_population_prepared(*f.db, prepared, 5s), "read-only resolve").status ==
|
|
Resolution::committed,
|
|
"commit unresolved");
|
|
auto future_batch = conversion.batch;
|
|
future_batch.format_version = 2;
|
|
auto future = prepared;
|
|
future.trace = take(encode_population_batch(future_batch), "future trace fixture");
|
|
auto future_manifest = take(decode_manifest(future.manifest), "future manifest fixture");
|
|
future_manifest.batch.trace_bytes = future.trace.size();
|
|
future_manifest.batch.trace_sha256 = take(sha256(future.trace), "future trace digest");
|
|
future.manifest = take(encode_manifest(future_manifest), "future manifest encoding");
|
|
auto rejected_future = retry_population_prepared(*f.db, future, 5s);
|
|
require(!rejected_future && rejected_future.error().kind == PublicationError::candidate,
|
|
"bundle 9 prepared retry admitted population trace 2");
|
|
require(!resolve_population_prepared(*f.db, future, 5s),
|
|
"bundle 9 prepared resolve admitted population trace 2");
|
|
auto rejected_sql = publish_population_checkpoint(*f.db, future.manifest, future.trace, 5s);
|
|
require(!rejected_sql && rejected_sql.error().kind == LeaseStoreError::invalid,
|
|
"schema 45 did not reject the unsupported trace format before commit lookup");
|
|
const auto current_manifest = take(decode_manifest(f.head.manifest), "upgrade source manifest");
|
|
const auto boundary_payload =
|
|
take(cellular_boundary_upgrade_payload(conversion.state->snapshot(), current_manifest),
|
|
"portable boundary upgrade source");
|
|
const auto boundary_manifest = take(
|
|
cellular_boundary_upgrade_manifest(
|
|
current_manifest, f.head.head.digest, f.id(), f.lease, f.build, boundary_payload.size(),
|
|
take(sha256(boundary_payload), "boundary payload digest")),
|
|
"portable boundary upgrade manifest");
|
|
const PopulationPreparedCommit boundary{
|
|
f.key,
|
|
boundary_manifest.commit,
|
|
take(encode_manifest(boundary_manifest), "boundary candidate encoding"),
|
|
{}};
|
|
auto rejected_boundary = retry_population_prepared(*f.db, boundary, 5s);
|
|
require(!rejected_boundary && rejected_boundary.error().kind == PublicationError::rejected &&
|
|
take(resolve_population_prepared(*f.db, boundary, 5s),
|
|
"upgrade through wrong journal resolution")
|
|
.status == Resolution::absent,
|
|
"accounting upgrade bypassed its dedicated version journal");
|
|
auto rejected_upgrade = publish_population_profile_transition(*f.db, boundary.manifest, 5s);
|
|
require(!rejected_upgrade,
|
|
"representation publisher admitted accounting upgrade without dedicated evidence");
|
|
require(!publish_region_checkpoint(*f.db, boundary.manifest, {}, 5s),
|
|
"generic publisher admitted boundary upgrade");
|
|
require(take(load_region_checkpoint(*f.db, f.key, 5s), "head after future rejection").head ==
|
|
f.head.head,
|
|
"future trace changed the retained durable head");
|
|
auto changed = prepared;
|
|
auto changed_manifest = take(decode_manifest(changed.manifest), "conflict manifest");
|
|
changed_manifest.payload_sha256[0] ^= 1;
|
|
changed.manifest = take(encode_manifest(changed_manifest), "conflict encoding");
|
|
require(take(resolve_population_prepared(*f.db, changed, 5s), "conflicting resolve").status ==
|
|
Resolution::conflict,
|
|
"changed commit was accepted");
|
|
require(!publish_region_checkpoint(*f.db, f.head.manifest, {}, 5s),
|
|
"generic publisher bypassed profile gate");
|
|
const auto expected = conversion.state->snapshot();
|
|
recover(f);
|
|
require(f.population->snapshot() == expected && !f.population->members().empty(),
|
|
"committed sample not recovered");
|
|
require(f.population->state().snapshot() == biology, "collapse healed/aged/moved biology");
|
|
for (unsigned oscillation = 0; oscillation < 8; ++oscillation) {
|
|
const bool collapse = oscillation % 2 != 0;
|
|
auto next =
|
|
take(prepare_population_conversion(*f.population, f.header(0), {genome, collapse}),
|
|
"conversion oscillation");
|
|
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, next.batch,
|
|
*next.state, f.id(), f.build, 5s),
|
|
"publish oscillation");
|
|
recover(f);
|
|
require(f.population->state().snapshot() == biology,
|
|
"durable conversion changed exact state");
|
|
}
|
|
}
|
|
void process_loss(Fixture& f, int point) {
|
|
auto next = unattended(f);
|
|
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, next.batch.header.end_tick,
|
|
5s),
|
|
"admit process-loss interval");
|
|
const auto predecessor = f.head.head;
|
|
const auto expected = next.state->snapshot();
|
|
const auto commit = f.id();
|
|
int channel[2]{};
|
|
require(::pipe(channel) == 0, "process acknowledgement pipe");
|
|
const auto child = ::fork();
|
|
require(child >= 0, "population fork");
|
|
if (child == 0) {
|
|
::close(channel[0]);
|
|
try {
|
|
auto db = connection();
|
|
PublishHooks hooks;
|
|
if (point == 0)
|
|
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
|
|
const auto published =
|
|
publish_population_batch(*db, *f.objects, f.lease, f.head, next.batch, *next.state,
|
|
commit, f.build, 5s, hooks);
|
|
if (!published)
|
|
::_exit(20);
|
|
if (point == 2) {
|
|
const char ack = 'A';
|
|
if (::write(channel[1], &ack, 1) != 1)
|
|
::_exit(21);
|
|
}
|
|
::kill(::getpid(), SIGKILL);
|
|
} catch (...) {
|
|
::_exit(22);
|
|
}
|
|
::_exit(23);
|
|
}
|
|
::close(channel[1]);
|
|
char ack{};
|
|
const auto acknowledged = ::read(channel[0], &ack, 1);
|
|
::close(channel[0]);
|
|
int status{};
|
|
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
|
|
WTERMSIG(status) == SIGKILL,
|
|
"child missed population crash boundary");
|
|
require((point == 2 && acknowledged == 1 && ack == 'A') || (point != 2 && acknowledged == 0),
|
|
"unexpected permanent acknowledgement");
|
|
recover(f);
|
|
const auto pending =
|
|
take(load_population_interval(*f.db, f.key, f.lease, 5s), "recover process interval");
|
|
require(pending.has_value() == (point == 0), "process loss changed interval settlement");
|
|
if (point == 0)
|
|
require(f.head.head == predecessor, "uncommitted process published a head");
|
|
else
|
|
require(f.head.head.commit == commit && f.population->snapshot() == expected,
|
|
"committed process loss changed state");
|
|
}
|
|
void ambiguous_publication(Fixture& f, bool terminate) {
|
|
auto next = unattended(f);
|
|
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, next.batch.header.end_tick,
|
|
5s),
|
|
"admit ambiguous interval");
|
|
const auto expected = next.state->snapshot();
|
|
auto locker = connection(true);
|
|
take(locker->query("BEGIN", {}, 5s), "population lease row transaction");
|
|
const std::array<std::string, 2> key{std::to_string(f.key.world), std::to_string(f.key.region)};
|
|
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=$1::evolution.u64 "
|
|
"AND region_id=$2::evolution.u64 FOR UPDATE",
|
|
key, 5s),
|
|
"hold population publication");
|
|
auto timed = connection();
|
|
auto unknown = publish_population_batch(*timed, *f.objects, f.lease, f.head, next.batch,
|
|
*next.state, f.id(), f.build, 300ms);
|
|
require(!unknown && unknown.error().kind == PublicationError::unknown,
|
|
"population publication failed to retain ambiguous request");
|
|
if (terminate) {
|
|
const std::array<std::string, 1> user{environment("EVO_PG_RUNTIME_USER")};
|
|
take(locker->query("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE "
|
|
"usename=$1 AND wait_event_type='Lock'",
|
|
user, 5s),
|
|
"terminate uncommitted population writer");
|
|
}
|
|
take(locker->query("ROLLBACK", {}, 5s), "release population publication");
|
|
auto resolver = connection();
|
|
bool resolved = false;
|
|
for (unsigned attempt = 0; attempt < 200; ++attempt) {
|
|
auto result = take(resolve_population_prepared(*resolver, unknown.error().prepared, 5s),
|
|
"resolve ambiguous population publication");
|
|
if (result.status == (terminate ? Resolution::absent : Resolution::committed)) {
|
|
resolved = true;
|
|
break;
|
|
}
|
|
std::this_thread::sleep_for(25ms);
|
|
}
|
|
require(resolved, "ambiguous population publication did not resolve");
|
|
f.head = take(retry_population_prepared(*resolver, unknown.error().prepared, 5s),
|
|
"retry exact population request");
|
|
recover(f);
|
|
require(f.population->snapshot() == expected, "ambiguous publication changed retained sample");
|
|
}
|
|
void interval_contract(Fixture& f) {
|
|
require(!take(load_population_interval(*f.db, f.key, f.lease, 5s), "empty interval journal")
|
|
.has_value(),
|
|
"settled source retained pending work");
|
|
auto next = unattended(f);
|
|
const auto source = f.head.head;
|
|
const auto admitted =
|
|
take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
|
|
"durable interval admission");
|
|
require(take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
|
|
"duplicate interval admission") == admitted,
|
|
"admission retry changed boundary");
|
|
auto changed =
|
|
admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 63, 5s);
|
|
require(!changed && changed.error().kind == LeaseStoreError::commit_conflict,
|
|
"source admitted two different intervals");
|
|
const auto expected = next.state->snapshot();
|
|
require(!f.population->members().empty(), "interval fixture lost its exact cohort");
|
|
const auto genome = f.population->members().front().genome;
|
|
auto conversion =
|
|
take(prepare_population_conversion(*f.population, f.header(0), {genome, false}),
|
|
"conversion during admitted work");
|
|
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, conversion.batch,
|
|
*conversion.state, f.id(), f.build, 5s),
|
|
"conversion skipped an admitted interval");
|
|
require(
|
|
!materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
|
|
"profile exit skipped an admitted interval");
|
|
auto shortened_header = f.header(63);
|
|
auto shortened =
|
|
take(prepare_population_batch(
|
|
*f.population,
|
|
take(unattended_population_inputs(f.population->state(), shortened_header),
|
|
"shortened input")),
|
|
"shortened interval");
|
|
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, shortened.batch,
|
|
*shortened.state, f.id(), f.build, 5s),
|
|
"publication partially consumed an interval");
|
|
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release interval owner");
|
|
const auto epoch = f.lease.epoch;
|
|
f.lease =
|
|
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "replace interval owner");
|
|
require(f.lease.epoch > epoch && take(take(load_population_interval(*f.db, f.key, f.lease, 5s),
|
|
"new owner interval load"),
|
|
"retained admission") == admitted,
|
|
"replacement owner lost admitted work");
|
|
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, next.batch,
|
|
*next.state, f.id(), f.build, 5s),
|
|
"settle exact admitted interval");
|
|
recover(f);
|
|
require(f.population->snapshot() == expected &&
|
|
!take(load_population_interval(*f.db, f.key, f.lease, 5s), "settled interval load")
|
|
.has_value(),
|
|
"settlement lost work or retained spendable pending state");
|
|
require(take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
|
|
"settled admission retry") == admitted &&
|
|
!take(load_population_interval(*f.db, f.key, f.lease, 5s), "settled retry pending")
|
|
.has_value(),
|
|
"old admission retry reopened settled work");
|
|
require(!admit_population_interval(*f.db, f.key, f.lease, f.head.head,
|
|
f.head.head.end_tick + 65, 5s),
|
|
"oversized interval admitted");
|
|
const auto denied = f.db->query(
|
|
"SELECT * FROM evolution.publish_population_checkpoint_v44(''::bytea,''::bytea)", {}, 5s);
|
|
require(!denied && std::string(denied.error().sqlstate.data(), 5) == "42501",
|
|
"runtime bypassed interval guard");
|
|
}
|
|
void isolation_and_reverse(Fixture& f) {
|
|
auto interval = unattended(f);
|
|
const auto old_lease = f.lease;
|
|
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release old authority");
|
|
auto new_db = connection();
|
|
f.lease = take(acquire_region_lease(*new_db, f.key, f.build, 60000ms, 5s),
|
|
"new population authority");
|
|
require(f.lease.epoch > old_lease.epoch, "authority epoch did not advance");
|
|
const auto old_head = f.head.head;
|
|
auto stale = publish_population_batch(*f.db, *f.objects, old_lease, f.head, interval.batch,
|
|
*interval.state, f.id(), f.build, 5s);
|
|
require(!stale && stale.error().store.kind == LeaseStoreError::fenced,
|
|
"stale population writer published");
|
|
recover(f);
|
|
require(f.head.head == old_head, "fenced writer changed head");
|
|
const auto expected = f.population->state().snapshot();
|
|
f.head = take(
|
|
materialize_population_profile(*new_db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
|
|
"return to cellular profile");
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
auto recovered =
|
|
take(recover_cellular_checkpoint(*new_db, *f.objects, f.key, f.lease, builds, 5s),
|
|
"cellular recovery after materialization");
|
|
require(recovered.state->snapshot() == expected &&
|
|
recovered.manifest.profile == CheckpointProfile::cellular_lives_v1,
|
|
"reverse transition changed biology or time");
|
|
f.raw = std::move(recovered.state);
|
|
}
|
|
void missing_and_corrupt_objects(Fixture& f) {
|
|
const auto manifest = take(decode_manifest(f.head.manifest), "object test manifest");
|
|
const auto path = f.objects->directory() / hexadecimal(manifest.payload_sha256, false);
|
|
const auto backup = path.string() + ".population-test";
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
std::filesystem::rename(path, backup);
|
|
auto missing = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
|
|
std::filesystem::rename(backup, path);
|
|
require(!missing && missing.error().kind == RecoveryError::missing_payload,
|
|
"missing population payload did not fail closed");
|
|
std::filesystem::rename(path, backup);
|
|
{
|
|
std::ofstream corrupt(path, std::ios::binary);
|
|
corrupt << 'X';
|
|
}
|
|
auto corrupt = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
|
|
std::filesystem::remove(path);
|
|
std::filesystem::rename(backup, path);
|
|
require(!corrupt && corrupt.error().kind == RecoveryError::corrupt_payload,
|
|
"corrupt population payload did not fail closed");
|
|
recover(f);
|
|
}
|
|
void history_pages(Fixture& f) {
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
const auto upper = f.head;
|
|
const auto biology = f.raw->snapshot();
|
|
const auto read = [&](const RegionHead& watermark, std::uint64_t first, std::size_t limit) {
|
|
return load_cellular_census_history(*f.db, *f.objects, f.key, f.lease, watermark, first,
|
|
limit, builds, 5s);
|
|
};
|
|
auto first = take(read(upper.head, 0, 2), "history first page");
|
|
require(first.entries.size() == 2 && first.next_sequence == 2 &&
|
|
first.entries.front().committed.head.sequence == 0 &&
|
|
first.entries.front().census.rows().empty(),
|
|
"history lost genesis or changed sequence pagination");
|
|
// Append two distinct no-tick cuts between requests; the old watermark stays pinned.
|
|
const auto population =
|
|
take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build,
|
|
{false, 1200, 1800}, 5s),
|
|
"history population cut");
|
|
f.head = take(
|
|
materialize_population_profile(*f.db, *f.objects, f.lease, population, f.id(), f.build, 5s),
|
|
"history raw cut");
|
|
std::uint64_t next = take(first.next_sequence, "history continuation cursor");
|
|
while (next <= upper.head.sequence) {
|
|
auto page = take(read(upper.head, next, 2), "pinned historical page");
|
|
require(page.watermark == upper.head && !page.entries.empty() && page.entries.size() <= 2,
|
|
"history changed its pinned upper cut");
|
|
for (const auto& entry : page.entries) {
|
|
const auto stored =
|
|
take(load_region_commit(*f.db, f.key, next, 5s), "history oracle cut");
|
|
const auto manifest = take(decode_manifest(stored.manifest), "history oracle manifest");
|
|
auto state =
|
|
take(restore_cellular_payload(
|
|
take(f.objects->get({manifest.payload_sha256, manifest.payload_bytes}),
|
|
"history oracle payload"),
|
|
manifest),
|
|
"history oracle biology");
|
|
const auto oracle = take(CellularCensus::observe(*state), "history oracle census");
|
|
require(entry.committed.head == stored.head &&
|
|
entry.committed.manifest == stored.manifest &&
|
|
std::ranges::equal(entry.census.rows(), oracle.rows()) &&
|
|
entry.census.boundary() == oracle.boundary(),
|
|
"history rebound old census to a newer or different committed cut");
|
|
++next;
|
|
}
|
|
require(
|
|
page.next_sequence ==
|
|
(next <= upper.head.sequence ? std::optional<std::uint64_t>{next} : std::nullopt),
|
|
"history cursor skipped or invented a commit");
|
|
}
|
|
const auto transitions = take(read(f.head.head, upper.head.sequence + 1, 8), "no-tick history");
|
|
require(transitions.entries.size() == 2 && !transitions.next_sequence &&
|
|
transitions.entries[0].committed.head == population.head &&
|
|
transitions.entries[1].committed.head == f.head.head &&
|
|
transitions.entries[0].census.boundary().population_format == 1 &&
|
|
transitions.entries[1].census.boundary().population_format == 0 &&
|
|
population.head.end_tick == upper.head.end_tick &&
|
|
f.head.head.end_tick == upper.head.end_tick,
|
|
"history collapsed distinct no-tick events or lost representation identity");
|
|
const auto repeat = take(read(upper.head, 0, 2), "repeat pinned history");
|
|
require(repeat.entries[0].committed.head == first.entries[0].committed.head &&
|
|
repeat.entries[1].committed.head == first.entries[1].committed.head &&
|
|
f.raw->snapshot() == biology &&
|
|
take(load_region_checkpoint(*f.db, f.key, 5s), "read-only history head").head ==
|
|
f.head.head,
|
|
"history read changed biology, head or event identity");
|
|
for (const auto limit :
|
|
{std::size_t{0}, max_cellular_history_page + 1, std::numeric_limits<std::size_t>::max()})
|
|
require(!read(upper.head, 0, limit) && read(upper.head, 0, limit).error().kind ==
|
|
CellularHistoryError::invalid_request,
|
|
"unbounded history request admitted");
|
|
require(!read(upper.head, upper.head.sequence + 1, 1), "history read beyond pinned cut");
|
|
auto wrong = upper.head;
|
|
wrong.digest.back() ^= 1;
|
|
require(!read(wrong, 0, 1) &&
|
|
read(wrong, 0, 1).error().kind == CellularHistoryError::cut_mismatch,
|
|
"history accepted a false watermark digest");
|
|
wrong = upper.head;
|
|
++wrong.end_tick;
|
|
require(!read(wrong, 0, 1), "history accepted a false watermark tick");
|
|
wrong = upper.head;
|
|
++wrong.frame;
|
|
require(!read(wrong, 0, 1), "history merged foreign frames");
|
|
require(!load_cellular_census_history(*f.db, *f.objects, {f.key.world, 999}, f.lease,
|
|
upper.head, 0, 1, builds, 5s) &&
|
|
!load_cellular_census_history(*f.db, *f.objects, f.key, f.lease, upper.head, 0, 1,
|
|
{}, 5s),
|
|
"foreign region or unretained build became an empty history");
|
|
const auto birth = take(load_region_commit(*f.db, f.key, 1, 5s), "historical birth cut");
|
|
const auto manifest = take(decode_manifest(birth.manifest), "historical birth manifest");
|
|
const auto path = f.objects->directory() / hexadecimal(manifest.payload_sha256, false);
|
|
const auto saved = path.string() + ".history-test";
|
|
const auto bytes = take(f.objects->get({manifest.payload_sha256, manifest.payload_bytes}),
|
|
"historical birth bytes");
|
|
std::filesystem::rename(path, saved);
|
|
const auto missing =
|
|
read(upper.head, 0, 2); // Genesis succeeded; second cut must fail the page.
|
|
std::filesystem::rename(saved, path);
|
|
require(!missing && missing.error().kind == CellularHistoryError::recovery,
|
|
"missing old payload returned a partial page or zero census");
|
|
{
|
|
std::ofstream file(path, std::ios::binary);
|
|
file << 'X';
|
|
require(file.good(), "corrupt old history payload fixture");
|
|
}
|
|
const auto corrupt = read(upper.head, 0, 2);
|
|
{
|
|
std::ofstream file(path, std::ios::binary);
|
|
file.write(reinterpret_cast<const char*>(bytes.data()),
|
|
static_cast<std::streamsize>(bytes.size()));
|
|
require(file.good(), "restore old history payload fixture");
|
|
}
|
|
require(!corrupt && corrupt.error().kind == CellularHistoryError::recovery,
|
|
"corrupt historical object returned a census");
|
|
const auto trace_path =
|
|
f.objects->directory() / hexadecimal(manifest.batch.trace_sha256, false);
|
|
const auto trace_saved = trace_path.string() + ".history-test";
|
|
std::filesystem::rename(trace_path, trace_saved);
|
|
const auto absent_trace = read(upper.head, 1, 1);
|
|
std::filesystem::rename(trace_saved, trace_path);
|
|
require(!absent_trace && absent_trace.error().recovery.kind == RecoveryError::tail,
|
|
"history trusted a payload without its replay evidence");
|
|
take(f.db->query("BEGIN", {}, 5s), "uncommitted history transaction");
|
|
const auto transactional = read(upper.head, 0, 1);
|
|
require(!f.db->transaction_idle() && !transactional &&
|
|
transactional.error().kind == CellularHistoryError::invalid_request,
|
|
"history admitted a caller-owned transaction");
|
|
const auto orphan =
|
|
take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build,
|
|
{false, 1200, 1800}, 5s),
|
|
"uncommitted history cut");
|
|
auto observer = connection();
|
|
const auto invisible = load_cellular_census_history(
|
|
*observer, *f.objects, f.key, f.lease, orphan.head, orphan.head.sequence, 1, builds, 5s);
|
|
take(f.db->query("ROLLBACK", {}, 5s), "rollback uncommitted history cut");
|
|
require(f.db->transaction_idle() && !invisible &&
|
|
take(load_region_checkpoint(*f.db, f.key, 5s), "orphan history head").head ==
|
|
f.head.head,
|
|
"uncommitted object/ghost became a durable history event");
|
|
auto old_lease = f.lease;
|
|
--old_lease.epoch;
|
|
const auto stale = load_cellular_census_history(*f.db, *f.objects, f.key, old_lease, upper.head,
|
|
1, 1, builds, 5s);
|
|
require(!stale && stale.error().recovery.store.kind == LeaseStoreError::fenced &&
|
|
take(read(upper.head, 0, 2), "current history reader").entries[1].committed.head ==
|
|
birth.head,
|
|
"historical evidence ignored a stale fence or rewrote original writers");
|
|
take(f.db->query("BEGIN READ ONLY", {}, 5s), "read-only history transaction");
|
|
const auto readonly = read(upper.head, 0, 1);
|
|
take(f.db->query("ROLLBACK", {}, 5s), "close read-only history transaction");
|
|
require(!readonly && readonly.error().kind == CellularHistoryError::invalid_request,
|
|
"history transaction requirement changed for read-only callers");
|
|
std::puts("committed census history pagination, replay, transaction and failure guards passed");
|
|
}
|
|
void registered_snapshot_coverage(Fixture& f) {
|
|
constexpr std::uint64_t seed = 31001, world = seed + 200;
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
auto reader = connection();
|
|
std::vector<std::unique_ptr<FilesystemObjectStore>> stores;
|
|
std::vector<CellularCensusRegionRequest> resources;
|
|
const auto enroll = [&](std::uint64_t region) {
|
|
const RegionKey key{world, region};
|
|
auto objects = take(
|
|
FilesystemObjectStore::open(environment("PGHOST"), key, max_cellular_population_bytes),
|
|
"registered empty region objects");
|
|
const auto state = opening(seed, true, 1, false, region);
|
|
take(provision_cellular_region(*f.owner, *objects, key, 1, *state, f.id(), f.build, 5s),
|
|
"registered empty region genesis");
|
|
const auto lease =
|
|
take(acquire_region_lease(*f.db, key, f.build, 60000ms, 5s), "registered region grant");
|
|
resources.push_back({region, lease, objects.get()});
|
|
stores.push_back(std::move(objects));
|
|
};
|
|
const auto read = [&](std::span<const CellularCensusRegionRequest> requests) {
|
|
return load_registered_cellular_census_snapshot(*reader, world, requests, builds, 5s);
|
|
};
|
|
const auto refused = [&](auto result, CellularCoverageError error) {
|
|
require(!result && result.error().kind == CellularSnapshotError::coverage &&
|
|
result.error().coverage.kind == error && reader->transaction_idle(),
|
|
"registered coverage omitted an owner or retained its transaction");
|
|
};
|
|
enroll(1);
|
|
const std::array first_request{resources.front()};
|
|
const auto before = take(read(first_request), "registered single-region census");
|
|
require(before.catalog == std::vector<RegisteredCellularRegion>{{1, 1}} &&
|
|
before.snapshot.cuts.size() == 1 && before.history_domain != Uuid{},
|
|
"registered snapshot lost catalog/cut evidence");
|
|
// Trusted owner metadata fault seam: change the namespace after head capture
|
|
// without changing biology. Real quarantined restore/fork is tested by the
|
|
// SQL and backup process qualifications, not by bypassing live authority here.
|
|
const auto domain_before =
|
|
take(f.owner->query("SELECT evolution.load_history_domain_v1()::text", {}, 5s),
|
|
"original domain");
|
|
const CellularCensusSnapshotHooks context_hook{[&](std::size_t) {
|
|
take(f.owner->query(
|
|
"WITH added AS (INSERT INTO evolution.history_domains(domain,parent,request,kind) "
|
|
"SELECT pg_catalog.gen_random_uuid(), domain, pg_catalog.gen_random_uuid(), "
|
|
"'restore' "
|
|
"FROM evolution.current_history_domain RETURNING domain) "
|
|
"UPDATE evolution.current_history_domain SET domain=(SELECT domain FROM added)",
|
|
{}, 5s),
|
|
"owner namespace change seam");
|
|
}};
|
|
const auto context_race = take(load_registered_cellular_census_snapshot(
|
|
*reader, world, first_request, builds, 5s, context_hook),
|
|
"context snapshot race");
|
|
const auto context_after = take(read(first_request), "next namespace snapshot");
|
|
require(context_race.history_domain == before.history_domain &&
|
|
context_after.history_domain != before.history_domain &&
|
|
context_race.snapshot.cuts[0].manifest == context_after.snapshot.cuts[0].manifest,
|
|
"namespace escaped the metadata snapshot or was inferred from unchanged cuts");
|
|
// Missing context must fail, never manufacture a namespace or a zero census.
|
|
take(f.owner->query("DELETE FROM evolution.current_history_domain", {}, 5s),
|
|
"owner missing context seam");
|
|
const auto absent_context = read(first_request);
|
|
require(!absent_context && absent_context.error().kind == CellularSnapshotError::context &&
|
|
reader->transaction_idle(),
|
|
"missing namespace was accepted");
|
|
require(domain_before.size() == 1 && domain_before[0].size() == 1, "original domain shape");
|
|
const std::array<std::string, 1> original_domain{
|
|
take(domain_before[0][0], "original domain value")};
|
|
take(f.owner->query("INSERT INTO evolution.current_history_domain VALUES(true,$1::uuid)",
|
|
original_domain, 5s),
|
|
"restore owner context seam");
|
|
const CellularCensusSnapshotHooks enrollment_hook{[&](std::size_t index) {
|
|
if (index == 0)
|
|
enroll(2); // Independent committed enrollment after the registry snapshot.
|
|
}};
|
|
const auto raced = take(load_registered_cellular_census_snapshot(*reader, world, first_request,
|
|
builds, 5s, enrollment_hook),
|
|
"registered snapshot concurrent enrollment");
|
|
require(raced.catalog == before.catalog &&
|
|
raced.snapshot.cuts[0].head == before.snapshot.cuts[0].head,
|
|
"concurrent enrollment changed an already selected registry cut");
|
|
refused(read(first_request), CellularCoverageError::regions);
|
|
auto extra = resources;
|
|
extra.push_back({999, resources.back().lease, stores.back().get()});
|
|
refused(read(extra), CellularCoverageError::regions);
|
|
auto wrong = resources;
|
|
++wrong.back().lease.frame;
|
|
refused(read(wrong), CellularCoverageError::frame);
|
|
wrong = resources;
|
|
wrong.back().region = 999;
|
|
refused(read(wrong), CellularCoverageError::regions);
|
|
wrong = resources;
|
|
wrong.back().region = wrong.front().region;
|
|
const auto duplicate = read(wrong);
|
|
require(!duplicate && duplicate.error().kind == CellularSnapshotError::invalid_request,
|
|
"registered snapshot accepted duplicate resources");
|
|
refused(
|
|
load_registered_cellular_census_snapshot(*reader, world + 1000, first_request, builds, 5s),
|
|
CellularCoverageError::unknown_world);
|
|
take(release_region_lease(*f.db, {world, 2}, resources[1].lease, 5s),
|
|
"release registered empty owner");
|
|
take(f.owner->query("UPDATE evolution.region_leases SET expires_at=clock_timestamp()-"
|
|
"interval '1 second' WHERE world_id=31201 AND region_id=1",
|
|
{}, 5s),
|
|
"expire registered empty owner");
|
|
const auto inactive = take(read(resources), "released and expired registry coverage");
|
|
require(inactive.catalog == std::vector<RegisteredCellularRegion>{{1, 1}, {2, 1}},
|
|
"registry hid released or expired holders");
|
|
// Genesis recovery has no fenced transition to read; this checks inclusion,
|
|
// not fresh ownership. No output grants authority to expired/released owners.
|
|
auto pending_store = take(FilesystemObjectStore::open(environment("PGHOST"), {world, 3},
|
|
max_cellular_population_bytes),
|
|
"pending registration objects");
|
|
const auto pending_state = opening(seed, true, 1, false, 3);
|
|
take(f.owner->query("BEGIN", {}, 5s), "pending registration transaction");
|
|
take(provision_cellular_region(*f.owner, *pending_store, {world, 3}, 1, *pending_state, f.id(),
|
|
f.build, 5s),
|
|
"pending registration genesis");
|
|
const auto isolated = read(resources);
|
|
take(f.owner->query("ROLLBACK", {}, 5s), "pending registration rollback");
|
|
require(take(isolated, "registered snapshot with pending enrollment").catalog ==
|
|
inactive.catalog,
|
|
"uncommitted enrollment entered a registered evidence cut");
|
|
take(reader->query("BEGIN READ ONLY", {}, 5s), "caller registered transaction");
|
|
const auto caller = read(resources);
|
|
take(reader->query("ROLLBACK", {}, 5s), "caller registered rollback");
|
|
require(!caller && caller.error().kind == CellularSnapshotError::invalid_request,
|
|
"registered snapshot took over a caller transaction");
|
|
for (std::uint64_t region = 3; region <= max_cellular_census_regions; ++region)
|
|
enroll(region);
|
|
const auto bounded = take(read(resources), "64 registered regions");
|
|
require(bounded.catalog.size() == max_cellular_census_regions &&
|
|
bounded.snapshot.cuts.size() == max_cellular_census_regions &&
|
|
bounded.catalog.back() == RegisteredCellularRegion{64, 1},
|
|
"registered coverage lost its last in-bound owner");
|
|
enroll(65);
|
|
refused(read(std::span{resources}.first(max_cellular_census_regions)),
|
|
CellularCoverageError::bounds);
|
|
const auto rows = take(reader->query("SELECT * FROM evolution.load_world_region_catalog_v1("
|
|
"31201::evolution.u64)",
|
|
{}, 5s),
|
|
"catalog overflow sentinel");
|
|
require(rows.size() == 65, "SQL registry silently truncated the overflow owner");
|
|
const RegionKey missing_key{world + 1, 1};
|
|
take(f.owner->query("INSERT INTO evolution.region_leases(world_id,region_id,frame_id) "
|
|
"VALUES(31202.00,1.0,1.00),(31202,18446744073709551615,1.000)",
|
|
{}, 5s),
|
|
"registered row with missing head fixture");
|
|
auto missing_store = take(FilesystemObjectStore::open(environment("PGHOST"), missing_key,
|
|
max_cellular_population_bytes),
|
|
"missing registered checkpoint store");
|
|
const std::array missing_resources{
|
|
CellularCensusRegionRequest{1, resources[0].lease, missing_store.get()},
|
|
CellularCensusRegionRequest{std::numeric_limits<std::uint64_t>::max(), resources[0].lease,
|
|
missing_store.get()}};
|
|
const auto missing = load_registered_cellular_census_snapshot(*reader, missing_key.world,
|
|
missing_resources, builds, 5s);
|
|
require(!missing && missing.error().kind == CellularSnapshotError::history &&
|
|
missing.error().region == 1 && reader->transaction_idle(),
|
|
"registered owner without a checkpoint disappeared from coverage");
|
|
const RegionKey unsupported_key{world + 2, 1};
|
|
const auto unsupported_manifest =
|
|
take(genesis_manifest(unsupported_key, 1, f.id(), 50000, f.build, 1, f.build),
|
|
"non-cellular registry manifest fixture");
|
|
const std::array<std::string, 1> manifest_parameter{
|
|
hexadecimal(take(encode_manifest(unsupported_manifest), "non-cellular fixture encoding"))};
|
|
take(f.owner->query("SELECT evolution.provision_region_checkpoint($1::bytea)",
|
|
manifest_parameter, 5s),
|
|
"non-cellular registered owner");
|
|
auto unsupported_store =
|
|
take(FilesystemObjectStore::open(environment("PGHOST"), unsupported_key,
|
|
max_cellular_population_bytes),
|
|
"unsupported registered store");
|
|
const std::array unsupported_resources{
|
|
CellularCensusRegionRequest{1, resources[0].lease, unsupported_store.get()}};
|
|
const auto unsupported = load_registered_cellular_census_snapshot(
|
|
*reader, unsupported_key.world, unsupported_resources, builds, 5s);
|
|
require(!unsupported && unsupported.error().kind == CellularSnapshotError::history &&
|
|
unsupported.error().history.recovery.kind == RecoveryError::unsupported &&
|
|
reader->transaction_idle(),
|
|
"unsupported registered owner was filtered out or reported as zero");
|
|
std::puts("registered world coverage, enrollment isolation and 64/65 bounds passed");
|
|
}
|
|
void snapshot_transport_faults(Fixture& f) {
|
|
if (!std::getenv("EVO_MIGRATION_TCP_PORT"))
|
|
return;
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
const std::array requests{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()}};
|
|
for (unsigned mode = 0; mode < 8; ++mode) {
|
|
test::PostgresRelay relay(migration_backend_port());
|
|
auto db = migration_tls_connection(relay);
|
|
const bool registered = mode >= 4;
|
|
const bool at_completion = mode % 4 >= 2;
|
|
const bool lost_request = mode % 2 == 0;
|
|
const auto partition = [&] {
|
|
if (lost_request)
|
|
relay.drop_requests(true);
|
|
else
|
|
relay.drop_responses(true);
|
|
};
|
|
if (!at_completion)
|
|
partition();
|
|
const CellularCensusSnapshotHooks hook{[&](std::size_t) {
|
|
if (at_completion)
|
|
partition();
|
|
}};
|
|
const auto failure = [](auto result) {
|
|
if (result)
|
|
throw std::runtime_error("lost capture I/O returned a census");
|
|
return result.error();
|
|
};
|
|
const auto lost = registered ? failure(load_registered_cellular_census_snapshot(
|
|
*db, f.key.world, requests, builds, 300ms, hook))
|
|
: failure(load_cellular_census_snapshot(
|
|
*db, f.key.world, requests, builds, 300ms, hook));
|
|
require(lost.kind == CellularSnapshotError::transaction &&
|
|
lost.transaction.kind == PgError::unknown && !db->transaction_idle() &&
|
|
relay.healthy() &&
|
|
(lost_request ? relay.dropped_requests() : relay.dropped_responses()) > 0 &&
|
|
take(load_region_checkpoint(*f.db, f.key, 5s), "head after lost capture I/O")
|
|
.head == f.head.head,
|
|
"lost snapshot BEGIN/COMMIT request/reply returned evidence or kept a session");
|
|
}
|
|
std::puts("snapshot TLS BEGIN/COMMIT request/reply loss fails closed");
|
|
}
|
|
void snapshot_guards(Fixture& f) {
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
const std::array requests{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()}};
|
|
const auto read = [&] {
|
|
return load_cellular_census_snapshot(*f.db, f.key.world, requests, builds, 5s);
|
|
};
|
|
const auto before = f.head;
|
|
const auto first = take(read(), "single-region captured snapshot");
|
|
const auto second = take(read(), "repeat captured snapshot");
|
|
require(first.cuts.size() == 1 && first.cuts[0].head == before.head &&
|
|
first.cuts[0].manifest == before.manifest &&
|
|
second.cuts[0].head == first.cuts[0].head &&
|
|
std::ranges::equal(
|
|
first.census.regions()[0].rows(),
|
|
take(CellularCensus::observe(*f.raw), "snapshot direct oracle").rows()) &&
|
|
f.db->transaction_idle(),
|
|
"captured snapshot lost its exact committed cut/biology or retained a transaction");
|
|
const auto invalid = [&](auto region_requests, std::uint64_t world = 30201,
|
|
std::chrono::milliseconds timeout = 5s) {
|
|
const auto result =
|
|
load_cellular_census_snapshot(*f.db, world, region_requests, builds, timeout);
|
|
require(!result && result.error().kind == CellularSnapshotError::invalid_request &&
|
|
f.db->transaction_idle(),
|
|
"invalid snapshot request started a transaction or returned a census");
|
|
};
|
|
invalid(std::vector<CellularCensusRegionRequest>{});
|
|
invalid(requests, 0);
|
|
invalid(requests, f.key.world, 0ms);
|
|
invalid(requests, f.key.world, 30001ms);
|
|
auto bad = requests;
|
|
bad[0].objects = nullptr;
|
|
invalid(bad);
|
|
bad = requests;
|
|
bad[0].region = 0;
|
|
invalid(bad);
|
|
bad = requests;
|
|
bad[0].lease.frame = 0;
|
|
invalid(bad);
|
|
invalid(std::array{requests[0], requests[0]});
|
|
auto higher = requests[0];
|
|
++higher.region;
|
|
invalid(std::array{higher, requests[0]});
|
|
invalid(std::vector<CellularCensusRegionRequest>(max_cellular_census_regions + 1, requests[0]));
|
|
higher.region = 999'999;
|
|
const auto missing = load_cellular_census_snapshot(*f.db, f.key.world,
|
|
std::array{requests[0], higher}, builds, 5s);
|
|
require(!missing && missing.error().kind == CellularSnapshotError::history &&
|
|
missing.error().region == higher.region && f.db->transaction_idle(),
|
|
"missing region became partial coverage or left capture open");
|
|
for (const auto* begin : {"BEGIN", "BEGIN READ ONLY"}) {
|
|
take(f.db->query(begin, {}, 5s), "caller snapshot transaction");
|
|
if (std::string_view{begin} == "BEGIN")
|
|
take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build,
|
|
{false, 1200, 1800}, 5s),
|
|
"pending caller snapshot publication");
|
|
const auto refused = read();
|
|
require(!refused && refused.error().kind == CellularSnapshotError::invalid_request &&
|
|
!f.db->transaction_idle(),
|
|
"snapshot took ownership of a caller transaction");
|
|
take(f.db->query("ROLLBACK", {}, 5s), "caller snapshot rollback");
|
|
}
|
|
const auto manifest = take(decode_manifest(before.manifest), "snapshot retained payload");
|
|
const auto path = f.objects->directory() / hexadecimal(manifest.payload_sha256, false);
|
|
const auto saved = path.string() + ".snapshot-test";
|
|
std::filesystem::rename(path, saved);
|
|
const auto missing_object = read();
|
|
std::filesystem::rename(saved, path);
|
|
require(!missing_object && missing_object.error().kind == CellularSnapshotError::history &&
|
|
f.db->transaction_idle(),
|
|
"missing replay payload became a snapshot or left capture open");
|
|
{
|
|
const UnreadableObject denied{path};
|
|
const auto unreadable = read();
|
|
require(!unreadable && unreadable.error().kind == CellularSnapshotError::history &&
|
|
f.db->transaction_idle(),
|
|
"unreadable captured payload became a census");
|
|
}
|
|
auto stale = requests;
|
|
--stale[0].lease.epoch;
|
|
const auto fenced = load_cellular_census_snapshot(*f.db, f.key.world, stale, builds, 5s);
|
|
require(!fenced && fenced.error().history.recovery.store.kind == LeaseStoreError::fenced,
|
|
"snapshot ignored stale evidence authority");
|
|
const std::array<Sha256Digest, 1> foreign_builds{};
|
|
require(!load_cellular_census_snapshot(*f.db, f.key.world, requests, foreign_builds, 5s),
|
|
"snapshot accepted unretained build evidence");
|
|
auto closed = connection();
|
|
CellularCensusSnapshotHooks close_hook{[&](std::size_t) { closed->disconnect(); }};
|
|
const auto lost =
|
|
load_cellular_census_snapshot(*closed, f.key.world, requests, builds, 5s, close_hook);
|
|
closed->disconnect();
|
|
const auto closed_query = closed->query("SELECT 1", {}, 5s);
|
|
require(!lost && lost.error().kind == CellularSnapshotError::transaction &&
|
|
!closed->transaction_idle() && !closed_query &&
|
|
closed_query.error().kind == PgError::closed,
|
|
"failed capture completion reused a dead session or returned a partial census");
|
|
auto throwing = connection();
|
|
bool threw = false;
|
|
bool hook_called = false;
|
|
try {
|
|
const CellularCensusSnapshotHooks hook{[&](std::size_t) {
|
|
hook_called = true;
|
|
throw std::runtime_error("capture hook test exception");
|
|
}};
|
|
const auto result =
|
|
load_cellular_census_snapshot(*throwing, f.key.world, requests, builds, 5s, hook);
|
|
require(!result, "throwing capture hook returned a snapshot");
|
|
} catch (const std::runtime_error&) {
|
|
threw = hook_called;
|
|
}
|
|
const auto after_throw = throwing->query("SELECT 1", {}, 5s);
|
|
require(threw && !after_throw && after_throw.error().kind == PgError::closed &&
|
|
!throwing->transaction_idle() &&
|
|
take(load_region_checkpoint(*f.db, f.key, 5s), "head after snapshot guards").head ==
|
|
before.head,
|
|
"capture exception retained its connection/transaction or changed a head");
|
|
std::puts("consistent census snapshot guards and connection cleanup passed");
|
|
snapshot_transport_faults(f);
|
|
}
|
|
void boundary_upgrades(Fixture& f, bool maturation = false) {
|
|
const auto migrate =
|
|
[maturation](Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
|
|
const CommittedCheckpoint& previous, Uuid commit, const Sha256Digest& build,
|
|
std::chrono::milliseconds timeout, const PublishHooks& hooks = {}) {
|
|
return (maturation ? migrate_cellular_maturation_profile
|
|
: migrate_cellular_boundary_profile)(db, objects, lease, previous,
|
|
commit, build, timeout, hooks);
|
|
};
|
|
const auto derive_manifest =
|
|
maturation ? cellular_maturation_upgrade_manifest : cellular_boundary_upgrade_manifest;
|
|
const auto target_bundle = maturation ? cellular_v11_versions : cellular_v10_versions;
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
const auto original = f.head;
|
|
if (maturation) {
|
|
const auto stale = f.lease;
|
|
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release pre-maturation authority");
|
|
f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s),
|
|
"new maturation authority");
|
|
const auto refused = migrate(*f.db, *f.objects, stale, f.head, f.id(), f.build, 5s);
|
|
require(!refused && refused.error().store.kind == LeaseStoreError::fenced &&
|
|
take(load_region_checkpoint(*f.db, f.key, 5s), "stale upgrade head").head ==
|
|
original.head,
|
|
"stale maturation writer changed head");
|
|
} else {
|
|
// A validly encoded target cannot skip the retained accounting predecessor.
|
|
const auto base = take(decode_manifest(f.head.manifest), "skipped upgrade source");
|
|
auto skipped = take(cellular_boundary_upgrade_manifest(base, f.head.head.digest, f.id(),
|
|
f.lease, f.build, base.payload_bytes,
|
|
base.payload_sha256),
|
|
"skipped target fixture");
|
|
skipped.cellular_versions = cellular_v11_versions;
|
|
require(!publish_cellular_profile_migration(
|
|
*f.db, take(encode_manifest(skipped), "skipped encode"), 5s) &&
|
|
take(load_region_checkpoint(*f.db, f.key, 5s), "skipped upgrade head").head ==
|
|
original.head,
|
|
"SQL maturation skipped bundle 10");
|
|
}
|
|
const auto expected_raw = take(maturation ? f.raw->maturation_upgrade_candidate()
|
|
: f.raw->migration_upgrade_candidate(),
|
|
"raw upgrade oracle")
|
|
->snapshot();
|
|
// Rehearse the raw upgrade, recovery and ordinary trace-3 publication in one
|
|
// transaction, then roll it back to retain a real bundle-9 source for profile 5.
|
|
take(f.db->query("BEGIN", {}, 5s), "raw upgrade rollback rehearsal");
|
|
auto raw_head = take(migrate(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
|
|
"raw accounting upgrade");
|
|
auto raw = take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s),
|
|
"raw accounting recovery");
|
|
require(raw.state->snapshot() == expected_raw &&
|
|
raw.manifest.cellular_versions == target_bundle &&
|
|
raw_head.head.end_tick == original.head.end_tick,
|
|
"raw accounting upgrade changed biology or time");
|
|
const CellularBatchHeader h{f.key.world,
|
|
f.key.region,
|
|
1,
|
|
raw_head.head.commit,
|
|
raw_head.head.digest,
|
|
raw_head.head.end_tick,
|
|
raw_head.head.end_tick + 1};
|
|
const auto inputs = take(unattended_population_inputs(*raw.state, h), "upgraded raw inputs");
|
|
auto advanced = take(replay_cellular_batch(*raw.state, inputs), "upgraded raw oracle");
|
|
take(publish_cellular_batch(*f.db, *f.objects, f.lease, raw_head, inputs, *advanced, f.id(),
|
|
f.build, 5s),
|
|
"upgraded raw ordinary publication");
|
|
require(take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s),
|
|
"upgraded raw ordinary recovery")
|
|
.state->snapshot() == advanced->snapshot(),
|
|
"raw accounting continuation differs");
|
|
take(f.db->query("ROLLBACK", {}, 5s), "roll back raw upgrade rehearsal");
|
|
require(take(load_region_checkpoint(*f.db, f.key, 5s), "raw rollback head").head ==
|
|
original.head,
|
|
"rolled-back upgrade moved the durable head");
|
|
|
|
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
|
|
f.build, {true, 1200, 1800}, 5s),
|
|
"population accounting source entry");
|
|
recover(f);
|
|
const auto member = std::ranges::find_if(f.population->state().lives(), [](const auto& life) {
|
|
return life.account == LifeAccount{};
|
|
});
|
|
require(member != f.population->state().lives().end(), "accounting source has no wildlife");
|
|
auto collapsed = take(
|
|
prepare_population_conversion(*f.population, f.header(0), {member->identity.genome, true}),
|
|
"accounting source cohort");
|
|
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, collapsed.batch,
|
|
*collapsed.state, f.id(), f.build, 5s),
|
|
"accounting source membership publication");
|
|
recover(f);
|
|
require(!f.population->members().empty() && f.population->conversion_sequence() != 0,
|
|
"accounting source lost cohort history");
|
|
auto interval = unattended(f);
|
|
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head,
|
|
interval.batch.header.end_tick, 5s),
|
|
"upgrade pending interval");
|
|
auto pending = migrate(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s);
|
|
require(!pending && std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18",
|
|
"accounting upgrade discarded admitted work");
|
|
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, interval.batch,
|
|
*interval.state, f.id(), f.build, 5s),
|
|
"settle pre-upgrade work");
|
|
recover(f);
|
|
const auto before = f.head;
|
|
const auto source = take(decode_manifest(before.manifest), "accounting source manifest");
|
|
const auto upgraded_population = take(maturation ? f.population->maturation_upgrade_candidate()
|
|
: f.population->migration_upgrade_candidate(),
|
|
"cohort upgrade oracle");
|
|
const auto expected = upgraded_population->snapshot();
|
|
const auto altered =
|
|
take(CellularPopulation::create(restore(upgraded_population->state()), {false, 1200, 1800}),
|
|
"valid unequal upgrade target");
|
|
require(altered->state().current_total() == upgraded_population->state().current_total() &&
|
|
altered->snapshot() != expected,
|
|
"unequal upgrade fixture must retain biological totals");
|
|
const auto altered_object = take(f.objects->put(altered->snapshot()), "unequal upgrade object");
|
|
const auto altered_manifest =
|
|
take(derive_manifest(source, before.head.digest, f.id(), f.lease, f.build,
|
|
altered_object.bytes, altered_object.sha256),
|
|
"unequal upgrade manifest");
|
|
take(f.db->query("BEGIN", {}, 5s), "unequal upgrade recovery rehearsal");
|
|
take(publish_cellular_profile_migration(
|
|
*f.db, take(encode_manifest(altered_manifest), "unequal upgrade encoding"), 5s),
|
|
"metadata-only unequal upgrade publication");
|
|
const auto unequal =
|
|
recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
|
|
take(f.db->query("ROLLBACK", {}, 5s), "undo unequal upgrade rehearsal");
|
|
require(!unequal && unequal.error().kind == RecoveryError::history &&
|
|
take(load_region_checkpoint(*f.db, f.key, 5s), "head after unequal upgrade").head ==
|
|
before.head,
|
|
"upgrade recovery compared only aggregate stock or rewrote the retained source");
|
|
const auto child = ::fork();
|
|
require(child >= 0, "upgrade pre-transaction fork");
|
|
if (child == 0) {
|
|
try {
|
|
auto db = connection();
|
|
PublishHooks hooks;
|
|
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
|
|
[[maybe_unused]] const auto result =
|
|
migrate(*db, *f.objects, f.lease, before, identity(60000), f.build, 5s, hooks);
|
|
} catch (...) {
|
|
::_exit(31);
|
|
}
|
|
::_exit(32);
|
|
}
|
|
int status{};
|
|
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
|
|
WTERMSIG(status) == SIGKILL,
|
|
"upgrade child missed pre-transaction kill");
|
|
recover(f);
|
|
require(f.head.head == before.head, "unpublished upgrade became authoritative");
|
|
|
|
auto locker = connection(true);
|
|
take(locker->query("BEGIN", {}, 5s), "accounting ambiguity lock transaction");
|
|
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 "
|
|
"AND region_id=1 FOR UPDATE",
|
|
{}, 5s),
|
|
"hold accounting reply");
|
|
auto timed = connection();
|
|
auto unknown = migrate(*timed, *f.objects, f.lease, before, f.id(), f.build, 300ms);
|
|
require(!unknown && unknown.error().kind == PublicationError::unknown &&
|
|
!unknown.error().prepared.manifest.empty(),
|
|
"uncertain accounting upgrade lost its exact request");
|
|
take(locker->query("ROLLBACK", {}, 5s), "release accounting reply");
|
|
auto resolver = connection();
|
|
f.head = take(retry_cellular_profile_migration(*resolver, unknown.error().prepared, 5s),
|
|
"retry exact accounting upgrade");
|
|
require(take(retry_cellular_profile_migration(*resolver, unknown.error().prepared, 5s),
|
|
"duplicate accounting upgrade")
|
|
.head == f.head.head,
|
|
"duplicate accounting upgrade advanced head");
|
|
recover(f);
|
|
require(f.population->snapshot() == expected && f.head.head.end_tick == before.head.end_tick &&
|
|
take(decode_manifest(f.head.manifest), "upgraded population manifest")
|
|
.cellular_versions == target_bundle,
|
|
"accounting upgrade changed biology, cohort history or time");
|
|
if (maturation) {
|
|
auto downgrade = take(decode_manifest(f.head.manifest), "downgrade fixture");
|
|
downgrade.predecessor_commit = downgrade.commit;
|
|
downgrade.predecessor_digest = f.head.head.digest;
|
|
downgrade.commit = f.id();
|
|
++downgrade.sequence;
|
|
downgrade.cellular_versions = cellular_v10_versions;
|
|
require(!publish_cellular_profile_migration(
|
|
*f.db, take(encode_manifest(downgrade), "downgrade encode"), 5s) &&
|
|
take(load_region_checkpoint(*f.db, f.key, 5s), "downgrade head").head ==
|
|
f.head.head,
|
|
"SQL downgraded maturation biology");
|
|
}
|
|
auto changed = unknown.error().prepared;
|
|
auto changed_manifest = take(decode_manifest(changed.manifest), "changed accounting manifest");
|
|
changed_manifest.payload_sha256[0] ^= 1;
|
|
changed.manifest = take(encode_manifest(changed_manifest), "changed accounting bytes");
|
|
auto conflict = retry_cellular_profile_migration(*resolver, changed, 5s);
|
|
require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict,
|
|
"accounting retry accepted changed bytes");
|
|
require(!migrate(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
|
|
"accounting upgrade applied twice");
|
|
const auto path = f.objects->directory() / hexadecimal(source.payload_sha256, false);
|
|
const auto backup = path.string() + ".upgrade-test";
|
|
std::filesystem::rename(path, backup);
|
|
const auto missing =
|
|
recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
|
|
std::filesystem::rename(backup, path);
|
|
require(!missing && missing.error().kind == RecoveryError::tail,
|
|
"accounting recovery trusted aggregate stock without its retained source");
|
|
missing_and_corrupt_objects(f);
|
|
const auto stale = f.lease;
|
|
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release accounting writer");
|
|
f.lease =
|
|
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "replace accounting writer");
|
|
auto stale_retry = retry_cellular_profile_migration(*f.db, unknown.error().prepared, 5s);
|
|
require(!stale_retry && stale_retry.error().store.kind == LeaseStoreError::fenced &&
|
|
!recover_population_checkpoint(*f.db, *f.objects, f.key, stale, builds, 5s),
|
|
"old accounting receipt granted current authority");
|
|
recover(f);
|
|
require(f.population->snapshot() == expected, "new epoch changed accounting upgrade");
|
|
auto future_interval = unattended(f);
|
|
auto old_trace = future_interval.batch;
|
|
old_trace.format_version = 1;
|
|
const auto old_bytes = take(encode_population_batch(old_trace), "wrong old trace fixture");
|
|
const auto upgraded_manifest = take(decode_manifest(f.head.manifest), "new accounting source");
|
|
auto wrong_trace_manifest =
|
|
take(population_successor_manifest(
|
|
upgraded_manifest, f.head.head.digest, f.id(), f.lease, old_trace.header.end_tick,
|
|
f.build, future_interval.state->snapshot().size(),
|
|
take(sha256(future_interval.state->snapshot()), "new payload SHA"),
|
|
{old_bytes.size(), take(sha256(old_bytes), "old trace SHA"), 0, {}}),
|
|
"wrong old trace manifest fixture");
|
|
const PopulationPreparedCommit wrong_trace{
|
|
f.key, wrong_trace_manifest.commit,
|
|
take(encode_manifest(wrong_trace_manifest), "wrong old trace manifest"), old_bytes};
|
|
auto refused_old = retry_population_prepared(*f.db, wrong_trace, 5s);
|
|
auto refused_old_sql =
|
|
publish_population_checkpoint(*f.db, wrong_trace.manifest, old_bytes, 5s);
|
|
require(!refused_old && refused_old.error().kind == PublicationError::candidate &&
|
|
!refused_old_sql && refused_old_sql.error().kind == LeaseStoreError::invalid &&
|
|
take(load_region_checkpoint(*f.db, f.key, 5s), "head after old trace").head ==
|
|
f.head.head,
|
|
"bundle 10 reinterpreted retained population trace 1");
|
|
if (maturation) {
|
|
auto growing = unattended(f);
|
|
bool grew = false;
|
|
for (const auto& life : growing.state->state().lives()) {
|
|
const auto old = std::ranges::find(f.population->state().lives(), life.identity.life,
|
|
[](const auto& cell) { return cell.identity.life; });
|
|
grew |= old != f.population->state().lives().end() &&
|
|
life.growth.realized_per_thousand > old->growth.realized_per_thousand;
|
|
}
|
|
require(grew, "durable maturation fixture never grew a real juvenile");
|
|
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head,
|
|
growing.batch.header.end_tick, 5s),
|
|
"admit funded growth interval");
|
|
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, growing.batch,
|
|
*growing.state, f.id(), f.build, 5s),
|
|
"funded maturation publication");
|
|
recover(f);
|
|
require(f.population->snapshot() == growing.state->snapshot(),
|
|
"funded growth recovery differs");
|
|
}
|
|
// Exercise the existing publication/process-loss matrix with batch 2 as well.
|
|
process_loss(f, 0);
|
|
process_loss(f, 1);
|
|
process_loss(f, 2);
|
|
isolation_and_reverse(f);
|
|
require(f.raw->format_version() == cellular_lives_migration_format,
|
|
"reverse transition downgraded accounting owner");
|
|
}
|
|
void migration_owner_partitions(Fixture& f, RegionKey destination_key,
|
|
FilesystemObjectStore& destination_objects,
|
|
LeaseGrant& destination_lease,
|
|
CommittedCheckpoint& destination_head, std::uint64_t genome) {
|
|
if (!std::getenv("EVO_MIGRATION_TCP_PORT"))
|
|
return;
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
for (unsigned lost_source = 0; lost_source < 2; ++lost_source) {
|
|
test::PostgresRelay source_relay(migration_backend_port()),
|
|
destination_relay(migration_backend_port());
|
|
const auto make_host = [&](RegionKey key, const test::PostgresRelay& relay) {
|
|
auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), key,
|
|
max_cellular_population_bytes),
|
|
"partition owner objects");
|
|
CellularHostConfig config{key, 1, {}, f.build, {f.build}};
|
|
config.lease = {3000ms, 400ms, 300ms, 150ms, 250ms};
|
|
const auto options = migration_tls_options(relay);
|
|
return take(CellularRegionHost::create(
|
|
config, [options] { return Postgres::connect(options, 500ms); },
|
|
std::move(objects)),
|
|
"partition owner host");
|
|
};
|
|
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release source to TLS owner");
|
|
take(release_region_lease(*f.db, destination_key, destination_lease, 5s),
|
|
"release destination to TLS owner");
|
|
auto source_host = make_host(f.key, source_relay),
|
|
destination_host = make_host(destination_key, destination_relay);
|
|
require(source_host->step().ready && destination_host->step().ready,
|
|
"TLS owners before freeze");
|
|
const auto sl = take(source_host->authority(), "TLS source fence"),
|
|
dl = take(destination_host->authority(), "TLS destination fence");
|
|
const auto source_bytes = source_host->state()->snapshot(),
|
|
destination_bytes = destination_host->state()->snapshot();
|
|
const auto source_before = f.head, destination_before = destination_head;
|
|
const CellularMigrationRequest request{
|
|
f.id(),
|
|
{f.key.region, f.head.head.commit, f.id(), f.head.head.digest},
|
|
{destination_key.region, destination_head.head.commit, f.id(),
|
|
destination_head.head.digest},
|
|
genome};
|
|
source_relay.drop_responses(true);
|
|
const auto unknown_prepare =
|
|
source_host->prepare_migration_freeze(destination_head, destination_objects, request);
|
|
require(!unknown_prepare &&
|
|
unknown_prepare.error().publication == PublicationError::unknown &&
|
|
!source_host->status().ready && source_relay.dropped_responses() > 0,
|
|
"lost preparation receipt did not pause source ownership");
|
|
const auto frozen =
|
|
take(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, sl, 5s),
|
|
"independent committed preparation observation"),
|
|
"missing preparation after TLS reply loss");
|
|
source_relay.drop_responses(false);
|
|
const auto reconcile_source_until = LeaseClock::now() + 2s;
|
|
while (!source_host->status().migration_frozen &&
|
|
LeaseClock::now() < reconcile_source_until) {
|
|
source_host->step();
|
|
destination_host->step();
|
|
std::this_thread::sleep_for(25ms);
|
|
}
|
|
require(source_host->status().migration_frozen && !source_host->status().ready,
|
|
"source lost the exact prepared request after TLS reconnect");
|
|
destination_relay.drop_responses(true);
|
|
const auto unknown_stage =
|
|
destination_host->stage_migration(f.head, *f.objects, frozen.trace);
|
|
require(!unknown_stage && unknown_stage.error().publication == PublicationError::unknown &&
|
|
!destination_host->status().ready &&
|
|
destination_relay.dropped_responses() > 0 &&
|
|
take(load_cellular_migration_stage(*f.db, destination_objects, destination_key,
|
|
dl, 5s),
|
|
"independent committed stage observation")
|
|
.has_value(),
|
|
"lost stage receipt activated the ghost or lost retained work");
|
|
destination_relay.drop_responses(false);
|
|
const auto reconcile_destination_until = LeaseClock::now() + 2s;
|
|
while (!destination_host->status().migration_frozen &&
|
|
LeaseClock::now() < reconcile_destination_until) {
|
|
source_host->step();
|
|
destination_host->step();
|
|
std::this_thread::sleep_for(25ms);
|
|
}
|
|
require(source_host->status().migration_frozen &&
|
|
destination_host->status().migration_frozen,
|
|
"TLS owners failed exact stage reconciliation");
|
|
auto& isolated = lost_source ? source_host : destination_host;
|
|
auto& healthy = lost_source ? destination_host : source_host;
|
|
auto& relay = lost_source ? source_relay : destination_relay;
|
|
const auto isolated_key = lost_source ? f.key : destination_key;
|
|
auto& replacement = lost_source ? f.lease : destination_lease;
|
|
const auto healthy_key = lost_source ? destination_key : f.key;
|
|
auto& healthy_lease = lost_source ? destination_lease : f.lease;
|
|
relay.drop_requests(true);
|
|
const auto expire_until = LeaseClock::now() + 3500ms;
|
|
while (LeaseClock::now() < expire_until) {
|
|
isolated->step();
|
|
healthy->step();
|
|
std::this_thread::sleep_for(25ms);
|
|
}
|
|
require(!isolated->status().ready && !isolated->authority().has_value() &&
|
|
!isolated->advance_unattended(64) && relay.dropped_requests() > 0 &&
|
|
healthy->status().phase == HostPhase::running &&
|
|
healthy->status().epoch == (lost_source ? dl.epoch : sl.epoch) &&
|
|
healthy->status().migration_frozen,
|
|
"isolated owner retained authority or healthy peer released a frozen ghost");
|
|
replacement = take(acquire_region_lease(*f.db, isolated_key, f.build, 60000ms, 5s),
|
|
"replacement after actual TLS partition expiry");
|
|
require(replacement.epoch > (lost_source ? sl.epoch : dl.epoch),
|
|
"replacement reused partitioned epoch");
|
|
if (lost_source)
|
|
require(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, replacement, 5s),
|
|
"replacement source retains freeze")
|
|
.has_value(),
|
|
"source lease expiry implicitly aborted");
|
|
else
|
|
require(take(load_cellular_migration_stage(*f.db, destination_objects, destination_key,
|
|
replacement, 5s),
|
|
"replacement destination retains stage")
|
|
.has_value(),
|
|
"destination lease expiry activated ghost");
|
|
const auto stale =
|
|
publish_cellular_migration(*f.db, *f.objects, destination_objects, sl, dl, f.head,
|
|
destination_head, frozen.trace, f.build, 5s);
|
|
require(!stale && stale.error().store.kind == LeaseStoreError::fenced &&
|
|
take(load_region_checkpoint(*f.db, f.key, 5s), "partition source head").head ==
|
|
source_before.head &&
|
|
take(load_region_checkpoint(*f.db, destination_key, 5s),
|
|
"partition destination head")
|
|
.head == destination_before.head,
|
|
"stale partitioned fence published a migration half");
|
|
healthy->step();
|
|
const auto current_source = lost_source ? replacement : sl;
|
|
if (lost_source) {
|
|
test::PostgresRelay abort_relay(migration_backend_port());
|
|
auto db = migration_tls_connection(abort_relay);
|
|
abort_relay.drop_responses(true);
|
|
const auto unknown =
|
|
abort_cellular_migration_freeze(*db, f.key, current_source, request.request, 300ms);
|
|
require(!unknown && unknown.error().kind == LeaseStoreError::unknown &&
|
|
abort_relay.dropped_responses() > 0 && abort_relay.healthy(),
|
|
"lost replacement abort receipt was not unknown");
|
|
take(abort_cellular_migration_freeze(*f.db, f.key, current_source, request.request, 5s),
|
|
"exact replacement abort reconciliation");
|
|
} else {
|
|
source_relay.drop_responses(true);
|
|
const auto unknown = source_host->abort_migration_freeze(request.request);
|
|
require(!unknown && !source_host->status().ready &&
|
|
source_relay.dropped_responses() > 0,
|
|
"lost owning-source abort receipt resumed the region");
|
|
source_relay.drop_responses(false);
|
|
}
|
|
require(!take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, current_source, 5s),
|
|
"independent abort source observation")
|
|
.has_value(),
|
|
"lost abort receipt reopened source stock");
|
|
const auto resume_until = LeaseClock::now() + 2s;
|
|
while (!healthy->step().ready && LeaseClock::now() < resume_until)
|
|
std::this_thread::sleep_for(25ms);
|
|
require(healthy->status().ready && healthy->state()->snapshot() ==
|
|
(lost_source ? destination_bytes : source_bytes),
|
|
"healthy owner resumed different stock after abort");
|
|
relay.drop_requests(false);
|
|
isolated->step();
|
|
require(!isolated->status().ready && !isolated->advance_unattended(64),
|
|
"healed stale owner resumed despite a replacement fence");
|
|
const auto isolated_manifest =
|
|
take(decode_manifest(lost_source ? f.head.manifest : destination_head.manifest),
|
|
"isolated head profile");
|
|
const auto& objects = lost_source ? *f.objects : destination_objects;
|
|
std::vector<std::uint8_t> recovered;
|
|
if (isolated_manifest.profile == CheckpointProfile::cellular_population_v1)
|
|
recovered = take(recover_population_checkpoint(*f.db, objects, isolated_key,
|
|
replacement, builds, 5s),
|
|
"replacement population replay")
|
|
.state->state()
|
|
.snapshot();
|
|
else
|
|
recovered = take(recover_cellular_checkpoint(*f.db, objects, isolated_key, replacement,
|
|
builds, 5s),
|
|
"replacement raw replay")
|
|
.state->snapshot();
|
|
require(recovered == (lost_source ? source_bytes : destination_bytes) &&
|
|
source_relay.healthy() && destination_relay.healthy(),
|
|
"replacement or relay lost exact stock");
|
|
source_host->drain();
|
|
destination_host->drain();
|
|
healthy_lease = take(acquire_region_lease(*f.db, healthy_key, f.build, 60000ms, 5s),
|
|
"return healthy TLS owner authority");
|
|
}
|
|
std::puts("migration independent TLS owner partitions, expiry, replacement and heal passed");
|
|
}
|
|
|
|
void migration_snapshot_race(Fixture& f, RegionKey destination_key,
|
|
FilesystemObjectStore& destination_objects,
|
|
const LeaseGrant& destination_lease,
|
|
CommittedCheckpoint& destination_head, std::uint64_t genome) {
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
const std::array requests{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()},
|
|
CellularCensusRegionRequest{destination_key.region, destination_lease,
|
|
&destination_objects}};
|
|
auto reader = connection();
|
|
for (const bool reverse : {false, true}) {
|
|
auto& sh = reverse ? destination_head : f.head;
|
|
auto& dh = reverse ? f.head : destination_head;
|
|
auto& so = reverse ? destination_objects : *f.objects;
|
|
auto& dst = reverse ? *f.objects : destination_objects;
|
|
const auto& sl = reverse ? destination_lease : f.lease;
|
|
const auto& dl = reverse ? f.lease : destination_lease;
|
|
const auto sk = reverse ? destination_key : f.key;
|
|
const auto dk = reverse ? f.key : destination_key;
|
|
const auto before_registered = take(
|
|
load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s),
|
|
"before snapshot migration race");
|
|
const auto& before = before_registered.snapshot;
|
|
CellularMigrationRequest request{f.id(),
|
|
{sk.region, sh.head.commit, f.id(), sh.head.digest},
|
|
{dk.region, dh.head.commit, f.id(), dh.head.digest},
|
|
genome};
|
|
const auto read = [&] {
|
|
return load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds,
|
|
5s);
|
|
};
|
|
const auto unchanged_stock = [&](const auto& snapshot) {
|
|
require(take(snapshot.snapshot.census.query(genome, CellularCensusScope::descendants),
|
|
"pending lineage") ==
|
|
take(before.census.query(genome, CellularCensusScope::descendants),
|
|
"before pending lineage"),
|
|
"pending evidence added stock");
|
|
};
|
|
const auto refused = [&](const auto& result, RecoveryError error) {
|
|
require(!result && result.error().kind == CellularSnapshotError::migration &&
|
|
result.error().migration.kind == error && reader->transaction_idle(),
|
|
"bad pending evidence returned a partial census or retained a transaction");
|
|
};
|
|
std::optional<CellularMigrationFreeze> preparation;
|
|
const CellularCensusSnapshotHooks prepare_hook{[&](std::size_t i) {
|
|
if (i == 0)
|
|
preparation =
|
|
take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s),
|
|
"capture prepare");
|
|
}};
|
|
const auto before_prepare =
|
|
take(load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds,
|
|
5s, prepare_hook),
|
|
"before concurrent prepare");
|
|
auto frozen = take(std::move(preparation), "preparation hook ran");
|
|
require(before_registered.migrations.empty() && before_prepare.migrations.empty(),
|
|
"concurrent preparation escaped its metadata view");
|
|
const auto prepared = take(read(), "prepared snapshot inventory");
|
|
require(prepared.migrations.size() == 1 && !prepared.migrations[0].staged &&
|
|
prepared.migrations[0].trace == frozen.trace,
|
|
"source-only inventory missing or treated as staged");
|
|
unchanged_stock(prepared);
|
|
const auto unretained = load_registered_cellular_census_snapshot(
|
|
*reader, f.key.world, requests, std::span<const Sha256Digest>{}, 5s);
|
|
refused(unretained, RecoveryError::unsupported);
|
|
const auto dm = take(decode_manifest(dh.manifest), "pending destination capsule");
|
|
{
|
|
UnreadableObject denied(so.directory() / hexadecimal(dm.payload_sha256, false));
|
|
refused(read(), RecoveryError::tail);
|
|
}
|
|
// Trusted owner fault seam: a pointer must not hide a foreign endpoint.
|
|
const std::array<std::string, 2> pointer{std::to_string(f.key.world),
|
|
std::to_string(sk.region)};
|
|
take(f.owner->query("UPDATE evolution.cellular_migration_pending SET region_id=99999 "
|
|
"WHERE world_id=$1::evolution.u64 AND region_id=$2::evolution.u64",
|
|
pointer, 5s),
|
|
"malformed pending pointer seam");
|
|
const auto malformed = read();
|
|
take(f.owner->query(
|
|
"UPDATE evolution.cellular_migration_pending SET region_id=$2::evolution.u64 "
|
|
"WHERE world_id=$1::evolution.u64 AND region_id=99999",
|
|
pointer, 5s),
|
|
"restore pending pointer seam");
|
|
refused(malformed, RecoveryError::history);
|
|
if (!reverse) {
|
|
const auto definition =
|
|
take(f.owner->query("SELECT "
|
|
"pg_get_functiondef('evolution.load_world_cellular_migrations_"
|
|
"v1(evolution.u64)'::regprocedure)",
|
|
{}, 5s),
|
|
"inventory definition");
|
|
const auto original = take(definition.at(0).at(0), "inventory definition value");
|
|
take(
|
|
f.owner->query(
|
|
"CREATE OR REPLACE FUNCTION evolution.load_world_cellular_migrations_v1(w "
|
|
"evolution.u64) "
|
|
"RETURNS TABLE(trace bytea,source_manifest bytea,destination_manifest bytea,"
|
|
"ghost_sha256 bytea,ghost_bytes evolution.u64,coherent boolean) "
|
|
"LANGUAGE sql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ "
|
|
"SELECT "
|
|
"NULL::bytea,NULL::bytea,NULL::bytea,NULL::bytea,NULL::evolution.u64,true "
|
|
"FROM pg_catalog.generate_series(1,65) $$",
|
|
{}, 5s),
|
|
"overflow response seam");
|
|
const auto overflow = read();
|
|
take(f.owner->query(original, {}, 5s), "restore bounded inventory function");
|
|
refused(overflow, RecoveryError::store);
|
|
require(overflow.error().migration.store.cause.kind == PgError::bounds,
|
|
"65th response was not rejected before parsing");
|
|
}
|
|
std::optional<CellularMigrationStage> stage;
|
|
const CellularCensusSnapshotHooks stage_hook{[&](std::size_t i) {
|
|
if (i == 0)
|
|
stage = take(stage_cellular_migration(*f.db, so, dst, dl, sh, dh, frozen.trace, 5s),
|
|
"capture stage");
|
|
}};
|
|
const auto before_stage = take(load_registered_cellular_census_snapshot(
|
|
*reader, f.key.world, requests, builds, 5s, stage_hook),
|
|
"before concurrent stage");
|
|
const auto staged = take(read(), "staged snapshot inventory");
|
|
require(before_stage.migrations.size() == 1 && !before_stage.migrations[0].staged &&
|
|
staged.migrations.size() == 1 && staged.migrations[0].staged,
|
|
"concurrent stage escaped the capture view");
|
|
unchanged_stock(staged);
|
|
const auto ghost = take(std::move(stage), "stage hook ran").ghost;
|
|
const auto ghost_path = dst.directory() / hexadecimal(ghost.sha256, false);
|
|
{
|
|
UnreadableObject denied(ghost_path);
|
|
refused(read(), RecoveryError::tail);
|
|
}
|
|
const CellularCensusSnapshotHooks abort_hook{[&](std::size_t i) {
|
|
if (i == 0)
|
|
take(abort_cellular_migration_freeze(*f.db, sk, sl, frozen.trace.request.request,
|
|
5s),
|
|
"capture abort");
|
|
}};
|
|
const auto before_abort = take(load_registered_cellular_census_snapshot(
|
|
*reader, f.key.world, requests, builds, 5s, abort_hook),
|
|
"before concurrent abort");
|
|
const auto after_abort = take(read(), "after abort inventory");
|
|
require(before_abort.migrations.size() == 1 && before_abort.migrations[0].staged &&
|
|
after_abort.migrations.empty(),
|
|
"abort escaped the selected inventory view");
|
|
unchanged_stock(before_abort);
|
|
unchanged_stock(after_abort);
|
|
request.request = f.id();
|
|
request.source.commit = f.id();
|
|
request.destination.commit = f.id();
|
|
frozen = take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s),
|
|
"snapshot race source freeze");
|
|
take(stage_cellular_migration(*f.db, so, dst, dl, sh, dh, frozen.trace, 5s),
|
|
"snapshot race destination stage");
|
|
std::optional<CellularMigrationCommitted> published;
|
|
const CellularCensusSnapshotHooks hook{[&](std::size_t index) {
|
|
if (index != 0)
|
|
return;
|
|
const auto settings =
|
|
take(reader->query("SELECT current_setting('transaction_isolation'), "
|
|
"current_setting('transaction_read_only')",
|
|
{}, 5s),
|
|
"capture transaction properties");
|
|
require(settings.size() == 1 && settings[0].size() == 2 &&
|
|
take(settings[0][0], "capture isolation") == "repeatable read" &&
|
|
take(settings[0][1], "capture access") == "on",
|
|
"capture did not own a repeatable read read-only snapshot");
|
|
auto publisher = connection();
|
|
published = take(publish_cellular_migration(*publisher, so, dst, sl, dl, sh, dh,
|
|
frozen.trace, f.build, 5s),
|
|
"publish real pair between snapshot head reads");
|
|
}};
|
|
const auto raced_registered = take(load_registered_cellular_census_snapshot(
|
|
*reader, f.key.world, requests, builds, 5s, hook),
|
|
"coherent census across a concurrent pair");
|
|
const auto& raced = raced_registered.snapshot;
|
|
const auto committed =
|
|
take(std::move(published), "pair was not published in snapshot hook");
|
|
sh = committed.source;
|
|
dh = committed.destination;
|
|
require(reader->transaction_idle() && raced.cuts.size() == 2 && before.cuts.size() == 2,
|
|
"raced capture retained its transaction or lost coverage");
|
|
for (std::size_t i = 0; i < before.cuts.size(); ++i)
|
|
require(raced.cuts[i].head == before.cuts[i].head &&
|
|
raced.cuts[i].manifest == before.cuts[i].manifest &&
|
|
std::ranges::equal(raced.census.regions()[i].rows(),
|
|
before.census.regions()[i].rows()),
|
|
"atomic pair split a captured cut or changed historical replay");
|
|
const auto after_registered = take(
|
|
load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s),
|
|
"snapshot after pair publication");
|
|
const auto& after = after_registered.snapshot;
|
|
require(raced_registered.migrations.size() == 1 && raced_registered.migrations[0].staged &&
|
|
raced_registered.migrations[0].trace == frozen.trace &&
|
|
after_registered.migrations.empty(),
|
|
"commit escaped the captured inventory or retained settled pointers");
|
|
require(before_registered.catalog == raced_registered.catalog &&
|
|
after_registered.catalog == before_registered.catalog &&
|
|
before_registered.catalog ==
|
|
std::vector<RegisteredCellularRegion>{
|
|
{f.key.region, f.lease.frame},
|
|
{destination_key.region, destination_lease.frame}},
|
|
"atomic migration changed or omitted registered owner coverage");
|
|
require(after.cuts[0].head == f.head.head && after.cuts[1].head == destination_head.head &&
|
|
after.cuts[0].head.sequence == before.cuts[0].head.sequence + 1 &&
|
|
after.cuts[1].head.sequence == before.cuts[1].head.sequence + 1,
|
|
"later snapshot omitted one committed migration endpoint");
|
|
for (const auto scope :
|
|
{CellularCensusScope::exact_genome, CellularCensusScope::descendants}) {
|
|
const auto initial = take(before.census.query(genome, scope), "before pair lineage");
|
|
const auto captured = take(raced.census.query(genome, scope), "captured pair lineage");
|
|
const auto settled = take(after.census.query(genome, scope), "settled pair lineage");
|
|
require(initial == captured && initial.resident == settled.resident,
|
|
"snapshot migration race lost/duplicated living count/body/reserve");
|
|
}
|
|
}
|
|
std::puts("consistent raw/cohort snapshots across atomic migration commits passed");
|
|
}
|
|
void migration_publication(Fixture& f, const RegionKey destination_key,
|
|
FilesystemObjectStore& destination_objects,
|
|
LeaseGrant& destination_lease, CommittedCheckpoint& destination_head) {
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
const auto wildlife = std::ranges::find_if(
|
|
f.raw->lives(), [](const auto& life) { return life.account == LifeAccount{}; });
|
|
require(wildlife != f.raw->lives().end(), "paired publication needs real wildlife");
|
|
const auto genome = wildlife->identity.genome;
|
|
const auto make_host = [&](RegionKey key) {
|
|
auto objects = take(
|
|
FilesystemObjectStore::open(environment("PGHOST"), key, max_cellular_population_bytes),
|
|
"paired host objects");
|
|
CellularHostConfig config{key, 1, {}, f.build, {f.build}};
|
|
config.lease.commit_timeout = 300ms;
|
|
return take(
|
|
CellularRegionHost::create(
|
|
config,
|
|
[] { return std::expected<std::unique_ptr<Postgres>, PgFailure>{connection()}; },
|
|
std::move(objects)),
|
|
"paired host");
|
|
};
|
|
for (unsigned representation = 0; representation < 2; ++representation) {
|
|
if (representation == 0) {
|
|
f.head = take(materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(),
|
|
f.build, 5s),
|
|
"paired raw source");
|
|
destination_head =
|
|
take(materialize_population_profile(*f.db, destination_objects, destination_lease,
|
|
destination_head, f.id(), f.build, 5s),
|
|
"paired raw destination");
|
|
} else {
|
|
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
|
|
f.build, {true, 1200, 1800}, 5s),
|
|
"paired population source");
|
|
destination_head = take(migrate_cellular_to_population(
|
|
*f.db, destination_objects, destination_lease,
|
|
destination_head, f.id(), f.build, {true, 1200, 1800}, 5s),
|
|
"paired population destination");
|
|
}
|
|
std::optional<CellularCensusHistoryEntry> outgoing_history, incoming_history;
|
|
for (unsigned direction = 0; direction < 2; ++direction) {
|
|
auto& sh = direction == 0 ? f.head : destination_head;
|
|
auto& dh = direction == 0 ? destination_head : f.head;
|
|
auto& so = direction == 0 ? *f.objects : destination_objects;
|
|
auto& dst = direction == 0 ? destination_objects : *f.objects;
|
|
auto& sl = direction == 0 ? f.lease : destination_lease;
|
|
auto& dl = direction == 0 ? destination_lease : f.lease;
|
|
const auto sk = direction == 0 ? f.key : destination_key;
|
|
const auto dk = direction == 0 ? destination_key : f.key;
|
|
take(release_region_lease(*f.db, sk, sl, 5s), "release paired source to host");
|
|
take(release_region_lease(*f.db, dk, dl, 5s), "release paired destination to host");
|
|
auto source_host = make_host(sk), destination_host = make_host(dk);
|
|
require(source_host->step().ready && destination_host->step().ready,
|
|
"paired hosts before preparation");
|
|
sl = take(source_host->authority(), "current source authority before pause");
|
|
dl = take(destination_host->authority(), "current destination authority before pause");
|
|
const auto source_before = sh, destination_before = dh;
|
|
const auto s = take(decode_manifest(sh.manifest), "paired predecessor source");
|
|
const auto d = take(decode_manifest(dh.manifest), "paired predecessor destination");
|
|
const auto sb =
|
|
take(so.get({s.payload_sha256, s.payload_bytes}), "paired source bytes");
|
|
const auto dbb =
|
|
take(dst.get({d.payload_sha256, d.payload_bytes}), "paired destination bytes");
|
|
const CellularMigrationRequest request{
|
|
f.id(),
|
|
{sk.region, sh.head.commit, f.id(), sh.head.digest},
|
|
{dk.region, dh.head.commit, f.id(), dh.head.digest},
|
|
genome};
|
|
ObjectStoreFaults source_faults, destination_faults;
|
|
auto failing_source =
|
|
take(FilesystemObjectStore::open(environment("PGHOST"), sk,
|
|
max_cellular_population_bytes, &source_faults),
|
|
"migration source sync fault store");
|
|
auto failing_destination = take(
|
|
FilesystemObjectStore::open(environment("PGHOST"), dk,
|
|
max_cellular_population_bytes, &destination_faults),
|
|
"migration destination sync fault store");
|
|
source_faults.fail_directory_syncs = 1;
|
|
const auto unsynced_preparation = prepare_cellular_migration_freeze(
|
|
*f.db, *failing_source, dst, sl, sh, dh, request, 5s);
|
|
require(!unsynced_preparation &&
|
|
unsynced_preparation.error().kind == PublicationError::upload &&
|
|
unsynced_preparation.error().object == ObjectError::io &&
|
|
source_faults.fail_directory_syncs == 0 &&
|
|
!take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s),
|
|
"unsynced preparation pointer"),
|
|
"failed capsule sync durably prepared a migration");
|
|
take(source_host->prepare_migration_freeze(dh, dst, request),
|
|
"owning paired source freeze");
|
|
const auto frozen = take(take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s),
|
|
"paired source evidence"),
|
|
"missing paired freeze");
|
|
// Preparation alone cannot authorize either result.
|
|
require(!publish_cellular_migration(*f.db, so, dst, sl, dl, sh, dh, frozen.trace,
|
|
f.build, 5s),
|
|
"unstaged migration published");
|
|
destination_faults.fail_directory_syncs = 1;
|
|
const auto unsynced_stage = stage_cellular_migration(*f.db, so, *failing_destination,
|
|
dl, sh, dh, frozen.trace, 5s);
|
|
require(!unsynced_stage && unsynced_stage.error().kind == PublicationError::upload &&
|
|
unsynced_stage.error().object == ObjectError::io &&
|
|
destination_faults.fail_directory_syncs == 0 &&
|
|
!take(load_cellular_migration_stage(*f.db, dst, dk, dl, 5s),
|
|
"unsynced destination stage pointer") &&
|
|
take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s),
|
|
"source freeze after stage sync failure")
|
|
.has_value(),
|
|
"failed stage capsule sync activated a ghost or released source ownership");
|
|
take(destination_host->stage_migration(sh, so, frozen.trace), "owning paired stage");
|
|
for (unsigned endpoint = 0; endpoint < 2; ++endpoint) {
|
|
auto& faults = endpoint == 0 ? source_faults : destination_faults;
|
|
faults.fail_directory_syncs = 1;
|
|
const auto unsynced_pair =
|
|
publish_cellular_migration(*f.db, *failing_source, *failing_destination, sl, dl,
|
|
sh, dh, frozen.trace, f.build, 5s);
|
|
require(
|
|
!unsynced_pair && unsynced_pair.error().kind == PublicationError::upload &&
|
|
unsynced_pair.error().object == ObjectError::io &&
|
|
faults.fail_directory_syncs == 0 &&
|
|
take(load_region_checkpoint(*f.db, sk, 5s), "unsynced source head").head ==
|
|
source_before.head &&
|
|
take(load_region_checkpoint(*f.db, dk, 5s), "unsynced destination head")
|
|
.head == destination_before.head &&
|
|
take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s),
|
|
"unsynced source freeze")
|
|
.has_value() &&
|
|
take(load_cellular_migration_stage(*f.db, dst, dk, dl, 5s),
|
|
"unsynced destination stage")
|
|
.has_value(),
|
|
"failed result sync published a half or released retained work");
|
|
}
|
|
std::vector<std::uint8_t> expected_source, expected_destination;
|
|
if (representation == 0) {
|
|
auto a = take(restore_cellular_payload(sb, s), "paired raw source oracle");
|
|
auto b = take(restore_cellular_payload(dbb, d), "paired raw destination oracle");
|
|
auto pair = take(replay_migration_trace(*a, *b, request, frozen.trace),
|
|
"paired raw oracle");
|
|
expected_source = pair.source->snapshot();
|
|
expected_destination = pair.destination->snapshot();
|
|
} else {
|
|
auto a = take(restore_population_payload(sb, s), "paired population source oracle");
|
|
auto b = take(restore_population_payload(dbb, d),
|
|
"paired population destination oracle");
|
|
auto pair = take(replay_migration_trace(*a, *b, request, frozen.trace),
|
|
"paired population oracle");
|
|
expected_source = pair.source->snapshot();
|
|
expected_destination = pair.destination->snapshot();
|
|
}
|
|
// Fail specifically on the SECOND result insert, after the first and receipt.
|
|
take(f.owner->query(
|
|
"CREATE FUNCTION evolution.test_refuse_second_migration() RETURNS trigger "
|
|
"LANGUAGE plpgsql AS $$ BEGIN IF NEW.commit_id='" +
|
|
hexadecimal(request.destination.commit, false) +
|
|
"'::uuid THEN RAISE EXCEPTION 'injected second insert' USING "
|
|
"ERRCODE='EVL06'; END IF; RETURN NEW; END $$",
|
|
{}, 5s),
|
|
"second insert fault function");
|
|
take(f.owner->query(
|
|
"CREATE TRIGGER test_refuse_second_migration BEFORE INSERT ON "
|
|
"evolution.region_commits "
|
|
"FOR EACH ROW EXECUTE FUNCTION evolution.test_refuse_second_migration()",
|
|
{}, 5s),
|
|
"second insert fault trigger");
|
|
const auto rejected = publish_cellular_migration(*f.db, so, dst, sl, dl, sh, dh,
|
|
frozen.trace, f.build, 5s);
|
|
take(f.owner->query(
|
|
"DROP TRIGGER test_refuse_second_migration ON evolution.region_commits", {},
|
|
5s),
|
|
"remove second insert fault");
|
|
take(f.owner->query("DROP FUNCTION evolution.test_refuse_second_migration()", {}, 5s),
|
|
"remove fault function");
|
|
require(
|
|
!rejected && rejected.error().store.kind == LeaseStoreError::unsupported &&
|
|
take(load_region_checkpoint(*f.db, sk, 5s), "rollback source head").head ==
|
|
source_before.head &&
|
|
take(load_region_checkpoint(*f.db, dk, 5s), "rollback destination head").head ==
|
|
destination_before.head &&
|
|
take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s),
|
|
"rollback source pause")
|
|
.has_value() &&
|
|
take(load_cellular_migration_stage(*f.db, dst, dk, dl, 5s),
|
|
"rollback destination pause")
|
|
.has_value(),
|
|
"failed second publication left half a migration or released a pause");
|
|
const auto prepared = rejected.error().prepared;
|
|
take(f.db->query("BEGIN", {}, 5s), "false paired biology transaction");
|
|
auto false_pair = prepared;
|
|
auto false_source =
|
|
take(decode_manifest(false_pair.source_manifest), "false source manifest");
|
|
false_source.payload_sha256 = s.payload_sha256;
|
|
false_source.payload_bytes = s.payload_bytes;
|
|
false_pair.source_manifest =
|
|
take(encode_manifest(false_source), "false source canonical bytes");
|
|
take(retry_cellular_migration_commit(*f.db, sl, dl, false_pair, 5s),
|
|
"metadata-only false pair");
|
|
const auto false_recovery = verify_cellular_migration_checkpoint(
|
|
*f.db, so, sl, take(load_region_checkpoint(*f.db, sk, 5s), "false paired head"), sb,
|
|
5s);
|
|
const auto false_head =
|
|
take(load_region_checkpoint(*f.db, sk, 5s), "false history cut");
|
|
const auto false_history = [&]() -> std::expected<void, RecoveryFailure> {
|
|
if (representation == 0) {
|
|
auto recovered = recover_cellular_commit(
|
|
*f.db, so, sk, false_head.head.sequence, sl, builds, 5s);
|
|
if (!recovered)
|
|
return std::unexpected(recovered.error());
|
|
} else {
|
|
auto recovered = recover_population_commit(
|
|
*f.db, so, sk, false_head.head.sequence, sl, builds, 5s);
|
|
if (!recovered)
|
|
return std::unexpected(recovered.error());
|
|
}
|
|
return {};
|
|
}();
|
|
take(f.db->query("ROLLBACK", {}, 5s), "rollback false paired biology fixture");
|
|
require(!false_recovery && false_recovery.error().kind == RecoveryError::history &&
|
|
!false_history && false_history.error().kind == RecoveryError::history,
|
|
"paired recovery accepted a valid old object as the migrated biology");
|
|
const std::array<std::string, 1> generic{hexadecimal(prepared.source_manifest)};
|
|
require(!f.db->query("SELECT * FROM evolution.publish_region_checkpoint($1::bytea)",
|
|
generic, 5s),
|
|
"generic publisher admitted a migration half");
|
|
const auto child = ::fork();
|
|
require(child >= 0, "pre-transaction paired fork");
|
|
if (child == 0) {
|
|
try {
|
|
auto db = connection();
|
|
PublishHooks hooks{[] { ::kill(::getpid(), SIGKILL); }};
|
|
if (!publish_cellular_migration(*db, so, dst, sl, dl, sh, dh, frozen.trace,
|
|
f.build, 5s, hooks))
|
|
::_exit(52);
|
|
} catch (...) {
|
|
::_exit(51);
|
|
}
|
|
::_exit(52);
|
|
}
|
|
int status{};
|
|
require(
|
|
::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
|
|
WTERMSIG(status) == SIGKILL &&
|
|
take(load_region_checkpoint(*f.db, sk, 5s), "killed pair source").head ==
|
|
source_before.head &&
|
|
take(load_region_checkpoint(*f.db, dk, 5s), "killed pair destination").head ==
|
|
destination_before.head,
|
|
"pre-transaction paired death changed a head");
|
|
if (direction == 0 && std::getenv("EVO_MIGRATION_TCP_PORT"))
|
|
migration_transport_commit(f, prepared, sk, dk, sl, dl, source_before.head,
|
|
destination_before.head);
|
|
if (direction == 0) {
|
|
auto locker = connection(true), timed = connection();
|
|
take(locker->query("BEGIN", {}, 5s), "pair ambiguity transaction");
|
|
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND "
|
|
"region_id=" +
|
|
std::to_string(sk.region) + " FOR UPDATE",
|
|
{}, 5s),
|
|
"hold paired reply");
|
|
const auto unknown =
|
|
retry_cellular_migration_commit(*timed, sl, dl, prepared, 300ms);
|
|
require(!unknown && unknown.error().kind == PublicationError::unknown &&
|
|
unknown.error().prepared.source_manifest == prepared.source_manifest &&
|
|
unknown.error().prepared.destination_manifest ==
|
|
prepared.destination_manifest,
|
|
"ambiguous pair lost its exact manifests");
|
|
take(locker->query("ROLLBACK", {}, 5s), "release paired ambiguity");
|
|
} else {
|
|
const auto committed_child = ::fork();
|
|
require(committed_child >= 0, "post-commit paired fork");
|
|
if (committed_child == 0) {
|
|
try {
|
|
auto db = connection();
|
|
if (!retry_cellular_migration_commit(*db, sl, dl, prepared, 5s))
|
|
::_exit(53);
|
|
::kill(::getpid(), SIGKILL);
|
|
} catch (...) {
|
|
::_exit(54);
|
|
}
|
|
::_exit(55);
|
|
}
|
|
require(::waitpid(committed_child, &status, 0) == committed_child &&
|
|
WIFSIGNALED(status) && WTERMSIG(status) == SIGKILL,
|
|
"paired publisher failed before post-commit kill");
|
|
}
|
|
const auto committed =
|
|
take(retry_cellular_migration_commit(*f.db, sl, dl, prepared, 5s),
|
|
"resolve exact paired commit");
|
|
sh = committed.source;
|
|
dh = committed.destination;
|
|
require(!abort_cellular_migration_freeze(*f.db, sk, sl, request.request, 5s),
|
|
"committed migration was aborted");
|
|
auto changed = prepared;
|
|
auto altered =
|
|
take(decode_manifest(changed.destination_manifest), "conflicting committed pair");
|
|
altered.payload_sha256.back() ^= 1;
|
|
changed.destination_manifest = take(encode_manifest(altered), "conflicting pair bytes");
|
|
const auto conflict = retry_cellular_migration_commit(*f.db, sl, dl, changed, 5s);
|
|
auto stale = sl;
|
|
--stale.epoch;
|
|
require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict &&
|
|
!retry_cellular_migration_commit(*f.db, stale, dl, prepared, 5s),
|
|
"changed pair or stale source authority was accepted");
|
|
const auto sm = take(decode_manifest(sh.manifest), "unavailable source result");
|
|
const auto dm = take(decode_manifest(dh.manifest), "unavailable destination result");
|
|
// Outbound: deny current results. Return: deny retained counterpart
|
|
// capsules. Both are required before either paused host may resume.
|
|
const VerifiedObject unavailable_source =
|
|
direction == 0 ? VerifiedObject{sm.payload_sha256, sm.payload_bytes}
|
|
: VerifiedObject{d.payload_sha256, d.payload_bytes};
|
|
const VerifiedObject unavailable_destination =
|
|
direction == 0 ? VerifiedObject{dm.payload_sha256, dm.payload_bytes}
|
|
: VerifiedObject{s.payload_sha256, s.payload_bytes};
|
|
{
|
|
UnreadableObject source_denial(so.directory() /
|
|
hexadecimal(unavailable_source.sha256, false));
|
|
UnreadableObject destination_denial(
|
|
dst.directory() / hexadecimal(unavailable_destination.sha256, false));
|
|
const auto denied_source = so.get(unavailable_source);
|
|
const auto denied_destination = dst.get(unavailable_destination);
|
|
require(!denied_source && denied_source.error() == ObjectError::io &&
|
|
!denied_destination && denied_destination.error() == ObjectError::io,
|
|
"object read denial did not produce actual I/O failures");
|
|
const auto source_failures = source_host->status().store_failures;
|
|
const auto destination_failures = destination_host->status().store_failures;
|
|
const auto observed = [&] {
|
|
if (direction == 0)
|
|
return source_host->status().store_failures > source_failures &&
|
|
destination_host->status().store_failures > destination_failures;
|
|
return source_host->status().phase == HostPhase::quarantined &&
|
|
destination_host->status().phase == HostPhase::quarantined;
|
|
};
|
|
for (unsigned attempt = 0; attempt < 200 && !observed(); ++attempt) {
|
|
source_host->step();
|
|
destination_host->step();
|
|
std::this_thread::sleep_for(10ms);
|
|
}
|
|
require(
|
|
observed() && !source_host->status().ready &&
|
|
!destination_host->status().ready && !source_host->advance_unattended(1) &&
|
|
!destination_host->advance_unattended(1) &&
|
|
take(load_region_checkpoint(*f.db, sk, 5s), "unavailable source head")
|
|
.head == sh.head &&
|
|
take(load_region_checkpoint(*f.db, dk, 5s), "unavailable destination head")
|
|
.head == dh.head,
|
|
"unreadable migration evidence resumed old ownership or changed a head");
|
|
}
|
|
if (direction != 0) {
|
|
require(!source_host->step().ready && !destination_host->step().ready,
|
|
"restoring storage silently cleared quarantine");
|
|
source_host->drain();
|
|
destination_host->drain();
|
|
// The quarantined local authorities cannot release themselves. This
|
|
// test coordinator explicitly retires their still-current DB leases.
|
|
take(release_region_lease(*f.db, sk, sl, 5s), "retire quarantined source lease");
|
|
take(release_region_lease(*f.db, dk, dl, 5s),
|
|
"retire quarantined destination lease");
|
|
source_host = make_host(sk);
|
|
destination_host = make_host(dk);
|
|
}
|
|
for (unsigned attempt = 0;
|
|
attempt < 200 && (!source_host->step().ready || !destination_host->step().ready);
|
|
++attempt)
|
|
std::this_thread::sleep_for(25ms);
|
|
require(source_host->status().ready && destination_host->status().ready &&
|
|
take(source_host->committed(), "adopted source").head == sh.head &&
|
|
take(destination_host->committed(), "adopted destination").head == dh.head,
|
|
"pointer release resumed a stale owner instead of recovering the pair");
|
|
const auto old_source_epoch = sl.epoch, old_destination_epoch = dl.epoch;
|
|
sl = take(source_host->authority(), "restored source object authority");
|
|
dl = take(destination_host->authority(), "restored destination object authority");
|
|
const bool expected_epochs =
|
|
direction == 0 ? sl.epoch == old_source_epoch && dl.epoch == old_destination_epoch
|
|
: sl.epoch > old_source_epoch && dl.epoch > old_destination_epoch;
|
|
require(expected_epochs &&
|
|
take(retry_cellular_migration_commit(*f.db, sl, dl, prepared, 5s),
|
|
"restored storage exact paired receipt")
|
|
.source.head == sh.head,
|
|
"storage recovery changed a committed pair or reused authority");
|
|
const auto check_bytes = [&](const CommittedCheckpoint& h, RegionKey key,
|
|
const FilesystemObjectStore& objects,
|
|
const LeaseGrant& lease) {
|
|
require(take(load_region_checkpoint(*f.db, key, 5s), "current paired recovery head")
|
|
.head == h.head,
|
|
"paired recovery selected a different head");
|
|
if (representation == 0)
|
|
return take(recover_cellular_checkpoint(*f.db, objects, key, lease, builds, 5s),
|
|
"recover raw paired result")
|
|
.state->snapshot();
|
|
return take(recover_population_checkpoint(*f.db, objects, key, lease, builds, 5s),
|
|
"recover cohort paired result")
|
|
.state->snapshot();
|
|
};
|
|
require(check_bytes(sh, sk, so, sl) == expected_source &&
|
|
check_bytes(dh, dk, dst, dl) == expected_destination,
|
|
"committed biology/member/material differs from the exact pair");
|
|
auto source_history =
|
|
take(load_cellular_census_history(*f.db, so, sk, sl, sh.head, sh.head.sequence, 1,
|
|
builds, 5s),
|
|
"paired source history");
|
|
auto destination_history =
|
|
take(load_cellular_census_history(*f.db, dst, dk, dl, dh.head, dh.head.sequence, 1,
|
|
builds, 5s),
|
|
"paired destination history");
|
|
const auto census_oracle = [&](const auto& bytes,
|
|
const CommittedCheckpoint& checkpoint) {
|
|
const auto manifest =
|
|
take(decode_manifest(checkpoint.manifest), "history pair oracle manifest");
|
|
if (representation == 0)
|
|
return take(
|
|
CellularCensus::observe(*take(restore_cellular_payload(bytes, manifest),
|
|
"history raw pair oracle")),
|
|
"history raw census oracle");
|
|
return take(
|
|
CellularCensus::observe(*take(restore_population_payload(bytes, manifest),
|
|
"history cohort pair oracle")),
|
|
"history cohort census oracle");
|
|
};
|
|
require(source_history.entries.size() == 1 && destination_history.entries.size() == 1 &&
|
|
std::ranges::equal(source_history.entries[0].census.rows(),
|
|
census_oracle(expected_source, sh).rows()) &&
|
|
std::ranges::equal(destination_history.entries[0].census.rows(),
|
|
census_oracle(expected_destination, dh).rows()),
|
|
"historical migration census differs from exact paired biology");
|
|
const auto& source_census = source_history.entries[0].census;
|
|
const auto& destination_census = destination_history.entries[0].census;
|
|
const auto& boundary = source_census.boundary();
|
|
std::array required_regions{sk.region, dk.region};
|
|
std::ranges::sort(required_regions);
|
|
const auto census_set =
|
|
take(CellularCensusSet::observe(
|
|
{boundary.world, boundary.frame, boundary.tick, boundary.owner_format,
|
|
boundary.rules, boundary.quantum_microseconds},
|
|
required_regions, std::array{source_census, destination_census}),
|
|
"committed paired history census set");
|
|
// Identity and complete per-region rows survive real raw/cohort
|
|
// publication, replay and repeated direction changes.
|
|
require(census_set.regions().size() == 2 &&
|
|
census_set.regions()[0].boundary().region == required_regions[0] &&
|
|
census_set.regions()[1].boundary().region == required_regions[1],
|
|
"paired history query lost declared region coverage");
|
|
for (const auto& row : destination_census.rows()) {
|
|
const auto combined =
|
|
take(census_set.query(row.genome, CellularCensusScope::descendants),
|
|
"committed cross-region lineage query");
|
|
const auto local =
|
|
take(destination_census.query(row.genome, CellularCensusScope::descendants),
|
|
"committed destination lineage");
|
|
const auto other =
|
|
source_census.query(row.genome, CellularCensusScope::descendants);
|
|
for (std::size_t kind = 0; kind < local.resident.size(); ++kind)
|
|
require(combined.resident[kind].count ==
|
|
local.resident[kind].count +
|
|
(other ? other->resident[kind].count : 0),
|
|
"committed set duplicated/dropped resident lineage stock");
|
|
}
|
|
if (direction == 0) {
|
|
outgoing_history = std::move(source_history.entries.front());
|
|
incoming_history = std::move(destination_history.entries.front());
|
|
}
|
|
const auto capsule_path = so.directory() / hexadecimal(d.payload_sha256, false);
|
|
const auto backup = capsule_path.string() + ".paired-test";
|
|
std::filesystem::rename(capsule_path, backup);
|
|
const auto missing =
|
|
verify_cellular_migration_checkpoint(*f.db, so, sl, sh, expected_source, 5s);
|
|
const auto missing_history = load_cellular_census_history(
|
|
*f.db, so, sk, sl, sh.head, sh.head.sequence, 1, builds, 5s);
|
|
std::filesystem::rename(backup, capsule_path);
|
|
require(!missing && missing.error().kind == RecoveryError::tail && !missing_history &&
|
|
missing_history.error().recovery.kind == RecoveryError::tail &&
|
|
!verify_cellular_migration_checkpoint(*f.db, so, sl, sh, sb, 5s),
|
|
"pair recovery ignored missing counterpart evidence or false local biology");
|
|
source_host->drain();
|
|
destination_host->drain();
|
|
source_host = make_host(sk);
|
|
destination_host = make_host(dk);
|
|
require(source_host->step().ready && destination_host->step().ready,
|
|
"replacement paired hosts failed complete recovery");
|
|
sl = take(source_host->authority(), "replacement source fence");
|
|
dl = take(destination_host->authority(), "replacement destination fence");
|
|
require(take(retry_cellular_migration_commit(*f.db, sl, dl, prepared, 5s),
|
|
"new epoch exact paired retry")
|
|
.source.head == sh.head,
|
|
"fresh authority changed a committed retry");
|
|
source_host->drain();
|
|
destination_host->drain();
|
|
sl = take(acquire_region_lease(*f.db, sk, f.build, 60000ms, 5s),
|
|
"return paired source authority");
|
|
dl = take(acquire_region_lease(*f.db, dk, f.build, 60000ms, 5s),
|
|
"return paired destination authority");
|
|
}
|
|
const auto outgoing = take(std::move(outgoing_history), "retained source historical cut");
|
|
const auto incoming =
|
|
take(std::move(incoming_history), "retained destination historical cut");
|
|
const auto old_source =
|
|
take(load_cellular_census_history(*f.db, *f.objects, f.key, f.lease, f.head.head,
|
|
outgoing.committed.head.sequence, 1, builds, 5s),
|
|
"source history after return");
|
|
const auto old_destination =
|
|
take(load_cellular_census_history(*f.db, destination_objects, destination_key,
|
|
destination_lease, destination_head.head,
|
|
incoming.committed.head.sequence, 1, builds, 5s),
|
|
"destination history after return");
|
|
require(
|
|
old_source.entries[0].committed.head == outgoing.committed.head &&
|
|
old_destination.entries[0].committed.head == incoming.committed.head &&
|
|
take(decode_manifest(outgoing.committed.manifest), "historical source writer")
|
|
.epoch < f.lease.epoch &&
|
|
take(decode_manifest(incoming.committed.manifest), "historical destination writer")
|
|
.epoch < destination_lease.epoch &&
|
|
std::ranges::equal(old_source.entries[0].census.rows(), outgoing.census.rows()) &&
|
|
std::ranges::equal(old_destination.entries[0].census.rows(),
|
|
incoming.census.rows()),
|
|
"return migration rewrote earlier committed ownership observations");
|
|
migration_owner_partitions(f, destination_key, destination_objects, destination_lease,
|
|
destination_head, genome);
|
|
migration_snapshot_race(f, destination_key, destination_objects, destination_lease,
|
|
destination_head, genome);
|
|
}
|
|
std::puts("migration object sync/read failures, quarantine and exact recovery passed");
|
|
// Race the actual two-head transaction against abort on independent connections.
|
|
const auto prepare_pair = [&](bool reverse) {
|
|
auto& sh = reverse ? destination_head : f.head;
|
|
auto& dh = reverse ? f.head : destination_head;
|
|
auto& so = reverse ? destination_objects : *f.objects;
|
|
auto& dst = reverse ? *f.objects : destination_objects;
|
|
auto& sl = reverse ? destination_lease : f.lease;
|
|
auto& dl = reverse ? f.lease : destination_lease;
|
|
const auto sk = reverse ? destination_key : f.key;
|
|
const auto dk = reverse ? f.key : destination_key;
|
|
CellularMigrationRequest request{f.id(),
|
|
{sk.region, sh.head.commit, f.id(), sh.head.digest},
|
|
{dk.region, dh.head.commit, f.id(), dh.head.digest},
|
|
genome};
|
|
const auto freeze =
|
|
take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s),
|
|
"commit/abort source preparation");
|
|
take(stage_cellular_migration(*f.db, so, dst, dl, sh, dh, freeze.trace, 5s),
|
|
"commit/abort destination stage");
|
|
return freeze.trace;
|
|
};
|
|
const auto aborted_trace = prepare_pair(false);
|
|
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, aborted_trace.request.request, 5s),
|
|
"abort wins before paired commit");
|
|
const auto after_abort = publish_cellular_migration(
|
|
*f.db, *f.objects, destination_objects, f.lease, destination_lease, f.head,
|
|
destination_head, aborted_trace, f.build, 5s);
|
|
require(!after_abort && after_abort.error().store.kind == LeaseStoreError::commit_conflict,
|
|
"paired publication reopened an aborted workflow");
|
|
for (unsigned race = 0; race < 4; ++race) {
|
|
const auto trace = prepare_pair(false);
|
|
auto committing = connection(), aborting = connection();
|
|
std::barrier start(2);
|
|
std::optional<std::expected<CellularMigrationCommitted, CellularMigrationCommitFailure>>
|
|
published;
|
|
std::optional<std::expected<CellularMigrationFreeze, LeaseStoreFailure>> aborted;
|
|
std::thread commit_thread([&] {
|
|
start.arrive_and_wait();
|
|
published = publish_cellular_migration(*committing, *f.objects, destination_objects,
|
|
f.lease, destination_lease, f.head,
|
|
destination_head, trace, f.build, 5s);
|
|
});
|
|
std::thread abort_thread([&] {
|
|
start.arrive_and_wait();
|
|
aborted = abort_cellular_migration_freeze(*aborting, f.key, f.lease,
|
|
trace.request.request, 5s);
|
|
});
|
|
commit_thread.join();
|
|
abort_thread.join();
|
|
require(published.has_value() && aborted.has_value() &&
|
|
(published->has_value() != aborted->has_value()),
|
|
"commit/abort race had two outcomes or no definitive outcome");
|
|
if (published->has_value()) {
|
|
require(aborted->error().kind == LeaseStoreError::commit_conflict,
|
|
"losing abort was not refused");
|
|
f.head = (*published)->source;
|
|
destination_head = (*published)->destination;
|
|
const auto returning = prepare_pair(true);
|
|
const auto returned =
|
|
take(publish_cellular_migration(*f.db, destination_objects, *f.objects,
|
|
destination_lease, f.lease, destination_head,
|
|
f.head, returning, f.build, 5s),
|
|
"return race winner's stock");
|
|
destination_head = returned.source;
|
|
f.head = returned.destination;
|
|
} else {
|
|
require(published->error().store.kind == LeaseStoreError::commit_conflict,
|
|
"losing paired publication was not refused");
|
|
}
|
|
require(
|
|
take(load_region_checkpoint(*f.db, f.key, 5s), "race source head").head ==
|
|
f.head.head &&
|
|
take(load_region_checkpoint(*f.db, destination_key, 5s), "race destination head")
|
|
.head == destination_head.head &&
|
|
!take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s),
|
|
"race source settled")
|
|
.has_value() &&
|
|
!take(load_cellular_migration_stage(*f.db, destination_objects, destination_key,
|
|
destination_lease, 5s),
|
|
"race destination settled")
|
|
.has_value(),
|
|
"commit/abort race left half a head or a pending pause");
|
|
}
|
|
recover(f);
|
|
f.raw = restore(f.population->state());
|
|
}
|
|
|
|
void migration_freezes(Fixture& f) {
|
|
const RegionKey destination_key{f.key.world, 2};
|
|
auto destination_objects =
|
|
take(FilesystemObjectStore::open(environment("PGHOST"), destination_key,
|
|
max_cellular_population_bytes),
|
|
"migration destination objects");
|
|
auto destination = opening(30001, true, 4, false, 2);
|
|
auto destination_head =
|
|
take(provision_cellular_region(*f.owner, *destination_objects, destination_key, 1,
|
|
*destination, f.id(), f.build, 5s),
|
|
"migration empty destination genesis");
|
|
auto destination_lease =
|
|
take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s),
|
|
"migration destination authority");
|
|
const auto unequal_cut = load_cellular_census_snapshot(
|
|
*f.db, f.key.world,
|
|
std::array{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()},
|
|
CellularCensusRegionRequest{destination_key.region, destination_lease,
|
|
destination_objects.get()}},
|
|
std::array<Sha256Digest, 1>{f.build}, 5s);
|
|
require(!unequal_cut && unequal_cut.error().kind == CellularSnapshotError::census &&
|
|
unequal_cut.error().census == CellularCensusSetError::boundary &&
|
|
f.db->transaction_idle(),
|
|
"snapshot silently reconciled different current ticks/owner versions");
|
|
while (destination->tick() < f.raw->tick()) {
|
|
const auto count = std::min(std::uint64_t{64}, f.raw->tick() - destination->tick());
|
|
CellularBatch inputs{{f.key.world, 2, 1, destination_head.head.commit,
|
|
destination_head.head.digest, destination->tick(),
|
|
destination->tick() + count},
|
|
{}};
|
|
inputs.ticks.resize(static_cast<std::size_t>(count));
|
|
destination =
|
|
take(replay_cellular_batch(*destination, inputs), "empty destination biology");
|
|
destination_head = take(publish_cellular_batch(*f.db, *destination_objects,
|
|
destination_lease, destination_head, inputs,
|
|
*destination, f.id(), f.build, 5s),
|
|
"empty destination tick publication")
|
|
.committed;
|
|
}
|
|
destination_head =
|
|
take(migrate_cellular_boundary_profile(*f.db, *destination_objects, destination_lease,
|
|
destination_head, f.id(), f.build, 5s),
|
|
"destination accounting upgrade");
|
|
destination = take(destination->migration_upgrade_candidate(), "destination upgraded oracle");
|
|
const std::array<Sha256Digest, 1> builds{f.build};
|
|
const auto request_for = [&] {
|
|
const auto member = std::ranges::find_if(
|
|
f.raw->lives(), [](const auto& life) { return life.account == LifeAccount{}; });
|
|
require(member != f.raw->lives().end(), "freeze fixture lacks real wildlife");
|
|
return CellularMigrationRequest{
|
|
f.id(),
|
|
{1, f.head.head.commit, f.id(), f.head.head.digest},
|
|
{2, destination_head.head.commit, f.id(), destination_head.head.digest},
|
|
member->identity.genome};
|
|
};
|
|
for (unsigned representation = 0; representation < 2; ++representation) {
|
|
if (representation == 1) {
|
|
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
|
|
f.build, {true, 1200, 1800}, 5s),
|
|
"freeze cohort source entry");
|
|
recover(f);
|
|
destination_head = take(migrate_cellular_to_population(
|
|
*f.db, *destination_objects, destination_lease,
|
|
destination_head, f.id(), f.build, {true, 1200, 1800}, 5s),
|
|
"freeze cohort destination entry");
|
|
}
|
|
const auto source_before = f.head;
|
|
const auto source_bytes = f.raw->snapshot();
|
|
const auto destination_before = destination_head;
|
|
const auto request = request_for();
|
|
const auto prepare = [&](const CellularMigrationRequest& requested,
|
|
std::chrono::milliseconds timeout = 5s) {
|
|
return prepare_cellular_migration_freeze(*f.db, *f.objects, *destination_objects,
|
|
f.lease, f.head, destination_head, requested,
|
|
timeout);
|
|
};
|
|
if (representation == 0) {
|
|
Uuid account{};
|
|
account.back() = 1;
|
|
take(f.db->query("BEGIN", {}, 5s), "pending player choice before freeze");
|
|
const auto control =
|
|
take(claim_account_control(*f.db, f.key, f.lease, account, 2, 9900, 60000ms, 5s),
|
|
"pending choice current control");
|
|
const auto input = encode_cellular_mutation_input(
|
|
{account, 9900, f.head.head.end_tick + 1, {100, 1, 1, 1}});
|
|
take(reserve_cellular_mutation(*f.db, f.key, f.lease, account, control.generation, 9900,
|
|
input, 5s),
|
|
"reserve pre-freeze player choice");
|
|
const auto pending = prepare(request);
|
|
take(f.db->query("ROLLBACK", {}, 5s), "rollback pre-freeze player choice");
|
|
require(!pending &&
|
|
std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18",
|
|
"preparation stranded a player choice");
|
|
} else {
|
|
take(f.db->query("BEGIN", {}, 5s), "prepare with admitted work transaction");
|
|
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head,
|
|
f.head.head.end_tick + 20, 5s),
|
|
"admitted work before freeze");
|
|
const auto pending = prepare(request);
|
|
take(f.db->query("ROLLBACK", {}, 5s), "rollback pending-work preparation fixture");
|
|
require(!pending &&
|
|
std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18",
|
|
"preparation skipped durable ecological work");
|
|
}
|
|
auto frozen = take(prepare(request), "durable source freeze");
|
|
const CellularMigrationPrepared prepared{
|
|
f.key, take(encode_migration_trace(frozen.trace), "frozen canonical trace")};
|
|
auto malformed = prepared.trace;
|
|
malformed.back() ^= 1;
|
|
const std::array<std::string, 3> malformed_args{hexadecimal(f.lease.holder, false),
|
|
std::to_string(f.lease.epoch),
|
|
hexadecimal(malformed)};
|
|
const auto malformed_sql = f.db->query("SELECT * FROM "
|
|
"evolution.prepare_cellular_migration_freeze(30201,"
|
|
"1,$1::uuid,$2::evolution.u64,$3::bytea)",
|
|
malformed_args, 5s);
|
|
require(!malformed_sql && std::string(malformed_sql.error().sqlstate.data(), 5) == "22023",
|
|
"SQL accepted an invalid trace checksum before duplicate lookup");
|
|
require(
|
|
!frozen.aborted && frozen.trace.request == request &&
|
|
take(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s),
|
|
"replay frozen capsule"),
|
|
"missing durable freeze")
|
|
.trace == frozen.trace &&
|
|
take(load_region_checkpoint(*f.db, f.key, 5s), "frozen source head").head ==
|
|
source_before.head &&
|
|
take(load_region_checkpoint(*f.db, destination_key, 5s), "unowned destination head")
|
|
.head == destination_before.head,
|
|
"preparation changed either owner or lost its full replay capsule");
|
|
require(
|
|
!take(retry_cellular_migration_freeze(*f.db, f.lease, prepared, 5s), "duplicate freeze")
|
|
.aborted,
|
|
"duplicate freeze lost its state");
|
|
const auto stage = [&](const CellularMigrationTrace& trace,
|
|
std::chrono::milliseconds timeout = 5s) {
|
|
return stage_cellular_migration(*f.db, *f.objects, *destination_objects,
|
|
destination_lease, f.head, destination_head, trace,
|
|
timeout);
|
|
};
|
|
// The optimistic destination cut must still be exact at first staging.
|
|
take(f.db->query("BEGIN", {}, 5s), "move destination before stage transaction");
|
|
if (representation == 0)
|
|
take(migrate_cellular_to_population(*f.db, *destination_objects, destination_lease,
|
|
destination_head, f.id(), f.build, {}, 5s),
|
|
"temporarily move destination profile");
|
|
else
|
|
take(materialize_population_profile(*f.db, *destination_objects, destination_lease,
|
|
destination_head, f.id(), f.build, 5s),
|
|
"temporarily move population destination");
|
|
const auto moved = stage(frozen.trace);
|
|
take(f.db->query("ROLLBACK", {}, 5s), "rollback moved destination fixture");
|
|
require(!moved && moved.error().store.kind == LeaseStoreError::head_mismatch,
|
|
"stage silently rebased a moved destination");
|
|
if (representation == 1) {
|
|
take(f.db->query("BEGIN", {}, 5s), "stage pending interval transaction");
|
|
take(admit_population_interval(*f.db, destination_key, destination_lease,
|
|
destination_head.head,
|
|
destination_head.head.end_tick + 20, 5s),
|
|
"destination admitted work before staging");
|
|
const auto pending = stage(frozen.trace);
|
|
take(f.db->query("ROLLBACK", {}, 5s), "rollback destination interval fixture");
|
|
require(!pending && pending.error().store.kind == LeaseStoreError::commit_conflict,
|
|
"stage stranded destination interval work");
|
|
}
|
|
const auto staged = take(stage(frozen.trace), "durable destination staging");
|
|
const CellularMigrationStagePrepared stage_request{destination_key, prepared.trace,
|
|
staged.ghost};
|
|
require(
|
|
!staged.freeze.aborted &&
|
|
take(take(load_cellular_migration_stage(*f.db, *destination_objects,
|
|
destination_key, destination_lease, 5s),
|
|
"recover destination ghost"),
|
|
"missing staged destination")
|
|
.ghost == staged.ghost &&
|
|
take(load_region_checkpoint(*f.db, destination_key, 5s),
|
|
"staged destination still owns original head")
|
|
.head == destination_before.head &&
|
|
take(retry_cellular_migration_stage(*f.db, destination_lease, stage_request, 5s),
|
|
"duplicate stage")
|
|
.ghost == staged.ghost,
|
|
"staging activated a ghost or lost exact replay");
|
|
auto changed_stage = stage_request;
|
|
++changed_stage.ghost.bytes;
|
|
const auto changed =
|
|
retry_cellular_migration_stage(*f.db, destination_lease, changed_stage, 5s);
|
|
require(!changed && changed.error().store.kind == LeaseStoreError::commit_conflict,
|
|
"stage accepted conflicting immutable ghost bytes");
|
|
auto reverse = frozen.trace;
|
|
reverse.request.request = f.id();
|
|
std::swap(reverse.request.source, reverse.request.destination);
|
|
const auto overlapping = retry_cellular_migration_freeze(
|
|
*f.db, destination_lease,
|
|
{destination_key, take(encode_migration_trace(reverse), "overlapping source trace")},
|
|
5s);
|
|
require(!overlapping && overlapping.error().store.kind == LeaseStoreError::commit_conflict,
|
|
"staged destination simultaneously prepared an outgoing workflow");
|
|
take(f.db->query("BEGIN", {}, 5s), "staged destination life reservation transaction");
|
|
Uuid new_account{};
|
|
new_account.back() = 5;
|
|
const auto staged_control =
|
|
take(claim_account_control(*f.db, destination_key, destination_lease, new_account, 1,
|
|
9950, 60000ms, 5s),
|
|
"staged destination reservation current control");
|
|
const auto staged_life =
|
|
reserve_primitive_life(*f.db, destination_key, destination_lease, new_account,
|
|
staged_control.generation, 1, 1, 5s);
|
|
take(f.db->query("ROLLBACK", {}, 5s), "rollback staged life reservation fixture");
|
|
require(!staged_life &&
|
|
std::string(staged_life.error().cause.sqlstate.data(), 5) == "EVL18",
|
|
"staged destination admitted a new player life reservation");
|
|
if (representation == 0) {
|
|
CellularBatch inputs{{f.key.world, 2, 1, destination_head.head.commit,
|
|
destination_head.head.digest, destination->tick(),
|
|
destination->tick() + 1},
|
|
{{}}};
|
|
const auto advanced = take(replay_cellular_batch(*destination, inputs),
|
|
"detached staged destination future");
|
|
const auto denied =
|
|
publish_cellular_batch(*f.db, *destination_objects, destination_lease,
|
|
destination_head, inputs, *advanced, f.id(), f.build, 5s);
|
|
require(!denied &&
|
|
std::string(denied.error().store.cause.sqlstate.data(), 5) == "EVL18",
|
|
"staged destination published an ordinary future");
|
|
} else {
|
|
require(!admit_population_interval(*f.db, destination_key, destination_lease,
|
|
destination_head.head,
|
|
destination_head.head.end_tick + 20, 5s) &&
|
|
!materialize_population_profile(*f.db, *destination_objects,
|
|
destination_lease, destination_head, f.id(),
|
|
f.build, 5s),
|
|
"staged destination progressed or changed profile");
|
|
}
|
|
const auto ghost_path =
|
|
destination_objects->directory() / hexadecimal(staged.ghost.sha256, false);
|
|
const auto ghost_backup = ghost_path.string() + ".stage-test";
|
|
std::filesystem::rename(ghost_path, ghost_backup);
|
|
const auto missing_ghost = load_cellular_migration_stage(
|
|
*f.db, *destination_objects, destination_key, destination_lease, 5s);
|
|
std::filesystem::rename(ghost_backup, ghost_path);
|
|
require(!missing_ghost && missing_ghost.error().kind == RecoveryError::tail,
|
|
"missing ghost fell back to the active destination");
|
|
char ghost_first{};
|
|
{
|
|
std::fstream file(ghost_path, std::ios::binary | std::ios::in | std::ios::out);
|
|
file.get(ghost_first);
|
|
file.seekp(0);
|
|
file.put(static_cast<char>(static_cast<unsigned char>(ghost_first) ^ 1U));
|
|
file.flush();
|
|
require(file.good(), "same-length ghost corruption fixture");
|
|
}
|
|
const auto corrupt_ghost = load_cellular_migration_stage(
|
|
*f.db, *destination_objects, destination_key, destination_lease, 5s);
|
|
{
|
|
std::fstream file(ghost_path, std::ios::binary | std::ios::in | std::ios::out);
|
|
file.put(ghost_first);
|
|
file.flush();
|
|
require(file.good(), "restore corrupt ghost fixture");
|
|
}
|
|
require(!corrupt_ghost && corrupt_ghost.error().kind == RecoveryError::tail,
|
|
"same-length corrupt ghost bypassed digest verification");
|
|
const auto source_manifest = take(decode_manifest(f.head.manifest), "stage source capsule");
|
|
const auto source_capsule_path =
|
|
destination_objects->directory() / hexadecimal(source_manifest.payload_sha256, false);
|
|
const auto source_capsule_backup = source_capsule_path.string() + ".stage-test";
|
|
std::filesystem::rename(source_capsule_path, source_capsule_backup);
|
|
const auto missing_source = load_cellular_migration_stage(
|
|
*f.db, *destination_objects, destination_key, destination_lease, 5s);
|
|
std::filesystem::rename(source_capsule_backup, source_capsule_path);
|
|
require(!missing_source && missing_source.error().kind == RecoveryError::tail,
|
|
"destination recovery used the live source instead of its retained capsule");
|
|
const auto old_destination_lease = destination_lease;
|
|
take(release_region_lease(*f.db, destination_key, destination_lease, 5s),
|
|
"release staged destination owner");
|
|
destination_lease = take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s),
|
|
"replacement staged destination owner");
|
|
require(
|
|
take(load_cellular_migration_stage(*f.db, *destination_objects, destination_key,
|
|
destination_lease, 5s),
|
|
"new epoch ghost replay")
|
|
.has_value() &&
|
|
!retry_cellular_migration_stage(*f.db, old_destination_lease, stage_request, 5s),
|
|
"replacement activated the ghost or stale destination staged it");
|
|
auto conflict_trace = frozen.trace;
|
|
++conflict_trace.transferred.energy.value;
|
|
auto conflict = retry_cellular_migration_freeze(
|
|
*f.db, f.lease, {f.key, take(encode_migration_trace(conflict_trace), "conflict trace")},
|
|
5s);
|
|
require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict &&
|
|
!prepare(request_for()),
|
|
"freeze accepted changed bytes or replaced pending request");
|
|
if (representation == 0) {
|
|
const auto h = f.header(1);
|
|
const auto inputs = take(unattended_population_inputs(*f.raw, h), "frozen raw inputs");
|
|
auto advanced = take(replay_cellular_batch(*f.raw, inputs), "frozen detached future");
|
|
auto denied = publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, inputs,
|
|
*advanced, f.id(), f.build, 5s);
|
|
require(!denied &&
|
|
std::string(denied.error().store.cause.sqlstate.data(), 5) == "EVL18" &&
|
|
!migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
|
|
f.build, {}, 5s),
|
|
"raw source progressed or changed profile while frozen");
|
|
Uuid account{};
|
|
account.back() = 1;
|
|
const auto input = encode_cellular_mutation_input(
|
|
{account, 9901, f.head.head.end_tick + 1, {100, 1, 1, 1}});
|
|
take(f.db->query("BEGIN", {}, 5s), "current control while frozen fixture");
|
|
const auto control =
|
|
take(claim_account_control(*f.db, f.key, f.lease, account, 2, 9901, 60000ms, 5s),
|
|
"current frozen-source account control");
|
|
const auto choice = reserve_cellular_mutation(*f.db, f.key, f.lease, account,
|
|
control.generation, 9901, input, 5s);
|
|
take(f.db->query("ROLLBACK", {}, 5s), "rollback frozen control fixture");
|
|
require(!choice, "frozen raw owner admitted a new mutation reservation");
|
|
if (std::string(choice.error().cause.sqlstate.data(), 5) != "EVL18")
|
|
throw std::runtime_error("frozen mutation guard SQLSTATE=" +
|
|
std::string(choice.error().cause.sqlstate.data(), 5));
|
|
} else {
|
|
require(!admit_population_interval(*f.db, f.key, f.lease, f.head.head,
|
|
f.head.head.end_tick + 20, 5s) &&
|
|
!materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(),
|
|
f.build, 5s),
|
|
"population source admitted work or changed profile while frozen");
|
|
}
|
|
const auto capsule_manifest =
|
|
take(decode_manifest(destination_head.manifest), "capsule manifest");
|
|
const auto denied_upgrade_id = f.id();
|
|
const auto denied_upgrade = migrate_cellular_maturation_profile(
|
|
*f.db, *f.objects, f.lease, f.head, denied_upgrade_id, f.build, 5s);
|
|
require(
|
|
!denied_upgrade &&
|
|
std::string(denied_upgrade.error().store.cause.sqlstate.data(), 5) == "EVL18" &&
|
|
!take(load_cellular_profile_migration(*f.db, f.key, f.lease, denied_upgrade_id, 5s),
|
|
"frozen maturation journal")
|
|
.has_value() &&
|
|
take(load_region_checkpoint(*f.db, f.key, 5s), "frozen maturation head").head ==
|
|
f.head.head,
|
|
"maturation escaped a frozen handoff or retained partial upgrade evidence");
|
|
const auto capsule_path =
|
|
f.objects->directory() / hexadecimal(capsule_manifest.payload_sha256, false);
|
|
const auto backup = capsule_path.string() + ".freeze-test";
|
|
std::filesystem::rename(capsule_path, backup);
|
|
const auto missing = load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s);
|
|
std::filesystem::rename(backup, capsule_path);
|
|
require(!missing && missing.error().kind == RecoveryError::tail,
|
|
"freeze recovery fell back to destination live state without the retained capsule");
|
|
char first{};
|
|
{
|
|
std::fstream file(capsule_path, std::ios::binary | std::ios::in | std::ios::out);
|
|
file.get(first);
|
|
file.seekp(0);
|
|
file.put(static_cast<char>(static_cast<unsigned char>(first) ^ 1U));
|
|
file.flush();
|
|
require(file.good(), "corrupt same-length capsule fixture");
|
|
}
|
|
const auto corrupt = load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s);
|
|
{
|
|
std::fstream file(capsule_path, std::ios::binary | std::ios::in | std::ios::out);
|
|
file.put(first);
|
|
file.flush();
|
|
require(file.good(), "restore same-length capsule fixture");
|
|
}
|
|
require(!corrupt && corrupt.error().kind == RecoveryError::tail,
|
|
"corrupt capsule bypassed strong object binding");
|
|
const auto old_lease = f.lease;
|
|
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release frozen source owner");
|
|
f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s),
|
|
"new frozen source owner");
|
|
require(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s),
|
|
"recover freeze under new epoch")
|
|
.has_value() &&
|
|
!abort_cellular_migration_freeze(*f.db, f.key, old_lease, request.request, 5s),
|
|
"new epoch released a freeze or old epoch aborted it");
|
|
require(
|
|
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, request.request, 5s),
|
|
"explicit source abort")
|
|
.aborted &&
|
|
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, request.request, 5s),
|
|
"duplicate source abort")
|
|
.aborted &&
|
|
take(retry_cellular_migration_freeze(*f.db, f.lease, prepared, 5s),
|
|
"retry aborted preparation")
|
|
.aborted &&
|
|
!take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s),
|
|
"aborted pointer load")
|
|
.has_value(),
|
|
"abort was not durable or preparation retry reopened it");
|
|
require(
|
|
!take(load_cellular_migration_stage(*f.db, *destination_objects, destination_key,
|
|
destination_lease, 5s),
|
|
"aborted destination pointer")
|
|
.has_value() &&
|
|
take(retry_cellular_migration_stage(*f.db, destination_lease, stage_request, 5s),
|
|
"retry aborted stage")
|
|
.freeze.aborted,
|
|
"source abort stranded the destination or stage retry reopened its pointer");
|
|
require(take(load_region_checkpoint(*f.db, f.key, 5s), "aborted source head").head ==
|
|
source_before.head &&
|
|
f.raw->snapshot() == source_bytes,
|
|
"abort changed biological stock/time/history");
|
|
|
|
auto false_trace = frozen.trace;
|
|
false_trace.request.request = f.id();
|
|
false_trace.request.source.commit = f.id();
|
|
false_trace.request.destination.commit = f.id();
|
|
++false_trace.transferred.energy.value;
|
|
take(retry_cellular_migration_freeze(*f.db, f.lease,
|
|
{f.key, take(encode_migration_trace(false_trace),
|
|
"canonical false inventory trace")},
|
|
5s),
|
|
"metadata-only false preparation");
|
|
const auto false_replay =
|
|
load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s);
|
|
require(!false_replay && false_replay.error().kind == RecoveryError::history,
|
|
"freeze recovery trusted declared material without complete biological replay");
|
|
const auto refuse_false_inventory = [&] {
|
|
const std::array resources{
|
|
CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()},
|
|
CellularCensusRegionRequest{destination_key.region, destination_lease,
|
|
destination_objects.get()}};
|
|
const auto result =
|
|
load_registered_cellular_census_snapshot(*f.db, f.key.world, resources, builds, 5s);
|
|
require(!result && result.error().kind == CellularSnapshotError::migration &&
|
|
result.error().migration.kind == RecoveryError::history &&
|
|
f.db->transaction_idle(),
|
|
"registered census accepted semantically false pending biology");
|
|
};
|
|
refuse_false_inventory();
|
|
take(
|
|
abort_cellular_migration_freeze(*f.db, f.key, f.lease, false_trace.request.request, 5s),
|
|
"explicitly abort false preparation without moving stock");
|
|
|
|
// A genuine unknown reply retains exactly the chosen cut and group.
|
|
auto locker = connection(true);
|
|
take(locker->query("BEGIN", {}, 5s), "freeze ambiguity lock");
|
|
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND "
|
|
"region_id=1 FOR UPDATE",
|
|
{}, 5s),
|
|
"hold freeze reply");
|
|
auto timed = connection();
|
|
auto unknown =
|
|
prepare_cellular_migration_freeze(*timed, *f.objects, *destination_objects, f.lease,
|
|
f.head, destination_head, request_for(), 300ms);
|
|
require(!unknown && unknown.error().kind == PublicationError::unknown &&
|
|
!unknown.error().prepared.trace.empty(),
|
|
"freeze ambiguity lost its immutable request");
|
|
take(locker->query("ROLLBACK", {}, 5s), "release freeze ambiguity");
|
|
frozen = take(retry_cellular_migration_freeze(*f.db, f.lease, unknown.error().prepared, 5s),
|
|
"reconcile unknown freeze");
|
|
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, frozen.trace.request.request,
|
|
5s),
|
|
"abort reconciled freeze");
|
|
|
|
const auto false_stage_freeze = take(prepare(request_for()), "false ghost preparation");
|
|
const auto original_destination =
|
|
take(decode_manifest(destination_head.manifest), "false ghost existing object");
|
|
const CellularMigrationStagePrepared false_ghost{
|
|
destination_key,
|
|
take(encode_migration_trace(false_stage_freeze.trace), "false ghost trace"),
|
|
{original_destination.payload_sha256, original_destination.payload_bytes}};
|
|
take(retry_cellular_migration_stage(*f.db, destination_lease, false_ghost, 5s),
|
|
"metadata-only false ghost stage");
|
|
const auto false_ghost_replay = load_cellular_migration_stage(
|
|
*f.db, *destination_objects, destination_key, destination_lease, 5s);
|
|
require(!false_ghost_replay && false_ghost_replay.error().kind == RecoveryError::history,
|
|
"stage recovery accepted a real object with the wrong complete biology");
|
|
refuse_false_inventory();
|
|
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
|
|
false_stage_freeze.trace.request.request, 5s),
|
|
"abort false ghost evidence");
|
|
|
|
const auto uncertain_stage_freeze =
|
|
take(prepare(request_for()), "unknown stage preparation");
|
|
take(locker->query("BEGIN", {}, 5s), "stage ambiguity lock");
|
|
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND "
|
|
"region_id=2 FOR UPDATE",
|
|
{}, 5s),
|
|
"hold stage reply");
|
|
auto stage_timed = connection();
|
|
const auto stage_unknown = stage_cellular_migration(
|
|
*stage_timed, *f.objects, *destination_objects, destination_lease, f.head,
|
|
destination_head, uncertain_stage_freeze.trace, 300ms);
|
|
require(!stage_unknown && stage_unknown.error().kind == PublicationError::unknown &&
|
|
!stage_unknown.error().prepared.trace.empty(),
|
|
"stage ambiguity lost exact bytes");
|
|
take(locker->query("ROLLBACK", {}, 5s), "release stage ambiguity");
|
|
take(retry_cellular_migration_stage(*f.db, destination_lease,
|
|
stage_unknown.error().prepared, 5s),
|
|
"resolve unknown stage");
|
|
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
|
|
uncertain_stage_freeze.trace.request.request, 5s),
|
|
"abort reconciled stage");
|
|
|
|
for (unsigned race = 0; race < 4; ++race) {
|
|
const auto race_freeze = take(prepare(request_for()), "stage race preparation");
|
|
auto stage_db = connection(), abort_db = connection();
|
|
std::barrier start(2);
|
|
std::optional<std::expected<CellularMigrationStage, CellularMigrationStageFailure>>
|
|
staged_race;
|
|
std::optional<std::expected<CellularMigrationFreeze, LeaseStoreFailure>> aborted_race;
|
|
std::thread staging([&] {
|
|
start.arrive_and_wait();
|
|
staged_race = stage_cellular_migration(*stage_db, *f.objects, *destination_objects,
|
|
destination_lease, f.head, destination_head,
|
|
race_freeze.trace, 5s);
|
|
});
|
|
std::thread aborting([&] {
|
|
start.arrive_and_wait();
|
|
aborted_race = abort_cellular_migration_freeze(
|
|
*abort_db, f.key, f.lease, race_freeze.trace.request.request, 5s);
|
|
});
|
|
staging.join();
|
|
aborting.join();
|
|
require(aborted_race.has_value() && aborted_race->has_value() &&
|
|
staged_race.has_value() &&
|
|
(staged_race->has_value() ||
|
|
staged_race->error().store.kind == LeaseStoreError::commit_conflict) &&
|
|
!take(load_cellular_migration_stage(*f.db, *destination_objects,
|
|
destination_key, destination_lease, 5s),
|
|
"stage race final pointer")
|
|
.has_value(),
|
|
"stage/abort race left an active ghost or stranded pause");
|
|
}
|
|
|
|
// Both sides of preparation's transaction boundary, with real process death.
|
|
for (unsigned point = 0; point < 2; ++point) {
|
|
const auto killed_request = request_for();
|
|
const auto child = ::fork();
|
|
require(child >= 0, "freeze publisher fork");
|
|
if (child == 0) {
|
|
try {
|
|
auto db = connection();
|
|
PublishHooks hooks;
|
|
if (point == 0)
|
|
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
|
|
if (!prepare_cellular_migration_freeze(*db, *f.objects, *destination_objects,
|
|
f.lease, f.head, destination_head,
|
|
killed_request, 5s, hooks))
|
|
::_exit(41);
|
|
::kill(::getpid(), SIGKILL);
|
|
} catch (...) {
|
|
::_exit(42);
|
|
}
|
|
::_exit(43);
|
|
}
|
|
int status{};
|
|
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
|
|
WTERMSIG(status) == SIGKILL &&
|
|
take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s),
|
|
"recover killed preparation")
|
|
.has_value() == (point == 1),
|
|
"publisher death changed source ownership freeze");
|
|
if (point == 1)
|
|
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease, killed_request.request,
|
|
5s),
|
|
"abort killed preparation");
|
|
}
|
|
for (unsigned point = 0; point < 2; ++point) {
|
|
const auto killed = take(prepare(request_for()), "stage process-loss preparation");
|
|
const auto child = ::fork();
|
|
require(child >= 0, "stage publisher fork");
|
|
if (child == 0) {
|
|
try {
|
|
auto db = connection();
|
|
PublishHooks hooks;
|
|
if (point == 0)
|
|
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
|
|
if (!stage_cellular_migration(*db, *f.objects, *destination_objects,
|
|
destination_lease, f.head, destination_head,
|
|
killed.trace, 5s, hooks))
|
|
::_exit(44);
|
|
::kill(::getpid(), SIGKILL);
|
|
} catch (...) {
|
|
::_exit(45);
|
|
}
|
|
::_exit(46);
|
|
}
|
|
int status{};
|
|
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
|
|
WTERMSIG(status) == SIGKILL &&
|
|
take(load_cellular_migration_stage(*f.db, *destination_objects,
|
|
destination_key, destination_lease, 5s),
|
|
"recover killed stage")
|
|
.has_value() == (point == 1),
|
|
"stage process loss changed ownership or lost the durable pause");
|
|
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
|
|
killed.trace.request.request, 5s),
|
|
"abort killed stage");
|
|
}
|
|
}
|
|
// The owning host must retain the freeze through drain/recreation with entry off.
|
|
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture to frozen host");
|
|
const auto make_host = [&] {
|
|
auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), f.key,
|
|
max_cellular_population_bytes),
|
|
"freeze host objects");
|
|
CellularHostConfig config{f.key, 1, {}, f.build, {f.build}};
|
|
config.lease.commit_timeout = 300ms;
|
|
return take(
|
|
CellularRegionHost::create(
|
|
config,
|
|
[] { return std::expected<std::unique_ptr<Postgres>, PgFailure>{connection()}; },
|
|
std::move(objects)),
|
|
"freeze host");
|
|
};
|
|
auto host = make_host();
|
|
require(host->step().ready, "source host before freeze");
|
|
const auto before = host->state()->snapshot();
|
|
const auto host_request = request_for();
|
|
take(host->prepare_migration_freeze(destination_head, *destination_objects, host_request),
|
|
"host source freeze");
|
|
require(!host->status().ready && host->status().migration_frozen &&
|
|
!host->advance_unattended(64) && host->state()->snapshot() == before,
|
|
"frozen host advanced or originated current output");
|
|
host->drain();
|
|
host = make_host();
|
|
require(!host->step().ready && host->status().migration_frozen &&
|
|
host->state()->snapshot() == before && !host->authority().has_value(),
|
|
"replacement host implicitly aborted or ignored its freeze");
|
|
take(host->abort_migration_freeze(host_request.request), "replacement host explicit abort");
|
|
require(host->step().ready && !host->status().migration_frozen &&
|
|
host->state()->snapshot() == before,
|
|
"host failed to resume the identical source after confirmed abort");
|
|
auto locker = connection(true);
|
|
take(locker->query("BEGIN", {}, 5s), "host unknown freeze lock");
|
|
take(
|
|
locker->query(
|
|
"SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE",
|
|
{}, 5s),
|
|
"hold host freeze reply");
|
|
const auto uncertain_request = request_for();
|
|
const auto uncertain =
|
|
host->prepare_migration_freeze(destination_head, *destination_objects, uncertain_request);
|
|
require(!uncertain && !host->status().ready, "host originated during uncertain freeze");
|
|
take(locker->query("ROLLBACK", {}, 5s), "release host freeze reply");
|
|
for (unsigned attempt = 0; attempt < 200 && !host->status().migration_frozen; ++attempt) {
|
|
host->step();
|
|
std::this_thread::sleep_for(25ms);
|
|
}
|
|
require(host->status().migration_frozen && !host->status().ready &&
|
|
host->state()->snapshot() == before,
|
|
"host chose different biology after unknown preparation");
|
|
take(locker->query("BEGIN", {}, 5s), "host unknown abort lock");
|
|
take(
|
|
locker->query(
|
|
"SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE",
|
|
{}, 5s),
|
|
"hold host abort reply");
|
|
const auto unknown_abort = host->abort_migration_freeze(uncertain_request.request);
|
|
require(!unknown_abort && !host->status().ready, "unconfirmed abort resumed the source");
|
|
take(locker->query("ROLLBACK", {}, 5s), "release host abort reply");
|
|
for (unsigned attempt = 0; attempt < 200 && !host->step().ready; ++attempt)
|
|
std::this_thread::sleep_for(25ms);
|
|
require(host->status().ready && !host->status().migration_frozen &&
|
|
host->state()->snapshot() == before,
|
|
"host failed exact ambiguous-abort reconciliation");
|
|
host->drain();
|
|
f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s),
|
|
"return freeze fixture authority");
|
|
const auto destination_host_freeze =
|
|
take(prepare_cellular_migration_freeze(*f.db, *f.objects, *destination_objects, f.lease,
|
|
f.head, destination_head, request_for(), 5s),
|
|
"destination host source preparation");
|
|
take(release_region_lease(*f.db, destination_key, destination_lease, 5s),
|
|
"release destination to owning host");
|
|
const auto make_destination_host = [&] {
|
|
auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), destination_key,
|
|
max_cellular_population_bytes),
|
|
"destination host objects");
|
|
CellularHostConfig config{destination_key, 1, {}, f.build, {f.build}};
|
|
config.lease.commit_timeout = 300ms;
|
|
return take(
|
|
CellularRegionHost::create(
|
|
config,
|
|
[] { return std::expected<std::unique_ptr<Postgres>, PgFailure>{connection()}; },
|
|
std::move(objects)),
|
|
"destination staging host");
|
|
};
|
|
auto destination_host = make_destination_host();
|
|
require(destination_host->step().ready, "destination host before staging");
|
|
const auto destination_bytes = destination_host->state()->snapshot();
|
|
take(locker->query("BEGIN", {}, 5s), "host stage ambiguity transaction");
|
|
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND "
|
|
"region_id=1 FOR UPDATE",
|
|
{}, 5s),
|
|
"hold owning destination stage");
|
|
const auto host_stage_unknown =
|
|
destination_host->stage_migration(f.head, *f.objects, destination_host_freeze.trace);
|
|
require(!host_stage_unknown && !destination_host->status().ready &&
|
|
destination_host->state()->snapshot() == destination_bytes,
|
|
"owning destination advanced during ambiguous staging");
|
|
take(locker->query("ROLLBACK", {}, 5s), "release owning stage reply");
|
|
for (unsigned attempt = 0; attempt < 200 && !destination_host->status().migration_frozen;
|
|
++attempt) {
|
|
destination_host->step();
|
|
std::this_thread::sleep_for(25ms);
|
|
}
|
|
require(destination_host->status().migration_frozen && !destination_host->status().ready &&
|
|
!destination_host->advance_unattended(64),
|
|
"owning destination activated its ghost");
|
|
destination_host->drain();
|
|
destination_host = make_destination_host();
|
|
require(!destination_host->step().ready && destination_host->status().migration_frozen &&
|
|
destination_host->state()->snapshot() == destination_bytes &&
|
|
!destination_host->authority().has_value(),
|
|
"replacement destination lost staging or adopted the uncommitted ghost");
|
|
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
|
|
destination_host_freeze.trace.request.request, 5s),
|
|
"source-authorized abort releases destination host");
|
|
for (unsigned attempt = 0; attempt < 200 && !destination_host->step().ready; ++attempt)
|
|
std::this_thread::sleep_for(25ms);
|
|
require(destination_host->status().ready && !destination_host->status().migration_frozen &&
|
|
destination_host->state()->snapshot() == destination_bytes,
|
|
"destination failed to resume its unchanged original state after source abort");
|
|
destination_host->drain();
|
|
destination_lease = take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s),
|
|
"return staged destination fixture authority");
|
|
migration_publication(f, destination_key, *destination_objects, destination_lease,
|
|
destination_head);
|
|
f.head = take(
|
|
materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
|
|
"resume frozen population as raw owner");
|
|
f.raw = take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s),
|
|
"source after frozen profile materialization")
|
|
.state;
|
|
take(release_region_lease(*f.db, destination_key, destination_lease, 5s),
|
|
"release freeze destination");
|
|
}
|
|
|
|
void host_ownership_and_admission(Fixture& f) {
|
|
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture host authority");
|
|
const auto make_host = [&](bool enabled) {
|
|
auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), f.key,
|
|
max_cellular_population_bytes),
|
|
"host objects");
|
|
CellularHostConfig config{f.key, 1, {}, f.build, {f.build}};
|
|
config.lease.commit_timeout = 300ms;
|
|
if (enabled)
|
|
config.population_policy = CellularTierPolicy{true, 1200, 1800};
|
|
return take(CellularRegionHost::create(
|
|
std::move(config),
|
|
[] {
|
|
return Postgres::connect({environment("PGHOST"),
|
|
{},
|
|
environment("PGPORT"),
|
|
environment("PGDATABASE"),
|
|
environment("EVO_PG_RUNTIME_USER"),
|
|
{},
|
|
{}},
|
|
5s);
|
|
},
|
|
std::move(objects)),
|
|
"population host");
|
|
};
|
|
auto host = make_host(false);
|
|
require(host->step().ready && !host->population_mode(), "raw host recovery");
|
|
require(!host->set_population_mode(true), "default host enabled population entry");
|
|
host->drain();
|
|
host = make_host(true);
|
|
require(host->step().ready, "opt-in host recovery");
|
|
const auto before = host->state()->snapshot();
|
|
const auto tick = host->state()->tick();
|
|
take(host->set_population_mode(true), "host population entry");
|
|
require(host->population_mode() && host->state()->snapshot() == before,
|
|
"host entry changed biology");
|
|
auto population =
|
|
take(CellularPopulation::create(
|
|
take(CellularLives::restore(before, f.key.world, f.key.region, 1, tick),
|
|
"host reference restore"),
|
|
{true, 1200, 1800}),
|
|
"host reference population");
|
|
f.head = take(host->committed(), "host population head");
|
|
auto next = unattended(f); // uses the same deterministic frozen/wildlife input contract.
|
|
// The fixture's old envelope may have a different membership history; biology is shared.
|
|
auto expected =
|
|
take(prepare_population_batch(*population, next.batch.biology), "host reference batch");
|
|
require(!take(host->advance_unattended(19), "collect bounded publication window").has_value() &&
|
|
host->state()->snapshot() == before,
|
|
"small local debt was admitted prematurely");
|
|
const auto published = take(host->advance_unattended(65), "host population publication");
|
|
require(published.has_value(), "host skipped admitted interval");
|
|
require(host->population_mode() &&
|
|
host->state()->snapshot() == expected.state->state().snapshot(),
|
|
"host population continuation differs");
|
|
const auto committed = take(host->committed(), "host continued head");
|
|
const auto continued = host->state()->snapshot();
|
|
const CellularBatchHeader recovered_header{f.key.world,
|
|
f.key.region,
|
|
1,
|
|
committed.head.commit,
|
|
committed.head.digest,
|
|
committed.head.end_tick,
|
|
committed.head.end_tick + 64};
|
|
auto recovered_expected = take(
|
|
prepare_population_batch(
|
|
*expected.state, take(unattended_population_inputs(*host->state(), recovered_header),
|
|
"recovered interval inputs")),
|
|
"recovered interval reference");
|
|
take(admit_population_interval(*f.db, f.key, take(host->authority(), "host interval authority"),
|
|
committed.head, committed.head.end_tick + 64, 5s),
|
|
"admit before host loss");
|
|
host->drain();
|
|
host = make_host(false);
|
|
require(host->step().ready && host->population_mode() &&
|
|
take(host->committed(), "host recovered head").head == committed.head &&
|
|
host->state()->snapshot() == continued,
|
|
"restarted host failed to recover population head");
|
|
require(host->status().ecological_pending_ticks == 64 && !host->set_population_mode(false),
|
|
"restarted host hid or skipped admitted work");
|
|
require(take(host->advance_unattended(0), "resume admitted work with zero new due time")
|
|
.has_value() &&
|
|
host->state()->snapshot() == recovered_expected.state->state().snapshot() &&
|
|
host->status().ecological_pending_ticks == 0,
|
|
"recovered interval did not settle exactly");
|
|
const auto recovered_head = take(host->committed(), "recovered interval head");
|
|
const auto recovered_bytes = host->state()->snapshot();
|
|
const CellularBatchHeader unknown_header{f.key.world,
|
|
f.key.region,
|
|
1,
|
|
recovered_head.head.commit,
|
|
recovered_head.head.digest,
|
|
recovered_head.head.end_tick,
|
|
recovered_head.head.end_tick + 64};
|
|
auto unknown_expected = take(
|
|
prepare_population_batch(*recovered_expected.state,
|
|
take(unattended_population_inputs(*host->state(), unknown_header),
|
|
"uncertain admission inputs")),
|
|
"uncertain admission reference");
|
|
auto locker = connection(true);
|
|
take(locker->query("BEGIN", {}, 5s), "host admission lock transaction");
|
|
take(
|
|
locker->query(
|
|
"SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE",
|
|
{}, 5s),
|
|
"hold host admission reply");
|
|
require(!host->advance_unattended(64) && !host->status().ready &&
|
|
host->state()->snapshot() == recovered_bytes,
|
|
"uncertain admission allowed simulation/output");
|
|
take(locker->query("ROLLBACK", {}, 5s), "release host admission reply");
|
|
for (unsigned attempt = 0; attempt < 200 && !host->step().ready; ++attempt)
|
|
std::this_thread::sleep_for(25ms);
|
|
require(host->status().ready && host->status().ecological_pending_ticks == 64,
|
|
"unknown admission failed exact reconciliation");
|
|
require(take(host->advance_unattended(0), "finish reconciled admission").has_value() &&
|
|
host->state()->snapshot() == unknown_expected.state->state().snapshot(),
|
|
"admission retry chose different work");
|
|
const auto final_bytes = host->state()->snapshot();
|
|
const auto final_authority = take(host->authority(), "final host authority");
|
|
Uuid account{};
|
|
account.back() = 1;
|
|
// A returning player's operation completes the durable reverse transition before lookup.
|
|
[[maybe_unused]] const auto queried = host->query_primitive_life(account, 1, 1);
|
|
require(!host->population_mode() && host->state()->snapshot() == final_bytes &&
|
|
host->state()->tick() == tick + 192,
|
|
"player operation did not materialize first");
|
|
host->drain();
|
|
// drain is best effort after the deliberate unknown-admission connection
|
|
// loss. Confirm the fixture handoff before another test acquires authority.
|
|
const auto released = release_region_lease(*f.db, f.key, final_authority, 5s);
|
|
require(released.has_value() || released.error().kind == LeaseStoreError::fenced,
|
|
"final host authority release failed");
|
|
}
|
|
} // namespace
|
|
void run_population_publication_tests() {
|
|
Fixture fixture;
|
|
funded_source(fixture);
|
|
history_pages(fixture);
|
|
snapshot_guards(fixture);
|
|
registered_snapshot_coverage(fixture);
|
|
conversions_and_retries(fixture);
|
|
interval_contract(fixture);
|
|
ambiguous_publication(fixture, false);
|
|
ambiguous_publication(fixture, true);
|
|
process_loss(fixture, 0);
|
|
process_loss(fixture, 1);
|
|
process_loss(fixture, 2);
|
|
missing_and_corrupt_objects(fixture);
|
|
isolation_and_reverse(fixture);
|
|
boundary_upgrades(fixture);
|
|
migration_freezes(fixture);
|
|
host_ownership_and_admission(fixture);
|
|
fixture.lease = take(acquire_region_lease(*fixture.db, fixture.key, fixture.build, 60000ms, 5s),
|
|
"maturation replacement authority");
|
|
const std::array<Sha256Digest, 1> retained{fixture.build};
|
|
auto maturation_source =
|
|
take(recover_cellular_checkpoint(*fixture.db, *fixture.objects, fixture.key, fixture.lease,
|
|
retained, 5s),
|
|
"maturation source recovery");
|
|
fixture.head = std::move(maturation_source.committed);
|
|
fixture.raw = std::move(maturation_source.state);
|
|
boundary_upgrades(fixture, true);
|
|
host_ownership_and_admission(fixture);
|
|
std::puts("population publication, exact recovery, process loss and fencing passed");
|
|
}
|