M12.6: capture consistent regional census snapshots
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m20s
Foundation / checks (network) (push) Successful in 21m3s
Foundation / checks (client) (push) Successful in 22m14s
Foundation / checks (tsan) (push) Successful in 24m52s
Foundation / checks (postgres) (push) Failing after 36m19s
Foundation / checks (checks) (push) Successful in 1h10m25s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped

This commit is contained in:
dingo
2026-10-18 15:56:58 -04:00
parent 82568da9cd
commit 7631f9d2e2
17 changed files with 648 additions and 3 deletions
+247
View File
@@ -1,4 +1,5 @@
#include "evo/persistence/account_control_store.hpp"
#include "evo/persistence/cellular_census_snapshot.hpp"
#include "evo/persistence/cellular_history_store.hpp"
#include "evo/persistence/population_checkpoint_store.hpp"
#include "evo/persistence/population_interval_store.hpp"
@@ -790,6 +791,158 @@ void history_pages(Fixture& f) {
"history transaction requirement changed for read-only callers");
std::puts("committed census history pagination, replay, transaction and failure guards passed");
}
void snapshot_transport_faults(Fixture& f) {
if (!std::getenv("EVO_MIGRATION_TCP_PORT"))
return;
const std::array<Sha256Digest, 1> builds{f.build};
const std::array requests{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()}};
for (unsigned mode = 0; mode < 4; ++mode) {
test::PostgresRelay relay(migration_backend_port());
auto db = migration_tls_connection(relay);
const bool at_completion = mode >= 2;
const bool lost_request = mode % 2 == 0;
const auto partition = [&] {
if (lost_request)
relay.drop_requests(true);
else
relay.drop_responses(true);
};
if (!at_completion)
partition();
const CellularCensusSnapshotHooks hook{[&](std::size_t) {
if (at_completion)
partition();
}};
const auto lost =
load_cellular_census_snapshot(*db, f.key.world, requests, builds, 300ms, hook);
require(!lost && lost.error().kind == CellularSnapshotError::transaction &&
lost.error().transaction.kind == PgError::unknown && !db->transaction_idle() &&
relay.healthy() &&
(lost_request ? relay.dropped_requests() : relay.dropped_responses()) > 0 &&
take(load_region_checkpoint(*f.db, f.key, 5s), "head after lost capture I/O")
.head == f.head.head,
"lost snapshot BEGIN/COMMIT request/reply returned evidence or kept a session");
}
std::puts("snapshot TLS BEGIN/COMMIT request/reply loss fails closed");
}
void snapshot_guards(Fixture& f) {
const std::array<Sha256Digest, 1> builds{f.build};
const std::array requests{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()}};
const auto read = [&] {
return load_cellular_census_snapshot(*f.db, f.key.world, requests, builds, 5s);
};
const auto before = f.head;
const auto first = take(read(), "single-region captured snapshot");
const auto second = take(read(), "repeat captured snapshot");
require(first.cuts.size() == 1 && first.cuts[0].head == before.head &&
first.cuts[0].manifest == before.manifest &&
second.cuts[0].head == first.cuts[0].head &&
std::ranges::equal(
first.census.regions()[0].rows(),
take(CellularCensus::observe(*f.raw), "snapshot direct oracle").rows()) &&
f.db->transaction_idle(),
"captured snapshot lost its exact committed cut/biology or retained a transaction");
const auto invalid = [&](auto region_requests, std::uint64_t world = 30201,
std::chrono::milliseconds timeout = 5s) {
const auto result =
load_cellular_census_snapshot(*f.db, world, region_requests, builds, timeout);
require(!result && result.error().kind == CellularSnapshotError::invalid_request &&
f.db->transaction_idle(),
"invalid snapshot request started a transaction or returned a census");
};
invalid(std::vector<CellularCensusRegionRequest>{});
invalid(requests, 0);
invalid(requests, f.key.world, 0ms);
invalid(requests, f.key.world, 30001ms);
auto bad = requests;
bad[0].objects = nullptr;
invalid(bad);
bad = requests;
bad[0].region = 0;
invalid(bad);
bad = requests;
bad[0].lease.frame = 0;
invalid(bad);
invalid(std::array{requests[0], requests[0]});
auto higher = requests[0];
++higher.region;
invalid(std::array{higher, requests[0]});
invalid(std::vector<CellularCensusRegionRequest>(max_cellular_census_regions + 1, requests[0]));
higher.region = 999'999;
const auto missing = load_cellular_census_snapshot(*f.db, f.key.world,
std::array{requests[0], higher}, builds, 5s);
require(!missing && missing.error().kind == CellularSnapshotError::history &&
missing.error().region == higher.region && f.db->transaction_idle(),
"missing region became partial coverage or left capture open");
for (const auto* begin : {"BEGIN", "BEGIN READ ONLY"}) {
take(f.db->query(begin, {}, 5s), "caller snapshot transaction");
if (std::string_view{begin} == "BEGIN")
take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build,
{false, 1200, 1800}, 5s),
"pending caller snapshot publication");
const auto refused = read();
require(!refused && refused.error().kind == CellularSnapshotError::invalid_request &&
!f.db->transaction_idle(),
"snapshot took ownership of a caller transaction");
take(f.db->query("ROLLBACK", {}, 5s), "caller snapshot rollback");
}
const auto manifest = take(decode_manifest(before.manifest), "snapshot retained payload");
const auto path = f.objects->directory() / hexadecimal(manifest.payload_sha256, false);
const auto saved = path.string() + ".snapshot-test";
std::filesystem::rename(path, saved);
const auto missing_object = read();
std::filesystem::rename(saved, path);
require(!missing_object && missing_object.error().kind == CellularSnapshotError::history &&
f.db->transaction_idle(),
"missing replay payload became a snapshot or left capture open");
{
const UnreadableObject denied{path};
const auto unreadable = read();
require(!unreadable && unreadable.error().kind == CellularSnapshotError::history &&
f.db->transaction_idle(),
"unreadable captured payload became a census");
}
auto stale = requests;
--stale[0].lease.epoch;
const auto fenced = load_cellular_census_snapshot(*f.db, f.key.world, stale, builds, 5s);
require(!fenced && fenced.error().history.recovery.store.kind == LeaseStoreError::fenced,
"snapshot ignored stale evidence authority");
const std::array<Sha256Digest, 1> foreign_builds{};
require(!load_cellular_census_snapshot(*f.db, f.key.world, requests, foreign_builds, 5s),
"snapshot accepted unretained build evidence");
auto closed = connection();
CellularCensusSnapshotHooks close_hook{[&](std::size_t) { closed->disconnect(); }};
const auto lost =
load_cellular_census_snapshot(*closed, f.key.world, requests, builds, 5s, close_hook);
closed->disconnect();
const auto closed_query = closed->query("SELECT 1", {}, 5s);
require(!lost && lost.error().kind == CellularSnapshotError::transaction &&
!closed->transaction_idle() && !closed_query &&
closed_query.error().kind == PgError::closed,
"failed capture completion reused a dead session or returned a partial census");
auto throwing = connection();
bool threw = false;
bool hook_called = false;
try {
const CellularCensusSnapshotHooks hook{[&](std::size_t) {
hook_called = true;
throw std::runtime_error("capture hook test exception");
}};
const auto result =
load_cellular_census_snapshot(*throwing, f.key.world, requests, builds, 5s, hook);
require(!result, "throwing capture hook returned a snapshot");
} catch (const std::runtime_error&) {
threw = hook_called;
}
const auto after_throw = throwing->query("SELECT 1", {}, 5s);
require(threw && !after_throw && after_throw.error().kind == PgError::closed &&
!throwing->transaction_idle() &&
take(load_region_checkpoint(*f.db, f.key, 5s), "head after snapshot guards").head ==
before.head,
"capture exception retained its connection/transaction or changed a head");
std::puts("consistent census snapshot guards and connection cleanup passed");
snapshot_transport_faults(f);
}
void boundary_upgrades(Fixture& f) {
const std::array<Sha256Digest, 1> builds{f.build};
const auto original = f.head;
@@ -1180,6 +1333,87 @@ void migration_owner_partitions(Fixture& f, RegionKey destination_key,
std::puts("migration independent TLS owner partitions, expiry, replacement and heal passed");
}
void migration_snapshot_race(Fixture& f, RegionKey destination_key,
FilesystemObjectStore& destination_objects,
const LeaseGrant& destination_lease,
CommittedCheckpoint& destination_head, std::uint64_t genome) {
const std::array<Sha256Digest, 1> builds{f.build};
const std::array requests{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()},
CellularCensusRegionRequest{destination_key.region, destination_lease,
&destination_objects}};
auto reader = connection();
for (const bool reverse : {false, true}) {
auto& sh = reverse ? destination_head : f.head;
auto& dh = reverse ? f.head : destination_head;
auto& so = reverse ? destination_objects : *f.objects;
auto& dst = reverse ? *f.objects : destination_objects;
const auto& sl = reverse ? destination_lease : f.lease;
const auto& dl = reverse ? f.lease : destination_lease;
const auto sk = reverse ? destination_key : f.key;
const auto dk = reverse ? f.key : destination_key;
const auto before =
take(load_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s),
"before snapshot migration race");
const CellularMigrationRequest request{f.id(),
{sk.region, sh.head.commit, f.id(), sh.head.digest},
{dk.region, dh.head.commit, f.id(), dh.head.digest},
genome};
const auto frozen =
take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s),
"snapshot race source freeze");
take(stage_cellular_migration(*f.db, so, dst, dl, sh, dh, frozen.trace, 5s),
"snapshot race destination stage");
std::optional<CellularMigrationCommitted> published;
const CellularCensusSnapshotHooks hook{[&](std::size_t index) {
if (index != 0)
return;
const auto settings =
take(reader->query("SELECT current_setting('transaction_isolation'), "
"current_setting('transaction_read_only')",
{}, 5s),
"capture transaction properties");
require(settings.size() == 1 && settings[0].size() == 2 &&
take(settings[0][0], "capture isolation") == "repeatable read" &&
take(settings[0][1], "capture access") == "on",
"capture did not own a repeatable read read-only snapshot");
auto publisher = connection();
published = take(publish_cellular_migration(*publisher, so, dst, sl, dl, sh, dh,
frozen.trace, f.build, 5s),
"publish real pair between snapshot head reads");
}};
const auto raced =
take(load_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s, hook),
"coherent census across a concurrent pair");
const auto committed =
take(std::move(published), "pair was not published in snapshot hook");
sh = committed.source;
dh = committed.destination;
require(reader->transaction_idle() && raced.cuts.size() == 2 && before.cuts.size() == 2,
"raced capture retained its transaction or lost coverage");
for (std::size_t i = 0; i < before.cuts.size(); ++i)
require(raced.cuts[i].head == before.cuts[i].head &&
raced.cuts[i].manifest == before.cuts[i].manifest &&
std::ranges::equal(raced.census.regions()[i].rows(),
before.census.regions()[i].rows()),
"atomic pair split a captured cut or changed historical replay");
const auto after =
take(load_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s),
"snapshot after pair publication");
require(after.cuts[0].head == f.head.head && after.cuts[1].head == destination_head.head &&
after.cuts[0].head.sequence == before.cuts[0].head.sequence + 1 &&
after.cuts[1].head.sequence == before.cuts[1].head.sequence + 1,
"later snapshot omitted one committed migration endpoint");
for (const auto scope :
{CellularCensusScope::exact_genome, CellularCensusScope::descendants}) {
const auto initial = take(before.census.query(genome, scope), "before pair lineage");
const auto captured = take(raced.census.query(genome, scope), "captured pair lineage");
const auto settled = take(after.census.query(genome, scope), "settled pair lineage");
require(initial == captured && initial.resident == settled.resident,
"snapshot migration race lost/duplicated living count/body/reserve");
}
}
std::puts("consistent raw/cohort snapshots across atomic migration commits passed");
}
void migration_publication(Fixture& f, const RegionKey destination_key,
FilesystemObjectStore& destination_objects,
LeaseGrant& destination_lease, CommittedCheckpoint& destination_head) {
@@ -1693,6 +1927,8 @@ void migration_publication(Fixture& f, const RegionKey destination_key,
"return migration rewrote earlier committed ownership observations");
migration_owner_partitions(f, destination_key, destination_objects, destination_lease,
destination_head, genome);
migration_snapshot_race(f, destination_key, destination_objects, destination_lease,
destination_head, genome);
}
std::puts("migration object sync/read failures, quarantine and exact recovery passed");
// Race the actual two-head transaction against abort on independent connections.
@@ -1796,6 +2032,16 @@ void migration_freezes(Fixture& f) {
auto destination_lease =
take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s),
"migration destination authority");
const auto unequal_cut = load_cellular_census_snapshot(
*f.db, f.key.world,
std::array{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()},
CellularCensusRegionRequest{destination_key.region, destination_lease,
destination_objects.get()}},
std::array<Sha256Digest, 1>{f.build}, 5s);
require(!unequal_cut && unequal_cut.error().kind == CellularSnapshotError::census &&
unequal_cut.error().census == CellularCensusSetError::boundary &&
f.db->transaction_idle(),
"snapshot silently reconciled different current ticks/owner versions");
while (destination->tick() < f.raw->tick()) {
const auto count = std::min(std::uint64_t{64}, f.raw->tick() - destination->tick());
CellularBatch inputs{{f.key.world, 2, 1, destination_head.head.commit,
@@ -2606,6 +2852,7 @@ void run_population_publication_tests() {
Fixture fixture;
funded_source(fixture);
history_pages(fixture);
snapshot_guards(fixture);
conversions_and_retries(fixture);
interval_contract(fixture);
ambiguous_publication(fixture, false);