M12.6: capture consistent regional census snapshots
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m20s
Foundation / checks (network) (push) Successful in 21m3s
Foundation / checks (client) (push) Successful in 22m14s
Foundation / checks (tsan) (push) Successful in 24m52s
Foundation / checks (postgres) (push) Failing after 36m19s
Foundation / checks (checks) (push) Successful in 1h10m25s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m20s
Foundation / checks (network) (push) Successful in 21m3s
Foundation / checks (client) (push) Successful in 22m14s
Foundation / checks (tsan) (push) Successful in 24m52s
Foundation / checks (postgres) (push) Failing after 36m19s
Foundation / checks (checks) (push) Successful in 1h10m25s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
#include "evo/persistence/account_control_store.hpp"
|
||||
#include "evo/persistence/cellular_census_snapshot.hpp"
|
||||
#include "evo/persistence/cellular_history_store.hpp"
|
||||
#include "evo/persistence/population_checkpoint_store.hpp"
|
||||
#include "evo/persistence/population_interval_store.hpp"
|
||||
@@ -790,6 +791,158 @@ void history_pages(Fixture& f) {
|
||||
"history transaction requirement changed for read-only callers");
|
||||
std::puts("committed census history pagination, replay, transaction and failure guards passed");
|
||||
}
|
||||
void snapshot_transport_faults(Fixture& f) {
|
||||
if (!std::getenv("EVO_MIGRATION_TCP_PORT"))
|
||||
return;
|
||||
const std::array<Sha256Digest, 1> builds{f.build};
|
||||
const std::array requests{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()}};
|
||||
for (unsigned mode = 0; mode < 4; ++mode) {
|
||||
test::PostgresRelay relay(migration_backend_port());
|
||||
auto db = migration_tls_connection(relay);
|
||||
const bool at_completion = mode >= 2;
|
||||
const bool lost_request = mode % 2 == 0;
|
||||
const auto partition = [&] {
|
||||
if (lost_request)
|
||||
relay.drop_requests(true);
|
||||
else
|
||||
relay.drop_responses(true);
|
||||
};
|
||||
if (!at_completion)
|
||||
partition();
|
||||
const CellularCensusSnapshotHooks hook{[&](std::size_t) {
|
||||
if (at_completion)
|
||||
partition();
|
||||
}};
|
||||
const auto lost =
|
||||
load_cellular_census_snapshot(*db, f.key.world, requests, builds, 300ms, hook);
|
||||
require(!lost && lost.error().kind == CellularSnapshotError::transaction &&
|
||||
lost.error().transaction.kind == PgError::unknown && !db->transaction_idle() &&
|
||||
relay.healthy() &&
|
||||
(lost_request ? relay.dropped_requests() : relay.dropped_responses()) > 0 &&
|
||||
take(load_region_checkpoint(*f.db, f.key, 5s), "head after lost capture I/O")
|
||||
.head == f.head.head,
|
||||
"lost snapshot BEGIN/COMMIT request/reply returned evidence or kept a session");
|
||||
}
|
||||
std::puts("snapshot TLS BEGIN/COMMIT request/reply loss fails closed");
|
||||
}
|
||||
void snapshot_guards(Fixture& f) {
|
||||
const std::array<Sha256Digest, 1> builds{f.build};
|
||||
const std::array requests{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()}};
|
||||
const auto read = [&] {
|
||||
return load_cellular_census_snapshot(*f.db, f.key.world, requests, builds, 5s);
|
||||
};
|
||||
const auto before = f.head;
|
||||
const auto first = take(read(), "single-region captured snapshot");
|
||||
const auto second = take(read(), "repeat captured snapshot");
|
||||
require(first.cuts.size() == 1 && first.cuts[0].head == before.head &&
|
||||
first.cuts[0].manifest == before.manifest &&
|
||||
second.cuts[0].head == first.cuts[0].head &&
|
||||
std::ranges::equal(
|
||||
first.census.regions()[0].rows(),
|
||||
take(CellularCensus::observe(*f.raw), "snapshot direct oracle").rows()) &&
|
||||
f.db->transaction_idle(),
|
||||
"captured snapshot lost its exact committed cut/biology or retained a transaction");
|
||||
const auto invalid = [&](auto region_requests, std::uint64_t world = 30201,
|
||||
std::chrono::milliseconds timeout = 5s) {
|
||||
const auto result =
|
||||
load_cellular_census_snapshot(*f.db, world, region_requests, builds, timeout);
|
||||
require(!result && result.error().kind == CellularSnapshotError::invalid_request &&
|
||||
f.db->transaction_idle(),
|
||||
"invalid snapshot request started a transaction or returned a census");
|
||||
};
|
||||
invalid(std::vector<CellularCensusRegionRequest>{});
|
||||
invalid(requests, 0);
|
||||
invalid(requests, f.key.world, 0ms);
|
||||
invalid(requests, f.key.world, 30001ms);
|
||||
auto bad = requests;
|
||||
bad[0].objects = nullptr;
|
||||
invalid(bad);
|
||||
bad = requests;
|
||||
bad[0].region = 0;
|
||||
invalid(bad);
|
||||
bad = requests;
|
||||
bad[0].lease.frame = 0;
|
||||
invalid(bad);
|
||||
invalid(std::array{requests[0], requests[0]});
|
||||
auto higher = requests[0];
|
||||
++higher.region;
|
||||
invalid(std::array{higher, requests[0]});
|
||||
invalid(std::vector<CellularCensusRegionRequest>(max_cellular_census_regions + 1, requests[0]));
|
||||
higher.region = 999'999;
|
||||
const auto missing = load_cellular_census_snapshot(*f.db, f.key.world,
|
||||
std::array{requests[0], higher}, builds, 5s);
|
||||
require(!missing && missing.error().kind == CellularSnapshotError::history &&
|
||||
missing.error().region == higher.region && f.db->transaction_idle(),
|
||||
"missing region became partial coverage or left capture open");
|
||||
for (const auto* begin : {"BEGIN", "BEGIN READ ONLY"}) {
|
||||
take(f.db->query(begin, {}, 5s), "caller snapshot transaction");
|
||||
if (std::string_view{begin} == "BEGIN")
|
||||
take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build,
|
||||
{false, 1200, 1800}, 5s),
|
||||
"pending caller snapshot publication");
|
||||
const auto refused = read();
|
||||
require(!refused && refused.error().kind == CellularSnapshotError::invalid_request &&
|
||||
!f.db->transaction_idle(),
|
||||
"snapshot took ownership of a caller transaction");
|
||||
take(f.db->query("ROLLBACK", {}, 5s), "caller snapshot rollback");
|
||||
}
|
||||
const auto manifest = take(decode_manifest(before.manifest), "snapshot retained payload");
|
||||
const auto path = f.objects->directory() / hexadecimal(manifest.payload_sha256, false);
|
||||
const auto saved = path.string() + ".snapshot-test";
|
||||
std::filesystem::rename(path, saved);
|
||||
const auto missing_object = read();
|
||||
std::filesystem::rename(saved, path);
|
||||
require(!missing_object && missing_object.error().kind == CellularSnapshotError::history &&
|
||||
f.db->transaction_idle(),
|
||||
"missing replay payload became a snapshot or left capture open");
|
||||
{
|
||||
const UnreadableObject denied{path};
|
||||
const auto unreadable = read();
|
||||
require(!unreadable && unreadable.error().kind == CellularSnapshotError::history &&
|
||||
f.db->transaction_idle(),
|
||||
"unreadable captured payload became a census");
|
||||
}
|
||||
auto stale = requests;
|
||||
--stale[0].lease.epoch;
|
||||
const auto fenced = load_cellular_census_snapshot(*f.db, f.key.world, stale, builds, 5s);
|
||||
require(!fenced && fenced.error().history.recovery.store.kind == LeaseStoreError::fenced,
|
||||
"snapshot ignored stale evidence authority");
|
||||
const std::array<Sha256Digest, 1> foreign_builds{};
|
||||
require(!load_cellular_census_snapshot(*f.db, f.key.world, requests, foreign_builds, 5s),
|
||||
"snapshot accepted unretained build evidence");
|
||||
auto closed = connection();
|
||||
CellularCensusSnapshotHooks close_hook{[&](std::size_t) { closed->disconnect(); }};
|
||||
const auto lost =
|
||||
load_cellular_census_snapshot(*closed, f.key.world, requests, builds, 5s, close_hook);
|
||||
closed->disconnect();
|
||||
const auto closed_query = closed->query("SELECT 1", {}, 5s);
|
||||
require(!lost && lost.error().kind == CellularSnapshotError::transaction &&
|
||||
!closed->transaction_idle() && !closed_query &&
|
||||
closed_query.error().kind == PgError::closed,
|
||||
"failed capture completion reused a dead session or returned a partial census");
|
||||
auto throwing = connection();
|
||||
bool threw = false;
|
||||
bool hook_called = false;
|
||||
try {
|
||||
const CellularCensusSnapshotHooks hook{[&](std::size_t) {
|
||||
hook_called = true;
|
||||
throw std::runtime_error("capture hook test exception");
|
||||
}};
|
||||
const auto result =
|
||||
load_cellular_census_snapshot(*throwing, f.key.world, requests, builds, 5s, hook);
|
||||
require(!result, "throwing capture hook returned a snapshot");
|
||||
} catch (const std::runtime_error&) {
|
||||
threw = hook_called;
|
||||
}
|
||||
const auto after_throw = throwing->query("SELECT 1", {}, 5s);
|
||||
require(threw && !after_throw && after_throw.error().kind == PgError::closed &&
|
||||
!throwing->transaction_idle() &&
|
||||
take(load_region_checkpoint(*f.db, f.key, 5s), "head after snapshot guards").head ==
|
||||
before.head,
|
||||
"capture exception retained its connection/transaction or changed a head");
|
||||
std::puts("consistent census snapshot guards and connection cleanup passed");
|
||||
snapshot_transport_faults(f);
|
||||
}
|
||||
void boundary_upgrades(Fixture& f) {
|
||||
const std::array<Sha256Digest, 1> builds{f.build};
|
||||
const auto original = f.head;
|
||||
@@ -1180,6 +1333,87 @@ void migration_owner_partitions(Fixture& f, RegionKey destination_key,
|
||||
std::puts("migration independent TLS owner partitions, expiry, replacement and heal passed");
|
||||
}
|
||||
|
||||
void migration_snapshot_race(Fixture& f, RegionKey destination_key,
|
||||
FilesystemObjectStore& destination_objects,
|
||||
const LeaseGrant& destination_lease,
|
||||
CommittedCheckpoint& destination_head, std::uint64_t genome) {
|
||||
const std::array<Sha256Digest, 1> builds{f.build};
|
||||
const std::array requests{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()},
|
||||
CellularCensusRegionRequest{destination_key.region, destination_lease,
|
||||
&destination_objects}};
|
||||
auto reader = connection();
|
||||
for (const bool reverse : {false, true}) {
|
||||
auto& sh = reverse ? destination_head : f.head;
|
||||
auto& dh = reverse ? f.head : destination_head;
|
||||
auto& so = reverse ? destination_objects : *f.objects;
|
||||
auto& dst = reverse ? *f.objects : destination_objects;
|
||||
const auto& sl = reverse ? destination_lease : f.lease;
|
||||
const auto& dl = reverse ? f.lease : destination_lease;
|
||||
const auto sk = reverse ? destination_key : f.key;
|
||||
const auto dk = reverse ? f.key : destination_key;
|
||||
const auto before =
|
||||
take(load_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s),
|
||||
"before snapshot migration race");
|
||||
const CellularMigrationRequest request{f.id(),
|
||||
{sk.region, sh.head.commit, f.id(), sh.head.digest},
|
||||
{dk.region, dh.head.commit, f.id(), dh.head.digest},
|
||||
genome};
|
||||
const auto frozen =
|
||||
take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s),
|
||||
"snapshot race source freeze");
|
||||
take(stage_cellular_migration(*f.db, so, dst, dl, sh, dh, frozen.trace, 5s),
|
||||
"snapshot race destination stage");
|
||||
std::optional<CellularMigrationCommitted> published;
|
||||
const CellularCensusSnapshotHooks hook{[&](std::size_t index) {
|
||||
if (index != 0)
|
||||
return;
|
||||
const auto settings =
|
||||
take(reader->query("SELECT current_setting('transaction_isolation'), "
|
||||
"current_setting('transaction_read_only')",
|
||||
{}, 5s),
|
||||
"capture transaction properties");
|
||||
require(settings.size() == 1 && settings[0].size() == 2 &&
|
||||
take(settings[0][0], "capture isolation") == "repeatable read" &&
|
||||
take(settings[0][1], "capture access") == "on",
|
||||
"capture did not own a repeatable read read-only snapshot");
|
||||
auto publisher = connection();
|
||||
published = take(publish_cellular_migration(*publisher, so, dst, sl, dl, sh, dh,
|
||||
frozen.trace, f.build, 5s),
|
||||
"publish real pair between snapshot head reads");
|
||||
}};
|
||||
const auto raced =
|
||||
take(load_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s, hook),
|
||||
"coherent census across a concurrent pair");
|
||||
const auto committed =
|
||||
take(std::move(published), "pair was not published in snapshot hook");
|
||||
sh = committed.source;
|
||||
dh = committed.destination;
|
||||
require(reader->transaction_idle() && raced.cuts.size() == 2 && before.cuts.size() == 2,
|
||||
"raced capture retained its transaction or lost coverage");
|
||||
for (std::size_t i = 0; i < before.cuts.size(); ++i)
|
||||
require(raced.cuts[i].head == before.cuts[i].head &&
|
||||
raced.cuts[i].manifest == before.cuts[i].manifest &&
|
||||
std::ranges::equal(raced.census.regions()[i].rows(),
|
||||
before.census.regions()[i].rows()),
|
||||
"atomic pair split a captured cut or changed historical replay");
|
||||
const auto after =
|
||||
take(load_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s),
|
||||
"snapshot after pair publication");
|
||||
require(after.cuts[0].head == f.head.head && after.cuts[1].head == destination_head.head &&
|
||||
after.cuts[0].head.sequence == before.cuts[0].head.sequence + 1 &&
|
||||
after.cuts[1].head.sequence == before.cuts[1].head.sequence + 1,
|
||||
"later snapshot omitted one committed migration endpoint");
|
||||
for (const auto scope :
|
||||
{CellularCensusScope::exact_genome, CellularCensusScope::descendants}) {
|
||||
const auto initial = take(before.census.query(genome, scope), "before pair lineage");
|
||||
const auto captured = take(raced.census.query(genome, scope), "captured pair lineage");
|
||||
const auto settled = take(after.census.query(genome, scope), "settled pair lineage");
|
||||
require(initial == captured && initial.resident == settled.resident,
|
||||
"snapshot migration race lost/duplicated living count/body/reserve");
|
||||
}
|
||||
}
|
||||
std::puts("consistent raw/cohort snapshots across atomic migration commits passed");
|
||||
}
|
||||
void migration_publication(Fixture& f, const RegionKey destination_key,
|
||||
FilesystemObjectStore& destination_objects,
|
||||
LeaseGrant& destination_lease, CommittedCheckpoint& destination_head) {
|
||||
@@ -1693,6 +1927,8 @@ void migration_publication(Fixture& f, const RegionKey destination_key,
|
||||
"return migration rewrote earlier committed ownership observations");
|
||||
migration_owner_partitions(f, destination_key, destination_objects, destination_lease,
|
||||
destination_head, genome);
|
||||
migration_snapshot_race(f, destination_key, destination_objects, destination_lease,
|
||||
destination_head, genome);
|
||||
}
|
||||
std::puts("migration object sync/read failures, quarantine and exact recovery passed");
|
||||
// Race the actual two-head transaction against abort on independent connections.
|
||||
@@ -1796,6 +2032,16 @@ void migration_freezes(Fixture& f) {
|
||||
auto destination_lease =
|
||||
take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s),
|
||||
"migration destination authority");
|
||||
const auto unequal_cut = load_cellular_census_snapshot(
|
||||
*f.db, f.key.world,
|
||||
std::array{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()},
|
||||
CellularCensusRegionRequest{destination_key.region, destination_lease,
|
||||
destination_objects.get()}},
|
||||
std::array<Sha256Digest, 1>{f.build}, 5s);
|
||||
require(!unequal_cut && unequal_cut.error().kind == CellularSnapshotError::census &&
|
||||
unequal_cut.error().census == CellularCensusSetError::boundary &&
|
||||
f.db->transaction_idle(),
|
||||
"snapshot silently reconciled different current ticks/owner versions");
|
||||
while (destination->tick() < f.raw->tick()) {
|
||||
const auto count = std::min(std::uint64_t{64}, f.raw->tick() - destination->tick());
|
||||
CellularBatch inputs{{f.key.world, 2, 1, destination_head.head.commit,
|
||||
@@ -2606,6 +2852,7 @@ void run_population_publication_tests() {
|
||||
Fixture fixture;
|
||||
funded_source(fixture);
|
||||
history_pages(fixture);
|
||||
snapshot_guards(fixture);
|
||||
conversions_and_retries(fixture);
|
||||
interval_contract(fixture);
|
||||
ambiguous_publication(fixture, false);
|
||||
|
||||
Reference in New Issue
Block a user