M12.7: require registered regions in census snapshots
Foundation / checks (deploy) (push) Successful in 58s
Foundation / checks (ecs) (push) Successful in 2m36s
Foundation / checks (network) (push) Successful in 20m56s
Foundation / checks (client) (push) Successful in 22m19s
Foundation / checks (tsan) (push) Successful in 25m18s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 44m49s
Foundation / checks (postgres) (push) Canceled after 44m54s
Foundation / checks (deploy) (push) Successful in 58s
Foundation / checks (ecs) (push) Successful in 2m36s
Foundation / checks (network) (push) Successful in 20m56s
Foundation / checks (client) (push) Successful in 22m19s
Foundation / checks (tsan) (push) Successful in 25m18s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 44m49s
Foundation / checks (postgres) (push) Canceled after 44m54s
This commit is contained in:
@@ -791,15 +791,161 @@ void history_pages(Fixture& f) {
|
||||
"history transaction requirement changed for read-only callers");
|
||||
std::puts("committed census history pagination, replay, transaction and failure guards passed");
|
||||
}
|
||||
void registered_snapshot_coverage(Fixture& f) {
|
||||
constexpr std::uint64_t seed = 31001, world = seed + 200;
|
||||
const std::array<Sha256Digest, 1> builds{f.build};
|
||||
auto reader = connection();
|
||||
std::vector<std::unique_ptr<FilesystemObjectStore>> stores;
|
||||
std::vector<CellularCensusRegionRequest> resources;
|
||||
const auto enroll = [&](std::uint64_t region) {
|
||||
const RegionKey key{world, region};
|
||||
auto objects = take(
|
||||
FilesystemObjectStore::open(environment("PGHOST"), key, max_cellular_population_bytes),
|
||||
"registered empty region objects");
|
||||
const auto state = opening(seed, true, 1, false, region);
|
||||
take(provision_cellular_region(*f.owner, *objects, key, 1, *state, f.id(), f.build, 5s),
|
||||
"registered empty region genesis");
|
||||
const auto lease =
|
||||
take(acquire_region_lease(*f.db, key, f.build, 60000ms, 5s), "registered region grant");
|
||||
resources.push_back({region, lease, objects.get()});
|
||||
stores.push_back(std::move(objects));
|
||||
};
|
||||
const auto read = [&](std::span<const CellularCensusRegionRequest> requests) {
|
||||
return load_registered_cellular_census_snapshot(*reader, world, requests, builds, 5s);
|
||||
};
|
||||
const auto refused = [&](auto result, CellularCoverageError error) {
|
||||
require(!result && result.error().kind == CellularSnapshotError::coverage &&
|
||||
result.error().coverage.kind == error && reader->transaction_idle(),
|
||||
"registered coverage omitted an owner or retained its transaction");
|
||||
};
|
||||
enroll(1);
|
||||
const std::array first_request{resources.front()};
|
||||
const auto before = take(read(first_request), "registered single-region census");
|
||||
require(before.catalog == std::vector<RegisteredCellularRegion>{{1, 1}} &&
|
||||
before.snapshot.cuts.size() == 1,
|
||||
"registered snapshot lost catalog/cut evidence");
|
||||
const CellularCensusSnapshotHooks enrollment_hook{[&](std::size_t index) {
|
||||
if (index == 0)
|
||||
enroll(2); // Independent committed enrollment after the registry snapshot.
|
||||
}};
|
||||
const auto raced = take(load_registered_cellular_census_snapshot(*reader, world, first_request,
|
||||
builds, 5s, enrollment_hook),
|
||||
"registered snapshot concurrent enrollment");
|
||||
require(raced.catalog == before.catalog &&
|
||||
raced.snapshot.cuts[0].head == before.snapshot.cuts[0].head,
|
||||
"concurrent enrollment changed an already selected registry cut");
|
||||
refused(read(first_request), CellularCoverageError::regions);
|
||||
auto extra = resources;
|
||||
extra.push_back({999, resources.back().lease, stores.back().get()});
|
||||
refused(read(extra), CellularCoverageError::regions);
|
||||
auto wrong = resources;
|
||||
++wrong.back().lease.frame;
|
||||
refused(read(wrong), CellularCoverageError::frame);
|
||||
wrong = resources;
|
||||
wrong.back().region = 999;
|
||||
refused(read(wrong), CellularCoverageError::regions);
|
||||
wrong = resources;
|
||||
wrong.back().region = wrong.front().region;
|
||||
const auto duplicate = read(wrong);
|
||||
require(!duplicate && duplicate.error().kind == CellularSnapshotError::invalid_request,
|
||||
"registered snapshot accepted duplicate resources");
|
||||
refused(
|
||||
load_registered_cellular_census_snapshot(*reader, world + 1000, first_request, builds, 5s),
|
||||
CellularCoverageError::unknown_world);
|
||||
take(release_region_lease(*f.db, {world, 2}, resources[1].lease, 5s),
|
||||
"release registered empty owner");
|
||||
take(f.owner->query("UPDATE evolution.region_leases SET expires_at=clock_timestamp()-"
|
||||
"interval '1 second' WHERE world_id=31201 AND region_id=1",
|
||||
{}, 5s),
|
||||
"expire registered empty owner");
|
||||
const auto inactive = take(read(resources), "released and expired registry coverage");
|
||||
require(inactive.catalog == std::vector<RegisteredCellularRegion>{{1, 1}, {2, 1}},
|
||||
"registry hid released or expired holders");
|
||||
// Genesis recovery has no fenced transition to read; this checks inclusion,
|
||||
// not fresh ownership. No output grants authority to expired/released owners.
|
||||
auto pending_store = take(FilesystemObjectStore::open(environment("PGHOST"), {world, 3},
|
||||
max_cellular_population_bytes),
|
||||
"pending registration objects");
|
||||
const auto pending_state = opening(seed, true, 1, false, 3);
|
||||
take(f.owner->query("BEGIN", {}, 5s), "pending registration transaction");
|
||||
take(provision_cellular_region(*f.owner, *pending_store, {world, 3}, 1, *pending_state, f.id(),
|
||||
f.build, 5s),
|
||||
"pending registration genesis");
|
||||
const auto isolated = read(resources);
|
||||
take(f.owner->query("ROLLBACK", {}, 5s), "pending registration rollback");
|
||||
require(take(isolated, "registered snapshot with pending enrollment").catalog ==
|
||||
inactive.catalog,
|
||||
"uncommitted enrollment entered a registered evidence cut");
|
||||
take(reader->query("BEGIN READ ONLY", {}, 5s), "caller registered transaction");
|
||||
const auto caller = read(resources);
|
||||
take(reader->query("ROLLBACK", {}, 5s), "caller registered rollback");
|
||||
require(!caller && caller.error().kind == CellularSnapshotError::invalid_request,
|
||||
"registered snapshot took over a caller transaction");
|
||||
for (std::uint64_t region = 3; region <= max_cellular_census_regions; ++region)
|
||||
enroll(region);
|
||||
const auto bounded = take(read(resources), "64 registered regions");
|
||||
require(bounded.catalog.size() == max_cellular_census_regions &&
|
||||
bounded.snapshot.cuts.size() == max_cellular_census_regions &&
|
||||
bounded.catalog.back() == RegisteredCellularRegion{64, 1},
|
||||
"registered coverage lost its last in-bound owner");
|
||||
enroll(65);
|
||||
refused(read(std::span{resources}.first(max_cellular_census_regions)),
|
||||
CellularCoverageError::bounds);
|
||||
const auto rows = take(reader->query("SELECT * FROM evolution.load_world_region_catalog_v1("
|
||||
"31201::evolution.u64)",
|
||||
{}, 5s),
|
||||
"catalog overflow sentinel");
|
||||
require(rows.size() == 65, "SQL registry silently truncated the overflow owner");
|
||||
const RegionKey missing_key{world + 1, 1};
|
||||
take(f.owner->query("INSERT INTO evolution.region_leases(world_id,region_id,frame_id) "
|
||||
"VALUES(31202.00,1.0,1.00),(31202,18446744073709551615,1.000)",
|
||||
{}, 5s),
|
||||
"registered row with missing head fixture");
|
||||
auto missing_store = take(FilesystemObjectStore::open(environment("PGHOST"), missing_key,
|
||||
max_cellular_population_bytes),
|
||||
"missing registered checkpoint store");
|
||||
const std::array missing_resources{
|
||||
CellularCensusRegionRequest{1, resources[0].lease, missing_store.get()},
|
||||
CellularCensusRegionRequest{std::numeric_limits<std::uint64_t>::max(), resources[0].lease,
|
||||
missing_store.get()}};
|
||||
const auto missing = load_registered_cellular_census_snapshot(*reader, missing_key.world,
|
||||
missing_resources, builds, 5s);
|
||||
require(!missing && missing.error().kind == CellularSnapshotError::history &&
|
||||
missing.error().region == 1 && reader->transaction_idle(),
|
||||
"registered owner without a checkpoint disappeared from coverage");
|
||||
const RegionKey unsupported_key{world + 2, 1};
|
||||
const auto unsupported_manifest =
|
||||
take(genesis_manifest(unsupported_key, 1, f.id(), 50000, f.build, 1, f.build),
|
||||
"non-cellular registry manifest fixture");
|
||||
const std::array<std::string, 1> manifest_parameter{
|
||||
hexadecimal(take(encode_manifest(unsupported_manifest), "non-cellular fixture encoding"))};
|
||||
take(f.owner->query("SELECT evolution.provision_region_checkpoint($1::bytea)",
|
||||
manifest_parameter, 5s),
|
||||
"non-cellular registered owner");
|
||||
auto unsupported_store =
|
||||
take(FilesystemObjectStore::open(environment("PGHOST"), unsupported_key,
|
||||
max_cellular_population_bytes),
|
||||
"unsupported registered store");
|
||||
const std::array unsupported_resources{
|
||||
CellularCensusRegionRequest{1, resources[0].lease, unsupported_store.get()}};
|
||||
const auto unsupported = load_registered_cellular_census_snapshot(
|
||||
*reader, unsupported_key.world, unsupported_resources, builds, 5s);
|
||||
require(!unsupported && unsupported.error().kind == CellularSnapshotError::history &&
|
||||
unsupported.error().history.recovery.kind == RecoveryError::unsupported &&
|
||||
reader->transaction_idle(),
|
||||
"unsupported registered owner was filtered out or reported as zero");
|
||||
std::puts("registered world coverage, enrollment isolation and 64/65 bounds passed");
|
||||
}
|
||||
void snapshot_transport_faults(Fixture& f) {
|
||||
if (!std::getenv("EVO_MIGRATION_TCP_PORT"))
|
||||
return;
|
||||
const std::array<Sha256Digest, 1> builds{f.build};
|
||||
const std::array requests{CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()}};
|
||||
for (unsigned mode = 0; mode < 4; ++mode) {
|
||||
for (unsigned mode = 0; mode < 8; ++mode) {
|
||||
test::PostgresRelay relay(migration_backend_port());
|
||||
auto db = migration_tls_connection(relay);
|
||||
const bool at_completion = mode >= 2;
|
||||
const bool registered = mode >= 4;
|
||||
const bool at_completion = mode % 4 >= 2;
|
||||
const bool lost_request = mode % 2 == 0;
|
||||
const auto partition = [&] {
|
||||
if (lost_request)
|
||||
@@ -813,10 +959,17 @@ void snapshot_transport_faults(Fixture& f) {
|
||||
if (at_completion)
|
||||
partition();
|
||||
}};
|
||||
const auto lost =
|
||||
load_cellular_census_snapshot(*db, f.key.world, requests, builds, 300ms, hook);
|
||||
require(!lost && lost.error().kind == CellularSnapshotError::transaction &&
|
||||
lost.error().transaction.kind == PgError::unknown && !db->transaction_idle() &&
|
||||
const auto failure = [](auto result) {
|
||||
if (result)
|
||||
throw std::runtime_error("lost capture I/O returned a census");
|
||||
return result.error();
|
||||
};
|
||||
const auto lost = registered ? failure(load_registered_cellular_census_snapshot(
|
||||
*db, f.key.world, requests, builds, 300ms, hook))
|
||||
: failure(load_cellular_census_snapshot(
|
||||
*db, f.key.world, requests, builds, 300ms, hook));
|
||||
require(lost.kind == CellularSnapshotError::transaction &&
|
||||
lost.transaction.kind == PgError::unknown && !db->transaction_idle() &&
|
||||
relay.healthy() &&
|
||||
(lost_request ? relay.dropped_requests() : relay.dropped_responses()) > 0 &&
|
||||
take(load_region_checkpoint(*f.db, f.key, 5s), "head after lost capture I/O")
|
||||
@@ -1351,9 +1504,10 @@ void migration_snapshot_race(Fixture& f, RegionKey destination_key,
|
||||
const auto& dl = reverse ? f.lease : destination_lease;
|
||||
const auto sk = reverse ? destination_key : f.key;
|
||||
const auto dk = reverse ? f.key : destination_key;
|
||||
const auto before =
|
||||
take(load_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s),
|
||||
"before snapshot migration race");
|
||||
const auto before_registered = take(
|
||||
load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s),
|
||||
"before snapshot migration race");
|
||||
const auto& before = before_registered.snapshot;
|
||||
const CellularMigrationRequest request{f.id(),
|
||||
{sk.region, sh.head.commit, f.id(), sh.head.digest},
|
||||
{dk.region, dh.head.commit, f.id(), dh.head.digest},
|
||||
@@ -1381,9 +1535,10 @@ void migration_snapshot_race(Fixture& f, RegionKey destination_key,
|
||||
frozen.trace, f.build, 5s),
|
||||
"publish real pair between snapshot head reads");
|
||||
}};
|
||||
const auto raced =
|
||||
take(load_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s, hook),
|
||||
"coherent census across a concurrent pair");
|
||||
const auto raced_registered = take(load_registered_cellular_census_snapshot(
|
||||
*reader, f.key.world, requests, builds, 5s, hook),
|
||||
"coherent census across a concurrent pair");
|
||||
const auto& raced = raced_registered.snapshot;
|
||||
const auto committed =
|
||||
take(std::move(published), "pair was not published in snapshot hook");
|
||||
sh = committed.source;
|
||||
@@ -1396,9 +1551,17 @@ void migration_snapshot_race(Fixture& f, RegionKey destination_key,
|
||||
std::ranges::equal(raced.census.regions()[i].rows(),
|
||||
before.census.regions()[i].rows()),
|
||||
"atomic pair split a captured cut or changed historical replay");
|
||||
const auto after =
|
||||
take(load_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s),
|
||||
"snapshot after pair publication");
|
||||
const auto after_registered = take(
|
||||
load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s),
|
||||
"snapshot after pair publication");
|
||||
const auto& after = after_registered.snapshot;
|
||||
require(before_registered.catalog == raced_registered.catalog &&
|
||||
after_registered.catalog == before_registered.catalog &&
|
||||
before_registered.catalog ==
|
||||
std::vector<RegisteredCellularRegion>{
|
||||
{f.key.region, f.lease.frame},
|
||||
{destination_key.region, destination_lease.frame}},
|
||||
"atomic migration changed or omitted registered owner coverage");
|
||||
require(after.cuts[0].head == f.head.head && after.cuts[1].head == destination_head.head &&
|
||||
after.cuts[0].head.sequence == before.cuts[0].head.sequence + 1 &&
|
||||
after.cuts[1].head.sequence == before.cuts[1].head.sequence + 1,
|
||||
@@ -2853,6 +3016,7 @@ void run_population_publication_tests() {
|
||||
funded_source(fixture);
|
||||
history_pages(fixture);
|
||||
snapshot_guards(fixture);
|
||||
registered_snapshot_coverage(fixture);
|
||||
conversions_and_retries(fixture);
|
||||
interval_contract(fixture);
|
||||
ambiguous_publication(fixture, false);
|
||||
|
||||
@@ -215,6 +215,7 @@ void upgrade_across_migration_and_barrier() {
|
||||
" -v migration_sha_47=$(s 0047_cellular_migration_freeze.sql)"
|
||||
" -v migration_sha_48=$(s 0048_cellular_migration_staging.sql)"
|
||||
" -v migration_sha_49=$(s 0049_cellular_migration_publication.sql)"
|
||||
" -v migration_sha_50=$(s 0050_world_region_catalog.sql)"
|
||||
" -v runtime_role=evo_runtime_test -f migrations/apply.sql"
|
||||
" > /dev/null 2>&1") != 0);
|
||||
CHECK(history(*owner_db) == "1,2,3,4,5,6,7");
|
||||
@@ -232,7 +233,7 @@ void upgrade_across_migration_and_barrier() {
|
||||
"tools/migrate-postgres.sh\" > /dev/null") == 0);
|
||||
CHECK(history(*owner_db) ==
|
||||
"1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,"
|
||||
"25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49");
|
||||
"25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50");
|
||||
CHECK(scalar(*owner_db, "SELECT (c.condeferrable AND c.condeferred AND "
|
||||
"c.confrelid = 'evolution.region_commits'::regclass)::text "
|
||||
"FROM pg_catalog.pg_constraint c WHERE c.conrelid = "
|
||||
|
||||
Reference in New Issue
Block a user