M12.7: reconcile pending migration evidence in census snapshots
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m26s
Foundation / checks (network) (push) Successful in 21m1s
Foundation / checks (client) (push) Successful in 22m9s
Foundation / checks (tsan) (push) Successful in 24m36s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (postgres) (push) Canceled after 33m38s
Foundation / checks (checks) (push) Canceled after 33m49s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m26s
Foundation / checks (network) (push) Successful in 21m1s
Foundation / checks (client) (push) Successful in 22m9s
Foundation / checks (tsan) (push) Successful in 24m36s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (postgres) (push) Canceled after 33m38s
Foundation / checks (checks) (push) Canceled after 33m49s
This commit is contained in:
@@ -1545,13 +1545,133 @@ void migration_snapshot_race(Fixture& f, RegionKey destination_key,
|
||||
load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s),
|
||||
"before snapshot migration race");
|
||||
const auto& before = before_registered.snapshot;
|
||||
const CellularMigrationRequest request{f.id(),
|
||||
{sk.region, sh.head.commit, f.id(), sh.head.digest},
|
||||
{dk.region, dh.head.commit, f.id(), dh.head.digest},
|
||||
genome};
|
||||
const auto frozen =
|
||||
take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s),
|
||||
"snapshot race source freeze");
|
||||
CellularMigrationRequest request{f.id(),
|
||||
{sk.region, sh.head.commit, f.id(), sh.head.digest},
|
||||
{dk.region, dh.head.commit, f.id(), dh.head.digest},
|
||||
genome};
|
||||
const auto read = [&] {
|
||||
return load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds,
|
||||
5s);
|
||||
};
|
||||
const auto unchanged_stock = [&](const auto& snapshot) {
|
||||
require(take(snapshot.snapshot.census.query(genome, CellularCensusScope::descendants),
|
||||
"pending lineage") ==
|
||||
take(before.census.query(genome, CellularCensusScope::descendants),
|
||||
"before pending lineage"),
|
||||
"pending evidence added stock");
|
||||
};
|
||||
const auto refused = [&](const auto& result, RecoveryError error) {
|
||||
require(!result && result.error().kind == CellularSnapshotError::migration &&
|
||||
result.error().migration.kind == error && reader->transaction_idle(),
|
||||
"bad pending evidence returned a partial census or retained a transaction");
|
||||
};
|
||||
std::optional<CellularMigrationFreeze> preparation;
|
||||
const CellularCensusSnapshotHooks prepare_hook{[&](std::size_t i) {
|
||||
if (i == 0)
|
||||
preparation =
|
||||
take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s),
|
||||
"capture prepare");
|
||||
}};
|
||||
const auto before_prepare =
|
||||
take(load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds,
|
||||
5s, prepare_hook),
|
||||
"before concurrent prepare");
|
||||
auto frozen = take(std::move(preparation), "preparation hook ran");
|
||||
require(before_registered.migrations.empty() && before_prepare.migrations.empty(),
|
||||
"concurrent preparation escaped its metadata view");
|
||||
const auto prepared = take(read(), "prepared snapshot inventory");
|
||||
require(prepared.migrations.size() == 1 && !prepared.migrations[0].staged &&
|
||||
prepared.migrations[0].trace == frozen.trace,
|
||||
"source-only inventory missing or treated as staged");
|
||||
unchanged_stock(prepared);
|
||||
const auto unretained = load_registered_cellular_census_snapshot(
|
||||
*reader, f.key.world, requests, std::span<const Sha256Digest>{}, 5s);
|
||||
refused(unretained, RecoveryError::unsupported);
|
||||
const auto dm = take(decode_manifest(dh.manifest), "pending destination capsule");
|
||||
{
|
||||
UnreadableObject denied(so.directory() / hexadecimal(dm.payload_sha256, false));
|
||||
refused(read(), RecoveryError::tail);
|
||||
}
|
||||
// Trusted owner fault seam: a pointer must not hide a foreign endpoint.
|
||||
const std::array<std::string, 2> pointer{std::to_string(f.key.world),
|
||||
std::to_string(sk.region)};
|
||||
take(f.owner->query("UPDATE evolution.cellular_migration_pending SET region_id=99999 "
|
||||
"WHERE world_id=$1::evolution.u64 AND region_id=$2::evolution.u64",
|
||||
pointer, 5s),
|
||||
"malformed pending pointer seam");
|
||||
const auto malformed = read();
|
||||
take(f.owner->query(
|
||||
"UPDATE evolution.cellular_migration_pending SET region_id=$2::evolution.u64 "
|
||||
"WHERE world_id=$1::evolution.u64 AND region_id=99999",
|
||||
pointer, 5s),
|
||||
"restore pending pointer seam");
|
||||
refused(malformed, RecoveryError::history);
|
||||
if (!reverse) {
|
||||
const auto definition =
|
||||
take(f.owner->query("SELECT "
|
||||
"pg_get_functiondef('evolution.load_world_cellular_migrations_"
|
||||
"v1(evolution.u64)'::regprocedure)",
|
||||
{}, 5s),
|
||||
"inventory definition");
|
||||
const auto original = take(definition.at(0).at(0), "inventory definition value");
|
||||
take(
|
||||
f.owner->query(
|
||||
"CREATE OR REPLACE FUNCTION evolution.load_world_cellular_migrations_v1(w "
|
||||
"evolution.u64) "
|
||||
"RETURNS TABLE(trace bytea,source_manifest bytea,destination_manifest bytea,"
|
||||
"ghost_sha256 bytea,ghost_bytes evolution.u64,coherent boolean) "
|
||||
"LANGUAGE sql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ "
|
||||
"SELECT "
|
||||
"NULL::bytea,NULL::bytea,NULL::bytea,NULL::bytea,NULL::evolution.u64,true "
|
||||
"FROM pg_catalog.generate_series(1,65) $$",
|
||||
{}, 5s),
|
||||
"overflow response seam");
|
||||
const auto overflow = read();
|
||||
take(f.owner->query(original, {}, 5s), "restore bounded inventory function");
|
||||
refused(overflow, RecoveryError::store);
|
||||
require(overflow.error().migration.store.cause.kind == PgError::bounds,
|
||||
"65th response was not rejected before parsing");
|
||||
}
|
||||
std::optional<CellularMigrationStage> stage;
|
||||
const CellularCensusSnapshotHooks stage_hook{[&](std::size_t i) {
|
||||
if (i == 0)
|
||||
stage = take(stage_cellular_migration(*f.db, so, dst, dl, sh, dh, frozen.trace, 5s),
|
||||
"capture stage");
|
||||
}};
|
||||
const auto before_stage = take(load_registered_cellular_census_snapshot(
|
||||
*reader, f.key.world, requests, builds, 5s, stage_hook),
|
||||
"before concurrent stage");
|
||||
const auto staged = take(read(), "staged snapshot inventory");
|
||||
require(before_stage.migrations.size() == 1 && !before_stage.migrations[0].staged &&
|
||||
staged.migrations.size() == 1 && staged.migrations[0].staged,
|
||||
"concurrent stage escaped the capture view");
|
||||
unchanged_stock(staged);
|
||||
const auto ghost = take(std::move(stage), "stage hook ran").ghost;
|
||||
const auto ghost_path = dst.directory() / hexadecimal(ghost.sha256, false);
|
||||
{
|
||||
UnreadableObject denied(ghost_path);
|
||||
refused(read(), RecoveryError::tail);
|
||||
}
|
||||
const CellularCensusSnapshotHooks abort_hook{[&](std::size_t i) {
|
||||
if (i == 0)
|
||||
take(abort_cellular_migration_freeze(*f.db, sk, sl, frozen.trace.request.request,
|
||||
5s),
|
||||
"capture abort");
|
||||
}};
|
||||
const auto before_abort = take(load_registered_cellular_census_snapshot(
|
||||
*reader, f.key.world, requests, builds, 5s, abort_hook),
|
||||
"before concurrent abort");
|
||||
const auto after_abort = take(read(), "after abort inventory");
|
||||
require(before_abort.migrations.size() == 1 && before_abort.migrations[0].staged &&
|
||||
after_abort.migrations.empty(),
|
||||
"abort escaped the selected inventory view");
|
||||
unchanged_stock(before_abort);
|
||||
unchanged_stock(after_abort);
|
||||
request.request = f.id();
|
||||
request.source.commit = f.id();
|
||||
request.destination.commit = f.id();
|
||||
frozen = take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s),
|
||||
"snapshot race source freeze");
|
||||
take(stage_cellular_migration(*f.db, so, dst, dl, sh, dh, frozen.trace, 5s),
|
||||
"snapshot race destination stage");
|
||||
std::optional<CellularMigrationCommitted> published;
|
||||
@@ -1592,6 +1712,10 @@ void migration_snapshot_race(Fixture& f, RegionKey destination_key,
|
||||
load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s),
|
||||
"snapshot after pair publication");
|
||||
const auto& after = after_registered.snapshot;
|
||||
require(raced_registered.migrations.size() == 1 && raced_registered.migrations[0].staged &&
|
||||
raced_registered.migrations[0].trace == frozen.trace &&
|
||||
after_registered.migrations.empty(),
|
||||
"commit escaped the captured inventory or retained settled pointers");
|
||||
require(before_registered.catalog == raced_registered.catalog &&
|
||||
after_registered.catalog == before_registered.catalog &&
|
||||
before_registered.catalog ==
|
||||
@@ -2620,6 +2744,19 @@ void migration_freezes(Fixture& f) {
|
||||
load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s);
|
||||
require(!false_replay && false_replay.error().kind == RecoveryError::history,
|
||||
"freeze recovery trusted declared material without complete biological replay");
|
||||
const auto refuse_false_inventory = [&] {
|
||||
const std::array resources{
|
||||
CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()},
|
||||
CellularCensusRegionRequest{destination_key.region, destination_lease,
|
||||
destination_objects.get()}};
|
||||
const auto result =
|
||||
load_registered_cellular_census_snapshot(*f.db, f.key.world, resources, builds, 5s);
|
||||
require(!result && result.error().kind == CellularSnapshotError::migration &&
|
||||
result.error().migration.kind == RecoveryError::history &&
|
||||
f.db->transaction_idle(),
|
||||
"registered census accepted semantically false pending biology");
|
||||
};
|
||||
refuse_false_inventory();
|
||||
take(
|
||||
abort_cellular_migration_freeze(*f.db, f.key, f.lease, false_trace.request.request, 5s),
|
||||
"explicitly abort false preparation without moving stock");
|
||||
@@ -2658,6 +2795,7 @@ void migration_freezes(Fixture& f) {
|
||||
*f.db, *destination_objects, destination_key, destination_lease, 5s);
|
||||
require(!false_ghost_replay && false_ghost_replay.error().kind == RecoveryError::history,
|
||||
"stage recovery accepted a real object with the wrong complete biology");
|
||||
refuse_false_inventory();
|
||||
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
|
||||
false_stage_freeze.trace.request.request, 5s),
|
||||
"abort false ghost evidence");
|
||||
|
||||
@@ -217,6 +217,7 @@ void upgrade_across_migration_and_barrier() {
|
||||
" -v migration_sha_49=$(s 0049_cellular_migration_publication.sql)"
|
||||
" -v migration_sha_50=$(s 0050_world_region_catalog.sql)"
|
||||
" -v migration_sha_51=$(s 0051_database_history_domain.sql)"
|
||||
" -v migration_sha_52=$(s 0052_census_pending_migrations.sql)"
|
||||
" -v runtime_role=evo_runtime_test -f migrations/apply.sql"
|
||||
" > /dev/null 2>&1") != 0);
|
||||
CHECK(history(*owner_db) == "1,2,3,4,5,6,7");
|
||||
@@ -234,7 +235,7 @@ void upgrade_across_migration_and_barrier() {
|
||||
"tools/migrate-postgres.sh\" > /dev/null") == 0);
|
||||
CHECK(history(*owner_db) ==
|
||||
"1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,"
|
||||
"25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51");
|
||||
"25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52");
|
||||
CHECK(scalar(*owner_db, "SELECT (c.condeferrable AND c.condeferred AND "
|
||||
"c.confrelid = 'evolution.region_commits'::regclass)::text "
|
||||
"FROM pg_catalog.pg_constraint c WHERE c.conrelid = "
|
||||
|
||||
Reference in New Issue
Block a user