M12.7: reconcile pending migration evidence in census snapshots
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m26s
Foundation / checks (network) (push) Successful in 21m1s
Foundation / checks (client) (push) Successful in 22m9s
Foundation / checks (tsan) (push) Successful in 24m36s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (postgres) (push) Canceled after 33m38s
Foundation / checks (checks) (push) Canceled after 33m49s

This commit is contained in:
dingo
2026-10-18 21:14:53 -04:00
parent 709dfb9557
commit 27ec5729bf
23 changed files with 592 additions and 23 deletions
+145 -7
View File
@@ -1545,13 +1545,133 @@ void migration_snapshot_race(Fixture& f, RegionKey destination_key,
load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s),
"before snapshot migration race");
const auto& before = before_registered.snapshot;
const CellularMigrationRequest request{f.id(),
{sk.region, sh.head.commit, f.id(), sh.head.digest},
{dk.region, dh.head.commit, f.id(), dh.head.digest},
genome};
const auto frozen =
take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s),
"snapshot race source freeze");
CellularMigrationRequest request{f.id(),
{sk.region, sh.head.commit, f.id(), sh.head.digest},
{dk.region, dh.head.commit, f.id(), dh.head.digest},
genome};
const auto read = [&] {
return load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds,
5s);
};
const auto unchanged_stock = [&](const auto& snapshot) {
require(take(snapshot.snapshot.census.query(genome, CellularCensusScope::descendants),
"pending lineage") ==
take(before.census.query(genome, CellularCensusScope::descendants),
"before pending lineage"),
"pending evidence added stock");
};
const auto refused = [&](const auto& result, RecoveryError error) {
require(!result && result.error().kind == CellularSnapshotError::migration &&
result.error().migration.kind == error && reader->transaction_idle(),
"bad pending evidence returned a partial census or retained a transaction");
};
std::optional<CellularMigrationFreeze> preparation;
const CellularCensusSnapshotHooks prepare_hook{[&](std::size_t i) {
if (i == 0)
preparation =
take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s),
"capture prepare");
}};
const auto before_prepare =
take(load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds,
5s, prepare_hook),
"before concurrent prepare");
auto frozen = take(std::move(preparation), "preparation hook ran");
require(before_registered.migrations.empty() && before_prepare.migrations.empty(),
"concurrent preparation escaped its metadata view");
const auto prepared = take(read(), "prepared snapshot inventory");
require(prepared.migrations.size() == 1 && !prepared.migrations[0].staged &&
prepared.migrations[0].trace == frozen.trace,
"source-only inventory missing or treated as staged");
unchanged_stock(prepared);
const auto unretained = load_registered_cellular_census_snapshot(
*reader, f.key.world, requests, std::span<const Sha256Digest>{}, 5s);
refused(unretained, RecoveryError::unsupported);
const auto dm = take(decode_manifest(dh.manifest), "pending destination capsule");
{
UnreadableObject denied(so.directory() / hexadecimal(dm.payload_sha256, false));
refused(read(), RecoveryError::tail);
}
// Trusted owner fault seam: a pointer must not hide a foreign endpoint.
const std::array<std::string, 2> pointer{std::to_string(f.key.world),
std::to_string(sk.region)};
take(f.owner->query("UPDATE evolution.cellular_migration_pending SET region_id=99999 "
"WHERE world_id=$1::evolution.u64 AND region_id=$2::evolution.u64",
pointer, 5s),
"malformed pending pointer seam");
const auto malformed = read();
take(f.owner->query(
"UPDATE evolution.cellular_migration_pending SET region_id=$2::evolution.u64 "
"WHERE world_id=$1::evolution.u64 AND region_id=99999",
pointer, 5s),
"restore pending pointer seam");
refused(malformed, RecoveryError::history);
if (!reverse) {
const auto definition =
take(f.owner->query("SELECT "
"pg_get_functiondef('evolution.load_world_cellular_migrations_"
"v1(evolution.u64)'::regprocedure)",
{}, 5s),
"inventory definition");
const auto original = take(definition.at(0).at(0), "inventory definition value");
take(
f.owner->query(
"CREATE OR REPLACE FUNCTION evolution.load_world_cellular_migrations_v1(w "
"evolution.u64) "
"RETURNS TABLE(trace bytea,source_manifest bytea,destination_manifest bytea,"
"ghost_sha256 bytea,ghost_bytes evolution.u64,coherent boolean) "
"LANGUAGE sql STABLE SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$ "
"SELECT "
"NULL::bytea,NULL::bytea,NULL::bytea,NULL::bytea,NULL::evolution.u64,true "
"FROM pg_catalog.generate_series(1,65) $$",
{}, 5s),
"overflow response seam");
const auto overflow = read();
take(f.owner->query(original, {}, 5s), "restore bounded inventory function");
refused(overflow, RecoveryError::store);
require(overflow.error().migration.store.cause.kind == PgError::bounds,
"65th response was not rejected before parsing");
}
std::optional<CellularMigrationStage> stage;
const CellularCensusSnapshotHooks stage_hook{[&](std::size_t i) {
if (i == 0)
stage = take(stage_cellular_migration(*f.db, so, dst, dl, sh, dh, frozen.trace, 5s),
"capture stage");
}};
const auto before_stage = take(load_registered_cellular_census_snapshot(
*reader, f.key.world, requests, builds, 5s, stage_hook),
"before concurrent stage");
const auto staged = take(read(), "staged snapshot inventory");
require(before_stage.migrations.size() == 1 && !before_stage.migrations[0].staged &&
staged.migrations.size() == 1 && staged.migrations[0].staged,
"concurrent stage escaped the capture view");
unchanged_stock(staged);
const auto ghost = take(std::move(stage), "stage hook ran").ghost;
const auto ghost_path = dst.directory() / hexadecimal(ghost.sha256, false);
{
UnreadableObject denied(ghost_path);
refused(read(), RecoveryError::tail);
}
const CellularCensusSnapshotHooks abort_hook{[&](std::size_t i) {
if (i == 0)
take(abort_cellular_migration_freeze(*f.db, sk, sl, frozen.trace.request.request,
5s),
"capture abort");
}};
const auto before_abort = take(load_registered_cellular_census_snapshot(
*reader, f.key.world, requests, builds, 5s, abort_hook),
"before concurrent abort");
const auto after_abort = take(read(), "after abort inventory");
require(before_abort.migrations.size() == 1 && before_abort.migrations[0].staged &&
after_abort.migrations.empty(),
"abort escaped the selected inventory view");
unchanged_stock(before_abort);
unchanged_stock(after_abort);
request.request = f.id();
request.source.commit = f.id();
request.destination.commit = f.id();
frozen = take(prepare_cellular_migration_freeze(*f.db, so, dst, sl, sh, dh, request, 5s),
"snapshot race source freeze");
take(stage_cellular_migration(*f.db, so, dst, dl, sh, dh, frozen.trace, 5s),
"snapshot race destination stage");
std::optional<CellularMigrationCommitted> published;
@@ -1592,6 +1712,10 @@ void migration_snapshot_race(Fixture& f, RegionKey destination_key,
load_registered_cellular_census_snapshot(*reader, f.key.world, requests, builds, 5s),
"snapshot after pair publication");
const auto& after = after_registered.snapshot;
require(raced_registered.migrations.size() == 1 && raced_registered.migrations[0].staged &&
raced_registered.migrations[0].trace == frozen.trace &&
after_registered.migrations.empty(),
"commit escaped the captured inventory or retained settled pointers");
require(before_registered.catalog == raced_registered.catalog &&
after_registered.catalog == before_registered.catalog &&
before_registered.catalog ==
@@ -2620,6 +2744,19 @@ void migration_freezes(Fixture& f) {
load_cellular_migration_freeze(*f.db, *f.objects, f.key, f.lease, 5s);
require(!false_replay && false_replay.error().kind == RecoveryError::history,
"freeze recovery trusted declared material without complete biological replay");
const auto refuse_false_inventory = [&] {
const std::array resources{
CellularCensusRegionRequest{f.key.region, f.lease, f.objects.get()},
CellularCensusRegionRequest{destination_key.region, destination_lease,
destination_objects.get()}};
const auto result =
load_registered_cellular_census_snapshot(*f.db, f.key.world, resources, builds, 5s);
require(!result && result.error().kind == CellularSnapshotError::migration &&
result.error().migration.kind == RecoveryError::history &&
f.db->transaction_idle(),
"registered census accepted semantically false pending biology");
};
refuse_false_inventory();
take(
abort_cellular_migration_freeze(*f.db, f.key, f.lease, false_trace.request.request, 5s),
"explicitly abort false preparation without moving stock");
@@ -2658,6 +2795,7 @@ void migration_freezes(Fixture& f) {
*f.db, *destination_objects, destination_key, destination_lease, 5s);
require(!false_ghost_replay && false_ghost_replay.error().kind == RecoveryError::history,
"stage recovery accepted a real object with the wrong complete biology");
refuse_false_inventory();
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
false_stage_freeze.trace.request.request, 5s),
"abort false ghost evidence");
+2 -1
View File
@@ -217,6 +217,7 @@ void upgrade_across_migration_and_barrier() {
" -v migration_sha_49=$(s 0049_cellular_migration_publication.sql)"
" -v migration_sha_50=$(s 0050_world_region_catalog.sql)"
" -v migration_sha_51=$(s 0051_database_history_domain.sql)"
" -v migration_sha_52=$(s 0052_census_pending_migrations.sql)"
" -v runtime_role=evo_runtime_test -f migrations/apply.sql"
" > /dev/null 2>&1") != 0);
CHECK(history(*owner_db) == "1,2,3,4,5,6,7");
@@ -234,7 +235,7 @@ void upgrade_across_migration_and_barrier() {
"tools/migrate-postgres.sh\" > /dev/null") == 0);
CHECK(history(*owner_db) ==
"1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,"
"25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51");
"25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52");
CHECK(scalar(*owner_db, "SELECT (c.condeferrable AND c.condeferred AND "
"c.confrelid = 'evolution.region_commits'::regclass)::text "
"FROM pg_catalog.pg_constraint c WHERE c.conrelid = "