M12.4: qualify migration storage failures and exact recovery
Foundation / checks (client) (push) Successful in 5s
Foundation / checks (network) (push) Successful in 5s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m12s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (postgres) (push) Canceled after 19m42s
Foundation / checks (checks) (push) Canceled after 19m48s
Foundation / checks (tsan) (push) Canceled after 19m49s

This commit is contained in:
dingo
2026-10-18 13:38:05 -04:00
parent e0a7c13f6b
commit df680810af
9 changed files with 313 additions and 4 deletions
+141
View File
@@ -98,6 +98,25 @@ struct Fixture {
head.head.end_tick + count};
}
};
// Deny actual reads by the unprivileged fixture user, then restore permissions
// even if a runtime requirement throws. This never changes immutable bytes.
class UnreadableObject {
public:
explicit UnreadableObject(std::filesystem::path path)
: path_(std::move(path)), permissions_(std::filesystem::status(path_).permissions()) {
std::filesystem::permissions(path_, std::filesystem::perms::none);
}
~UnreadableObject() {
std::error_code error;
std::filesystem::permissions(path_, permissions_, error);
}
UnreadableObject(const UnreadableObject&) = delete;
UnreadableObject& operator=(const UnreadableObject&) = delete;
private:
std::filesystem::path path_;
std::filesystem::perms permissions_;
};
std::uint16_t migration_backend_port() {
const auto port = environment("EVO_MIGRATION_TCP_PORT");
unsigned value{};
@@ -1072,6 +1091,25 @@ void migration_publication(Fixture& f, const RegionKey destination_key,
{sk.region, sh.head.commit, f.id(), sh.head.digest},
{dk.region, dh.head.commit, f.id(), dh.head.digest},
genome};
ObjectStoreFaults source_faults, destination_faults;
auto failing_source =
take(FilesystemObjectStore::open(environment("PGHOST"), sk,
max_cellular_population_bytes, &source_faults),
"migration source sync fault store");
auto failing_destination = take(
FilesystemObjectStore::open(environment("PGHOST"), dk,
max_cellular_population_bytes, &destination_faults),
"migration destination sync fault store");
source_faults.fail_directory_syncs = 1;
const auto unsynced_preparation = prepare_cellular_migration_freeze(
*f.db, *failing_source, dst, sl, sh, dh, request, 5s);
require(!unsynced_preparation &&
unsynced_preparation.error().kind == PublicationError::upload &&
unsynced_preparation.error().object == ObjectError::io &&
source_faults.fail_directory_syncs == 0 &&
!take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s),
"unsynced preparation pointer"),
"failed capsule sync durably prepared a migration");
take(source_host->prepare_migration_freeze(dh, dst, request),
"owning paired source freeze");
const auto frozen = take(take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s),
@@ -1081,7 +1119,41 @@ void migration_publication(Fixture& f, const RegionKey destination_key,
require(!publish_cellular_migration(*f.db, so, dst, sl, dl, sh, dh, frozen.trace,
f.build, 5s),
"unstaged migration published");
destination_faults.fail_directory_syncs = 1;
const auto unsynced_stage = stage_cellular_migration(*f.db, so, *failing_destination,
dl, sh, dh, frozen.trace, 5s);
require(!unsynced_stage && unsynced_stage.error().kind == PublicationError::upload &&
unsynced_stage.error().object == ObjectError::io &&
destination_faults.fail_directory_syncs == 0 &&
!take(load_cellular_migration_stage(*f.db, dst, dk, dl, 5s),
"unsynced destination stage pointer") &&
take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s),
"source freeze after stage sync failure")
.has_value(),
"failed stage capsule sync activated a ghost or released source ownership");
take(destination_host->stage_migration(sh, so, frozen.trace), "owning paired stage");
for (unsigned endpoint = 0; endpoint < 2; ++endpoint) {
auto& faults = endpoint == 0 ? source_faults : destination_faults;
faults.fail_directory_syncs = 1;
const auto unsynced_pair =
publish_cellular_migration(*f.db, *failing_source, *failing_destination, sl, dl,
sh, dh, frozen.trace, f.build, 5s);
require(
!unsynced_pair && unsynced_pair.error().kind == PublicationError::upload &&
unsynced_pair.error().object == ObjectError::io &&
faults.fail_directory_syncs == 0 &&
take(load_region_checkpoint(*f.db, sk, 5s), "unsynced source head").head ==
source_before.head &&
take(load_region_checkpoint(*f.db, dk, 5s), "unsynced destination head")
.head == destination_before.head &&
take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s),
"unsynced source freeze")
.has_value() &&
take(load_cellular_migration_stage(*f.db, dst, dk, dl, 5s),
"unsynced destination stage")
.has_value(),
"failed result sync published a half or released retained work");
}
std::vector<std::uint8_t> expected_source, expected_destination;
if (representation == 0) {
auto a = take(restore_cellular_payload(sb, s), "paired raw source oracle");
@@ -1234,6 +1306,63 @@ void migration_publication(Fixture& f, const RegionKey destination_key,
require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict &&
!retry_cellular_migration_commit(*f.db, stale, dl, prepared, 5s),
"changed pair or stale source authority was accepted");
const auto sm = take(decode_manifest(sh.manifest), "unavailable source result");
const auto dm = take(decode_manifest(dh.manifest), "unavailable destination result");
// Outbound: deny current results. Return: deny retained counterpart
// capsules. Both are required before either paused host may resume.
const VerifiedObject unavailable_source =
direction == 0 ? VerifiedObject{sm.payload_sha256, sm.payload_bytes}
: VerifiedObject{d.payload_sha256, d.payload_bytes};
const VerifiedObject unavailable_destination =
direction == 0 ? VerifiedObject{dm.payload_sha256, dm.payload_bytes}
: VerifiedObject{s.payload_sha256, s.payload_bytes};
{
UnreadableObject source_denial(so.directory() /
hexadecimal(unavailable_source.sha256, false));
UnreadableObject destination_denial(
dst.directory() / hexadecimal(unavailable_destination.sha256, false));
const auto denied_source = so.get(unavailable_source);
const auto denied_destination = dst.get(unavailable_destination);
require(!denied_source && denied_source.error() == ObjectError::io &&
!denied_destination && denied_destination.error() == ObjectError::io,
"object read denial did not produce actual I/O failures");
const auto source_failures = source_host->status().store_failures;
const auto destination_failures = destination_host->status().store_failures;
const auto observed = [&] {
if (direction == 0)
return source_host->status().store_failures > source_failures &&
destination_host->status().store_failures > destination_failures;
return source_host->status().phase == HostPhase::quarantined &&
destination_host->status().phase == HostPhase::quarantined;
};
for (unsigned attempt = 0; attempt < 200 && !observed(); ++attempt) {
source_host->step();
destination_host->step();
std::this_thread::sleep_for(10ms);
}
require(
observed() && !source_host->status().ready &&
!destination_host->status().ready && !source_host->advance_unattended(1) &&
!destination_host->advance_unattended(1) &&
take(load_region_checkpoint(*f.db, sk, 5s), "unavailable source head")
.head == sh.head &&
take(load_region_checkpoint(*f.db, dk, 5s), "unavailable destination head")
.head == dh.head,
"unreadable migration evidence resumed old ownership or changed a head");
}
if (direction != 0) {
require(!source_host->step().ready && !destination_host->step().ready,
"restoring storage silently cleared quarantine");
source_host->drain();
destination_host->drain();
// The quarantined local authorities cannot release themselves. This
// test coordinator explicitly retires their still-current DB leases.
take(release_region_lease(*f.db, sk, sl, 5s), "retire quarantined source lease");
take(release_region_lease(*f.db, dk, dl, 5s),
"retire quarantined destination lease");
source_host = make_host(sk);
destination_host = make_host(dk);
}
for (unsigned attempt = 0;
attempt < 200 && (!source_host->step().ready || !destination_host->step().ready);
++attempt)
@@ -1242,6 +1371,17 @@ void migration_publication(Fixture& f, const RegionKey destination_key,
take(source_host->committed(), "adopted source").head == sh.head &&
take(destination_host->committed(), "adopted destination").head == dh.head,
"pointer release resumed a stale owner instead of recovering the pair");
const auto old_source_epoch = sl.epoch, old_destination_epoch = dl.epoch;
sl = take(source_host->authority(), "restored source object authority");
dl = take(destination_host->authority(), "restored destination object authority");
const bool expected_epochs =
direction == 0 ? sl.epoch == old_source_epoch && dl.epoch == old_destination_epoch
: sl.epoch > old_source_epoch && dl.epoch > old_destination_epoch;
require(expected_epochs &&
take(retry_cellular_migration_commit(*f.db, sl, dl, prepared, 5s),
"restored storage exact paired receipt")
.source.head == sh.head,
"storage recovery changed a committed pair or reused authority");
const auto check_bytes = [&](const CommittedCheckpoint& h, RegionKey key,
const FilesystemObjectStore& objects,
const LeaseGrant& lease) {
@@ -1290,6 +1430,7 @@ void migration_publication(Fixture& f, const RegionKey destination_key,
migration_owner_partitions(f, destination_key, destination_objects, destination_lease,
destination_head, genome);
}
std::puts("migration object sync/read failures, quarantine and exact recovery passed");
// Race the actual two-head transaction against abort on independent connections.
const auto prepare_pair = [&](bool reverse) {
auto& sh = reverse ? destination_head : f.head;