M12.4: qualify migration storage failures and exact recovery
Foundation / checks (client) (push) Successful in 5s
Foundation / checks (network) (push) Successful in 5s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m12s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (postgres) (push) Canceled after 19m42s
Foundation / checks (checks) (push) Canceled after 19m48s
Foundation / checks (tsan) (push) Canceled after 19m49s
Foundation / checks (client) (push) Successful in 5s
Foundation / checks (network) (push) Successful in 5s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m12s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (postgres) (push) Canceled after 19m42s
Foundation / checks (checks) (push) Canceled after 19m48s
Foundation / checks (tsan) (push) Canceled after 19m49s
This commit is contained in:
@@ -98,6 +98,25 @@ struct Fixture {
|
||||
head.head.end_tick + count};
|
||||
}
|
||||
};
|
||||
// Deny actual reads by the unprivileged fixture user, then restore permissions
|
||||
// even if a runtime requirement throws. This never changes immutable bytes.
|
||||
class UnreadableObject {
|
||||
public:
|
||||
explicit UnreadableObject(std::filesystem::path path)
|
||||
: path_(std::move(path)), permissions_(std::filesystem::status(path_).permissions()) {
|
||||
std::filesystem::permissions(path_, std::filesystem::perms::none);
|
||||
}
|
||||
~UnreadableObject() {
|
||||
std::error_code error;
|
||||
std::filesystem::permissions(path_, permissions_, error);
|
||||
}
|
||||
UnreadableObject(const UnreadableObject&) = delete;
|
||||
UnreadableObject& operator=(const UnreadableObject&) = delete;
|
||||
|
||||
private:
|
||||
std::filesystem::path path_;
|
||||
std::filesystem::perms permissions_;
|
||||
};
|
||||
std::uint16_t migration_backend_port() {
|
||||
const auto port = environment("EVO_MIGRATION_TCP_PORT");
|
||||
unsigned value{};
|
||||
@@ -1072,6 +1091,25 @@ void migration_publication(Fixture& f, const RegionKey destination_key,
|
||||
{sk.region, sh.head.commit, f.id(), sh.head.digest},
|
||||
{dk.region, dh.head.commit, f.id(), dh.head.digest},
|
||||
genome};
|
||||
ObjectStoreFaults source_faults, destination_faults;
|
||||
auto failing_source =
|
||||
take(FilesystemObjectStore::open(environment("PGHOST"), sk,
|
||||
max_cellular_population_bytes, &source_faults),
|
||||
"migration source sync fault store");
|
||||
auto failing_destination = take(
|
||||
FilesystemObjectStore::open(environment("PGHOST"), dk,
|
||||
max_cellular_population_bytes, &destination_faults),
|
||||
"migration destination sync fault store");
|
||||
source_faults.fail_directory_syncs = 1;
|
||||
const auto unsynced_preparation = prepare_cellular_migration_freeze(
|
||||
*f.db, *failing_source, dst, sl, sh, dh, request, 5s);
|
||||
require(!unsynced_preparation &&
|
||||
unsynced_preparation.error().kind == PublicationError::upload &&
|
||||
unsynced_preparation.error().object == ObjectError::io &&
|
||||
source_faults.fail_directory_syncs == 0 &&
|
||||
!take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s),
|
||||
"unsynced preparation pointer"),
|
||||
"failed capsule sync durably prepared a migration");
|
||||
take(source_host->prepare_migration_freeze(dh, dst, request),
|
||||
"owning paired source freeze");
|
||||
const auto frozen = take(take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s),
|
||||
@@ -1081,7 +1119,41 @@ void migration_publication(Fixture& f, const RegionKey destination_key,
|
||||
require(!publish_cellular_migration(*f.db, so, dst, sl, dl, sh, dh, frozen.trace,
|
||||
f.build, 5s),
|
||||
"unstaged migration published");
|
||||
destination_faults.fail_directory_syncs = 1;
|
||||
const auto unsynced_stage = stage_cellular_migration(*f.db, so, *failing_destination,
|
||||
dl, sh, dh, frozen.trace, 5s);
|
||||
require(!unsynced_stage && unsynced_stage.error().kind == PublicationError::upload &&
|
||||
unsynced_stage.error().object == ObjectError::io &&
|
||||
destination_faults.fail_directory_syncs == 0 &&
|
||||
!take(load_cellular_migration_stage(*f.db, dst, dk, dl, 5s),
|
||||
"unsynced destination stage pointer") &&
|
||||
take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s),
|
||||
"source freeze after stage sync failure")
|
||||
.has_value(),
|
||||
"failed stage capsule sync activated a ghost or released source ownership");
|
||||
take(destination_host->stage_migration(sh, so, frozen.trace), "owning paired stage");
|
||||
for (unsigned endpoint = 0; endpoint < 2; ++endpoint) {
|
||||
auto& faults = endpoint == 0 ? source_faults : destination_faults;
|
||||
faults.fail_directory_syncs = 1;
|
||||
const auto unsynced_pair =
|
||||
publish_cellular_migration(*f.db, *failing_source, *failing_destination, sl, dl,
|
||||
sh, dh, frozen.trace, f.build, 5s);
|
||||
require(
|
||||
!unsynced_pair && unsynced_pair.error().kind == PublicationError::upload &&
|
||||
unsynced_pair.error().object == ObjectError::io &&
|
||||
faults.fail_directory_syncs == 0 &&
|
||||
take(load_region_checkpoint(*f.db, sk, 5s), "unsynced source head").head ==
|
||||
source_before.head &&
|
||||
take(load_region_checkpoint(*f.db, dk, 5s), "unsynced destination head")
|
||||
.head == destination_before.head &&
|
||||
take(load_cellular_migration_freeze(*f.db, so, sk, sl, 5s),
|
||||
"unsynced source freeze")
|
||||
.has_value() &&
|
||||
take(load_cellular_migration_stage(*f.db, dst, dk, dl, 5s),
|
||||
"unsynced destination stage")
|
||||
.has_value(),
|
||||
"failed result sync published a half or released retained work");
|
||||
}
|
||||
std::vector<std::uint8_t> expected_source, expected_destination;
|
||||
if (representation == 0) {
|
||||
auto a = take(restore_cellular_payload(sb, s), "paired raw source oracle");
|
||||
@@ -1234,6 +1306,63 @@ void migration_publication(Fixture& f, const RegionKey destination_key,
|
||||
require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict &&
|
||||
!retry_cellular_migration_commit(*f.db, stale, dl, prepared, 5s),
|
||||
"changed pair or stale source authority was accepted");
|
||||
const auto sm = take(decode_manifest(sh.manifest), "unavailable source result");
|
||||
const auto dm = take(decode_manifest(dh.manifest), "unavailable destination result");
|
||||
// Outbound: deny current results. Return: deny retained counterpart
|
||||
// capsules. Both are required before either paused host may resume.
|
||||
const VerifiedObject unavailable_source =
|
||||
direction == 0 ? VerifiedObject{sm.payload_sha256, sm.payload_bytes}
|
||||
: VerifiedObject{d.payload_sha256, d.payload_bytes};
|
||||
const VerifiedObject unavailable_destination =
|
||||
direction == 0 ? VerifiedObject{dm.payload_sha256, dm.payload_bytes}
|
||||
: VerifiedObject{s.payload_sha256, s.payload_bytes};
|
||||
{
|
||||
UnreadableObject source_denial(so.directory() /
|
||||
hexadecimal(unavailable_source.sha256, false));
|
||||
UnreadableObject destination_denial(
|
||||
dst.directory() / hexadecimal(unavailable_destination.sha256, false));
|
||||
const auto denied_source = so.get(unavailable_source);
|
||||
const auto denied_destination = dst.get(unavailable_destination);
|
||||
require(!denied_source && denied_source.error() == ObjectError::io &&
|
||||
!denied_destination && denied_destination.error() == ObjectError::io,
|
||||
"object read denial did not produce actual I/O failures");
|
||||
const auto source_failures = source_host->status().store_failures;
|
||||
const auto destination_failures = destination_host->status().store_failures;
|
||||
const auto observed = [&] {
|
||||
if (direction == 0)
|
||||
return source_host->status().store_failures > source_failures &&
|
||||
destination_host->status().store_failures > destination_failures;
|
||||
return source_host->status().phase == HostPhase::quarantined &&
|
||||
destination_host->status().phase == HostPhase::quarantined;
|
||||
};
|
||||
for (unsigned attempt = 0; attempt < 200 && !observed(); ++attempt) {
|
||||
source_host->step();
|
||||
destination_host->step();
|
||||
std::this_thread::sleep_for(10ms);
|
||||
}
|
||||
require(
|
||||
observed() && !source_host->status().ready &&
|
||||
!destination_host->status().ready && !source_host->advance_unattended(1) &&
|
||||
!destination_host->advance_unattended(1) &&
|
||||
take(load_region_checkpoint(*f.db, sk, 5s), "unavailable source head")
|
||||
.head == sh.head &&
|
||||
take(load_region_checkpoint(*f.db, dk, 5s), "unavailable destination head")
|
||||
.head == dh.head,
|
||||
"unreadable migration evidence resumed old ownership or changed a head");
|
||||
}
|
||||
if (direction != 0) {
|
||||
require(!source_host->step().ready && !destination_host->step().ready,
|
||||
"restoring storage silently cleared quarantine");
|
||||
source_host->drain();
|
||||
destination_host->drain();
|
||||
// The quarantined local authorities cannot release themselves. This
|
||||
// test coordinator explicitly retires their still-current DB leases.
|
||||
take(release_region_lease(*f.db, sk, sl, 5s), "retire quarantined source lease");
|
||||
take(release_region_lease(*f.db, dk, dl, 5s),
|
||||
"retire quarantined destination lease");
|
||||
source_host = make_host(sk);
|
||||
destination_host = make_host(dk);
|
||||
}
|
||||
for (unsigned attempt = 0;
|
||||
attempt < 200 && (!source_host->step().ready || !destination_host->step().ready);
|
||||
++attempt)
|
||||
@@ -1242,6 +1371,17 @@ void migration_publication(Fixture& f, const RegionKey destination_key,
|
||||
take(source_host->committed(), "adopted source").head == sh.head &&
|
||||
take(destination_host->committed(), "adopted destination").head == dh.head,
|
||||
"pointer release resumed a stale owner instead of recovering the pair");
|
||||
const auto old_source_epoch = sl.epoch, old_destination_epoch = dl.epoch;
|
||||
sl = take(source_host->authority(), "restored source object authority");
|
||||
dl = take(destination_host->authority(), "restored destination object authority");
|
||||
const bool expected_epochs =
|
||||
direction == 0 ? sl.epoch == old_source_epoch && dl.epoch == old_destination_epoch
|
||||
: sl.epoch > old_source_epoch && dl.epoch > old_destination_epoch;
|
||||
require(expected_epochs &&
|
||||
take(retry_cellular_migration_commit(*f.db, sl, dl, prepared, 5s),
|
||||
"restored storage exact paired receipt")
|
||||
.source.head == sh.head,
|
||||
"storage recovery changed a committed pair or reused authority");
|
||||
const auto check_bytes = [&](const CommittedCheckpoint& h, RegionKey key,
|
||||
const FilesystemObjectStore& objects,
|
||||
const LeaseGrant& lease) {
|
||||
@@ -1290,6 +1430,7 @@ void migration_publication(Fixture& f, const RegionKey destination_key,
|
||||
migration_owner_partitions(f, destination_key, destination_objects, destination_lease,
|
||||
destination_head, genome);
|
||||
}
|
||||
std::puts("migration object sync/read failures, quarantine and exact recovery passed");
|
||||
// Race the actual two-head transaction against abort on independent connections.
|
||||
const auto prepare_pair = [&](bool reverse) {
|
||||
auto& sh = reverse ? destination_head : f.head;
|
||||
|
||||
Reference in New Issue
Block a user