Fix object reuse durability and refuse unsafe pre-v5 upgrades
Review P1 (object durability): a name installed by a put whose directory fsync failed was later reused as success without syncing again, so a publication could acknowledge an object that might vanish on power loss. Reuse (existing and lost-link-race paths) now fsyncs the verified file and the directory before succeeding, and namespace opens always sync their parent. A test-only ObjectStoreFaults hook injects directory sync failures; tests cover failed sync then retries, namespace creation syncs, and publication refusing to proceed until durability holds. Review P1 (pre-v5 history): v5 recovery requires one durable decision per terminal actor, but v3/v4 published checkpoints carry none, so an accepted upgrade could make regions unrecoverable. pre_v5_guard.sql now runs before 0005 in every upgrade path and refuses (EVL10) when any published pre-v5 checkpoint exists; genesis-only regions pass. 0005 is unchanged. CI publishes a real v4 checkpoint and proves the upgrade is refused with nothing changed; a committed death without a decision is shown to recover as quarantined history. Validated: PostgreSQL suites Release x3, clang-tidy, ASan/UBSan x2; reverting each fix is caught by its test. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -143,6 +143,35 @@ void object_store_is_immutable_and_verified() {
|
||||
}
|
||||
}
|
||||
|
||||
// Review P1: a name made visible by an install whose directory sync failed is not
|
||||
// durable. Neither a retry that finds it nor a namespace reopen may report success
|
||||
// until a directory sync actually succeeds.
|
||||
void directory_sync_failures_are_never_acknowledged() {
|
||||
const auto root = scratch("syncs");
|
||||
ObjectStoreFaults faults;
|
||||
faults.fail_directory_syncs = 1; // Creating w1 under the root.
|
||||
CHECK(FilesystemObjectStore::open(root, {1, 5}, 1024, &faults).error() == ObjectError::io);
|
||||
CHECK(fs::exists(root / "w1")); // Created, but its entry is unconfirmed.
|
||||
faults.fail_directory_syncs = 1; // Reopen finds it and must sync again.
|
||||
CHECK(FilesystemObjectStore::open(root, {1, 5}, 1024, &faults).error() == ObjectError::io);
|
||||
auto store = FilesystemObjectStore::open(root, {1, 5}, 1024, &faults);
|
||||
CHECK(store && faults.directory_syncs == 2); // root->w1 and w1->r5.
|
||||
|
||||
const auto payload = bytes_of("durable only after a directory sync succeeds");
|
||||
const VerifiedObject object{*sha256(payload), payload.size()};
|
||||
faults.fail_directory_syncs = 1;
|
||||
CHECK((*store)->put(payload).error() == ObjectError::io);
|
||||
CHECK((*store)->get(object)); // Visible, yet never acknowledged.
|
||||
faults.fail_directory_syncs = 3; // Retries while the directory still cannot sync.
|
||||
for (int attempt = 0; attempt < 3; ++attempt)
|
||||
CHECK((*store)->put(payload).error() == ObjectError::io);
|
||||
CHECK(faults.directory_syncs == 2);
|
||||
auto retried = (*store)->put(payload);
|
||||
CHECK(retried && *retried == object && faults.directory_syncs == 3);
|
||||
// Every reuse re-establishes durability rather than trusting the visible name.
|
||||
CHECK((*store)->put(payload) && faults.directory_syncs == 4);
|
||||
}
|
||||
|
||||
struct Region {
|
||||
RegionKey key{7, 3};
|
||||
std::unique_ptr<Postgres> owner, runtime;
|
||||
@@ -675,13 +704,76 @@ void durable_outcomes_and_ambiguity() {
|
||||
CHECK(!undo && std::string(undo.error().sqlstate.data()) == "EVL09"); // Stays quarantined.
|
||||
CHECK(release_region_lease(*r.runtime, r.key, *lease, 5s));
|
||||
}
|
||||
// Publication cannot acknowledge a checkpoint whose objects are not yet durable.
|
||||
void publication_waits_for_directory_durability() {
|
||||
auto r = provisioned("sync-publish", 9, 0xA8);
|
||||
ObjectStoreFaults faults;
|
||||
auto faulty = FilesystemObjectStore::open(r.objects->directory().parent_path().parent_path(),
|
||||
r.key, max_environmental_payload_bytes, &faults);
|
||||
CHECK(faulty);
|
||||
auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s);
|
||||
CHECK(lease);
|
||||
auto genesis = load_region_checkpoint(*r.runtime, r.key, 5s);
|
||||
CHECK(genesis);
|
||||
auto state = evo::test::region_community();
|
||||
EnvBatch batch{*state};
|
||||
run_batch(batch, 8);
|
||||
for (const std::uint32_t failures : {1U, 1U, 2U}) {
|
||||
faults.fail_directory_syncs = failures;
|
||||
auto attempt = publish_environmental_batch(*r.runtime, **faulty, *lease, *genesis, batch,
|
||||
filled<16>(0xA9), build, budget, 5s);
|
||||
CHECK(!attempt && attempt.error().kind == PublicationError::upload &&
|
||||
attempt.error().object == ObjectError::io);
|
||||
CHECK(load_region_head(*r.runtime, r.key, 5s)->sequence == 0); // Nothing sent.
|
||||
}
|
||||
faults.fail_directory_syncs = 0;
|
||||
auto published = publish_environmental_batch(*r.runtime, **faulty, *lease, *genesis, batch,
|
||||
filled<16>(0xA9), build, budget, 5s);
|
||||
CHECK(published && published->committed.head.sequence == 1);
|
||||
CHECK(release_region_lease(*r.runtime, r.key, *lease, 5s));
|
||||
}
|
||||
|
||||
// Review P1: a committed death with no durable decision (what an accepted pre-v5
|
||||
// upgrade would leave, or a host that skipped derivation) is never served as a
|
||||
// recovered region. The database cannot see payload deaths; recovery must.
|
||||
void undecided_death_is_quarantined() {
|
||||
auto r = provisioned("undecided", 10, 0xB6);
|
||||
auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s);
|
||||
CHECK(lease);
|
||||
auto genesis = load_region_checkpoint(*r.runtime, r.key, 5s);
|
||||
CHECK(genesis);
|
||||
auto state = evo::test::region_community();
|
||||
EnvBatch batch{*state};
|
||||
run_batch(batch, 64, true); // The consumer dies at tick 20.
|
||||
CHECK(batch.end_tick() == 20 && state->progress().consumer_terminal == 1);
|
||||
const auto previous = *decode_manifest(genesis->manifest);
|
||||
auto payload = state->snapshot();
|
||||
auto trace =
|
||||
encode_trace({1, r.key, previous.commit, genesis->head.digest, 0, 20}, batch.steps());
|
||||
CHECK(payload && trace && r.objects->put(*payload) && r.objects->put(*trace));
|
||||
auto manifest = successor_manifest(previous, genesis->head.digest, filled<16>(0xB7), *lease, 20,
|
||||
build, payload->size(), *sha256(*payload),
|
||||
{trace->size(), *sha256(*trace), 0, {}});
|
||||
CHECK(manifest);
|
||||
auto bytes = encode_manifest(*manifest);
|
||||
CHECK(bytes && publish_region_checkpoint(*r.runtime, *bytes, {}, 5s));
|
||||
const std::array<Sha256Digest, 1> retained{build};
|
||||
auto recovered =
|
||||
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s);
|
||||
CHECK(!recovered && recovered.error().kind == RecoveryError::history);
|
||||
CHECK(release_region_lease(*r.runtime, r.key, *lease, 5s));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void run_region_checkpoint_store_tests() {
|
||||
sha256_vectors_and_database_agreement();
|
||||
object_store_is_immutable_and_verified();
|
||||
directory_sync_failures_are_never_acknowledged();
|
||||
publish_recover_and_continue();
|
||||
failures_leave_the_previous_head();
|
||||
lineage_publish_and_recover();
|
||||
durable_outcomes_and_ambiguity();
|
||||
publication_waits_for_directory_durability();
|
||||
undecided_death_is_quarantined();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user