Fix object reuse durability and refuse unsafe pre-v5 upgrades

Review P1 (object durability): a name installed by a put whose directory
fsync failed was later reused as success without syncing again, so a
publication could acknowledge an object that might vanish on power
loss. Reuse (existing and lost-link-race paths) now fsyncs the verified
file and the directory before succeeding, and namespace opens always
sync their parent. A test-only ObjectStoreFaults hook injects directory
sync failures; tests cover failed sync then retries, namespace creation
syncs, and publication refusing to proceed until durability holds.

Review P1 (pre-v5 history): v5 recovery requires one durable decision
per terminal actor, but v3/v4 published checkpoints carry none, so an
accepted upgrade could make regions unrecoverable. pre_v5_guard.sql now
runs before 0005 in every upgrade path and refuses (EVL10) when any
published pre-v5 checkpoint exists; genesis-only regions pass. 0005 is
unchanged. CI publishes a real v4 checkpoint and proves the upgrade is
refused with nothing changed; a committed death without a decision is
shown to recover as quarantined history.

Validated: PostgreSQL suites Release x3, clang-tidy, ASan/UBSan x2;
reverting each fix is caught by its test.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-28 19:20:35 -04:00
co-authored by Claude Opus 5.5
parent dd7e8028b0
commit ab4bf5355f
10 changed files with 259 additions and 32 deletions
+92
View File
@@ -143,6 +143,35 @@ void object_store_is_immutable_and_verified() {
}
}
// Review P1: a name made visible by an install whose directory sync failed is not
// durable. Neither a retry that finds it nor a namespace reopen may report success
// until a directory sync actually succeeds.
void directory_sync_failures_are_never_acknowledged() {
const auto root = scratch("syncs");
ObjectStoreFaults faults;
faults.fail_directory_syncs = 1; // Creating w1 under the root.
CHECK(FilesystemObjectStore::open(root, {1, 5}, 1024, &faults).error() == ObjectError::io);
CHECK(fs::exists(root / "w1")); // Created, but its entry is unconfirmed.
faults.fail_directory_syncs = 1; // Reopen finds it and must sync again.
CHECK(FilesystemObjectStore::open(root, {1, 5}, 1024, &faults).error() == ObjectError::io);
auto store = FilesystemObjectStore::open(root, {1, 5}, 1024, &faults);
CHECK(store && faults.directory_syncs == 2); // root->w1 and w1->r5.
const auto payload = bytes_of("durable only after a directory sync succeeds");
const VerifiedObject object{*sha256(payload), payload.size()};
faults.fail_directory_syncs = 1;
CHECK((*store)->put(payload).error() == ObjectError::io);
CHECK((*store)->get(object)); // Visible, yet never acknowledged.
faults.fail_directory_syncs = 3; // Retries while the directory still cannot sync.
for (int attempt = 0; attempt < 3; ++attempt)
CHECK((*store)->put(payload).error() == ObjectError::io);
CHECK(faults.directory_syncs == 2);
auto retried = (*store)->put(payload);
CHECK(retried && *retried == object && faults.directory_syncs == 3);
// Every reuse re-establishes durability rather than trusting the visible name.
CHECK((*store)->put(payload) && faults.directory_syncs == 4);
}
struct Region {
RegionKey key{7, 3};
std::unique_ptr<Postgres> owner, runtime;
@@ -675,13 +704,76 @@ void durable_outcomes_and_ambiguity() {
CHECK(!undo && std::string(undo.error().sqlstate.data()) == "EVL09"); // Stays quarantined.
CHECK(release_region_lease(*r.runtime, r.key, *lease, 5s));
}
// Publication cannot acknowledge a checkpoint whose objects are not yet durable.
void publication_waits_for_directory_durability() {
auto r = provisioned("sync-publish", 9, 0xA8);
ObjectStoreFaults faults;
auto faulty = FilesystemObjectStore::open(r.objects->directory().parent_path().parent_path(),
r.key, max_environmental_payload_bytes, &faults);
CHECK(faulty);
auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s);
CHECK(lease);
auto genesis = load_region_checkpoint(*r.runtime, r.key, 5s);
CHECK(genesis);
auto state = evo::test::region_community();
EnvBatch batch{*state};
run_batch(batch, 8);
for (const std::uint32_t failures : {1U, 1U, 2U}) {
faults.fail_directory_syncs = failures;
auto attempt = publish_environmental_batch(*r.runtime, **faulty, *lease, *genesis, batch,
filled<16>(0xA9), build, budget, 5s);
CHECK(!attempt && attempt.error().kind == PublicationError::upload &&
attempt.error().object == ObjectError::io);
CHECK(load_region_head(*r.runtime, r.key, 5s)->sequence == 0); // Nothing sent.
}
faults.fail_directory_syncs = 0;
auto published = publish_environmental_batch(*r.runtime, **faulty, *lease, *genesis, batch,
filled<16>(0xA9), build, budget, 5s);
CHECK(published && published->committed.head.sequence == 1);
CHECK(release_region_lease(*r.runtime, r.key, *lease, 5s));
}
// Review P1: a committed death with no durable decision (what an accepted pre-v5
// upgrade would leave, or a host that skipped derivation) is never served as a
// recovered region. The database cannot see payload deaths; recovery must.
void undecided_death_is_quarantined() {
auto r = provisioned("undecided", 10, 0xB6);
auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s);
CHECK(lease);
auto genesis = load_region_checkpoint(*r.runtime, r.key, 5s);
CHECK(genesis);
auto state = evo::test::region_community();
EnvBatch batch{*state};
run_batch(batch, 64, true); // The consumer dies at tick 20.
CHECK(batch.end_tick() == 20 && state->progress().consumer_terminal == 1);
const auto previous = *decode_manifest(genesis->manifest);
auto payload = state->snapshot();
auto trace =
encode_trace({1, r.key, previous.commit, genesis->head.digest, 0, 20}, batch.steps());
CHECK(payload && trace && r.objects->put(*payload) && r.objects->put(*trace));
auto manifest = successor_manifest(previous, genesis->head.digest, filled<16>(0xB7), *lease, 20,
build, payload->size(), *sha256(*payload),
{trace->size(), *sha256(*trace), 0, {}});
CHECK(manifest);
auto bytes = encode_manifest(*manifest);
CHECK(bytes && publish_region_checkpoint(*r.runtime, *bytes, {}, 5s));
const std::array<Sha256Digest, 1> retained{build};
auto recovered =
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s);
CHECK(!recovered && recovered.error().kind == RecoveryError::history);
CHECK(release_region_lease(*r.runtime, r.key, *lease, 5s));
}
} // namespace
void run_region_checkpoint_store_tests() {
sha256_vectors_and_database_agreement();
object_store_is_immutable_and_verified();
directory_sync_failures_are_never_acknowledged();
publish_recover_and_continue();
failures_leave_the_previous_head();
lineage_publish_and_recover();
durable_outcomes_and_ambiguity();
publication_waits_for_directory_durability();
undecided_death_is_quarantined();
}