From a2a12fdc696c005bd3feb496f6efcdc58c28262c Mon Sep 17 00:00:00 2001 From: dingo Date: Sat, 26 Sep 2026 19:40:16 -0400 Subject: [PATCH] Fix M5.7 compaction and failure recovery --- docs/adr/0048-local-publication.md | 28 ++++++--- docs/engineering/VALIDATION.md | 35 ++++++++--- docs/world/WORLD_CODEC.md | 5 +- include/evo/world/local_files.hpp | 16 ++++- src/world/local_files.cpp | 35 +++++++++-- tests/world_local_files_tests.cpp | 97 +++++++++++++++++++++++++++++- 6 files changed, 187 insertions(+), 29 deletions(-) diff --git a/docs/adr/0048-local-publication.md b/docs/adr/0048-local-publication.md index e9c96a8..c725a39 100644 --- a/docs/adr/0048-local-publication.md +++ b/docs/adr/0048-local-publication.md @@ -27,20 +27,27 @@ state silently. No automatic orphan deletion in this gate. Snapshot checkpoint validates bytes by restoring them under budget before publication. A journal commit must match the current checkpoint's exact bytes and high-water, and -pass detached replay. Only one journal segment is active per checkpoint; checkpoint -after replay to compact it. Recovery validates the selected manifest and referenced -payloads using M5.6, then replays the optional segment before returning a candidate. -Existing valid manifest stays the source until new publication completes. +pass detached replay. Only one journal segment is active per checkpoint. While one is +published, compaction accepts only the byte-identical canonical snapshot of recovery's +replayed result; command high-water alone cannot prove continuation or prevent a fork. +Recovery validates the selected manifest and referenced payloads using M5.6, then +replays the optional segment before returning a candidate. Existing valid manifest +stays the source until new publication completes. A poisoned handle refuses recovery, +reports no current generation, and requires reopen to resolve an ambiguous publication +error. Implement the minimal native OS boundary directly: POSIX lock/write/fsync/link/rename/ directory fsync; Windows exclusive CreateFile and write-through replacement. No third party library. Root directory is trusted and locally administered. Files are not network-safe/authenticated; FNV detects accidental corruption only. -Provide one-shot failure injection at payload sync, manifest sync, and manifest publish -for deterministic crash-boundary tests. After injected interruption, discard the handle -and reopen. A failure before manifest replacement yields old state; after replacement -yields new state. Allocation/OS errors do not clear dirty state. +Provide one-shot failure injection at payload sync, manifest sync, manifest publish, +partial payload/manifest write (out of space), and post-replacement directory sync for +deterministic crash-boundary tests. Partial-write failures leave only an ignored temp +file and allow retry; injected interruptions poison the handle. After an interruption +or ambiguous sync failure, discard the handle and reopen. A failure before manifest +replacement yields old state; after replacement yields new state. Allocation/OS errors +do not clear dirty state. ## Alternatives Considered In-place overwrite can destroy the old checkpoint on short write. A single manifest can @@ -60,6 +67,7 @@ differ. Same-host files do not survive host loss. Fail-closed corruption may req operator recovery. No production DR/RPO claim. ## Consequences -Test first checkpoint, journal, compaction, restart continuation, competing lock, corrupt -payload/manifest, every injected boundary, missing sequence and full-disk/OS errors. +Test first checkpoint, journal, exact replay compaction, divergent-state rejection, +restart continuation, competing lock, corrupt payload/manifest, every injected boundary, +partial-write recovery, missing sequence and full-disk/OS errors. Record Linux checks separately. M5.8 performs the whole create/edit/compact/restart proof. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 3638fad..0bf68f8 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -28,14 +28,33 @@ Executed on Windows x64 / MSVC 19.44.35207: - `tools/windows-check.ps1 -Preset release -Action format`: passed. - `git diff --check`: passed before commit. -New coverage validates checkpoint/journal/recovery, compaction over a published -journal, exclusive writer lock, all three injected publication boundaries, corrupt -manifest and missing published payload failure. Before-manifest failures recover -the previous generation; after-manifest replacement recovers the new generation. -Actual power-cut/full-disk behavior, POSIX syscall path, Linux CI/libFuzzer, and -filesystem-controller flush guarantees remain unverified locally. There is no -automatic orphan cleanup, production storage adapter, Kubernetes deployment or -World0 change. +New coverage validates checkpoint/journal/recovery, exact compaction over a published +journal, divergent-state rejection, exclusive writer lock, injected publication +boundaries, partial-write/out-of-space outcomes, corrupt manifest and missing payload. +Before-manifest failures recover the previous state; after replacement, reopen +recovers the new state. Actual power-cut/disk-full behavior, POSIX syscall path, +Linux CI/libFuzzer, and filesystem-controller flush guarantees remain unverified +locally. There is no automatic orphan cleanup, production storage adapter, +Kubernetes deployment or World0 change. + +### M5.7 review corrections + +The follow-up review found that command high-water alone allowed a divergent +checkpoint to replace an active journal. Checkpoint now compares its canonical bytes +with the fully recovered journal result before compaction. Poisoned handles reject +recovery; reopen resolves ambiguous post-replacement directory-sync failures. +Interruption tests now use a different edited payload and compare recovered canonical +state and debris across the commit point. Deterministic partial-write/out-of-space +injection covers payload and manifest temporaries, including retry, and a directory +sync failure hook verifies poison/reopen behavior. The injection models a partial +write outcome; it is not a physical disk-full or power-cut test. + +Revalidated after these corrections on Windows x64 / MSVC 19.44.35207: + +- Release configure/build and 47/47 CTests passed. +- clang-tidy analysis build and 47/47 CTests passed. +- ASan build and 47/47 CTests passed. +- clang-format and `git diff --check` passed. ## Tools and host diff --git a/docs/world/WORLD_CODEC.md b/docs/world/WORLD_CODEC.md index f727ba9..2fdbacb 100644 --- a/docs/world/WORLD_CODEC.md +++ b/docs/world/WORLD_CODEC.md @@ -82,6 +82,9 @@ and [the manifest schema](../../schemas/local-world-manifest.md). Two alternatin manifest slots select immutable M5.6 snapshot/journal payloads. Recovery validates the selected snapshot and optional journal into a detached store. The manifest replacement is the local commit point; this provides no multi-host durability, -production lease, authentication or disaster recovery guarantee. +production lease, authentication or disaster recovery guarantee. Compaction while a +journal is active accepts only its byte-identical replay result, and poisoned handles +must reopen before recovery. Partial-write and post-replacement sync failures have +deterministic test injection. M5.6 does not write files, fsync, delete journals, clear dirty state or recover a live server process. M7 supplies the production durability/ownership path. diff --git a/include/evo/world/local_files.hpp b/include/evo/world/local_files.hpp index 2b244e7..2ac4aec 100644 --- a/include/evo/world/local_files.hpp +++ b/include/evo/world/local_files.hpp @@ -31,6 +31,12 @@ enum class PublicationInterruption : std::uint8_t { after_manifest_sync, after_manifest_publish }; +enum class PublicationFailure : std::uint8_t { + none, + payload_out_of_space, + manifest_out_of_space, + manifest_directory_sync +}; struct FileCommit { std::uint64_t generation{}, covered_command{}, snapshot_checksum{}; bool has_journal{}; @@ -40,17 +46,21 @@ class LocalWorldFiles { public: [[nodiscard]] static std::expected, LocalFileError> open(const std::filesystem::path& trusted_root, WorldIdentity identity, std::size_t byte_budget, - PublicationInterruption interrupt_once = PublicationInterruption::none); + PublicationInterruption interrupt_once = PublicationInterruption::none, + PublicationFailure fail_once = PublicationFailure::none); LocalWorldFiles(const LocalWorldFiles&) = delete; LocalWorldFiles& operator=(const LocalWorldFiles&) = delete; LocalWorldFiles(LocalWorldFiles&&) = delete; LocalWorldFiles& operator=(LocalWorldFiles&&) = delete; ~LocalWorldFiles(); + // An ambiguous publication failure poisons this handle; reopen before recovery. [[nodiscard]] std::expected, LocalFileError> recover() const; + // While a journal is committed, compaction requires its byte-identical replay result. [[nodiscard]] std::expected checkpoint(const ChunkStore& store); [[nodiscard]] std::expected publish_journal(std::span checkpoint, std::span journal); + // Empty on an uninitialized or poisoned handle. [[nodiscard]] std::optional current() const noexcept; private: @@ -61,7 +71,8 @@ class LocalWorldFiles { bool has_journal{}; }; LocalWorldFiles(std::filesystem::path directory, WorldIdentity identity, std::size_t budget, - PublicationInterruption interruption, std::unique_ptr lock); + PublicationInterruption interruption, PublicationFailure failure, + std::unique_ptr lock); [[nodiscard]] std::expected publish(std::span payload, bool journal, std::uint64_t covered, std::uint64_t payload_checksum); @@ -69,6 +80,7 @@ class LocalWorldFiles { WorldIdentity identity_; std::size_t budget_; PublicationInterruption interruption_; + PublicationFailure failure_; std::unique_ptr lock_; std::optional manifest_; std::size_t active_slot_{}; diff --git a/src/world/local_files.cpp b/src/world/local_files.cpp index e4d11c0..8c1db43 100644 --- a/src/world/local_files.cpp +++ b/src/world/local_files.cpp @@ -211,14 +211,14 @@ struct LocalWorldFiles::Lock { LocalWorldFiles::LocalWorldFiles(std::filesystem::path directory, WorldIdentity identity, std::size_t budget, PublicationInterruption interruption, - std::unique_ptr lock) + PublicationFailure failure, std::unique_ptr lock) : directory_{std::move(directory)}, identity_{identity}, budget_{budget}, - interruption_{interruption}, lock_{std::move(lock)} {} + interruption_{interruption}, failure_{failure}, lock_{std::move(lock)} {} LocalWorldFiles::~LocalWorldFiles() = default; std::expected, LocalFileError> LocalWorldFiles::open(const std::filesystem::path& root, WorldIdentity identity, std::size_t budget, - PublicationInterruption interruption) { + PublicationInterruption interruption, PublicationFailure failure) { if (!identity.world || !identity.frame || !budget) return std::unexpected(LocalFileError::invalid_identity); const auto directory = root / ("world-" + hex(identity.world) + "-" + hex(identity.frame)); @@ -238,7 +238,7 @@ LocalWorldFiles::open(const std::filesystem::path& root, WorldIdentity identity, return std::unexpected(LocalFileError::busy); #endif auto result = std::unique_ptr( - new LocalWorldFiles(directory, identity, budget, interruption, std::move(lock))); + new LocalWorldFiles(directory, identity, budget, interruption, failure, std::move(lock))); for (std::size_t slot = 0; slot < 2; ++slot) { auto bytes = read_file(directory / ("manifest." + std::to_string(slot)), manifest_size); if (!bytes) { @@ -260,13 +260,15 @@ LocalWorldFiles::open(const std::filesystem::path& root, WorldIdentity identity, } std::optional LocalWorldFiles::current() const noexcept { - if (!manifest_) + if (poisoned_ || !manifest_) return std::nullopt; return FileCommit{manifest_->generation, manifest_->covered_command, manifest_->snapshot_checksum, manifest_->has_journal}; } std::expected, LocalFileError> LocalWorldFiles::recover() const { + if (poisoned_) + return std::unexpected(LocalFileError::poisoned); if (!manifest_) return std::unexpected(LocalFileError::empty); const auto snapshot_path = @@ -314,6 +316,12 @@ LocalWorldFiles::publish(std::span payload, bool journal, st const auto temp = directory_ / (std::string{prefix} + ".payload.tmp"); std::filesystem::remove(temp, ec); ec.clear(); + if (failure_ == PublicationFailure::payload_out_of_space) { + failure_ = PublicationFailure::none; + const auto partial_size = std::max(1, payload.size() / 2); + (void)write_synced(temp, payload.first(partial_size)); + return std::unexpected(LocalFileError::io); + } if (!write_synced(temp, payload)) return std::unexpected(LocalFileError::io); if (interruption_ == PublicationInterruption::after_payload_sync) { @@ -349,6 +357,12 @@ LocalWorldFiles::publish(std::span payload, bool journal, st const auto encoded = encode_manifest(identity_, candidate); std::filesystem::remove(temp_manifest, ec); ec.clear(); + if (failure_ == PublicationFailure::manifest_out_of_space) { + failure_ = PublicationFailure::none; + const auto partial_size = std::max(1, encoded.size() / 2); + (void)write_synced(temp_manifest, std::span{encoded}.first(partial_size)); + return std::unexpected(LocalFileError::io); + } if (!write_synced(temp_manifest, encoded)) return std::unexpected(LocalFileError::io); if (interruption_ == PublicationInterruption::after_manifest_sync) { @@ -358,7 +372,8 @@ LocalWorldFiles::publish(std::span payload, bool journal, st } if (!replace_file(temp_manifest, dest_manifest)) return std::unexpected(LocalFileError::io); - if (!sync_directory(directory_)) { + if (failure_ == PublicationFailure::manifest_directory_sync || !sync_directory(directory_)) { + failure_ = PublicationFailure::none; poisoned_ = true; return std::unexpected(LocalFileError::io); } @@ -389,6 +404,14 @@ std::expected LocalWorldFiles::checkpoint(const Chun : 0; if (covered < required) return std::unexpected(LocalFileError::stale_checkpoint); + if (manifest_ && manifest_->has_journal) { + auto committed = recover(); + if (!committed) + return std::unexpected(committed.error()); + const auto committed_bytes = (*committed)->snapshot(identity_); + if (!committed_bytes || !std::ranges::equal(*bytes, *committed_bytes)) + return std::unexpected(LocalFileError::stale_checkpoint); + } return publish(*bytes, false, covered, hash(*bytes)); } diff --git a/tests/world_local_files_tests.cpp b/tests/world_local_files_tests.cpp index bc50d58..fca7f7f 100644 --- a/tests/world_local_files_tests.cpp +++ b/tests/world_local_files_tests.cpp @@ -37,6 +37,18 @@ std::unique_ptr make_store() { CHECK((*result)->load({0, 0, -2})); return std::move(*result); } +std::unique_ptr edited_store(std::uint32_t amount) { + auto result = make_store(); + const auto view = result->view({0, 0, -2}); + CHECK(view); + const std::array cuts{{{{0, 0, -2}, {0, 0, 0}, view->revision(), amount}}}; + ExcavationRequest request{1, 1, cuts, 0}; + const auto digest = excavation_digest(request); + CHECK(digest); + request.digest = *digest; + CHECK(result->excavate(request)); + return result; +} std::filesystem::path world_dir(const std::filesystem::path& root) { return root / "world-0000000000000047-000000000000005b"; } @@ -65,6 +77,10 @@ void checkpoint_and_journal_round_trip() { CHECK(!(*files)->publish_journal(*checkpoint, *journal)); const auto stale_checkpoint = (*files)->checkpoint(*original); CHECK(!stale_checkpoint && stale_checkpoint.error() == LocalFileError::stale_checkpoint); + auto divergent = edited_store(3); + const auto divergent_checkpoint = (*files)->checkpoint(*divergent); + CHECK(!divergent_checkpoint && + divergent_checkpoint.error() == LocalFileError::stale_checkpoint); auto recovered = (*files)->recover(); CHECK(recovered); auto expected = replay_edit_journal(*checkpoint, *journal, identity, budget); @@ -87,18 +103,27 @@ void interruptions_preserve_commit_point() { {PublicationInterruption::after_payload_sync, PublicationInterruption::after_manifest_sync, PublicationInterruption::after_manifest_publish}) { TempRoot root; + WorldBytes old_snapshot; { auto files = LocalWorldFiles::open(root.path, identity, budget); CHECK(files); auto initial = make_store(); + auto snapshot = initial->snapshot(identity); + CHECK(snapshot); + old_snapshot = *snapshot; CHECK((*files)->checkpoint(*initial)); } auto interrupted = LocalWorldFiles::open(root.path, identity, budget, point); CHECK(interrupted); - auto changed = make_store(); + auto changed = edited_store(2); + auto new_snapshot = changed->snapshot(identity); + CHECK(new_snapshot); const auto result = (*interrupted)->checkpoint(*changed); CHECK(!result && result.error() == LocalFileError::interrupted); CHECK(!(*interrupted)->checkpoint(*changed)); + const auto poisoned_recovery = (*interrupted)->recover(); + CHECK(!poisoned_recovery && poisoned_recovery.error() == LocalFileError::poisoned); + CHECK(!(*interrupted)->current()); (*interrupted).reset(); auto recovered = LocalWorldFiles::open(root.path, identity, budget); CHECK(recovered); @@ -106,9 +131,75 @@ void interruptions_preserve_commit_point() { CHECK(commit.has_value()); const auto generation = commit.value_or(FileCommit{}).generation; CHECK(generation == (point == PublicationInterruption::after_manifest_publish ? 2 : 1)); - CHECK((*recovered)->recover()); + auto recovered_state = (*recovered)->recover(); + CHECK(recovered_state); + const auto& expected = + point == PublicationInterruption::after_manifest_publish ? *new_snapshot : old_snapshot; + CHECK((**recovered_state).snapshot(identity) == expected); + const auto expected_debris = + point == PublicationInterruption::after_manifest_publish ? 2U : 0U; + const auto debris = (**recovered_state).debris({1}); + CHECK(debris && *debris == VolumeUnits{expected_debris}); } } +void partial_write_failures_preserve_and_retry() { + for (const auto failure : + {PublicationFailure::payload_out_of_space, PublicationFailure::manifest_out_of_space}) { + TempRoot root; + WorldBytes old_snapshot; + { + auto files = LocalWorldFiles::open(root.path, identity, budget); + CHECK(files); + auto initial = make_store(); + auto snapshot = initial->snapshot(identity); + CHECK(snapshot); + old_snapshot = *snapshot; + CHECK((*files)->checkpoint(*initial)); + } + auto files = LocalWorldFiles::open(root.path, identity, budget, + PublicationInterruption::none, failure); + CHECK(files); + auto changed = edited_store(2); + auto new_snapshot = changed->snapshot(identity); + CHECK(new_snapshot); + const auto failed = (*files)->checkpoint(*changed); + CHECK(!failed && failed.error() == LocalFileError::io); + auto before_retry = (*files)->recover(); + CHECK(before_retry); + CHECK((**before_retry).snapshot(identity) == old_snapshot); + auto retried = (*files)->checkpoint(*changed); + CHECK(retried); + auto after_retry = (*files)->recover(); + CHECK(after_retry); + CHECK((**after_retry).snapshot(identity) == *new_snapshot); + } +} +void manifest_directory_sync_failure_requires_reopen() { + TempRoot root; + { + auto files = LocalWorldFiles::open(root.path, identity, budget); + CHECK(files); + auto initial = make_store(); + CHECK((*files)->checkpoint(*initial)); + } + auto files = LocalWorldFiles::open(root.path, identity, budget, PublicationInterruption::none, + PublicationFailure::manifest_directory_sync); + CHECK(files); + auto changed = edited_store(2); + auto expected = changed->snapshot(identity); + CHECK(expected); + const auto failed = (*files)->checkpoint(*changed); + CHECK(!failed && failed.error() == LocalFileError::io); + const auto poisoned = (*files)->recover(); + CHECK(!poisoned && poisoned.error() == LocalFileError::poisoned); + CHECK(!(*files)->current()); + (*files).reset(); + auto reopened = LocalWorldFiles::open(root.path, identity, budget); + CHECK(reopened); + auto recovered = (*reopened)->recover(); + CHECK(recovered); + CHECK((**recovered).snapshot(identity) == *expected); +} void lock_and_corruption_fail_closed() { TempRoot root; auto first = LocalWorldFiles::open(root.path, identity, budget); @@ -164,6 +255,8 @@ void missing_published_payload_fails_closed() { int main() try { checkpoint_and_journal_round_trip(); interruptions_preserve_commit_point(); + partial_write_failures_preserve_and_retry(); + manifest_directory_sync_failure_requires_reopen(); lock_and_corruption_fail_closed(); missing_published_payload_fails_closed(); } catch (const std::exception&) {