From 207f9e6953b0563487f27669fb317b451ffc9513 Mon Sep 17 00:00:00 2001 From: dingo Date: Sat, 26 Sep 2026 19:05:29 -0400 Subject: [PATCH] Implement M5.6 canonical world snapshots and edit journals --- CMakeLists.txt | 20 +- README.md | 1 + docs/ROADMAP.md | 4 +- docs/STATUS.md | 12 +- docs/adr/0047-world-codec.md | 61 ++ docs/adr/README.md | 2 + docs/architecture/PERSISTENCE_ARCHITECTURE.md | 8 + docs/architecture/WORLD_ARCHITECTURE.md | 8 + docs/engineering/VALIDATION.md | 35 ++ docs/world/WORLD_CODEC.md | 82 +++ include/evo/world/chunks.hpp | 31 +- schemas/world-edits.md | 6 +- schemas/world-journal.md | 31 ++ schemas/world-snapshot.md | 58 ++ src/world/codec.cpp | 524 ++++++++++++++++++ src/world/edits.cpp | 1 + tests/fixtures/world_empty_v1.hpp | 15 + tests/fuzz/world_codec_fuzz.cpp | 91 +++ tests/fuzz/world_codec_smoke.cpp | 18 + tests/world_codec_tests.cpp | 233 ++++++++ tools/ci/linux-check.sh | 4 +- tools/world_snapshot_reference.py | 26 + 22 files changed, 1262 insertions(+), 9 deletions(-) create mode 100644 docs/adr/0047-world-codec.md create mode 100644 docs/world/WORLD_CODEC.md create mode 100644 schemas/world-journal.md create mode 100644 schemas/world-snapshot.md create mode 100644 src/world/codec.cpp create mode 100644 tests/fixtures/world_empty_v1.hpp create mode 100644 tests/fuzz/world_codec_fuzz.cpp create mode 100644 tests/fuzz/world_codec_smoke.cpp create mode 100644 tests/world_codec_tests.cpp create mode 100644 tools/world_snapshot_reference.py diff --git a/CMakeLists.txt b/CMakeLists.txt index e9f6676..33cabce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,7 +34,8 @@ target_include_directories(evo_simulation PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/in target_compile_features(evo_simulation PUBLIC cxx_std_23) evo_owned_target(evo_simulation) -add_library(evo_world STATIC src/world/coordinates.cpp src/world/generation.cpp src/world/chunks.cpp src/world/edits.cpp) +set(evo_world_sources src/world/coordinates.cpp src/world/generation.cpp src/world/chunks.cpp src/world/edits.cpp src/world/codec.cpp) +add_library(evo_world STATIC ${evo_world_sources}) add_library(evo::world ALIAS evo_world) target_include_directories(evo_world PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") target_compile_features(evo_world PUBLIC cxx_std_23) @@ -71,6 +72,18 @@ evo_owned_target(evo-sim) include(CTest) if(BUILD_TESTING) + add_executable(evo-world-codec-fuzz-smoke tests/fuzz/world_codec_fuzz.cpp tests/fuzz/world_codec_smoke.cpp) + target_link_libraries(evo-world-codec-fuzz-smoke PRIVATE evo::world) + evo_owned_target(evo-world-codec-fuzz-smoke) + add_test(NAME world-codec-fuzz-smoke COMMAND evo-world-codec-fuzz-smoke) + set_tests_properties(world-codec-fuzz-smoke PROPERTIES TIMEOUT 120) + + add_executable(evo-world-codec-tests tests/world_codec_tests.cpp) + target_link_libraries(evo-world-codec-tests PRIVATE evo::world) + evo_owned_target(evo-world-codec-tests) + add_test(NAME world-codec-contracts COMMAND evo-world-codec-tests) + set_tests_properties(world-codec-contracts PROPERTIES TIMEOUT 120) + add_executable(evo-world-edit-tests tests/world_edit_tests.cpp) target_link_libraries(evo-world-edit-tests PRIVATE evo::world) evo_owned_target(evo-world-edit-tests) @@ -254,6 +267,11 @@ if(EVO_BUILD_FUZZER) if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR MSVC OR EVO_SANITIZER STREQUAL "thread") message(FATAL_ERROR "Fuzzing requires Linux Clang and is incompatible with TSan") endif() + add_executable(evo-world-codec-fuzz tests/fuzz/world_codec_fuzz.cpp ${evo_world_sources}) + target_include_directories(evo-world-codec-fuzz PRIVATE include) + evo_owned_target(evo-world-codec-fuzz) + target_compile_options(evo-world-codec-fuzz PRIVATE -fsanitize=fuzzer,address,undefined -fno-omit-frame-pointer) + target_link_options(evo-world-codec-fuzz PRIVATE -fsanitize=fuzzer,address,undefined) # Compile the kernel itself with coverage and sanitizer instrumentation. add_executable(evo-snapshot-fuzz tests/fuzz/snapshot_fuzz.cpp ${evo_simulation_sources}) target_include_directories(evo-snapshot-fuzz PRIVATE include) diff --git a/README.md b/README.md index 5e7257e..284eb16 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ network service. Dev verification manifests are separate from the simulation. - [Architecture decisions and proposals](docs/adr/README.md) - [Actionable milestones 0–4 and later gates](docs/ROADMAP.md) - [Open design questions](docs/architecture/OPEN_QUESTIONS.md) +- [World snapshots and edit journals](docs/world/WORLD_CODEC.md) - [Transactional terrain excavation](docs/world/TERRAIN_EDITS.md) - [Bounded chunk ownership](docs/world/CHUNK_OWNERSHIP.md) - [Versioned abiotic base generation](docs/world/BASE_GENERATION.md) diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index b660cda..f15205d 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -266,7 +266,9 @@ M5.4 is complete under [ADR-0045](adr/0045-bounded-chunk-ownership.md): [bounded ownership](world/CHUNK_OWNERSHIP.md), compact storage, validated revision tokens and dirty eviction refusal. M5.5 now implements [atomic excavation](world/TERRAIN_EDITS.md) with per-material volume conservation under [ADR-0046](adr/0046-transactional-excavation.md). -Next is 5.6: canonical snapshots and edit journals. +M5.6 is complete under [ADR-0047](adr/0047-world-codec.md): +[canonical snapshots and edit journals](world/WORLD_CODEC.md) validate geometry, debris +and retry state together. Next is 5.7: local publication, compaction and recovery. **Decisions still open:** final spherical layout, region dimensions, adaptive LOD, field quantization beyond the initial experiment, hydrology and surface extraction. diff --git a/docs/STATUS.md b/docs/STATUS.md index e4086f5..0cd2eed 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -7,7 +7,7 @@ persistent simulation server. The founding vision remains unchanged. ## Completed implementation - Vision, architectural pillars, risks, open-question register, architecture - diagrams, biological contracts, engineering constitution and 46 ADRs. + diagrams, biological contracts, engineering constitution and 47 ADRs. Proposed/deferred choices remain distinct from accepted proof decisions. - M0–M4 task breakdowns with dependencies, acceptance evidence and risks, plus M5–M14 gates in [ROADMAP.md](ROADMAP.md). @@ -61,7 +61,7 @@ for the bounded proof; the general production ECS remains a later measured choic | Priority | Work | Completion evidence | |---|---|---| -| 1 | M5.6 snapshots and edit journals | Bounded canonical format, geometry/debris/retry-state consistency and malformed-input rejection | +| 1 | M5.7 local publication and compaction | Verified snapshot/journal replacement, recovery and injected filesystem failures | | Research | Improve aggregate fidelity and measure Linux/target-host costs | M4.6 found no qualifying width; see the aggregate budget report | | 2 | Revisit the ADR-0031/0032 reproduction cost model | Reproduction is affordable only at the genome schema's floor; decide whether that is intended before M4 builds cohorts on it | | Delivery gate | Add/update the dev workload when a deployable increment needs it; Argo is ready | Observe Job/pod completion separately from image publication; no World0 changes | @@ -576,3 +576,11 @@ volume into finite per-material debris. Revision/digest/full-payload retry check 64-command retention and peak allocation backpressure are explicit. The [edit contract](world/TERRAIN_EDITS.md) records excavation-only scope and persistence obligations. No mass conversion or durable acknowledgement is claimed. Next: M5.6. + +## M5.6 snapshots and edit journals + +Added bounded canonical world snapshot v1 and edit journal v1, world/frame and base +version checks, full geometry/debris/retry restore, semantic validation and detached +all-or-nothing replay. [Codec contract](world/WORLD_CODEC.md) and schemas define +memory/trust limits. Dirty flags remain set; no file publication or durable ACK is +implemented. Next: M5.7 local publication, compaction and recovery. diff --git a/docs/adr/0047-world-codec.md b/docs/adr/0047-world-codec.md new file mode 100644 index 0000000..8bc18bd --- /dev/null +++ b/docs/adr/0047-world-codec.md @@ -0,0 +1,61 @@ +# ADR-0047: Bounded world snapshot and edit-journal codecs + +## Status +Accepted for M5.6 local binary buffers. File publication/crash consistency is M5.7. + +## Context +Geometry, debris and retry state form one consistency boundary. Host allocation +budgets are runtime policy, not saved physical laws. Unknown versions fail closed. + +## Decision +Implement explicit little-endian snapshot v1 and journal v1; no native object dumps. +Bind nonzero world/frame IDs supplied by the host and retain all base versions. +Snapshots sort chunks by address and successful replay records by command ID, record +slot capacity, revision high-water, command high-water, successful edit count, +debris capacities/balances, dirty state, logical samples and full retained requests +and receipts. Covered sequence is the command high-water (IDs may have gaps). +Use logical cells for the initial codec; resident compression is independent. + +Bounds: 4096 slots, 64 replay records, 70 MiB snapshot, 256 KiB journal, at most 64 +journal commands. Total length and FNV-1a corruption checksum precede semantic +acceptance. FNV is not authenticity or a security primitive. The host must provide +trusted provenance for durable data; future production manifests need strong digests. +Input/output buffers and bounded codec workspace are separate from resident budget. +No unchecked length controls allocation or iteration. Restore takes an explicit host +byte budget, builds an unpublished store and validates all state before return. + +Validate cell structure/materials against the recorded procedural base: excavation +cannot change material, add fill, or alter clean chunks. Removed volume across resident +chunks must equal debris exactly. All changed chunks remain dirty/resident. Validate +replay shape/digest/receipt amounts, chronology, current revision linkage and reverse +volume feasibility for retained commands. Persist successful edit count to distinguish +a full bounded replay window from accidentally dropped retry records. + +Journals contain predecessor snapshot checksum, base command high-water, ordered full +fresh edit requests and expected final snapshot checksum. Replay restores a detached +base then applies ordinary atomic excavation, verifies the final canonical snapshot, +and publishes only success. No partial replay escapes. Journal construction requires +already-loaded chunks and no intervening load/evict/pin operations; those require a +new snapshot. Empty journals are valid. Retry requests are not fresh journal events. + +## Alternatives Considered +Native structs are ABI-dependent. Geometry-only snapshots lose conservation/retry +state. Reconstructing everything from 64 retained requests loses older edits. A full +database/event-sourcing backend is outside this gate. Compression is deferred. + +## Benefits +Canonical round trips, strict version/identity boundaries and real snapshot+edit replay. + +## Costs +Dense logical wire cells and full exact retry payloads trade bytes for simple validation. +Restore recomputes base geometry and validates history; this is not a latency SLA. + +## Risks +Checksums do not stop malicious forgery. This codec does not fsync, atomically publish, +clear dirty flags, authenticate a client or acknowledge permanent outcomes durably. +Allocation exceptions propagate to the host; failed candidates are discarded. + +## Consequences +Test canonical round trips, post-restore continuation/retries, journal chaining, +truncation/corruption/oversize/version/identity/budget and semantic rejections with +recomputed checksums. Add a bounded fuzz entry point; preserve existing goldens. diff --git a/docs/adr/README.md b/docs/adr/README.md index 60d03c2..3c5b10c 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -51,3 +51,5 @@ design. No unadopted candidate is a link dependency of the proof. - [ADR-0045: Bounded chunk ownership](0045-bounded-chunk-ownership.md) - accepted for M5.4 local memory, revision tokens and dirty eviction guards. - [ADR-0046: Transactional excavation](0046-transactional-excavation.md) - accepted for M5.5 atomic local volume transfers and bounded exact-request replay. + +- [ADR-0047: World codecs](0047-world-codec.md) - accepted for M5.6 bounded canonical snapshots and detached edit-journal replay. diff --git a/docs/architecture/PERSISTENCE_ARCHITECTURE.md b/docs/architecture/PERSISTENCE_ARCHITECTURE.md index dcc5fde..986364d 100644 --- a/docs/architecture/PERSISTENCE_ARCHITECTURE.md +++ b/docs/architecture/PERSISTENCE_ARCHITECTURE.md @@ -142,3 +142,11 @@ host loss. Exact guarantees for acknowledged outcomes require an explicitly verified durable failure domain containing the complete causal recovery chain and referenced payloads before acknowledgment; restoring an older backup cannot silently undo permadeath. That admission/DR requirement is a World0 launch gate. + +## Implemented M5.6 local world codec boundary + +[ADR-0047](../adr/0047-world-codec.md) and [the codec contract](../world/WORLD_CODEC.md) +implement bounded canonical buffers carrying world/frame IDs, base versions, covered +command high-water, resident cells, debris and exact retry state. Detached replay +verifies predecessor and final state. This is in-memory codec validation, not the +production durability contract above. File publication/crash recovery is M5.7. diff --git a/docs/architecture/WORLD_ARCHITECTURE.md b/docs/architecture/WORLD_ARCHITECTURE.md index 62c44dd..96a94d3 100644 --- a/docs/architecture/WORLD_ARCHITECTURE.md +++ b/docs/architecture/WORLD_ARCHITECTURE.md @@ -141,3 +141,11 @@ request digest plus full-payload retry identity, and bounded history with a comm high-water mark. [The contract](../world/TERRAIN_EDITS.md) specifies failure semantics. No mass/density model or persistence is implied. Debris/retry state must accompany geometry in M5.6/5.7 recovery, and M6 must reconcile volume with chemical inventories. + +## Implemented M5.6 snapshot/replay boundary + +The [world codec](../world/WORLD_CODEC.md) preserves loaded membership, dirty cells, +versions, revisions, finite debris and retry history in one validated snapshot. +Journal replay uses the existing excavation transaction on a detached store. Dirty +chunks remain pinned; encoding does not acknowledge durability. M5.7 supplies local +file publication and compaction; no production persistence dependency was added. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 72aedd7..91d5dfd 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -1532,3 +1532,38 @@ claimed. The source/replacement commit operations have compile-time nonthrowing Linux CI and UBSan were not run locally. These are typed volume transfers, not a validated density/mass/chemistry model. No deployment or World0 modification occurred. Next: M5.6 canonical snapshots/edit journals preserving geometry, debris and retry state. + +## M5.6 canonical world snapshots and edit journals - 2026-09-26 + +Added ADR-0047, byte schemas, canonical snapshot/restore and detached atomic journal +replay. State includes base versions, identity, membership, revisions, geometry, +debris, successful edit count and exact bounded retry records. No existing golden +or biological simulation format changed. New snapshot v1 has an independently encoded +132-byte empty fixture from tools/world_snapshot_reference.py. + +Executed on Windows x64 / MSVC 19.44: + +- Release configure/build and 46/46 CTest passed on the final source/tests. +- Analysis configure/build (clang-tidy 19) and 46/46 CTest passed after explicit + size/iterator conversion fixes. Final history-corruption additions rebuilt and + passed both tests selected by `ctest --test-dir build/analysis -R world-codec + --output-on-failure`. +- ASan configure/build and 46/46 CTest passed. After final size/iterator/golden and + history-corruption additions, rebuilt ASan and reran both world-codec tests: passed. +- `tools/windows-check.ps1 -Preset release -Action format`: passed. +- `python tools/world_snapshot_reference.py`: generated independent golden bytes; + encoder equality and decoder acceptance pass in CTest. +- `git diff --check`: passed. + +New coverage includes 65-edit history rollover, deterministic continuation, historical +retry receipts, exact re-encoding, snapshot+journal replay, wrong predecessor/final +state, every fixture truncation, oversized/trailing data, version/identity/budget +rejection, recomputed-checksum geometry/debris/receipt/revision/history corruption, +and a deterministic 128-input semantic fuzz smoke. The fuzz harness mutates valid +fixtures and repairs checksums so validation below the envelope is exercised. + +Linux libFuzzer/UBSan jobs are wired for 10000 cases but were not executed locally. +Native Windows ASan is not UBSan. Full-size throughput, allocation-failure injection +and crash-atomic files remain unverified/later gates. Codecs write no files, clear no +dirty flags and issue no durable acknowledgements. No Argo/World0 operation occurred. +Next: M5.7 local snapshot/journal publication, compaction and recovery. diff --git a/docs/world/WORLD_CODEC.md b/docs/world/WORLD_CODEC.md new file mode 100644 index 0000000..a57d3a2 --- /dev/null +++ b/docs/world/WORLD_CODEC.md @@ -0,0 +1,82 @@ +# M5.6 world snapshot and edit-journal contract + +[ADR-0047](../adr/0047-world-codec.md) accepts bounded local byte codecs. Public API +is in evo/world/chunks.hpp; evo::world remains free of database, networking, rendering +and Kubernetes dependencies. No existing biology/particle snapshot format changed. + +## API and ownership + +- `store.snapshot(identity)` returns canonical bytes for the current coherent store. + Caller supplies nonzero world/frame IDs and owns their association with this store. +- `ChunkStore::restore(bytes, expected_identity, byte_budget)` returns a new owning + store only on full validation. Never replaces an existing live store in-place. +- `make_edit_journal(base, identity, requests, byte_budget)` validates/restores a base, + executes up to 64 fresh requests privately and encodes their expected result. +- `replay_edit_journal(base, journal, identity, byte_budget)` restores the base, applies + the whole bounded journal and verifies the final state before returning a new store. + +All are single-owner operations; input spans remain immutable/readable through return. +Use typed WorldCodecError for invalid input/budget/state. Allocation exceptions reach +the host. Views of the source store never become valid views of the restored store. +Budget is host policy, supplied explicitly and excluded from canonical state. Restore +preserves slot capacity, versions, revisions, successful edit count, debris and exact +bounded replay history. The snapshot covered sequence is command high-water, not a +tick or contiguous ordinal. Journal requests cannot contain retries or cache operations. + +```mermaid +flowchart LR + S[Canonical snapshot] --> V[Envelope and semantic validation] + V --> C[Detached compact store] + J[Ordered journal] --> R[Predecessor and command validation] + C --> R + R --> H[Final canonical checksum verification] + H --> O[Return owning candidate] +``` + +The snapshot carries all loaded chunks including clean ones because loaded membership +and revisions affect admitted edits. Unloaded clean regions remain procedural. Dirty +chunks cannot be evicted; all removed material therefore remains accountable against +resident geometry. No implicit clean/regenerate operation discards edits. + +## Version and conservation rules + +[Snapshot schema](../../schemas/world-snapshot.md) and +[journal schema](../../schemas/world-journal.md) specify every byte and bound. +Unknown versions, identities, materials and malformed/trailing data fail closed. +Restore compares cells against deterministic generation: no new fill or changed +material; clean chunks must match base. Summed removed volume equals stored debris +per material. Replay receipts and revision chains are checked; no digest-only retry. +All retained history can be undone without exceeding original base fill. + +FNV is an accidental-corruption check, not security. Untrusted/unauthenticated world +state is not authorized by a correct checksum. M5.7 and production infrastructure +must establish trusted manifests and complete journal coverage. Creating bytes alone +is neither durable acknowledgement nor permission to clear dirty flags. + +## Resource bounds and limitations + +Snapshot <=70 MiB, journal <=256 KiB, slots <=4096, retained/journal commands <=64, +cuts <=64 each, chunks per edit <=4. Restore validates framing/count bounds before +allocation and enforces the candidate's resident budget for metadata and payloads. +Codec input/output buffers are outside that budget: the host must separately admit +up to one input and one output snapshot plus journal and fixed codec workspace. +Largest explicit encoding workspace is a 4096-entry index array plus a 64-entry +history order array; restoration uses one 4096-word chunk and bounded history scans. +Reserve 64 KiB for codec workspace in addition to ordinary runtime/allocator overhead. +No compression or performance SLA is claimed; full restore recomputes base geometry +and checks at most 4096 retained cuts against later retained cuts. + +## Evidence and next step + +Tests cover independently encoded empty v1 bytes, canonical round trips, 65 edits +and history wrap, post-restore continuation/retries, seam journal replay, empty +segments, every truncation of fixtures, checksums, recomputed-checksum semantic +corruption, unknown versions, wrong identity, counts, trailing/oversized input and +insufficient memory. A 128-input deterministic fuzz smoke is in CTest; Linux CI adds +10000 libFuzzer cases. The harness repairs mutated checksums to exercise semantic +validation instead of stopping only at the envelope. Local Windows smoke is not a +claim that Linux coverage-guided fuzzing has executed. + +Next M5.7: publish/compact local files with verified replacement and failure injection. +M5.6 does not write files, fsync, delete journals, clear dirty state or recover a live +server process. M7 supplies the production durability/ownership path. diff --git a/include/evo/world/chunks.hpp b/include/evo/world/chunks.hpp index d4b3ebd..6fe3e4c 100644 --- a/include/evo/world/chunks.hpp +++ b/include/evo/world/chunks.hpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace evo::world { enum class ChunkError : std::uint8_t { @@ -64,6 +65,24 @@ struct EditReceipt { // Validates request shape; digest excludes the digest field itself. Not authentication. [[nodiscard]] std::expected excavation_digest(const ExcavationRequest& request) noexcept; +struct WorldIdentity { + std::uint64_t world{}, frame{}; + auto operator<=>(const WorldIdentity&) const = default; +}; +enum class WorldCodecError : std::uint8_t { + format, + version, + checksum, + identity, + bounds, + state, + budget, + edit, + predecessor +}; +using WorldBytes = std::vector; +inline constexpr std::size_t max_world_snapshot_bytes = std::size_t{70} * 1024 * 1024; +inline constexpr std::size_t max_world_journal_bytes = std::size_t{256} * 1024; class ChunkStore; // Immutable owned geometry. Construction does not authorize a terrain edit. class CompactCells { @@ -130,6 +149,9 @@ class ChunkStore { [[nodiscard]] std::expected evict(ChunkView view) noexcept; [[nodiscard]] std::expected excavate(const ExcavationRequest& request); [[nodiscard]] std::expected debris(MaterialId material) const noexcept; + [[nodiscard]] std::expected snapshot(WorldIdentity identity) const; + [[nodiscard]] static std::expected, WorldCodecError> + restore(std::span bytes, WorldIdentity identity, std::size_t byte_budget); [[nodiscard]] std::size_t accounted_bytes() const noexcept { return bytes_; } @@ -149,8 +171,15 @@ class ChunkStore { std::unique_ptr entries_; DebrisCapacity debris_capacity_; std::array debris_{}; - std::uint64_t last_command_{}; + std::uint64_t last_command_{}, committed_edits_{}; std::size_t history_next_{}; std::unique_ptr history_; }; +// Journals admit only fresh commands against the supplied loaded base, no cache operations. +[[nodiscard]] std::expected +make_edit_journal(std::span base, WorldIdentity identity, + std::span requests, std::size_t byte_budget); +[[nodiscard]] std::expected, WorldCodecError> +replay_edit_journal(std::span base, std::span journal, + WorldIdentity identity, std::size_t byte_budget); } // namespace evo::world diff --git a/schemas/world-edits.md b/schemas/world-edits.md index ebed05d..fe48d4d 100644 --- a/schemas/world-edits.md +++ b/schemas/world-edits.md @@ -18,7 +18,7 @@ FNV-1a uint64 offset=14695981039346656037, prime=1099511628211; for each byte b: Digest field is excluded. Canonical order is chunk lexicographic x/y/z then X-fast local ordinal, with no duplicates. Store identity/descriptor is bound by the target -store, not this digest; neither digest nor revision authenticates a caller. M5.6 must -add world/frame/base versions, bounds, integrity checks and transaction framing to -its separately reviewed persistent format. Geometry, debris and retry metadata form +store, not this digest; neither digest nor revision authenticates a caller. M5.6 now supplies +world/frame/base versions, bounds, integrity checks and transaction framing in +world-snapshot.md and world-journal.md. Geometry, debris and retry metadata form one consistent persistence unit; recording only a request digest cannot recover it. diff --git a/schemas/world-journal.md b/schemas/world-journal.md new file mode 100644 index 0000000..2394c1a --- /dev/null +++ b/schemas/world-journal.md @@ -0,0 +1,31 @@ +# Local edit journal v1 + +Scope: ordered fresh excavation requests against one already-loaded snapshot. Cache +load/evict/pin operations require a new snapshot and are not silently omitted replay. +All fields are explicit little endian; identity and digest trust limits match snapshot v1. + +| Offset | Field | Width | +|---:|---|---| +| 0 | Magic 0x314c4e4a57564541 | u64 | +| 8 | Version 1 | u32 | +| 12 | Total length including checksum | u32 | +| 16 | World, frame | 2*u64 | +| 32 | Predecessor snapshot checksum | u64 | +| 40 | Predecessor covered command high-water | u64 | +| 48 | Request count (0..64) | u32 | +| 52 | Requests | variable | +| before checksum | Resulting canonical snapshot checksum | u64 | +| final 8 | FNV-1a over all preceding bytes | u64 | + +Request: command u64, request digest u64, destination u32, count u32, then 48-byte +cuts defined in world-snapshot.md. No receipt is trusted from the journal: ordinary +excavation computes it. IDs strictly increase above the predecessor high-water; gaps +are allowed, retries are not journal events. Empty journal length is 68. Max length +is 256 KiB. Exact consumption is required; partial/trailing/oversized data is rejected. + +Restore predecessor into a detached store under an explicit host budget; verify +identity/checksum/high-water; apply every command; compare final canonical snapshot +checksum before returning. On any failure, discard the entire candidate. Input and +any existing live store remain unchanged. The caller must select a complete committed +journal chain via M5.7 manifests; codec validation cannot know about omitted external +segments. No files, fsync, crash-atomic publication or durable ACK are provided here. diff --git a/schemas/world-snapshot.md b/schemas/world-snapshot.md new file mode 100644 index 0000000..d6a52b6 --- /dev/null +++ b/schemas/world-snapshot.md @@ -0,0 +1,58 @@ +# Local world snapshot v1 + +Scope: ADR-0047; canonical bounded buffers, not native structs or a database format. +All integers are unsigned little endian. Signed chunk axes encode modulo 2^64 and +decode without implementation-defined unsigned-to-signed overflow. No optional fields. + +| Offset | Field | Width | +|---:|---|---| +| 0 | Magic 0x31504e5357564541 | u64 | +| 8 | Format version 1 | u32 | +| 12 | Total byte length including checksum | u32 | +| 16 | World ID, frame ID (both nonzero) | 2*u64 | +| 32 | Base seed | u64 | +| 40 | Coordinate, generator, material versions; edge | 4*u32 | +| 56 | Slot capacity; resident chunk count | 2*u32 | +| 64 | Local revision high-water | u64 | +| 72 | Covered command high-water (IDs can have gaps) | u64 | +| 80 | Number of successful fresh edits since store creation | u64 | +| 88 | Rock and regolith debris capacities | 2*u64 | +| 104 | Rock and regolith debris balances | 2*u64 | +| 120 | Retained request count, min(successful edits,64) | u32 | +| 124 | Chunk records, then replay records | variable | +| final 8 | FNV-1a checksum of all preceding bytes | u64 | + +Chunk records are strictly lexicographic by signed x/y/z: address 3*u64, revision +u64, dirty u8 (0 or 1), then 4096 X-fast logical cells (material u16, fill u16). +Each chunk is 16417 bytes; native resident compression does not affect encoding. +Revisions are nonzero, unique among current chunks and <= revision high-water. +All geometry must be an excavation-only reduction of the recorded base. Clean chunks +must equal the base. Dirty pins can still contain unchanged base geometry. + +Replay records are strictly increasing command ID: command u64, request digest u64, +destination u32, cut count u32, followed by cuts. Each cut is address 3*u64, local +x/y/z 3*u32, expected revision u64, removed fill u32 (48 bytes). Then receipt removed +volumes 2*u64, receipt chunk count u32, and receipt chunks (address 3*u64, revision +u64). This is 44+48*cuts+32*receipt_chunks bytes. Digest rules are in world-edits.md. +Unused array tails, directory slots, replay ring cursor, pointers, C++ padding and +host byte budget are not encoded. Restore rebuilds compact data and canonical ring +ordering. Recipe/version and logical geometry are preserved, not allocation layout. + +Bounds: 1..4096 slots, resident count <= slots, 0..64 replay records, 1..64 cuts per +record, 1..4 receipt chunks. Empty snapshot length is 132. Total <=70 MiB, with exact +length/no trailing bytes. Validate envelope before counts allocate state. Compare +supplied world/frame IDs exactly and reject unsupported base/format versions. + +FNV-1a uses offset 14695981039346656037, prime 1099511628211, unsigned 64-bit wrap. +It detects accidental corruption and is NOT authentication. Durable manifests need +trusted provenance and a later strong digest; never authorize a client from this hash. + +Semantic checks include terrain loss==debris per material, balance<=capacity, +request/receipt consistency, history counts/high-water, receipt revision chronology, +current/next-request revision linkage, and undo feasibility of retained cuts. When +all edits remain retained, receipt totals equal debris; otherwise they cannot exceed +it. Edited history chunks cannot be absent or clean. Serialization never clears dirty. + +Independent empty fixture: tools/world_snapshot_reference.py and + tests/fixtures/world_empty_v1.hpp. Changes to bytes/semantics require format/version +review; do not update a golden merely to satisfy tests. diff --git a/src/world/codec.cpp b/src/world/codec.cpp new file mode 100644 index 0000000..81950e3 --- /dev/null +++ b/src/world/codec.cpp @@ -0,0 +1,524 @@ +#include "chunk_state.hpp" +#include +#include +#include +#include + +namespace evo::world { +namespace { +constexpr std::uint64_t snapshot_magic = 0x31504e5357564541ULL; +constexpr std::uint64_t journal_magic = 0x314c4e4a57564541ULL; +std::uint64_t checksum(std::span data) noexcept { + std::uint64_t h = 14695981039346656037ULL; + for (const auto byte : data) { + h = (h ^ byte) * 1099511628211ULL; + } + return h; +} +struct Reader { + std::span data; + std::size_t at{}; + bool ok{true}; + std::uint64_t get(unsigned width) noexcept { + if (!ok || width > data.size() - at) { + ok = false; + return 0; + } + std::uint64_t value = 0; + for (unsigned i = 0; i < width; ++i) { + value |= static_cast(data[at++]) << (8U * i); + } + return value; + } + std::uint32_t u32() noexcept { + return static_cast(get(4)); + } + std::int64_t signed64() noexcept { + const auto value = get(8); + return value <= static_cast(std::numeric_limits::max()) + ? static_cast(value) + : -1 - static_cast(~value); + } + ChunkAddress address() noexcept { + return {signed64(), signed64(), signed64()}; + } +}; +struct Writer { + WorldBytes data; + explicit Writer(std::size_t bytes) { + data.reserve(bytes); + } + void put(std::uint64_t value, unsigned width) { + for (unsigned i = 0; i < width; ++i) { + data.push_back(static_cast(value & 255U)); + value >>= 8U; + } + } + void address(ChunkAddress a) { + put(static_cast(a.x), 8); + put(static_cast(a.y), 8); + put(static_cast(a.z), 8); + } + void header(std::uint64_t magic, WorldIdentity id) { + put(magic, 8); + put(1, 4); + put(0, 4); + put(id.world, 8); + put(id.frame, 8); + } + WorldBytes finish() { + const auto length = data.size() + 8; + for (unsigned i = 0; i < 4; ++i) { + data[12 + i] = static_cast((length >> (8U * i)) & 255U); + } + const auto hash = checksum(data); + put(hash, 8); + return std::move(data); + } +}; +std::expected open(std::span bytes, + std::uint64_t magic, std::size_t maximum, + WorldIdentity id) noexcept { + if (bytes.size() < 40 || bytes.size() > maximum) { + return std::unexpected{WorldCodecError::bounds}; + } + Reader r{bytes.first(bytes.size() - 8)}; + if (r.get(8) != magic) { + return std::unexpected{WorldCodecError::format}; + } + if (r.u32() != 1) { + return std::unexpected{WorldCodecError::version}; + } + if (r.u32() != bytes.size()) { + return std::unexpected{WorldCodecError::bounds}; + } + Reader tail{bytes.last(8)}; + if (checksum(r.data) != tail.get(8)) { + return std::unexpected{WorldCodecError::checksum}; + } + const WorldIdentity stored{r.get(8), r.get(8)}; + if (id.world == 0 || id.frame == 0 || stored != id) { + return std::unexpected{WorldCodecError::identity}; + } + return r; +} +void write_request(Writer& w, const ExcavationRequest& r) { + w.put(r.command, 8); + w.put(r.digest, 8); + w.put(r.destination, 4); + w.put(r.cuts.size(), 4); + for (const auto& cut : r.cuts) { + w.address(cut.chunk); + w.put(cut.local.x, 4); + w.put(cut.local.y, 4); + w.put(cut.local.z, 4); + w.put(cut.expected_revision, 8); + w.put(cut.removed_fill, 4); + } +} +std::expected +read_request(Reader& r, std::array& cuts) noexcept { + ExcavationRequest request{}; + request.command = r.get(8); + request.digest = r.get(8); + request.destination = r.u32(); + const auto count = r.u32(); + if (!r.ok || count == 0 || count > 64 || + static_cast(count) * 48 > r.data.size() - r.at) { + return std::unexpected{WorldCodecError::bounds}; + } + for (std::size_t i = 0; i < count; ++i) { + cuts[i] = {r.address(), {r.u32(), r.u32(), r.u32()}, r.get(8), r.u32()}; + } + request.cuts = std::span{cuts}.first(count); + const auto hash = excavation_digest(request); + if (!r.ok || !hash || *hash != request.digest) { + return std::unexpected{WorldCodecError::state}; + } + return request; +} +void write_receipt(Writer& w, const EditReceipt& r) { + for (const auto amount : r.removed) { + w.put(amount.value, 8); + } + w.put(r.chunk_count, 4); + for (std::size_t i = 0; i < r.chunk_count; ++i) { + w.address(r.chunks[i].address); + w.put(r.chunks[i].revision, 8); + } +} +std::uint64_t covered(std::span bytes) noexcept { + Reader r{bytes.subspan(72, 8)}; + return r.get(8); // only after successful snapshot restore +} +std::uint64_t fingerprint(std::span bytes) noexcept { + Reader r{bytes.last(8)}; + return r.get(8); // only after envelope validation +} +} // namespace +std::expected ChunkStore::snapshot(WorldIdentity identity) const { + if (identity.world == 0 || identity.frame == 0) { + return std::unexpected{WorldCodecError::identity}; + } + std::array order{}; + std::size_t count = 0; + for (std::size_t i = 0; i < slots_; ++i) { + if (entries_[i].cells) { + order[count++] = i; + } + } + std::ranges::sort(std::span{order}.first(count), + [&](auto a, auto b) { return entries_[a].address < entries_[b].address; }); + std::array history{}; + std::size_t records = 0, length = 132 + count * (33 + 16384); + for (std::size_t i = 0; i < retained_commands; ++i) { + if (history_[i].receipt.command != 0) { + history[records++] = i; + length += 44 + 48 * history_[i].count + 32 * history_[i].receipt.chunk_count; + } + } + std::ranges::sort(std::span{history}.first(records), [&](auto a, auto b) { + return history_[a].receipt.command < history_[b].receipt.command; + }); + if (length > max_world_snapshot_bytes) { + return std::unexpected{WorldCodecError::bounds}; + } + Writer w{length}; + w.header(snapshot_magic, identity); + w.put(descriptor_.seed, 8); + w.put(descriptor_.coordinate_version, 4); + w.put(descriptor_.generator_version, 4); + w.put(descriptor_.material_version, 4); + w.put(descriptor_.edge, 4); + w.put(slots_, 4); + w.put(count, 4); + w.put(revision_, 8); + w.put(last_command_, 8); + w.put(committed_edits_, 8); + for (const auto cap : debris_capacity_.materials) { + w.put(cap.value, 8); + } + for (const auto amount : debris_) { + w.put(amount.value, 8); + } + w.put(records, 4); + for (std::size_t i = 0; i < count; ++i) { + const auto& entry = entries_[order[i]]; + w.address(entry.address); + w.put(entry.revision, 8); + w.put(entry.dirty ? 1 : 0, 1); + if (!entry.cells) { + return std::unexpected{WorldCodecError::state}; + } + for (std::uint32_t j = 0; j < base_chunk_cells; ++j) { + const auto cell = *entry.cells->read(j); + w.put(cell.material().value, 2); + w.put(cell.fill(), 2); + } + } + for (std::size_t i = 0; i < records; ++i) { + const auto& h = history_[history[i]]; + write_request(w, {h.receipt.command, h.destination, std::span{h.cuts}.first(h.count), + h.receipt.digest}); + write_receipt(w, h.receipt); + } + return w.finish(); +} +std::expected, WorldCodecError> +ChunkStore::restore(std::span bytes, WorldIdentity identity, + std::size_t budget) { + const auto envelope = open(bytes, snapshot_magic, max_world_snapshot_bytes, identity); + if (!envelope) { + return std::unexpected{envelope.error()}; + } + auto r = *envelope; + const BaseDescriptor descriptor{r.get(8), r.u32(), r.u32(), r.u32(), r.u32()}; + const auto slots = r.u32(), count = r.u32(); + const auto revision = r.get(8), command = r.get(8), commits = r.get(8); + const DebrisCapacity capacity{{VolumeUnits{r.get(8)}, VolumeUnits{r.get(8)}}}; + const std::array debris{VolumeUnits{r.get(8)}, VolumeUnits{r.get(8)}}; + const auto records = r.u32(); + if (!r.ok || slots == 0 || slots > max_slots || count > slots || records > retained_commands || + static_cast(count) * (33 + 16384) > r.data.size() - r.at) { + return std::unexpected{WorldCodecError::bounds}; + } + if (records != std::min(commits, retained_commands) || commits > command || + ((commits == 0) != (command == 0)) || debris[0].value > capacity.materials[0].value || + debris[1].value > capacity.materials[1].value) { + return std::unexpected{WorldCodecError::state}; + } + auto made = create(descriptor, slots, budget, capacity); + if (!made) { + return std::unexpected{made.error() == ChunkError::memory_budget + ? WorldCodecError::budget + : WorldCodecError::version}; + } + auto& store = **made; + std::array removed{}; + for (std::size_t i = 0; i < count; ++i) { + auto& entry = store.entries_[i]; + entry.address = r.address(); + entry.revision = r.get(8); + const auto dirty = r.get(1); + const auto origin = join({entry.address, {}}, base_chunk_edge); + if (!r.ok || !origin || dirty > 1 || entry.revision == 0 || entry.revision > revision || + (i > 0 && entry.address <= store.entries_[i - 1].address)) { + return std::unexpected{WorldCodecError::state}; + } + for (std::size_t j = 0; j < i; ++j) { + if (store.entries_[j].revision == entry.revision) { + return std::unexpected{WorldCodecError::state}; + } + } + entry.dirty = dirty == 1; + std::array words{}; + for (std::uint32_t j = 0; j < base_chunk_cells; ++j) { + const auto material = static_cast(r.get(2)); + const auto fill = static_cast(r.get(2)); + const auto cell = CellSample::create({material}, fill); + const auto local = *local_from_ordinal(j, base_chunk_edge); + const auto base = *base_sample( + descriptor, {origin->x + local.x, origin->y + local.y, origin->z + local.z}); + if (!r.ok || !cell || (material != 0 && material != base.material().value) || + fill > base.fill() || (!entry.dirty && *cell != base)) { + return std::unexpected{WorldCodecError::state}; + } + if (base.material().value != 0) { + removed[base.material().value - 1] += base.fill() - fill; + } + words[j] = material | (fill << 16U); + } + const auto payload = CompactCells::required_bytes(words); + if (payload > budget - store.bytes_) { + return std::unexpected{WorldCodecError::budget}; + } + entry.cells.emplace(CompactCells::encode(words)); + store.bytes_ += payload; + ++store.size_; + } + if (removed[0] != debris[0].value || removed[1] != debris[1].value) { + return std::unexpected{WorldCodecError::state}; + } + std::uint64_t prior_command = 0, prior_revision = 0; + for (std::size_t i = 0; i < records; ++i) { + auto& h = store.history_[i]; + const auto request = read_request(r, h.cuts); + if (!request) { + return std::unexpected{request.error()}; + } + h.count = request->cuts.size(); + h.destination = request->destination; + h.receipt.command = request->command; + h.receipt.digest = request->digest; + h.receipt.removed = {VolumeUnits{r.get(8)}, VolumeUnits{r.get(8)}}; + h.receipt.chunk_count = r.u32(); + if (!r.ok || h.receipt.chunk_count == 0 || h.receipt.chunk_count > 4 || + request->command <= prior_command || request->command > command) { + return std::unexpected{WorldCodecError::state}; + } + std::size_t expected_chunks = 0; + std::array amounts{}; + for (std::size_t j = 0; j < h.count; ++j) { + const auto& cut = h.cuts[j]; + const auto at = join({cut.chunk, cut.local}, base_chunk_edge); + const auto base = *base_sample(descriptor, *at); + if (base.material().value == 0) { + return std::unexpected{WorldCodecError::state}; + } + amounts[base.material().value - 1] += cut.removed_fill; + if (j == 0 || cut.chunk != h.cuts[j - 1].chunk) { + if (expected_chunks >= h.receipt.chunk_count) { + return std::unexpected{WorldCodecError::state}; + } + auto& edited = h.receipt.chunks[expected_chunks]; + edited = {r.address(), r.get(8)}; + if (edited.address != cut.chunk || edited.revision <= prior_revision || + edited.revision > revision || + (expected_chunks > 0 && + edited.revision != h.receipt.chunks[expected_chunks - 1].revision + 1)) { + return std::unexpected{WorldCodecError::state}; + } + ++expected_chunks; + } + } + if (!r.ok || expected_chunks != h.receipt.chunk_count || + amounts[0] != h.receipt.removed[0].value || amounts[1] != h.receipt.removed[1].value) { + return std::unexpected{WorldCodecError::state}; + } + for (const auto& cut : request->cuts) { + if (cut.expected_revision >= h.receipt.chunks[0].revision) { + return std::unexpected{WorldCodecError::state}; + } + } + prior_revision = h.receipt.chunks[h.receipt.chunk_count - 1].revision; + prior_command = request->command; + } + if (!r.ok || r.at != r.data.size() || prior_command != command) { + return std::unexpected{WorldCodecError::state}; + } + // Verify history against current geometry and subsequent requests without storing + // an unbounded undo world. At most 4096 retained cuts participate in these scans. + for (std::size_t i = 0; i < records; ++i) { + const auto& h = store.history_[i]; + for (std::size_t j = 0; j < h.receipt.chunk_count; ++j) { + const auto edited = h.receipt.chunks[j]; + const auto token = store.view(edited.address); + if (!token || !store.entries_[token->slot_].dirty) { + return std::unexpected{WorldCodecError::state}; + } + auto next_revision = token->revision(); + for (std::size_t later = i + 1; later < records; ++later) { + const auto& next = store.history_[later]; + bool found = false; + for (std::size_t k = 0; k < next.count; ++k) { + if (next.cuts[k].chunk == edited.address) { + next_revision = next.cuts[k].expected_revision; + found = true; + break; + } + } + if (found) { + break; + } + } + if (next_revision != edited.revision) { + return std::unexpected{WorldCodecError::state}; + } + for (std::size_t k = 0; k < count; ++k) { + if (store.entries_[k].revision == edited.revision && + store.entries_[k].address != edited.address) { + return std::unexpected{WorldCodecError::state}; + } + } + } + for (std::size_t j = 0; j < h.count; ++j) { + const auto& cut = h.cuts[j]; + const auto token = store.view(cut.chunk); + if (!token) { + return std::unexpected{WorldCodecError::state}; + } + const auto current = store.read(*token, cut.local); + if (!current) { + return std::unexpected{WorldCodecError::state}; + } + std::uint64_t undo = current->fill(); + for (std::size_t later = i; later < records; ++later) { + const auto& next = store.history_[later]; + for (std::size_t k = 0; k < next.count; ++k) { + if (next.cuts[k].chunk == cut.chunk && next.cuts[k].local == cut.local) { + undo += next.cuts[k].removed_fill; + } + } + } + const auto base = + *base_sample(descriptor, *join({cut.chunk, cut.local}, base_chunk_edge)); + if (undo > base.fill()) { + return std::unexpected{WorldCodecError::state}; + } + } + } + std::array recorded_removed{}; + for (std::size_t i = 0; i < records; ++i) { + for (std::size_t m = 0; m < 2; ++m) { + recorded_removed[m] += store.history_[i].receipt.removed[m].value; + } + } + for (std::size_t m = 0; m < 2; ++m) { + if (recorded_removed[m] > debris[m].value || + (commits <= retained_commands && recorded_removed[m] != debris[m].value)) { + return std::unexpected{WorldCodecError::state}; + } + } + if (commits > revision) { + return std::unexpected{WorldCodecError::state}; + } + store.revision_ = revision; + store.last_command_ = command; + store.committed_edits_ = commits; + store.debris_ = debris; + store.history_next_ = records % retained_commands; + return std::move(*made); +} +std::expected +make_edit_journal(std::span base, WorldIdentity identity, + std::span requests, std::size_t budget) { + if (requests.size() > 64) { + return std::unexpected{WorldCodecError::bounds}; + } + auto store = ChunkStore::restore(base, identity, budget); + if (!store) { + return std::unexpected{store.error()}; + } + std::size_t length = 68; + for (const auto& r : requests) { + if (r.cuts.size() > 64) { + return std::unexpected{WorldCodecError::bounds}; + } + length += 24 + 48 * r.cuts.size(); + } + Writer w{length}; + w.header(journal_magic, identity); + w.put(fingerprint(base), 8); + w.put(covered(base), 8); + w.put(requests.size(), 4); + auto last = covered(base); + for (const auto& r : requests) { + if (r.command <= last || !(**store).excavate(r)) { + return std::unexpected{WorldCodecError::edit}; + } + write_request(w, r); + last = r.command; + } + const auto final = (**store).snapshot(identity); + if (!final) { + return std::unexpected{final.error()}; + } + w.put(fingerprint(*final), 8); + return w.finish(); +} +std::expected, WorldCodecError> +replay_edit_journal(std::span base, std::span journal, + WorldIdentity identity, std::size_t budget) { + const auto envelope = open(journal, journal_magic, max_world_journal_bytes, identity); + if (!envelope) { + return std::unexpected{envelope.error()}; + } + auto r = *envelope; + const auto previous = r.get(8), last = r.get(8); + const auto count = r.u32(); + if (!r.ok || count > 64 || static_cast(count) * 72 + 8 > r.data.size() - r.at) { + return std::unexpected{WorldCodecError::bounds}; + } + auto store = ChunkStore::restore(base, identity, budget); + if (!store) { + return std::unexpected{store.error()}; + } + if (previous != fingerprint(base) || last != covered(base)) { + return std::unexpected{WorldCodecError::predecessor}; + } + auto prior = last; + for (std::size_t i = 0; i < count; ++i) { + std::array cuts{}; + const auto request = read_request(r, cuts); + if (!request) { + return std::unexpected{request.error()}; + } + if (request->command <= prior || !(**store).excavate(*request)) { + return std::unexpected{WorldCodecError::edit}; + } + prior = request->command; + } + const auto expected = r.get(8); + if (!r.ok || r.at != r.data.size()) { + return std::unexpected{WorldCodecError::format}; + } + const auto final = (**store).snapshot(identity); + if (!final) { + return std::unexpected{final.error()}; + } + if (fingerprint(*final) != expected) { + return std::unexpected{WorldCodecError::state}; + } + return std::move(*store); +} +} // namespace evo::world diff --git a/src/world/edits.cpp b/src/world/edits.cpp index 95b2860..f95f671 100644 --- a/src/world/edits.cpp +++ b/src/world/edits.cpp @@ -167,6 +167,7 @@ std::expected ChunkStore::excavate(const ExcavationRequ record.receipt = receipt; history_next_ = (history_next_ + 1) % retained_commands; last_command_ = r.command; + ++committed_edits_; return receipt; } } // namespace evo::world diff --git a/tests/fixtures/world_empty_v1.hpp b/tests/fixtures/world_empty_v1.hpp new file mode 100644 index 0000000..a081c02 --- /dev/null +++ b/tests/fixtures/world_empty_v1.hpp @@ -0,0 +1,15 @@ +#pragma once +#include +#include +// Frozen from tools/world_snapshot_reference.py; format v1. +inline constexpr std::array world_empty_v1{ + 0x41, 0x45, 0x56, 0x57, 0x53, 0x4e, 0x50, 0x31, 0x01, 0x00, 0x00, 0x00, 0x84, 0x00, 0x00, + 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x42, + 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x42, 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x50, 0x84, 0x9a, 0xf6, 0x48, 0x9d, 0x21, 0x96, +}; diff --git a/tests/fuzz/world_codec_fuzz.cpp b/tests/fuzz/world_codec_fuzz.cpp new file mode 100644 index 0000000..710be6a --- /dev/null +++ b/tests/fuzz/world_codec_fuzz.cpp @@ -0,0 +1,91 @@ +#include "evo/world/chunks.hpp" +#include +#include +#include +#include +#include +#include + +namespace { +using namespace evo::world; +constexpr WorldIdentity identity{7, 9}; +constexpr std::size_t budget = 2000000; +struct Seeds { + WorldBytes snapshot, journal; +}; +Seeds seeds() { + auto store = ChunkStore::create({}, 2, budget, {{VolumeUnits{1000000}, VolumeUnits{1000000}}}); + if (!store) { + std::abort(); + } + const auto view = (**store).load({0, 0, -2}); + if (!view) { + std::abort(); + } + std::array cuts{{{{0, 0, -2}, {}, view->revision(), 10}}}; + ExcavationRequest r{1, 1, cuts, 0}; + r.digest = *excavation_digest(r); + if (!(**store).excavate(r)) { + std::abort(); + } + const auto base = (**store).snapshot(identity); + if (!base) { + std::abort(); + } + cuts[0].expected_revision = (**store).view({0, 0, -2})->revision(); + r.command = 2; + r.digest = *excavation_digest(r); + const std::array requests{r}; + const auto journal = make_edit_journal(*base, identity, requests, budget); + if (!journal) { + std::abort(); + } + return {*base, *journal}; +} +WorldBytes mutate(WorldBytes bytes, std::span input) { + // Triples choose bounded offsets and replacement bytes; repair the envelope + // checksum so fuzz mutations exercise semantic validation as well as framing. + for (std::size_t i = 0; i + 2 < input.size(); i += 3) { + const auto index = + (static_cast(input[i]) * 256 + input[i + 1]) % (bytes.size() - 8); + bytes[index] = input[i + 2]; + } + std::uint64_t hash = 14695981039346656037ULL; + for (std::size_t i = 0; i < bytes.size() - 8; ++i) { + hash = (hash ^ bytes[i]) * 1099511628211ULL; + } + for (unsigned i = 0; i < 8; ++i) { + bytes[bytes.size() - 8 + i] = static_cast((hash >> (8U * i)) & 255U); + } + return bytes; +} +void check_snapshot(std::span bytes) { + auto restored = ChunkStore::restore(bytes, identity, budget); + if (restored) { + const auto canonical = (**restored).snapshot(identity); + if (!canonical || + !std::equal(canonical->begin(), canonical->end(), bytes.begin(), bytes.end())) { + std::abort(); + } + } +} +} // namespace +extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, std::size_t size) { + if (size > 4096) { + return 0; + } + const auto input = std::span{data, size}; + static const auto fixtures = seeds(); + check_snapshot(input); + const auto mutated = mutate(fixtures.snapshot, input); + check_snapshot(mutated); + const auto journal = mutate(fixtures.journal, input); + auto replay = replay_edit_journal(fixtures.snapshot, journal, identity, budget); + if (replay) { + const auto bytes = (**replay).snapshot(identity); + if (!bytes || !ChunkStore::restore(*bytes, identity, budget)) { + std::abort(); + } + } + return 0; +} diff --git a/tests/fuzz/world_codec_smoke.cpp b/tests/fuzz/world_codec_smoke.cpp new file mode 100644 index 0000000..a2eec9e --- /dev/null +++ b/tests/fuzz/world_codec_smoke.cpp @@ -0,0 +1,18 @@ +#include "evo/core/random.hpp" +#include +#include +#include +extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t*, std::size_t); +int main() { + std::array bytes{}; + evo::Random random{123456}; + for (std::size_t sample = 0; sample < 128; ++sample) { + for (auto& byte : bytes) { + byte = static_cast(random.next() & 255U); + } + if (LLVMFuzzerTestOneInput(bytes.data(), sample % 97) != 0) { + return 1; + } + } + return 0; +} diff --git a/tests/world_codec_tests.cpp b/tests/world_codec_tests.cpp new file mode 100644 index 0000000..809c739 --- /dev/null +++ b/tests/world_codec_tests.cpp @@ -0,0 +1,233 @@ +#include "evo/world/chunks.hpp" +#include "fixtures/world_empty_v1.hpp" +#include +#include +#include +#include +#include + +namespace { +using namespace evo::world; +constexpr WorldIdentity identity{7, 9}; +constexpr std::size_t budget = 2000000; +void check(bool condition, std::source_location at = std::source_location::current()) { + if (!condition) { + throw std::runtime_error("world codec line " + std::to_string(at.line())); + } +} +#define CHECK(...) check(static_cast(__VA_ARGS__)) +std::unique_ptr store() { + auto made = ChunkStore::create({}, 4, budget, {{VolumeUnits{1000000}, VolumeUnits{1000000}}}); + CHECK(made); + return std::move(*made); +} +void put(WorldBytes& b, std::size_t offset, std::uint64_t n, unsigned width) { + for (unsigned i = 0; i < width; ++i) { + b.at(offset + i) = static_cast(n & 255U); + n >>= 8U; + } +} +void seal(WorldBytes& bytes) { + put(bytes, 12, bytes.size(), 4); + std::uint64_t h = 14695981039346656037ULL; + for (std::size_t i = 0; i < bytes.size() - 8; ++i) { + h = (h ^ bytes[i]) * 1099511628211ULL; + } + put(bytes, bytes.size() - 8, h, 8); +} +ExcavationRequest request(std::uint64_t id, std::span cuts) { + ExcavationRequest r{id, 1, cuts, 0}; + const auto hash = excavation_digest(r); + CHECK(hash); + r.digest = *hash; + return r; +} +void round_trip_and_history() { + auto s = store(); + CHECK(s->load({-1, 0, -2})); + CHECK(s->load({0, 0, -2})); + std::array first{}; + std::array latest{}; + EditReceipt receipt{}; + for (std::uint64_t id = 1; id <= 65; ++id) { + latest = {CellCut{{-1, 0, -2}, {15, 0, 0}, s->view({-1, 0, -2})->revision(), 1}, + CellCut{{0, 0, -2}, {0, 0, 0}, s->view({0, 0, -2})->revision(), 2}}; + if (id == 1) { + first = latest; + } + const auto done = s->excavate(request(id, latest)); + CHECK(done); + receipt = *done; + } + const auto pinned = s->load({5, 5, 5}); + CHECK(pinned && s->mark_dirty(*pinned)); + const auto clean = s->load({-5, -5, 5}); + CHECK(clean); + const auto bytes = s->snapshot(identity); + CHECK(bytes); + auto restored = ChunkStore::restore(*bytes, identity, budget); + CHECK(restored); + auto& copy = **restored; + CHECK(copy.snapshot(identity) == bytes); + CHECK(copy.debris({1}) == s->debris({1})); + CHECK(*copy.excavate(request(65, latest)) == receipt); + const auto stale = copy.excavate(request(1, first)); + CHECK(!stale && stale.error() == ChunkError::stale_command); + latest[0].removed_fill = 2; + const auto conflict = copy.excavate(request(65, latest)); + CHECK(!conflict && conflict.error() == ChunkError::command_conflict); + latest[0].expected_revision = copy.view({-1, 0, -2})->revision(); + latest[1].expected_revision = copy.view({0, 0, -2})->revision(); + CHECK(copy.excavate(request(66, latest)) == s->excavate(request(66, latest))); + CHECK(copy.snapshot(identity) == s->snapshot(identity)); + CHECK(!copy.evict(*copy.view({-1, 0, -2}))); + CHECK(copy.evict(*copy.view({-5, -5, 5}))); +} +void journals() { + auto s = store(); + CHECK(s->load({-1, 0, -2})); + CHECK(s->load({0, 0, -2})); + const auto base = s->snapshot(identity); + CHECK(base); + std::array a{{{{-1, 0, -2}, {15, 0, 0}, s->view({-1, 0, -2})->revision(), 10}, + {{0, 0, -2}, {0, 0, 0}, s->view({0, 0, -2})->revision(), 20}}}; + const auto one = request(3, a); + CHECK(s->excavate(one)); + auto b = a; + for (auto& cut : b) { + cut.expected_revision = s->view(cut.chunk)->revision(); + } + const auto two = request(8, b); + CHECK(s->excavate(two)); + const std::array batch{one, two}; + const auto journal = make_edit_journal(*base, identity, batch, budget); + CHECK(journal); + auto replay = replay_edit_journal(*base, *journal, identity, budget); + CHECK(replay); + CHECK((**replay).snapshot(identity) == s->snapshot(identity)); + const auto nextbase = (**replay).snapshot(identity); + CHECK(nextbase); + const auto empty = make_edit_journal(*nextbase, identity, {}, budget); + CHECK(empty); + const auto unchanged = replay_edit_journal(*nextbase, *empty, identity, budget); + CHECK(unchanged); + CHECK((**unchanged).snapshot(identity) == nextbase); + CHECK(!replay_edit_journal(*nextbase, *journal, identity, budget)); + CHECK(!make_edit_journal(*nextbase, identity, batch, budget)); + for (std::size_t n = 0; n < journal->size(); ++n) { + CHECK(!replay_edit_journal(*base, std::span{*journal}.first(n), identity, budget)); + } + auto corrupt = *journal; + corrupt[corrupt.size() - 16] ^= 1; + seal(corrupt); + CHECK(!replay_edit_journal(*base, corrupt, identity, budget)); + // A valid first request followed by stale state must not publish a partial candidate. + const std::array bad{one, request(8, a)}; + CHECK(!make_edit_journal(*base, identity, bad, budget)); + CHECK(ChunkStore::restore(*base, identity, budget)); +} +void malformed() { + auto s = store(); + const auto v = s->load({0, 0, -2}); + CHECK(v); + const std::array cuts{{{{0, 0, -2}, {}, v->revision(), 100}}}; + CHECK(s->excavate(request(1, cuts))); + const auto saved = s->snapshot(identity); + CHECK(saved); + for (std::size_t n = 0; n < saved->size(); ++n) { + CHECK(!ChunkStore::restore(std::span{*saved}.first(n), identity, budget)); + } + CHECK(!ChunkStore::restore(*saved, {8, 9}, budget)); + CHECK(!s->snapshot({0, 9})); + CHECK(!ChunkStore::restore(*saved, identity, *ChunkStore::minimum_bytes(4))); + auto corrupt = *saved; + corrupt[200] ^= 1; + CHECK(!ChunkStore::restore(corrupt, identity, budget)); + // Semantic failures must reject even when the corruption checksum is recomputed. + for (const auto offset : {8U, 40U, 44U, 48U, 52U}) { + corrupt = *saved; + put(corrupt, offset, 99, 4); + seal(corrupt); + CHECK(!ChunkStore::restore(corrupt, identity, budget)); + } + for (const auto offset : {56U, 60U, 120U}) { + corrupt = *saved; + put(corrupt, offset, 0xffffffffU, 4); + seal(corrupt); + CHECK(!ChunkStore::restore(corrupt, identity, budget)); + } + corrupt = *saved; + put(corrupt, 104, 101, 8); + seal(corrupt); + CHECK(!ChunkStore::restore(corrupt, identity, budget)); + corrupt = *saved; + put(corrupt, 80, 0, 8); + seal(corrupt); + CHECK(!ChunkStore::restore(corrupt, identity, budget)); + corrupt = *saved; + put(corrupt, 124, 65536, 8); + seal(corrupt); + CHECK(!ChunkStore::restore(corrupt, identity, budget)); + corrupt = *saved; + corrupt[156] = 0; + seal(corrupt); + CHECK(!ChunkStore::restore(corrupt, identity, budget)); + corrupt = *saved; + put(corrupt, 157, 3, 2); + seal(corrupt); + CHECK(!ChunkStore::restore(corrupt, identity, budget)); + corrupt = *saved; + corrupt.insert(corrupt.end() - 8, 0); + seal(corrupt); + CHECK(!ChunkStore::restore(corrupt, identity, budget)); + // First retained record follows the one canonical chunk. Preserve checksum + // validity while breaking receipt accounting or revision chronology. + constexpr std::size_t history_start = 124 + 16417; + corrupt = *saved; + put(corrupt, history_start + 24 + 48, 101, 8); + seal(corrupt); + CHECK(!ChunkStore::restore(corrupt, identity, budget)); + corrupt = *saved; + put(corrupt, history_start + 24 + 48 + 20 + 24, 1, 8); + seal(corrupt); + CHECK(!ChunkStore::restore(corrupt, identity, budget)); + // Removing retry history must not yield an apparently valid genesis snapshot + // while edited geometry and its debris remain. + corrupt = *saved; + corrupt.resize(history_start + 8); + put(corrupt, 72, 0, 8); + put(corrupt, 80, 0, 8); + put(corrupt, 120, 0, 4); + seal(corrupt); + CHECK(!ChunkStore::restore(corrupt, identity, budget)); + WorldBytes huge(max_world_snapshot_bytes + 1, 0); + CHECK(!ChunkStore::restore(huge, identity, budget)); + CHECK(s->snapshot(identity) == saved); +} +void empty_and_clean() { + auto s = store(); + const auto empty = s->snapshot(identity); + CHECK(empty && *empty == WorldBytes(world_empty_v1.begin(), world_empty_v1.end())); + CHECK(ChunkStore::restore(world_empty_v1, identity, budget)); + auto restored = ChunkStore::restore(*empty, identity, budget); + CHECK(restored && (**restored).snapshot(identity) == empty); + CHECK(s->load({0, 0, 0})); + CHECK(s->load({-3, 7, -5})); + const auto clean = s->snapshot(identity); + CHECK(clean); + restored = ChunkStore::restore(*clean, identity, budget); + CHECK(restored && (**restored).snapshot(identity) == clean); +} +} // namespace +int main() try { + empty_and_clean(); + round_trip_and_history(); + journals(); + malformed(); + return 0; +} catch (const std::exception& e) { + std::fprintf(stderr, "%s\n", e.what()); + return 1; +} catch (...) { + return 1; +} diff --git a/tools/ci/linux-check.sh b/tools/ci/linux-check.sh index 5bfa4d0..eaff8e5 100644 --- a/tools/ci/linux-check.sh +++ b/tools/ci/linux-check.sh @@ -31,7 +31,9 @@ case "$suite" in cmake --preset debug -B build/fuzz -DCMAKE_CXX_COMPILER=clang++-19 \ -DEVO_BUILD_FUZZER=ON - cmake --build build/fuzz --target evo-snapshot-fuzz --parallel 2 + cmake --build build/fuzz --target evo-snapshot-fuzz evo-world-codec-fuzz --parallel 2 + ./build/fuzz/evo-world-codec-fuzz -seed=123456 -runs=10000 \ + -max_len=96 -timeout=10 -rss_limit_mb=1024 mkdir -p build/fuzz/corpus ./build/release/evo-sim --seed 0 --entities 0 --ticks 0 \ --save build/fuzz/corpus/empty.evo diff --git a/tools/world_snapshot_reference.py b/tools/world_snapshot_reference.py new file mode 100644 index 0000000..4706ef3 --- /dev/null +++ b/tools/world_snapshot_reference.py @@ -0,0 +1,26 @@ +"""Independent empty world-snapshot v1 fixture, derived from schemas/world-snapshot.md. +Prints C++ bytes only; never changes a golden file. +""" +import struct + +payload = bytearray() +def put(value, code): + payload.extend(struct.pack('<' + code, value)) + +put(0x31504e5357564541, 'Q') +put(1, 'I') +put(132, 'I') +for value in [7, 9, 0]: # world, frame, seed + put(value, 'Q') +for value in [1, 1, 1, 16, 4, 0]: # versions, edge, slots, chunks + put(value, 'I') +for value in [0, 0, 0, 1000000, 1000000, 0, 0]: + put(value, 'Q') +put(0, 'I') # retained records +assert len(payload) == 124 +checksum = 14695981039346656037 +for byte in payload: + checksum = ((checksum ^ byte) * 1099511628211) & ((1 << 64) - 1) +put(checksum, 'Q') +for start in range(0, len(payload), 16): + print(', '.join(f'0x{b:02x}' for b in payload[start:start+16]) + ',')