From 13273b46f484bbb18b8302fd32d48b697d1bb767 Mon Sep 17 00:00:00 2001 From: dingo Date: Sat, 10 Oct 2026 23:23:17 -0400 Subject: [PATCH] Recover cellular checkpoints with action evidence --- CMakeLists.txt | 1 + docs/engineering/M11_3_DELIVERY_PLAN.md | 6 + docs/engineering/VALIDATION.md | 14 +++ include/evo/persistence/checkpoint_store.hpp | 11 ++ src/persistence/cellular_checkpoint_store.cpp | 118 ++++++++++++++++++ tests/cellular_publication_tests.cpp | 104 +++++++++++++++ tests/postgres_tests.cpp | 2 + 7 files changed, 256 insertions(+) create mode 100644 tests/cellular_publication_tests.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 88447bf..17819bd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -271,6 +271,7 @@ if(EVO_BUILD_POSTGRES) if(BUILD_TESTING) # Contention tests use one thread and one connection per contender. add_executable(evo-postgres-tests tests/postgres_tests.cpp tests/region_lease_store_tests.cpp + tests/cellular_publication_tests.cpp tests/account_control_store_tests.cpp tests/probe_handoff_store_tests.cpp tests/life_reservation_store_tests.cpp tests/primitive_publication_tests.cpp diff --git a/docs/engineering/M11_3_DELIVERY_PLAN.md b/docs/engineering/M11_3_DELIVERY_PLAN.md index 42a1d5b..8a4d3ef 100644 --- a/docs/engineering/M11_3_DELIVERY_PLAN.md +++ b/docs/engineering/M11_3_DELIVERY_PLAN.md @@ -243,6 +243,12 @@ scoped change before starting the next segment. manifest validator now applies ADR-0113's variable cellular trace bounds, matching migration 21. Result-row reconciliation and full restart recovery, host integration, and CI execution remain open. +- 11.3d-6f: the cellular recovery adapter now verifies the committed payload, + current tail trace and predecessor link, loads the immutable action block under + the lease, replays the batch and checks every action's persisted kind, result, + tick, sequence and canonical input bytes. A PostgreSQL integration test covers + empty-cell genesis through publish and restart-style recovery. CI execution is + pending; populated-life action-result cases and process fault recovery remain. ## Exit boundary diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 0f12101..5cba4d3 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -6270,3 +6270,17 @@ manifest validator, and boundary tests pass. Full CMake build/CTest and PostgreS integration are pending CI. The documented debug preset currently selects the cached MSVC 19.44 compiler and fails CMake's C++23 feature check; this environment also has no PostgreSQL runtime on `PATH`. + +## 2026-10-10 — M11.3d-6f cellular action result recovery (CI pending) + +Added lease-fenced cellular checkpoint recovery. It verifies the head payload, +trace/header and predecessor digest, replays the complete batch, and checks each +action row against the action block's account/action identity, cellular kind, +committed result, intended tick, head sequence and exact canonical input. The +PostgreSQL integration suite now exercises profile-4 genesis, a two-tick +publication, and recovery of that committed head. + +`git diff --check` and GCC 13.2 C++23 syntax checks with warnings as errors pass +for the changed adapter and integration test. Full build, PostgreSQL execution, +and sanitizer checks are pending CI; the local CMake preset remains unable to +configure with its cached MSVC compiler selection. diff --git a/include/evo/persistence/checkpoint_store.hpp b/include/evo/persistence/checkpoint_store.hpp index 426c49a..1219698 100644 --- a/include/evo/persistence/checkpoint_store.hpp +++ b/include/evo/persistence/checkpoint_store.hpp @@ -115,6 +115,12 @@ struct CellularPublicationFailure { struct PublishedCellularBatch { CommittedCheckpoint committed; }; +struct RecoveredCellularCheckpoint { + CommittedCheckpoint committed; + CheckpointManifest manifest; + std::unique_ptr state; + std::vector actions; +}; // Owner/operator connection only: uploads a tick-0 state and records genesis. [[nodiscard]] std::expected provision_environmental_region( @@ -204,6 +210,11 @@ retry_cellular_prepared(Postgres& db, const CellularPreparedCommit& prepared, [[nodiscard]] std::expected resolve_cellular_prepared(Postgres& db, const CellularPreparedCommit& prepared, std::chrono::milliseconds timeout); +[[nodiscard]] std::expected +recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, + const LeaseGrant& lease, + std::span retained_builds, + std::chrono::milliseconds timeout); // After an `unknown` publication: resolve exactly the prepared bytes. A committed // result yields the final batch; absent may be retried with the same prepared bytes diff --git a/src/persistence/cellular_checkpoint_store.cpp b/src/persistence/cellular_checkpoint_store.cpp index 59b3c3a..ae746ef 100644 --- a/src/persistence/cellular_checkpoint_store.cpp +++ b/src/persistence/cellular_checkpoint_store.cpp @@ -8,6 +8,13 @@ namespace evo::persistence { namespace { using std::chrono::milliseconds; +std::uint64_t input_tick(const server::CellularActionRecord& action) { + std::uint64_t tick{}; + for (std::size_t i = 0; i < 8; ++i) + tick |= static_cast(action.input[i]) << (i * 8U); + return tick; +} + CellularPublicationFailure failure(PublicationError kind, ObjectError object = {}, LeaseStoreFailure store = {}, CellularPreparedCommit prepared = {}) { @@ -150,4 +157,115 @@ publish_cellular_batch(Postgres& db, const FilesystemObjectStore& objects, return std::unexpected{committed.error()}; return PublishedCellularBatch{std::move(*committed)}; } + +std::expected +recover_cellular_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, + const LeaseGrant& lease, + std::span retained_builds, + milliseconds timeout) { + const auto fail_with = [](RecoveryError kind) { + return std::unexpected{RecoveryFailure{kind, {}}}; + }; + auto committed = load_region_checkpoint(db, key, timeout); + if (!committed) + return std::unexpected{RecoveryFailure{ + committed.error().kind == LeaseStoreError::not_provisioned + ? RecoveryError::not_provisioned + : RecoveryError::store, + committed.error()}}; + const auto digest = sha256(committed->manifest); + auto manifest = decode_manifest(committed->manifest); + if (!digest || *digest != committed->head.digest || !manifest || manifest->key != key || + manifest->sequence != committed->head.sequence || + manifest->commit != committed->head.commit || manifest->end_tick != committed->head.end_tick || + manifest->frame != committed->head.frame) + return fail_with(RecoveryError::manifest); + if (!supported(*manifest) || manifest->profile != CheckpointProfile::cellular_lives_v1 || + std::ranges::find(retained_builds, manifest->build) == retained_builds.end()) + return fail_with(RecoveryError::unsupported); + if (lease.frame != manifest->frame) + return fail_with(RecoveryError::unsupported); + + auto payload = objects.get({manifest->payload_sha256, manifest->payload_bytes}); + if (!payload) + return fail_with(payload.error() == ObjectError::io ? RecoveryError::store + : RecoveryError::corrupt_payload); + auto state = restore_cellular_payload(*payload, *manifest); + if (!state) + return fail_with(RecoveryError::payload); + + std::vector actions; + if (manifest->sequence != 0) { + auto action_bytes = load_cellular_publication_actions(db, key, lease, manifest->commit, timeout); + if (!action_bytes) + return std::unexpected{RecoveryFailure{RecoveryError::store, action_bytes.error()}}; + if (!*action_bytes) + return fail_with(RecoveryError::history); + + auto trace_bytes = objects.get({manifest->batch.trace_sha256, manifest->batch.trace_bytes}); + if (!trace_bytes) + return fail_with(trace_bytes.error() == ObjectError::io ? RecoveryError::store + : RecoveryError::tail); + auto trace = server::decode_cellular_batch(*trace_bytes); + if (!trace) + return fail_with(RecoveryError::tail); + if (trace->header.world != key.world || trace->header.region != key.region || + trace->header.frame != manifest->frame || + trace->header.predecessor != manifest->predecessor_commit || + trace->header.predecessor_digest != manifest->predecessor_digest || + trace->header.begin_tick != manifest->begin_tick || + trace->header.end_tick != manifest->end_tick) + return fail_with(RecoveryError::tail); + + auto predecessor = load_region_commit(db, key, manifest->sequence - 1, timeout); + if (!predecessor) + return std::unexpected{RecoveryFailure{RecoveryError::tail, predecessor.error()}}; + const auto predecessor_digest = sha256(predecessor->manifest); + auto previous = decode_manifest(predecessor->manifest); + if (!predecessor_digest || *predecessor_digest != manifest->predecessor_digest || + predecessor->head.digest != manifest->predecessor_digest || + predecessor->head.commit != manifest->predecessor_commit || + predecessor->head.sequence != manifest->sequence - 1 || !previous || + previous->profile != CheckpointProfile::cellular_lives_v1 || + previous->end_tick != manifest->begin_tick) + return fail_with(RecoveryError::tail); + auto base_bytes = objects.get({previous->payload_sha256, previous->payload_bytes}); + if (!base_bytes) + return fail_with(base_bytes.error() == ObjectError::io ? RecoveryError::store + : RecoveryError::tail); + auto base = restore_cellular_payload(*base_bytes, *previous); + if (!base) + return fail_with(RecoveryError::tail); + auto batch_actions = server::decode_cellular_action_block( + **action_bytes, manifest->begin_tick, manifest->end_tick); + if (!batch_actions || !server::match_cellular_actions_to_trace(**base, *trace, *batch_actions)) + return fail_with(RecoveryError::history); + auto replay = server::replay_cellular_batch(**base, *trace); + if (!replay || (*replay)->snapshot() != *payload) + return fail_with(RecoveryError::tail); + + std::vector refs; + refs.reserve(batch_actions->size()); + for (const auto& action : *batch_actions) + refs.push_back({action.account, action.action_id}); + auto records = find_region_action_records(db, key, lease, refs, timeout); + if (!records) + return std::unexpected{RecoveryFailure{RecoveryError::store, records.error()}}; + for (std::size_t i = 0; i < batch_actions->size(); ++i) { + if (!(*records)[i]) + return fail_with(RecoveryError::history); + const auto& record = *(*records)[i]; + const auto& action = (*batch_actions)[i]; + if (record.kind != 2 || record.result != 5 || !record.cellular_input || + *record.cellular_input != + std::vector(action.input.begin(), action.input.end()) || + record.action.region != key.region || record.action.tick != input_tick(action) || + record.action.commit_sequence != manifest->sequence) + return fail_with(RecoveryError::history); + } + actions = std::move(*batch_actions); + } + return RecoveredCellularCheckpoint{std::move(*committed), *manifest, std::move(*state), + std::move(actions)}; +} } // namespace evo::persistence diff --git a/tests/cellular_publication_tests.cpp b/tests/cellular_publication_tests.cpp new file mode 100644 index 0000000..8a5c4e0 --- /dev/null +++ b/tests/cellular_publication_tests.cpp @@ -0,0 +1,104 @@ +// Profile-4 publication and recovery against the migration-backed PostgreSQL adapter. +#include "evo/persistence/checkpoint_store.hpp" +#include "evo/persistence/sha256.hpp" + +#include +#include +#include + +void run_cellular_publication_tests(); +namespace { +using namespace evo::persistence; +using namespace evo::server; +using namespace std::chrono_literals; +void check(bool value, std::source_location at = std::source_location::current()) { + if (!value) + throw std::runtime_error("cellular publication test line " + std::to_string(at.line())); +} +#define CHECK(...) check(static_cast(__VA_ARGS__)) +std::string env(const char* name) { + const auto* value = std::getenv(name); + if (!value) + throw std::runtime_error("missing PostgreSQL test environment"); + return value; +} +auto connect_as(const std::string& user) { + auto db = Postgres::connect( + {env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}}, 5s); + CHECK(db); + return std::move(*db); +} +template std::array filled(std::uint8_t value) { + std::array out{}; + out.fill(value); + return out; +} +std::string hex(std::span bytes) { + constexpr char digits[] = "0123456789abcdef"; + std::string out = "\\x"; + for (const auto byte : bytes) { + out.push_back(digits[byte >> 4U]); + out.push_back(digits[byte & 15U]); + } + return out; +} +} // namespace + +void run_cellular_publication_tests() { + auto owner = connect_as(env("PGUSER")); + auto db = connect_as(env("EVO_PG_RUNTIME_USER")); + constexpr RegionKey key{9756, 1}; + const auto build = filled<32>(0xD9); + CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " + "(decode(repeat('d9',32),'hex')) ON CONFLICT DO NOTHING", + {}, 5s)); + + auto endowed = evo::biology::primitive_endowment(); + CHECK(endowed); + auto funding = endowed->body; + funding.energy.value = endowed->reserve.energy.value + endowed->construction.value; + auto primitive = PrimitiveLives::create({key.world, key.region, 1, 1, funding, {{0, 0}}}); + CHECK(primitive); + auto habitat = evo::biology::CellularHabitat::create( + {key.world, key.region, 1, 1'000, 1'000, {}}); + CHECK(habitat); + auto opening = CellularLives::create(**primitive, std::move(*habitat), + {{1'000, 1'000}, 0, 293'150}); + CHECK(opening && (*opening)->lives().empty()); + + const auto payload = (*opening)->snapshot(); + const auto payload_digest = sha256(payload); + CHECK(payload_digest); + const auto genesis = genesis_manifest(key, 1, filled<16>(0x21), 50'000, build, + payload.size(), *payload_digest, + CheckpointProfile::cellular_lives_v1); + CHECK(genesis); + const auto genesis_bytes = encode_manifest(*genesis); + CHECK(genesis_bytes); + auto objects = FilesystemObjectStore::open(env("PGHOST"), key, max_cellular_lives_bytes); + CHECK(objects && (*objects)->put(payload)); + CHECK(owner->query("SELECT evolution.provision_region_checkpoint($1::bytea)", + std::array{hex(*genesis_bytes)}, 5s)); + auto predecessor = load_region_checkpoint(*owner, key, 5s); + CHECK(predecessor); + const auto lease = acquire_region_lease(*db, key, build, 20'000ms, 5s); + CHECK(lease); + + const auto predecessor_digest = sha256(predecessor->manifest); + CHECK(predecessor_digest); + CellularBatch batch; + batch.header = {key.world, key.region, 1, predecessor->head.commit, *predecessor_digest, 0, 2}; + batch.ticks.resize(2); // The funded genesis has no player-controlled lives. + const auto replay = replay_cellular_batch(**opening, batch); + CHECK(replay); + auto published = publish_cellular_batch(*db, **objects, *lease, *predecessor, batch, + **replay, filled<16>(0x22), build, 5s); + CHECK(published && published->committed.head.sequence == 1 && + published->committed.head.end_tick == 2); + + auto recovered = recover_cellular_checkpoint(*db, **objects, key, *lease, + std::span{&build, 1}, 5s); + CHECK(recovered && recovered->manifest.sequence == 1 && recovered->state->tick() == 2 && + recovered->state->snapshot() == (**replay).snapshot() && recovered->actions.empty()); + CHECK(release_region_lease(*db, key, *lease, 5s)); +} diff --git a/tests/postgres_tests.cpp b/tests/postgres_tests.cpp index 2a3de86..c739877 100644 --- a/tests/postgres_tests.cpp +++ b/tests/postgres_tests.cpp @@ -8,6 +8,7 @@ void run_region_lease_store_tests(); void run_account_control_store_tests(); void run_life_reservation_store_tests(); void run_primitive_publication_tests(); +void run_cellular_publication_tests(); void run_probe_handoff_store_tests(); void run_region_action_tests(); void run_region_checkpoint_store_tests(); @@ -137,6 +138,7 @@ int main(int argc, char** argv) { run_account_control_store_tests(); run_life_reservation_store_tests(); run_primitive_publication_tests(); + run_cellular_publication_tests(); run_probe_handoff_store_tests(); run_region_action_tests(); std::puts("region lease fencing integration passed");