From 0ddba31d42da6af7a41065d90ce3186e929c8a57 Mon Sep 17 00:00:00 2001 From: dingo Date: Sat, 10 Oct 2026 23:27:26 -0400 Subject: [PATCH] Add fenced cellular region host --- CMakeLists.txt | 3 +- docs/engineering/M11_3_DELIVERY_PLAN.md | 7 + docs/engineering/VALIDATION.md | 15 + include/evo/server/cellular_region_host.hpp | 79 +++++ src/server/cellular_region_host.cpp | 351 ++++++++++++++++++++ tests/cellular_publication_tests.cpp | 52 +-- 6 files changed, 487 insertions(+), 20 deletions(-) create mode 100644 include/evo/server/cellular_region_host.hpp create mode 100644 src/server/cellular_region_host.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 17819bd..72efe27 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -239,7 +239,8 @@ if(EVO_BUILD_POSTGRES) evo_owned_target(evo_region_store) # M7.6 authoritative region host and its process (ADR-0064). add_library(evo_region_host STATIC src/server/region_host.cpp - src/server/primitive_region_host.cpp src/server/ops_http.cpp) + src/server/primitive_region_host.cpp src/server/cellular_region_host.cpp + src/server/ops_http.cpp) add_library(evo::region_host ALIAS evo_region_host) target_link_libraries(evo_region_host PUBLIC evo::region_store evo::ops) evo_owned_target(evo_region_host) diff --git a/docs/engineering/M11_3_DELIVERY_PLAN.md b/docs/engineering/M11_3_DELIVERY_PLAN.md index 608a61d..f7eaee8 100644 --- a/docs/engineering/M11_3_DELIVERY_PLAN.md +++ b/docs/engineering/M11_3_DELIVERY_PLAN.md @@ -250,6 +250,13 @@ scoped change before starting the next segment. funded-founder genesis, one committed neutral action, publication and restart-style recovery. CI execution is pending; action conflicts and process fault recovery remain. +- 11.3d-7a: `CellularRegionHost` now owns a fenced profile-4 lease, recovers before + readiness, replays submitted cellular batches, publishes only through the + profile-4 adapter, pauses on ambiguous commits and resolves/retries the exact + prepared bytes. It exposes final action acknowledgements only after a confirmed + receipt. A real-PostgreSQL test exercises host publication followed by a new + host instance recovering the result. Session v7 admission, scheduler + ownership, process kill tests and CI execution remain open. ## Exit boundary diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index da69d10..6a49b5b 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -6285,3 +6285,18 @@ publication, and recovery including its durable result row. for the changed adapter and integration test. Full build, PostgreSQL execution, and sanitizer checks are pending CI; the local CMake preset remains unable to configure with its cached MSVC compiler selection. + +## 2026-10-10 — M11.3d-7a cellular region host (CI pending) + +Added a profile-4 single-writer host that obtains and renews the region lease, +fully recovers before readiness, replays each supplied batch, and advances its +in-memory state only after a confirmed checkpoint receipt. An uncertain response +pauses admission and resolves or retries the exact prepared manifest, trace and +action block. Final action acknowledgements are exposed only after a confirmed +receipt. The PostgreSQL integration test exercises publication through the host, +its action result, and recovery through a fresh host instance. + +`git diff --check` and GCC 13.2 C++23 syntax checks with warnings as errors pass +for the host and integration test. Full CMake build, PostgreSQL execution and +process fault tests remain pending CI; CMake's documented preset still fails its +cached MSVC C++23 feature check. diff --git a/include/evo/server/cellular_region_host.hpp b/include/evo/server/cellular_region_host.hpp new file mode 100644 index 0000000..b8bcc57 --- /dev/null +++ b/include/evo/server/cellular_region_host.hpp @@ -0,0 +1,79 @@ +#pragma once + +#include "evo/persistence/checkpoint_store.hpp" +#include "evo/server/ops.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace evo::server { +using CellularConnect = + std::function, persistence::PgFailure>()>; +struct CellularHostConfig { + persistence::RegionKey key{}; + std::uint64_t frame{}; + persistence::LeasePolicy lease{}; + persistence::Sha256Digest build{}; + std::vector retained_builds; +}; +enum class CellularHostError : std::uint8_t { configuration, not_ready, invalid_batch, publication }; +struct FinalCellularAction { + CellularActionRecord action; + std::uint64_t commit_sequence{}; + friend bool operator==(const FinalCellularAction&, const FinalCellularAction&) = default; +}; + +// Single-writer profile-4 authority. The caller supplies a complete bounded trace; +// the host replays it from its recovered state and advances only on a confirmed head. +class CellularRegionHost { + public: + [[nodiscard]] static std::expected, + CellularHostError> + create(CellularHostConfig config, CellularConnect connect, + std::unique_ptr objects); + CellularRegionHost(const CellularRegionHost&) = delete; + CellularRegionHost& operator=(const CellularRegionHost&) = delete; + ~CellularRegionHost(); + + // Acquires/renews authority, recovers fully, or resolves the exact pending commit. + const HostStatus& step(); + [[nodiscard]] std::expected + publish(const CellularBatch& batch, + std::span actions = {}); + [[nodiscard]] std::vector take_finalized_actions(); + void drain(); + [[nodiscard]] std::optional authority() const; + [[nodiscard]] const HostStatus& status() const noexcept { return status_; } + [[nodiscard]] const CellularLives* state() const noexcept { return state_.get(); } + [[nodiscard]] const std::optional& committed() const noexcept { + return committed_; + } + + private: + struct Pending; + CellularRegionHost(CellularHostConfig config, CellularConnect connect, + std::unique_ptr objects); + bool database(); + void acquire(); + void upkeep(); + void resolve(); + void refresh(); + + CellularHostConfig config_; + CellularConnect connect_; + std::unique_ptr objects_; + std::unique_ptr db_; + persistence::LeaseAuthority authority_; + std::unique_ptr state_; + std::optional committed_; + std::unique_ptr pending_; + std::vector finalized_actions_; + HostStatus status_{}; +}; +} // namespace evo::server diff --git a/src/server/cellular_region_host.cpp b/src/server/cellular_region_host.cpp new file mode 100644 index 0000000..98a00cc --- /dev/null +++ b/src/server/cellular_region_host.cpp @@ -0,0 +1,351 @@ +#include "evo/server/cellular_region_host.hpp" + +#include "evo/persistence/sha256.hpp" + +#include +#include +#include +#include + +namespace evo::server { +struct CellularRegionHost::Pending { + persistence::CellularPreparedCommit prepared; + std::unique_ptr candidate; + std::vector actions; +}; + +namespace { +using persistence::LeaseClock; +using persistence::LeaseStoreError; +using std::chrono::milliseconds; + +bool valid(const CellularHostConfig& config) { + return config.key.world != 0 && config.key.region != 0 && config.frame != 0 && + persistence::valid(config.lease) && config.build != persistence::Sha256Digest{} && + std::ranges::find(config.retained_builds, config.build) != config.retained_builds.end(); +} + +std::optional commit_id(const persistence::CommittedCheckpoint& previous, + std::span trace, + std::span actions) { + std::vector identity; + identity.reserve(previous.head.digest.size() + 16 + trace.size() + actions.size()); + identity.insert(identity.end(), previous.head.digest.begin(), previous.head.digest.end()); + const auto append_length = [&](std::size_t size) { + const auto length = static_cast(size); + for (unsigned i = 0; i < 8; ++i) + identity.push_back(static_cast(length >> (i * 8U))); + }; + append_length(trace.size()); + identity.insert(identity.end(), trace.begin(), trace.end()); + append_length(actions.size()); + identity.insert(identity.end(), actions.begin(), actions.end()); + const auto digest = persistence::sha256(identity); + if (!digest) + return std::nullopt; + persistence::Uuid out{}; + std::ranges::copy_n(digest->begin(), out.size(), out.begin()); + if (out == persistence::Uuid{}) + out.back() = 1; + return out; +} +} // namespace + +CellularRegionHost::CellularRegionHost( + CellularHostConfig config, CellularConnect connect, + std::unique_ptr objects) + : config_{std::move(config)}, connect_{std::move(connect)}, objects_{std::move(objects)}, + authority_{config_.lease} {} + +CellularRegionHost::~CellularRegionHost() = default; + +std::expected, CellularHostError> +CellularRegionHost::create(CellularHostConfig config, CellularConnect connect, + std::unique_ptr objects) { + if (!valid(config) || !connect || !objects) + return std::unexpected{CellularHostError::configuration}; + return std::unique_ptr{ + new CellularRegionHost{std::move(config), std::move(connect), std::move(objects)}}; +} + +bool CellularRegionHost::database() { + if (db_) + return true; + auto connection = connect_(); + if (!connection) { + ++status_.store_failures; + return false; + } + db_ = std::move(*connection); + return true; +} + +void CellularRegionHost::acquire() { + if (!database()) + return; + const auto sent = LeaseClock::now(); + auto lease = persistence::acquire_region_lease(*db_, config_.key, config_.build, + config_.lease.duration, + config_.lease.renew_timeout); + if (!lease) { + if (lease.error().kind == LeaseStoreError::not_provisioned || + lease.error().kind == LeaseStoreError::unsupported) { + status_.phase = HostPhase::quarantined; + } else if (lease.error().kind != LeaseStoreError::busy) { + db_.reset(); + ++status_.store_failures; + } + return; + } + if (lease->frame != config_.frame || + !authority_.acquired(*lease, sent, LeaseClock::now())) { + status_.phase = HostPhase::quarantined; + return; + } + auto recovered = persistence::recover_cellular_checkpoint( + *db_, *objects_, config_.key, *lease, config_.retained_builds, + config_.lease.commit_timeout); + if (!recovered) { + if (recovered.error().kind == persistence::RecoveryError::store) { + db_.reset(); + ++status_.store_failures; + return; + } + authority_.fence(); + status_.phase = HostPhase::quarantined; + return; + } + state_ = std::move(recovered->state); + committed_ = std::move(recovered->committed); + status_.phase = HostPhase::running; + status_.simulated_tick = committed_->head.end_tick; + status_.committed_tick = committed_->head.end_tick; + status_.committed_sequence = committed_->head.sequence; +} + +void CellularRegionHost::upkeep() { + if (authority_.state(LeaseClock::now()) == persistence::LeaseState::fenced) { + authority_.fence(); + state_.reset(); + committed_.reset(); + pending_.reset(); + status_.ready = false; + status_.phase = HostPhase::acquiring; + ++status_.fences; + return; + } + if (authority_.state(LeaseClock::now()) != persistence::LeaseState::renew_due || !database()) + return; + const auto grant = *authority_.grant(); + const auto sent = LeaseClock::now(); + auto renewed = persistence::renew_region_lease(*db_, config_.key, grant, config_.lease.duration, + config_.lease.renew_timeout); + if (renewed) { + if (!authority_.renewed(grant.epoch, *renewed, sent, LeaseClock::now())) { + authority_.fence(); + state_.reset(); + committed_.reset(); + pending_.reset(); + status_.ready = false; + status_.phase = HostPhase::acquiring; + } + return; + } + if (renewed.error().kind == LeaseStoreError::unknown) + authority_.renewal_unknown(config_.lease.duration, sent); + else if (renewed.error().kind == LeaseStoreError::fenced) { + authority_.fence(); + state_.reset(); + committed_.reset(); + pending_.reset(); + status_.ready = false; + status_.phase = HostPhase::acquiring; + ++status_.fences; + } + db_.reset(); + ++status_.store_failures; +} + +void CellularRegionHost::resolve() { + if (!pending_ || !database()) + return; + if (!authority_.may_begin_commit(LeaseClock::now())) { + authority_.fence(); + state_.reset(); + committed_.reset(); + pending_.reset(); + status_.phase = HostPhase::acquiring; + status_.ready = false; + ++status_.fences; + return; + } + const auto resolved = persistence::resolve_cellular_prepared( + *db_, pending_->prepared, config_.lease.commit_timeout); + if (!resolved) { + if (resolved.error().kind == LeaseStoreError::fenced) { + authority_.fence(); + state_.reset(); + committed_.reset(); + pending_.reset(); + status_.phase = HostPhase::acquiring; + status_.ready = false; + ++status_.fences; + } else { + db_.reset(); + ++status_.store_failures; + } + return; + } + if (resolved->status == persistence::Resolution::conflict) { + authority_.fence(); + state_.reset(); + committed_.reset(); + pending_.reset(); + status_.phase = HostPhase::quarantined; + status_.ready = false; + return; + } + if (resolved->status == persistence::Resolution::absent && + !authority_.may_begin_commit(LeaseClock::now())) + return; + auto retried = persistence::retry_cellular_prepared( + *db_, pending_->prepared, config_.lease.commit_timeout); + if (!retried) { + if (retried.error().kind == persistence::PublicationError::unknown) { + db_.reset(); + ++status_.unknown_outcomes; + status_.phase = HostPhase::resolving; + status_.ready = false; + } else if (retried.error().store.kind == LeaseStoreError::fenced) { + authority_.fence(); + state_.reset(); + committed_.reset(); + pending_.reset(); + status_.phase = HostPhase::acquiring; + status_.ready = false; + ++status_.fences; + } else { + authority_.fence(); + state_.reset(); + committed_.reset(); + pending_.reset(); + status_.phase = HostPhase::quarantined; + status_.ready = false; + } + return; + } + state_ = std::move(pending_->candidate); + committed_ = std::move(*retried); + for (const auto& action : pending_->actions) + finalized_actions_.push_back({action, committed_->head.sequence}); + pending_.reset(); + ++status_.commits; + status_.committed_sequence = committed_->head.sequence; + status_.committed_tick = committed_->head.end_tick; + status_.simulated_tick = committed_->head.end_tick; + status_.phase = HostPhase::running; +} + +void CellularRegionHost::refresh() { + const auto& grant = authority_.grant(); + status_.ready = status_.phase == HostPhase::running && state_ && committed_ && grant && + authority_.may_originate(LeaseClock::now()); + status_.epoch = grant ? grant->epoch : 0; + status_.lease_remaining_ms = + status_.ready + ? static_cast(std::chrono::duration_cast( + authority_.deadline() - LeaseClock::now()) + .count()) + : 0; +} + +const HostStatus& CellularRegionHost::step() { + if (status_.phase == HostPhase::quarantined || status_.phase == HostPhase::stopped) + return status_; + if (authority_.grant()) + upkeep(); + if (pending_) + resolve(); + else if (!state_) + acquire(); + refresh(); + return status_; +} + +std::expected +CellularRegionHost::publish(const CellularBatch& batch, + std::span actions) { + if (!status_.ready || !state_ || !committed_ || + !authority_.may_begin_commit(LeaseClock::now())) + return std::unexpected{CellularHostError::not_ready}; + auto candidate = replay_cellular_batch(*state_, batch); + if (!candidate) + return std::unexpected{CellularHostError::invalid_batch}; + auto trace = encode_cellular_batch(batch); + auto action_block = encode_cellular_action_block(actions, batch.header.begin_tick, + batch.header.end_tick); + const auto commit = trace && action_block + ? commit_id(*committed_, *trace, *action_block) + : std::nullopt; + if (!commit) + return std::unexpected{CellularHostError::invalid_batch}; + auto published = persistence::publish_cellular_batch( + *db_, *objects_, *authority_.grant(), *committed_, batch, **candidate, *commit, + config_.build, config_.lease.commit_timeout, {}, actions); + if (!published) { + if (published.error().kind == persistence::PublicationError::unknown) { + pending_ = std::make_unique( + Pending{published.error().prepared, std::move(*candidate), + {actions.begin(), actions.end()}}); + status_.phase = HostPhase::resolving; + status_.ready = false; + ++status_.unknown_outcomes; + db_.reset(); + } else if (published.error().store.kind == LeaseStoreError::fenced) { + authority_.fence(); + state_.reset(); + committed_.reset(); + status_.phase = HostPhase::acquiring; + status_.ready = false; + ++status_.fences; + } + return std::unexpected{CellularHostError::publication}; + } + state_ = std::move(*candidate); + committed_ = published->committed; + for (const auto& action : actions) + finalized_actions_.push_back({action, committed_->head.sequence}); + ++status_.commits; + status_.committed_sequence = committed_->head.sequence; + status_.committed_tick = committed_->head.end_tick; + status_.simulated_tick = committed_->head.end_tick; + refresh(); + return *published; +} + +std::vector CellularRegionHost::take_finalized_actions() { + return std::exchange(finalized_actions_, {}); +} + +void CellularRegionHost::drain() { + status_.ready = false; + status_.phase = HostPhase::draining; + pending_.reset(); + const auto grant = authority_.grant(); + if (grant && authority_.may_originate(LeaseClock::now()) && database()) + static_cast(persistence::release_region_lease( + *db_, config_.key, *grant, config_.lease.renew_timeout)); + authority_.fence(); + state_.reset(); + committed_.reset(); + status_.phase = HostPhase::stopped; + status_.epoch = 0; + status_.lease_remaining_ms = 0; +} + +std::optional CellularRegionHost::authority() const { + if (!status_.ready || !authority_.may_originate(LeaseClock::now())) + return std::nullopt; + return authority_.grant(); +} +} // namespace evo::server diff --git a/tests/cellular_publication_tests.cpp b/tests/cellular_publication_tests.cpp index 4355df7..d6abe53 100644 --- a/tests/cellular_publication_tests.cpp +++ b/tests/cellular_publication_tests.cpp @@ -1,6 +1,7 @@ // Profile-4 publication and recovery against the migration-backed PostgreSQL adapter. #include "evo/persistence/checkpoint_store.hpp" #include "evo/persistence/sha256.hpp" +#include "evo/server/cellular_region_host.hpp" #include #include @@ -46,7 +47,6 @@ std::string hex(std::span bytes) { void run_cellular_publication_tests() { auto owner = connect_as(env("PGUSER")); - auto db = connect_as(env("EVO_PG_RUNTIME_USER")); constexpr RegionKey key{9756, 1}; const auto build = filled<32>(0xD9); CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " @@ -83,34 +83,48 @@ void run_cellular_publication_tests() { CHECK(objects && (*objects)->put(payload)); CHECK(owner->query("SELECT evolution.provision_region_checkpoint($1::bytea)", std::array{hex(*genesis_bytes)}, 5s)); - auto predecessor = load_region_checkpoint(*owner, key, 5s); - CHECK(predecessor); - const auto lease = acquire_region_lease(*db, key, build, 20'000ms, 5s); - CHECK(lease); - - const auto predecessor_digest = sha256(predecessor->manifest); - CHECK(predecessor_digest); + const auto connect = [&]() -> std::expected, PgFailure> { + return Postgres::connect({env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), + env("EVO_PG_RUNTIME_USER"), {}, {}}, 5s); + }; + CellularHostConfig config{key, 1, {}, build, {build}}; + auto made_host = CellularRegionHost::create(config, connect, std::move(*objects)); + CHECK(made_host); + auto host = std::move(*made_host); + CHECK(host->step().ready); CellularBatch batch; - batch.header = {key.world, key.region, 1, predecessor->head.commit, *predecessor_digest, 0, 2}; + const auto predecessor_digest = sha256(host->committed()->manifest); + CHECK(predecessor_digest); + batch.header = {key.world, key.region, 1, host->committed()->head.commit, + *predecessor_digest, 0, 2}; batch.ticks.resize(2); for (auto& tick : batch.ticks) tick.lives.push_back({(*opening)->lives()[0].identity.life, {}}); - const auto replay = replay_cellular_batch(**opening, batch); + const auto replay = replay_cellular_batch(*host->state(), batch); CHECK(replay); CellularActionRecord action; action.account = founder.account; action.action_id = 77; action.input[0] = 1; // Tick 1, neutral movement, no patch target. const std::array actions{action}; - auto published = publish_cellular_batch(*db, **objects, *lease, *predecessor, batch, - **replay, filled<16>(0x22), build, 5s, {}, actions); + auto published = host->publish(batch, actions); CHECK(published && published->committed.head.sequence == 1 && - published->committed.head.end_tick == 2); + published->committed.head.end_tick == 2 && + host->state()->snapshot() == (**replay).snapshot()); + CHECK(host->take_finalized_actions() == + std::vector{{action, 1}}); + host->drain(); + host.reset(); - auto recovered = recover_cellular_checkpoint(*db, **objects, key, *lease, - std::span{&build, 1}, 5s); - CHECK(recovered && recovered->manifest.sequence == 1 && recovered->state->tick() == 2 && - recovered->state->snapshot() == (**replay).snapshot() && - recovered->actions == std::vector{action}); - CHECK(release_region_lease(*db, key, *lease, 5s)); + auto restarted_objects = FilesystemObjectStore::open(env("PGHOST"), key, + max_cellular_lives_bytes); + CHECK(restarted_objects); + auto made_restarted = CellularRegionHost::create(config, connect, + std::move(*restarted_objects)); + CHECK(made_restarted); + auto restarted = std::move(*made_restarted); + CHECK(restarted->step().ready && restarted->state()->tick() == 2 && + restarted->state()->snapshot() == (**replay).snapshot() && + restarted->committed()->head.sequence == 1); + restarted->drain(); }