Add fenced cellular region host
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (checks) (push) Failing after 41s
Foundation / checks (ecs) (push) Successful in 1m54s
Foundation / checks (postgres) (push) Failing after 53s
Foundation / checks (tsan) (push) Failing after 33s
Foundation / checks (network) (push) Failing after 3m4s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (client) (push) Canceled after 4m14s

This commit is contained in:
dingo
2026-10-10 23:27:26 -04:00
parent 4930fd7159
commit 0ddba31d42
6 changed files with 487 additions and 20 deletions
+2 -1
View File
@@ -239,7 +239,8 @@ if(EVO_BUILD_POSTGRES)
evo_owned_target(evo_region_store)
# M7.6 authoritative region host and its process (ADR-0064).
add_library(evo_region_host STATIC src/server/region_host.cpp
src/server/primitive_region_host.cpp src/server/ops_http.cpp)
src/server/primitive_region_host.cpp src/server/cellular_region_host.cpp
src/server/ops_http.cpp)
add_library(evo::region_host ALIAS evo_region_host)
target_link_libraries(evo_region_host PUBLIC evo::region_store evo::ops)
evo_owned_target(evo_region_host)
+7
View File
@@ -250,6 +250,13 @@ scoped change before starting the next segment.
funded-founder genesis, one committed neutral action, publication and
restart-style recovery. CI execution is pending; action conflicts and process
fault recovery remain.
- 11.3d-7a: `CellularRegionHost` now owns a fenced profile-4 lease, recovers before
readiness, replays submitted cellular batches, publishes only through the
profile-4 adapter, pauses on ambiguous commits and resolves/retries the exact
prepared bytes. It exposes final action acknowledgements only after a confirmed
receipt. A real-PostgreSQL test exercises host publication followed by a new
host instance recovering the result. Session v7 admission, scheduler
ownership, process kill tests and CI execution remain open.
## Exit boundary
+15
View File
@@ -6285,3 +6285,18 @@ publication, and recovery including its durable result row.
for the changed adapter and integration test. Full build, PostgreSQL execution,
and sanitizer checks are pending CI; the local CMake preset remains unable to
configure with its cached MSVC compiler selection.
## 2026-10-10 — M11.3d-7a cellular region host (CI pending)
Added a profile-4 single-writer host that obtains and renews the region lease,
fully recovers before readiness, replays each supplied batch, and advances its
in-memory state only after a confirmed checkpoint receipt. An uncertain response
pauses admission and resolves or retries the exact prepared manifest, trace and
action block. Final action acknowledgements are exposed only after a confirmed
receipt. The PostgreSQL integration test exercises publication through the host,
its action result, and recovery through a fresh host instance.
`git diff --check` and GCC 13.2 C++23 syntax checks with warnings as errors pass
for the host and integration test. Full CMake build, PostgreSQL execution and
process fault tests remain pending CI; CMake's documented preset still fails its
cached MSVC C++23 feature check.
@@ -0,0 +1,79 @@
#pragma once
#include "evo/persistence/checkpoint_store.hpp"
#include "evo/server/ops.hpp"
#include <chrono>
#include <cstdint>
#include <expected>
#include <functional>
#include <memory>
#include <optional>
#include <span>
#include <vector>
namespace evo::server {
using CellularConnect =
std::function<std::expected<std::unique_ptr<persistence::Postgres>, persistence::PgFailure>()>;
struct CellularHostConfig {
persistence::RegionKey key{};
std::uint64_t frame{};
persistence::LeasePolicy lease{};
persistence::Sha256Digest build{};
std::vector<persistence::Sha256Digest> retained_builds;
};
enum class CellularHostError : std::uint8_t { configuration, not_ready, invalid_batch, publication };
struct FinalCellularAction {
CellularActionRecord action;
std::uint64_t commit_sequence{};
friend bool operator==(const FinalCellularAction&, const FinalCellularAction&) = default;
};
// Single-writer profile-4 authority. The caller supplies a complete bounded trace;
// the host replays it from its recovered state and advances only on a confirmed head.
class CellularRegionHost {
public:
[[nodiscard]] static std::expected<std::unique_ptr<CellularRegionHost>,
CellularHostError>
create(CellularHostConfig config, CellularConnect connect,
std::unique_ptr<persistence::FilesystemObjectStore> objects);
CellularRegionHost(const CellularRegionHost&) = delete;
CellularRegionHost& operator=(const CellularRegionHost&) = delete;
~CellularRegionHost();
// Acquires/renews authority, recovers fully, or resolves the exact pending commit.
const HostStatus& step();
[[nodiscard]] std::expected<persistence::PublishedCellularBatch, CellularHostError>
publish(const CellularBatch& batch,
std::span<const CellularActionRecord> actions = {});
[[nodiscard]] std::vector<FinalCellularAction> take_finalized_actions();
void drain();
[[nodiscard]] std::optional<persistence::LeaseGrant> authority() const;
[[nodiscard]] const HostStatus& status() const noexcept { return status_; }
[[nodiscard]] const CellularLives* state() const noexcept { return state_.get(); }
[[nodiscard]] const std::optional<persistence::CommittedCheckpoint>& committed() const noexcept {
return committed_;
}
private:
struct Pending;
CellularRegionHost(CellularHostConfig config, CellularConnect connect,
std::unique_ptr<persistence::FilesystemObjectStore> objects);
bool database();
void acquire();
void upkeep();
void resolve();
void refresh();
CellularHostConfig config_;
CellularConnect connect_;
std::unique_ptr<persistence::FilesystemObjectStore> objects_;
std::unique_ptr<persistence::Postgres> db_;
persistence::LeaseAuthority authority_;
std::unique_ptr<CellularLives> state_;
std::optional<persistence::CommittedCheckpoint> committed_;
std::unique_ptr<Pending> pending_;
std::vector<FinalCellularAction> finalized_actions_;
HostStatus status_{};
};
} // namespace evo::server
+351
View File
@@ -0,0 +1,351 @@
#include "evo/server/cellular_region_host.hpp"
#include "evo/persistence/sha256.hpp"
#include <algorithm>
#include <array>
#include <chrono>
#include <utility>
namespace evo::server {
struct CellularRegionHost::Pending {
persistence::CellularPreparedCommit prepared;
std::unique_ptr<CellularLives> candidate;
std::vector<CellularActionRecord> actions;
};
namespace {
using persistence::LeaseClock;
using persistence::LeaseStoreError;
using std::chrono::milliseconds;
bool valid(const CellularHostConfig& config) {
return config.key.world != 0 && config.key.region != 0 && config.frame != 0 &&
persistence::valid(config.lease) && config.build != persistence::Sha256Digest{} &&
std::ranges::find(config.retained_builds, config.build) != config.retained_builds.end();
}
std::optional<persistence::Uuid> commit_id(const persistence::CommittedCheckpoint& previous,
std::span<const std::uint8_t> trace,
std::span<const std::uint8_t> actions) {
std::vector<std::uint8_t> identity;
identity.reserve(previous.head.digest.size() + 16 + trace.size() + actions.size());
identity.insert(identity.end(), previous.head.digest.begin(), previous.head.digest.end());
const auto append_length = [&](std::size_t size) {
const auto length = static_cast<std::uint64_t>(size);
for (unsigned i = 0; i < 8; ++i)
identity.push_back(static_cast<std::uint8_t>(length >> (i * 8U)));
};
append_length(trace.size());
identity.insert(identity.end(), trace.begin(), trace.end());
append_length(actions.size());
identity.insert(identity.end(), actions.begin(), actions.end());
const auto digest = persistence::sha256(identity);
if (!digest)
return std::nullopt;
persistence::Uuid out{};
std::ranges::copy_n(digest->begin(), out.size(), out.begin());
if (out == persistence::Uuid{})
out.back() = 1;
return out;
}
} // namespace
CellularRegionHost::CellularRegionHost(
CellularHostConfig config, CellularConnect connect,
std::unique_ptr<persistence::FilesystemObjectStore> objects)
: config_{std::move(config)}, connect_{std::move(connect)}, objects_{std::move(objects)},
authority_{config_.lease} {}
CellularRegionHost::~CellularRegionHost() = default;
std::expected<std::unique_ptr<CellularRegionHost>, CellularHostError>
CellularRegionHost::create(CellularHostConfig config, CellularConnect connect,
std::unique_ptr<persistence::FilesystemObjectStore> objects) {
if (!valid(config) || !connect || !objects)
return std::unexpected{CellularHostError::configuration};
return std::unique_ptr<CellularRegionHost>{
new CellularRegionHost{std::move(config), std::move(connect), std::move(objects)}};
}
bool CellularRegionHost::database() {
if (db_)
return true;
auto connection = connect_();
if (!connection) {
++status_.store_failures;
return false;
}
db_ = std::move(*connection);
return true;
}
void CellularRegionHost::acquire() {
if (!database())
return;
const auto sent = LeaseClock::now();
auto lease = persistence::acquire_region_lease(*db_, config_.key, config_.build,
config_.lease.duration,
config_.lease.renew_timeout);
if (!lease) {
if (lease.error().kind == LeaseStoreError::not_provisioned ||
lease.error().kind == LeaseStoreError::unsupported) {
status_.phase = HostPhase::quarantined;
} else if (lease.error().kind != LeaseStoreError::busy) {
db_.reset();
++status_.store_failures;
}
return;
}
if (lease->frame != config_.frame ||
!authority_.acquired(*lease, sent, LeaseClock::now())) {
status_.phase = HostPhase::quarantined;
return;
}
auto recovered = persistence::recover_cellular_checkpoint(
*db_, *objects_, config_.key, *lease, config_.retained_builds,
config_.lease.commit_timeout);
if (!recovered) {
if (recovered.error().kind == persistence::RecoveryError::store) {
db_.reset();
++status_.store_failures;
return;
}
authority_.fence();
status_.phase = HostPhase::quarantined;
return;
}
state_ = std::move(recovered->state);
committed_ = std::move(recovered->committed);
status_.phase = HostPhase::running;
status_.simulated_tick = committed_->head.end_tick;
status_.committed_tick = committed_->head.end_tick;
status_.committed_sequence = committed_->head.sequence;
}
void CellularRegionHost::upkeep() {
if (authority_.state(LeaseClock::now()) == persistence::LeaseState::fenced) {
authority_.fence();
state_.reset();
committed_.reset();
pending_.reset();
status_.ready = false;
status_.phase = HostPhase::acquiring;
++status_.fences;
return;
}
if (authority_.state(LeaseClock::now()) != persistence::LeaseState::renew_due || !database())
return;
const auto grant = *authority_.grant();
const auto sent = LeaseClock::now();
auto renewed = persistence::renew_region_lease(*db_, config_.key, grant, config_.lease.duration,
config_.lease.renew_timeout);
if (renewed) {
if (!authority_.renewed(grant.epoch, *renewed, sent, LeaseClock::now())) {
authority_.fence();
state_.reset();
committed_.reset();
pending_.reset();
status_.ready = false;
status_.phase = HostPhase::acquiring;
}
return;
}
if (renewed.error().kind == LeaseStoreError::unknown)
authority_.renewal_unknown(config_.lease.duration, sent);
else if (renewed.error().kind == LeaseStoreError::fenced) {
authority_.fence();
state_.reset();
committed_.reset();
pending_.reset();
status_.ready = false;
status_.phase = HostPhase::acquiring;
++status_.fences;
}
db_.reset();
++status_.store_failures;
}
void CellularRegionHost::resolve() {
if (!pending_ || !database())
return;
if (!authority_.may_begin_commit(LeaseClock::now())) {
authority_.fence();
state_.reset();
committed_.reset();
pending_.reset();
status_.phase = HostPhase::acquiring;
status_.ready = false;
++status_.fences;
return;
}
const auto resolved = persistence::resolve_cellular_prepared(
*db_, pending_->prepared, config_.lease.commit_timeout);
if (!resolved) {
if (resolved.error().kind == LeaseStoreError::fenced) {
authority_.fence();
state_.reset();
committed_.reset();
pending_.reset();
status_.phase = HostPhase::acquiring;
status_.ready = false;
++status_.fences;
} else {
db_.reset();
++status_.store_failures;
}
return;
}
if (resolved->status == persistence::Resolution::conflict) {
authority_.fence();
state_.reset();
committed_.reset();
pending_.reset();
status_.phase = HostPhase::quarantined;
status_.ready = false;
return;
}
if (resolved->status == persistence::Resolution::absent &&
!authority_.may_begin_commit(LeaseClock::now()))
return;
auto retried = persistence::retry_cellular_prepared(
*db_, pending_->prepared, config_.lease.commit_timeout);
if (!retried) {
if (retried.error().kind == persistence::PublicationError::unknown) {
db_.reset();
++status_.unknown_outcomes;
status_.phase = HostPhase::resolving;
status_.ready = false;
} else if (retried.error().store.kind == LeaseStoreError::fenced) {
authority_.fence();
state_.reset();
committed_.reset();
pending_.reset();
status_.phase = HostPhase::acquiring;
status_.ready = false;
++status_.fences;
} else {
authority_.fence();
state_.reset();
committed_.reset();
pending_.reset();
status_.phase = HostPhase::quarantined;
status_.ready = false;
}
return;
}
state_ = std::move(pending_->candidate);
committed_ = std::move(*retried);
for (const auto& action : pending_->actions)
finalized_actions_.push_back({action, committed_->head.sequence});
pending_.reset();
++status_.commits;
status_.committed_sequence = committed_->head.sequence;
status_.committed_tick = committed_->head.end_tick;
status_.simulated_tick = committed_->head.end_tick;
status_.phase = HostPhase::running;
}
void CellularRegionHost::refresh() {
const auto& grant = authority_.grant();
status_.ready = status_.phase == HostPhase::running && state_ && committed_ && grant &&
authority_.may_originate(LeaseClock::now());
status_.epoch = grant ? grant->epoch : 0;
status_.lease_remaining_ms =
status_.ready
? static_cast<std::uint64_t>(std::chrono::duration_cast<milliseconds>(
authority_.deadline() - LeaseClock::now())
.count())
: 0;
}
const HostStatus& CellularRegionHost::step() {
if (status_.phase == HostPhase::quarantined || status_.phase == HostPhase::stopped)
return status_;
if (authority_.grant())
upkeep();
if (pending_)
resolve();
else if (!state_)
acquire();
refresh();
return status_;
}
std::expected<persistence::PublishedCellularBatch, CellularHostError>
CellularRegionHost::publish(const CellularBatch& batch,
std::span<const CellularActionRecord> actions) {
if (!status_.ready || !state_ || !committed_ ||
!authority_.may_begin_commit(LeaseClock::now()))
return std::unexpected{CellularHostError::not_ready};
auto candidate = replay_cellular_batch(*state_, batch);
if (!candidate)
return std::unexpected{CellularHostError::invalid_batch};
auto trace = encode_cellular_batch(batch);
auto action_block = encode_cellular_action_block(actions, batch.header.begin_tick,
batch.header.end_tick);
const auto commit = trace && action_block
? commit_id(*committed_, *trace, *action_block)
: std::nullopt;
if (!commit)
return std::unexpected{CellularHostError::invalid_batch};
auto published = persistence::publish_cellular_batch(
*db_, *objects_, *authority_.grant(), *committed_, batch, **candidate, *commit,
config_.build, config_.lease.commit_timeout, {}, actions);
if (!published) {
if (published.error().kind == persistence::PublicationError::unknown) {
pending_ = std::make_unique<Pending>(
Pending{published.error().prepared, std::move(*candidate),
{actions.begin(), actions.end()}});
status_.phase = HostPhase::resolving;
status_.ready = false;
++status_.unknown_outcomes;
db_.reset();
} else if (published.error().store.kind == LeaseStoreError::fenced) {
authority_.fence();
state_.reset();
committed_.reset();
status_.phase = HostPhase::acquiring;
status_.ready = false;
++status_.fences;
}
return std::unexpected{CellularHostError::publication};
}
state_ = std::move(*candidate);
committed_ = published->committed;
for (const auto& action : actions)
finalized_actions_.push_back({action, committed_->head.sequence});
++status_.commits;
status_.committed_sequence = committed_->head.sequence;
status_.committed_tick = committed_->head.end_tick;
status_.simulated_tick = committed_->head.end_tick;
refresh();
return *published;
}
std::vector<FinalCellularAction> CellularRegionHost::take_finalized_actions() {
return std::exchange(finalized_actions_, {});
}
void CellularRegionHost::drain() {
status_.ready = false;
status_.phase = HostPhase::draining;
pending_.reset();
const auto grant = authority_.grant();
if (grant && authority_.may_originate(LeaseClock::now()) && database())
static_cast<void>(persistence::release_region_lease(
*db_, config_.key, *grant, config_.lease.renew_timeout));
authority_.fence();
state_.reset();
committed_.reset();
status_.phase = HostPhase::stopped;
status_.epoch = 0;
status_.lease_remaining_ms = 0;
}
std::optional<persistence::LeaseGrant> CellularRegionHost::authority() const {
if (!status_.ready || !authority_.may_originate(LeaseClock::now()))
return std::nullopt;
return authority_.grant();
}
} // namespace evo::server
+33 -19
View File
@@ -1,6 +1,7 @@
// Profile-4 publication and recovery against the migration-backed PostgreSQL adapter.
#include "evo/persistence/checkpoint_store.hpp"
#include "evo/persistence/sha256.hpp"
#include "evo/server/cellular_region_host.hpp"
#include <cstdlib>
#include <source_location>
@@ -46,7 +47,6 @@ std::string hex(std::span<const std::uint8_t> bytes) {
void run_cellular_publication_tests() {
auto owner = connect_as(env("PGUSER"));
auto db = connect_as(env("EVO_PG_RUNTIME_USER"));
constexpr RegionKey key{9756, 1};
const auto build = filled<32>(0xD9);
CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
@@ -83,34 +83,48 @@ void run_cellular_publication_tests() {
CHECK(objects && (*objects)->put(payload));
CHECK(owner->query("SELECT evolution.provision_region_checkpoint($1::bytea)",
std::array{hex(*genesis_bytes)}, 5s));
auto predecessor = load_region_checkpoint(*owner, key, 5s);
CHECK(predecessor);
const auto lease = acquire_region_lease(*db, key, build, 20'000ms, 5s);
CHECK(lease);
const auto predecessor_digest = sha256(predecessor->manifest);
CHECK(predecessor_digest);
const auto connect = [&]() -> std::expected<std::unique_ptr<Postgres>, PgFailure> {
return Postgres::connect({env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"),
env("EVO_PG_RUNTIME_USER"), {}, {}}, 5s);
};
CellularHostConfig config{key, 1, {}, build, {build}};
auto made_host = CellularRegionHost::create(config, connect, std::move(*objects));
CHECK(made_host);
auto host = std::move(*made_host);
CHECK(host->step().ready);
CellularBatch batch;
batch.header = {key.world, key.region, 1, predecessor->head.commit, *predecessor_digest, 0, 2};
const auto predecessor_digest = sha256(host->committed()->manifest);
CHECK(predecessor_digest);
batch.header = {key.world, key.region, 1, host->committed()->head.commit,
*predecessor_digest, 0, 2};
batch.ticks.resize(2);
for (auto& tick : batch.ticks)
tick.lives.push_back({(*opening)->lives()[0].identity.life, {}});
const auto replay = replay_cellular_batch(**opening, batch);
const auto replay = replay_cellular_batch(*host->state(), batch);
CHECK(replay);
CellularActionRecord action;
action.account = founder.account;
action.action_id = 77;
action.input[0] = 1; // Tick 1, neutral movement, no patch target.
const std::array actions{action};
auto published = publish_cellular_batch(*db, **objects, *lease, *predecessor, batch,
**replay, filled<16>(0x22), build, 5s, {}, actions);
auto published = host->publish(batch, actions);
CHECK(published && published->committed.head.sequence == 1 &&
published->committed.head.end_tick == 2);
published->committed.head.end_tick == 2 &&
host->state()->snapshot() == (**replay).snapshot());
CHECK(host->take_finalized_actions() ==
std::vector<FinalCellularAction>{{action, 1}});
host->drain();
host.reset();
auto recovered = recover_cellular_checkpoint(*db, **objects, key, *lease,
std::span{&build, 1}, 5s);
CHECK(recovered && recovered->manifest.sequence == 1 && recovered->state->tick() == 2 &&
recovered->state->snapshot() == (**replay).snapshot() &&
recovered->actions == std::vector<CellularActionRecord>{action});
CHECK(release_region_lease(*db, key, *lease, 5s));
auto restarted_objects = FilesystemObjectStore::open(env("PGHOST"), key,
max_cellular_lives_bytes);
CHECK(restarted_objects);
auto made_restarted = CellularRegionHost::create(config, connect,
std::move(*restarted_objects));
CHECK(made_restarted);
auto restarted = std::move(*made_restarted);
CHECK(restarted->step().ready && restarted->state()->tick() == 2 &&
restarted->state()->snapshot() == (**replay).snapshot() &&
restarted->committed()->head.sequence == 1);
restarted->drain();
}