Complete M7.6 bounded authoritative region host

ADR-0064: RegionHost<State> (environmental and lineage profiles) is a
single-owner, single-threaded state machine advanced by step(now):
acquire the fenced lease and recover the verified head, run, resolve
ambiguous commits, drain and stop, or quarantine. Admission is
provisional and bounded (64 inputs, 64 KiB) with backpressure; FinalAck
and FinalOutcome are emitted only after a confirmed commit, and inputs
that will never be acknowledged are reported with a reason. Ticks follow
the caller's scheduling clock with bounded catch-up and measured tick
debt, never invented inputs. A death seals the batch and commits in the
same step, so no dependent tick uses an uncommitted outcome. Storage
outages pause and retry, unknown outcomes resolve by exact bytes,
fencing discards unconfirmed work and re-acquires; readiness fails
closed. Drain commits pending ticks and releases the lease.

evo-server hosts one region with a bounded stdin input feed, stdout
event lines, a readiness file and SIGTERM drain (exit 0/3/4).

Validated: Windows CTest 58/58; PostgreSQL suites under GCC Release,
clang-tidy and ASan/UBSan including real-process SIGTERM drain and
SIGKILL with successor recovery; four host mutations each caught. No
gateway, metrics, Docker/Gitea CI or deployment.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-28 20:53:11 -04:00
co-authored by Claude Opus 5.5
parent ab4bf5355f
commit d6d0408aae
15 changed files with 1934 additions and 3 deletions
+471
View File
@@ -0,0 +1,471 @@
#include "evo/server/region_host.hpp"
#include "evo/persistence/sha256.hpp"
#include <algorithm>
#include <utility>
namespace evo::server {
namespace {
using persistence::LeaseClock;
using persistence::LeaseStoreError;
using persistence::PublicationError;
using persistence::RecoveryError;
// Per-profile calls into the persistence layer.
template <typename State> struct Ops;
template <> struct Ops<processes::EnvironmentalCommunity> {
using State = processes::EnvironmentalCommunity;
static std::uint64_t tick(const State& s) {
return s.progress().tick;
}
static std::uint64_t quantum(const State& s) {
return s.tick_microseconds();
}
template <typename... A> static auto recover(A&&... a) {
return persistence::recover_environmental_checkpoint(std::forward<A>(a)...);
}
template <typename... A> static auto publish(A&&... a) {
return persistence::publish_environmental_batch(std::forward<A>(a)...);
}
};
template <> struct Ops<processes::LineageRegion> {
using State = processes::LineageRegion;
static std::uint64_t tick(const State& s) {
return s.community().progress().tick;
}
static std::uint64_t quantum(const State& s) {
return s.community().tick_microseconds();
}
template <typename... A> static auto recover(A&&... a) {
return persistence::recover_lineage_checkpoint(std::forward<A>(a)...);
}
template <typename... A> static auto publish(A&&... a) {
return persistence::publish_lineage_batch(std::forward<A>(a)...);
}
};
bool valid(const HostConfig& c) {
constexpr std::size_t max_backlog_inputs = 64;
constexpr std::size_t max_backlog_bytes = 65536;
return c.key.world != 0 && c.key.region != 0 && c.frame != 0 && persistence::valid(c.lease) &&
c.tick_quantum.count() > 0 && c.flush_interval.count() > 0 && c.max_queued_inputs >= 1 &&
c.max_queued_inputs <= max_backlog_inputs && c.max_queued_bytes >= tick_input_bytes &&
c.max_queued_bytes <= max_backlog_bytes && c.max_catch_up_ticks >= 1 &&
c.max_catch_up_ticks <= 64 && c.build != persistence::Sha256Digest{} &&
std::ranges::find(c.retained_builds, c.build) != c.retained_builds.end() &&
c.restore_budget > 0;
}
} // namespace
template <typename State>
RegionHost<State>::RegionHost(HostConfig config, Connect connect,
std::unique_ptr<persistence::FilesystemObjectStore> objects)
: config_(std::move(config)), connect_(std::move(connect)), objects_(std::move(objects)),
authority_(config_.lease) {}
template <typename State> RegionHost<State>::~RegionHost() = default;
template <typename State>
std::expected<std::unique_ptr<RegionHost<State>>, HostConfigError>
RegionHost<State>::create(HostConfig config, Connect connect,
std::unique_ptr<persistence::FilesystemObjectStore> objects) {
if (!valid(config) || !connect || !objects)
return std::unexpected{HostConfigError::invalid};
return std::unique_ptr<RegionHost>(
new RegionHost(std::move(config), std::move(connect), std::move(objects)));
}
template <typename State>
std::expected<void, AdmissionError> RegionHost<State>::submit(const TickInput& input) {
if (input.request == persistence::Uuid{})
return std::unexpected{AdmissionError::invalid};
if (draining_)
return std::unexpected{AdmissionError::draining};
if (!status_.ready)
return std::unexpected{AdmissionError::not_ready};
const auto same = [&](const TickInput& x) { return x.request == input.request; };
if (std::ranges::any_of(queue_, same) ||
std::ranges::any_of(applied_, [&](const Applied& a) { return same(a.input); }))
return std::unexpected{AdmissionError::duplicate};
if (queue_.size() >= config_.max_queued_inputs ||
(queue_.size() + 1) * tick_input_bytes > config_.max_queued_bytes)
return std::unexpected{AdmissionError::full};
queue_.push_back(input);
status_.queued = queue_.size();
status_.queued_bytes = queue_.size() * tick_input_bytes;
return {};
}
template <typename State> bool RegionHost<State>::database() {
if (db_)
return true;
auto made = connect_();
if (!made) {
++status_.store_failures;
return false;
}
db_ = std::move(*made);
return true;
}
template <typename State>
void RegionHost<State>::lose_authority(Unacknowledged reason, HostPhase next) {
for (auto& a : applied_)
dropped_.push_back({a.input, reason});
for (auto& q : queue_)
dropped_.push_back({q, reason});
applied_.clear();
queue_.clear();
batch_.reset(); // References state_: reset first.
state_.reset();
committed_.reset();
prepared_.reset();
// Best effort: a quarantined region's lease is released so peers see it free.
const auto& grant = authority_.grant();
if (next == HostPhase::quarantined && db_ && grant &&
authority_.may_originate(LeaseClock::now())) {
[[maybe_unused]] const auto released = persistence::release_region_lease(
*db_, config_.key, *grant, config_.lease.renew_timeout);
}
authority_.fence();
if (reason == Unacknowledged::fenced)
++status_.fences;
status_.phase = draining_ && next == HostPhase::acquiring ? HostPhase::stopped : next;
}
template <typename State> void RegionHost<State>::acquire(SchedulerTime now) {
if (!database())
return;
// A grant survives a transient recovery failure; only re-acquire without one.
if (!authority_.may_originate(LeaseClock::now())) {
const auto sent = LeaseClock::now();
auto grant = persistence::acquire_region_lease(*db_, config_.key, config_.lease.duration,
config_.lease.renew_timeout);
if (!grant) {
if (grant.error().kind == LeaseStoreError::not_provisioned)
status_.phase = HostPhase::quarantined;
else if (grant.error().kind != LeaseStoreError::busy)
db_.reset();
return;
}
if (grant->frame != config_.frame) {
status_.phase = HostPhase::quarantined;
return;
}
if (!authority_.acquired(*grant, sent, LeaseClock::now()))
return; // Reply too late to use; the lease expires unused.
}
auto recovered = Ops<State>::recover(*db_, *objects_, config_.key, config_.retained_builds,
config_.restore_budget, config_.lease.commit_timeout);
if (!recovered) {
if (recovered.error().kind == RecoveryError::store) {
db_.reset(); // Transient: keep the grant and retry recovery.
++status_.store_failures;
return;
}
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
return;
}
if (Ops<State>::quantum(*recovered->state) !=
static_cast<std::uint64_t>(config_.tick_quantum.count())) {
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
return;
}
state_ = std::move(recovered->state);
committed_ = std::move(recovered->committed);
batch_.emplace(*state_);
origin_ = now;
origin_tick_ = Ops<State>::tick(*state_);
status_.max_tick_debt = 0;
status_.phase = draining_ ? HostPhase::draining : HostPhase::running;
}
template <typename State> void RegionHost<State>::upkeep() {
const auto state = authority_.state(LeaseClock::now());
if (state == persistence::LeaseState::fenced) {
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
return;
}
if (state != persistence::LeaseState::renew_due || !database())
return;
const auto grant = *authority_.grant();
const auto sent = LeaseClock::now();
auto renewed = persistence::renew_region_lease(*db_, config_.key, grant, config_.lease.duration,
config_.lease.renew_timeout);
if (renewed) {
if (!authority_.renewed(grant.epoch, *renewed, sent, LeaseClock::now()))
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
return;
}
switch (renewed.error().kind) {
case LeaseStoreError::unknown:
// It may have shortened or extended nothing: keep the earlier deadline.
authority_.renewal_unknown(config_.lease.duration, sent);
db_.reset();
++status_.store_failures;
return;
case LeaseStoreError::fenced:
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
return;
default:
db_.reset();
++status_.store_failures;
return;
}
}
template <typename State> persistence::Uuid RegionHost<State>::next_commit_id() const {
// Unique: the DB-generated holder token names this acquisition; the
// sequence names the link. Stable across retries of the same batch.
const auto& grant = authority_.grant();
if (!grant || !committed_)
return {};
auto id = grant->holder;
const auto sequence = committed_->head.sequence + 1;
for (unsigned i = 0; i < 8; ++i)
id[8 + i] = static_cast<std::uint8_t>((sequence >> (8U * i)) & 0xFFU);
return id;
}
template <typename State>
void RegionHost<State>::confirmed(const persistence::PublishedBatch& published, SchedulerTime now) {
const auto& head = published.committed.head;
if (!batch_ || !batch_->confirm(head.end_tick)) {
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
return;
}
committed_ = published.committed;
for (const auto& a : applied_)
acks_.push_back({a.input.request, a.tick, head.sequence});
applied_.clear();
for (const auto& d : published.decisions)
outcomes_.push_back({d, head.sequence});
prepared_.reset();
batch_started_ = now;
++status_.commits;
status_.phase = draining_ ? HostPhase::draining : HostPhase::running;
}
template <typename State> void RegionHost<State>::commit(SchedulerTime now) {
if (!authority_.may_begin_commit(LeaseClock::now()))
return; // Renewal first; upkeep fences if the deadline passes.
if (!database())
return;
const auto& grant = authority_.grant();
if (!grant || !committed_ || !batch_)
return;
const auto commit_id = next_commit_id();
batch_->seal();
auto published =
Ops<State>::publish(*db_, *objects_, *grant, *committed_, *batch_, commit_id, config_.build,
config_.restore_budget, config_.lease.commit_timeout);
if (published) {
confirmed(*published, now);
return;
}
const auto& f = published.error();
switch (f.kind) {
case PublicationError::upload:
case PublicationError::verify:
case PublicationError::digest:
++status_.store_failures; // Nothing sent: retry the same batch later.
return;
case PublicationError::unknown:
prepared_ = f.prepared;
db_.reset();
++status_.unknown_outcomes;
status_.phase = HostPhase::resolving;
return;
case PublicationError::rejected:
if (f.store.kind == LeaseStoreError::fenced) {
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
return;
}
if (f.store.kind == LeaseStoreError::store || f.store.kind == LeaseStoreError::invalid) {
// SQL/connection error before any mutation: retry with the same bytes.
db_.reset();
++status_.store_failures;
return;
}
[[fallthrough]];
default: // candidate, predecessor, receipt, head moved, conflicts: fail closed.
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
return;
}
}
template <typename State> void RegionHost<State>::resolve(SchedulerTime now) {
if (!database() || !prepared_ || !batch_)
return;
const auto& prepared = *prepared_;
auto resolved = persistence::resolve_prepared(*db_, prepared, config_.lease.commit_timeout);
if (!resolved) {
db_.reset();
++status_.store_failures;
return;
}
const auto adopt = [&](const persistence::RegionHead& receipt) {
auto decisions = persistence::decode_decisions(prepared.decisions, batch_->begin_tick(),
batch_->end_tick());
const auto digest = persistence::sha256(prepared.manifest);
if (!decisions || !digest || *digest != receipt.digest ||
receipt.end_tick != batch_->end_tick()) {
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
return;
}
confirmed({{receipt, prepared.manifest}, *decisions}, now);
};
switch (resolved->status) {
case persistence::Resolution::committed:
adopt(resolved->receipt);
return;
case persistence::Resolution::conflict:
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
return;
case persistence::Resolution::absent:
break;
}
// Absent now: retry exactly the same bytes, only while still authorized.
if (!authority_.may_begin_commit(LeaseClock::now()))
return;
auto retried = persistence::retry_prepared(*db_, prepared, config_.lease.commit_timeout);
if (retried) {
adopt(*retried);
return;
}
switch (retried.error().kind) {
case LeaseStoreError::fenced:
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
return;
case LeaseStoreError::unknown:
case LeaseStoreError::store:
db_.reset();
++status_.store_failures;
return;
default:
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
return;
}
}
template <typename State> void RegionHost<State>::simulate(SchedulerTime now) {
const auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(now - origin_);
const auto due = elapsed.count() <= 0
? std::uint64_t{0}
: static_cast<std::uint64_t>(elapsed / config_.tick_quantum);
if (!batch_ || !state_)
return;
auto& batch = *batch_;
const auto& state = *state_;
for (std::uint32_t k = 0; k < config_.max_catch_up_ticks && !batch.sealed(); ++k) {
if (Ops<State>::tick(state) - origin_tick_ >= due || queue_.empty() ||
!authority_.may_originate(LeaseClock::now()))
break; // Not due, starved (debt accrues; inputs are never invented) or unauthorized.
const auto input = queue_.front();
queue_.pop_front();
if (batch.steps().empty())
batch_started_ = now;
auto report = batch.advance(input.input);
if (!report) {
dropped_.push_back({input, Unacknowledged::refused});
continue;
}
applied_.push_back({input, report->tick});
}
}
template <typename State> void RegionHost<State>::refresh(SchedulerTime now) {
status_.queued = queue_.size();
status_.queued_bytes = queue_.size() * tick_input_bytes;
const auto& grant = authority_.grant();
status_.epoch = grant ? grant->epoch : 0;
status_.committed_tick = committed_ ? committed_->head.end_tick : 0;
status_.committed_sequence = committed_ ? committed_->head.sequence : 0;
status_.simulated_tick = state_ ? Ops<State>::tick(*state_) : status_.committed_tick;
if (state_ && status_.phase == HostPhase::running) {
const auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(now - origin_);
const auto due = elapsed.count() <= 0
? std::uint64_t{0}
: static_cast<std::uint64_t>(elapsed / config_.tick_quantum);
const auto done = Ops<State>::tick(*state_) - origin_tick_;
status_.tick_debt = due > done ? due - done : 0;
status_.max_tick_debt = std::max(status_.max_tick_debt, status_.tick_debt);
}
status_.ready = status_.phase == HostPhase::running && state_ && !draining_ &&
authority_.may_originate(LeaseClock::now());
}
template <typename State> const HostStatus& RegionHost<State>::step(SchedulerTime now) {
switch (status_.phase) {
case HostPhase::acquiring:
acquire(now);
break;
case HostPhase::resolving:
upkeep();
if (status_.phase == HostPhase::resolving)
resolve(now);
break;
case HostPhase::running:
case HostPhase::draining: {
upkeep();
if (status_.phase != HostPhase::running && status_.phase != HostPhase::draining)
break;
if (!draining_)
simulate(now);
if (!batch_)
break; // Running always has a batch; defensive.
const bool flush =
!batch_->steps().empty() &&
(batch_->sealed() || draining_ || now - batch_started_ >= config_.flush_interval);
if (flush)
commit(now); // May lose authority and reset the batch.
if (draining_ && status_.phase == HostPhase::draining && batch_ &&
batch_->steps().empty()) {
const auto& grant = authority_.grant();
if (database() && grant && authority_.may_originate(LeaseClock::now())) {
// Best effort: an unreleased lease simply expires.
[[maybe_unused]] const auto released = persistence::release_region_lease(
*db_, config_.key, *grant, config_.lease.renew_timeout);
}
batch_.reset();
state_.reset();
authority_.fence();
db_.reset();
status_.phase = HostPhase::stopped;
}
break;
}
case HostPhase::stopped:
case HostPhase::quarantined:
break;
}
refresh(now);
return status_;
}
template <typename State> void RegionHost<State>::drain() {
if (draining_)
return;
draining_ = true;
for (auto& q : queue_)
dropped_.push_back({q, Unacknowledged::drained});
queue_.clear();
if (status_.phase == HostPhase::running)
status_.phase = HostPhase::draining;
else if (status_.phase == HostPhase::acquiring)
status_.phase = HostPhase::stopped;
status_.ready = false;
}
template <typename State> std::vector<FinalAck> RegionHost<State>::take_acks() {
return std::exchange(acks_, {});
}
template <typename State> std::vector<FinalOutcome> RegionHost<State>::take_outcomes() {
return std::exchange(outcomes_, {});
}
template <typename State> std::vector<Dropped> RegionHost<State>::take_dropped() {
return std::exchange(dropped_, {});
}
template class RegionHost<processes::EnvironmentalCommunity>;
template class RegionHost<processes::LineageRegion>;
} // namespace evo::server