Complete M7.6 bounded authoritative region host
ADR-0064: RegionHost<State> (environmental and lineage profiles) is a single-owner, single-threaded state machine advanced by step(now): acquire the fenced lease and recover the verified head, run, resolve ambiguous commits, drain and stop, or quarantine. Admission is provisional and bounded (64 inputs, 64 KiB) with backpressure; FinalAck and FinalOutcome are emitted only after a confirmed commit, and inputs that will never be acknowledged are reported with a reason. Ticks follow the caller's scheduling clock with bounded catch-up and measured tick debt, never invented inputs. A death seals the batch and commits in the same step, so no dependent tick uses an uncommitted outcome. Storage outages pause and retry, unknown outcomes resolve by exact bytes, fencing discards unconfirmed work and re-acquires; readiness fails closed. Drain commits pending ticks and releases the lease. evo-server hosts one region with a bounded stdin input feed, stdout event lines, a readiness file and SIGTERM drain (exit 0/3/4). Validated: Windows CTest 58/58; PostgreSQL suites under GCC Release, clang-tidy and ASan/UBSan including real-process SIGTERM drain and SIGKILL with successor recovery; four host mutations each caught. No gateway, metrics, Docker/Gitea CI or deployment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,471 @@
|
||||
#include "evo/server/region_host.hpp"
|
||||
#include "evo/persistence/sha256.hpp"
|
||||
#include <algorithm>
|
||||
#include <utility>
|
||||
|
||||
namespace evo::server {
|
||||
namespace {
|
||||
using persistence::LeaseClock;
|
||||
using persistence::LeaseStoreError;
|
||||
using persistence::PublicationError;
|
||||
using persistence::RecoveryError;
|
||||
|
||||
// Per-profile calls into the persistence layer.
|
||||
template <typename State> struct Ops;
|
||||
template <> struct Ops<processes::EnvironmentalCommunity> {
|
||||
using State = processes::EnvironmentalCommunity;
|
||||
static std::uint64_t tick(const State& s) {
|
||||
return s.progress().tick;
|
||||
}
|
||||
static std::uint64_t quantum(const State& s) {
|
||||
return s.tick_microseconds();
|
||||
}
|
||||
template <typename... A> static auto recover(A&&... a) {
|
||||
return persistence::recover_environmental_checkpoint(std::forward<A>(a)...);
|
||||
}
|
||||
template <typename... A> static auto publish(A&&... a) {
|
||||
return persistence::publish_environmental_batch(std::forward<A>(a)...);
|
||||
}
|
||||
};
|
||||
template <> struct Ops<processes::LineageRegion> {
|
||||
using State = processes::LineageRegion;
|
||||
static std::uint64_t tick(const State& s) {
|
||||
return s.community().progress().tick;
|
||||
}
|
||||
static std::uint64_t quantum(const State& s) {
|
||||
return s.community().tick_microseconds();
|
||||
}
|
||||
template <typename... A> static auto recover(A&&... a) {
|
||||
return persistence::recover_lineage_checkpoint(std::forward<A>(a)...);
|
||||
}
|
||||
template <typename... A> static auto publish(A&&... a) {
|
||||
return persistence::publish_lineage_batch(std::forward<A>(a)...);
|
||||
}
|
||||
};
|
||||
|
||||
bool valid(const HostConfig& c) {
|
||||
constexpr std::size_t max_backlog_inputs = 64;
|
||||
constexpr std::size_t max_backlog_bytes = 65536;
|
||||
return c.key.world != 0 && c.key.region != 0 && c.frame != 0 && persistence::valid(c.lease) &&
|
||||
c.tick_quantum.count() > 0 && c.flush_interval.count() > 0 && c.max_queued_inputs >= 1 &&
|
||||
c.max_queued_inputs <= max_backlog_inputs && c.max_queued_bytes >= tick_input_bytes &&
|
||||
c.max_queued_bytes <= max_backlog_bytes && c.max_catch_up_ticks >= 1 &&
|
||||
c.max_catch_up_ticks <= 64 && c.build != persistence::Sha256Digest{} &&
|
||||
std::ranges::find(c.retained_builds, c.build) != c.retained_builds.end() &&
|
||||
c.restore_budget > 0;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
template <typename State>
|
||||
RegionHost<State>::RegionHost(HostConfig config, Connect connect,
|
||||
std::unique_ptr<persistence::FilesystemObjectStore> objects)
|
||||
: config_(std::move(config)), connect_(std::move(connect)), objects_(std::move(objects)),
|
||||
authority_(config_.lease) {}
|
||||
|
||||
template <typename State> RegionHost<State>::~RegionHost() = default;
|
||||
|
||||
template <typename State>
|
||||
std::expected<std::unique_ptr<RegionHost<State>>, HostConfigError>
|
||||
RegionHost<State>::create(HostConfig config, Connect connect,
|
||||
std::unique_ptr<persistence::FilesystemObjectStore> objects) {
|
||||
if (!valid(config) || !connect || !objects)
|
||||
return std::unexpected{HostConfigError::invalid};
|
||||
return std::unique_ptr<RegionHost>(
|
||||
new RegionHost(std::move(config), std::move(connect), std::move(objects)));
|
||||
}
|
||||
|
||||
template <typename State>
|
||||
std::expected<void, AdmissionError> RegionHost<State>::submit(const TickInput& input) {
|
||||
if (input.request == persistence::Uuid{})
|
||||
return std::unexpected{AdmissionError::invalid};
|
||||
if (draining_)
|
||||
return std::unexpected{AdmissionError::draining};
|
||||
if (!status_.ready)
|
||||
return std::unexpected{AdmissionError::not_ready};
|
||||
const auto same = [&](const TickInput& x) { return x.request == input.request; };
|
||||
if (std::ranges::any_of(queue_, same) ||
|
||||
std::ranges::any_of(applied_, [&](const Applied& a) { return same(a.input); }))
|
||||
return std::unexpected{AdmissionError::duplicate};
|
||||
if (queue_.size() >= config_.max_queued_inputs ||
|
||||
(queue_.size() + 1) * tick_input_bytes > config_.max_queued_bytes)
|
||||
return std::unexpected{AdmissionError::full};
|
||||
queue_.push_back(input);
|
||||
status_.queued = queue_.size();
|
||||
status_.queued_bytes = queue_.size() * tick_input_bytes;
|
||||
return {};
|
||||
}
|
||||
|
||||
template <typename State> bool RegionHost<State>::database() {
|
||||
if (db_)
|
||||
return true;
|
||||
auto made = connect_();
|
||||
if (!made) {
|
||||
++status_.store_failures;
|
||||
return false;
|
||||
}
|
||||
db_ = std::move(*made);
|
||||
return true;
|
||||
}
|
||||
|
||||
template <typename State>
|
||||
void RegionHost<State>::lose_authority(Unacknowledged reason, HostPhase next) {
|
||||
for (auto& a : applied_)
|
||||
dropped_.push_back({a.input, reason});
|
||||
for (auto& q : queue_)
|
||||
dropped_.push_back({q, reason});
|
||||
applied_.clear();
|
||||
queue_.clear();
|
||||
batch_.reset(); // References state_: reset first.
|
||||
state_.reset();
|
||||
committed_.reset();
|
||||
prepared_.reset();
|
||||
// Best effort: a quarantined region's lease is released so peers see it free.
|
||||
const auto& grant = authority_.grant();
|
||||
if (next == HostPhase::quarantined && db_ && grant &&
|
||||
authority_.may_originate(LeaseClock::now())) {
|
||||
[[maybe_unused]] const auto released = persistence::release_region_lease(
|
||||
*db_, config_.key, *grant, config_.lease.renew_timeout);
|
||||
}
|
||||
authority_.fence();
|
||||
if (reason == Unacknowledged::fenced)
|
||||
++status_.fences;
|
||||
status_.phase = draining_ && next == HostPhase::acquiring ? HostPhase::stopped : next;
|
||||
}
|
||||
|
||||
template <typename State> void RegionHost<State>::acquire(SchedulerTime now) {
|
||||
if (!database())
|
||||
return;
|
||||
// A grant survives a transient recovery failure; only re-acquire without one.
|
||||
if (!authority_.may_originate(LeaseClock::now())) {
|
||||
const auto sent = LeaseClock::now();
|
||||
auto grant = persistence::acquire_region_lease(*db_, config_.key, config_.lease.duration,
|
||||
config_.lease.renew_timeout);
|
||||
if (!grant) {
|
||||
if (grant.error().kind == LeaseStoreError::not_provisioned)
|
||||
status_.phase = HostPhase::quarantined;
|
||||
else if (grant.error().kind != LeaseStoreError::busy)
|
||||
db_.reset();
|
||||
return;
|
||||
}
|
||||
if (grant->frame != config_.frame) {
|
||||
status_.phase = HostPhase::quarantined;
|
||||
return;
|
||||
}
|
||||
if (!authority_.acquired(*grant, sent, LeaseClock::now()))
|
||||
return; // Reply too late to use; the lease expires unused.
|
||||
}
|
||||
auto recovered = Ops<State>::recover(*db_, *objects_, config_.key, config_.retained_builds,
|
||||
config_.restore_budget, config_.lease.commit_timeout);
|
||||
if (!recovered) {
|
||||
if (recovered.error().kind == RecoveryError::store) {
|
||||
db_.reset(); // Transient: keep the grant and retry recovery.
|
||||
++status_.store_failures;
|
||||
return;
|
||||
}
|
||||
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
|
||||
return;
|
||||
}
|
||||
if (Ops<State>::quantum(*recovered->state) !=
|
||||
static_cast<std::uint64_t>(config_.tick_quantum.count())) {
|
||||
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
|
||||
return;
|
||||
}
|
||||
state_ = std::move(recovered->state);
|
||||
committed_ = std::move(recovered->committed);
|
||||
batch_.emplace(*state_);
|
||||
origin_ = now;
|
||||
origin_tick_ = Ops<State>::tick(*state_);
|
||||
status_.max_tick_debt = 0;
|
||||
status_.phase = draining_ ? HostPhase::draining : HostPhase::running;
|
||||
}
|
||||
|
||||
template <typename State> void RegionHost<State>::upkeep() {
|
||||
const auto state = authority_.state(LeaseClock::now());
|
||||
if (state == persistence::LeaseState::fenced) {
|
||||
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
|
||||
return;
|
||||
}
|
||||
if (state != persistence::LeaseState::renew_due || !database())
|
||||
return;
|
||||
const auto grant = *authority_.grant();
|
||||
const auto sent = LeaseClock::now();
|
||||
auto renewed = persistence::renew_region_lease(*db_, config_.key, grant, config_.lease.duration,
|
||||
config_.lease.renew_timeout);
|
||||
if (renewed) {
|
||||
if (!authority_.renewed(grant.epoch, *renewed, sent, LeaseClock::now()))
|
||||
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
|
||||
return;
|
||||
}
|
||||
switch (renewed.error().kind) {
|
||||
case LeaseStoreError::unknown:
|
||||
// It may have shortened or extended nothing: keep the earlier deadline.
|
||||
authority_.renewal_unknown(config_.lease.duration, sent);
|
||||
db_.reset();
|
||||
++status_.store_failures;
|
||||
return;
|
||||
case LeaseStoreError::fenced:
|
||||
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
|
||||
return;
|
||||
default:
|
||||
db_.reset();
|
||||
++status_.store_failures;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename State> persistence::Uuid RegionHost<State>::next_commit_id() const {
|
||||
// Unique: the DB-generated holder token names this acquisition; the
|
||||
// sequence names the link. Stable across retries of the same batch.
|
||||
const auto& grant = authority_.grant();
|
||||
if (!grant || !committed_)
|
||||
return {};
|
||||
auto id = grant->holder;
|
||||
const auto sequence = committed_->head.sequence + 1;
|
||||
for (unsigned i = 0; i < 8; ++i)
|
||||
id[8 + i] = static_cast<std::uint8_t>((sequence >> (8U * i)) & 0xFFU);
|
||||
return id;
|
||||
}
|
||||
|
||||
template <typename State>
|
||||
void RegionHost<State>::confirmed(const persistence::PublishedBatch& published, SchedulerTime now) {
|
||||
const auto& head = published.committed.head;
|
||||
if (!batch_ || !batch_->confirm(head.end_tick)) {
|
||||
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
|
||||
return;
|
||||
}
|
||||
committed_ = published.committed;
|
||||
for (const auto& a : applied_)
|
||||
acks_.push_back({a.input.request, a.tick, head.sequence});
|
||||
applied_.clear();
|
||||
for (const auto& d : published.decisions)
|
||||
outcomes_.push_back({d, head.sequence});
|
||||
prepared_.reset();
|
||||
batch_started_ = now;
|
||||
++status_.commits;
|
||||
status_.phase = draining_ ? HostPhase::draining : HostPhase::running;
|
||||
}
|
||||
|
||||
template <typename State> void RegionHost<State>::commit(SchedulerTime now) {
|
||||
if (!authority_.may_begin_commit(LeaseClock::now()))
|
||||
return; // Renewal first; upkeep fences if the deadline passes.
|
||||
if (!database())
|
||||
return;
|
||||
const auto& grant = authority_.grant();
|
||||
if (!grant || !committed_ || !batch_)
|
||||
return;
|
||||
const auto commit_id = next_commit_id();
|
||||
batch_->seal();
|
||||
auto published =
|
||||
Ops<State>::publish(*db_, *objects_, *grant, *committed_, *batch_, commit_id, config_.build,
|
||||
config_.restore_budget, config_.lease.commit_timeout);
|
||||
if (published) {
|
||||
confirmed(*published, now);
|
||||
return;
|
||||
}
|
||||
const auto& f = published.error();
|
||||
switch (f.kind) {
|
||||
case PublicationError::upload:
|
||||
case PublicationError::verify:
|
||||
case PublicationError::digest:
|
||||
++status_.store_failures; // Nothing sent: retry the same batch later.
|
||||
return;
|
||||
case PublicationError::unknown:
|
||||
prepared_ = f.prepared;
|
||||
db_.reset();
|
||||
++status_.unknown_outcomes;
|
||||
status_.phase = HostPhase::resolving;
|
||||
return;
|
||||
case PublicationError::rejected:
|
||||
if (f.store.kind == LeaseStoreError::fenced) {
|
||||
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
|
||||
return;
|
||||
}
|
||||
if (f.store.kind == LeaseStoreError::store || f.store.kind == LeaseStoreError::invalid) {
|
||||
// SQL/connection error before any mutation: retry with the same bytes.
|
||||
db_.reset();
|
||||
++status_.store_failures;
|
||||
return;
|
||||
}
|
||||
[[fallthrough]];
|
||||
default: // candidate, predecessor, receipt, head moved, conflicts: fail closed.
|
||||
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename State> void RegionHost<State>::resolve(SchedulerTime now) {
|
||||
if (!database() || !prepared_ || !batch_)
|
||||
return;
|
||||
const auto& prepared = *prepared_;
|
||||
auto resolved = persistence::resolve_prepared(*db_, prepared, config_.lease.commit_timeout);
|
||||
if (!resolved) {
|
||||
db_.reset();
|
||||
++status_.store_failures;
|
||||
return;
|
||||
}
|
||||
const auto adopt = [&](const persistence::RegionHead& receipt) {
|
||||
auto decisions = persistence::decode_decisions(prepared.decisions, batch_->begin_tick(),
|
||||
batch_->end_tick());
|
||||
const auto digest = persistence::sha256(prepared.manifest);
|
||||
if (!decisions || !digest || *digest != receipt.digest ||
|
||||
receipt.end_tick != batch_->end_tick()) {
|
||||
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
|
||||
return;
|
||||
}
|
||||
confirmed({{receipt, prepared.manifest}, *decisions}, now);
|
||||
};
|
||||
switch (resolved->status) {
|
||||
case persistence::Resolution::committed:
|
||||
adopt(resolved->receipt);
|
||||
return;
|
||||
case persistence::Resolution::conflict:
|
||||
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
|
||||
return;
|
||||
case persistence::Resolution::absent:
|
||||
break;
|
||||
}
|
||||
// Absent now: retry exactly the same bytes, only while still authorized.
|
||||
if (!authority_.may_begin_commit(LeaseClock::now()))
|
||||
return;
|
||||
auto retried = persistence::retry_prepared(*db_, prepared, config_.lease.commit_timeout);
|
||||
if (retried) {
|
||||
adopt(*retried);
|
||||
return;
|
||||
}
|
||||
switch (retried.error().kind) {
|
||||
case LeaseStoreError::fenced:
|
||||
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
|
||||
return;
|
||||
case LeaseStoreError::unknown:
|
||||
case LeaseStoreError::store:
|
||||
db_.reset();
|
||||
++status_.store_failures;
|
||||
return;
|
||||
default:
|
||||
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename State> void RegionHost<State>::simulate(SchedulerTime now) {
|
||||
const auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(now - origin_);
|
||||
const auto due = elapsed.count() <= 0
|
||||
? std::uint64_t{0}
|
||||
: static_cast<std::uint64_t>(elapsed / config_.tick_quantum);
|
||||
if (!batch_ || !state_)
|
||||
return;
|
||||
auto& batch = *batch_;
|
||||
const auto& state = *state_;
|
||||
for (std::uint32_t k = 0; k < config_.max_catch_up_ticks && !batch.sealed(); ++k) {
|
||||
if (Ops<State>::tick(state) - origin_tick_ >= due || queue_.empty() ||
|
||||
!authority_.may_originate(LeaseClock::now()))
|
||||
break; // Not due, starved (debt accrues; inputs are never invented) or unauthorized.
|
||||
const auto input = queue_.front();
|
||||
queue_.pop_front();
|
||||
if (batch.steps().empty())
|
||||
batch_started_ = now;
|
||||
auto report = batch.advance(input.input);
|
||||
if (!report) {
|
||||
dropped_.push_back({input, Unacknowledged::refused});
|
||||
continue;
|
||||
}
|
||||
applied_.push_back({input, report->tick});
|
||||
}
|
||||
}
|
||||
|
||||
template <typename State> void RegionHost<State>::refresh(SchedulerTime now) {
|
||||
status_.queued = queue_.size();
|
||||
status_.queued_bytes = queue_.size() * tick_input_bytes;
|
||||
const auto& grant = authority_.grant();
|
||||
status_.epoch = grant ? grant->epoch : 0;
|
||||
status_.committed_tick = committed_ ? committed_->head.end_tick : 0;
|
||||
status_.committed_sequence = committed_ ? committed_->head.sequence : 0;
|
||||
status_.simulated_tick = state_ ? Ops<State>::tick(*state_) : status_.committed_tick;
|
||||
if (state_ && status_.phase == HostPhase::running) {
|
||||
const auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(now - origin_);
|
||||
const auto due = elapsed.count() <= 0
|
||||
? std::uint64_t{0}
|
||||
: static_cast<std::uint64_t>(elapsed / config_.tick_quantum);
|
||||
const auto done = Ops<State>::tick(*state_) - origin_tick_;
|
||||
status_.tick_debt = due > done ? due - done : 0;
|
||||
status_.max_tick_debt = std::max(status_.max_tick_debt, status_.tick_debt);
|
||||
}
|
||||
status_.ready = status_.phase == HostPhase::running && state_ && !draining_ &&
|
||||
authority_.may_originate(LeaseClock::now());
|
||||
}
|
||||
|
||||
template <typename State> const HostStatus& RegionHost<State>::step(SchedulerTime now) {
|
||||
switch (status_.phase) {
|
||||
case HostPhase::acquiring:
|
||||
acquire(now);
|
||||
break;
|
||||
case HostPhase::resolving:
|
||||
upkeep();
|
||||
if (status_.phase == HostPhase::resolving)
|
||||
resolve(now);
|
||||
break;
|
||||
case HostPhase::running:
|
||||
case HostPhase::draining: {
|
||||
upkeep();
|
||||
if (status_.phase != HostPhase::running && status_.phase != HostPhase::draining)
|
||||
break;
|
||||
if (!draining_)
|
||||
simulate(now);
|
||||
if (!batch_)
|
||||
break; // Running always has a batch; defensive.
|
||||
const bool flush =
|
||||
!batch_->steps().empty() &&
|
||||
(batch_->sealed() || draining_ || now - batch_started_ >= config_.flush_interval);
|
||||
if (flush)
|
||||
commit(now); // May lose authority and reset the batch.
|
||||
if (draining_ && status_.phase == HostPhase::draining && batch_ &&
|
||||
batch_->steps().empty()) {
|
||||
const auto& grant = authority_.grant();
|
||||
if (database() && grant && authority_.may_originate(LeaseClock::now())) {
|
||||
// Best effort: an unreleased lease simply expires.
|
||||
[[maybe_unused]] const auto released = persistence::release_region_lease(
|
||||
*db_, config_.key, *grant, config_.lease.renew_timeout);
|
||||
}
|
||||
batch_.reset();
|
||||
state_.reset();
|
||||
authority_.fence();
|
||||
db_.reset();
|
||||
status_.phase = HostPhase::stopped;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case HostPhase::stopped:
|
||||
case HostPhase::quarantined:
|
||||
break;
|
||||
}
|
||||
refresh(now);
|
||||
return status_;
|
||||
}
|
||||
|
||||
template <typename State> void RegionHost<State>::drain() {
|
||||
if (draining_)
|
||||
return;
|
||||
draining_ = true;
|
||||
for (auto& q : queue_)
|
||||
dropped_.push_back({q, Unacknowledged::drained});
|
||||
queue_.clear();
|
||||
if (status_.phase == HostPhase::running)
|
||||
status_.phase = HostPhase::draining;
|
||||
else if (status_.phase == HostPhase::acquiring)
|
||||
status_.phase = HostPhase::stopped;
|
||||
status_.ready = false;
|
||||
}
|
||||
|
||||
template <typename State> std::vector<FinalAck> RegionHost<State>::take_acks() {
|
||||
return std::exchange(acks_, {});
|
||||
}
|
||||
template <typename State> std::vector<FinalOutcome> RegionHost<State>::take_outcomes() {
|
||||
return std::exchange(outcomes_, {});
|
||||
}
|
||||
template <typename State> std::vector<Dropped> RegionHost<State>::take_dropped() {
|
||||
return std::exchange(dropped_, {});
|
||||
}
|
||||
|
||||
template class RegionHost<processes::EnvironmentalCommunity>;
|
||||
template class RegionHost<processes::LineageRegion>;
|
||||
} // namespace evo::server
|
||||
Reference in New Issue
Block a user