Files
aevolution/src/server/region_host.cpp
T
dingoandClaude Opus 5.5 f59cca2c3b Complete M7.9b upgrade barrier and durability qualification
Schema v8 binds authority to builds (ADR-0068): acquisition names the
host build and only retained, unretired builds may hold a lease; no
commit may name a retired build; upgrade_barrier retires a build and
fences every holder in one transaction, while build-less pre-v8 hosts
keep working until the first barrier. Tests cover a writer surviving a
failed and then a real online migration, old-writer fencing on every
path, whole-history recovery by the new build, and the M7 qualification
fixture (19.6 s SIGKILL-to-ready under the production lease, no lost
acknowledgements).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-29 07:31:04 -04:00

489 lines
19 KiB
C++

#include "evo/server/region_host.hpp"
#include "evo/persistence/sha256.hpp"
#include <algorithm>
#include <utility>
namespace evo::server {
namespace {
using persistence::LeaseClock;
using persistence::LeaseStoreError;
using persistence::PublicationError;
using persistence::RecoveryError;
// Per-profile calls into the persistence layer.
template <typename State> struct Ops;
template <> struct Ops<processes::EnvironmentalCommunity> {
using State = processes::EnvironmentalCommunity;
static std::uint64_t tick(const State& s) {
return s.progress().tick;
}
static std::uint64_t quantum(const State& s) {
return s.tick_microseconds();
}
template <typename... A> static auto recover(A&&... a) {
return persistence::recover_environmental_checkpoint(std::forward<A>(a)...);
}
template <typename... A> static auto publish(A&&... a) {
return persistence::publish_environmental_batch(std::forward<A>(a)...);
}
};
template <> struct Ops<processes::LineageRegion> {
using State = processes::LineageRegion;
static std::uint64_t tick(const State& s) {
return s.community().progress().tick;
}
static std::uint64_t quantum(const State& s) {
return s.community().tick_microseconds();
}
template <typename... A> static auto recover(A&&... a) {
return persistence::recover_lineage_checkpoint(std::forward<A>(a)...);
}
template <typename... A> static auto publish(A&&... a) {
return persistence::publish_lineage_batch(std::forward<A>(a)...);
}
};
bool valid(const HostConfig& c) {
constexpr std::size_t max_backlog_inputs = 64;
constexpr std::size_t max_backlog_bytes = 65536;
return c.key.world != 0 && c.key.region != 0 && c.frame != 0 && persistence::valid(c.lease) &&
c.tick_quantum.count() > 0 && c.flush_interval.count() > 0 && c.max_queued_inputs >= 1 &&
c.max_queued_inputs <= max_backlog_inputs && c.max_queued_bytes >= tick_input_bytes &&
c.max_queued_bytes <= max_backlog_bytes && c.max_catch_up_ticks >= 1 &&
c.max_catch_up_ticks <= 64 && c.build != persistence::Sha256Digest{} &&
std::ranges::find(c.retained_builds, c.build) != c.retained_builds.end() &&
c.restore_budget > 0;
}
} // namespace
template <typename State>
RegionHost<State>::RegionHost(HostConfig config, Connect connect,
std::unique_ptr<persistence::FilesystemObjectStore> objects)
: config_(std::move(config)), connect_(std::move(connect)), objects_(std::move(objects)),
authority_(config_.lease) {}
template <typename State> RegionHost<State>::~RegionHost() = default;
template <typename State>
std::expected<std::unique_ptr<RegionHost<State>>, HostConfigError>
RegionHost<State>::create(HostConfig config, Connect connect,
std::unique_ptr<persistence::FilesystemObjectStore> objects) {
if (!valid(config) || !connect || !objects)
return std::unexpected{HostConfigError::invalid};
return std::unique_ptr<RegionHost>(
new RegionHost(std::move(config), std::move(connect), std::move(objects)));
}
template <typename State>
std::expected<void, AdmissionError> RegionHost<State>::submit(const TickInput& input) {
if (input.request == persistence::Uuid{})
return std::unexpected{AdmissionError::invalid};
if (draining_)
return std::unexpected{AdmissionError::draining};
if (!status_.ready)
return std::unexpected{AdmissionError::not_ready};
const auto same = [&](const TickInput& x) { return x.request == input.request; };
if (std::ranges::any_of(queue_, same) ||
std::ranges::any_of(applied_, [&](const Applied& a) { return same(a.input); }))
return std::unexpected{AdmissionError::duplicate};
if (queue_.size() >= config_.max_queued_inputs ||
(queue_.size() + 1) * tick_input_bytes > config_.max_queued_bytes)
return std::unexpected{AdmissionError::full};
queue_.push_back(input);
status_.queued = queue_.size();
status_.queued_bytes = queue_.size() * tick_input_bytes;
return {};
}
template <typename State> bool RegionHost<State>::database() {
if (db_)
return true;
auto made = connect_();
if (!made) {
++status_.store_failures;
return false;
}
db_ = std::move(*made);
return true;
}
template <typename State>
void RegionHost<State>::lose_authority(Unacknowledged reason, HostPhase next) {
for (auto& a : applied_)
dropped_.push_back({a.input, reason});
for (auto& q : queue_)
dropped_.push_back({q, reason});
applied_.clear();
queue_.clear();
batch_.reset(); // References state_: reset first.
state_.reset();
committed_.reset();
prepared_.reset();
// Best effort: a quarantined region's lease is released so peers see it free.
const auto& grant = authority_.grant();
if (next == HostPhase::quarantined && db_ && grant &&
authority_.may_originate(LeaseClock::now())) {
[[maybe_unused]] const auto released = persistence::release_region_lease(
*db_, config_.key, *grant, config_.lease.renew_timeout);
}
authority_.fence();
if (reason == Unacknowledged::fenced)
++status_.fences;
status_.phase = draining_ && next == HostPhase::acquiring ? HostPhase::stopped : next;
}
template <typename State> void RegionHost<State>::acquire(SchedulerTime now) {
if (!database())
return;
// A grant survives a transient recovery failure; only re-acquire without one.
if (!authority_.may_originate(LeaseClock::now())) {
const auto sent = LeaseClock::now();
auto grant = persistence::acquire_region_lease(
*db_, config_.key, config_.build, config_.lease.duration, config_.lease.renew_timeout);
if (!grant) {
// No region, or this build is retired/unknown (ADR-0068): never serve.
if (grant.error().kind == LeaseStoreError::not_provisioned ||
grant.error().kind == LeaseStoreError::unsupported)
status_.phase = HostPhase::quarantined;
else if (grant.error().kind != LeaseStoreError::busy)
db_.reset();
return;
}
if (grant->frame != config_.frame) {
status_.phase = HostPhase::quarantined;
return;
}
if (!authority_.acquired(*grant, sent, LeaseClock::now()))
return; // Reply too late to use; the lease expires unused.
}
auto recovered = Ops<State>::recover(*db_, *objects_, config_.key, config_.retained_builds,
config_.restore_budget, config_.lease.commit_timeout);
if (!recovered) {
if (recovered.error().kind == RecoveryError::store) {
db_.reset(); // Transient: keep the grant and retry recovery.
++status_.store_failures;
return;
}
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
return;
}
if (Ops<State>::quantum(*recovered->state) !=
static_cast<std::uint64_t>(config_.tick_quantum.count())) {
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
return;
}
state_ = std::move(recovered->state);
committed_ = std::move(recovered->committed);
batch_.emplace(*state_);
origin_ = now;
origin_tick_ = Ops<State>::tick(*state_);
status_.max_tick_debt = 0;
status_.phase = draining_ ? HostPhase::draining : HostPhase::running;
}
template <typename State> void RegionHost<State>::upkeep() {
const auto state = authority_.state(LeaseClock::now());
if (state == persistence::LeaseState::fenced) {
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
return;
}
if (state != persistence::LeaseState::renew_due || !database())
return;
const auto grant = *authority_.grant();
const auto sent = LeaseClock::now();
auto renewed = persistence::renew_region_lease(*db_, config_.key, grant, config_.lease.duration,
config_.lease.renew_timeout);
if (renewed) {
if (!authority_.renewed(grant.epoch, *renewed, sent, LeaseClock::now()))
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
return;
}
switch (renewed.error().kind) {
case LeaseStoreError::unknown:
// It may have shortened or extended nothing: keep the earlier deadline.
authority_.renewal_unknown(config_.lease.duration, sent);
db_.reset();
++status_.store_failures;
return;
case LeaseStoreError::fenced:
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
return;
default:
db_.reset();
++status_.store_failures;
return;
}
}
template <typename State> persistence::Uuid RegionHost<State>::next_commit_id() const {
// Unique: the DB-generated holder token names this acquisition; the
// sequence names the link. Stable across retries of the same batch.
const auto& grant = authority_.grant();
if (!grant || !committed_)
return {};
auto id = grant->holder;
const auto sequence = committed_->head.sequence + 1;
for (unsigned i = 0; i < 8; ++i)
id[8 + i] = static_cast<std::uint8_t>((sequence >> (8U * i)) & 0xFFU);
return id;
}
template <typename State>
void RegionHost<State>::confirmed(const persistence::PublishedBatch& published, SchedulerTime now) {
if (config_.crash_hook)
config_.crash_hook(CrashPoint::after_commit);
const auto& head = published.committed.head;
if (!batch_ || !batch_->confirm(head.end_tick)) {
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
return;
}
committed_ = published.committed;
for (const auto& a : applied_)
acks_.push_back({a.input.request, a.tick, head.sequence});
applied_.clear();
for (const auto& d : published.decisions)
outcomes_.push_back({d, head.sequence});
prepared_.reset();
batch_started_ = now;
++status_.commits;
status_.phase = draining_ ? HostPhase::draining : HostPhase::running;
}
template <typename State> void RegionHost<State>::commit(SchedulerTime now) {
if (!authority_.may_begin_commit(LeaseClock::now()))
return; // Renewal first; upkeep fences if the deadline passes.
if (!database())
return;
const auto& grant = authority_.grant();
if (!grant || !committed_ || !batch_)
return;
const auto commit_id = next_commit_id();
batch_->seal();
const auto& crash = config_.crash_hook;
if (crash)
crash(CrashPoint::before_upload);
persistence::PublishHooks hooks;
if (crash)
hooks.before_transaction = [&crash] { crash(CrashPoint::before_transaction); };
auto published =
Ops<State>::publish(*db_, *objects_, *grant, *committed_, *batch_, commit_id, config_.build,
config_.restore_budget, config_.lease.commit_timeout, hooks);
if (published) {
confirmed(*published, now);
return;
}
const auto& f = published.error();
switch (f.kind) {
case PublicationError::upload:
case PublicationError::verify:
case PublicationError::digest:
++status_.store_failures; // Nothing sent: retry the same batch later.
return;
case PublicationError::unknown:
prepared_ = f.prepared;
db_.reset();
++status_.unknown_outcomes;
status_.phase = HostPhase::resolving;
return;
case PublicationError::rejected:
if (f.store.kind == LeaseStoreError::fenced) {
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
return;
}
if (f.store.kind == LeaseStoreError::store || f.store.kind == LeaseStoreError::invalid) {
// SQL/connection error before any mutation: retry with the same bytes.
db_.reset();
++status_.store_failures;
return;
}
[[fallthrough]];
default: // candidate, predecessor, receipt, head moved, conflicts: fail closed.
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
return;
}
}
template <typename State> void RegionHost<State>::resolve(SchedulerTime now) {
if (!database() || !prepared_ || !batch_)
return;
const auto& prepared = *prepared_;
auto resolved = persistence::resolve_prepared(*db_, prepared, config_.lease.commit_timeout);
if (!resolved) {
db_.reset();
++status_.store_failures;
return;
}
const auto adopt = [&](const persistence::RegionHead& receipt) {
auto decisions = persistence::decode_decisions(prepared.decisions, batch_->begin_tick(),
batch_->end_tick());
const auto digest = persistence::sha256(prepared.manifest);
if (!decisions || !digest || *digest != receipt.digest ||
receipt.end_tick != batch_->end_tick()) {
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
return;
}
confirmed({{receipt, prepared.manifest}, *decisions}, now);
};
switch (resolved->status) {
case persistence::Resolution::committed:
adopt(resolved->receipt);
return;
case persistence::Resolution::conflict:
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
return;
case persistence::Resolution::absent:
break;
}
// Absent now: retry exactly the same bytes, only while still authorized.
if (!authority_.may_begin_commit(LeaseClock::now()))
return;
auto retried = persistence::retry_prepared(*db_, prepared, config_.lease.commit_timeout);
if (retried) {
adopt(*retried);
return;
}
switch (retried.error().kind) {
case LeaseStoreError::fenced:
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
return;
case LeaseStoreError::unknown:
case LeaseStoreError::store:
db_.reset();
++status_.store_failures;
return;
default:
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
return;
}
}
template <typename State> void RegionHost<State>::simulate(SchedulerTime now) {
const auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(now - origin_);
const auto due = elapsed.count() <= 0
? std::uint64_t{0}
: static_cast<std::uint64_t>(elapsed / config_.tick_quantum);
if (!batch_ || !state_)
return;
auto& batch = *batch_;
const auto& state = *state_;
for (std::uint32_t k = 0; k < config_.max_catch_up_ticks && !batch.sealed(); ++k) {
if (Ops<State>::tick(state) - origin_tick_ >= due || queue_.empty() ||
!authority_.may_originate(LeaseClock::now()))
break; // Not due, starved (debt accrues; inputs are never invented) or unauthorized.
const auto input = queue_.front();
queue_.pop_front();
if (batch.steps().empty())
batch_started_ = now;
auto report = batch.advance(input.input);
if (!report) {
dropped_.push_back({input, Unacknowledged::refused});
continue;
}
applied_.push_back({input, report->tick});
}
}
template <typename State> void RegionHost<State>::refresh(SchedulerTime now) {
status_.queued = queue_.size();
status_.queued_bytes = queue_.size() * tick_input_bytes;
const auto& grant = authority_.grant();
status_.epoch = grant ? grant->epoch : 0;
const auto lease_now = LeaseClock::now();
status_.lease_remaining_ms =
grant && authority_.may_originate(lease_now)
? static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
authority_.deadline() - lease_now)
.count())
: 0;
status_.committed_tick = committed_ ? committed_->head.end_tick : 0;
status_.committed_sequence = committed_ ? committed_->head.sequence : 0;
status_.simulated_tick = state_ ? Ops<State>::tick(*state_) : status_.committed_tick;
if (state_ && status_.phase == HostPhase::running) {
const auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(now - origin_);
const auto due = elapsed.count() <= 0
? std::uint64_t{0}
: static_cast<std::uint64_t>(elapsed / config_.tick_quantum);
const auto done = Ops<State>::tick(*state_) - origin_tick_;
status_.tick_debt = due > done ? due - done : 0;
status_.max_tick_debt = std::max(status_.max_tick_debt, status_.tick_debt);
}
status_.ready = status_.phase == HostPhase::running && state_ && !draining_ &&
authority_.may_originate(LeaseClock::now());
}
template <typename State> const HostStatus& RegionHost<State>::step(SchedulerTime now) {
switch (status_.phase) {
case HostPhase::acquiring:
acquire(now);
break;
case HostPhase::resolving:
upkeep();
if (status_.phase == HostPhase::resolving)
resolve(now);
break;
case HostPhase::running:
case HostPhase::draining: {
upkeep();
if (status_.phase != HostPhase::running && status_.phase != HostPhase::draining)
break;
if (!draining_)
simulate(now);
if (!batch_)
break; // Running always has a batch; defensive.
const bool flush =
!batch_->steps().empty() &&
(batch_->sealed() || draining_ || now - batch_started_ >= config_.flush_interval);
if (flush)
commit(now); // May lose authority and reset the batch.
if (draining_ && status_.phase == HostPhase::draining && batch_ &&
batch_->steps().empty()) {
const auto& grant = authority_.grant();
if (database() && grant && authority_.may_originate(LeaseClock::now())) {
// Best effort: an unreleased lease simply expires.
[[maybe_unused]] const auto released = persistence::release_region_lease(
*db_, config_.key, *grant, config_.lease.renew_timeout);
}
batch_.reset();
state_.reset();
authority_.fence();
db_.reset();
status_.phase = HostPhase::stopped;
}
break;
}
case HostPhase::stopped:
case HostPhase::quarantined:
break;
}
refresh(now);
return status_;
}
template <typename State> void RegionHost<State>::drain() {
if (draining_)
return;
draining_ = true;
for (auto& q : queue_)
dropped_.push_back({q, Unacknowledged::drained});
queue_.clear();
if (status_.phase == HostPhase::running)
status_.phase = HostPhase::draining;
else if (status_.phase == HostPhase::acquiring)
status_.phase = HostPhase::stopped;
status_.ready = false;
}
template <typename State> std::vector<FinalAck> RegionHost<State>::take_acks() {
return std::exchange(acks_, {});
}
template <typename State> std::vector<FinalOutcome> RegionHost<State>::take_outcomes() {
return std::exchange(outcomes_, {});
}
template <typename State> std::vector<Dropped> RegionHost<State>::take_dropped() {
return std::exchange(dropped_, {});
}
template class RegionHost<processes::EnvironmentalCommunity>;
template class RegionHost<processes::LineageRegion>;
} // namespace evo::server