Schema v8 binds authority to builds (ADR-0068): acquisition names the host build and only retained, unretired builds may hold a lease; no commit may name a retired build; upgrade_barrier retires a build and fences every holder in one transaction, while build-less pre-v8 hosts keep working until the first barrier. Tests cover a writer surviving a failed and then a real online migration, old-writer fencing on every path, whole-history recovery by the new build, and the M7 qualification fixture (19.6 s SIGKILL-to-ready under the production lease, no lost acknowledgements). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
489 lines
19 KiB
C++
489 lines
19 KiB
C++
#include "evo/server/region_host.hpp"
|
|
#include "evo/persistence/sha256.hpp"
|
|
#include <algorithm>
|
|
#include <utility>
|
|
|
|
namespace evo::server {
|
|
namespace {
|
|
using persistence::LeaseClock;
|
|
using persistence::LeaseStoreError;
|
|
using persistence::PublicationError;
|
|
using persistence::RecoveryError;
|
|
|
|
// Per-profile calls into the persistence layer.
|
|
template <typename State> struct Ops;
|
|
template <> struct Ops<processes::EnvironmentalCommunity> {
|
|
using State = processes::EnvironmentalCommunity;
|
|
static std::uint64_t tick(const State& s) {
|
|
return s.progress().tick;
|
|
}
|
|
static std::uint64_t quantum(const State& s) {
|
|
return s.tick_microseconds();
|
|
}
|
|
template <typename... A> static auto recover(A&&... a) {
|
|
return persistence::recover_environmental_checkpoint(std::forward<A>(a)...);
|
|
}
|
|
template <typename... A> static auto publish(A&&... a) {
|
|
return persistence::publish_environmental_batch(std::forward<A>(a)...);
|
|
}
|
|
};
|
|
template <> struct Ops<processes::LineageRegion> {
|
|
using State = processes::LineageRegion;
|
|
static std::uint64_t tick(const State& s) {
|
|
return s.community().progress().tick;
|
|
}
|
|
static std::uint64_t quantum(const State& s) {
|
|
return s.community().tick_microseconds();
|
|
}
|
|
template <typename... A> static auto recover(A&&... a) {
|
|
return persistence::recover_lineage_checkpoint(std::forward<A>(a)...);
|
|
}
|
|
template <typename... A> static auto publish(A&&... a) {
|
|
return persistence::publish_lineage_batch(std::forward<A>(a)...);
|
|
}
|
|
};
|
|
|
|
bool valid(const HostConfig& c) {
|
|
constexpr std::size_t max_backlog_inputs = 64;
|
|
constexpr std::size_t max_backlog_bytes = 65536;
|
|
return c.key.world != 0 && c.key.region != 0 && c.frame != 0 && persistence::valid(c.lease) &&
|
|
c.tick_quantum.count() > 0 && c.flush_interval.count() > 0 && c.max_queued_inputs >= 1 &&
|
|
c.max_queued_inputs <= max_backlog_inputs && c.max_queued_bytes >= tick_input_bytes &&
|
|
c.max_queued_bytes <= max_backlog_bytes && c.max_catch_up_ticks >= 1 &&
|
|
c.max_catch_up_ticks <= 64 && c.build != persistence::Sha256Digest{} &&
|
|
std::ranges::find(c.retained_builds, c.build) != c.retained_builds.end() &&
|
|
c.restore_budget > 0;
|
|
}
|
|
} // namespace
|
|
|
|
template <typename State>
|
|
RegionHost<State>::RegionHost(HostConfig config, Connect connect,
|
|
std::unique_ptr<persistence::FilesystemObjectStore> objects)
|
|
: config_(std::move(config)), connect_(std::move(connect)), objects_(std::move(objects)),
|
|
authority_(config_.lease) {}
|
|
|
|
template <typename State> RegionHost<State>::~RegionHost() = default;
|
|
|
|
template <typename State>
|
|
std::expected<std::unique_ptr<RegionHost<State>>, HostConfigError>
|
|
RegionHost<State>::create(HostConfig config, Connect connect,
|
|
std::unique_ptr<persistence::FilesystemObjectStore> objects) {
|
|
if (!valid(config) || !connect || !objects)
|
|
return std::unexpected{HostConfigError::invalid};
|
|
return std::unique_ptr<RegionHost>(
|
|
new RegionHost(std::move(config), std::move(connect), std::move(objects)));
|
|
}
|
|
|
|
template <typename State>
|
|
std::expected<void, AdmissionError> RegionHost<State>::submit(const TickInput& input) {
|
|
if (input.request == persistence::Uuid{})
|
|
return std::unexpected{AdmissionError::invalid};
|
|
if (draining_)
|
|
return std::unexpected{AdmissionError::draining};
|
|
if (!status_.ready)
|
|
return std::unexpected{AdmissionError::not_ready};
|
|
const auto same = [&](const TickInput& x) { return x.request == input.request; };
|
|
if (std::ranges::any_of(queue_, same) ||
|
|
std::ranges::any_of(applied_, [&](const Applied& a) { return same(a.input); }))
|
|
return std::unexpected{AdmissionError::duplicate};
|
|
if (queue_.size() >= config_.max_queued_inputs ||
|
|
(queue_.size() + 1) * tick_input_bytes > config_.max_queued_bytes)
|
|
return std::unexpected{AdmissionError::full};
|
|
queue_.push_back(input);
|
|
status_.queued = queue_.size();
|
|
status_.queued_bytes = queue_.size() * tick_input_bytes;
|
|
return {};
|
|
}
|
|
|
|
template <typename State> bool RegionHost<State>::database() {
|
|
if (db_)
|
|
return true;
|
|
auto made = connect_();
|
|
if (!made) {
|
|
++status_.store_failures;
|
|
return false;
|
|
}
|
|
db_ = std::move(*made);
|
|
return true;
|
|
}
|
|
|
|
template <typename State>
|
|
void RegionHost<State>::lose_authority(Unacknowledged reason, HostPhase next) {
|
|
for (auto& a : applied_)
|
|
dropped_.push_back({a.input, reason});
|
|
for (auto& q : queue_)
|
|
dropped_.push_back({q, reason});
|
|
applied_.clear();
|
|
queue_.clear();
|
|
batch_.reset(); // References state_: reset first.
|
|
state_.reset();
|
|
committed_.reset();
|
|
prepared_.reset();
|
|
// Best effort: a quarantined region's lease is released so peers see it free.
|
|
const auto& grant = authority_.grant();
|
|
if (next == HostPhase::quarantined && db_ && grant &&
|
|
authority_.may_originate(LeaseClock::now())) {
|
|
[[maybe_unused]] const auto released = persistence::release_region_lease(
|
|
*db_, config_.key, *grant, config_.lease.renew_timeout);
|
|
}
|
|
authority_.fence();
|
|
if (reason == Unacknowledged::fenced)
|
|
++status_.fences;
|
|
status_.phase = draining_ && next == HostPhase::acquiring ? HostPhase::stopped : next;
|
|
}
|
|
|
|
template <typename State> void RegionHost<State>::acquire(SchedulerTime now) {
|
|
if (!database())
|
|
return;
|
|
// A grant survives a transient recovery failure; only re-acquire without one.
|
|
if (!authority_.may_originate(LeaseClock::now())) {
|
|
const auto sent = LeaseClock::now();
|
|
auto grant = persistence::acquire_region_lease(
|
|
*db_, config_.key, config_.build, config_.lease.duration, config_.lease.renew_timeout);
|
|
if (!grant) {
|
|
// No region, or this build is retired/unknown (ADR-0068): never serve.
|
|
if (grant.error().kind == LeaseStoreError::not_provisioned ||
|
|
grant.error().kind == LeaseStoreError::unsupported)
|
|
status_.phase = HostPhase::quarantined;
|
|
else if (grant.error().kind != LeaseStoreError::busy)
|
|
db_.reset();
|
|
return;
|
|
}
|
|
if (grant->frame != config_.frame) {
|
|
status_.phase = HostPhase::quarantined;
|
|
return;
|
|
}
|
|
if (!authority_.acquired(*grant, sent, LeaseClock::now()))
|
|
return; // Reply too late to use; the lease expires unused.
|
|
}
|
|
auto recovered = Ops<State>::recover(*db_, *objects_, config_.key, config_.retained_builds,
|
|
config_.restore_budget, config_.lease.commit_timeout);
|
|
if (!recovered) {
|
|
if (recovered.error().kind == RecoveryError::store) {
|
|
db_.reset(); // Transient: keep the grant and retry recovery.
|
|
++status_.store_failures;
|
|
return;
|
|
}
|
|
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
|
|
return;
|
|
}
|
|
if (Ops<State>::quantum(*recovered->state) !=
|
|
static_cast<std::uint64_t>(config_.tick_quantum.count())) {
|
|
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
|
|
return;
|
|
}
|
|
state_ = std::move(recovered->state);
|
|
committed_ = std::move(recovered->committed);
|
|
batch_.emplace(*state_);
|
|
origin_ = now;
|
|
origin_tick_ = Ops<State>::tick(*state_);
|
|
status_.max_tick_debt = 0;
|
|
status_.phase = draining_ ? HostPhase::draining : HostPhase::running;
|
|
}
|
|
|
|
template <typename State> void RegionHost<State>::upkeep() {
|
|
const auto state = authority_.state(LeaseClock::now());
|
|
if (state == persistence::LeaseState::fenced) {
|
|
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
|
|
return;
|
|
}
|
|
if (state != persistence::LeaseState::renew_due || !database())
|
|
return;
|
|
const auto grant = *authority_.grant();
|
|
const auto sent = LeaseClock::now();
|
|
auto renewed = persistence::renew_region_lease(*db_, config_.key, grant, config_.lease.duration,
|
|
config_.lease.renew_timeout);
|
|
if (renewed) {
|
|
if (!authority_.renewed(grant.epoch, *renewed, sent, LeaseClock::now()))
|
|
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
|
|
return;
|
|
}
|
|
switch (renewed.error().kind) {
|
|
case LeaseStoreError::unknown:
|
|
// It may have shortened or extended nothing: keep the earlier deadline.
|
|
authority_.renewal_unknown(config_.lease.duration, sent);
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
return;
|
|
case LeaseStoreError::fenced:
|
|
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
|
|
return;
|
|
default:
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
return;
|
|
}
|
|
}
|
|
|
|
template <typename State> persistence::Uuid RegionHost<State>::next_commit_id() const {
|
|
// Unique: the DB-generated holder token names this acquisition; the
|
|
// sequence names the link. Stable across retries of the same batch.
|
|
const auto& grant = authority_.grant();
|
|
if (!grant || !committed_)
|
|
return {};
|
|
auto id = grant->holder;
|
|
const auto sequence = committed_->head.sequence + 1;
|
|
for (unsigned i = 0; i < 8; ++i)
|
|
id[8 + i] = static_cast<std::uint8_t>((sequence >> (8U * i)) & 0xFFU);
|
|
return id;
|
|
}
|
|
|
|
template <typename State>
|
|
void RegionHost<State>::confirmed(const persistence::PublishedBatch& published, SchedulerTime now) {
|
|
if (config_.crash_hook)
|
|
config_.crash_hook(CrashPoint::after_commit);
|
|
const auto& head = published.committed.head;
|
|
if (!batch_ || !batch_->confirm(head.end_tick)) {
|
|
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
|
|
return;
|
|
}
|
|
committed_ = published.committed;
|
|
for (const auto& a : applied_)
|
|
acks_.push_back({a.input.request, a.tick, head.sequence});
|
|
applied_.clear();
|
|
for (const auto& d : published.decisions)
|
|
outcomes_.push_back({d, head.sequence});
|
|
prepared_.reset();
|
|
batch_started_ = now;
|
|
++status_.commits;
|
|
status_.phase = draining_ ? HostPhase::draining : HostPhase::running;
|
|
}
|
|
|
|
template <typename State> void RegionHost<State>::commit(SchedulerTime now) {
|
|
if (!authority_.may_begin_commit(LeaseClock::now()))
|
|
return; // Renewal first; upkeep fences if the deadline passes.
|
|
if (!database())
|
|
return;
|
|
const auto& grant = authority_.grant();
|
|
if (!grant || !committed_ || !batch_)
|
|
return;
|
|
const auto commit_id = next_commit_id();
|
|
batch_->seal();
|
|
const auto& crash = config_.crash_hook;
|
|
if (crash)
|
|
crash(CrashPoint::before_upload);
|
|
persistence::PublishHooks hooks;
|
|
if (crash)
|
|
hooks.before_transaction = [&crash] { crash(CrashPoint::before_transaction); };
|
|
auto published =
|
|
Ops<State>::publish(*db_, *objects_, *grant, *committed_, *batch_, commit_id, config_.build,
|
|
config_.restore_budget, config_.lease.commit_timeout, hooks);
|
|
if (published) {
|
|
confirmed(*published, now);
|
|
return;
|
|
}
|
|
const auto& f = published.error();
|
|
switch (f.kind) {
|
|
case PublicationError::upload:
|
|
case PublicationError::verify:
|
|
case PublicationError::digest:
|
|
++status_.store_failures; // Nothing sent: retry the same batch later.
|
|
return;
|
|
case PublicationError::unknown:
|
|
prepared_ = f.prepared;
|
|
db_.reset();
|
|
++status_.unknown_outcomes;
|
|
status_.phase = HostPhase::resolving;
|
|
return;
|
|
case PublicationError::rejected:
|
|
if (f.store.kind == LeaseStoreError::fenced) {
|
|
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
|
|
return;
|
|
}
|
|
if (f.store.kind == LeaseStoreError::store || f.store.kind == LeaseStoreError::invalid) {
|
|
// SQL/connection error before any mutation: retry with the same bytes.
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
return;
|
|
}
|
|
[[fallthrough]];
|
|
default: // candidate, predecessor, receipt, head moved, conflicts: fail closed.
|
|
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
|
|
return;
|
|
}
|
|
}
|
|
|
|
template <typename State> void RegionHost<State>::resolve(SchedulerTime now) {
|
|
if (!database() || !prepared_ || !batch_)
|
|
return;
|
|
const auto& prepared = *prepared_;
|
|
auto resolved = persistence::resolve_prepared(*db_, prepared, config_.lease.commit_timeout);
|
|
if (!resolved) {
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
return;
|
|
}
|
|
const auto adopt = [&](const persistence::RegionHead& receipt) {
|
|
auto decisions = persistence::decode_decisions(prepared.decisions, batch_->begin_tick(),
|
|
batch_->end_tick());
|
|
const auto digest = persistence::sha256(prepared.manifest);
|
|
if (!decisions || !digest || *digest != receipt.digest ||
|
|
receipt.end_tick != batch_->end_tick()) {
|
|
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
|
|
return;
|
|
}
|
|
confirmed({{receipt, prepared.manifest}, *decisions}, now);
|
|
};
|
|
switch (resolved->status) {
|
|
case persistence::Resolution::committed:
|
|
adopt(resolved->receipt);
|
|
return;
|
|
case persistence::Resolution::conflict:
|
|
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
|
|
return;
|
|
case persistence::Resolution::absent:
|
|
break;
|
|
}
|
|
// Absent now: retry exactly the same bytes, only while still authorized.
|
|
if (!authority_.may_begin_commit(LeaseClock::now()))
|
|
return;
|
|
auto retried = persistence::retry_prepared(*db_, prepared, config_.lease.commit_timeout);
|
|
if (retried) {
|
|
adopt(*retried);
|
|
return;
|
|
}
|
|
switch (retried.error().kind) {
|
|
case LeaseStoreError::fenced:
|
|
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
|
|
return;
|
|
case LeaseStoreError::unknown:
|
|
case LeaseStoreError::store:
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
return;
|
|
default:
|
|
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
|
|
return;
|
|
}
|
|
}
|
|
|
|
template <typename State> void RegionHost<State>::simulate(SchedulerTime now) {
|
|
const auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(now - origin_);
|
|
const auto due = elapsed.count() <= 0
|
|
? std::uint64_t{0}
|
|
: static_cast<std::uint64_t>(elapsed / config_.tick_quantum);
|
|
if (!batch_ || !state_)
|
|
return;
|
|
auto& batch = *batch_;
|
|
const auto& state = *state_;
|
|
for (std::uint32_t k = 0; k < config_.max_catch_up_ticks && !batch.sealed(); ++k) {
|
|
if (Ops<State>::tick(state) - origin_tick_ >= due || queue_.empty() ||
|
|
!authority_.may_originate(LeaseClock::now()))
|
|
break; // Not due, starved (debt accrues; inputs are never invented) or unauthorized.
|
|
const auto input = queue_.front();
|
|
queue_.pop_front();
|
|
if (batch.steps().empty())
|
|
batch_started_ = now;
|
|
auto report = batch.advance(input.input);
|
|
if (!report) {
|
|
dropped_.push_back({input, Unacknowledged::refused});
|
|
continue;
|
|
}
|
|
applied_.push_back({input, report->tick});
|
|
}
|
|
}
|
|
|
|
template <typename State> void RegionHost<State>::refresh(SchedulerTime now) {
|
|
status_.queued = queue_.size();
|
|
status_.queued_bytes = queue_.size() * tick_input_bytes;
|
|
const auto& grant = authority_.grant();
|
|
status_.epoch = grant ? grant->epoch : 0;
|
|
const auto lease_now = LeaseClock::now();
|
|
status_.lease_remaining_ms =
|
|
grant && authority_.may_originate(lease_now)
|
|
? static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
authority_.deadline() - lease_now)
|
|
.count())
|
|
: 0;
|
|
status_.committed_tick = committed_ ? committed_->head.end_tick : 0;
|
|
status_.committed_sequence = committed_ ? committed_->head.sequence : 0;
|
|
status_.simulated_tick = state_ ? Ops<State>::tick(*state_) : status_.committed_tick;
|
|
if (state_ && status_.phase == HostPhase::running) {
|
|
const auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(now - origin_);
|
|
const auto due = elapsed.count() <= 0
|
|
? std::uint64_t{0}
|
|
: static_cast<std::uint64_t>(elapsed / config_.tick_quantum);
|
|
const auto done = Ops<State>::tick(*state_) - origin_tick_;
|
|
status_.tick_debt = due > done ? due - done : 0;
|
|
status_.max_tick_debt = std::max(status_.max_tick_debt, status_.tick_debt);
|
|
}
|
|
status_.ready = status_.phase == HostPhase::running && state_ && !draining_ &&
|
|
authority_.may_originate(LeaseClock::now());
|
|
}
|
|
|
|
template <typename State> const HostStatus& RegionHost<State>::step(SchedulerTime now) {
|
|
switch (status_.phase) {
|
|
case HostPhase::acquiring:
|
|
acquire(now);
|
|
break;
|
|
case HostPhase::resolving:
|
|
upkeep();
|
|
if (status_.phase == HostPhase::resolving)
|
|
resolve(now);
|
|
break;
|
|
case HostPhase::running:
|
|
case HostPhase::draining: {
|
|
upkeep();
|
|
if (status_.phase != HostPhase::running && status_.phase != HostPhase::draining)
|
|
break;
|
|
if (!draining_)
|
|
simulate(now);
|
|
if (!batch_)
|
|
break; // Running always has a batch; defensive.
|
|
const bool flush =
|
|
!batch_->steps().empty() &&
|
|
(batch_->sealed() || draining_ || now - batch_started_ >= config_.flush_interval);
|
|
if (flush)
|
|
commit(now); // May lose authority and reset the batch.
|
|
if (draining_ && status_.phase == HostPhase::draining && batch_ &&
|
|
batch_->steps().empty()) {
|
|
const auto& grant = authority_.grant();
|
|
if (database() && grant && authority_.may_originate(LeaseClock::now())) {
|
|
// Best effort: an unreleased lease simply expires.
|
|
[[maybe_unused]] const auto released = persistence::release_region_lease(
|
|
*db_, config_.key, *grant, config_.lease.renew_timeout);
|
|
}
|
|
batch_.reset();
|
|
state_.reset();
|
|
authority_.fence();
|
|
db_.reset();
|
|
status_.phase = HostPhase::stopped;
|
|
}
|
|
break;
|
|
}
|
|
case HostPhase::stopped:
|
|
case HostPhase::quarantined:
|
|
break;
|
|
}
|
|
refresh(now);
|
|
return status_;
|
|
}
|
|
|
|
template <typename State> void RegionHost<State>::drain() {
|
|
if (draining_)
|
|
return;
|
|
draining_ = true;
|
|
for (auto& q : queue_)
|
|
dropped_.push_back({q, Unacknowledged::drained});
|
|
queue_.clear();
|
|
if (status_.phase == HostPhase::running)
|
|
status_.phase = HostPhase::draining;
|
|
else if (status_.phase == HostPhase::acquiring)
|
|
status_.phase = HostPhase::stopped;
|
|
status_.ready = false;
|
|
}
|
|
|
|
template <typename State> std::vector<FinalAck> RegionHost<State>::take_acks() {
|
|
return std::exchange(acks_, {});
|
|
}
|
|
template <typename State> std::vector<FinalOutcome> RegionHost<State>::take_outcomes() {
|
|
return std::exchange(outcomes_, {});
|
|
}
|
|
template <typename State> std::vector<Dropped> RegionHost<State>::take_dropped() {
|
|
return std::exchange(dropped_, {});
|
|
}
|
|
|
|
template class RegionHost<processes::EnvironmentalCommunity>;
|
|
template class RegionHost<processes::LineageRegion>;
|
|
} // namespace evo::server
|