#include "evo/server/region_host.hpp" #include "evo/persistence/sha256.hpp" #include #include namespace evo::server { namespace { using persistence::LeaseClock; using persistence::LeaseStoreError; using persistence::PublicationError; using persistence::RecoveryError; // Commit retries after a store failure: 100 ms doubling to 2 s (well inside a // lease, so recovery after an outage is prompt; bounded so a full volume or an // unreachable store costs at most one upload attempt every 2 s). constexpr std::chrono::milliseconds commit_retry_initial{100}; constexpr std::chrono::milliseconds commit_retry_limit{2000}; // Per-profile calls into the persistence layer. template struct Ops; template <> struct Ops { using State = processes::EnvironmentalCommunity; static std::uint64_t tick(const State& s) { return s.progress().tick; } static std::uint64_t quantum(const State& s) { return s.tick_microseconds(); } template static auto recover(A&&... a) { return persistence::recover_environmental_checkpoint(std::forward(a)...); } template static auto publish(A&&... a) { return persistence::publish_environmental_batch(std::forward(a)...); } }; template <> struct Ops { using State = processes::LineageRegion; static std::uint64_t tick(const State& s) { return s.community().progress().tick; } static std::uint64_t quantum(const State& s) { return s.community().tick_microseconds(); } template static auto recover(A&&... a) { return persistence::recover_lineage_checkpoint(std::forward(a)...); } template static auto publish(A&&... a) { return persistence::publish_lineage_batch(std::forward(a)...); } }; bool valid(const HostConfig& c) { constexpr std::size_t max_backlog_inputs = 64; constexpr std::size_t max_backlog_bytes = 65536; return c.key.world != 0 && c.key.region != 0 && c.frame != 0 && persistence::valid(c.lease) && c.tick_quantum.count() > 0 && c.flush_interval.count() > 0 && c.max_queued_inputs >= 1 && c.max_queued_inputs <= max_backlog_inputs && c.max_queued_bytes >= tick_input_bytes && c.max_queued_bytes <= max_backlog_bytes && c.max_catch_up_ticks >= 1 && c.max_catch_up_ticks <= 64 && c.build != persistence::Sha256Digest{} && std::ranges::find(c.retained_builds, c.build) != c.retained_builds.end() && c.restore_budget > 0; } } // namespace template RegionHost::RegionHost(HostConfig config, Connect connect, std::unique_ptr objects) : config_(std::move(config)), connect_(std::move(connect)), objects_(std::move(objects)), authority_(config_.lease) {} template RegionHost::~RegionHost() = default; template std::expected>, HostConfigError> RegionHost::create(HostConfig config, Connect connect, std::unique_ptr objects) { if (!valid(config) || !connect || !objects) return std::unexpected{HostConfigError::invalid}; return std::unique_ptr( new RegionHost(std::move(config), std::move(connect), std::move(objects))); } template std::expected RegionHost::submit(const TickInput& input) { if (input.request == persistence::Uuid{}) return std::unexpected{AdmissionError::invalid}; if (draining_) return std::unexpected{AdmissionError::draining}; if (!status_.ready) return std::unexpected{AdmissionError::not_ready}; const auto same = [&](const TickInput& x) { return x.request == input.request; }; if (std::ranges::any_of(queue_, same) || std::ranges::any_of(applied_, [&](const Applied& a) { return same(a.input); })) return std::unexpected{AdmissionError::duplicate}; if (queue_.size() >= config_.max_queued_inputs || (queue_.size() + 1) * tick_input_bytes > config_.max_queued_bytes) return std::unexpected{AdmissionError::full}; queue_.push_back(input); status_.queued = queue_.size(); status_.queued_bytes = queue_.size() * tick_input_bytes; return {}; } template std::expected RegionHost::submit_action(const persistence::ActionRecord& record) { if (record.account == persistence::Uuid{} || record.action_id == 0 || record.kind != 1 || record.tick == 0) return std::unexpected{AdmissionError::invalid}; if (draining_) return std::unexpected{AdmissionError::draining}; if (!status_.ready) return std::unexpected{AdmissionError::not_ready}; const auto same = [&](const persistence::ActionRecord& a) { return a.account == record.account && a.action_id == record.action_id; }; if (std::ranges::any_of(action_queue_, same) || std::ranges::any_of(batch_actions_, same)) return std::unexpected{AdmissionError::duplicate}; if (action_queue_.size() + batch_actions_.size() >= config_.max_queued_actions) return std::unexpected{AdmissionError::full}; action_queue_.push_back(record); return {}; } template bool RegionHost::database() { if (db_) return true; auto made = connect_(); if (!made) { ++status_.store_failures; return false; } db_ = std::move(*made); return true; } template void RegionHost::lose_authority(Unacknowledged reason, HostPhase next) { for (auto& a : applied_) dropped_.push_back({a.input, reason}); for (auto& q : queue_) dropped_.push_back({q, reason}); // Actions of a batch whose publication outcome is unknown may be durable: not final. for (const auto& a : batch_actions_) dropped_actions_.push_back({a, !prepared_.has_value(), reason}); for (const auto& a : action_queue_) dropped_actions_.push_back({a, true, reason}); batch_actions_.clear(); action_queue_.clear(); applied_.clear(); queue_.clear(); batch_.reset(); // References state_: reset first. state_.reset(); committed_.reset(); prepared_.reset(); // Best effort: a quarantined region's lease is released so peers see it free. const auto& grant = authority_.grant(); if (next == HostPhase::quarantined && db_ && grant && authority_.may_originate(LeaseClock::now())) { [[maybe_unused]] const auto released = persistence::release_region_lease( *db_, config_.key, *grant, config_.lease.renew_timeout); } authority_.fence(); if (reason == Unacknowledged::fenced) ++status_.fences; status_.phase = draining_ && next == HostPhase::acquiring ? HostPhase::stopped : next; } template void RegionHost::acquire(SchedulerTime now) { if (!database()) return; // A grant survives a transient recovery failure; only re-acquire without one. if (!authority_.may_originate(LeaseClock::now())) { const auto sent = LeaseClock::now(); auto grant = persistence::acquire_region_lease( *db_, config_.key, config_.build, config_.lease.duration, config_.lease.renew_timeout); if (!grant) { // No region, or this build is retired/unknown (ADR-0068): never serve. if (grant.error().kind == LeaseStoreError::not_provisioned || grant.error().kind == LeaseStoreError::unsupported) status_.phase = HostPhase::quarantined; else if (grant.error().kind != LeaseStoreError::busy) db_.reset(); return; } if (grant->frame != config_.frame) { status_.phase = HostPhase::quarantined; return; } if (!authority_.acquired(*grant, sent, LeaseClock::now())) return; // Reply too late to use; the lease expires unused. } auto recovered = Ops::recover(*db_, *objects_, config_.key, config_.retained_builds, config_.restore_budget, config_.lease.commit_timeout); if (!recovered) { if (recovered.error().kind == RecoveryError::store) { db_.reset(); // Transient: keep the grant and retry recovery. ++status_.store_failures; return; } lose_authority(Unacknowledged::quarantined, HostPhase::quarantined); return; } if (Ops::quantum(*recovered->state) != static_cast(config_.tick_quantum.count())) { lose_authority(Unacknowledged::quarantined, HostPhase::quarantined); return; } state_ = std::move(recovered->state); committed_ = std::move(recovered->committed); batch_.emplace(*state_); origin_ = now; origin_tick_ = Ops::tick(*state_); status_.max_tick_debt = 0; status_.phase = draining_ ? HostPhase::draining : HostPhase::running; } template void RegionHost::upkeep() { const auto state = authority_.state(LeaseClock::now()); if (state == persistence::LeaseState::fenced) { lose_authority(Unacknowledged::fenced, HostPhase::acquiring); return; } if (state != persistence::LeaseState::renew_due || !database()) return; const auto grant = *authority_.grant(); const auto sent = LeaseClock::now(); auto renewed = persistence::renew_region_lease(*db_, config_.key, grant, config_.lease.duration, config_.lease.renew_timeout); if (renewed) { if (!authority_.renewed(grant.epoch, *renewed, sent, LeaseClock::now())) lose_authority(Unacknowledged::fenced, HostPhase::acquiring); return; } switch (renewed.error().kind) { case LeaseStoreError::unknown: // It may have shortened or extended nothing: keep the earlier deadline. authority_.renewal_unknown(config_.lease.duration, sent); db_.reset(); ++status_.store_failures; return; case LeaseStoreError::fenced: lose_authority(Unacknowledged::fenced, HostPhase::acquiring); return; default: db_.reset(); ++status_.store_failures; return; } } template persistence::Uuid RegionHost::next_commit_id() const { // Unique: the DB-generated holder token names this acquisition; the // sequence names the link. Stable across retries of the same batch. const auto& grant = authority_.grant(); if (!grant || !committed_) return {}; auto id = grant->holder; const auto sequence = committed_->head.sequence + 1; for (unsigned i = 0; i < 8; ++i) id[8 + i] = static_cast((sequence >> (8U * i)) & 0xFFU); return id; } template void RegionHost::confirmed(const persistence::PublishedBatch& published, SchedulerTime now) { if (config_.crash_hook) config_.crash_hook(CrashPoint::after_commit); const auto& head = published.committed.head; if (!batch_ || !batch_->confirm(head.end_tick)) { lose_authority(Unacknowledged::quarantined, HostPhase::quarantined); return; } committed_ = published.committed; for (const auto& a : applied_) acks_.push_back({a.input.request, a.tick, head.sequence}); applied_.clear(); for (const auto& d : published.decisions) outcomes_.push_back({d, head.sequence}); for (const auto& a : batch_actions_) action_acks_.push_back({a, head.sequence}); batch_actions_.clear(); prepared_.reset(); batch_started_ = now; retry_at_ = {}; retry_delay_ = {}; ++status_.commits; status_.phase = draining_ ? HostPhase::draining : HostPhase::running; } template void RegionHost::commit(SchedulerTime now) { if (!authority_.may_begin_commit(LeaseClock::now())) return; // Renewal first; upkeep fences if the deadline passes. if (!database()) return; const auto& grant = authority_.grant(); if (!grant || !committed_ || !batch_) return; const auto commit_id = next_commit_id(); batch_->seal(); const auto& crash = config_.crash_hook; if (crash) crash(CrashPoint::before_upload); persistence::PublishHooks hooks; if (crash) hooks.before_transaction = [&crash] { crash(CrashPoint::before_transaction); }; auto published = Ops::publish(*db_, *objects_, *grant, *committed_, *batch_, commit_id, config_.build, config_.restore_budget, config_.lease.commit_timeout, hooks, batch_actions_); if (published) { confirmed(*published, now); return; } const auto& f = published.error(); switch (f.kind) { case PublicationError::upload: case PublicationError::verify: case PublicationError::digest: ++status_.store_failures; // Nothing sent: retry the same batch later. return; case PublicationError::unknown: prepared_ = f.prepared; db_.reset(); ++status_.unknown_outcomes; status_.phase = HostPhase::resolving; return; case PublicationError::rejected: if (f.store.kind == LeaseStoreError::fenced) { lose_authority(Unacknowledged::fenced, HostPhase::acquiring); return; } if (f.store.kind == LeaseStoreError::store || f.store.kind == LeaseStoreError::invalid) { // SQL/connection error before any mutation: retry with the same bytes. db_.reset(); ++status_.store_failures; return; } [[fallthrough]]; default: // candidate, predecessor, receipt, head moved, conflicts: fail closed. lose_authority(Unacknowledged::quarantined, HostPhase::quarantined); return; } } template void RegionHost::resolve(SchedulerTime now) { if (!database() || !prepared_ || !batch_) return; const auto& prepared = *prepared_; auto resolved = persistence::resolve_prepared(*db_, prepared, config_.lease.commit_timeout); if (!resolved) { db_.reset(); ++status_.store_failures; return; } const auto adopt = [&](const persistence::RegionHead& receipt) { auto decisions = persistence::decode_decisions(prepared.decisions, batch_->begin_tick(), batch_->end_tick()); const auto digest = persistence::sha256(prepared.manifest); if (!decisions || !digest || *digest != receipt.digest || receipt.end_tick != batch_->end_tick()) { lose_authority(Unacknowledged::quarantined, HostPhase::quarantined); return; } confirmed({{receipt, prepared.manifest}, *decisions}, now); }; switch (resolved->status) { case persistence::Resolution::committed: adopt(resolved->receipt); return; case persistence::Resolution::conflict: lose_authority(Unacknowledged::quarantined, HostPhase::quarantined); return; case persistence::Resolution::absent: break; } // Absent now: retry exactly the same bytes, only while still authorized. if (!authority_.may_begin_commit(LeaseClock::now())) return; auto retried = persistence::retry_prepared(*db_, prepared, config_.lease.commit_timeout); if (retried) { adopt(*retried); return; } switch (retried.error().kind) { case LeaseStoreError::fenced: lose_authority(Unacknowledged::fenced, HostPhase::acquiring); return; case LeaseStoreError::unknown: case LeaseStoreError::store: db_.reset(); ++status_.store_failures; return; default: lose_authority(Unacknowledged::quarantined, HostPhase::quarantined); return; } } template void RegionHost::simulate(SchedulerTime now) { const auto elapsed = std::chrono::duration_cast(now - origin_); const auto due = elapsed.count() <= 0 ? std::uint64_t{0} : static_cast(elapsed / config_.tick_quantum); if (!batch_ || !state_) return; auto& batch = *batch_; const auto& state = *state_; for (std::uint32_t k = 0; k < config_.max_catch_up_ticks && !batch.sealed(); ++k) { if (Ops::tick(state) - origin_tick_ >= due || queue_.empty() || !authority_.may_originate(LeaseClock::now())) break; // Not due, starved (debt accrues; inputs are never invented) or unauthorized. const auto input = queue_.front(); queue_.pop_front(); if (batch.steps().empty()) batch_started_ = now; auto report = batch.advance(input.input); if (!report) { dropped_.push_back({input, Unacknowledged::refused}); continue; } applied_.push_back({input, report->tick}); // Actions due at or before this tick ride it (late ones ride the first tick run). std::erase_if(action_queue_, [&](persistence::ActionRecord action) { if (action.tick > report->tick) return false; action.tick = report->tick; batch_actions_.push_back(action); return true; }); } } template void RegionHost::refresh(SchedulerTime now) { status_.queued = queue_.size(); status_.queued_bytes = queue_.size() * tick_input_bytes; const auto& grant = authority_.grant(); status_.epoch = grant ? grant->epoch : 0; const auto lease_now = LeaseClock::now(); status_.lease_remaining_ms = grant && authority_.may_originate(lease_now) ? static_cast(std::chrono::duration_cast( authority_.deadline() - lease_now) .count()) : 0; status_.committed_tick = committed_ ? committed_->head.end_tick : 0; status_.committed_sequence = committed_ ? committed_->head.sequence : 0; status_.simulated_tick = state_ ? Ops::tick(*state_) : status_.committed_tick; if (state_ && status_.phase == HostPhase::running) { const auto elapsed = std::chrono::duration_cast(now - origin_); const auto due = elapsed.count() <= 0 ? std::uint64_t{0} : static_cast(elapsed / config_.tick_quantum); const auto done = Ops::tick(*state_) - origin_tick_; status_.tick_debt = due > done ? due - done : 0; status_.max_tick_debt = std::max(status_.max_tick_debt, status_.tick_debt); } status_.ready = status_.phase == HostPhase::running && state_ && !draining_ && authority_.may_originate(LeaseClock::now()); } template const HostStatus& RegionHost::step(SchedulerTime now) { switch (status_.phase) { case HostPhase::acquiring: acquire(now); break; case HostPhase::resolving: upkeep(); if (status_.phase == HostPhase::resolving) resolve(now); break; case HostPhase::running: case HostPhase::draining: { upkeep(); if (status_.phase != HostPhase::running && status_.phase != HostPhase::draining) break; if (!draining_) simulate(now); if (!batch_) break; // Running always has a batch; defensive. const bool flush = !batch_->steps().empty() && now >= retry_at_ && (batch_->sealed() || draining_ || now - batch_started_ >= config_.flush_interval); if (flush) { const auto failures = status_.store_failures; commit(now); // May lose authority and reset the batch. if (status_.store_failures != failures) { // Every retry re-uploads the batch: back off instead of spinning. retry_delay_ = retry_delay_ == std::chrono::milliseconds{} ? commit_retry_initial : std::min(retry_delay_ * 2, commit_retry_limit); retry_at_ = now + retry_delay_; } } if (draining_ && status_.phase == HostPhase::draining && batch_ && batch_->steps().empty()) { const auto& grant = authority_.grant(); if (database() && grant && authority_.may_originate(LeaseClock::now())) { // Best effort: an unreleased lease simply expires. [[maybe_unused]] const auto released = persistence::release_region_lease( *db_, config_.key, *grant, config_.lease.renew_timeout); } batch_.reset(); state_.reset(); authority_.fence(); db_.reset(); status_.phase = HostPhase::stopped; } break; } case HostPhase::stopped: case HostPhase::quarantined: break; } refresh(now); return status_; } template void RegionHost::drain() { if (draining_) return; draining_ = true; for (auto& q : queue_) dropped_.push_back({q, Unacknowledged::drained}); queue_.clear(); for (const auto& a : action_queue_) dropped_actions_.push_back({a, true, Unacknowledged::drained}); action_queue_.clear(); if (status_.phase == HostPhase::running) status_.phase = HostPhase::draining; else if (status_.phase == HostPhase::acquiring) status_.phase = HostPhase::stopped; status_.ready = false; } template std::optional RegionHost::authority() const { if (!status_.ready || !authority_.may_originate(LeaseClock::now())) return std::nullopt; return authority_.grant(); } template std::vector RegionHost::take_acks() { return std::exchange(acks_, {}); } template std::vector RegionHost::take_outcomes() { return std::exchange(outcomes_, {}); } template std::vector RegionHost::take_dropped() { return std::exchange(dropped_, {}); } template std::vector RegionHost::take_action_acks() { return std::exchange(action_acks_, {}); } template std::vector RegionHost::take_dropped_actions() { return std::exchange(dropped_actions_, {}); } template class RegionHost; template class RegionHost; } // namespace evo::server