Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m25s
Foundation / checks (network) (push) Successful in 20m44s
Foundation / checks (client) (push) Successful in 21m57s
Foundation / checks (tsan) (push) Successful in 22m46s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 30m2s
Foundation / checks (postgres) (push) Canceled after 29m56s
1327 lines
59 KiB
C++
1327 lines
59 KiB
C++
#include "evo/server/cellular_region_host.hpp"
|
|
|
|
#include "evo/biology/founder.hpp"
|
|
#include "evo/persistence/sha256.hpp"
|
|
|
|
#include <algorithm>
|
|
#include <array>
|
|
#include <chrono>
|
|
#include <limits>
|
|
#include <utility>
|
|
|
|
namespace evo::server {
|
|
struct CellularRegionHost::Pending {
|
|
persistence::CellularPreparedCommit prepared;
|
|
OwnedState candidate;
|
|
std::vector<CellularActionRecord> actions;
|
|
std::vector<CellularTickIntents::Birth> births;
|
|
std::vector<CellularMutationActionRecord> mutation_actions;
|
|
std::optional<persistence::PopulationPreparedCommit> population_prepared{};
|
|
};
|
|
|
|
namespace {
|
|
using persistence::LeaseClock;
|
|
using persistence::LeaseStoreError;
|
|
using std::chrono::milliseconds;
|
|
|
|
bool valid(const CellularHostConfig& config) {
|
|
return config.key.world != 0 && config.key.region != 0 && config.frame != 0 &&
|
|
persistence::valid(config.lease) && config.build != persistence::Sha256Digest{} &&
|
|
std::ranges::find(config.retained_builds, config.build) != config.retained_builds.end();
|
|
}
|
|
|
|
network::PrimitiveLifeResult life_result(persistence::LifeRefusal refusal) {
|
|
switch (refusal) {
|
|
case persistence::LifeRefusal::none:
|
|
return network::PrimitiveLifeResult::pending;
|
|
case persistence::LifeRefusal::insufficient_funding:
|
|
return network::PrimitiveLifeResult::insufficient_funding;
|
|
case persistence::LifeRefusal::no_placement:
|
|
return network::PrimitiveLifeResult::no_placement;
|
|
case persistence::LifeRefusal::cancelled:
|
|
return network::PrimitiveLifeResult::cancelled;
|
|
case persistence::LifeRefusal::existing_life:
|
|
return network::PrimitiveLifeResult::existing_life;
|
|
}
|
|
return network::PrimitiveLifeResult::unknown;
|
|
}
|
|
|
|
std::vector<CellularTickIntents::Birth> births_in(const CellularBatch& batch) {
|
|
std::vector<CellularTickIntents::Birth> births;
|
|
for (const auto& tick : batch.ticks)
|
|
births.insert(births.end(), tick.births.begin(), tick.births.end());
|
|
return births;
|
|
}
|
|
|
|
std::optional<persistence::Uuid> commit_id(const persistence::CommittedCheckpoint& previous,
|
|
std::span<const std::uint8_t> trace,
|
|
std::span<const std::uint8_t> actions) {
|
|
std::vector<std::uint8_t> identity;
|
|
identity.reserve(previous.head.digest.size() + 16 + trace.size() + actions.size());
|
|
identity.insert(identity.end(), previous.head.digest.begin(), previous.head.digest.end());
|
|
const auto append_length = [&](std::size_t size) {
|
|
const auto length = static_cast<std::uint64_t>(size);
|
|
for (unsigned i = 0; i < 8; ++i)
|
|
identity.push_back(static_cast<std::uint8_t>(length >> (i * 8U)));
|
|
};
|
|
append_length(trace.size());
|
|
identity.insert(identity.end(), trace.begin(), trace.end());
|
|
append_length(actions.size());
|
|
identity.insert(identity.end(), actions.begin(), actions.end());
|
|
const auto digest = persistence::sha256(identity);
|
|
if (!digest)
|
|
return std::nullopt;
|
|
persistence::Uuid out{};
|
|
std::ranges::copy_n(digest->begin(), out.size(), out.begin());
|
|
if (out == persistence::Uuid{})
|
|
out.back() = 1;
|
|
return out;
|
|
}
|
|
} // namespace
|
|
|
|
network::AdmitStatus admit_status(CellularMutationAdmission admission) noexcept {
|
|
switch (admission) {
|
|
case CellularMutationAdmission::admitted:
|
|
return network::AdmitStatus::admitted;
|
|
case CellularMutationAdmission::not_ready:
|
|
return network::AdmitStatus::not_ready;
|
|
case CellularMutationAdmission::busy:
|
|
return network::AdmitStatus::too_many_unresolved;
|
|
case CellularMutationAdmission::final:
|
|
return network::AdmitStatus::duplicate;
|
|
case CellularMutationAdmission::invalid:
|
|
case CellularMutationAdmission::conflict:
|
|
break;
|
|
}
|
|
return network::AdmitStatus::invalid_action;
|
|
}
|
|
|
|
network::ActionOutcome action_outcome(const FinalCellularMutation& mutation) noexcept {
|
|
return {mutation.action_id,
|
|
mutation.result == CellularMutationResult::applied ||
|
|
mutation.result == CellularMutationResult::declined
|
|
? network::ActionResult::committed
|
|
: network::ActionResult::committed_no_effect,
|
|
mutation.tick, mutation.commit_sequence};
|
|
}
|
|
|
|
network::CellularMutationOffer offer_view(const CellularLives& state, const CellularOwnedLife& life,
|
|
std::uint64_t tick) {
|
|
network::CellularMutationOffer view{tick, life.identity.life, 0, 0, {}};
|
|
const auto& offer = life.mutation.offer;
|
|
if (!offer)
|
|
return view;
|
|
view.event_index = life.mutation.event_index;
|
|
view.offer_tick = life.mutation.offer_tick;
|
|
const auto genome = state.archive().find(life.identity.genome);
|
|
// The cost does not depend on the reserve: fund against an ample one and read what it
|
|
// takes. The owner funds the real choice against the real reserve.
|
|
constexpr std::uint64_t ample = std::uint64_t{1} << 60U;
|
|
const accounting::Inventory unlimited{
|
|
{ample},
|
|
{accounting::Femtograms{ample}, accounting::Femtograms{ample},
|
|
accounting::Femtograms{ample}, accounting::Femtograms{ample}}};
|
|
constexpr auto probe_identity = std::numeric_limits<std::uint64_t>::max();
|
|
for (const auto& candidate : offer->candidates) {
|
|
if (view.candidates.size() >= network::max_cellular_offered_candidates)
|
|
break;
|
|
network::CellularOfferedCandidate shown{candidate.operator_id,
|
|
static_cast<std::uint8_t>(candidate.target),
|
|
static_cast<std::uint8_t>(candidate.effect),
|
|
candidate.parameter,
|
|
candidate.maintenance,
|
|
candidate.total_mass,
|
|
0,
|
|
{}};
|
|
if (genome) {
|
|
const auto funded =
|
|
biology::fund_mutation(*genome, offer->context, *offer, candidate.operator_id,
|
|
probe_identity, probe_identity, unlimited);
|
|
if (funded) {
|
|
shown.cost_energy_pj = funded->intake.energy.value;
|
|
for (std::size_t channel = 0; channel < shown.cost_matter_fg.size(); ++channel)
|
|
shown.cost_matter_fg[channel] = funded->intake.matter[channel].value;
|
|
}
|
|
}
|
|
view.candidates.push_back(shown);
|
|
}
|
|
std::ranges::sort(view.candidates, {}, &network::CellularOfferedCandidate::operator_id);
|
|
return view;
|
|
}
|
|
|
|
CellularRegionHost::CellularRegionHost(CellularHostConfig config, CellularConnect connect,
|
|
std::unique_ptr<persistence::FilesystemObjectStore> objects)
|
|
: config_{std::move(config)}, connect_{std::move(connect)}, objects_{std::move(objects)},
|
|
authority_{config_.lease} {}
|
|
|
|
CellularRegionHost::~CellularRegionHost() = default;
|
|
|
|
std::expected<std::unique_ptr<CellularRegionHost>, CellularHostError>
|
|
CellularRegionHost::create(CellularHostConfig config, CellularConnect connect,
|
|
std::unique_ptr<persistence::FilesystemObjectStore> objects) {
|
|
if (!valid(config) || !connect || !objects)
|
|
return std::unexpected{CellularHostError::configuration};
|
|
return std::unique_ptr<CellularRegionHost>{
|
|
new CellularRegionHost{std::move(config), std::move(connect), std::move(objects)}};
|
|
}
|
|
|
|
bool CellularRegionHost::database() {
|
|
if (db_)
|
|
return true;
|
|
auto connection = connect_();
|
|
if (!connection) {
|
|
++status_.store_failures;
|
|
return false;
|
|
}
|
|
db_ = std::move(*connection);
|
|
return true;
|
|
}
|
|
|
|
void CellularRegionHost::acquire() {
|
|
if (!database())
|
|
return;
|
|
const auto sent = LeaseClock::now();
|
|
auto lease = persistence::acquire_region_lease(
|
|
*db_, config_.key, config_.build, config_.lease.duration, config_.lease.renew_timeout);
|
|
if (!lease) {
|
|
if (lease.error().kind == LeaseStoreError::not_provisioned ||
|
|
lease.error().kind == LeaseStoreError::unsupported) {
|
|
status_.phase = HostPhase::quarantined;
|
|
} else if (lease.error().kind != LeaseStoreError::busy) {
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
}
|
|
return;
|
|
}
|
|
if (lease->frame != config_.frame || !authority_.acquired(*lease, sent, LeaseClock::now())) {
|
|
status_.phase = HostPhase::quarantined;
|
|
return;
|
|
}
|
|
if (!recover_state(*lease) || !committed_.has_value() || !state_)
|
|
return;
|
|
migration_reload_ = true;
|
|
migration_frozen_ = false;
|
|
migration_poll_at_ = {};
|
|
interval_reload_ = population_mode();
|
|
admitted_interval_.reset();
|
|
interval_request_.reset();
|
|
interval_failure_.reset();
|
|
queued_mutations_.clear();
|
|
mutations_uncertain_ = true;
|
|
pending_endings_.clear();
|
|
// Reconcile retained owned deaths after either profile's recovery, including
|
|
// older profile-5 records that admitted lingering cells. Existing endings
|
|
// are idempotent; no ending is inferred for a living suspended cell.
|
|
for (const auto& corpse : state_->corpses())
|
|
if (corpse.parent_life == 0)
|
|
pending_endings_.push_back({committed_->head.commit, corpse.life_id});
|
|
status_.phase = HostPhase::running;
|
|
status_.simulated_tick = committed_->head.end_tick;
|
|
status_.committed_tick = committed_->head.end_tick;
|
|
status_.committed_sequence = committed_->head.sequence;
|
|
}
|
|
|
|
bool CellularRegionHost::recover_state(const persistence::LeaseGrant& lease) {
|
|
auto head =
|
|
persistence::load_region_checkpoint(*db_, config_.key, config_.lease.commit_timeout);
|
|
if (!head) {
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
return false;
|
|
}
|
|
const auto manifest = persistence::decode_manifest(head->manifest);
|
|
if (!manifest) {
|
|
authority_.fence();
|
|
status_.phase = HostPhase::quarantined;
|
|
return false;
|
|
}
|
|
if (manifest->profile == persistence::CheckpointProfile::cellular_population_v1) {
|
|
auto recovered = persistence::recover_population_checkpoint(*db_, *objects_, config_.key,
|
|
lease, config_.retained_builds,
|
|
config_.lease.commit_timeout);
|
|
if (!recovered) {
|
|
if (recovered.error().kind == persistence::RecoveryError::store) {
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
} else {
|
|
authority_.fence();
|
|
status_.phase = HostPhase::quarantined;
|
|
}
|
|
return false;
|
|
}
|
|
state_ = std::move(recovered->state);
|
|
committed_ = std::move(recovered->committed);
|
|
return true;
|
|
}
|
|
auto recovered = persistence::recover_cellular_checkpoint(
|
|
*db_, *objects_, config_.key, lease, config_.retained_builds, config_.lease.commit_timeout);
|
|
if (!recovered) {
|
|
if (recovered.error().kind == persistence::RecoveryError::store) {
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
return false;
|
|
}
|
|
authority_.fence();
|
|
status_.phase = HostPhase::quarantined;
|
|
return false;
|
|
}
|
|
state_ = std::move(recovered->state);
|
|
committed_ = std::move(recovered->committed);
|
|
return true;
|
|
}
|
|
|
|
void CellularRegionHost::upkeep() {
|
|
if (authority_.state(LeaseClock::now()) == persistence::LeaseState::fenced) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
pending_.reset();
|
|
status_.ready = false;
|
|
status_.phase = HostPhase::acquiring;
|
|
++status_.fences;
|
|
return;
|
|
}
|
|
if (authority_.state(LeaseClock::now()) != persistence::LeaseState::renew_due || !database())
|
|
return;
|
|
const auto grant = *authority_.grant();
|
|
const auto sent = LeaseClock::now();
|
|
auto renewed = persistence::renew_region_lease(*db_, config_.key, grant, config_.lease.duration,
|
|
config_.lease.renew_timeout);
|
|
if (renewed) {
|
|
if (!authority_.renewed(grant.epoch, *renewed, sent, LeaseClock::now())) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
pending_.reset();
|
|
status_.ready = false;
|
|
status_.phase = HostPhase::acquiring;
|
|
}
|
|
return;
|
|
}
|
|
if (renewed.error().kind == LeaseStoreError::unknown)
|
|
authority_.renewal_unknown(config_.lease.duration, sent);
|
|
else if (renewed.error().kind == LeaseStoreError::fenced) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
pending_.reset();
|
|
status_.ready = false;
|
|
status_.phase = HostPhase::acquiring;
|
|
++status_.fences;
|
|
}
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
}
|
|
|
|
void CellularRegionHost::resolve() {
|
|
if (!pending_ || !database())
|
|
return;
|
|
if (!authority_.may_begin_commit(LeaseClock::now())) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
pending_.reset();
|
|
status_.phase = HostPhase::acquiring;
|
|
status_.ready = false;
|
|
++status_.fences;
|
|
return;
|
|
}
|
|
const auto& population_prepared = pending_->population_prepared;
|
|
const auto resolved = population_prepared.has_value()
|
|
? persistence::resolve_population_prepared(
|
|
*db_, *population_prepared, config_.lease.commit_timeout)
|
|
: persistence::resolve_cellular_prepared(
|
|
*db_, pending_->prepared, config_.lease.commit_timeout);
|
|
if (!resolved) {
|
|
if (resolved.error().kind == LeaseStoreError::fenced) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
pending_.reset();
|
|
status_.phase = HostPhase::acquiring;
|
|
status_.ready = false;
|
|
++status_.fences;
|
|
} else {
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
}
|
|
return;
|
|
}
|
|
if (resolved->status == persistence::Resolution::conflict) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
pending_.reset();
|
|
status_.phase = HostPhase::quarantined;
|
|
status_.ready = false;
|
|
return;
|
|
}
|
|
if (resolved->status == persistence::Resolution::absent &&
|
|
!authority_.may_begin_commit(LeaseClock::now()))
|
|
return;
|
|
auto retried = [&]() -> std::expected<persistence::CommittedCheckpoint,
|
|
persistence::CellularPublicationFailure> {
|
|
if (!pending_->population_prepared)
|
|
return persistence::retry_cellular_prepared(*db_, pending_->prepared,
|
|
config_.lease.commit_timeout);
|
|
auto result = persistence::retry_population_prepared(*db_, *pending_->population_prepared,
|
|
config_.lease.commit_timeout);
|
|
if (!result)
|
|
return std::unexpected(persistence::CellularPublicationFailure{
|
|
result.error().kind, result.error().store, result.error().object, {}});
|
|
return std::move(*result);
|
|
}();
|
|
if (!retried) {
|
|
if (retried.error().kind == persistence::PublicationError::unknown) {
|
|
db_.reset();
|
|
++status_.unknown_outcomes;
|
|
status_.phase = HostPhase::resolving;
|
|
status_.ready = false;
|
|
} else if (retried.error().store.kind == LeaseStoreError::fenced) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
pending_.reset();
|
|
status_.phase = HostPhase::acquiring;
|
|
status_.ready = false;
|
|
++status_.fences;
|
|
} else {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
pending_.reset();
|
|
status_.phase = HostPhase::quarantined;
|
|
status_.ready = false;
|
|
}
|
|
return;
|
|
}
|
|
note_endings(*state_, *pending_->candidate, retried->head.commit);
|
|
state_ = std::move(pending_->candidate);
|
|
committed_ = std::move(*retried);
|
|
finish_interval();
|
|
for (const auto& action : pending_->actions)
|
|
finalized_actions_.push_back({action, committed_->head.sequence});
|
|
finalize_births(pending_->births);
|
|
finalize_mutations(pending_->mutation_actions, pending_->prepared.results);
|
|
pending_.reset();
|
|
++status_.commits;
|
|
status_.committed_sequence = committed_->head.sequence;
|
|
status_.committed_tick = committed_->head.end_tick;
|
|
status_.simulated_tick = committed_->head.end_tick;
|
|
status_.phase = HostPhase::running;
|
|
}
|
|
|
|
void CellularRegionHost::refresh() {
|
|
const auto& grant = authority_.grant();
|
|
status_.ready = status_.phase == HostPhase::running && state_ && committed_ && grant &&
|
|
authority_.may_originate(LeaseClock::now()) && !interval_reload_ &&
|
|
!interval_request_.has_value() && !migration_reload_ && !migration_frozen_ &&
|
|
!migration_request_.has_value() && !migration_stage_request_.has_value() &&
|
|
!migration_abort_.has_value();
|
|
status_.migration_frozen = migration_frozen_;
|
|
const auto& interval = admitted_interval_;
|
|
status_.ecological_pending_ticks =
|
|
interval.has_value() ? interval->end_tick - interval->begin_tick : 0;
|
|
status_.epoch = grant ? grant->epoch : 0;
|
|
status_.lease_remaining_ms =
|
|
status_.ready ? static_cast<std::uint64_t>(std::chrono::duration_cast<milliseconds>(
|
|
authority_.deadline() - LeaseClock::now())
|
|
.count())
|
|
: 0;
|
|
}
|
|
|
|
void CellularRegionHost::reconcile_migration_freeze() {
|
|
if ((!migration_reload_ && !migration_frozen_ && !migration_request_ &&
|
|
!migration_stage_request_ && !migration_abort_) ||
|
|
LeaseClock::now() < migration_poll_at_)
|
|
return;
|
|
const auto grant = authority_.grant();
|
|
if (!grant || !authority_.may_begin_commit(LeaseClock::now()) || !database())
|
|
return;
|
|
migration_poll_at_ = LeaseClock::now() + std::chrono::seconds{1};
|
|
if (migration_request_) {
|
|
auto result = persistence::retry_cellular_migration_freeze(
|
|
*db_, *grant, *migration_request_, config_.lease.commit_timeout);
|
|
if (!result) {
|
|
if (result.error().kind == persistence::PublicationError::unknown) {
|
|
db_.reset();
|
|
++status_.unknown_outcomes;
|
|
return;
|
|
}
|
|
if (result.error().store.kind == LeaseStoreError::fenced ||
|
|
result.error().store.kind == LeaseStoreError::head_mismatch) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
status_.phase = HostPhase::acquiring;
|
|
} else {
|
|
authority_.fence();
|
|
status_.phase = HostPhase::quarantined;
|
|
}
|
|
return;
|
|
}
|
|
migration_request_.reset();
|
|
}
|
|
if (migration_stage_request_) {
|
|
const auto staged = persistence::retry_cellular_migration_stage(
|
|
*db_, *grant, *migration_stage_request_, config_.lease.commit_timeout);
|
|
if (!staged) {
|
|
if (staged.error().kind == persistence::PublicationError::unknown) {
|
|
db_.reset();
|
|
++status_.unknown_outcomes;
|
|
return;
|
|
}
|
|
// A concurrent source abort can win before the uncertain stage.
|
|
// Reload authoritative pointers before making the region ready.
|
|
if (staged.error().store.kind != LeaseStoreError::commit_conflict) {
|
|
authority_.fence();
|
|
status_.phase = staged.error().store.kind == LeaseStoreError::fenced ||
|
|
staged.error().store.kind == LeaseStoreError::head_mismatch
|
|
? HostPhase::acquiring
|
|
: HostPhase::quarantined;
|
|
state_.reset();
|
|
committed_.reset();
|
|
return;
|
|
}
|
|
}
|
|
migration_stage_request_.reset();
|
|
}
|
|
if (migration_abort_) {
|
|
auto result = persistence::abort_cellular_migration_freeze(
|
|
*db_, config_.key, *grant, *migration_abort_, config_.lease.commit_timeout);
|
|
if (!result) {
|
|
if (result.error().kind == LeaseStoreError::unknown) {
|
|
db_.reset();
|
|
++status_.unknown_outcomes;
|
|
return;
|
|
}
|
|
authority_.fence();
|
|
status_.phase = result.error().kind == LeaseStoreError::fenced ? HostPhase::acquiring
|
|
: HostPhase::quarantined;
|
|
state_.reset();
|
|
committed_.reset();
|
|
return;
|
|
}
|
|
migration_abort_.reset();
|
|
}
|
|
const auto frozen = persistence::load_cellular_migration_freeze(
|
|
*db_, *objects_, config_.key, *grant, config_.lease.commit_timeout);
|
|
if (!frozen) {
|
|
migration_reload_ = true;
|
|
if (frozen.error().kind == persistence::RecoveryError::store) {
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
} else {
|
|
authority_.fence();
|
|
status_.phase = HostPhase::quarantined;
|
|
}
|
|
return;
|
|
}
|
|
migration_frozen_ = frozen->has_value();
|
|
if (!migration_frozen_) {
|
|
const auto staged = persistence::load_cellular_migration_stage(
|
|
*db_, *objects_, config_.key, *grant, config_.lease.commit_timeout);
|
|
if (!staged) {
|
|
migration_reload_ = true;
|
|
if (staged.error().kind == persistence::RecoveryError::store) {
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
} else {
|
|
authority_.fence();
|
|
status_.phase = HostPhase::quarantined;
|
|
}
|
|
return;
|
|
}
|
|
migration_frozen_ = staged->has_value();
|
|
}
|
|
migration_reload_ = false;
|
|
}
|
|
|
|
std::expected<void, CellularPublishFailure>
|
|
CellularRegionHost::stage_migration(const persistence::CommittedCheckpoint& source,
|
|
const persistence::FilesystemObjectStore& source_objects,
|
|
const CellularMigrationTrace& trace) {
|
|
const auto grant = authority_.grant();
|
|
if (!grant.has_value() || !status_.ready || pending_ || !state_ || !committed_ ||
|
|
!queued_births_.empty() || !queued_mutations_.empty() || mutations_uncertain_ ||
|
|
!pending_endings_.empty() || interval_request_ || admitted_interval_ || interval_reload_ ||
|
|
!population_interval_safe(*state_) || !authority_.may_begin_commit(LeaseClock::now()) ||
|
|
!database())
|
|
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
|
migration_reload_ = true;
|
|
refresh();
|
|
const auto staged =
|
|
persistence::stage_cellular_migration(*db_, source_objects, *objects_, *grant, source,
|
|
*committed_, trace, config_.lease.commit_timeout);
|
|
if (!staged) {
|
|
if (staged.error().kind == persistence::PublicationError::unknown) {
|
|
migration_stage_request_ = staged.error().prepared;
|
|
db_.reset();
|
|
++status_.unknown_outcomes;
|
|
} else if (staged.error().store.kind == LeaseStoreError::fenced ||
|
|
staged.error().store.kind == LeaseStoreError::head_mismatch) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
status_.phase = HostPhase::acquiring;
|
|
}
|
|
migration_poll_at_ = {};
|
|
refresh();
|
|
return std::unexpected(CellularPublishFailure{CellularHostError::publication,
|
|
staged.error().kind, staged.error().store,
|
|
staged.error().object});
|
|
}
|
|
migration_poll_at_ = {};
|
|
reconcile_migration_freeze();
|
|
refresh();
|
|
return {};
|
|
}
|
|
|
|
std::expected<void, CellularPublishFailure> CellularRegionHost::prepare_migration_freeze(
|
|
const persistence::CommittedCheckpoint& destination,
|
|
const persistence::FilesystemObjectStore& destination_objects,
|
|
const CellularMigrationRequest& request) {
|
|
const auto grant = authority_.grant();
|
|
if (!grant.has_value() || !status_.ready || pending_ || !state_ || !committed_ ||
|
|
!queued_births_.empty() || !queued_mutations_.empty() || mutations_uncertain_ ||
|
|
!pending_endings_.empty() || interval_request_ || admitted_interval_ || interval_reload_ ||
|
|
!population_interval_safe(*state_) || !authority_.may_begin_commit(LeaseClock::now()) ||
|
|
!database())
|
|
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
|
migration_reload_ = true;
|
|
refresh();
|
|
const auto result = persistence::prepare_cellular_migration_freeze(
|
|
*db_, *objects_, destination_objects, *grant, *committed_, destination, request,
|
|
config_.lease.commit_timeout);
|
|
if (!result) {
|
|
if (result.error().kind == persistence::PublicationError::unknown) {
|
|
migration_request_ = result.error().prepared;
|
|
db_.reset();
|
|
++status_.unknown_outcomes;
|
|
} else if (result.error().store.kind == LeaseStoreError::head_mismatch ||
|
|
result.error().store.kind == LeaseStoreError::fenced) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
status_.phase = HostPhase::acquiring;
|
|
}
|
|
migration_poll_at_ = {};
|
|
refresh();
|
|
return std::unexpected(CellularPublishFailure{CellularHostError::publication,
|
|
result.error().kind, result.error().store,
|
|
result.error().object});
|
|
}
|
|
migration_poll_at_ = {};
|
|
reconcile_migration_freeze();
|
|
refresh();
|
|
return {};
|
|
}
|
|
|
|
std::expected<void, CellularPublishFailure>
|
|
CellularRegionHost::abort_migration_freeze(const persistence::Uuid& request) {
|
|
const auto grant = authority_.grant();
|
|
if (!grant || !state_ || pending_ || status_.phase != HostPhase::running ||
|
|
!authority_.may_begin_commit(LeaseClock::now()) || !database())
|
|
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
|
migration_reload_ = true;
|
|
refresh();
|
|
const auto result = persistence::abort_cellular_migration_freeze(
|
|
*db_, config_.key, *grant, request, config_.lease.commit_timeout);
|
|
if (!result) {
|
|
if (result.error().kind == LeaseStoreError::unknown) {
|
|
migration_abort_ = request;
|
|
db_.reset();
|
|
++status_.unknown_outcomes;
|
|
}
|
|
migration_poll_at_ = {};
|
|
refresh();
|
|
return std::unexpected(
|
|
CellularPublishFailure{CellularHostError::publication, {}, result.error(), {}});
|
|
}
|
|
migration_poll_at_ = {};
|
|
reconcile_migration_freeze();
|
|
refresh();
|
|
return {};
|
|
}
|
|
|
|
const HostStatus& CellularRegionHost::step() {
|
|
if (status_.phase == HostPhase::quarantined || status_.phase == HostPhase::stopped)
|
|
return status_;
|
|
if (authority_.grant())
|
|
upkeep();
|
|
if (pending_)
|
|
resolve();
|
|
else if (!state_)
|
|
acquire();
|
|
if (!pending_ && state_ && status_.phase == HostPhase::running)
|
|
reconcile_migration_freeze();
|
|
if (migration_reload_ || migration_frozen_ || migration_request_ || migration_stage_request_ ||
|
|
migration_abort_) {
|
|
refresh();
|
|
return status_;
|
|
}
|
|
if (!pending_ && state_ && status_.phase == HostPhase::running)
|
|
reconcile_interval();
|
|
if (!pending_ && state_ && mutations_uncertain_ && status_.phase == HostPhase::running)
|
|
reload_mutations();
|
|
if (!pending_ && state_ && status_.phase == HostPhase::running)
|
|
flush_endings();
|
|
refresh();
|
|
return status_;
|
|
}
|
|
|
|
void CellularRegionHost::note_endings(const CellularLives& before, const CellularLives& after,
|
|
const persistence::Uuid& commit) {
|
|
for (const auto& life : before.lives())
|
|
if (life.account != LifeAccount{} &&
|
|
std::ranges::find(after.lives(), life.identity.life, [](const CellularOwnedLife& next) {
|
|
return next.identity.life;
|
|
}) == after.lives().end())
|
|
pending_endings_.push_back({commit, life.identity.life});
|
|
}
|
|
|
|
void CellularRegionHost::flush_endings() {
|
|
while (!pending_endings_.empty()) {
|
|
const auto grant = authority_.grant();
|
|
if (!grant || !authority_.may_begin_commit(LeaseClock::now()) || !database())
|
|
return;
|
|
// One call per commit, at most the SQL bound of identities.
|
|
const auto commit = pending_endings_.front().commit;
|
|
std::vector<std::uint64_t> lives;
|
|
for (const auto& ending : pending_endings_)
|
|
if (ending.commit == commit && lives.size() < persistence::max_life_endings_per_call)
|
|
lives.push_back(ending.life);
|
|
const auto recorded = persistence::end_cellular_lives(*db_, config_.key, *grant, commit,
|
|
lives, config_.lease.commit_timeout);
|
|
if (!recorded) {
|
|
if (recorded.error().kind == persistence::LifeReservationError::fenced) {
|
|
authority_.fence();
|
|
status_.ready = false;
|
|
}
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
return; // Retried on a later step; a new life waits for it.
|
|
}
|
|
std::erase_if(pending_endings_, [&](const Ending& ending) {
|
|
return ending.commit == commit && std::ranges::find(lives, ending.life) != lives.end();
|
|
});
|
|
}
|
|
}
|
|
|
|
void CellularRegionHost::reload_mutations() {
|
|
const auto grant = authority_.grant();
|
|
if (!grant || !committed_ || !database())
|
|
return;
|
|
auto pending = persistence::load_pending_cellular_mutations(*db_, config_.key, *grant,
|
|
config_.lease.commit_timeout);
|
|
if (!pending) {
|
|
if (pending.error().kind == persistence::LifeReservationError::fenced) {
|
|
authority_.fence();
|
|
status_.ready = false;
|
|
}
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
return;
|
|
}
|
|
// Every reservation targets the tick after the head it was made at, and SQL refuses a
|
|
// checkpoint that skips one, so an unbound choice is always for the next tick.
|
|
queued_mutations_.clear();
|
|
for (const auto& reservation : *pending)
|
|
if (reservation.target_tick == committed_->head.end_tick + 1)
|
|
queued_mutations_.push_back(reservation);
|
|
mutations_uncertain_ = false;
|
|
}
|
|
|
|
void CellularRegionHost::finalize_mutations(std::span<const CellularMutationActionRecord> actions,
|
|
std::span<const std::uint8_t> results) {
|
|
if (!committed_)
|
|
return;
|
|
const auto sequence = committed_->head.sequence;
|
|
for (std::size_t i = 0; i < actions.size() && i < results.size(); ++i) {
|
|
const auto& action = actions[i];
|
|
std::erase_if(queued_mutations_, [&](const auto& queued) {
|
|
return queued.account == action.account && queued.action_id == action.action_id;
|
|
});
|
|
const FinalCellularMutation done{action.account, action.action_id, action.tick, sequence,
|
|
static_cast<CellularMutationResult>(results[i])};
|
|
finalized_mutations_.push_back(done);
|
|
std::erase_if(recent_mutations_,
|
|
[&](const auto& recent) { return recent.account == done.account; });
|
|
if (recent_mutations_.size() >= 1024)
|
|
recent_mutations_.erase(recent_mutations_.begin());
|
|
recent_mutations_.push_back(done);
|
|
}
|
|
}
|
|
|
|
std::vector<FinalCellularMutation> CellularRegionHost::take_finalized_mutations() {
|
|
return std::exchange(finalized_mutations_, {});
|
|
}
|
|
|
|
CellularMutationReply
|
|
CellularRegionHost::select_mutation(const persistence::Uuid& account, std::uint64_t generation,
|
|
std::uint64_t action_id,
|
|
const CellularMutationActionInput& input) {
|
|
using enum CellularMutationAdmission;
|
|
if (account == persistence::Uuid{} || generation == 0 || action_id == 0)
|
|
return {invalid, 0, {}};
|
|
if (const auto recent = std::ranges::find_if(recent_mutations_,
|
|
[&](const auto& done) {
|
|
return done.account == account &&
|
|
done.action_id == action_id;
|
|
});
|
|
recent != recent_mutations_.end())
|
|
return {final, recent->tick, *recent};
|
|
if (!status_.ready || mutations_uncertain_ || pending_ || !state_ || !committed_ ||
|
|
!authority_.may_begin_commit(LeaseClock::now()) || !database())
|
|
return {not_ready, 0, {}};
|
|
if (population_mode() && !set_population_mode(false))
|
|
return {not_ready, 0, {}};
|
|
const auto tick = committed_->head.end_tick + 1;
|
|
const CellularMutationActionRecord record{account, action_id, tick, input};
|
|
const auto bytes = encode_cellular_mutation_input(record);
|
|
for (const auto& queued : queued_mutations_) {
|
|
if (queued.account != account)
|
|
continue;
|
|
if (queued.action_id != action_id)
|
|
return {busy, queued.target_tick, {}};
|
|
return queued.input == bytes ? CellularMutationReply{admitted, tick, {}}
|
|
: CellularMutationReply{conflict, 0, {}};
|
|
}
|
|
// The choice must name this account's living cell and its exact outstanding offer.
|
|
const auto life = std::ranges::find(state_->lives(), account, &CellularOwnedLife::account);
|
|
if (life == state_->lives().end() || life->identity.life != input.life_id ||
|
|
input.event_index == 0 || life->mutation.event_index != input.event_index ||
|
|
life->mutation.offer_tick != input.offer_tick || input.offer_tick >= tick ||
|
|
life->presence != CellularPresence::active) // ADR-0128: only an active life chooses.
|
|
return {invalid, 0, {}};
|
|
const auto& offer = life->mutation.offer;
|
|
// Operator 0 declines the exact offer (ADR-0120 amendment); otherwise it must be offered.
|
|
if (!offer || (input.operator_id != 0 &&
|
|
!std::ranges::any_of(offer->candidates, [&](const auto& candidate) {
|
|
return candidate.operator_id == input.operator_id;
|
|
})))
|
|
return {invalid, 0, {}};
|
|
const auto grant = authority_.grant();
|
|
if (!grant)
|
|
return {not_ready, 0, {}};
|
|
auto reserved =
|
|
persistence::reserve_cellular_mutation(*db_, config_.key, *grant, account, generation,
|
|
action_id, bytes, config_.lease.commit_timeout);
|
|
if (!reserved) {
|
|
switch (reserved.error().kind) {
|
|
case persistence::LifeReservationError::pending_mutation:
|
|
return {busy, 0, {}};
|
|
case persistence::LifeReservationError::identity_conflict:
|
|
return {conflict, 0, {}};
|
|
case persistence::LifeReservationError::fenced:
|
|
authority_.fence();
|
|
status_.ready = false;
|
|
return {not_ready, 0, {}};
|
|
case persistence::LifeReservationError::unknown:
|
|
case persistence::LifeReservationError::store:
|
|
// The reservation may have committed: nothing publishes until it is reloaded.
|
|
mutations_uncertain_ = true;
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
return {not_ready, 0, {}};
|
|
default:
|
|
return {invalid, 0, {}};
|
|
}
|
|
}
|
|
if (reserved->region != config_.key.region || reserved->target_tick != tick ||
|
|
reserved->input != bytes) {
|
|
mutations_uncertain_ = true;
|
|
return {not_ready, 0, {}};
|
|
}
|
|
queued_mutations_.push_back(*reserved);
|
|
return {admitted, tick, {}};
|
|
}
|
|
|
|
std::expected<persistence::PublishedCellularBatch, CellularPublishFailure>
|
|
CellularRegionHost::publish(const CellularBatch& batch,
|
|
std::span<const CellularActionRecord> actions,
|
|
std::span<const CellularMutationActionRecord> mutation_actions) {
|
|
const auto grant = authority_.grant();
|
|
if (!status_.ready || mutations_uncertain_ || !state_ || !committed_ || !grant ||
|
|
!authority_.may_begin_commit(LeaseClock::now()))
|
|
return std::unexpected{CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}};
|
|
if (population_mode()) {
|
|
if (!actions.empty() || !mutation_actions.empty())
|
|
return std::unexpected(
|
|
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}});
|
|
return publish_population(batch);
|
|
}
|
|
auto candidate = replay_cellular_batch(*state_, batch);
|
|
if (!candidate)
|
|
return std::unexpected{
|
|
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}}};
|
|
auto trace = encode_cellular_batch(batch);
|
|
auto action_block =
|
|
encode_cellular_action_block(actions, batch.header.begin_tick, batch.header.end_tick);
|
|
const auto commit =
|
|
trace && action_block ? commit_id(*committed_, *trace, *action_block) : std::nullopt;
|
|
if (!commit)
|
|
return std::unexpected{
|
|
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}}};
|
|
auto published = persistence::publish_cellular_batch(
|
|
*db_, *objects_, grant.value_or(persistence::LeaseGrant{}), *committed_, batch, **candidate,
|
|
*commit, config_.build, config_.lease.commit_timeout, {}, actions, mutation_actions);
|
|
if (!published) {
|
|
if (published.error().kind == persistence::PublicationError::unknown) {
|
|
pending_ = std::make_unique<Pending>(
|
|
Pending{published.error().prepared,
|
|
std::move(*candidate),
|
|
{actions.begin(), actions.end()},
|
|
births_in(batch),
|
|
{mutation_actions.begin(), mutation_actions.end()}});
|
|
status_.phase = HostPhase::resolving;
|
|
status_.ready = false;
|
|
++status_.unknown_outcomes;
|
|
db_.reset();
|
|
} else if (published.error().store.kind == LeaseStoreError::fenced) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
status_.phase = HostPhase::acquiring;
|
|
status_.ready = false;
|
|
++status_.fences;
|
|
} else {
|
|
mutations_uncertain_ = true; // A refusal may name a choice the queue lacks.
|
|
}
|
|
return std::unexpected{
|
|
CellularPublishFailure{CellularHostError::publication, published.error().kind,
|
|
published.error().store, published.error().object}};
|
|
}
|
|
note_endings(*state_, **candidate, published->committed.head.commit);
|
|
state_ = std::move(*candidate);
|
|
committed_ = published->committed;
|
|
flush_endings();
|
|
for (const auto& action : actions)
|
|
finalized_actions_.push_back({action, committed_->head.sequence});
|
|
const auto births = births_in(batch);
|
|
finalize_births(births);
|
|
std::vector<std::uint8_t> results;
|
|
results.reserve(published->mutation_results.size());
|
|
for (const auto result : published->mutation_results)
|
|
results.push_back(static_cast<std::uint8_t>(result));
|
|
finalize_mutations(mutation_actions, results);
|
|
++status_.commits;
|
|
status_.committed_sequence = committed_->head.sequence;
|
|
status_.committed_tick = committed_->head.end_tick;
|
|
status_.simulated_tick = committed_->head.end_tick;
|
|
refresh();
|
|
return *published;
|
|
}
|
|
|
|
void CellularRegionHost::queue_birth(const CellularTickIntents::Birth& birth) {
|
|
if (std::ranges::any_of(pending_birth_refusals_, [&](const auto& refusal) {
|
|
return refusal.birth.account == birth.account && refusal.birth.request == birth.request;
|
|
}))
|
|
return;
|
|
const auto found = std::ranges::find_if(queued_births_, [&](const auto& queued) {
|
|
return queued.account == birth.account && queued.request == birth.request;
|
|
});
|
|
if (found == queued_births_.end())
|
|
queued_births_.push_back(birth);
|
|
}
|
|
|
|
void CellularRegionHost::finalize_births(std::span<const CellularTickIntents::Birth> births) {
|
|
for (const auto& birth : births)
|
|
std::erase_if(queued_births_, [&](const auto& queued) {
|
|
return queued.account == birth.account && queued.request == birth.request &&
|
|
queued.generation == birth.generation && queued.identity == birth.identity;
|
|
});
|
|
}
|
|
|
|
void CellularRegionHost::refuse_births(std::span<const CellularBirthRefusal> refusals) {
|
|
for (const auto& refusal : refusals) {
|
|
if (refusal.reason != CellularLivesError::insufficient_funding &&
|
|
refusal.reason != CellularLivesError::no_placement &&
|
|
refusal.reason != CellularLivesError::life_exists)
|
|
continue;
|
|
if (std::ranges::find(queued_births_, refusal.birth) == queued_births_.end())
|
|
continue;
|
|
// Remove before the database call: an uncertain refusal must never race a spawn.
|
|
std::erase(queued_births_, refusal.birth);
|
|
std::erase_if(pending_birth_refusals_, [&](const auto& previous) {
|
|
return previous.birth.account == refusal.birth.account;
|
|
});
|
|
if (pending_birth_refusals_.size() == biology::max_cellular_tick_lives)
|
|
pending_birth_refusals_.erase(pending_birth_refusals_.begin());
|
|
pending_birth_refusals_.push_back(refusal);
|
|
static_cast<void>(query_birth(refusal.birth.account, refusal.birth.generation,
|
|
refusal.birth.request, false));
|
|
}
|
|
}
|
|
|
|
network::PrimitiveLifeOutcome CellularRegionHost::query_birth(const persistence::Uuid& account,
|
|
std::uint64_t generation,
|
|
std::uint64_t request, bool enqueue) {
|
|
using network::PrimitiveLifeResult;
|
|
const auto unknown = [&] {
|
|
return network::PrimitiveLifeOutcome{request, PrimitiveLifeResult::unknown, 0, 0};
|
|
};
|
|
if (!status_.ready || !authority_.may_begin_commit(LeaseClock::now()) ||
|
|
account == persistence::Uuid{} || generation == 0 || request == 0 || !database())
|
|
return unknown();
|
|
const auto grant = authority_.grant();
|
|
if (!grant)
|
|
return unknown();
|
|
auto queried = persistence::query_primitive_life(*db_, config_.key, *grant, account, generation,
|
|
request, config_.lease.commit_timeout);
|
|
if (!queried) {
|
|
if (queried.error().kind == persistence::LifeReservationError::unknown ||
|
|
queried.error().kind == persistence::LifeReservationError::store)
|
|
db_.reset();
|
|
if (queried.error().kind == persistence::LifeReservationError::fenced) {
|
|
authority_.fence();
|
|
status_.ready = false;
|
|
}
|
|
return unknown();
|
|
}
|
|
if (!queried->has_value())
|
|
return unknown();
|
|
const auto outcome = queried->value_or(persistence::PrimitiveLifeOutcome{});
|
|
if (outcome.published)
|
|
return {request, PrimitiveLifeResult::born, outcome.published->identity,
|
|
outcome.published->commit_sequence};
|
|
if (outcome.reservation.region != config_.key.region)
|
|
return unknown();
|
|
const auto result = life_result(outcome.reservation.refusal);
|
|
const auto refused = std::ranges::find_if(pending_birth_refusals_, [&](const auto& entry) {
|
|
return entry.birth.account == account && entry.birth.request == request;
|
|
});
|
|
if (refused != pending_birth_refusals_.end()) {
|
|
if (result != PrimitiveLifeResult::pending) {
|
|
pending_birth_refusals_.erase(refused);
|
|
} else {
|
|
const auto reason = refused->reason == CellularLivesError::insufficient_funding
|
|
? persistence::LifeRefusal::insufficient_funding
|
|
: refused->reason == CellularLivesError::no_placement
|
|
? persistence::LifeRefusal::no_placement
|
|
: persistence::LifeRefusal::existing_life;
|
|
const auto recorded =
|
|
persistence::reject_primitive_life(*db_, config_.key, *grant, account, generation,
|
|
request, reason, config_.lease.commit_timeout);
|
|
if (!recorded) {
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
return unknown();
|
|
}
|
|
pending_birth_refusals_.erase(refused);
|
|
return {request, life_result(*recorded), outcome.reservation.identity, 0};
|
|
}
|
|
}
|
|
if (result == PrimitiveLifeResult::pending && enqueue)
|
|
queue_birth({account, request, generation, outcome.reservation.identity});
|
|
return {request, result, outcome.reservation.identity, 0};
|
|
}
|
|
|
|
network::PrimitiveLifeOutcome
|
|
CellularRegionHost::create_primitive_life(const persistence::Uuid& account,
|
|
std::uint64_t generation, std::uint64_t request) {
|
|
using network::PrimitiveLifeResult;
|
|
if (population_mode() && !set_population_mode(false))
|
|
return {request, PrimitiveLifeResult::unknown, 0, 0};
|
|
const auto existing = query_birth(account, generation, request, true);
|
|
if (existing.result != PrimitiveLifeResult::unknown)
|
|
return existing;
|
|
if (!status_.ready || !authority_.may_begin_commit(LeaseClock::now()) || !database())
|
|
return {request, PrimitiveLifeResult::unknown, 0, 0};
|
|
const auto grant = authority_.grant();
|
|
if (!grant)
|
|
return {request, PrimitiveLifeResult::unknown, 0, 0};
|
|
auto reserved = persistence::reserve_primitive_life(
|
|
*db_, config_.key, *grant, account, generation, request, biology::founder_life_rules,
|
|
config_.lease.commit_timeout);
|
|
if (!reserved) {
|
|
if (reserved.error().kind == persistence::LifeReservationError::unknown ||
|
|
reserved.error().kind == persistence::LifeReservationError::store)
|
|
db_.reset();
|
|
if (reserved.error().kind == persistence::LifeReservationError::fenced) {
|
|
authority_.fence();
|
|
status_.ready = false;
|
|
}
|
|
return {request, PrimitiveLifeResult::unknown, 0, 0};
|
|
}
|
|
if (reserved->region != config_.key.region)
|
|
return {request, PrimitiveLifeResult::unknown, reserved->identity, 0};
|
|
const auto result = life_result(reserved->refusal);
|
|
if (result == PrimitiveLifeResult::pending)
|
|
queue_birth({account, request, generation, reserved->identity});
|
|
return {request, result, reserved->identity, 0};
|
|
}
|
|
|
|
network::PrimitiveLifeOutcome
|
|
CellularRegionHost::query_primitive_life(const persistence::Uuid& account, std::uint64_t generation,
|
|
std::uint64_t request) {
|
|
if (population_mode() && !set_population_mode(false))
|
|
return {request, network::PrimitiveLifeResult::unknown, 0, 0};
|
|
return query_birth(account, generation, request, true);
|
|
}
|
|
|
|
std::vector<FinalCellularAction> CellularRegionHost::take_finalized_actions() {
|
|
return std::exchange(finalized_actions_, {});
|
|
}
|
|
|
|
void CellularRegionHost::drain() {
|
|
status_.ready = false;
|
|
status_.phase = HostPhase::draining;
|
|
pending_.reset();
|
|
const auto grant = authority_.grant();
|
|
if (grant && authority_.may_originate(LeaseClock::now()) && database()) {
|
|
// Best effort during shutdown; the lease expires if release cannot be confirmed.
|
|
[[maybe_unused]] const auto released = persistence::release_region_lease(
|
|
*db_, config_.key, *grant, config_.lease.renew_timeout);
|
|
}
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
admitted_interval_.reset();
|
|
interval_request_.reset();
|
|
interval_reload_ = false;
|
|
status_.phase = HostPhase::stopped;
|
|
status_.epoch = 0;
|
|
status_.lease_remaining_ms = 0;
|
|
}
|
|
|
|
std::optional<persistence::LeaseGrant> CellularRegionHost::authority() const {
|
|
if (!status_.ready || !authority_.may_originate(LeaseClock::now()))
|
|
return std::nullopt;
|
|
return authority_.grant();
|
|
}
|
|
bool CellularRegionHost::population_mode() const noexcept {
|
|
return state_.population != nullptr;
|
|
}
|
|
|
|
std::expected<persistence::CommittedCheckpoint, CellularPublishFailure>
|
|
CellularRegionHost::accept_population_publication(
|
|
std::expected<persistence::CommittedCheckpoint, persistence::PopulationPublicationFailure>
|
|
result,
|
|
OwnedState candidate) {
|
|
if (!result) {
|
|
const auto& error = result.error();
|
|
if (error.kind == persistence::PublicationError::unknown) {
|
|
pending_ = std::make_unique<Pending>(
|
|
Pending{{}, std::move(candidate), {}, {}, {}, error.prepared});
|
|
status_.phase = HostPhase::resolving;
|
|
status_.ready = false;
|
|
++status_.unknown_outcomes;
|
|
db_.reset();
|
|
} else if (error.store.kind == LeaseStoreError::fenced) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
status_.phase = HostPhase::acquiring;
|
|
status_.ready = false;
|
|
++status_.fences;
|
|
} else if (error.kind == persistence::PublicationError::predecessor ||
|
|
error.kind == persistence::PublicationError::verify ||
|
|
error.kind == persistence::PublicationError::receipt) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
status_.phase = HostPhase::quarantined;
|
|
status_.ready = false;
|
|
} else {
|
|
mutations_uncertain_ = true;
|
|
}
|
|
return std::unexpected(CellularPublishFailure{CellularHostError::publication, error.kind,
|
|
error.store, error.object});
|
|
}
|
|
note_endings(*state_, *candidate, result->head.commit);
|
|
state_ = std::move(candidate);
|
|
committed_ = *result;
|
|
finish_interval();
|
|
++status_.commits;
|
|
status_.committed_sequence = committed_->head.sequence;
|
|
status_.committed_tick = status_.simulated_tick = committed_->head.end_tick;
|
|
refresh();
|
|
return std::move(*result);
|
|
}
|
|
|
|
std::expected<void, CellularPublishFailure> CellularRegionHost::set_population_mode(bool enabled) {
|
|
if (!status_.ready || pending_ || mutations_uncertain_ || admitted_interval_.has_value() ||
|
|
interval_request_.has_value() || interval_reload_ || !state_ || !committed_.has_value() ||
|
|
!authority_.may_begin_commit(LeaseClock::now()) || !database())
|
|
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
|
if (population_mode() == enabled)
|
|
return {};
|
|
if (enabled &&
|
|
(!config_.population_policy || !queued_mutations_.empty() || !pending_endings_.empty() ||
|
|
state_->rules_version() != cellular_lives_foraging_rules))
|
|
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
|
if (enabled && !population_interval_safe(*state_))
|
|
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
|
const auto policy = config_.population_policy.value_or(CellularTierPolicy{});
|
|
std::array<std::uint8_t, 10> marker{};
|
|
marker[0] = enabled ? 1 : 0;
|
|
marker[1] = policy.automatic ? 1 : 0;
|
|
for (unsigned i = 0; i < 4; ++i) {
|
|
marker[2 + i] = static_cast<std::uint8_t>(policy.materialize_radius_um >> (i * 8U));
|
|
marker[6 + i] = static_cast<std::uint8_t>(policy.collapse_radius_um >> (i * 8U));
|
|
}
|
|
const auto id = commit_id(*committed_, marker, {});
|
|
auto raw = CellularLives::restore(state_->snapshot(), state_->world(), state_->region(),
|
|
state_->frame(), state_->tick());
|
|
if (!id || !raw)
|
|
return std::unexpected(
|
|
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}});
|
|
OwnedState candidate;
|
|
if (enabled) {
|
|
auto population = CellularPopulation::create(std::move(*raw), policy);
|
|
if (!population)
|
|
return std::unexpected(
|
|
CellularPublishFailure{CellularHostError::configuration, {}, {}, {}});
|
|
candidate = std::move(*population);
|
|
} else
|
|
candidate = std::move(*raw);
|
|
const auto grant = authority_.grant();
|
|
if (!grant.has_value())
|
|
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
|
auto result =
|
|
enabled ? persistence::migrate_cellular_to_population(*db_, *objects_, *grant, *committed_,
|
|
*id, config_.build, policy,
|
|
config_.lease.commit_timeout)
|
|
: persistence::materialize_population_profile(*db_, *objects_, *grant, *committed_,
|
|
*id, config_.build,
|
|
config_.lease.commit_timeout);
|
|
auto accepted = accept_population_publication(std::move(result), std::move(candidate));
|
|
if (!accepted)
|
|
return std::unexpected(accepted.error());
|
|
return {};
|
|
}
|
|
|
|
std::expected<persistence::PublishedCellularBatch, CellularPublishFailure>
|
|
CellularRegionHost::publish_population(const CellularBatch& batch) {
|
|
const auto grant = authority_.grant();
|
|
if (!state_.population || !queued_mutations_.empty() || !committed_.has_value() ||
|
|
!grant.has_value() || !database())
|
|
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
|
auto execution = prepare_population_batch(*state_.population, batch);
|
|
if (!execution)
|
|
return std::unexpected(
|
|
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}});
|
|
const auto trace = encode_population_batch(execution->batch);
|
|
const auto id = trace ? commit_id(*committed_, *trace, {}) : std::nullopt;
|
|
if (!id)
|
|
return std::unexpected(
|
|
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}});
|
|
auto result = persistence::publish_population_batch(
|
|
*db_, *objects_, *grant, *committed_, execution->batch, *execution->state, *id,
|
|
config_.build, config_.lease.commit_timeout);
|
|
auto accepted =
|
|
accept_population_publication(std::move(result), OwnedState(std::move(execution->state)));
|
|
if (!accepted)
|
|
return std::unexpected(accepted.error());
|
|
return persistence::PublishedCellularBatch{std::move(*accepted), {}};
|
|
}
|
|
|
|
void CellularRegionHost::finish_interval() {
|
|
const auto& interval = admitted_interval_;
|
|
if (interval.has_value() && committed_.has_value() &&
|
|
committed_->head.end_tick == interval->end_tick)
|
|
admitted_interval_.reset();
|
|
}
|
|
|
|
void CellularRegionHost::reconcile_interval() {
|
|
if (!interval_reload_ && !interval_request_.has_value())
|
|
return;
|
|
const auto grant = authority_.grant();
|
|
if (!population_mode() || !committed_.has_value() || !grant.has_value() ||
|
|
!authority_.may_begin_commit(LeaseClock::now()) || !database())
|
|
return;
|
|
auto result = [&]() -> std::expected<std::optional<persistence::PopulationInterval>,
|
|
persistence::LeaseStoreFailure> {
|
|
const auto& request = interval_request_;
|
|
if (request.has_value()) {
|
|
auto admitted = persistence::admit_population_interval(
|
|
*db_, config_.key, *grant, request->source, request->end_tick,
|
|
config_.lease.commit_timeout);
|
|
if (!admitted)
|
|
return std::unexpected(admitted.error());
|
|
return std::optional{*admitted};
|
|
}
|
|
return persistence::load_population_interval(*db_, config_.key, *grant,
|
|
config_.lease.commit_timeout);
|
|
}();
|
|
if (!result) {
|
|
interval_failure_ = result.error();
|
|
if (result.error().kind == LeaseStoreError::unknown ||
|
|
result.error().kind == LeaseStoreError::store) {
|
|
if (result.error().kind == LeaseStoreError::unknown)
|
|
++status_.unknown_outcomes;
|
|
else
|
|
++status_.store_failures;
|
|
db_.reset();
|
|
return;
|
|
}
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
admitted_interval_.reset();
|
|
interval_request_.reset();
|
|
status_.ready = false;
|
|
status_.phase = result.error().kind == LeaseStoreError::fenced ? HostPhase::acquiring
|
|
: HostPhase::quarantined;
|
|
++status_.fences;
|
|
return;
|
|
}
|
|
auto interval = *result;
|
|
if (interval.has_value() &&
|
|
(interval->source_commit != committed_->head.commit ||
|
|
interval->source_manifest_sha256 != committed_->head.digest ||
|
|
interval->begin_tick != committed_->head.end_tick || interval->controller_version != 1)) {
|
|
authority_.fence();
|
|
status_.ready = false;
|
|
status_.phase = HostPhase::quarantined;
|
|
return;
|
|
}
|
|
admitted_interval_ = interval;
|
|
interval_request_.reset();
|
|
interval_failure_.reset();
|
|
interval_reload_ = false;
|
|
}
|
|
|
|
std::expected<std::optional<persistence::CommittedCheckpoint>, CellularPublishFailure>
|
|
CellularRegionHost::advance_unattended(std::uint64_t due_ticks) {
|
|
if (!status_.ready || !population_mode() || !committed_.has_value() || pending_ ||
|
|
!authority_.may_begin_commit(LeaseClock::now()) || !population_interval_safe(*state_))
|
|
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
|
if (!admitted_interval_.has_value()) {
|
|
const auto remaining = ~std::uint64_t{0} - committed_->head.end_tick;
|
|
const auto count = std::min({due_ticks, remaining, std::uint64_t{64}});
|
|
if (count == 0 || (due_ticks < 20 && count != remaining))
|
|
return std::optional<persistence::CommittedCheckpoint>{};
|
|
interval_request_ = IntervalRequest{committed_->head, committed_->head.end_tick + count};
|
|
reconcile_interval();
|
|
refresh();
|
|
if (!status_.ready || !admitted_interval_.has_value())
|
|
return std::unexpected(
|
|
CellularPublishFailure{CellularHostError::publication, {}, interval_failure_, {}});
|
|
}
|
|
const auto interval = admitted_interval_;
|
|
if (!interval.has_value())
|
|
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
|
const CellularBatchHeader header{config_.key.world,
|
|
config_.key.region,
|
|
config_.frame,
|
|
interval->source_commit,
|
|
interval->source_manifest_sha256,
|
|
interval->begin_tick,
|
|
interval->end_tick};
|
|
const auto inputs = unattended_population_inputs(*state_, header);
|
|
if (!inputs) {
|
|
authority_.fence();
|
|
status_.ready = false;
|
|
status_.phase = HostPhase::quarantined;
|
|
return std::unexpected(
|
|
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}});
|
|
}
|
|
auto published = publish(*inputs);
|
|
if (!published)
|
|
return std::unexpected(published.error());
|
|
return std::optional{std::move(published->committed)};
|
|
}
|
|
} // namespace evo::server
|