Foundation / checks (deploy) (push) Successful in 6s
Foundation / checks (ecs) (push) Successful in 1m58s
Foundation / checks (tsan) (push) Successful in 8m55s
Foundation / checks (client) (push) Successful in 13m17s
Foundation / checks (network) (push) Successful in 15m14s
Foundation / checks (postgres) (push) Failing after 34m1s
Foundation / checks (checks) (push) Successful in 38m3s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
477 lines
19 KiB
C++
477 lines
19 KiB
C++
#include "evo/server/cellular_region_host.hpp"
|
|
|
|
#include "evo/biology/founder.hpp"
|
|
#include "evo/persistence/sha256.hpp"
|
|
|
|
#include <algorithm>
|
|
#include <array>
|
|
#include <chrono>
|
|
#include <utility>
|
|
|
|
namespace evo::server {
|
|
struct CellularRegionHost::Pending {
|
|
persistence::CellularPreparedCommit prepared;
|
|
std::unique_ptr<CellularLives> candidate;
|
|
std::vector<CellularActionRecord> actions;
|
|
std::vector<CellularTickIntents::Birth> births;
|
|
};
|
|
|
|
namespace {
|
|
using persistence::LeaseClock;
|
|
using persistence::LeaseStoreError;
|
|
using std::chrono::milliseconds;
|
|
|
|
bool valid(const CellularHostConfig& config) {
|
|
return config.key.world != 0 && config.key.region != 0 && config.frame != 0 &&
|
|
persistence::valid(config.lease) && config.build != persistence::Sha256Digest{} &&
|
|
std::ranges::find(config.retained_builds, config.build) != config.retained_builds.end();
|
|
}
|
|
|
|
network::PrimitiveLifeResult life_result(persistence::LifeRefusal refusal) {
|
|
switch (refusal) {
|
|
case persistence::LifeRefusal::none:
|
|
return network::PrimitiveLifeResult::pending;
|
|
case persistence::LifeRefusal::insufficient_funding:
|
|
return network::PrimitiveLifeResult::insufficient_funding;
|
|
case persistence::LifeRefusal::no_placement:
|
|
return network::PrimitiveLifeResult::no_placement;
|
|
case persistence::LifeRefusal::cancelled:
|
|
return network::PrimitiveLifeResult::cancelled;
|
|
case persistence::LifeRefusal::existing_life:
|
|
return network::PrimitiveLifeResult::existing_life;
|
|
}
|
|
return network::PrimitiveLifeResult::unknown;
|
|
}
|
|
|
|
std::vector<CellularTickIntents::Birth> births_in(const CellularBatch& batch) {
|
|
std::vector<CellularTickIntents::Birth> births;
|
|
for (const auto& tick : batch.ticks)
|
|
births.insert(births.end(), tick.births.begin(), tick.births.end());
|
|
return births;
|
|
}
|
|
|
|
std::optional<persistence::Uuid> commit_id(const persistence::CommittedCheckpoint& previous,
|
|
std::span<const std::uint8_t> trace,
|
|
std::span<const std::uint8_t> actions) {
|
|
std::vector<std::uint8_t> identity;
|
|
identity.reserve(previous.head.digest.size() + 16 + trace.size() + actions.size());
|
|
identity.insert(identity.end(), previous.head.digest.begin(), previous.head.digest.end());
|
|
const auto append_length = [&](std::size_t size) {
|
|
const auto length = static_cast<std::uint64_t>(size);
|
|
for (unsigned i = 0; i < 8; ++i)
|
|
identity.push_back(static_cast<std::uint8_t>(length >> (i * 8U)));
|
|
};
|
|
append_length(trace.size());
|
|
identity.insert(identity.end(), trace.begin(), trace.end());
|
|
append_length(actions.size());
|
|
identity.insert(identity.end(), actions.begin(), actions.end());
|
|
const auto digest = persistence::sha256(identity);
|
|
if (!digest)
|
|
return std::nullopt;
|
|
persistence::Uuid out{};
|
|
std::ranges::copy_n(digest->begin(), out.size(), out.begin());
|
|
if (out == persistence::Uuid{})
|
|
out.back() = 1;
|
|
return out;
|
|
}
|
|
} // namespace
|
|
|
|
CellularRegionHost::CellularRegionHost(CellularHostConfig config, CellularConnect connect,
|
|
std::unique_ptr<persistence::FilesystemObjectStore> objects)
|
|
: config_{std::move(config)}, connect_{std::move(connect)}, objects_{std::move(objects)},
|
|
authority_{config_.lease} {}
|
|
|
|
CellularRegionHost::~CellularRegionHost() = default;
|
|
|
|
std::expected<std::unique_ptr<CellularRegionHost>, CellularHostError>
|
|
CellularRegionHost::create(CellularHostConfig config, CellularConnect connect,
|
|
std::unique_ptr<persistence::FilesystemObjectStore> objects) {
|
|
if (!valid(config) || !connect || !objects)
|
|
return std::unexpected{CellularHostError::configuration};
|
|
return std::unique_ptr<CellularRegionHost>{
|
|
new CellularRegionHost{std::move(config), std::move(connect), std::move(objects)}};
|
|
}
|
|
|
|
bool CellularRegionHost::database() {
|
|
if (db_)
|
|
return true;
|
|
auto connection = connect_();
|
|
if (!connection) {
|
|
++status_.store_failures;
|
|
return false;
|
|
}
|
|
db_ = std::move(*connection);
|
|
return true;
|
|
}
|
|
|
|
void CellularRegionHost::acquire() {
|
|
if (!database())
|
|
return;
|
|
const auto sent = LeaseClock::now();
|
|
auto lease = persistence::acquire_region_lease(
|
|
*db_, config_.key, config_.build, config_.lease.duration, config_.lease.renew_timeout);
|
|
if (!lease) {
|
|
if (lease.error().kind == LeaseStoreError::not_provisioned ||
|
|
lease.error().kind == LeaseStoreError::unsupported) {
|
|
status_.phase = HostPhase::quarantined;
|
|
} else if (lease.error().kind != LeaseStoreError::busy) {
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
}
|
|
return;
|
|
}
|
|
if (lease->frame != config_.frame || !authority_.acquired(*lease, sent, LeaseClock::now())) {
|
|
status_.phase = HostPhase::quarantined;
|
|
return;
|
|
}
|
|
auto recovered = persistence::recover_cellular_checkpoint(*db_, *objects_, config_.key, *lease,
|
|
config_.retained_builds,
|
|
config_.lease.commit_timeout);
|
|
if (!recovered) {
|
|
if (recovered.error().kind == persistence::RecoveryError::store) {
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
return;
|
|
}
|
|
authority_.fence();
|
|
status_.phase = HostPhase::quarantined;
|
|
return;
|
|
}
|
|
state_ = std::move(recovered->state);
|
|
committed_ = std::move(recovered->committed);
|
|
status_.phase = HostPhase::running;
|
|
status_.simulated_tick = committed_->head.end_tick;
|
|
status_.committed_tick = committed_->head.end_tick;
|
|
status_.committed_sequence = committed_->head.sequence;
|
|
}
|
|
|
|
void CellularRegionHost::upkeep() {
|
|
if (authority_.state(LeaseClock::now()) == persistence::LeaseState::fenced) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
pending_.reset();
|
|
status_.ready = false;
|
|
status_.phase = HostPhase::acquiring;
|
|
++status_.fences;
|
|
return;
|
|
}
|
|
if (authority_.state(LeaseClock::now()) != persistence::LeaseState::renew_due || !database())
|
|
return;
|
|
const auto grant = *authority_.grant();
|
|
const auto sent = LeaseClock::now();
|
|
auto renewed = persistence::renew_region_lease(*db_, config_.key, grant, config_.lease.duration,
|
|
config_.lease.renew_timeout);
|
|
if (renewed) {
|
|
if (!authority_.renewed(grant.epoch, *renewed, sent, LeaseClock::now())) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
pending_.reset();
|
|
status_.ready = false;
|
|
status_.phase = HostPhase::acquiring;
|
|
}
|
|
return;
|
|
}
|
|
if (renewed.error().kind == LeaseStoreError::unknown)
|
|
authority_.renewal_unknown(config_.lease.duration, sent);
|
|
else if (renewed.error().kind == LeaseStoreError::fenced) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
pending_.reset();
|
|
status_.ready = false;
|
|
status_.phase = HostPhase::acquiring;
|
|
++status_.fences;
|
|
}
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
}
|
|
|
|
void CellularRegionHost::resolve() {
|
|
if (!pending_ || !database())
|
|
return;
|
|
if (!authority_.may_begin_commit(LeaseClock::now())) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
pending_.reset();
|
|
status_.phase = HostPhase::acquiring;
|
|
status_.ready = false;
|
|
++status_.fences;
|
|
return;
|
|
}
|
|
const auto resolved = persistence::resolve_cellular_prepared(*db_, pending_->prepared,
|
|
config_.lease.commit_timeout);
|
|
if (!resolved) {
|
|
if (resolved.error().kind == LeaseStoreError::fenced) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
pending_.reset();
|
|
status_.phase = HostPhase::acquiring;
|
|
status_.ready = false;
|
|
++status_.fences;
|
|
} else {
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
}
|
|
return;
|
|
}
|
|
if (resolved->status == persistence::Resolution::conflict) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
pending_.reset();
|
|
status_.phase = HostPhase::quarantined;
|
|
status_.ready = false;
|
|
return;
|
|
}
|
|
if (resolved->status == persistence::Resolution::absent &&
|
|
!authority_.may_begin_commit(LeaseClock::now()))
|
|
return;
|
|
auto retried = persistence::retry_cellular_prepared(*db_, pending_->prepared,
|
|
config_.lease.commit_timeout);
|
|
if (!retried) {
|
|
if (retried.error().kind == persistence::PublicationError::unknown) {
|
|
db_.reset();
|
|
++status_.unknown_outcomes;
|
|
status_.phase = HostPhase::resolving;
|
|
status_.ready = false;
|
|
} else if (retried.error().store.kind == LeaseStoreError::fenced) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
pending_.reset();
|
|
status_.phase = HostPhase::acquiring;
|
|
status_.ready = false;
|
|
++status_.fences;
|
|
} else {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
pending_.reset();
|
|
status_.phase = HostPhase::quarantined;
|
|
status_.ready = false;
|
|
}
|
|
return;
|
|
}
|
|
state_ = std::move(pending_->candidate);
|
|
committed_ = std::move(*retried);
|
|
for (const auto& action : pending_->actions)
|
|
finalized_actions_.push_back({action, committed_->head.sequence});
|
|
finalize_births(pending_->births);
|
|
pending_.reset();
|
|
++status_.commits;
|
|
status_.committed_sequence = committed_->head.sequence;
|
|
status_.committed_tick = committed_->head.end_tick;
|
|
status_.simulated_tick = committed_->head.end_tick;
|
|
status_.phase = HostPhase::running;
|
|
}
|
|
|
|
void CellularRegionHost::refresh() {
|
|
const auto& grant = authority_.grant();
|
|
status_.ready = status_.phase == HostPhase::running && state_ && committed_ && grant &&
|
|
authority_.may_originate(LeaseClock::now());
|
|
status_.epoch = grant ? grant->epoch : 0;
|
|
status_.lease_remaining_ms =
|
|
status_.ready ? static_cast<std::uint64_t>(std::chrono::duration_cast<milliseconds>(
|
|
authority_.deadline() - LeaseClock::now())
|
|
.count())
|
|
: 0;
|
|
}
|
|
|
|
const HostStatus& CellularRegionHost::step() {
|
|
if (status_.phase == HostPhase::quarantined || status_.phase == HostPhase::stopped)
|
|
return status_;
|
|
if (authority_.grant())
|
|
upkeep();
|
|
if (pending_)
|
|
resolve();
|
|
else if (!state_)
|
|
acquire();
|
|
refresh();
|
|
return status_;
|
|
}
|
|
|
|
std::expected<persistence::PublishedCellularBatch, CellularPublishFailure>
|
|
CellularRegionHost::publish(const CellularBatch& batch,
|
|
std::span<const CellularActionRecord> actions) {
|
|
const auto grant = authority_.grant();
|
|
if (!status_.ready || !state_ || !committed_ || !grant ||
|
|
!authority_.may_begin_commit(LeaseClock::now()))
|
|
return std::unexpected{CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}};
|
|
auto candidate = replay_cellular_batch(*state_, batch);
|
|
if (!candidate)
|
|
return std::unexpected{
|
|
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}}};
|
|
auto trace = encode_cellular_batch(batch);
|
|
auto action_block =
|
|
encode_cellular_action_block(actions, batch.header.begin_tick, batch.header.end_tick);
|
|
const auto commit =
|
|
trace && action_block ? commit_id(*committed_, *trace, *action_block) : std::nullopt;
|
|
if (!commit)
|
|
return std::unexpected{
|
|
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}}};
|
|
auto published = persistence::publish_cellular_batch(
|
|
*db_, *objects_, grant.value_or(persistence::LeaseGrant{}), *committed_, batch, **candidate,
|
|
*commit, config_.build, config_.lease.commit_timeout, {}, actions);
|
|
if (!published) {
|
|
if (published.error().kind == persistence::PublicationError::unknown) {
|
|
pending_ = std::make_unique<Pending>(Pending{published.error().prepared,
|
|
std::move(*candidate),
|
|
{actions.begin(), actions.end()},
|
|
births_in(batch)});
|
|
status_.phase = HostPhase::resolving;
|
|
status_.ready = false;
|
|
++status_.unknown_outcomes;
|
|
db_.reset();
|
|
} else if (published.error().store.kind == LeaseStoreError::fenced) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
status_.phase = HostPhase::acquiring;
|
|
status_.ready = false;
|
|
++status_.fences;
|
|
}
|
|
return std::unexpected{
|
|
CellularPublishFailure{CellularHostError::publication, published.error().kind,
|
|
published.error().store, published.error().object}};
|
|
}
|
|
state_ = std::move(*candidate);
|
|
committed_ = published->committed;
|
|
for (const auto& action : actions)
|
|
finalized_actions_.push_back({action, committed_->head.sequence});
|
|
const auto births = births_in(batch);
|
|
finalize_births(births);
|
|
++status_.commits;
|
|
status_.committed_sequence = committed_->head.sequence;
|
|
status_.committed_tick = committed_->head.end_tick;
|
|
status_.simulated_tick = committed_->head.end_tick;
|
|
refresh();
|
|
return *published;
|
|
}
|
|
|
|
void CellularRegionHost::queue_birth(const CellularTickIntents::Birth& birth) {
|
|
const auto found = std::ranges::find_if(queued_births_, [&](const auto& queued) {
|
|
return queued.account == birth.account && queued.request == birth.request;
|
|
});
|
|
if (found == queued_births_.end())
|
|
queued_births_.push_back(birth);
|
|
}
|
|
|
|
void CellularRegionHost::finalize_births(std::span<const CellularTickIntents::Birth> births) {
|
|
for (const auto& birth : births)
|
|
std::erase_if(queued_births_, [&](const auto& queued) {
|
|
return queued.account == birth.account && queued.request == birth.request &&
|
|
queued.generation == birth.generation && queued.identity == birth.identity;
|
|
});
|
|
}
|
|
|
|
network::PrimitiveLifeOutcome CellularRegionHost::query_birth(const persistence::Uuid& account,
|
|
std::uint64_t generation,
|
|
std::uint64_t request, bool enqueue) {
|
|
using network::PrimitiveLifeResult;
|
|
const auto unknown = [&] {
|
|
return network::PrimitiveLifeOutcome{request, PrimitiveLifeResult::unknown, 0, 0};
|
|
};
|
|
if (!status_.ready || !authority_.may_begin_commit(LeaseClock::now()) ||
|
|
account == persistence::Uuid{} || generation == 0 || request == 0 || !database())
|
|
return unknown();
|
|
const auto grant = authority_.grant();
|
|
if (!grant)
|
|
return unknown();
|
|
auto queried = persistence::query_primitive_life(*db_, config_.key, *grant, account, generation,
|
|
request, config_.lease.commit_timeout);
|
|
if (!queried) {
|
|
if (queried.error().kind == persistence::LifeReservationError::unknown ||
|
|
queried.error().kind == persistence::LifeReservationError::store)
|
|
db_.reset();
|
|
if (queried.error().kind == persistence::LifeReservationError::fenced) {
|
|
authority_.fence();
|
|
status_.ready = false;
|
|
}
|
|
return unknown();
|
|
}
|
|
if (!queried->has_value())
|
|
return unknown();
|
|
const auto outcome = queried->value_or(persistence::PrimitiveLifeOutcome{});
|
|
if (outcome.published)
|
|
return {request, PrimitiveLifeResult::born, outcome.published->identity,
|
|
outcome.published->commit_sequence};
|
|
if (outcome.reservation.region != config_.key.region)
|
|
return unknown();
|
|
const auto result = life_result(outcome.reservation.refusal);
|
|
if (result == PrimitiveLifeResult::pending && enqueue)
|
|
queue_birth({account, request, generation, outcome.reservation.identity});
|
|
return {request, result, outcome.reservation.identity, 0};
|
|
}
|
|
|
|
network::PrimitiveLifeOutcome
|
|
CellularRegionHost::create_primitive_life(const persistence::Uuid& account,
|
|
std::uint64_t generation, std::uint64_t request) {
|
|
using network::PrimitiveLifeResult;
|
|
const auto existing = query_birth(account, generation, request, true);
|
|
if (existing.result != PrimitiveLifeResult::unknown)
|
|
return existing;
|
|
if (!status_.ready || !authority_.may_begin_commit(LeaseClock::now()) || !database())
|
|
return {request, PrimitiveLifeResult::unknown, 0, 0};
|
|
const auto grant = authority_.grant();
|
|
if (!grant)
|
|
return {request, PrimitiveLifeResult::unknown, 0, 0};
|
|
auto reserved = persistence::reserve_primitive_life(
|
|
*db_, config_.key, *grant, account, generation, request, biology::founder_life_rules,
|
|
config_.lease.commit_timeout);
|
|
if (!reserved) {
|
|
if (reserved.error().kind == persistence::LifeReservationError::unknown ||
|
|
reserved.error().kind == persistence::LifeReservationError::store)
|
|
db_.reset();
|
|
if (reserved.error().kind == persistence::LifeReservationError::fenced) {
|
|
authority_.fence();
|
|
status_.ready = false;
|
|
}
|
|
return {request, PrimitiveLifeResult::unknown, 0, 0};
|
|
}
|
|
if (reserved->region != config_.key.region)
|
|
return {request, PrimitiveLifeResult::unknown, reserved->identity, 0};
|
|
const auto result = life_result(reserved->refusal);
|
|
if (result == PrimitiveLifeResult::pending)
|
|
queue_birth({account, request, generation, reserved->identity});
|
|
return {request, result, reserved->identity, 0};
|
|
}
|
|
|
|
network::PrimitiveLifeOutcome
|
|
CellularRegionHost::query_primitive_life(const persistence::Uuid& account, std::uint64_t generation,
|
|
std::uint64_t request) {
|
|
return query_birth(account, generation, request, true);
|
|
}
|
|
|
|
std::vector<FinalCellularAction> CellularRegionHost::take_finalized_actions() {
|
|
return std::exchange(finalized_actions_, {});
|
|
}
|
|
|
|
void CellularRegionHost::drain() {
|
|
status_.ready = false;
|
|
status_.phase = HostPhase::draining;
|
|
pending_.reset();
|
|
const auto grant = authority_.grant();
|
|
if (grant && authority_.may_originate(LeaseClock::now()) && database()) {
|
|
// Best effort during shutdown; the lease expires if release cannot be confirmed.
|
|
[[maybe_unused]] const auto released = persistence::release_region_lease(
|
|
*db_, config_.key, *grant, config_.lease.renew_timeout);
|
|
}
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
status_.phase = HostPhase::stopped;
|
|
status_.epoch = 0;
|
|
status_.lease_remaining_ms = 0;
|
|
}
|
|
|
|
std::optional<persistence::LeaseGrant> CellularRegionHost::authority() const {
|
|
if (!status_.ready || !authority_.may_originate(LeaseClock::now()))
|
|
return std::nullopt;
|
|
return authority_.grant();
|
|
}
|
|
} // namespace evo::server
|