Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 1m59s
Foundation / checks (postgres) (push) Canceled after 0s
Foundation / checks (tsan) (push) Canceled after 0s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 6m2s
Foundation / checks (network) (push) Canceled after 3m53s
Foundation / checks (client) (push) Canceled after 6m8s
511 lines
22 KiB
C++
511 lines
22 KiB
C++
#include "evo/server/primitive_region_host.hpp"
|
|
|
|
#include "evo/persistence/sha256.hpp"
|
|
#include "evo/server/lifecycle/primitive_batch.hpp"
|
|
|
|
#include <algorithm>
|
|
|
|
namespace evo::server {
|
|
struct PrimitiveRegionHost::Pending {
|
|
persistence::Uuid account{};
|
|
std::uint64_t generation{}, request{};
|
|
persistence::LifeReservation reservation{};
|
|
PrimitiveBatch batch{};
|
|
std::unique_ptr<PrimitiveLives> candidate;
|
|
persistence::Uuid commit{};
|
|
std::optional<persistence::PrimitivePreparedCommit> prepared;
|
|
};
|
|
|
|
namespace {
|
|
using namespace std::chrono_literals;
|
|
using persistence::LeaseClock;
|
|
using persistence::LeaseStoreError;
|
|
|
|
bool valid(const PrimitiveHostConfig& c) {
|
|
return c.key.world != 0 && c.key.region != 0 && c.frame != 0 && persistence::valid(c.lease) &&
|
|
c.build != persistence::Sha256Digest{} &&
|
|
std::ranges::find(c.retained_builds, c.build) != c.retained_builds.end();
|
|
}
|
|
PrimitiveHostReply reply(PrimitiveHostReplyKind kind, persistence::LifeReservation reservation,
|
|
std::optional<persistence::PrimitiveLifeBinding> published = {}) {
|
|
return {kind, reservation, published};
|
|
}
|
|
} // namespace
|
|
|
|
PrimitiveRegionHost::PrimitiveRegionHost(
|
|
PrimitiveHostConfig config, PrimitiveConnect connect,
|
|
std::unique_ptr<persistence::FilesystemObjectStore> objects)
|
|
: config_{std::move(config)}, connect_{std::move(connect)}, objects_{std::move(objects)},
|
|
authority_{config_.lease} {}
|
|
|
|
PrimitiveRegionHost::~PrimitiveRegionHost() = default;
|
|
|
|
std::expected<std::unique_ptr<PrimitiveRegionHost>, PrimitiveHostError>
|
|
PrimitiveRegionHost::create(PrimitiveHostConfig config, PrimitiveConnect connect,
|
|
std::unique_ptr<persistence::FilesystemObjectStore> objects) {
|
|
if (!valid(config) || !connect || !objects)
|
|
return std::unexpected{PrimitiveHostError::configuration};
|
|
return std::unique_ptr<PrimitiveRegionHost>{
|
|
new PrimitiveRegionHost{std::move(config), std::move(connect), std::move(objects)}};
|
|
}
|
|
|
|
bool PrimitiveRegionHost::database() {
|
|
if (db_)
|
|
return true;
|
|
auto connection = connect_();
|
|
if (!connection) {
|
|
++status_.store_failures;
|
|
return false;
|
|
}
|
|
db_ = std::move(*connection);
|
|
return true;
|
|
}
|
|
|
|
void PrimitiveRegionHost::acquire() {
|
|
if (!database())
|
|
return;
|
|
if (!authority_.may_originate(LeaseClock::now())) {
|
|
const auto sent = LeaseClock::now();
|
|
auto lease = persistence::acquire_region_lease(
|
|
*db_, config_.key, config_.build, config_.lease.duration, config_.lease.renew_timeout);
|
|
if (!lease) {
|
|
if (lease.error().kind == LeaseStoreError::not_provisioned ||
|
|
lease.error().kind == LeaseStoreError::unsupported)
|
|
status_.phase = HostPhase::quarantined;
|
|
else if (lease.error().kind != LeaseStoreError::busy)
|
|
db_.reset();
|
|
return;
|
|
}
|
|
if (lease->frame != config_.frame ||
|
|
!authority_.acquired(*lease, sent, LeaseClock::now())) {
|
|
status_.phase = HostPhase::quarantined;
|
|
return;
|
|
}
|
|
}
|
|
auto recovered = persistence::recover_primitive_checkpoint(
|
|
*db_, *objects_, config_.key, config_.retained_builds, config_.lease.commit_timeout);
|
|
if (!recovered) {
|
|
if (recovered.error().kind == persistence::RecoveryError::store) {
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
return;
|
|
}
|
|
authority_.fence();
|
|
status_.phase = HostPhase::quarantined;
|
|
state_.reset();
|
|
committed_.reset();
|
|
return;
|
|
}
|
|
if (recovered->manifest.frame != config_.frame ||
|
|
recovered->manifest.tick_microseconds != 50000 ||
|
|
recovered->manifest.profile != persistence::CheckpointProfile::primitive_lives_v1) {
|
|
authority_.fence();
|
|
status_.phase = HostPhase::quarantined;
|
|
state_.reset();
|
|
committed_.reset();
|
|
return;
|
|
}
|
|
state_ = std::move(recovered->state);
|
|
committed_ = std::move(recovered->committed);
|
|
pending_.reset();
|
|
const auto grant = authority_.grant();
|
|
if (!grant) {
|
|
authority_.fence();
|
|
status_.phase = HostPhase::quarantined;
|
|
state_.reset();
|
|
committed_.reset();
|
|
return;
|
|
}
|
|
status_.epoch = grant->epoch;
|
|
status_.committed_tick = committed_->head.end_tick;
|
|
status_.simulated_tick = committed_->head.end_tick;
|
|
status_.committed_sequence = committed_->head.sequence;
|
|
status_.phase = HostPhase::running;
|
|
status_.ready = true;
|
|
}
|
|
|
|
void PrimitiveRegionHost::upkeep() {
|
|
const auto state = authority_.state(LeaseClock::now());
|
|
if (state == persistence::LeaseState::fenced) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
pending_.reset();
|
|
status_.ready = false;
|
|
status_.phase = HostPhase::acquiring;
|
|
return;
|
|
}
|
|
if (state != persistence::LeaseState::renew_due || !database())
|
|
return;
|
|
const auto grant = *authority_.grant();
|
|
const auto sent = LeaseClock::now();
|
|
auto renewed = persistence::renew_region_lease(*db_, config_.key, grant, config_.lease.duration,
|
|
config_.lease.renew_timeout);
|
|
if (renewed) {
|
|
if (!authority_.renewed(grant.epoch, *renewed, sent, LeaseClock::now())) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
pending_.reset();
|
|
status_.ready = false;
|
|
status_.phase = HostPhase::acquiring;
|
|
}
|
|
return;
|
|
}
|
|
if (renewed.error().kind == LeaseStoreError::unknown) {
|
|
authority_.renewal_unknown(config_.lease.duration, sent);
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
} else if (renewed.error().kind == LeaseStoreError::fenced) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
pending_.reset();
|
|
status_.ready = false;
|
|
status_.phase = HostPhase::acquiring;
|
|
} else {
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
}
|
|
}
|
|
|
|
void PrimitiveRegionHost::refresh() {
|
|
const auto& lease = authority_.grant();
|
|
status_.ready = status_.phase == HostPhase::running && state_ && committed_ && lease &&
|
|
authority_.may_originate(LeaseClock::now());
|
|
status_.epoch = lease ? lease->epoch : 0;
|
|
status_.lease_remaining_ms =
|
|
status_.ready
|
|
? static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
authority_.deadline() - LeaseClock::now())
|
|
.count())
|
|
: 0;
|
|
}
|
|
|
|
const HostStatus& PrimitiveRegionHost::step() {
|
|
if (status_.phase == HostPhase::quarantined)
|
|
return status_;
|
|
if (state_)
|
|
upkeep();
|
|
if (pending_ && authority_.may_begin_commit(LeaseClock::now())) {
|
|
status_.phase = HostPhase::running;
|
|
status_.ready = true;
|
|
const auto account = pending_->account;
|
|
const auto generation = pending_->generation;
|
|
const auto request = pending_->request;
|
|
(void)new_life(account, generation, request, 1);
|
|
}
|
|
if (!state_ && status_.phase != HostPhase::quarantined)
|
|
acquire();
|
|
refresh();
|
|
return status_;
|
|
}
|
|
|
|
void PrimitiveRegionHost::drain() {
|
|
status_.ready = false;
|
|
status_.phase = HostPhase::draining;
|
|
pending_.reset();
|
|
const auto grant = authority_.grant();
|
|
if (grant && authority_.may_originate(LeaseClock::now()) && database()) {
|
|
const auto released = persistence::release_region_lease(*db_, config_.key, *grant,
|
|
config_.lease.renew_timeout);
|
|
if (!released)
|
|
db_.reset();
|
|
}
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
status_.phase = HostPhase::stopped;
|
|
status_.epoch = 0;
|
|
status_.lease_remaining_ms = 0;
|
|
}
|
|
|
|
std::optional<persistence::LeaseGrant> PrimitiveRegionHost::authority() const {
|
|
if (!status_.ready || !authority_.may_originate(LeaseClock::now()))
|
|
return std::nullopt;
|
|
return authority_.grant();
|
|
}
|
|
|
|
PrimitiveHostReply PrimitiveRegionHost::query(const persistence::Uuid& account,
|
|
std::uint64_t generation, std::uint64_t request) {
|
|
if (!database())
|
|
return reply(PrimitiveHostReplyKind::unavailable, {});
|
|
const auto grant = authority_.grant();
|
|
if (!grant)
|
|
return reply(PrimitiveHostReplyKind::unavailable, {});
|
|
auto outcome = persistence::query_primitive_life(*db_, config_.key, *grant, account, generation,
|
|
request, config_.lease.commit_timeout);
|
|
if (!outcome) {
|
|
if (outcome.error().kind == persistence::LifeReservationError::unknown ||
|
|
outcome.error().kind == persistence::LifeReservationError::store)
|
|
db_.reset();
|
|
if (outcome.error().kind == persistence::LifeReservationError::fenced)
|
|
status_.ready = false;
|
|
return reply(PrimitiveHostReplyKind::unknown, {});
|
|
}
|
|
const auto& maybe_value = outcome.value();
|
|
if (!maybe_value.has_value())
|
|
return reply(PrimitiveHostReplyKind::unknown, {});
|
|
const auto& value = maybe_value.value();
|
|
if (value.published)
|
|
return reply(PrimitiveHostReplyKind::born, value.reservation, value.published);
|
|
if (value.reservation.region != config_.key.region)
|
|
return reply(PrimitiveHostReplyKind::routed, value.reservation);
|
|
if (value.reservation.refusal != persistence::LifeRefusal::none)
|
|
return reply(PrimitiveHostReplyKind::refused, value.reservation);
|
|
return reply(PrimitiveHostReplyKind::pending, value.reservation);
|
|
}
|
|
|
|
PrimitiveHostReply PrimitiveRegionHost::query_life(const persistence::Uuid& account,
|
|
std::uint64_t generation,
|
|
std::uint64_t request) {
|
|
if (!status_.ready || !authority_.may_begin_commit(LeaseClock::now()))
|
|
return reply(PrimitiveHostReplyKind::unavailable, {});
|
|
return query(account, generation, request);
|
|
}
|
|
|
|
PrimitiveHostReply PrimitiveRegionHost::new_life(const persistence::Uuid& account,
|
|
std::uint64_t generation, std::uint64_t request,
|
|
std::uint32_t rules) {
|
|
if (!status_.ready || !state_ || !committed_ || !authority_.may_begin_commit(LeaseClock::now()))
|
|
return reply(PrimitiveHostReplyKind::unavailable, {});
|
|
const auto grant = authority_.grant();
|
|
if (!grant)
|
|
return reply(PrimitiveHostReplyKind::unavailable, {});
|
|
if (account == persistence::Uuid{} || generation == 0 || request == 0 || rules == 0)
|
|
return reply(PrimitiveHostReplyKind::refused, {});
|
|
if (pending_) {
|
|
if (pending_->account != account || pending_->generation != generation ||
|
|
pending_->request != request)
|
|
return reply(PrimitiveHostReplyKind::pending, pending_->reservation);
|
|
if (!database()) {
|
|
status_.phase = HostPhase::resolving;
|
|
status_.ready = false;
|
|
return reply(PrimitiveHostReplyKind::unknown, pending_->reservation);
|
|
}
|
|
persistence::PrimitivePreparedCommit prepared;
|
|
const auto prepared_value = pending_->prepared;
|
|
if (prepared_value.has_value()) {
|
|
prepared = prepared_value.value();
|
|
} else {
|
|
// Re-entering the adapter with the same predecessor, trace and commit recreates
|
|
// the exact canonical bytes when the earlier failure happened before SQL.
|
|
auto published = persistence::publish_primitive_batch(
|
|
*db_, *objects_, *grant, *committed_, pending_->batch, *pending_->candidate,
|
|
pending_->commit, config_.build, config_.lease.commit_timeout);
|
|
if (published) {
|
|
state_ = std::move(pending_->candidate);
|
|
committed_ = published->committed;
|
|
pending_.reset();
|
|
++status_.commits;
|
|
status_.committed_sequence = committed_->head.sequence;
|
|
status_.committed_tick = committed_->head.end_tick;
|
|
status_.simulated_tick = committed_->head.end_tick;
|
|
return query(account, generation, request);
|
|
}
|
|
if (published.error().kind == persistence::PublicationError::unknown) {
|
|
pending_->prepared = published.error().prepared;
|
|
db_.reset();
|
|
++status_.unknown_outcomes;
|
|
status_.phase = HostPhase::resolving;
|
|
status_.ready = false;
|
|
return reply(PrimitiveHostReplyKind::unknown, pending_->reservation);
|
|
}
|
|
if (published.error().kind == persistence::PublicationError::rejected &&
|
|
published.error().store.kind == LeaseStoreError::fenced) {
|
|
authority_.fence();
|
|
status_.ready = false;
|
|
status_.phase = HostPhase::acquiring;
|
|
return reply(PrimitiveHostReplyKind::unknown, pending_->reservation);
|
|
}
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
const auto reservation = pending_->reservation;
|
|
pending_.reset();
|
|
authority_.fence();
|
|
status_.ready = false;
|
|
status_.phase = HostPhase::quarantined;
|
|
return reply(PrimitiveHostReplyKind::unavailable, reservation);
|
|
}
|
|
auto resolved =
|
|
persistence::retry_primitive_prepared(*db_, prepared, config_.lease.commit_timeout);
|
|
if (!resolved) {
|
|
if (resolved.error().kind == persistence::PublicationError::unknown) {
|
|
db_.reset();
|
|
++status_.store_failures;
|
|
status_.phase = HostPhase::resolving;
|
|
status_.ready = false;
|
|
return reply(PrimitiveHostReplyKind::unknown, pending_->reservation);
|
|
}
|
|
const auto reservation = pending_->reservation;
|
|
pending_.reset();
|
|
status_.phase = HostPhase::quarantined;
|
|
status_.ready = false;
|
|
authority_.fence();
|
|
return reply(PrimitiveHostReplyKind::unavailable, reservation);
|
|
}
|
|
state_ = std::move(pending_->candidate);
|
|
committed_ = std::move(*resolved);
|
|
pending_.reset();
|
|
++status_.commits;
|
|
status_.phase = HostPhase::running;
|
|
status_.committed_sequence = committed_->head.sequence;
|
|
status_.committed_tick = committed_->head.end_tick;
|
|
status_.simulated_tick = committed_->head.end_tick;
|
|
refresh();
|
|
return query(account, generation, request);
|
|
}
|
|
|
|
if (!database())
|
|
return reply(PrimitiveHostReplyKind::unavailable, {});
|
|
auto existing = query(account, generation, request);
|
|
if (existing.kind != PrimitiveHostReplyKind::unknown &&
|
|
existing.kind != PrimitiveHostReplyKind::pending)
|
|
return existing;
|
|
auto reserved =
|
|
persistence::reserve_primitive_life(*db_, config_.key, *grant, account, generation, request,
|
|
rules, config_.lease.commit_timeout);
|
|
if (!reserved) {
|
|
if (reserved.error().kind == persistence::LifeReservationError::unknown) {
|
|
db_.reset();
|
|
++status_.unknown_outcomes;
|
|
return reply(PrimitiveHostReplyKind::unknown, {});
|
|
}
|
|
if (reserved.error().kind == persistence::LifeReservationError::existing_life)
|
|
return query(account, generation, request);
|
|
if (reserved.error().kind == persistence::LifeReservationError::fenced) {
|
|
status_.ready = false;
|
|
authority_.fence();
|
|
}
|
|
return reply(PrimitiveHostReplyKind::unavailable, {});
|
|
}
|
|
if (reserved->region != config_.key.region)
|
|
return reply(PrimitiveHostReplyKind::routed, *reserved);
|
|
if (reserved->refusal != persistence::LifeRefusal::none)
|
|
return reply(PrimitiveHostReplyKind::refused, *reserved);
|
|
const auto refuse = [&](persistence::LifeRefusal reason) {
|
|
const auto recorded =
|
|
persistence::reject_primitive_life(*db_, config_.key, *grant, account, generation,
|
|
request, reason, config_.lease.commit_timeout);
|
|
if (!recorded) {
|
|
if (recorded.error().kind == persistence::LifeReservationError::unknown)
|
|
db_.reset();
|
|
return reply(PrimitiveHostReplyKind::unknown, *reserved);
|
|
}
|
|
auto final = *reserved;
|
|
final.refusal = *recorded;
|
|
return reply(PrimitiveHostReplyKind::refused, final);
|
|
};
|
|
if (state_->lives().size() >= max_primitive_lives ||
|
|
state_->lives().size() >= state_->config().sites.size())
|
|
return refuse(persistence::LifeRefusal::no_placement);
|
|
const auto endowed = biology::primitive_endowment();
|
|
if (!endowed || state_->balances().empty()) {
|
|
status_.phase = HostPhase::quarantined;
|
|
status_.ready = false;
|
|
authority_.fence();
|
|
return reply(PrimitiveHostReplyKind::unavailable, *reserved);
|
|
}
|
|
auto& source = state_->balances().front();
|
|
const auto enough = [&] {
|
|
if (source.energy.value < endowed->body.energy.value + endowed->reserve.energy.value +
|
|
endowed->construction.value)
|
|
return false;
|
|
for (std::size_t i = 0; i < source.matter.size(); ++i)
|
|
if (source.matter[i].value < endowed->body.matter[i].value)
|
|
return false;
|
|
return true;
|
|
};
|
|
if (!enough())
|
|
return refuse(persistence::LifeRefusal::insufficient_funding);
|
|
if (committed_->head.end_tick == UINT64_MAX || committed_->head.sequence == UINT64_MAX) {
|
|
status_.phase = HostPhase::quarantined;
|
|
status_.ready = false;
|
|
authority_.fence();
|
|
return reply(PrimitiveHostReplyKind::unavailable, *reserved);
|
|
}
|
|
const auto decoded = persistence::decode_manifest(committed_->manifest);
|
|
const auto predecessor_digest = persistence::sha256(committed_->manifest);
|
|
if (!decoded || !predecessor_digest) {
|
|
status_.phase = HostPhase::quarantined;
|
|
status_.ready = false;
|
|
authority_.fence();
|
|
return reply(PrimitiveHostReplyKind::unavailable, *reserved);
|
|
}
|
|
const auto end_tick = committed_->head.end_tick + 1;
|
|
PrimitiveBatch batch{{config_.key.world, config_.key.region, config_.frame,
|
|
committed_->head.commit, *predecessor_digest, committed_->head.end_tick,
|
|
end_tick},
|
|
{{{account, request}, reserved->identity, generation, {end_tick}}}};
|
|
auto candidate = replay_primitive_batch(*state_, batch);
|
|
if (!candidate) {
|
|
if (candidate.error() == PrimitiveBatchError::state) {
|
|
status_.phase = HostPhase::quarantined;
|
|
status_.ready = false;
|
|
authority_.fence();
|
|
return reply(PrimitiveHostReplyKind::unavailable, *reserved);
|
|
}
|
|
status_.phase = HostPhase::quarantined;
|
|
status_.ready = false;
|
|
authority_.fence();
|
|
return reply(PrimitiveHostReplyKind::unavailable, *reserved);
|
|
}
|
|
auto commit = grant->holder;
|
|
const auto sequence = committed_->head.sequence + 1;
|
|
for (unsigned i = 0; i < 8; ++i)
|
|
commit[8 + i] = static_cast<std::uint8_t>((sequence >> (i * 8U)) & 0xffU);
|
|
pending_ = std::make_unique<Pending>(Pending{account,
|
|
generation,
|
|
request,
|
|
*reserved,
|
|
std::move(batch),
|
|
std::move(*candidate),
|
|
commit,
|
|
{}});
|
|
auto published = persistence::publish_primitive_batch(
|
|
*db_, *objects_, *grant, *committed_, pending_->batch, *pending_->candidate,
|
|
pending_->commit, config_.build, config_.lease.commit_timeout);
|
|
if (!published) {
|
|
const auto& failure = published.error();
|
|
if (failure.kind == persistence::PublicationError::unknown) {
|
|
pending_->prepared = failure.prepared;
|
|
db_.reset();
|
|
++status_.unknown_outcomes;
|
|
status_.phase = HostPhase::resolving;
|
|
status_.ready = false;
|
|
return reply(PrimitiveHostReplyKind::unknown, *reserved);
|
|
}
|
|
if (failure.kind == persistence::PublicationError::rejected &&
|
|
failure.store.kind == LeaseStoreError::fenced) {
|
|
authority_.fence();
|
|
state_.reset();
|
|
committed_.reset();
|
|
pending_.reset();
|
|
status_.ready = false;
|
|
status_.phase = HostPhase::acquiring;
|
|
return reply(PrimitiveHostReplyKind::unknown, *reserved);
|
|
}
|
|
++status_.store_failures;
|
|
if (failure.kind == persistence::PublicationError::candidate ||
|
|
failure.kind == persistence::PublicationError::predecessor ||
|
|
failure.kind == persistence::PublicationError::receipt ||
|
|
failure.kind == persistence::PublicationError::rejected) {
|
|
pending_.reset();
|
|
authority_.fence();
|
|
status_.phase = HostPhase::quarantined;
|
|
status_.ready = false;
|
|
return reply(PrimitiveHostReplyKind::unavailable, *reserved);
|
|
}
|
|
return reply(PrimitiveHostReplyKind::pending, *reserved);
|
|
}
|
|
state_ = std::move(pending_->candidate);
|
|
committed_ = published->committed;
|
|
pending_.reset();
|
|
++status_.commits;
|
|
status_.committed_sequence = committed_->head.sequence;
|
|
status_.committed_tick = committed_->head.end_tick;
|
|
status_.simulated_tick = committed_->head.end_tick;
|
|
return query(account, generation, request);
|
|
}
|
|
} // namespace evo::server
|