Add primitive profile region host
Foundation / checks (deploy) (push) Successful in 6s
Foundation / checks (ecs) (push) Successful in 1m57s
Foundation / checks (checks) (push) Failing after 3m43s
Foundation / checks (tsan) (push) Canceled after 0s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (postgres) (push) Canceled after 3m2s
Foundation / checks (network) (push) Canceled after 4m39s
Foundation / checks (client) (push) Canceled after 6m48s
Foundation / checks (deploy) (push) Successful in 6s
Foundation / checks (ecs) (push) Successful in 1m57s
Foundation / checks (checks) (push) Failing after 3m43s
Foundation / checks (tsan) (push) Canceled after 0s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (postgres) (push) Canceled after 3m2s
Foundation / checks (network) (push) Canceled after 4m39s
Foundation / checks (client) (push) Canceled after 6m48s
This commit is contained in:
@@ -0,0 +1,482 @@
|
||||
#include "evo/server/primitive_region_host.hpp"
|
||||
|
||||
#include "evo/persistence/sha256.hpp"
|
||||
#include "evo/server/lifecycle/primitive_batch.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
namespace evo::server {
|
||||
struct PrimitiveRegionHost::Pending {
|
||||
persistence::Uuid account{};
|
||||
std::uint64_t generation{}, request{};
|
||||
persistence::LifeReservation reservation{};
|
||||
PrimitiveBatch batch{};
|
||||
std::unique_ptr<PrimitiveLives> candidate;
|
||||
persistence::Uuid commit{};
|
||||
std::optional<persistence::PrimitivePreparedCommit> prepared;
|
||||
};
|
||||
|
||||
namespace {
|
||||
using namespace std::chrono_literals;
|
||||
using persistence::LeaseClock;
|
||||
using persistence::LeaseStoreError;
|
||||
|
||||
bool valid(const PrimitiveHostConfig& c) {
|
||||
return c.key.world != 0 && c.key.region != 0 && c.frame != 0 &&
|
||||
persistence::valid(c.lease) && c.build != persistence::Sha256Digest{} &&
|
||||
std::ranges::find(c.retained_builds, c.build) != c.retained_builds.end();
|
||||
}
|
||||
PrimitiveHostReply reply(PrimitiveHostReplyKind kind, persistence::LifeReservation reservation,
|
||||
std::optional<persistence::PrimitiveLifeBinding> published = {}) {
|
||||
return {kind, reservation, std::move(published)};
|
||||
}
|
||||
} // namespace
|
||||
|
||||
PrimitiveRegionHost::PrimitiveRegionHost(PrimitiveHostConfig config, PrimitiveConnect connect,
|
||||
std::unique_ptr<persistence::FilesystemObjectStore> objects)
|
||||
: config_{std::move(config)}, connect_{std::move(connect)}, objects_{std::move(objects)},
|
||||
authority_{config_.lease} {}
|
||||
|
||||
PrimitiveRegionHost::~PrimitiveRegionHost() = default;
|
||||
|
||||
std::expected<std::unique_ptr<PrimitiveRegionHost>, PrimitiveHostError>
|
||||
PrimitiveRegionHost::create(PrimitiveHostConfig config, PrimitiveConnect connect,
|
||||
std::unique_ptr<persistence::FilesystemObjectStore> objects) {
|
||||
if (!valid(config) || !connect || !objects)
|
||||
return std::unexpected{PrimitiveHostError::configuration};
|
||||
return std::unique_ptr<PrimitiveRegionHost>{new PrimitiveRegionHost{
|
||||
std::move(config), std::move(connect), std::move(objects)}};
|
||||
}
|
||||
|
||||
bool PrimitiveRegionHost::database() {
|
||||
if (db_)
|
||||
return true;
|
||||
auto connection = connect_();
|
||||
if (!connection) {
|
||||
++status_.store_failures;
|
||||
return false;
|
||||
}
|
||||
db_ = std::move(*connection);
|
||||
return true;
|
||||
}
|
||||
|
||||
void PrimitiveRegionHost::acquire() {
|
||||
if (!database())
|
||||
return;
|
||||
if (!authority_.may_originate(LeaseClock::now())) {
|
||||
const auto sent = LeaseClock::now();
|
||||
auto lease = persistence::acquire_region_lease(
|
||||
*db_, config_.key, config_.build, config_.lease.duration, config_.lease.renew_timeout);
|
||||
if (!lease) {
|
||||
if (lease.error().kind == LeaseStoreError::not_provisioned ||
|
||||
lease.error().kind == LeaseStoreError::unsupported)
|
||||
status_.phase = HostPhase::quarantined;
|
||||
else if (lease.error().kind != LeaseStoreError::busy)
|
||||
db_.reset();
|
||||
return;
|
||||
}
|
||||
if (lease->frame != config_.frame ||
|
||||
!authority_.acquired(*lease, sent, LeaseClock::now())) {
|
||||
status_.phase = HostPhase::quarantined;
|
||||
return;
|
||||
}
|
||||
}
|
||||
auto recovered = persistence::recover_primitive_checkpoint(
|
||||
*db_, *objects_, config_.key, config_.retained_builds, config_.lease.commit_timeout);
|
||||
if (!recovered) {
|
||||
if (recovered.error().kind == persistence::RecoveryError::store) {
|
||||
db_.reset();
|
||||
++status_.store_failures;
|
||||
return;
|
||||
}
|
||||
authority_.fence();
|
||||
status_.phase = HostPhase::quarantined;
|
||||
state_.reset();
|
||||
committed_.reset();
|
||||
return;
|
||||
}
|
||||
if (recovered->manifest.frame != config_.frame ||
|
||||
recovered->manifest.tick_microseconds != 50000 ||
|
||||
recovered->manifest.profile != persistence::CheckpointProfile::primitive_lives_v1) {
|
||||
authority_.fence();
|
||||
status_.phase = HostPhase::quarantined;
|
||||
state_.reset();
|
||||
committed_.reset();
|
||||
return;
|
||||
}
|
||||
state_ = std::move(recovered->state);
|
||||
committed_ = std::move(recovered->committed);
|
||||
pending_.reset();
|
||||
status_.epoch = authority_.grant()->epoch;
|
||||
status_.committed_tick = committed_->head.end_tick;
|
||||
status_.simulated_tick = committed_->head.end_tick;
|
||||
status_.committed_sequence = committed_->head.sequence;
|
||||
status_.phase = HostPhase::running;
|
||||
status_.ready = true;
|
||||
}
|
||||
|
||||
void PrimitiveRegionHost::upkeep() {
|
||||
const auto state = authority_.state(LeaseClock::now());
|
||||
if (state == persistence::LeaseState::fenced) {
|
||||
authority_.fence();
|
||||
state_.reset();
|
||||
committed_.reset();
|
||||
pending_.reset();
|
||||
status_.ready = false;
|
||||
status_.phase = HostPhase::acquiring;
|
||||
return;
|
||||
}
|
||||
if (state != persistence::LeaseState::renew_due || !database())
|
||||
return;
|
||||
const auto grant = *authority_.grant();
|
||||
const auto sent = LeaseClock::now();
|
||||
auto renewed = persistence::renew_region_lease(*db_, config_.key, grant, config_.lease.duration,
|
||||
config_.lease.renew_timeout);
|
||||
if (renewed) {
|
||||
if (!authority_.renewed(grant.epoch, *renewed, sent, LeaseClock::now())) {
|
||||
authority_.fence();
|
||||
state_.reset();
|
||||
committed_.reset();
|
||||
pending_.reset();
|
||||
status_.ready = false;
|
||||
status_.phase = HostPhase::acquiring;
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (renewed.error().kind == LeaseStoreError::unknown) {
|
||||
authority_.renewal_unknown(config_.lease.duration, sent);
|
||||
db_.reset();
|
||||
++status_.store_failures;
|
||||
} else if (renewed.error().kind == LeaseStoreError::fenced) {
|
||||
authority_.fence();
|
||||
state_.reset();
|
||||
committed_.reset();
|
||||
pending_.reset();
|
||||
status_.ready = false;
|
||||
status_.phase = HostPhase::acquiring;
|
||||
} else {
|
||||
db_.reset();
|
||||
++status_.store_failures;
|
||||
}
|
||||
}
|
||||
|
||||
void PrimitiveRegionHost::refresh() {
|
||||
const auto& lease = authority_.grant();
|
||||
status_.ready = status_.phase == HostPhase::running && state_ && committed_ && lease &&
|
||||
authority_.may_originate(LeaseClock::now());
|
||||
status_.epoch = lease ? lease->epoch : 0;
|
||||
status_.lease_remaining_ms =
|
||||
status_.ready ? static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
authority_.deadline() - LeaseClock::now()).count())
|
||||
: 0;
|
||||
}
|
||||
|
||||
const HostStatus& PrimitiveRegionHost::step() {
|
||||
if (status_.phase == HostPhase::quarantined)
|
||||
return status_;
|
||||
if (state_)
|
||||
upkeep();
|
||||
if (pending_ && authority_.may_begin_commit(LeaseClock::now())) {
|
||||
status_.phase = HostPhase::running;
|
||||
status_.ready = true;
|
||||
const auto account = pending_->account;
|
||||
const auto generation = pending_->generation;
|
||||
const auto request = pending_->request;
|
||||
(void)new_life(account, generation, request, 1);
|
||||
}
|
||||
if (!state_ && status_.phase != HostPhase::quarantined)
|
||||
acquire();
|
||||
refresh();
|
||||
return status_;
|
||||
}
|
||||
|
||||
void PrimitiveRegionHost::drain() {
|
||||
status_.ready = false;
|
||||
status_.phase = HostPhase::draining;
|
||||
pending_.reset();
|
||||
const auto grant = authority_.grant();
|
||||
if (grant && authority_.may_originate(LeaseClock::now()) && database()) {
|
||||
const auto released = persistence::release_region_lease(
|
||||
*db_, config_.key, *grant, config_.lease.renew_timeout);
|
||||
if (!released)
|
||||
db_.reset();
|
||||
}
|
||||
authority_.fence();
|
||||
state_.reset();
|
||||
committed_.reset();
|
||||
status_.phase = HostPhase::stopped;
|
||||
status_.epoch = 0;
|
||||
status_.lease_remaining_ms = 0;
|
||||
}
|
||||
|
||||
std::optional<persistence::LeaseGrant> PrimitiveRegionHost::authority() const {
|
||||
if (!status_.ready || !authority_.may_originate(LeaseClock::now()))
|
||||
return std::nullopt;
|
||||
return authority_.grant();
|
||||
}
|
||||
|
||||
PrimitiveHostReply PrimitiveRegionHost::query(const persistence::Uuid& account,
|
||||
std::uint64_t generation,
|
||||
std::uint64_t request) {
|
||||
if (!database() || !authority_.grant())
|
||||
return reply(PrimitiveHostReplyKind::unavailable, {});
|
||||
auto outcome = persistence::query_primitive_life(*db_, config_.key, *authority_.grant(),
|
||||
account, generation, request,
|
||||
config_.lease.commit_timeout);
|
||||
if (!outcome) {
|
||||
if (outcome.error().kind == persistence::LifeReservationError::unknown ||
|
||||
outcome.error().kind == persistence::LifeReservationError::store)
|
||||
db_.reset();
|
||||
if (outcome.error().kind == persistence::LifeReservationError::fenced)
|
||||
status_.ready = false;
|
||||
return reply(PrimitiveHostReplyKind::unknown, {});
|
||||
}
|
||||
if (!*outcome)
|
||||
return reply(PrimitiveHostReplyKind::unknown, {});
|
||||
const auto& value = **outcome;
|
||||
if (value.published)
|
||||
return reply(PrimitiveHostReplyKind::born, value.reservation, value.published);
|
||||
if (value.reservation.region != config_.key.region)
|
||||
return reply(PrimitiveHostReplyKind::routed, value.reservation);
|
||||
if (value.reservation.refusal != persistence::LifeRefusal::none)
|
||||
return reply(PrimitiveHostReplyKind::refused, value.reservation);
|
||||
return reply(PrimitiveHostReplyKind::pending, value.reservation);
|
||||
}
|
||||
|
||||
PrimitiveHostReply PrimitiveRegionHost::new_life(const persistence::Uuid& account,
|
||||
std::uint64_t generation,
|
||||
std::uint64_t request,
|
||||
std::uint32_t rules) {
|
||||
if (!status_.ready || !state_ || !committed_ || !authority_.may_begin_commit(LeaseClock::now()))
|
||||
return reply(PrimitiveHostReplyKind::unavailable, {});
|
||||
if (account == persistence::Uuid{} || generation == 0 || request == 0 || rules == 0)
|
||||
return reply(PrimitiveHostReplyKind::refused, {});
|
||||
if (pending_) {
|
||||
if (pending_->account != account || pending_->generation != generation ||
|
||||
pending_->request != request)
|
||||
return reply(PrimitiveHostReplyKind::pending, pending_->reservation);
|
||||
if (!database()) {
|
||||
status_.phase = HostPhase::resolving;
|
||||
status_.ready = false;
|
||||
return reply(PrimitiveHostReplyKind::unknown, pending_->reservation);
|
||||
}
|
||||
persistence::PrimitivePreparedCommit prepared;
|
||||
if (pending_->prepared) {
|
||||
prepared = *pending_->prepared;
|
||||
} else {
|
||||
// Re-entering the adapter with the same predecessor, trace and commit recreates
|
||||
// the exact canonical bytes when the earlier failure happened before SQL.
|
||||
auto published = persistence::publish_primitive_batch(
|
||||
*db_, *objects_, *authority_.grant(), *committed_, pending_->batch,
|
||||
*pending_->candidate, pending_->commit, config_.build,
|
||||
config_.lease.commit_timeout);
|
||||
if (published) {
|
||||
state_ = std::move(pending_->candidate);
|
||||
committed_ = published->committed;
|
||||
pending_.reset();
|
||||
++status_.commits;
|
||||
status_.committed_sequence = committed_->head.sequence;
|
||||
status_.committed_tick = committed_->head.end_tick;
|
||||
status_.simulated_tick = committed_->head.end_tick;
|
||||
return query(account, generation, request);
|
||||
}
|
||||
if (published.error().kind == persistence::PublicationError::unknown) {
|
||||
pending_->prepared = published.error().prepared;
|
||||
db_.reset();
|
||||
++status_.unknown_outcomes;
|
||||
status_.phase = HostPhase::resolving;
|
||||
status_.ready = false;
|
||||
return reply(PrimitiveHostReplyKind::unknown, pending_->reservation);
|
||||
}
|
||||
if (published.error().kind == persistence::PublicationError::rejected &&
|
||||
published.error().store.kind == LeaseStoreError::fenced) {
|
||||
authority_.fence();
|
||||
status_.ready = false;
|
||||
status_.phase = HostPhase::acquiring;
|
||||
return reply(PrimitiveHostReplyKind::unknown, pending_->reservation);
|
||||
}
|
||||
db_.reset();
|
||||
++status_.store_failures;
|
||||
const auto reservation = pending_->reservation;
|
||||
pending_.reset();
|
||||
authority_.fence();
|
||||
status_.ready = false;
|
||||
status_.phase = HostPhase::quarantined;
|
||||
return reply(PrimitiveHostReplyKind::unavailable, reservation);
|
||||
}
|
||||
auto resolved = persistence::retry_primitive_prepared(
|
||||
*db_, prepared, config_.lease.commit_timeout);
|
||||
if (!resolved) {
|
||||
if (resolved.error().kind == persistence::PublicationError::unknown) {
|
||||
db_.reset();
|
||||
++status_.store_failures;
|
||||
status_.phase = HostPhase::resolving;
|
||||
status_.ready = false;
|
||||
return reply(PrimitiveHostReplyKind::unknown, pending_->reservation);
|
||||
}
|
||||
const auto reservation = pending_->reservation;
|
||||
pending_.reset();
|
||||
status_.phase = HostPhase::quarantined;
|
||||
status_.ready = false;
|
||||
authority_.fence();
|
||||
return reply(PrimitiveHostReplyKind::unavailable, reservation);
|
||||
}
|
||||
state_ = std::move(pending_->candidate);
|
||||
committed_ = std::move(*resolved);
|
||||
pending_.reset();
|
||||
++status_.commits;
|
||||
status_.phase = HostPhase::running;
|
||||
status_.committed_sequence = committed_->head.sequence;
|
||||
status_.committed_tick = committed_->head.end_tick;
|
||||
status_.simulated_tick = committed_->head.end_tick;
|
||||
refresh();
|
||||
return query(account, generation, request);
|
||||
}
|
||||
|
||||
if (!database())
|
||||
return reply(PrimitiveHostReplyKind::unavailable, {});
|
||||
auto existing = query(account, generation, request);
|
||||
if (existing.kind != PrimitiveHostReplyKind::unknown &&
|
||||
existing.kind != PrimitiveHostReplyKind::pending)
|
||||
return existing;
|
||||
auto reserved = persistence::reserve_primitive_life(
|
||||
*db_, config_.key, *authority_.grant(), account, generation, request, rules,
|
||||
config_.lease.commit_timeout);
|
||||
if (!reserved) {
|
||||
if (reserved.error().kind == persistence::LifeReservationError::unknown) {
|
||||
db_.reset();
|
||||
++status_.unknown_outcomes;
|
||||
return reply(PrimitiveHostReplyKind::unknown, {});
|
||||
}
|
||||
if (reserved.error().kind == persistence::LifeReservationError::existing_life)
|
||||
return query(account, generation, request);
|
||||
if (reserved.error().kind == persistence::LifeReservationError::fenced) {
|
||||
status_.ready = false;
|
||||
authority_.fence();
|
||||
}
|
||||
return reply(PrimitiveHostReplyKind::unavailable, {});
|
||||
}
|
||||
if (reserved->region != config_.key.region)
|
||||
return reply(PrimitiveHostReplyKind::routed, *reserved);
|
||||
if (reserved->refusal != persistence::LifeRefusal::none)
|
||||
return reply(PrimitiveHostReplyKind::refused, *reserved);
|
||||
const auto refuse = [&](persistence::LifeRefusal reason) {
|
||||
const auto recorded = persistence::reject_primitive_life(
|
||||
*db_, config_.key, *authority_.grant(), account, generation, request, reason,
|
||||
config_.lease.commit_timeout);
|
||||
if (!recorded) {
|
||||
if (recorded.error().kind == persistence::LifeReservationError::unknown)
|
||||
db_.reset();
|
||||
return reply(PrimitiveHostReplyKind::unknown, *reserved);
|
||||
}
|
||||
auto final = *reserved;
|
||||
final.refusal = *recorded;
|
||||
return reply(PrimitiveHostReplyKind::refused, final);
|
||||
};
|
||||
if (state_->lives().size() >= max_primitive_lives ||
|
||||
state_->lives().size() >= state_->config().sites.size())
|
||||
return refuse(persistence::LifeRefusal::no_placement);
|
||||
const auto endowed = biology::primitive_endowment();
|
||||
if (!endowed || state_->balances().empty()) {
|
||||
status_.phase = HostPhase::quarantined;
|
||||
status_.ready = false;
|
||||
authority_.fence();
|
||||
return reply(PrimitiveHostReplyKind::unavailable, *reserved);
|
||||
}
|
||||
auto& source = state_->balances().front();
|
||||
const auto enough = [&] {
|
||||
if (source.energy.value < endowed->body.energy.value + endowed->reserve.energy.value +
|
||||
endowed->construction.value)
|
||||
return false;
|
||||
for (std::size_t i = 0; i < source.matter.size(); ++i)
|
||||
if (source.matter[i].value < endowed->body.matter[i].value)
|
||||
return false;
|
||||
return true;
|
||||
};
|
||||
if (!enough())
|
||||
return refuse(persistence::LifeRefusal::insufficient_funding);
|
||||
if (committed_->head.end_tick == UINT64_MAX ||
|
||||
committed_->head.sequence == UINT64_MAX) {
|
||||
status_.phase = HostPhase::quarantined;
|
||||
status_.ready = false;
|
||||
authority_.fence();
|
||||
return reply(PrimitiveHostReplyKind::unavailable, *reserved);
|
||||
}
|
||||
const auto decoded = persistence::decode_manifest(committed_->manifest);
|
||||
const auto predecessor_digest = persistence::sha256(committed_->manifest);
|
||||
if (!decoded || !predecessor_digest) {
|
||||
status_.phase = HostPhase::quarantined;
|
||||
status_.ready = false;
|
||||
authority_.fence();
|
||||
return reply(PrimitiveHostReplyKind::unavailable, *reserved);
|
||||
}
|
||||
const auto end_tick = committed_->head.end_tick + 1;
|
||||
PrimitiveBatch batch{{config_.key.world, config_.key.region, config_.frame,
|
||||
committed_->head.commit, *predecessor_digest,
|
||||
committed_->head.end_tick, end_tick},
|
||||
{{{account, request}, reserved->identity, generation, {end_tick}}}};
|
||||
auto candidate = replay_primitive_batch(*state_, batch);
|
||||
if (!candidate) {
|
||||
if (candidate.error() == PrimitiveBatchError::state) {
|
||||
status_.phase = HostPhase::quarantined;
|
||||
status_.ready = false;
|
||||
authority_.fence();
|
||||
return reply(PrimitiveHostReplyKind::unavailable, *reserved);
|
||||
}
|
||||
status_.phase = HostPhase::quarantined;
|
||||
status_.ready = false;
|
||||
authority_.fence();
|
||||
return reply(PrimitiveHostReplyKind::unavailable, *reserved);
|
||||
}
|
||||
auto commit = authority_.grant()->holder;
|
||||
const auto sequence = committed_->head.sequence + 1;
|
||||
for (unsigned i = 0; i < 8; ++i)
|
||||
commit[8 + i] = static_cast<std::uint8_t>((sequence >> (i * 8U)) & 0xffU);
|
||||
pending_ = std::make_unique<Pending>(Pending{account, generation, request, *reserved,
|
||||
std::move(batch), std::move(*candidate), commit, {}});
|
||||
auto published = persistence::publish_primitive_batch(
|
||||
*db_, *objects_, *authority_.grant(), *committed_, pending_->batch, *pending_->candidate,
|
||||
pending_->commit, config_.build, config_.lease.commit_timeout);
|
||||
if (!published) {
|
||||
const auto& failure = published.error();
|
||||
if (failure.kind == persistence::PublicationError::unknown) {
|
||||
pending_->prepared = failure.prepared;
|
||||
db_.reset();
|
||||
++status_.unknown_outcomes;
|
||||
status_.phase = HostPhase::resolving;
|
||||
status_.ready = false;
|
||||
return reply(PrimitiveHostReplyKind::unknown, *reserved);
|
||||
}
|
||||
if (failure.kind == persistence::PublicationError::rejected &&
|
||||
failure.store.kind == LeaseStoreError::fenced) {
|
||||
authority_.fence();
|
||||
state_.reset();
|
||||
committed_.reset();
|
||||
pending_.reset();
|
||||
status_.ready = false;
|
||||
status_.phase = HostPhase::acquiring;
|
||||
return reply(PrimitiveHostReplyKind::unknown, *reserved);
|
||||
}
|
||||
++status_.store_failures;
|
||||
if (failure.kind == persistence::PublicationError::candidate ||
|
||||
failure.kind == persistence::PublicationError::predecessor ||
|
||||
failure.kind == persistence::PublicationError::receipt ||
|
||||
failure.kind == persistence::PublicationError::rejected) {
|
||||
pending_.reset();
|
||||
authority_.fence();
|
||||
status_.phase = HostPhase::quarantined;
|
||||
status_.ready = false;
|
||||
return reply(PrimitiveHostReplyKind::unavailable, *reserved);
|
||||
}
|
||||
return reply(PrimitiveHostReplyKind::pending, *reserved);
|
||||
}
|
||||
state_ = std::move(pending_->candidate);
|
||||
committed_ = published->committed;
|
||||
pending_.reset();
|
||||
++status_.commits;
|
||||
status_.committed_sequence = committed_->head.sequence;
|
||||
status_.committed_tick = committed_->head.end_tick;
|
||||
status_.simulated_tick = committed_->head.end_tick;
|
||||
return query(account, generation, request);
|
||||
}
|
||||
} // namespace evo::server
|
||||
Reference in New Issue
Block a user