feat(server): commit cellular batches on a background thread
Foundation / checks (checks) (push) Successful in 5s
Foundation / checks (client) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 4s
Foundation / checks (network) (push) Successful in 4s
Foundation / checks (postgres) (push) Successful in 4s
Foundation / checks (tsan) (push) Successful in 4s
Foundation / checks (deploy) (push) Successful in 4m28s
Foundation / postgres-tools (push) Successful in 23s
Foundation / server-image (push) Successful in 4m27s
Foundation / image (push) Successful in 4m31s
Foundation / checks (checks) (push) Successful in 5s
Foundation / checks (client) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 4s
Foundation / checks (network) (push) Successful in 4s
Foundation / checks (postgres) (push) Successful in 4s
Foundation / checks (tsan) (push) Successful in 4s
Foundation / checks (deploy) (push) Successful in 4m28s
Foundation / postgres-tools (push) Successful in 23s
Foundation / server-image (push) Successful in 4m27s
Foundation / image (push) Successful in 4m31s
The rules-12 loop blocked for ~411 ms per 20-tick commit, so players saw one-second bursts. The host now commits a batch on a worker thread with its own connection (begin_publish / poll_publish), using the builder's already-simulated candidate. The next batch builds on that candidate under a provisional head and is rebased when the commit lands. One commit is in flight at a time; births and mutation choices wait for a batch that starts on the durable head, and host reconciliation pauses meanwhile. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -5,8 +5,10 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <limits>
|
||||
#include <thread>
|
||||
#include <utility>
|
||||
|
||||
namespace evo::server {
|
||||
@@ -18,6 +20,22 @@ struct CellularRegionHost::Pending {
|
||||
std::vector<CellularMutationActionRecord> mutation_actions;
|
||||
std::optional<persistence::PopulationPreparedCommit> population_prepared{};
|
||||
};
|
||||
// A write-behind commit: everything the worker reads is owned here and untouched by the host
|
||||
// thread until `done`, and the worker touches nothing else.
|
||||
struct CellularRegionHost::InFlight {
|
||||
CellularBatch batch;
|
||||
std::vector<CellularActionRecord> actions;
|
||||
std::vector<CellularMutationActionRecord> mutation_actions;
|
||||
std::unique_ptr<CellularLives> candidate;
|
||||
persistence::CommittedCheckpoint predecessor;
|
||||
persistence::LeaseGrant grant;
|
||||
persistence::Uuid commit{};
|
||||
std::optional<
|
||||
std::expected<persistence::PublishedCellularBatch, persistence::CellularPublicationFailure>>
|
||||
result;
|
||||
std::atomic<bool> done{false};
|
||||
std::thread worker;
|
||||
};
|
||||
|
||||
namespace {
|
||||
using persistence::LeaseClock;
|
||||
@@ -155,7 +173,10 @@ CellularRegionHost::CellularRegionHost(CellularHostConfig config, CellularConnec
|
||||
: config_{std::move(config)}, connect_{std::move(connect)}, objects_{std::move(objects)},
|
||||
authority_{config_.lease} {}
|
||||
|
||||
CellularRegionHost::~CellularRegionHost() = default;
|
||||
CellularRegionHost::~CellularRegionHost() {
|
||||
if (in_flight_ && in_flight_->worker.joinable())
|
||||
in_flight_->worker.join();
|
||||
}
|
||||
|
||||
std::expected<std::unique_ptr<CellularRegionHost>, CellularHostError>
|
||||
CellularRegionHost::create(CellularHostConfig config, CellularConnect connect,
|
||||
@@ -676,8 +697,13 @@ const HostStatus& CellularRegionHost::step() {
|
||||
upkeep();
|
||||
if (pending_)
|
||||
resolve();
|
||||
else if (!state_)
|
||||
else if (!state_ && !in_flight_)
|
||||
acquire();
|
||||
// A write-behind commit owns the next head: nothing else commits or reloads meanwhile.
|
||||
if (in_flight_) {
|
||||
refresh();
|
||||
return status_;
|
||||
}
|
||||
if (!pending_ && state_ && status_.phase == HostPhase::running)
|
||||
reconcile_migration_freeze();
|
||||
if (migration_reload_ || migration_frozen_ || migration_request_ || migration_stage_request_ ||
|
||||
@@ -867,7 +893,7 @@ CellularRegionHost::publish(const CellularBatch& batch,
|
||||
std::span<const CellularActionRecord> actions,
|
||||
std::span<const CellularMutationActionRecord> mutation_actions) {
|
||||
const auto grant = authority_.grant();
|
||||
if (!status_.ready || mutations_uncertain_ || !state_ || !committed_ || !grant ||
|
||||
if (in_flight_ || !status_.ready || mutations_uncertain_ || !state_ || !committed_ || !grant ||
|
||||
!authority_.may_begin_commit(LeaseClock::now()))
|
||||
return std::unexpected{CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}};
|
||||
if (population_mode()) {
|
||||
@@ -891,11 +917,94 @@ CellularRegionHost::publish(const CellularBatch& batch,
|
||||
auto published = persistence::publish_cellular_batch(
|
||||
*db_, *objects_, grant.value_or(persistence::LeaseGrant{}), *committed_, batch, **candidate,
|
||||
*commit, config_.build, config_.lease.commit_timeout, {}, actions, mutation_actions);
|
||||
return finish_publication(batch, actions, mutation_actions, std::move(*candidate),
|
||||
std::move(published));
|
||||
}
|
||||
|
||||
std::expected<void, CellularPublishFailure>
|
||||
CellularRegionHost::begin_publish(CellularBatch batch, std::vector<CellularActionRecord> actions,
|
||||
std::vector<CellularMutationActionRecord> mutation_actions,
|
||||
std::unique_ptr<CellularLives> candidate) {
|
||||
const auto grant = authority_.grant();
|
||||
if (in_flight_ || !candidate || !status_.ready || mutations_uncertain_ || !state_ ||
|
||||
!committed_ || !grant || population_mode() ||
|
||||
!authority_.may_begin_commit(LeaseClock::now()))
|
||||
return std::unexpected{CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}};
|
||||
const auto& head = committed_->head;
|
||||
if (candidate->tick() != batch.header.end_tick || batch.header.begin_tick != head.end_tick ||
|
||||
batch.header.predecessor != head.commit || batch.header.predecessor_digest != head.digest)
|
||||
return std::unexpected{
|
||||
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}}};
|
||||
auto trace = encode_cellular_batch(batch);
|
||||
auto action_block =
|
||||
encode_cellular_action_block(actions, batch.header.begin_tick, batch.header.end_tick);
|
||||
const auto commit =
|
||||
trace && action_block ? commit_id(*committed_, *trace, *action_block) : std::nullopt;
|
||||
if (!commit)
|
||||
return std::unexpected{
|
||||
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}}};
|
||||
if (!commit_db_) {
|
||||
auto connection = connect_();
|
||||
if (!connection) {
|
||||
++status_.store_failures;
|
||||
return std::unexpected{
|
||||
CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}};
|
||||
}
|
||||
commit_db_ = std::move(*connection);
|
||||
}
|
||||
auto flight = std::make_unique<InFlight>();
|
||||
flight->batch = std::move(batch);
|
||||
flight->actions = std::move(actions);
|
||||
flight->mutation_actions = std::move(mutation_actions);
|
||||
flight->candidate = std::move(candidate);
|
||||
flight->predecessor = *committed_;
|
||||
flight->grant = *grant;
|
||||
flight->commit = *commit;
|
||||
auto* work = flight.get();
|
||||
auto* db = commit_db_.get();
|
||||
const auto* objects = objects_.get();
|
||||
const auto build = config_.build;
|
||||
const auto timeout = config_.lease.commit_timeout;
|
||||
work->worker = std::thread([work, db, objects, build, timeout] {
|
||||
work->result = persistence::publish_cellular_batch(
|
||||
*db, *objects, work->grant, work->predecessor, work->batch, *work->candidate,
|
||||
work->commit, build, timeout, {}, work->actions, work->mutation_actions);
|
||||
work->done.store(true, std::memory_order_release);
|
||||
});
|
||||
in_flight_ = std::move(flight);
|
||||
return {};
|
||||
}
|
||||
|
||||
std::optional<std::expected<persistence::PublishedCellularBatch, CellularPublishFailure>>
|
||||
CellularRegionHost::poll_publish() {
|
||||
if (!in_flight_ || !in_flight_->done.load(std::memory_order_acquire))
|
||||
return std::nullopt;
|
||||
in_flight_->worker.join();
|
||||
const auto flight = std::move(in_flight_);
|
||||
auto& result = flight->result;
|
||||
if (!result.has_value() || !result->has_value())
|
||||
commit_db_.reset(); // A failed connection is never reused.
|
||||
// Authority lost while the commit ran: recovery reloads whatever the database holds.
|
||||
if (!state_ || !committed_ || committed_->head != flight->predecessor.head)
|
||||
return std::unexpected{CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}};
|
||||
if (!result.has_value())
|
||||
return std::unexpected{CellularPublishFailure{CellularHostError::publication, {}, {}, {}}};
|
||||
return finish_publication(flight->batch, flight->actions, flight->mutation_actions,
|
||||
std::move(flight->candidate), std::move(*result));
|
||||
}
|
||||
|
||||
std::expected<persistence::PublishedCellularBatch, CellularPublishFailure>
|
||||
CellularRegionHost::finish_publication(
|
||||
const CellularBatch& batch, std::span<const CellularActionRecord> actions,
|
||||
std::span<const CellularMutationActionRecord> mutation_actions,
|
||||
std::unique_ptr<CellularLives> candidate,
|
||||
std::expected<persistence::PublishedCellularBatch, persistence::CellularPublicationFailure>
|
||||
published) {
|
||||
if (!published) {
|
||||
if (published.error().kind == persistence::PublicationError::unknown) {
|
||||
pending_ = std::make_unique<Pending>(
|
||||
Pending{published.error().prepared,
|
||||
std::move(*candidate),
|
||||
std::move(candidate),
|
||||
{actions.begin(), actions.end()},
|
||||
births_in(batch),
|
||||
{mutation_actions.begin(), mutation_actions.end()}});
|
||||
@@ -917,8 +1026,8 @@ CellularRegionHost::publish(const CellularBatch& batch,
|
||||
CellularPublishFailure{CellularHostError::publication, published.error().kind,
|
||||
published.error().store, published.error().object}};
|
||||
}
|
||||
note_endings(*state_, **candidate, published->committed.head.commit);
|
||||
state_ = std::move(*candidate);
|
||||
note_endings(*state_, *candidate, published->committed.head.commit);
|
||||
state_ = std::move(candidate);
|
||||
committed_ = published->committed;
|
||||
flush_endings();
|
||||
for (const auto& action : actions)
|
||||
@@ -1093,6 +1202,11 @@ void CellularRegionHost::drain() {
|
||||
status_.ready = false;
|
||||
status_.phase = HostPhase::draining;
|
||||
pending_.reset();
|
||||
// A running commit finishes (or times out) before the lease is released.
|
||||
if (in_flight_ && in_flight_->worker.joinable())
|
||||
in_flight_->worker.join();
|
||||
in_flight_.reset();
|
||||
commit_db_.reset();
|
||||
const auto grant = authority_.grant();
|
||||
if (grant && authority_.may_originate(LeaseClock::now()) && database()) {
|
||||
// Best effort during shutdown; the lease expires if release cannot be confirmed.
|
||||
@@ -1166,9 +1280,10 @@ CellularRegionHost::accept_population_publication(
|
||||
}
|
||||
|
||||
std::expected<void, CellularPublishFailure> CellularRegionHost::set_population_mode(bool enabled) {
|
||||
if (!status_.ready || pending_ || mutations_uncertain_ || admitted_interval_.has_value() ||
|
||||
interval_request_.has_value() || interval_reload_ || !state_ || !committed_.has_value() ||
|
||||
!authority_.may_begin_commit(LeaseClock::now()) || !database())
|
||||
if (in_flight_ || !status_.ready || pending_ || mutations_uncertain_ ||
|
||||
admitted_interval_.has_value() || interval_request_.has_value() || interval_reload_ ||
|
||||
!state_ || !committed_.has_value() || !authority_.may_begin_commit(LeaseClock::now()) ||
|
||||
!database())
|
||||
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
||||
if (population_mode() == enabled)
|
||||
return {};
|
||||
|
||||
Reference in New Issue
Block a user