feat(server): commit cellular batches on a background thread
Foundation / checks (checks) (push) Successful in 5s
Foundation / checks (client) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 4s
Foundation / checks (network) (push) Successful in 4s
Foundation / checks (postgres) (push) Successful in 4s
Foundation / checks (tsan) (push) Successful in 4s
Foundation / checks (deploy) (push) Successful in 4m28s
Foundation / postgres-tools (push) Successful in 23s
Foundation / server-image (push) Successful in 4m27s
Foundation / image (push) Successful in 4m31s

The rules-12 loop blocked for ~411 ms per 20-tick commit, so players saw
one-second bursts. The host now commits a batch on a worker thread with
its own connection (begin_publish / poll_publish), using the builder's
already-simulated candidate. The next batch builds on that candidate
under a provisional head and is rebased when the commit lands. One commit
is in flight at a time; births and mutation choices wait for a batch that
starts on the durable head, and host reconciliation pauses meanwhile.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-22 08:41:56 -04:00
co-authored by Claude Opus 5.5
parent 4112bef52c
commit ff333d97c2
6 changed files with 403 additions and 37 deletions
+124 -9
View File
@@ -5,8 +5,10 @@
#include <algorithm>
#include <array>
#include <atomic>
#include <chrono>
#include <limits>
#include <thread>
#include <utility>
namespace evo::server {
@@ -18,6 +20,22 @@ struct CellularRegionHost::Pending {
std::vector<CellularMutationActionRecord> mutation_actions;
std::optional<persistence::PopulationPreparedCommit> population_prepared{};
};
// A write-behind commit: everything the worker reads is owned here and untouched by the host
// thread until `done`, and the worker touches nothing else.
struct CellularRegionHost::InFlight {
CellularBatch batch;
std::vector<CellularActionRecord> actions;
std::vector<CellularMutationActionRecord> mutation_actions;
std::unique_ptr<CellularLives> candidate;
persistence::CommittedCheckpoint predecessor;
persistence::LeaseGrant grant;
persistence::Uuid commit{};
std::optional<
std::expected<persistence::PublishedCellularBatch, persistence::CellularPublicationFailure>>
result;
std::atomic<bool> done{false};
std::thread worker;
};
namespace {
using persistence::LeaseClock;
@@ -155,7 +173,10 @@ CellularRegionHost::CellularRegionHost(CellularHostConfig config, CellularConnec
: config_{std::move(config)}, connect_{std::move(connect)}, objects_{std::move(objects)},
authority_{config_.lease} {}
CellularRegionHost::~CellularRegionHost() = default;
CellularRegionHost::~CellularRegionHost() {
if (in_flight_ && in_flight_->worker.joinable())
in_flight_->worker.join();
}
std::expected<std::unique_ptr<CellularRegionHost>, CellularHostError>
CellularRegionHost::create(CellularHostConfig config, CellularConnect connect,
@@ -676,8 +697,13 @@ const HostStatus& CellularRegionHost::step() {
upkeep();
if (pending_)
resolve();
else if (!state_)
else if (!state_ && !in_flight_)
acquire();
// A write-behind commit owns the next head: nothing else commits or reloads meanwhile.
if (in_flight_) {
refresh();
return status_;
}
if (!pending_ && state_ && status_.phase == HostPhase::running)
reconcile_migration_freeze();
if (migration_reload_ || migration_frozen_ || migration_request_ || migration_stage_request_ ||
@@ -867,7 +893,7 @@ CellularRegionHost::publish(const CellularBatch& batch,
std::span<const CellularActionRecord> actions,
std::span<const CellularMutationActionRecord> mutation_actions) {
const auto grant = authority_.grant();
if (!status_.ready || mutations_uncertain_ || !state_ || !committed_ || !grant ||
if (in_flight_ || !status_.ready || mutations_uncertain_ || !state_ || !committed_ || !grant ||
!authority_.may_begin_commit(LeaseClock::now()))
return std::unexpected{CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}};
if (population_mode()) {
@@ -891,11 +917,94 @@ CellularRegionHost::publish(const CellularBatch& batch,
auto published = persistence::publish_cellular_batch(
*db_, *objects_, grant.value_or(persistence::LeaseGrant{}), *committed_, batch, **candidate,
*commit, config_.build, config_.lease.commit_timeout, {}, actions, mutation_actions);
return finish_publication(batch, actions, mutation_actions, std::move(*candidate),
std::move(published));
}
std::expected<void, CellularPublishFailure>
CellularRegionHost::begin_publish(CellularBatch batch, std::vector<CellularActionRecord> actions,
std::vector<CellularMutationActionRecord> mutation_actions,
std::unique_ptr<CellularLives> candidate) {
const auto grant = authority_.grant();
if (in_flight_ || !candidate || !status_.ready || mutations_uncertain_ || !state_ ||
!committed_ || !grant || population_mode() ||
!authority_.may_begin_commit(LeaseClock::now()))
return std::unexpected{CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}};
const auto& head = committed_->head;
if (candidate->tick() != batch.header.end_tick || batch.header.begin_tick != head.end_tick ||
batch.header.predecessor != head.commit || batch.header.predecessor_digest != head.digest)
return std::unexpected{
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}}};
auto trace = encode_cellular_batch(batch);
auto action_block =
encode_cellular_action_block(actions, batch.header.begin_tick, batch.header.end_tick);
const auto commit =
trace && action_block ? commit_id(*committed_, *trace, *action_block) : std::nullopt;
if (!commit)
return std::unexpected{
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}}};
if (!commit_db_) {
auto connection = connect_();
if (!connection) {
++status_.store_failures;
return std::unexpected{
CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}};
}
commit_db_ = std::move(*connection);
}
auto flight = std::make_unique<InFlight>();
flight->batch = std::move(batch);
flight->actions = std::move(actions);
flight->mutation_actions = std::move(mutation_actions);
flight->candidate = std::move(candidate);
flight->predecessor = *committed_;
flight->grant = *grant;
flight->commit = *commit;
auto* work = flight.get();
auto* db = commit_db_.get();
const auto* objects = objects_.get();
const auto build = config_.build;
const auto timeout = config_.lease.commit_timeout;
work->worker = std::thread([work, db, objects, build, timeout] {
work->result = persistence::publish_cellular_batch(
*db, *objects, work->grant, work->predecessor, work->batch, *work->candidate,
work->commit, build, timeout, {}, work->actions, work->mutation_actions);
work->done.store(true, std::memory_order_release);
});
in_flight_ = std::move(flight);
return {};
}
std::optional<std::expected<persistence::PublishedCellularBatch, CellularPublishFailure>>
CellularRegionHost::poll_publish() {
if (!in_flight_ || !in_flight_->done.load(std::memory_order_acquire))
return std::nullopt;
in_flight_->worker.join();
const auto flight = std::move(in_flight_);
auto& result = flight->result;
if (!result.has_value() || !result->has_value())
commit_db_.reset(); // A failed connection is never reused.
// Authority lost while the commit ran: recovery reloads whatever the database holds.
if (!state_ || !committed_ || committed_->head != flight->predecessor.head)
return std::unexpected{CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}};
if (!result.has_value())
return std::unexpected{CellularPublishFailure{CellularHostError::publication, {}, {}, {}}};
return finish_publication(flight->batch, flight->actions, flight->mutation_actions,
std::move(flight->candidate), std::move(*result));
}
std::expected<persistence::PublishedCellularBatch, CellularPublishFailure>
CellularRegionHost::finish_publication(
const CellularBatch& batch, std::span<const CellularActionRecord> actions,
std::span<const CellularMutationActionRecord> mutation_actions,
std::unique_ptr<CellularLives> candidate,
std::expected<persistence::PublishedCellularBatch, persistence::CellularPublicationFailure>
published) {
if (!published) {
if (published.error().kind == persistence::PublicationError::unknown) {
pending_ = std::make_unique<Pending>(
Pending{published.error().prepared,
std::move(*candidate),
std::move(candidate),
{actions.begin(), actions.end()},
births_in(batch),
{mutation_actions.begin(), mutation_actions.end()}});
@@ -917,8 +1026,8 @@ CellularRegionHost::publish(const CellularBatch& batch,
CellularPublishFailure{CellularHostError::publication, published.error().kind,
published.error().store, published.error().object}};
}
note_endings(*state_, **candidate, published->committed.head.commit);
state_ = std::move(*candidate);
note_endings(*state_, *candidate, published->committed.head.commit);
state_ = std::move(candidate);
committed_ = published->committed;
flush_endings();
for (const auto& action : actions)
@@ -1093,6 +1202,11 @@ void CellularRegionHost::drain() {
status_.ready = false;
status_.phase = HostPhase::draining;
pending_.reset();
// A running commit finishes (or times out) before the lease is released.
if (in_flight_ && in_flight_->worker.joinable())
in_flight_->worker.join();
in_flight_.reset();
commit_db_.reset();
const auto grant = authority_.grant();
if (grant && authority_.may_originate(LeaseClock::now()) && database()) {
// Best effort during shutdown; the lease expires if release cannot be confirmed.
@@ -1166,9 +1280,10 @@ CellularRegionHost::accept_population_publication(
}
std::expected<void, CellularPublishFailure> CellularRegionHost::set_population_mode(bool enabled) {
if (!status_.ready || pending_ || mutations_uncertain_ || admitted_interval_.has_value() ||
interval_request_.has_value() || interval_reload_ || !state_ || !committed_.has_value() ||
!authority_.may_begin_commit(LeaseClock::now()) || !database())
if (in_flight_ || !status_.ready || pending_ || mutations_uncertain_ ||
admitted_interval_.has_value() || interval_request_.has_value() || interval_reload_ ||
!state_ || !committed_.has_value() || !authority_.may_begin_commit(LeaseClock::now()) ||
!database())
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
if (population_mode() == enabled)
return {};