feat(server): commit cellular batches on a background thread
Foundation / checks (checks) (push) Successful in 5s
Foundation / checks (client) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 4s
Foundation / checks (network) (push) Successful in 4s
Foundation / checks (postgres) (push) Successful in 4s
Foundation / checks (tsan) (push) Successful in 4s
Foundation / checks (deploy) (push) Successful in 4m28s
Foundation / postgres-tools (push) Successful in 23s
Foundation / server-image (push) Successful in 4m27s
Foundation / image (push) Successful in 4m31s

The rules-12 loop blocked for ~411 ms per 20-tick commit, so players saw
one-second bursts. The host now commits a batch on a worker thread with
its own connection (begin_publish / poll_publish), using the builder's
already-simulated candidate. The next batch builds on that candidate
under a provisional head and is rebased when the commit lands. One commit
is in flight at a time; births and mutation choices wait for a batch that
starts on the durable head, and host reconciliation pauses meanwhile.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-22 08:41:56 -04:00
co-authored by Claude Opus 5.5
parent 4112bef52c
commit ff333d97c2
6 changed files with 403 additions and 37 deletions
+102 -26
View File
@@ -942,6 +942,20 @@ int serve_cellular(Options& o) {
std::vector<network::CarriedAction> batch_actions;
std::uint64_t pace_epoch = 0, pace_tick = 0, flushes_logged = 0;
std::vector<network::CarriedAction> awaiting_outcomes;
// The batch being committed on the host's worker thread while the next one builds on its
// candidate: its carried actions, birth refusals and operator timing.
std::vector<network::CarriedAction> flight_actions;
std::vector<server::CellularBirthRefusal> flight_refusals;
std::size_t flight_ticks = 0;
long long flight_handoff_ms = 0;
auto flight_start = std::chrono::steady_clock::now();
const auto drop_carried = [&](std::vector<network::CarriedAction>& carried) {
bridge->dropped(carried, true);
std::erase_if(awaiting_outcomes, [&](const auto& action) {
return std::ranges::find(carried, action) != carried.end();
});
carried.clear();
};
// A tick that cannot be built is retried every 50 ms; name each distinct cause once.
std::optional<std::pair<std::string_view, server::CellularSessionBatchError>> stalled;
const auto report_stall = [&](std::string_view where, server::CellularSessionBatchError error,
@@ -1008,6 +1022,35 @@ int serve_cellular(Options& o) {
host.drain();
return finish(0);
}
// A finished background commit lands first, so this step sees its head.
if (auto landed = host.poll_publish(); landed) {
const auto commit_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - flight_start)
.count();
// Operator evidence: the loop is blocked only for the hand-off, not the commit.
if (flight_handoff_ms > 50 || !*landed || ++flushes_logged % 30 == 0)
std::printf("cellular commit ticks=%zu ms=%lld handoff_ms=%lld ok=%d\n",
flight_ticks, static_cast<long long>(commit_ms), flight_handoff_ms,
*landed ? 1 : 0);
if (*landed) {
host.refuse_births(flight_refusals);
flight_actions.clear();
if (batch_builder && !batch_builder->rebase(host.committed()->head)) {
drop_carried(batch_actions);
batch_builder.reset();
}
} else {
// An unresolved outcome keeps its actions for the host to finalize.
if (host.status().phase == server::HostPhase::resolving)
flight_actions.clear();
else
drop_carried(flight_actions);
// The open batch continued from a state that is not durable.
drop_carried(batch_actions);
batch_builder.reset();
}
flight_refusals.clear();
}
const auto& status = host.step();
const auto wall =
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(
@@ -1115,34 +1158,60 @@ int serve_cellular(Options& o) {
}
const auto flush_ticks = std::clamp<std::uint64_t>(
static_cast<std::uint64_t>(o.host.flush_interval.count()) / 50, 1, 60);
// Births and mutation choices wait for a batch that starts on the durable head.
const bool admissions_waiting = !held_births.empty() || !host.queued_mutations().empty();
// Write-behind commit: the open batch goes to the host's worker thread and the next
// batch continues from its candidate under a provisional head, rebased when the commit
// lands. One commit is in flight; meanwhile the open batch simply keeps growing.
const auto flush = [&] {
if (!batch_builder || batch_builder->batch().ticks.empty()) {
batch_builder.reset();
if (!host.commit_in_flight())
batch_builder.reset();
return;
}
const auto flush_start = std::chrono::steady_clock::now();
const auto flush_ticks_held = batch_builder->batch().ticks.size();
auto published = host.publish(batch_builder->batch(), batch_builder->actions(),
batch_builder->mutation_actions());
// Operator evidence for commit stalls: every slow commit, and one in 60 otherwise.
const auto flush_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - flush_start)
.count();
if (flush_ms > 50 || ++flushes_logged % 60 == 0)
std::printf("cellular commit ticks=%zu ms=%lld ok=%d\n", flush_ticks_held,
static_cast<long long>(flush_ms), published ? 1 : 0);
if (published)
host.refuse_births(batch_builder->refused_births());
if (!published && host.status().phase != server::HostPhase::resolving) {
bridge->dropped(batch_actions, true);
std::erase_if(awaiting_outcomes, [&](const auto& action) {
return std::ranges::find(batch_actions, action) != batch_actions.end();
});
if (host.commit_in_flight())
return;
const auto handoff_start = std::chrono::steady_clock::now();
const auto& candidate = batch_builder->candidate();
std::optional<server::CellularSessionBatchBuilder> next;
if (!admissions_waiting && candidate.tick() != UINT64_MAX) {
persistence::RegionHead provisional{candidate.frame(),
host.committed()->head.sequence + 1,
{},
{},
candidate.tick()};
provisional.commit.fill(0xFF);
provisional.digest.fill(0xFF);
if (auto created =
server::CellularSessionBatchBuilder::create(candidate, provisional);
created)
next.emplace(std::move(*created));
}
flight_ticks = batch_builder->batch().ticks.size();
const auto refusals = batch_builder->refused_births();
flight_refusals.assign(refusals.begin(), refusals.end());
const auto actions = batch_builder->actions();
const auto mutation_actions = batch_builder->mutation_actions();
auto begun =
host.begin_publish(batch_builder->batch(), {actions.begin(), actions.end()},
{mutation_actions.begin(), mutation_actions.end()},
batch_builder->take_candidate());
batch_builder.reset();
flight_actions = std::move(batch_actions);
batch_actions.clear();
deliver_finalized();
state = host.state(); // Publication replaced the committed state.
flight_start = std::chrono::steady_clock::now();
flight_handoff_ms =
std::chrono::duration_cast<std::chrono::milliseconds>(flight_start - handoff_start)
.count();
if (!begun) {
std::printf("cellular commit refused handoff_ms=%lld error=%u\n", flight_handoff_ms,
static_cast<unsigned>(begun.error().kind));
drop_carried(flight_actions);
flight_refusals.clear();
return;
}
if (next)
batch_builder = std::move(next);
};
// A batch cannot outlive the authority or the raw mode it was built under; when demand
// ends, what it holds is committed.
@@ -1168,11 +1237,16 @@ int serve_cellular(Options& o) {
// large: commit every flush interval (ADR-0089: 1 s), not every tick. Players
// see the batch's working state at once; outcomes become final at its commit.
// Births and mutation choices commit promptly, in their own fresh batch.
if (batch_builder && (!held_births.empty() || !host.queued_mutations().empty()))
if (batch_builder && admissions_waiting)
flush();
const auto* current = batch_builder ? &batch_builder->candidate() : state;
// A failed commit can leave the host without state until it recovers.
if (current && host.committed() && paced->due && needed &&
// A failed commit can leave the host without state until it recovers. While a
// commit is in flight the open batch is the only place to simulate, and it is
// bounded below the batch limit.
const bool can_build = batch_builder ? batch_builder->batch().ticks.size() + 4 <
server::max_cellular_batch_ticks
: !host.commit_in_flight();
if (current && host.committed() && paced->due && needed && can_build &&
current->tick() != UINT64_MAX) {
const auto tick = current->tick() + 1;
std::vector<server::CellularLifeIntent> baseline;
@@ -1227,8 +1301,10 @@ int serve_cellular(Options& o) {
if (!batch_builder) {
bridge->dropped(actions, true);
} else {
// Births and mutation choices only ever enter a fresh batch.
const bool fresh = batch_builder->batch().ticks.empty();
// Births and mutation choices only ever enter a fresh batch on the
// durable head.
const bool fresh =
batch_builder->batch().ticks.empty() && !host.commit_in_flight();
const std::span<const server::CellularTickIntents::Birth> births =
fresh ? std::span<const server::CellularTickIntents::Birth>{held_births}
: std::span<const server::CellularTickIntents::Birth>{};