feat(server): commit cellular batches on a background thread
Foundation / checks (checks) (push) Successful in 5s
Foundation / checks (client) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 4s
Foundation / checks (network) (push) Successful in 4s
Foundation / checks (postgres) (push) Successful in 4s
Foundation / checks (tsan) (push) Successful in 4s
Foundation / checks (deploy) (push) Successful in 4m28s
Foundation / postgres-tools (push) Successful in 23s
Foundation / server-image (push) Successful in 4m27s
Foundation / image (push) Successful in 4m31s
Foundation / checks (checks) (push) Successful in 5s
Foundation / checks (client) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 4s
Foundation / checks (network) (push) Successful in 4s
Foundation / checks (postgres) (push) Successful in 4s
Foundation / checks (tsan) (push) Successful in 4s
Foundation / checks (deploy) (push) Successful in 4m28s
Foundation / postgres-tools (push) Successful in 23s
Foundation / server-image (push) Successful in 4m27s
Foundation / image (push) Successful in 4m31s
The rules-12 loop blocked for ~411 ms per 20-tick commit, so players saw one-second bursts. The host now commits a batch on a worker thread with its own connection (begin_publish / poll_publish), using the builder's already-simulated candidate. The next batch builds on that candidate under a provisional head and is rebased when the commit lands. One commit is in flight at a time; births and mutation choices wait for a batch that starts on the durable head, and host reconciliation pauses meanwhile. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+102
-26
@@ -942,6 +942,20 @@ int serve_cellular(Options& o) {
|
||||
std::vector<network::CarriedAction> batch_actions;
|
||||
std::uint64_t pace_epoch = 0, pace_tick = 0, flushes_logged = 0;
|
||||
std::vector<network::CarriedAction> awaiting_outcomes;
|
||||
// The batch being committed on the host's worker thread while the next one builds on its
|
||||
// candidate: its carried actions, birth refusals and operator timing.
|
||||
std::vector<network::CarriedAction> flight_actions;
|
||||
std::vector<server::CellularBirthRefusal> flight_refusals;
|
||||
std::size_t flight_ticks = 0;
|
||||
long long flight_handoff_ms = 0;
|
||||
auto flight_start = std::chrono::steady_clock::now();
|
||||
const auto drop_carried = [&](std::vector<network::CarriedAction>& carried) {
|
||||
bridge->dropped(carried, true);
|
||||
std::erase_if(awaiting_outcomes, [&](const auto& action) {
|
||||
return std::ranges::find(carried, action) != carried.end();
|
||||
});
|
||||
carried.clear();
|
||||
};
|
||||
// A tick that cannot be built is retried every 50 ms; name each distinct cause once.
|
||||
std::optional<std::pair<std::string_view, server::CellularSessionBatchError>> stalled;
|
||||
const auto report_stall = [&](std::string_view where, server::CellularSessionBatchError error,
|
||||
@@ -1008,6 +1022,35 @@ int serve_cellular(Options& o) {
|
||||
host.drain();
|
||||
return finish(0);
|
||||
}
|
||||
// A finished background commit lands first, so this step sees its head.
|
||||
if (auto landed = host.poll_publish(); landed) {
|
||||
const auto commit_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now() - flight_start)
|
||||
.count();
|
||||
// Operator evidence: the loop is blocked only for the hand-off, not the commit.
|
||||
if (flight_handoff_ms > 50 || !*landed || ++flushes_logged % 30 == 0)
|
||||
std::printf("cellular commit ticks=%zu ms=%lld handoff_ms=%lld ok=%d\n",
|
||||
flight_ticks, static_cast<long long>(commit_ms), flight_handoff_ms,
|
||||
*landed ? 1 : 0);
|
||||
if (*landed) {
|
||||
host.refuse_births(flight_refusals);
|
||||
flight_actions.clear();
|
||||
if (batch_builder && !batch_builder->rebase(host.committed()->head)) {
|
||||
drop_carried(batch_actions);
|
||||
batch_builder.reset();
|
||||
}
|
||||
} else {
|
||||
// An unresolved outcome keeps its actions for the host to finalize.
|
||||
if (host.status().phase == server::HostPhase::resolving)
|
||||
flight_actions.clear();
|
||||
else
|
||||
drop_carried(flight_actions);
|
||||
// The open batch continued from a state that is not durable.
|
||||
drop_carried(batch_actions);
|
||||
batch_builder.reset();
|
||||
}
|
||||
flight_refusals.clear();
|
||||
}
|
||||
const auto& status = host.step();
|
||||
const auto wall =
|
||||
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(
|
||||
@@ -1115,34 +1158,60 @@ int serve_cellular(Options& o) {
|
||||
}
|
||||
const auto flush_ticks = std::clamp<std::uint64_t>(
|
||||
static_cast<std::uint64_t>(o.host.flush_interval.count()) / 50, 1, 60);
|
||||
// Births and mutation choices wait for a batch that starts on the durable head.
|
||||
const bool admissions_waiting = !held_births.empty() || !host.queued_mutations().empty();
|
||||
// Write-behind commit: the open batch goes to the host's worker thread and the next
|
||||
// batch continues from its candidate under a provisional head, rebased when the commit
|
||||
// lands. One commit is in flight; meanwhile the open batch simply keeps growing.
|
||||
const auto flush = [&] {
|
||||
if (!batch_builder || batch_builder->batch().ticks.empty()) {
|
||||
batch_builder.reset();
|
||||
if (!host.commit_in_flight())
|
||||
batch_builder.reset();
|
||||
return;
|
||||
}
|
||||
const auto flush_start = std::chrono::steady_clock::now();
|
||||
const auto flush_ticks_held = batch_builder->batch().ticks.size();
|
||||
auto published = host.publish(batch_builder->batch(), batch_builder->actions(),
|
||||
batch_builder->mutation_actions());
|
||||
// Operator evidence for commit stalls: every slow commit, and one in 60 otherwise.
|
||||
const auto flush_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now() - flush_start)
|
||||
.count();
|
||||
if (flush_ms > 50 || ++flushes_logged % 60 == 0)
|
||||
std::printf("cellular commit ticks=%zu ms=%lld ok=%d\n", flush_ticks_held,
|
||||
static_cast<long long>(flush_ms), published ? 1 : 0);
|
||||
if (published)
|
||||
host.refuse_births(batch_builder->refused_births());
|
||||
if (!published && host.status().phase != server::HostPhase::resolving) {
|
||||
bridge->dropped(batch_actions, true);
|
||||
std::erase_if(awaiting_outcomes, [&](const auto& action) {
|
||||
return std::ranges::find(batch_actions, action) != batch_actions.end();
|
||||
});
|
||||
if (host.commit_in_flight())
|
||||
return;
|
||||
const auto handoff_start = std::chrono::steady_clock::now();
|
||||
const auto& candidate = batch_builder->candidate();
|
||||
std::optional<server::CellularSessionBatchBuilder> next;
|
||||
if (!admissions_waiting && candidate.tick() != UINT64_MAX) {
|
||||
persistence::RegionHead provisional{candidate.frame(),
|
||||
host.committed()->head.sequence + 1,
|
||||
{},
|
||||
{},
|
||||
candidate.tick()};
|
||||
provisional.commit.fill(0xFF);
|
||||
provisional.digest.fill(0xFF);
|
||||
if (auto created =
|
||||
server::CellularSessionBatchBuilder::create(candidate, provisional);
|
||||
created)
|
||||
next.emplace(std::move(*created));
|
||||
}
|
||||
flight_ticks = batch_builder->batch().ticks.size();
|
||||
const auto refusals = batch_builder->refused_births();
|
||||
flight_refusals.assign(refusals.begin(), refusals.end());
|
||||
const auto actions = batch_builder->actions();
|
||||
const auto mutation_actions = batch_builder->mutation_actions();
|
||||
auto begun =
|
||||
host.begin_publish(batch_builder->batch(), {actions.begin(), actions.end()},
|
||||
{mutation_actions.begin(), mutation_actions.end()},
|
||||
batch_builder->take_candidate());
|
||||
batch_builder.reset();
|
||||
flight_actions = std::move(batch_actions);
|
||||
batch_actions.clear();
|
||||
deliver_finalized();
|
||||
state = host.state(); // Publication replaced the committed state.
|
||||
flight_start = std::chrono::steady_clock::now();
|
||||
flight_handoff_ms =
|
||||
std::chrono::duration_cast<std::chrono::milliseconds>(flight_start - handoff_start)
|
||||
.count();
|
||||
if (!begun) {
|
||||
std::printf("cellular commit refused handoff_ms=%lld error=%u\n", flight_handoff_ms,
|
||||
static_cast<unsigned>(begun.error().kind));
|
||||
drop_carried(flight_actions);
|
||||
flight_refusals.clear();
|
||||
return;
|
||||
}
|
||||
if (next)
|
||||
batch_builder = std::move(next);
|
||||
};
|
||||
// A batch cannot outlive the authority or the raw mode it was built under; when demand
|
||||
// ends, what it holds is committed.
|
||||
@@ -1168,11 +1237,16 @@ int serve_cellular(Options& o) {
|
||||
// large: commit every flush interval (ADR-0089: 1 s), not every tick. Players
|
||||
// see the batch's working state at once; outcomes become final at its commit.
|
||||
// Births and mutation choices commit promptly, in their own fresh batch.
|
||||
if (batch_builder && (!held_births.empty() || !host.queued_mutations().empty()))
|
||||
if (batch_builder && admissions_waiting)
|
||||
flush();
|
||||
const auto* current = batch_builder ? &batch_builder->candidate() : state;
|
||||
// A failed commit can leave the host without state until it recovers.
|
||||
if (current && host.committed() && paced->due && needed &&
|
||||
// A failed commit can leave the host without state until it recovers. While a
|
||||
// commit is in flight the open batch is the only place to simulate, and it is
|
||||
// bounded below the batch limit.
|
||||
const bool can_build = batch_builder ? batch_builder->batch().ticks.size() + 4 <
|
||||
server::max_cellular_batch_ticks
|
||||
: !host.commit_in_flight();
|
||||
if (current && host.committed() && paced->due && needed && can_build &&
|
||||
current->tick() != UINT64_MAX) {
|
||||
const auto tick = current->tick() + 1;
|
||||
std::vector<server::CellularLifeIntent> baseline;
|
||||
@@ -1227,8 +1301,10 @@ int serve_cellular(Options& o) {
|
||||
if (!batch_builder) {
|
||||
bridge->dropped(actions, true);
|
||||
} else {
|
||||
// Births and mutation choices only ever enter a fresh batch.
|
||||
const bool fresh = batch_builder->batch().ticks.empty();
|
||||
// Births and mutation choices only ever enter a fresh batch on the
|
||||
// durable head.
|
||||
const bool fresh =
|
||||
batch_builder->batch().ticks.empty() && !host.commit_in_flight();
|
||||
const std::span<const server::CellularTickIntents::Birth> births =
|
||||
fresh ? std::span<const server::CellularTickIntents::Birth>{held_births}
|
||||
: std::span<const server::CellularTickIntents::Birth>{};
|
||||
|
||||
Reference in New Issue
Block a user