fix(server): hold mutation choices while ticks past the head are open
Foundation / checks (checks) (push) Successful in 5s
Foundation / checks (client) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 4s
Foundation / checks (postgres) (push) Successful in 4s
Foundation / checks (network) (push) Successful in 5s
Foundation / checks (tsan) (push) Successful in 4s
Foundation / checks (deploy) (push) Successful in 4m27s
Foundation / postgres-tools (push) Successful in 25s
Foundation / server-image (push) Successful in 4m9s
Foundation / image (push) Successful in 4m33s

A choice reserves the tick after the committed head. With an open batch
or a background commit, that tick was already being simulated: the batch
committed without the choice (refused, losing its ticks) or, racing the
reservation, committed past it and left a reservation for a tick that had
gone, which stalled the region at its next fresh batch.

While a batch is open or a commit is in flight, the host now holds new
choices and answers not_ready (the client repeats them every 2 s). The
loop commits what it holds, waits for the head, then reserves the held
choices and starts a fresh batch with them. A queued choice the builder
still refuses makes the host reload its queue instead of stalling.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-22 09:32:39 -04:00
co-authored by Claude Opus 5.5
parent 8ce609d344
commit d06c781819
5 changed files with 70 additions and 2 deletions
+12 -1
View File
@@ -1063,6 +1063,8 @@ int serve_cellular(Options& o) {
if (batch_builder)
session_status.simulated_tick =
std::max(session_status.simulated_tick, batch_builder->candidate().tick());
// Mutation choices are reserved only while nothing is simulated past the head.
host.hold_mutation_selections(batch_builder.has_value() || host.commit_in_flight());
bridge->poll(session_status, wall);
deliver_finalized();
set_ready(status.ready);
@@ -1159,7 +1161,8 @@ int serve_cellular(Options& o) {
const auto flush_ticks = std::clamp<std::uint64_t>(
static_cast<std::uint64_t>(o.host.flush_interval.count()) / 50, 1, 60);
// Births and mutation choices wait for a batch that starts on the durable head.
const bool admissions_waiting = !held_births.empty() || !host.queued_mutations().empty();
const bool admissions_waiting =
!held_births.empty() || !host.queued_mutations().empty() || host.selections_held();
// Write-behind commit: the open batch goes to the host's worker thread and the next
// batch continues from its candidate under a provisional head, rebased when the commit
// lands. One commit is in flight; meanwhile the open batch simply keeps growing.
@@ -1239,6 +1242,12 @@ int serve_cellular(Options& o) {
// Births and mutation choices commit promptly, in their own fresh batch.
if (batch_builder && admissions_waiting)
flush();
// The head is current: held choices reserve its next tick and join the
// fresh batch below.
if (!batch_builder && !host.commit_in_flight() && host.selections_held()) {
host.hold_mutation_selections(false);
host.admit_held_selections();
}
const auto* current = batch_builder ? &batch_builder->candidate() : state;
// A failed commit can leave the host without state until it recovers. While a
// commit is in flight the open batch is the only place to simulate, and it is
@@ -1333,6 +1342,8 @@ int serve_cellular(Options& o) {
const auto lives = batch_builder->last_lives_error();
report_stall("append", appended.error(), tick,
lives ? static_cast<int>(*lives) : -1);
if (appended.error() == server::CellularSessionBatchError::mutation)
host.recheck_mutations();
bridge->dropped(actions, true);
flush(); // Commit what is good; the next tick starts afresh.
} else {
@@ -12,7 +12,7 @@ spec:
env:
- {name: REGION_SESSION, value: "dev"}
- {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"}
- {name: REGION_SESSION_WINDOW, value: "ui-20261022T130803-a1d37601:1792688883"}
- {name: REGION_SESSION_WINDOW, value: "dev-20261022T133227-896f7eec:1792690347"}
- {name: REGION_M10_MAX_GRANT_LIFETIME, value: "14400"}
ports:
# QUIC (ADR-0090): UDP only; nothing listens on TCP 7400.
@@ -139,6 +139,23 @@ class CellularRegionHost {
std::uint64_t generation,
std::uint64_t action_id,
const CellularMutationActionInput& input);
// A choice reserves the tick after the committed head, so it must not be reserved while
// ticks past that head are simulated (an open batch, or a commit in flight): the batch
// would commit without it, or skip its tick. While held, a new choice is remembered and
// answered not_ready (the client repeats it); admit_held_selections reserves them once
// nothing is simulated past the head.
void hold_mutation_selections(bool hold) noexcept {
hold_selections_ = hold;
}
[[nodiscard]] bool selections_held() const noexcept {
return !held_selections_.empty();
}
void admit_held_selections();
// A queued choice the builder refused (its tick passed): reload the queue from the
// store, which keeps only reservations for the head's next tick.
void recheck_mutations() noexcept {
mutations_uncertain_ = true;
}
// Reserved choices for the next tick, in reservation order; pass them to the builder.
[[nodiscard]] std::span<const persistence::CellularMutationReservation>
queued_mutations() const noexcept {
@@ -277,8 +294,15 @@ class CellularRegionHost {
std::vector<CellularTickIntents::Birth> queued_births_;
std::vector<CellularBirthRefusal> pending_birth_refusals_;
std::vector<persistence::CellularMutationReservation> queued_mutations_;
struct HeldSelection {
persistence::Uuid account{};
std::uint64_t generation{}, action_id{};
CellularMutationActionInput input{};
};
std::vector<HeldSelection> held_selections_;
// A reservation may exist that the queue does not show; reload before publishing.
bool mutations_uncertain_{};
bool hold_selections_{};
std::vector<FinalCellularMutation> finalized_mutations_;
std::vector<FinalCellularMutation> recent_mutations_; // Bounded, for exact repeats.
struct Ending {
+23
View File
@@ -827,6 +827,20 @@ CellularRegionHost::select_mutation(const persistence::Uuid& account, std::uint6
return {not_ready, 0, {}};
if (population_mode() && !set_population_mode(false))
return {not_ready, 0, {}};
if (hold_selections_ || in_flight_) {
// Already reserved: repeats keep their answer. Otherwise remember the latest choice
// per account (bounded like a tick's lives) until the head is current.
const auto queued = std::ranges::find(queued_mutations_, account,
&persistence::CellularMutationReservation::account);
if (queued == queued_mutations_.end()) {
const auto held = std::ranges::find(held_selections_, account, &HeldSelection::account);
if (held != held_selections_.end())
*held = {account, generation, action_id, input};
else if (held_selections_.size() < biology::max_cellular_tick_lives)
held_selections_.push_back({account, generation, action_id, input});
return {not_ready, 0, {}};
}
}
const auto tick = committed_->head.end_tick + 1;
const CellularMutationActionRecord record{account, action_id, tick, input};
const auto bytes = encode_cellular_mutation_input(record);
@@ -888,6 +902,15 @@ CellularRegionHost::select_mutation(const persistence::Uuid& account, std::uint6
return {admitted, tick, {}};
}
void CellularRegionHost::admit_held_selections() {
if (hold_selections_ || in_flight_)
return;
// The clients repeat their choices; a refusal here is answered on that repeat.
for (const auto& held : std::exchange(held_selections_, {}))
static_cast<void>(
select_mutation(held.account, held.generation, held.action_id, held.input));
}
std::expected<persistence::PublishedCellularBatch, CellularPublishFailure>
CellularRegionHost::publish(const CellularBatch& batch,
std::span<const CellularActionRecord> actions,
+10
View File
@@ -607,6 +607,16 @@ void cellular_host_admits_mutation_choices() {
++stale.event_index;
CHECK(host->select_mutation(account, control->generation, 31, stale).status ==
CellularMutationAdmission::invalid);
// While ticks past the head are being simulated, a choice is held, not reserved.
host->hold_mutation_selections(true);
CHECK(host->select_mutation(account, control->generation, 31, choice).status ==
CellularMutationAdmission::not_ready &&
host->selections_held() && host->queued_mutations().empty());
host->admit_held_selections(); // Still held: nothing is reserved.
CHECK(host->selections_held() && host->queued_mutations().empty());
host->hold_mutation_selections(false);
host->admit_held_selections();
CHECK(!host->selections_held() && host->queued_mutations().size() == 1);
const auto admitted = host->select_mutation(account, control->generation, 31, choice);
CHECK(admitted.status == CellularMutationAdmission::admitted && admitted.tick == next);
const auto repeated = host->select_mutation(account, control->generation, 31, choice);