fix(server): hold mutation choices while ticks past the head are open
Foundation / checks (checks) (push) Successful in 5s
Foundation / checks (client) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 4s
Foundation / checks (postgres) (push) Successful in 4s
Foundation / checks (network) (push) Successful in 5s
Foundation / checks (tsan) (push) Successful in 4s
Foundation / checks (deploy) (push) Successful in 4m27s
Foundation / postgres-tools (push) Successful in 25s
Foundation / server-image (push) Successful in 4m9s
Foundation / image (push) Successful in 4m33s
Foundation / checks (checks) (push) Successful in 5s
Foundation / checks (client) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 4s
Foundation / checks (postgres) (push) Successful in 4s
Foundation / checks (network) (push) Successful in 5s
Foundation / checks (tsan) (push) Successful in 4s
Foundation / checks (deploy) (push) Successful in 4m27s
Foundation / postgres-tools (push) Successful in 25s
Foundation / server-image (push) Successful in 4m9s
Foundation / image (push) Successful in 4m33s
A choice reserves the tick after the committed head. With an open batch or a background commit, that tick was already being simulated: the batch committed without the choice (refused, losing its ticks) or, racing the reservation, committed past it and left a reservation for a tick that had gone, which stalled the region at its next fresh batch. While a batch is open or a commit is in flight, the host now holds new choices and answers not_ready (the client repeats them every 2 s). The loop commits what it holds, waits for the head, then reserves the held choices and starts a fresh batch with them. A queued choice the builder still refuses makes the host reload its queue instead of stalling. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1063,6 +1063,8 @@ int serve_cellular(Options& o) {
|
||||
if (batch_builder)
|
||||
session_status.simulated_tick =
|
||||
std::max(session_status.simulated_tick, batch_builder->candidate().tick());
|
||||
// Mutation choices are reserved only while nothing is simulated past the head.
|
||||
host.hold_mutation_selections(batch_builder.has_value() || host.commit_in_flight());
|
||||
bridge->poll(session_status, wall);
|
||||
deliver_finalized();
|
||||
set_ready(status.ready);
|
||||
@@ -1159,7 +1161,8 @@ int serve_cellular(Options& o) {
|
||||
const auto flush_ticks = std::clamp<std::uint64_t>(
|
||||
static_cast<std::uint64_t>(o.host.flush_interval.count()) / 50, 1, 60);
|
||||
// Births and mutation choices wait for a batch that starts on the durable head.
|
||||
const bool admissions_waiting = !held_births.empty() || !host.queued_mutations().empty();
|
||||
const bool admissions_waiting =
|
||||
!held_births.empty() || !host.queued_mutations().empty() || host.selections_held();
|
||||
// Write-behind commit: the open batch goes to the host's worker thread and the next
|
||||
// batch continues from its candidate under a provisional head, rebased when the commit
|
||||
// lands. One commit is in flight; meanwhile the open batch simply keeps growing.
|
||||
@@ -1239,6 +1242,12 @@ int serve_cellular(Options& o) {
|
||||
// Births and mutation choices commit promptly, in their own fresh batch.
|
||||
if (batch_builder && admissions_waiting)
|
||||
flush();
|
||||
// The head is current: held choices reserve its next tick and join the
|
||||
// fresh batch below.
|
||||
if (!batch_builder && !host.commit_in_flight() && host.selections_held()) {
|
||||
host.hold_mutation_selections(false);
|
||||
host.admit_held_selections();
|
||||
}
|
||||
const auto* current = batch_builder ? &batch_builder->candidate() : state;
|
||||
// A failed commit can leave the host without state until it recovers. While a
|
||||
// commit is in flight the open batch is the only place to simulate, and it is
|
||||
@@ -1333,6 +1342,8 @@ int serve_cellular(Options& o) {
|
||||
const auto lives = batch_builder->last_lives_error();
|
||||
report_stall("append", appended.error(), tick,
|
||||
lives ? static_cast<int>(*lives) : -1);
|
||||
if (appended.error() == server::CellularSessionBatchError::mutation)
|
||||
host.recheck_mutations();
|
||||
bridge->dropped(actions, true);
|
||||
flush(); // Commit what is good; the next tick starts afresh.
|
||||
} else {
|
||||
|
||||
@@ -12,7 +12,7 @@ spec:
|
||||
env:
|
||||
- {name: REGION_SESSION, value: "dev"}
|
||||
- {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"}
|
||||
- {name: REGION_SESSION_WINDOW, value: "ui-20261022T130803-a1d37601:1792688883"}
|
||||
- {name: REGION_SESSION_WINDOW, value: "dev-20261022T133227-896f7eec:1792690347"}
|
||||
- {name: REGION_M10_MAX_GRANT_LIFETIME, value: "14400"}
|
||||
ports:
|
||||
# QUIC (ADR-0090): UDP only; nothing listens on TCP 7400.
|
||||
|
||||
@@ -139,6 +139,23 @@ class CellularRegionHost {
|
||||
std::uint64_t generation,
|
||||
std::uint64_t action_id,
|
||||
const CellularMutationActionInput& input);
|
||||
// A choice reserves the tick after the committed head, so it must not be reserved while
|
||||
// ticks past that head are simulated (an open batch, or a commit in flight): the batch
|
||||
// would commit without it, or skip its tick. While held, a new choice is remembered and
|
||||
// answered not_ready (the client repeats it); admit_held_selections reserves them once
|
||||
// nothing is simulated past the head.
|
||||
void hold_mutation_selections(bool hold) noexcept {
|
||||
hold_selections_ = hold;
|
||||
}
|
||||
[[nodiscard]] bool selections_held() const noexcept {
|
||||
return !held_selections_.empty();
|
||||
}
|
||||
void admit_held_selections();
|
||||
// A queued choice the builder refused (its tick passed): reload the queue from the
|
||||
// store, which keeps only reservations for the head's next tick.
|
||||
void recheck_mutations() noexcept {
|
||||
mutations_uncertain_ = true;
|
||||
}
|
||||
// Reserved choices for the next tick, in reservation order; pass them to the builder.
|
||||
[[nodiscard]] std::span<const persistence::CellularMutationReservation>
|
||||
queued_mutations() const noexcept {
|
||||
@@ -277,8 +294,15 @@ class CellularRegionHost {
|
||||
std::vector<CellularTickIntents::Birth> queued_births_;
|
||||
std::vector<CellularBirthRefusal> pending_birth_refusals_;
|
||||
std::vector<persistence::CellularMutationReservation> queued_mutations_;
|
||||
struct HeldSelection {
|
||||
persistence::Uuid account{};
|
||||
std::uint64_t generation{}, action_id{};
|
||||
CellularMutationActionInput input{};
|
||||
};
|
||||
std::vector<HeldSelection> held_selections_;
|
||||
// A reservation may exist that the queue does not show; reload before publishing.
|
||||
bool mutations_uncertain_{};
|
||||
bool hold_selections_{};
|
||||
std::vector<FinalCellularMutation> finalized_mutations_;
|
||||
std::vector<FinalCellularMutation> recent_mutations_; // Bounded, for exact repeats.
|
||||
struct Ending {
|
||||
|
||||
@@ -827,6 +827,20 @@ CellularRegionHost::select_mutation(const persistence::Uuid& account, std::uint6
|
||||
return {not_ready, 0, {}};
|
||||
if (population_mode() && !set_population_mode(false))
|
||||
return {not_ready, 0, {}};
|
||||
if (hold_selections_ || in_flight_) {
|
||||
// Already reserved: repeats keep their answer. Otherwise remember the latest choice
|
||||
// per account (bounded like a tick's lives) until the head is current.
|
||||
const auto queued = std::ranges::find(queued_mutations_, account,
|
||||
&persistence::CellularMutationReservation::account);
|
||||
if (queued == queued_mutations_.end()) {
|
||||
const auto held = std::ranges::find(held_selections_, account, &HeldSelection::account);
|
||||
if (held != held_selections_.end())
|
||||
*held = {account, generation, action_id, input};
|
||||
else if (held_selections_.size() < biology::max_cellular_tick_lives)
|
||||
held_selections_.push_back({account, generation, action_id, input});
|
||||
return {not_ready, 0, {}};
|
||||
}
|
||||
}
|
||||
const auto tick = committed_->head.end_tick + 1;
|
||||
const CellularMutationActionRecord record{account, action_id, tick, input};
|
||||
const auto bytes = encode_cellular_mutation_input(record);
|
||||
@@ -888,6 +902,15 @@ CellularRegionHost::select_mutation(const persistence::Uuid& account, std::uint6
|
||||
return {admitted, tick, {}};
|
||||
}
|
||||
|
||||
void CellularRegionHost::admit_held_selections() {
|
||||
if (hold_selections_ || in_flight_)
|
||||
return;
|
||||
// The clients repeat their choices; a refusal here is answered on that repeat.
|
||||
for (const auto& held : std::exchange(held_selections_, {}))
|
||||
static_cast<void>(
|
||||
select_mutation(held.account, held.generation, held.action_id, held.input));
|
||||
}
|
||||
|
||||
std::expected<persistence::PublishedCellularBatch, CellularPublishFailure>
|
||||
CellularRegionHost::publish(const CellularBatch& batch,
|
||||
std::span<const CellularActionRecord> actions,
|
||||
|
||||
@@ -607,6 +607,16 @@ void cellular_host_admits_mutation_choices() {
|
||||
++stale.event_index;
|
||||
CHECK(host->select_mutation(account, control->generation, 31, stale).status ==
|
||||
CellularMutationAdmission::invalid);
|
||||
// While ticks past the head are being simulated, a choice is held, not reserved.
|
||||
host->hold_mutation_selections(true);
|
||||
CHECK(host->select_mutation(account, control->generation, 31, choice).status ==
|
||||
CellularMutationAdmission::not_ready &&
|
||||
host->selections_held() && host->queued_mutations().empty());
|
||||
host->admit_held_selections(); // Still held: nothing is reserved.
|
||||
CHECK(host->selections_held() && host->queued_mutations().empty());
|
||||
host->hold_mutation_selections(false);
|
||||
host->admit_held_selections();
|
||||
CHECK(!host->selections_held() && host->queued_mutations().size() == 1);
|
||||
const auto admitted = host->select_mutation(account, control->generation, 31, choice);
|
||||
CHECK(admitted.status == CellularMutationAdmission::admitted && admitted.tick == next);
|
||||
const auto repeated = host->select_mutation(account, control->generation, 31, choice);
|
||||
|
||||
Reference in New Issue
Block a user