feat: durable mutation queue (schema 59)
Foundation / checks (checks) (push) Successful in 5s
Foundation / checks (client) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 4s
Foundation / checks (network) (push) Successful in 5s
Foundation / checks (postgres) (push) Successful in 4s
Foundation / checks (tsan) (push) Successful in 4s
Foundation / checks (deploy) (push) Successful in 4m24s
Foundation / postgres-tools (push) Successful in 1m31s
Foundation / server-image (push) Successful in 4m25s
Foundation / image (push) Successful in 4m50s

A mutation choice's reserved tick is now the earliest tick it may bind
at, not the only one (ADR-0121 amendment). With write-behind commits, a
choice reserved for head + 1 landed in ticks already being simulated or
committed: the batch was refused, or the reservation was orphaned for a
past tick and stalled the region.

Schema 59: a bound record carries the tick it applied at and the rest of
its input must equal the reservation's; the "omits a reserved mutation"
refusals are removed. Existing rows are unchanged, so an orphaned
reservation binds at the next commit that carries it.

The host keeps every unbound reservation queued and the open batch binds
those the commit in flight does not carry, at its next tick; the earlier
hold on new choices is removed. A choice that outlives its life binds as
life_died (owner, trace matching and recovery accept it when no life with
that ID remains). Recovery takes a bound choice's tick from its trace.

Tests: a choice skipped by one checkpoint binds a tick late and recovers;
builder late binding, refusal before the earliest tick, and a departed
life's life_died outcome. Full local PostgreSQL qualification passes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-22 10:27:09 -04:00
co-authored by Claude Opus 5.5
parent 8a7e944942
commit 71d195d4e8
17 changed files with 409 additions and 114 deletions
+13 -24
View File
@@ -774,11 +774,11 @@ void CellularRegionHost::reload_mutations() {
++status_.store_failures;
return;
}
// Every reservation targets the tick after the head it was made at, and SQL refuses a
// checkpoint that skips one, so an unbound choice is always for the next tick.
// Schema 59: every unbound reservation waits in the queue; its tick is the earliest it
// may bind at, which is never past the head's next tick.
queued_mutations_.clear();
for (const auto& reservation : *pending)
if (reservation.target_tick == committed_->head.end_tick + 1)
if (reservation.target_tick <= committed_->head.end_tick + 1)
queued_mutations_.push_back(reservation);
mutations_uncertain_ = false;
}
@@ -827,20 +827,6 @@ CellularRegionHost::select_mutation(const persistence::Uuid& account, std::uint6
return {not_ready, 0, {}};
if (population_mode() && !set_population_mode(false))
return {not_ready, 0, {}};
if (hold_selections_ || in_flight_) {
// Already reserved: repeats keep their answer. Otherwise remember the latest choice
// per account (bounded like a tick's lives) until the head is current.
const auto queued = std::ranges::find(queued_mutations_, account,
&persistence::CellularMutationReservation::account);
if (queued == queued_mutations_.end()) {
const auto held = std::ranges::find(held_selections_, account, &HeldSelection::account);
if (held != held_selections_.end())
*held = {account, generation, action_id, input};
else if (held_selections_.size() < biology::max_cellular_tick_lives)
held_selections_.push_back({account, generation, action_id, input});
return {not_ready, 0, {}};
}
}
const auto tick = committed_->head.end_tick + 1;
const CellularMutationActionRecord record{account, action_id, tick, input};
const auto bytes = encode_cellular_mutation_input(record);
@@ -902,13 +888,16 @@ CellularRegionHost::select_mutation(const persistence::Uuid& account, std::uint6
return {admitted, tick, {}};
}
void CellularRegionHost::admit_held_selections() {
if (hold_selections_ || in_flight_)
return;
// The clients repeat their choices; a refusal here is answered on that repeat.
for (const auto& held : std::exchange(held_selections_, {}))
static_cast<void>(
select_mutation(held.account, held.generation, held.action_id, held.input));
std::vector<persistence::CellularMutationReservation>
CellularRegionHost::unbound_mutations() const {
std::vector<persistence::CellularMutationReservation> unbound;
for (const auto& queued : queued_mutations_)
if (!in_flight_ ||
std::ranges::none_of(in_flight_->mutation_actions, [&](const auto& bound) {
return bound.account == queued.account && bound.action_id == queued.action_id;
}))
unbound.push_back(queued);
return unbound;
}
std::expected<persistence::PublishedCellularBatch, CellularPublishFailure>
+23 -8
View File
@@ -606,11 +606,10 @@ match_cellular_mutation_actions_to_trace(const CellularLives& predecessor,
const auto life = std::ranges::find(
predecessor.lives(), selection.life_id,
[](const CellularOwnedLife& value) { return value.identity.life; });
if (life == predecessor.lives().end())
return std::unexpected{CellularActionBlockError::mismatch};
// A life that ended before this batch: its queued choice names it directly.
const auto matches = std::ranges::count_if(actions, [&](const auto& action) {
return action.account == life->account && action.tick == tick &&
action.input.life_id == selection.life_id &&
return (life == predecessor.lives().end() || action.account == life->account) &&
action.tick == tick && action.input.life_id == selection.life_id &&
action.input.event_index == selection.event_index &&
action.input.offer_tick == selection.offer_tick &&
action.input.operator_id == selection.operator_id;
@@ -620,13 +619,29 @@ match_cellular_mutation_actions_to_trace(const CellularLives& predecessor,
}
}
for (const auto& action : actions) {
const auto life =
std::ranges::find(predecessor.lives(), action.account, &CellularOwnedLife::account);
if (life == predecessor.lives().end() ||
(predecessor.format_version() < cellular_lives_revision_format))
if (predecessor.format_version() < cellular_lives_revision_format)
return std::unexpected{CellularActionBlockError::mismatch};
const auto tick_index = static_cast<std::size_t>(action.tick - batch.header.begin_tick - 1);
const auto& mutations = batch.ticks[tick_index].mutations;
const auto named =
std::ranges::find(predecessor.lives(), action.input.life_id,
[](const CellularOwnedLife& value) { return value.identity.life; });
if (named == predecessor.lives().end()) {
// ADR-0121 amendment (schema 59): the life ended before this batch; the owner
// records life_died, so only the selection itself must match.
const auto departed = std::ranges::lower_bound(mutations, action.input.life_id, {},
&CellularMutationSelection::life_id);
if (departed == mutations.end() || departed->life_id != action.input.life_id ||
departed->event_index != action.input.event_index ||
departed->offer_tick != action.input.offer_tick ||
departed->operator_id != action.input.operator_id)
return std::unexpected{CellularActionBlockError::mismatch};
continue;
}
const auto life =
std::ranges::find(predecessor.lives(), action.account, &CellularOwnedLife::account);
if (life == predecessor.lives().end())
return std::unexpected{CellularActionBlockError::mismatch};
const auto selection = std::ranges::lower_bound(mutations, life->identity.life, {},
&CellularMutationSelection::life_id);
if (!life->mutation.offer)
+8 -2
View File
@@ -1912,8 +1912,14 @@ CellularLives::advance_tick_with_mutations(std::span<const CellularLifeIntent> i
const auto life = std::ranges::lower_bound(
state_->lives, selection.life_id, {},
[](const CellularOwnedLife& value) { return value.identity.life; });
if (life == state_->lives.end() || life->identity.life != selection.life_id ||
life->mutation.event_index != selection.event_index ||
// ADR-0121 amendment (schema 59): a queued choice can outlive its life; it binds
// with the life_died result and changes nothing.
if (life == state_->lives.end() || life->identity.life != selection.life_id) {
if (state_->archive.find(selection.successor_id))
return std::unexpected{CellularLivesError::mutation};
continue;
}
if (life->mutation.event_index != selection.event_index ||
life->mutation.offer_tick != selection.offer_tick)
return std::unexpected{CellularLivesError::mutation};
if (!life->mutation.offer.has_value())
+19 -13
View File
@@ -102,23 +102,29 @@ prepare_cellular_session_tick(const CellularLives& state, std::uint64_t tick,
const auto event_index = little_endian(bytes, 16, 8);
const auto offer_tick = little_endian(bytes, 24, 8);
const auto operator_id = static_cast<std::uint16_t>(little_endian(bytes, 32, 2));
// ADR-0121 amendment (schema 59): the reservation's tick is the earliest the choice
// may bind at; the queue binds it at the first tick simulated after admission.
if (reservation.account == persistence::Uuid{} || reservation.action_id == 0 ||
reservation.region != state.region() || reservation.generation == 0 ||
reservation.successor_identity == 0 || target_tick != tick ||
reservation.target_tick != tick || life_id == 0 || event_index == 0 ||
offer_tick == 0 || offer_tick >= tick)
reservation.successor_identity == 0 || target_tick != reservation.target_tick ||
reservation.target_tick > tick || life_id == 0 || event_index == 0 || offer_tick == 0 ||
offer_tick >= reservation.target_tick)
return std::unexpected{CellularSessionBatchError::mutation};
// A life that ended while its choice waited binds the choice as life_died.
const auto life =
std::ranges::find(state.lives(), reservation.account, &CellularOwnedLife::account);
if (life == state.lives().end() || life->identity.life != life_id ||
life->mutation.event_index != event_index || life->mutation.offer_tick != offer_tick)
return std::unexpected{CellularSessionBatchError::mutation};
const auto& offer = life->mutation.offer;
if (!offer || (operator_id != 0 && // Operator 0 declines the offer.
!std::ranges::any_of(offer->candidates, [&](const auto& candidate) {
return candidate.operator_id == operator_id;
})))
return std::unexpected{CellularSessionBatchError::mutation};
std::ranges::find(state.lives(), life_id,
[](const CellularOwnedLife& value) { return value.identity.life; });
if (life != state.lives().end()) {
if (life->account != reservation.account || life->mutation.event_index != event_index ||
life->mutation.offer_tick != offer_tick)
return std::unexpected{CellularSessionBatchError::mutation};
const auto& offer = life->mutation.offer;
if (!offer || (operator_id != 0 && // Operator 0 declines the offer.
!std::ranges::any_of(offer->candidates, [&](const auto& candidate) {
return candidate.operator_id == operator_id;
})))
return std::unexpected{CellularSessionBatchError::mutation};
}
if (std::ranges::any_of(prepared.intents.mutations,
[&](const auto& previous) {