feat: durable mutation queue (schema 59)
Foundation / checks (checks) (push) Successful in 5s
Foundation / checks (client) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 4s
Foundation / checks (network) (push) Successful in 5s
Foundation / checks (postgres) (push) Successful in 4s
Foundation / checks (tsan) (push) Successful in 4s
Foundation / checks (deploy) (push) Successful in 4m24s
Foundation / postgres-tools (push) Successful in 1m31s
Foundation / server-image (push) Successful in 4m25s
Foundation / image (push) Successful in 4m50s
Foundation / checks (checks) (push) Successful in 5s
Foundation / checks (client) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 4s
Foundation / checks (network) (push) Successful in 5s
Foundation / checks (postgres) (push) Successful in 4s
Foundation / checks (tsan) (push) Successful in 4s
Foundation / checks (deploy) (push) Successful in 4m24s
Foundation / postgres-tools (push) Successful in 1m31s
Foundation / server-image (push) Successful in 4m25s
Foundation / image (push) Successful in 4m50s
A mutation choice's reserved tick is now the earliest tick it may bind at, not the only one (ADR-0121 amendment). With write-behind commits, a choice reserved for head + 1 landed in ticks already being simulated or committed: the batch was refused, or the reservation was orphaned for a past tick and stalled the region. Schema 59: a bound record carries the tick it applied at and the rest of its input must equal the reservation's; the "omits a reserved mutation" refusals are removed. Existing rows are unchanged, so an orphaned reservation binds at the next commit that carries it. The host keeps every unbound reservation queued and the open batch binds those the commit in flight does not carry, at its next tick; the earlier hold on new choices is removed. A choice that outlives its life binds as life_died (owner, trace matching and recovery accept it when no life with that ID remains). Recovery takes a bound choice's tick from its trace. Tests: a choice skipped by one checkpoint binds a tick late and recovers; builder late binding, refusal before the earliest tick, and a departed life's life_died outcome. Full local PostgreSQL qualification passes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -6,9 +6,12 @@
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <functional>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace evo::persistence {
|
||||
namespace {
|
||||
@@ -64,9 +67,12 @@ mutation_results(const server::CellularLives& base,
|
||||
std::vector<std::uint8_t> results;
|
||||
results.reserve(actions.size());
|
||||
for (const auto& action : actions) {
|
||||
const auto life =
|
||||
std::ranges::find(base.lives(), action.account, &server::CellularOwnedLife::account);
|
||||
if (life == base.lives().end() || life->identity.life != action.input.life_id)
|
||||
// A life that ended before the batch (ADR-0121 amendment, schema 59) has no owner
|
||||
// entry; one still present must be the account's.
|
||||
const auto life = std::ranges::find(
|
||||
base.lives(), action.input.life_id,
|
||||
[](const server::CellularOwnedLife& value) { return value.identity.life; });
|
||||
if (life != base.lives().end() && life->account != action.account)
|
||||
return std::nullopt;
|
||||
const auto outcome = std::ranges::find_if(outcomes, [&](const auto& candidate) {
|
||||
return candidate.tick == action.tick && candidate.life_id == action.input.life_id;
|
||||
@@ -78,6 +84,23 @@ mutation_results(const server::CellularLives& base,
|
||||
return results;
|
||||
}
|
||||
|
||||
// The tick of the trace selection a bound choice names (its life, offer and operator).
|
||||
std::optional<std::uint64_t> applied_tick(const server::CellularBatch& trace,
|
||||
const server::CellularMutationActionRecord& action) {
|
||||
std::optional<std::uint64_t> found;
|
||||
for (std::size_t i = 0; i < trace.ticks.size(); ++i)
|
||||
for (const auto& selection : trace.ticks[i].mutations)
|
||||
if (selection.life_id == action.input.life_id &&
|
||||
selection.event_index == action.input.event_index &&
|
||||
selection.offer_tick == action.input.offer_tick &&
|
||||
selection.operator_id == action.input.operator_id) {
|
||||
if (found)
|
||||
return std::nullopt;
|
||||
found = trace.header.begin_tick + i + 1;
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
std::size_t selection_count(const server::CellularBatch& trace) {
|
||||
std::size_t count{};
|
||||
for (const auto& tick : trace.ticks)
|
||||
@@ -844,9 +867,34 @@ recover_cellular_loaded(Postgres& db, const FilesystemObjectStore& objects, Regi
|
||||
server::decode_cellular_mutation_input(row.account, row.action_id, row.input);
|
||||
if (!action)
|
||||
return fail_with(RecoveryError::history);
|
||||
// The stored input names the earliest tick; the choice bound at its selection's.
|
||||
const auto bound = applied_tick(*trace, *action);
|
||||
if (!bound || *bound < action->tick)
|
||||
return fail_with(RecoveryError::history);
|
||||
action->tick = *bound;
|
||||
mutation_actions.push_back(*action);
|
||||
stored_results.push_back(row.result);
|
||||
}
|
||||
// Results follow the block's canonical (tick, account, action) order.
|
||||
std::vector<std::size_t> order(mutation_actions.size());
|
||||
for (std::size_t i = 0; i < order.size(); ++i)
|
||||
order[i] = i;
|
||||
std::ranges::sort(order, [&](std::size_t left, std::size_t right) {
|
||||
const auto& a = mutation_actions[left];
|
||||
const auto& b = mutation_actions[right];
|
||||
return std::tie(a.tick, a.account, a.action_id) <
|
||||
std::tie(b.tick, b.account, b.action_id);
|
||||
});
|
||||
{
|
||||
std::vector<server::CellularMutationActionRecord> sorted_actions;
|
||||
std::vector<std::uint8_t> sorted_results;
|
||||
for (const auto i : order) {
|
||||
sorted_actions.push_back(mutation_actions[i]);
|
||||
sorted_results.push_back(stored_results[i]);
|
||||
}
|
||||
mutation_actions = std::move(sorted_actions);
|
||||
stored_results = std::move(sorted_results);
|
||||
}
|
||||
if (mutation_actions.size() != selection_count(*trace) ||
|
||||
(!mutation_actions.empty() &&
|
||||
(!server::encode_cellular_mutation_action_block(mutation_actions, manifest->begin_tick,
|
||||
|
||||
@@ -774,11 +774,11 @@ void CellularRegionHost::reload_mutations() {
|
||||
++status_.store_failures;
|
||||
return;
|
||||
}
|
||||
// Every reservation targets the tick after the head it was made at, and SQL refuses a
|
||||
// checkpoint that skips one, so an unbound choice is always for the next tick.
|
||||
// Schema 59: every unbound reservation waits in the queue; its tick is the earliest it
|
||||
// may bind at, which is never past the head's next tick.
|
||||
queued_mutations_.clear();
|
||||
for (const auto& reservation : *pending)
|
||||
if (reservation.target_tick == committed_->head.end_tick + 1)
|
||||
if (reservation.target_tick <= committed_->head.end_tick + 1)
|
||||
queued_mutations_.push_back(reservation);
|
||||
mutations_uncertain_ = false;
|
||||
}
|
||||
@@ -827,20 +827,6 @@ CellularRegionHost::select_mutation(const persistence::Uuid& account, std::uint6
|
||||
return {not_ready, 0, {}};
|
||||
if (population_mode() && !set_population_mode(false))
|
||||
return {not_ready, 0, {}};
|
||||
if (hold_selections_ || in_flight_) {
|
||||
// Already reserved: repeats keep their answer. Otherwise remember the latest choice
|
||||
// per account (bounded like a tick's lives) until the head is current.
|
||||
const auto queued = std::ranges::find(queued_mutations_, account,
|
||||
&persistence::CellularMutationReservation::account);
|
||||
if (queued == queued_mutations_.end()) {
|
||||
const auto held = std::ranges::find(held_selections_, account, &HeldSelection::account);
|
||||
if (held != held_selections_.end())
|
||||
*held = {account, generation, action_id, input};
|
||||
else if (held_selections_.size() < biology::max_cellular_tick_lives)
|
||||
held_selections_.push_back({account, generation, action_id, input});
|
||||
return {not_ready, 0, {}};
|
||||
}
|
||||
}
|
||||
const auto tick = committed_->head.end_tick + 1;
|
||||
const CellularMutationActionRecord record{account, action_id, tick, input};
|
||||
const auto bytes = encode_cellular_mutation_input(record);
|
||||
@@ -902,13 +888,16 @@ CellularRegionHost::select_mutation(const persistence::Uuid& account, std::uint6
|
||||
return {admitted, tick, {}};
|
||||
}
|
||||
|
||||
void CellularRegionHost::admit_held_selections() {
|
||||
if (hold_selections_ || in_flight_)
|
||||
return;
|
||||
// The clients repeat their choices; a refusal here is answered on that repeat.
|
||||
for (const auto& held : std::exchange(held_selections_, {}))
|
||||
static_cast<void>(
|
||||
select_mutation(held.account, held.generation, held.action_id, held.input));
|
||||
std::vector<persistence::CellularMutationReservation>
|
||||
CellularRegionHost::unbound_mutations() const {
|
||||
std::vector<persistence::CellularMutationReservation> unbound;
|
||||
for (const auto& queued : queued_mutations_)
|
||||
if (!in_flight_ ||
|
||||
std::ranges::none_of(in_flight_->mutation_actions, [&](const auto& bound) {
|
||||
return bound.account == queued.account && bound.action_id == queued.action_id;
|
||||
}))
|
||||
unbound.push_back(queued);
|
||||
return unbound;
|
||||
}
|
||||
|
||||
std::expected<persistence::PublishedCellularBatch, CellularPublishFailure>
|
||||
|
||||
@@ -606,11 +606,10 @@ match_cellular_mutation_actions_to_trace(const CellularLives& predecessor,
|
||||
const auto life = std::ranges::find(
|
||||
predecessor.lives(), selection.life_id,
|
||||
[](const CellularOwnedLife& value) { return value.identity.life; });
|
||||
if (life == predecessor.lives().end())
|
||||
return std::unexpected{CellularActionBlockError::mismatch};
|
||||
// A life that ended before this batch: its queued choice names it directly.
|
||||
const auto matches = std::ranges::count_if(actions, [&](const auto& action) {
|
||||
return action.account == life->account && action.tick == tick &&
|
||||
action.input.life_id == selection.life_id &&
|
||||
return (life == predecessor.lives().end() || action.account == life->account) &&
|
||||
action.tick == tick && action.input.life_id == selection.life_id &&
|
||||
action.input.event_index == selection.event_index &&
|
||||
action.input.offer_tick == selection.offer_tick &&
|
||||
action.input.operator_id == selection.operator_id;
|
||||
@@ -620,13 +619,29 @@ match_cellular_mutation_actions_to_trace(const CellularLives& predecessor,
|
||||
}
|
||||
}
|
||||
for (const auto& action : actions) {
|
||||
const auto life =
|
||||
std::ranges::find(predecessor.lives(), action.account, &CellularOwnedLife::account);
|
||||
if (life == predecessor.lives().end() ||
|
||||
(predecessor.format_version() < cellular_lives_revision_format))
|
||||
if (predecessor.format_version() < cellular_lives_revision_format)
|
||||
return std::unexpected{CellularActionBlockError::mismatch};
|
||||
const auto tick_index = static_cast<std::size_t>(action.tick - batch.header.begin_tick - 1);
|
||||
const auto& mutations = batch.ticks[tick_index].mutations;
|
||||
const auto named =
|
||||
std::ranges::find(predecessor.lives(), action.input.life_id,
|
||||
[](const CellularOwnedLife& value) { return value.identity.life; });
|
||||
if (named == predecessor.lives().end()) {
|
||||
// ADR-0121 amendment (schema 59): the life ended before this batch; the owner
|
||||
// records life_died, so only the selection itself must match.
|
||||
const auto departed = std::ranges::lower_bound(mutations, action.input.life_id, {},
|
||||
&CellularMutationSelection::life_id);
|
||||
if (departed == mutations.end() || departed->life_id != action.input.life_id ||
|
||||
departed->event_index != action.input.event_index ||
|
||||
departed->offer_tick != action.input.offer_tick ||
|
||||
departed->operator_id != action.input.operator_id)
|
||||
return std::unexpected{CellularActionBlockError::mismatch};
|
||||
continue;
|
||||
}
|
||||
const auto life =
|
||||
std::ranges::find(predecessor.lives(), action.account, &CellularOwnedLife::account);
|
||||
if (life == predecessor.lives().end())
|
||||
return std::unexpected{CellularActionBlockError::mismatch};
|
||||
const auto selection = std::ranges::lower_bound(mutations, life->identity.life, {},
|
||||
&CellularMutationSelection::life_id);
|
||||
if (!life->mutation.offer)
|
||||
|
||||
@@ -1912,8 +1912,14 @@ CellularLives::advance_tick_with_mutations(std::span<const CellularLifeIntent> i
|
||||
const auto life = std::ranges::lower_bound(
|
||||
state_->lives, selection.life_id, {},
|
||||
[](const CellularOwnedLife& value) { return value.identity.life; });
|
||||
if (life == state_->lives.end() || life->identity.life != selection.life_id ||
|
||||
life->mutation.event_index != selection.event_index ||
|
||||
// ADR-0121 amendment (schema 59): a queued choice can outlive its life; it binds
|
||||
// with the life_died result and changes nothing.
|
||||
if (life == state_->lives.end() || life->identity.life != selection.life_id) {
|
||||
if (state_->archive.find(selection.successor_id))
|
||||
return std::unexpected{CellularLivesError::mutation};
|
||||
continue;
|
||||
}
|
||||
if (life->mutation.event_index != selection.event_index ||
|
||||
life->mutation.offer_tick != selection.offer_tick)
|
||||
return std::unexpected{CellularLivesError::mutation};
|
||||
if (!life->mutation.offer.has_value())
|
||||
|
||||
@@ -102,23 +102,29 @@ prepare_cellular_session_tick(const CellularLives& state, std::uint64_t tick,
|
||||
const auto event_index = little_endian(bytes, 16, 8);
|
||||
const auto offer_tick = little_endian(bytes, 24, 8);
|
||||
const auto operator_id = static_cast<std::uint16_t>(little_endian(bytes, 32, 2));
|
||||
// ADR-0121 amendment (schema 59): the reservation's tick is the earliest the choice
|
||||
// may bind at; the queue binds it at the first tick simulated after admission.
|
||||
if (reservation.account == persistence::Uuid{} || reservation.action_id == 0 ||
|
||||
reservation.region != state.region() || reservation.generation == 0 ||
|
||||
reservation.successor_identity == 0 || target_tick != tick ||
|
||||
reservation.target_tick != tick || life_id == 0 || event_index == 0 ||
|
||||
offer_tick == 0 || offer_tick >= tick)
|
||||
reservation.successor_identity == 0 || target_tick != reservation.target_tick ||
|
||||
reservation.target_tick > tick || life_id == 0 || event_index == 0 || offer_tick == 0 ||
|
||||
offer_tick >= reservation.target_tick)
|
||||
return std::unexpected{CellularSessionBatchError::mutation};
|
||||
// A life that ended while its choice waited binds the choice as life_died.
|
||||
const auto life =
|
||||
std::ranges::find(state.lives(), reservation.account, &CellularOwnedLife::account);
|
||||
if (life == state.lives().end() || life->identity.life != life_id ||
|
||||
life->mutation.event_index != event_index || life->mutation.offer_tick != offer_tick)
|
||||
return std::unexpected{CellularSessionBatchError::mutation};
|
||||
const auto& offer = life->mutation.offer;
|
||||
if (!offer || (operator_id != 0 && // Operator 0 declines the offer.
|
||||
!std::ranges::any_of(offer->candidates, [&](const auto& candidate) {
|
||||
return candidate.operator_id == operator_id;
|
||||
})))
|
||||
return std::unexpected{CellularSessionBatchError::mutation};
|
||||
std::ranges::find(state.lives(), life_id,
|
||||
[](const CellularOwnedLife& value) { return value.identity.life; });
|
||||
if (life != state.lives().end()) {
|
||||
if (life->account != reservation.account || life->mutation.event_index != event_index ||
|
||||
life->mutation.offer_tick != offer_tick)
|
||||
return std::unexpected{CellularSessionBatchError::mutation};
|
||||
const auto& offer = life->mutation.offer;
|
||||
if (!offer || (operator_id != 0 && // Operator 0 declines the offer.
|
||||
!std::ranges::any_of(offer->candidates, [&](const auto& candidate) {
|
||||
return candidate.operator_id == operator_id;
|
||||
})))
|
||||
return std::unexpected{CellularSessionBatchError::mutation};
|
||||
}
|
||||
|
||||
if (std::ranges::any_of(prepared.intents.mutations,
|
||||
[&](const auto& previous) {
|
||||
|
||||
Reference in New Issue
Block a user