M12.3: publish and recover durable exact populations
Foundation / checks (ecs) (push) Successful in 2m41s
Foundation / checks (tsan) (push) Failing after 18m12s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (network) (push) Canceled after 20m41s
Foundation / checks (postgres) (push) Canceled after 20m43s
Foundation / checks (checks) (push) Canceled after 20m49s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (client) (push) Failing after 17m4s
Foundation / postgres-tools (push) Canceled after 0s

This commit is contained in:
dingo
2026-10-17 17:10:17 -04:00
parent d5d10a0cc7
commit 627411983f
35 changed files with 2637 additions and 102 deletions
+210 -23
View File
@@ -12,10 +12,11 @@
namespace evo::server {
struct CellularRegionHost::Pending {
persistence::CellularPreparedCommit prepared;
std::unique_ptr<CellularLives> candidate;
OwnedState candidate;
std::vector<CellularActionRecord> actions;
std::vector<CellularTickIntents::Birth> births;
std::vector<CellularMutationActionRecord> mutation_actions;
std::optional<persistence::PopulationPreparedCommit> population_prepared{};
};
namespace {
@@ -196,34 +197,72 @@ void CellularRegionHost::acquire() {
status_.phase = HostPhase::quarantined;
return;
}
auto recovered = persistence::recover_cellular_checkpoint(*db_, *objects_, config_.key, *lease,
config_.retained_builds,
config_.lease.commit_timeout);
if (!recovered) {
if (recovered.error().kind == persistence::RecoveryError::store) {
db_.reset();
++status_.store_failures;
return;
}
authority_.fence();
status_.phase = HostPhase::quarantined;
if (!recover_state(*lease) || !committed_.has_value() || !state_)
return;
}
state_ = std::move(recovered->state);
committed_ = std::move(recovered->committed);
queued_mutations_.clear();
mutations_uncertain_ = true; // Reserved choices are reloaded under the new lease.
// ADR-0127: every founder corpse is an ended life; record any ending a crash lost.
mutations_uncertain_ = true;
pending_endings_.clear();
for (const auto& corpse : state_->corpses())
if (corpse.parent_life == 0)
pending_endings_.push_back({committed_->head.commit, corpse.life_id});
// Controlled cells cannot die in unattended profile 5; their earlier raw
// endings were drained before entry. Do not invent a new death commit.
if (!population_mode())
for (const auto& corpse : state_->corpses())
if (corpse.parent_life == 0)
pending_endings_.push_back({committed_->head.commit, corpse.life_id});
status_.phase = HostPhase::running;
status_.simulated_tick = committed_->head.end_tick;
status_.committed_tick = committed_->head.end_tick;
status_.committed_sequence = committed_->head.sequence;
}
bool CellularRegionHost::recover_state(const persistence::LeaseGrant& lease) {
auto head =
persistence::load_region_checkpoint(*db_, config_.key, config_.lease.commit_timeout);
if (!head) {
db_.reset();
++status_.store_failures;
return false;
}
const auto manifest = persistence::decode_manifest(head->manifest);
if (!manifest) {
authority_.fence();
status_.phase = HostPhase::quarantined;
return false;
}
if (manifest->profile == persistence::CheckpointProfile::cellular_population_v1) {
auto recovered = persistence::recover_population_checkpoint(*db_, *objects_, config_.key,
lease, config_.retained_builds,
config_.lease.commit_timeout);
if (!recovered) {
if (recovered.error().kind == persistence::RecoveryError::store) {
db_.reset();
++status_.store_failures;
} else {
authority_.fence();
status_.phase = HostPhase::quarantined;
}
return false;
}
state_ = std::move(recovered->state);
committed_ = std::move(recovered->committed);
return true;
}
auto recovered = persistence::recover_cellular_checkpoint(
*db_, *objects_, config_.key, lease, config_.retained_builds, config_.lease.commit_timeout);
if (!recovered) {
if (recovered.error().kind == persistence::RecoveryError::store) {
db_.reset();
++status_.store_failures;
return false;
}
authority_.fence();
status_.phase = HostPhase::quarantined;
return false;
}
state_ = std::move(recovered->state);
committed_ = std::move(recovered->committed);
return true;
}
void CellularRegionHost::upkeep() {
if (authority_.state(LeaseClock::now()) == persistence::LeaseState::fenced) {
authority_.fence();
@@ -280,8 +319,12 @@ void CellularRegionHost::resolve() {
++status_.fences;
return;
}
const auto resolved = persistence::resolve_cellular_prepared(*db_, pending_->prepared,
config_.lease.commit_timeout);
const auto& population_prepared = pending_->population_prepared;
const auto resolved = population_prepared.has_value()
? persistence::resolve_population_prepared(
*db_, *population_prepared, config_.lease.commit_timeout)
: persistence::resolve_cellular_prepared(
*db_, pending_->prepared, config_.lease.commit_timeout);
if (!resolved) {
if (resolved.error().kind == LeaseStoreError::fenced) {
authority_.fence();
@@ -309,8 +352,18 @@ void CellularRegionHost::resolve() {
if (resolved->status == persistence::Resolution::absent &&
!authority_.may_begin_commit(LeaseClock::now()))
return;
auto retried = persistence::retry_cellular_prepared(*db_, pending_->prepared,
auto retried = [&]() -> std::expected<persistence::CommittedCheckpoint,
persistence::CellularPublicationFailure> {
if (!pending_->population_prepared)
return persistence::retry_cellular_prepared(*db_, pending_->prepared,
config_.lease.commit_timeout);
auto result = persistence::retry_population_prepared(*db_, *pending_->population_prepared,
config_.lease.commit_timeout);
if (!result)
return std::unexpected(persistence::CellularPublicationFailure{
result.error().kind, result.error().store, result.error().object, {}});
return std::move(*result);
}();
if (!retried) {
if (retried.error().kind == persistence::PublicationError::unknown) {
db_.reset();
@@ -483,6 +536,8 @@ CellularRegionHost::select_mutation(const persistence::Uuid& account, std::uint6
if (!status_.ready || mutations_uncertain_ || pending_ || !state_ || !committed_ ||
!authority_.may_begin_commit(LeaseClock::now()) || !database())
return {not_ready, 0, {}};
if (population_mode() && !set_population_mode(false))
return {not_ready, 0, {}};
const auto tick = committed_->head.end_tick + 1;
const CellularMutationActionRecord record{account, action_id, tick, input};
const auto bytes = encode_cellular_mutation_input(record);
@@ -552,6 +607,12 @@ CellularRegionHost::publish(const CellularBatch& batch,
if (!status_.ready || mutations_uncertain_ || !state_ || !committed_ || !grant ||
!authority_.may_begin_commit(LeaseClock::now()))
return std::unexpected{CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}};
if (population_mode()) {
if (!actions.empty() || !mutation_actions.empty())
return std::unexpected(
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}});
return publish_population(batch);
}
auto candidate = replay_cellular_batch(*state_, batch);
if (!candidate)
return std::unexpected{
@@ -722,6 +783,8 @@ network::PrimitiveLifeOutcome
CellularRegionHost::create_primitive_life(const persistence::Uuid& account,
std::uint64_t generation, std::uint64_t request) {
using network::PrimitiveLifeResult;
if (population_mode() && !set_population_mode(false))
return {request, PrimitiveLifeResult::unknown, 0, 0};
const auto existing = query_birth(account, generation, request, true);
if (existing.result != PrimitiveLifeResult::unknown)
return existing;
@@ -754,6 +817,8 @@ CellularRegionHost::create_primitive_life(const persistence::Uuid& account,
network::PrimitiveLifeOutcome
CellularRegionHost::query_primitive_life(const persistence::Uuid& account, std::uint64_t generation,
std::uint64_t request) {
if (population_mode() && !set_population_mode(false))
return {request, network::PrimitiveLifeResult::unknown, 0, 0};
return query_birth(account, generation, request, true);
}
@@ -784,4 +849,126 @@ std::optional<persistence::LeaseGrant> CellularRegionHost::authority() const {
return std::nullopt;
return authority_.grant();
}
bool CellularRegionHost::population_mode() const noexcept {
return state_.population != nullptr;
}
std::expected<persistence::CommittedCheckpoint, CellularPublishFailure>
CellularRegionHost::accept_population_publication(
std::expected<persistence::CommittedCheckpoint, persistence::PopulationPublicationFailure>
result,
OwnedState candidate) {
if (!result) {
const auto& error = result.error();
if (error.kind == persistence::PublicationError::unknown) {
pending_ = std::make_unique<Pending>(
Pending{{}, std::move(candidate), {}, {}, {}, error.prepared});
status_.phase = HostPhase::resolving;
status_.ready = false;
++status_.unknown_outcomes;
db_.reset();
} else if (error.store.kind == LeaseStoreError::fenced) {
authority_.fence();
state_.reset();
committed_.reset();
status_.phase = HostPhase::acquiring;
status_.ready = false;
++status_.fences;
} else if (error.kind == persistence::PublicationError::predecessor ||
error.kind == persistence::PublicationError::verify ||
error.kind == persistence::PublicationError::receipt) {
authority_.fence();
state_.reset();
committed_.reset();
status_.phase = HostPhase::quarantined;
status_.ready = false;
} else {
mutations_uncertain_ = true;
}
return std::unexpected(CellularPublishFailure{CellularHostError::publication, error.kind,
error.store, error.object});
}
note_endings(*state_, *candidate, result->head.commit);
state_ = std::move(candidate);
committed_ = *result;
++status_.commits;
status_.committed_sequence = committed_->head.sequence;
status_.committed_tick = status_.simulated_tick = committed_->head.end_tick;
refresh();
return std::move(*result);
}
std::expected<void, CellularPublishFailure> CellularRegionHost::set_population_mode(bool enabled) {
if (!status_.ready || pending_ || mutations_uncertain_ || !state_ || !committed_.has_value() ||
!authority_.may_begin_commit(LeaseClock::now()) || !database())
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
if (population_mode() == enabled)
return {};
if (enabled &&
(!config_.population_policy || !queued_mutations_.empty() || !pending_endings_.empty() ||
state_->rules_version() != cellular_lives_foraging_rules))
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
const auto policy = config_.population_policy.value_or(CellularTierPolicy{});
std::array<std::uint8_t, 10> marker{};
marker[0] = enabled ? 1 : 0;
marker[1] = policy.automatic ? 1 : 0;
for (unsigned i = 0; i < 4; ++i) {
marker[2 + i] = static_cast<std::uint8_t>(policy.materialize_radius_um >> (i * 8U));
marker[6 + i] = static_cast<std::uint8_t>(policy.collapse_radius_um >> (i * 8U));
}
const auto id = commit_id(*committed_, marker, {});
auto raw = CellularLives::restore(state_->snapshot(), state_->world(), state_->region(),
state_->frame(), state_->tick());
if (!id || !raw)
return std::unexpected(
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}});
OwnedState candidate;
if (enabled) {
auto population = CellularPopulation::create(std::move(*raw), policy);
if (!population)
return std::unexpected(
CellularPublishFailure{CellularHostError::configuration, {}, {}, {}});
candidate = std::move(*population);
} else
candidate = std::move(*raw);
const auto grant = authority_.grant();
if (!grant.has_value())
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
auto result =
enabled ? persistence::migrate_cellular_to_population(*db_, *objects_, *grant, *committed_,
*id, config_.build, policy,
config_.lease.commit_timeout)
: persistence::materialize_population_profile(*db_, *objects_, *grant, *committed_,
*id, config_.build,
config_.lease.commit_timeout);
auto accepted = accept_population_publication(std::move(result), std::move(candidate));
if (!accepted)
return std::unexpected(accepted.error());
return {};
}
std::expected<persistence::PublishedCellularBatch, CellularPublishFailure>
CellularRegionHost::publish_population(const CellularBatch& batch) {
const auto grant = authority_.grant();
if (!state_.population || !queued_mutations_.empty() || !committed_.has_value() ||
!grant.has_value() || !database())
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
auto execution = prepare_population_batch(*state_.population, batch);
if (!execution)
return std::unexpected(
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}});
const auto trace = encode_population_batch(execution->batch);
const auto id = trace ? commit_id(*committed_, *trace, {}) : std::nullopt;
if (!id)
return std::unexpected(
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}});
auto result = persistence::publish_population_batch(
*db_, *objects_, *grant, *committed_, execution->batch, *execution->state, *id,
config_.build, config_.lease.commit_timeout);
auto accepted =
accept_population_publication(std::move(result), OwnedState(std::move(execution->state)));
if (!accepted)
return std::unexpected(accepted.error());
return persistence::PublishedCellularBatch{std::move(*accepted), {}};
}
} // namespace evo::server