M12.3: publish and recover durable exact populations
Foundation / checks (ecs) (push) Successful in 2m41s
Foundation / checks (tsan) (push) Failing after 18m12s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (network) (push) Canceled after 20m41s
Foundation / checks (postgres) (push) Canceled after 20m43s
Foundation / checks (checks) (push) Canceled after 20m49s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (client) (push) Failing after 17m4s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / checks (ecs) (push) Successful in 2m41s
Foundation / checks (tsan) (push) Failing after 18m12s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (network) (push) Canceled after 20m41s
Foundation / checks (postgres) (push) Canceled after 20m43s
Foundation / checks (checks) (push) Canceled after 20m49s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (client) (push) Failing after 17m4s
Foundation / postgres-tools (push) Canceled after 0s
This commit is contained in:
@@ -12,10 +12,11 @@
|
||||
namespace evo::server {
|
||||
struct CellularRegionHost::Pending {
|
||||
persistence::CellularPreparedCommit prepared;
|
||||
std::unique_ptr<CellularLives> candidate;
|
||||
OwnedState candidate;
|
||||
std::vector<CellularActionRecord> actions;
|
||||
std::vector<CellularTickIntents::Birth> births;
|
||||
std::vector<CellularMutationActionRecord> mutation_actions;
|
||||
std::optional<persistence::PopulationPreparedCommit> population_prepared{};
|
||||
};
|
||||
|
||||
namespace {
|
||||
@@ -196,34 +197,72 @@ void CellularRegionHost::acquire() {
|
||||
status_.phase = HostPhase::quarantined;
|
||||
return;
|
||||
}
|
||||
auto recovered = persistence::recover_cellular_checkpoint(*db_, *objects_, config_.key, *lease,
|
||||
config_.retained_builds,
|
||||
config_.lease.commit_timeout);
|
||||
if (!recovered) {
|
||||
if (recovered.error().kind == persistence::RecoveryError::store) {
|
||||
db_.reset();
|
||||
++status_.store_failures;
|
||||
return;
|
||||
}
|
||||
authority_.fence();
|
||||
status_.phase = HostPhase::quarantined;
|
||||
if (!recover_state(*lease) || !committed_.has_value() || !state_)
|
||||
return;
|
||||
}
|
||||
state_ = std::move(recovered->state);
|
||||
committed_ = std::move(recovered->committed);
|
||||
queued_mutations_.clear();
|
||||
mutations_uncertain_ = true; // Reserved choices are reloaded under the new lease.
|
||||
// ADR-0127: every founder corpse is an ended life; record any ending a crash lost.
|
||||
mutations_uncertain_ = true;
|
||||
pending_endings_.clear();
|
||||
for (const auto& corpse : state_->corpses())
|
||||
if (corpse.parent_life == 0)
|
||||
pending_endings_.push_back({committed_->head.commit, corpse.life_id});
|
||||
// Controlled cells cannot die in unattended profile 5; their earlier raw
|
||||
// endings were drained before entry. Do not invent a new death commit.
|
||||
if (!population_mode())
|
||||
for (const auto& corpse : state_->corpses())
|
||||
if (corpse.parent_life == 0)
|
||||
pending_endings_.push_back({committed_->head.commit, corpse.life_id});
|
||||
status_.phase = HostPhase::running;
|
||||
status_.simulated_tick = committed_->head.end_tick;
|
||||
status_.committed_tick = committed_->head.end_tick;
|
||||
status_.committed_sequence = committed_->head.sequence;
|
||||
}
|
||||
|
||||
bool CellularRegionHost::recover_state(const persistence::LeaseGrant& lease) {
|
||||
auto head =
|
||||
persistence::load_region_checkpoint(*db_, config_.key, config_.lease.commit_timeout);
|
||||
if (!head) {
|
||||
db_.reset();
|
||||
++status_.store_failures;
|
||||
return false;
|
||||
}
|
||||
const auto manifest = persistence::decode_manifest(head->manifest);
|
||||
if (!manifest) {
|
||||
authority_.fence();
|
||||
status_.phase = HostPhase::quarantined;
|
||||
return false;
|
||||
}
|
||||
if (manifest->profile == persistence::CheckpointProfile::cellular_population_v1) {
|
||||
auto recovered = persistence::recover_population_checkpoint(*db_, *objects_, config_.key,
|
||||
lease, config_.retained_builds,
|
||||
config_.lease.commit_timeout);
|
||||
if (!recovered) {
|
||||
if (recovered.error().kind == persistence::RecoveryError::store) {
|
||||
db_.reset();
|
||||
++status_.store_failures;
|
||||
} else {
|
||||
authority_.fence();
|
||||
status_.phase = HostPhase::quarantined;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
state_ = std::move(recovered->state);
|
||||
committed_ = std::move(recovered->committed);
|
||||
return true;
|
||||
}
|
||||
auto recovered = persistence::recover_cellular_checkpoint(
|
||||
*db_, *objects_, config_.key, lease, config_.retained_builds, config_.lease.commit_timeout);
|
||||
if (!recovered) {
|
||||
if (recovered.error().kind == persistence::RecoveryError::store) {
|
||||
db_.reset();
|
||||
++status_.store_failures;
|
||||
return false;
|
||||
}
|
||||
authority_.fence();
|
||||
status_.phase = HostPhase::quarantined;
|
||||
return false;
|
||||
}
|
||||
state_ = std::move(recovered->state);
|
||||
committed_ = std::move(recovered->committed);
|
||||
return true;
|
||||
}
|
||||
|
||||
void CellularRegionHost::upkeep() {
|
||||
if (authority_.state(LeaseClock::now()) == persistence::LeaseState::fenced) {
|
||||
authority_.fence();
|
||||
@@ -280,8 +319,12 @@ void CellularRegionHost::resolve() {
|
||||
++status_.fences;
|
||||
return;
|
||||
}
|
||||
const auto resolved = persistence::resolve_cellular_prepared(*db_, pending_->prepared,
|
||||
config_.lease.commit_timeout);
|
||||
const auto& population_prepared = pending_->population_prepared;
|
||||
const auto resolved = population_prepared.has_value()
|
||||
? persistence::resolve_population_prepared(
|
||||
*db_, *population_prepared, config_.lease.commit_timeout)
|
||||
: persistence::resolve_cellular_prepared(
|
||||
*db_, pending_->prepared, config_.lease.commit_timeout);
|
||||
if (!resolved) {
|
||||
if (resolved.error().kind == LeaseStoreError::fenced) {
|
||||
authority_.fence();
|
||||
@@ -309,8 +352,18 @@ void CellularRegionHost::resolve() {
|
||||
if (resolved->status == persistence::Resolution::absent &&
|
||||
!authority_.may_begin_commit(LeaseClock::now()))
|
||||
return;
|
||||
auto retried = persistence::retry_cellular_prepared(*db_, pending_->prepared,
|
||||
auto retried = [&]() -> std::expected<persistence::CommittedCheckpoint,
|
||||
persistence::CellularPublicationFailure> {
|
||||
if (!pending_->population_prepared)
|
||||
return persistence::retry_cellular_prepared(*db_, pending_->prepared,
|
||||
config_.lease.commit_timeout);
|
||||
auto result = persistence::retry_population_prepared(*db_, *pending_->population_prepared,
|
||||
config_.lease.commit_timeout);
|
||||
if (!result)
|
||||
return std::unexpected(persistence::CellularPublicationFailure{
|
||||
result.error().kind, result.error().store, result.error().object, {}});
|
||||
return std::move(*result);
|
||||
}();
|
||||
if (!retried) {
|
||||
if (retried.error().kind == persistence::PublicationError::unknown) {
|
||||
db_.reset();
|
||||
@@ -483,6 +536,8 @@ CellularRegionHost::select_mutation(const persistence::Uuid& account, std::uint6
|
||||
if (!status_.ready || mutations_uncertain_ || pending_ || !state_ || !committed_ ||
|
||||
!authority_.may_begin_commit(LeaseClock::now()) || !database())
|
||||
return {not_ready, 0, {}};
|
||||
if (population_mode() && !set_population_mode(false))
|
||||
return {not_ready, 0, {}};
|
||||
const auto tick = committed_->head.end_tick + 1;
|
||||
const CellularMutationActionRecord record{account, action_id, tick, input};
|
||||
const auto bytes = encode_cellular_mutation_input(record);
|
||||
@@ -552,6 +607,12 @@ CellularRegionHost::publish(const CellularBatch& batch,
|
||||
if (!status_.ready || mutations_uncertain_ || !state_ || !committed_ || !grant ||
|
||||
!authority_.may_begin_commit(LeaseClock::now()))
|
||||
return std::unexpected{CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}}};
|
||||
if (population_mode()) {
|
||||
if (!actions.empty() || !mutation_actions.empty())
|
||||
return std::unexpected(
|
||||
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}});
|
||||
return publish_population(batch);
|
||||
}
|
||||
auto candidate = replay_cellular_batch(*state_, batch);
|
||||
if (!candidate)
|
||||
return std::unexpected{
|
||||
@@ -722,6 +783,8 @@ network::PrimitiveLifeOutcome
|
||||
CellularRegionHost::create_primitive_life(const persistence::Uuid& account,
|
||||
std::uint64_t generation, std::uint64_t request) {
|
||||
using network::PrimitiveLifeResult;
|
||||
if (population_mode() && !set_population_mode(false))
|
||||
return {request, PrimitiveLifeResult::unknown, 0, 0};
|
||||
const auto existing = query_birth(account, generation, request, true);
|
||||
if (existing.result != PrimitiveLifeResult::unknown)
|
||||
return existing;
|
||||
@@ -754,6 +817,8 @@ CellularRegionHost::create_primitive_life(const persistence::Uuid& account,
|
||||
network::PrimitiveLifeOutcome
|
||||
CellularRegionHost::query_primitive_life(const persistence::Uuid& account, std::uint64_t generation,
|
||||
std::uint64_t request) {
|
||||
if (population_mode() && !set_population_mode(false))
|
||||
return {request, network::PrimitiveLifeResult::unknown, 0, 0};
|
||||
return query_birth(account, generation, request, true);
|
||||
}
|
||||
|
||||
@@ -784,4 +849,126 @@ std::optional<persistence::LeaseGrant> CellularRegionHost::authority() const {
|
||||
return std::nullopt;
|
||||
return authority_.grant();
|
||||
}
|
||||
bool CellularRegionHost::population_mode() const noexcept {
|
||||
return state_.population != nullptr;
|
||||
}
|
||||
|
||||
std::expected<persistence::CommittedCheckpoint, CellularPublishFailure>
|
||||
CellularRegionHost::accept_population_publication(
|
||||
std::expected<persistence::CommittedCheckpoint, persistence::PopulationPublicationFailure>
|
||||
result,
|
||||
OwnedState candidate) {
|
||||
if (!result) {
|
||||
const auto& error = result.error();
|
||||
if (error.kind == persistence::PublicationError::unknown) {
|
||||
pending_ = std::make_unique<Pending>(
|
||||
Pending{{}, std::move(candidate), {}, {}, {}, error.prepared});
|
||||
status_.phase = HostPhase::resolving;
|
||||
status_.ready = false;
|
||||
++status_.unknown_outcomes;
|
||||
db_.reset();
|
||||
} else if (error.store.kind == LeaseStoreError::fenced) {
|
||||
authority_.fence();
|
||||
state_.reset();
|
||||
committed_.reset();
|
||||
status_.phase = HostPhase::acquiring;
|
||||
status_.ready = false;
|
||||
++status_.fences;
|
||||
} else if (error.kind == persistence::PublicationError::predecessor ||
|
||||
error.kind == persistence::PublicationError::verify ||
|
||||
error.kind == persistence::PublicationError::receipt) {
|
||||
authority_.fence();
|
||||
state_.reset();
|
||||
committed_.reset();
|
||||
status_.phase = HostPhase::quarantined;
|
||||
status_.ready = false;
|
||||
} else {
|
||||
mutations_uncertain_ = true;
|
||||
}
|
||||
return std::unexpected(CellularPublishFailure{CellularHostError::publication, error.kind,
|
||||
error.store, error.object});
|
||||
}
|
||||
note_endings(*state_, *candidate, result->head.commit);
|
||||
state_ = std::move(candidate);
|
||||
committed_ = *result;
|
||||
++status_.commits;
|
||||
status_.committed_sequence = committed_->head.sequence;
|
||||
status_.committed_tick = status_.simulated_tick = committed_->head.end_tick;
|
||||
refresh();
|
||||
return std::move(*result);
|
||||
}
|
||||
|
||||
std::expected<void, CellularPublishFailure> CellularRegionHost::set_population_mode(bool enabled) {
|
||||
if (!status_.ready || pending_ || mutations_uncertain_ || !state_ || !committed_.has_value() ||
|
||||
!authority_.may_begin_commit(LeaseClock::now()) || !database())
|
||||
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
||||
if (population_mode() == enabled)
|
||||
return {};
|
||||
if (enabled &&
|
||||
(!config_.population_policy || !queued_mutations_.empty() || !pending_endings_.empty() ||
|
||||
state_->rules_version() != cellular_lives_foraging_rules))
|
||||
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
||||
const auto policy = config_.population_policy.value_or(CellularTierPolicy{});
|
||||
std::array<std::uint8_t, 10> marker{};
|
||||
marker[0] = enabled ? 1 : 0;
|
||||
marker[1] = policy.automatic ? 1 : 0;
|
||||
for (unsigned i = 0; i < 4; ++i) {
|
||||
marker[2 + i] = static_cast<std::uint8_t>(policy.materialize_radius_um >> (i * 8U));
|
||||
marker[6 + i] = static_cast<std::uint8_t>(policy.collapse_radius_um >> (i * 8U));
|
||||
}
|
||||
const auto id = commit_id(*committed_, marker, {});
|
||||
auto raw = CellularLives::restore(state_->snapshot(), state_->world(), state_->region(),
|
||||
state_->frame(), state_->tick());
|
||||
if (!id || !raw)
|
||||
return std::unexpected(
|
||||
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}});
|
||||
OwnedState candidate;
|
||||
if (enabled) {
|
||||
auto population = CellularPopulation::create(std::move(*raw), policy);
|
||||
if (!population)
|
||||
return std::unexpected(
|
||||
CellularPublishFailure{CellularHostError::configuration, {}, {}, {}});
|
||||
candidate = std::move(*population);
|
||||
} else
|
||||
candidate = std::move(*raw);
|
||||
const auto grant = authority_.grant();
|
||||
if (!grant.has_value())
|
||||
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
||||
auto result =
|
||||
enabled ? persistence::migrate_cellular_to_population(*db_, *objects_, *grant, *committed_,
|
||||
*id, config_.build, policy,
|
||||
config_.lease.commit_timeout)
|
||||
: persistence::materialize_population_profile(*db_, *objects_, *grant, *committed_,
|
||||
*id, config_.build,
|
||||
config_.lease.commit_timeout);
|
||||
auto accepted = accept_population_publication(std::move(result), std::move(candidate));
|
||||
if (!accepted)
|
||||
return std::unexpected(accepted.error());
|
||||
return {};
|
||||
}
|
||||
|
||||
std::expected<persistence::PublishedCellularBatch, CellularPublishFailure>
|
||||
CellularRegionHost::publish_population(const CellularBatch& batch) {
|
||||
const auto grant = authority_.grant();
|
||||
if (!state_.population || !queued_mutations_.empty() || !committed_.has_value() ||
|
||||
!grant.has_value() || !database())
|
||||
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
||||
auto execution = prepare_population_batch(*state_.population, batch);
|
||||
if (!execution)
|
||||
return std::unexpected(
|
||||
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}});
|
||||
const auto trace = encode_population_batch(execution->batch);
|
||||
const auto id = trace ? commit_id(*committed_, *trace, {}) : std::nullopt;
|
||||
if (!id)
|
||||
return std::unexpected(
|
||||
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}});
|
||||
auto result = persistence::publish_population_batch(
|
||||
*db_, *objects_, *grant, *committed_, execution->batch, *execution->state, *id,
|
||||
config_.build, config_.lease.commit_timeout);
|
||||
auto accepted =
|
||||
accept_population_publication(std::move(result), OwnedState(std::move(execution->state)));
|
||||
if (!accepted)
|
||||
return std::unexpected(accepted.error());
|
||||
return persistence::PublishedCellularBatch{std::move(*accepted), {}};
|
||||
}
|
||||
} // namespace evo::server
|
||||
|
||||
Reference in New Issue
Block a user