M12.4: retain and settle owned population intervals
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m28s
Foundation / checks (tsan) (push) Failing after 18m22s
Foundation / checks (network) (push) Successful in 20m38s
Foundation / checks (client) (push) Failing after 21m6s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (postgres) (push) Canceled after 46m54s
Foundation / checks (checks) (push) Canceled after 47m3s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m28s
Foundation / checks (tsan) (push) Failing after 18m22s
Foundation / checks (network) (push) Successful in 20m38s
Foundation / checks (client) (push) Failing after 21m6s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (postgres) (push) Canceled after 46m54s
Foundation / checks (checks) (push) Canceled after 47m3s
This commit is contained in:
@@ -199,15 +199,19 @@ void CellularRegionHost::acquire() {
|
||||
}
|
||||
if (!recover_state(*lease) || !committed_.has_value() || !state_)
|
||||
return;
|
||||
interval_reload_ = population_mode();
|
||||
admitted_interval_.reset();
|
||||
interval_request_.reset();
|
||||
interval_failure_.reset();
|
||||
queued_mutations_.clear();
|
||||
mutations_uncertain_ = true;
|
||||
pending_endings_.clear();
|
||||
// Controlled cells cannot die in unattended profile 5; their earlier raw
|
||||
// endings were drained before entry. Do not invent a new death commit.
|
||||
if (!population_mode())
|
||||
for (const auto& corpse : state_->corpses())
|
||||
if (corpse.parent_life == 0)
|
||||
pending_endings_.push_back({committed_->head.commit, corpse.life_id});
|
||||
// Reconcile retained owned deaths after either profile's recovery, including
|
||||
// older profile-5 records that admitted lingering cells. Existing endings
|
||||
// are idempotent; no ending is inferred for a living suspended cell.
|
||||
for (const auto& corpse : state_->corpses())
|
||||
if (corpse.parent_life == 0)
|
||||
pending_endings_.push_back({committed_->head.commit, corpse.life_id});
|
||||
status_.phase = HostPhase::running;
|
||||
status_.simulated_tick = committed_->head.end_tick;
|
||||
status_.committed_tick = committed_->head.end_tick;
|
||||
@@ -391,6 +395,7 @@ void CellularRegionHost::resolve() {
|
||||
note_endings(*state_, *pending_->candidate, retried->head.commit);
|
||||
state_ = std::move(pending_->candidate);
|
||||
committed_ = std::move(*retried);
|
||||
finish_interval();
|
||||
for (const auto& action : pending_->actions)
|
||||
finalized_actions_.push_back({action, committed_->head.sequence});
|
||||
finalize_births(pending_->births);
|
||||
@@ -406,7 +411,11 @@ void CellularRegionHost::resolve() {
|
||||
void CellularRegionHost::refresh() {
|
||||
const auto& grant = authority_.grant();
|
||||
status_.ready = status_.phase == HostPhase::running && state_ && committed_ && grant &&
|
||||
authority_.may_originate(LeaseClock::now());
|
||||
authority_.may_originate(LeaseClock::now()) && !interval_reload_ &&
|
||||
!interval_request_.has_value();
|
||||
const auto& interval = admitted_interval_;
|
||||
status_.ecological_pending_ticks =
|
||||
interval.has_value() ? interval->end_tick - interval->begin_tick : 0;
|
||||
status_.epoch = grant ? grant->epoch : 0;
|
||||
status_.lease_remaining_ms =
|
||||
status_.ready ? static_cast<std::uint64_t>(std::chrono::duration_cast<milliseconds>(
|
||||
@@ -424,6 +433,8 @@ const HostStatus& CellularRegionHost::step() {
|
||||
resolve();
|
||||
else if (!state_)
|
||||
acquire();
|
||||
if (!pending_ && state_ && status_.phase == HostPhase::running)
|
||||
reconcile_interval();
|
||||
if (!pending_ && state_ && mutations_uncertain_ && status_.phase == HostPhase::running)
|
||||
reload_mutations();
|
||||
if (!pending_ && state_ && status_.phase == HostPhase::running)
|
||||
@@ -839,6 +850,9 @@ void CellularRegionHost::drain() {
|
||||
authority_.fence();
|
||||
state_.reset();
|
||||
committed_.reset();
|
||||
admitted_interval_.reset();
|
||||
interval_request_.reset();
|
||||
interval_reload_ = false;
|
||||
status_.phase = HostPhase::stopped;
|
||||
status_.epoch = 0;
|
||||
status_.lease_remaining_ms = 0;
|
||||
@@ -891,6 +905,7 @@ CellularRegionHost::accept_population_publication(
|
||||
note_endings(*state_, *candidate, result->head.commit);
|
||||
state_ = std::move(candidate);
|
||||
committed_ = *result;
|
||||
finish_interval();
|
||||
++status_.commits;
|
||||
status_.committed_sequence = committed_->head.sequence;
|
||||
status_.committed_tick = status_.simulated_tick = committed_->head.end_tick;
|
||||
@@ -899,7 +914,8 @@ CellularRegionHost::accept_population_publication(
|
||||
}
|
||||
|
||||
std::expected<void, CellularPublishFailure> CellularRegionHost::set_population_mode(bool enabled) {
|
||||
if (!status_.ready || pending_ || mutations_uncertain_ || !state_ || !committed_.has_value() ||
|
||||
if (!status_.ready || pending_ || mutations_uncertain_ || admitted_interval_.has_value() ||
|
||||
interval_request_.has_value() || interval_reload_ || !state_ || !committed_.has_value() ||
|
||||
!authority_.may_begin_commit(LeaseClock::now()) || !database())
|
||||
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
||||
if (population_mode() == enabled)
|
||||
@@ -908,6 +924,8 @@ std::expected<void, CellularPublishFailure> CellularRegionHost::set_population_m
|
||||
(!config_.population_policy || !queued_mutations_.empty() || !pending_endings_.empty() ||
|
||||
state_->rules_version() != cellular_lives_foraging_rules))
|
||||
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
||||
if (enabled && !population_interval_safe(*state_))
|
||||
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
||||
const auto policy = config_.population_policy.value_or(CellularTierPolicy{});
|
||||
std::array<std::uint8_t, 10> marker{};
|
||||
marker[0] = enabled ? 1 : 0;
|
||||
@@ -971,4 +989,111 @@ CellularRegionHost::publish_population(const CellularBatch& batch) {
|
||||
return std::unexpected(accepted.error());
|
||||
return persistence::PublishedCellularBatch{std::move(*accepted), {}};
|
||||
}
|
||||
|
||||
void CellularRegionHost::finish_interval() {
|
||||
const auto& interval = admitted_interval_;
|
||||
if (interval.has_value() && committed_.has_value() &&
|
||||
committed_->head.end_tick == interval->end_tick)
|
||||
admitted_interval_.reset();
|
||||
}
|
||||
|
||||
void CellularRegionHost::reconcile_interval() {
|
||||
if (!interval_reload_ && !interval_request_.has_value())
|
||||
return;
|
||||
const auto grant = authority_.grant();
|
||||
if (!population_mode() || !committed_.has_value() || !grant.has_value() ||
|
||||
!authority_.may_begin_commit(LeaseClock::now()) || !database())
|
||||
return;
|
||||
auto result = [&]() -> std::expected<std::optional<persistence::PopulationInterval>,
|
||||
persistence::LeaseStoreFailure> {
|
||||
const auto& request = interval_request_;
|
||||
if (request.has_value()) {
|
||||
auto admitted = persistence::admit_population_interval(
|
||||
*db_, config_.key, *grant, request->source, request->end_tick,
|
||||
config_.lease.commit_timeout);
|
||||
if (!admitted)
|
||||
return std::unexpected(admitted.error());
|
||||
return std::optional{*admitted};
|
||||
}
|
||||
return persistence::load_population_interval(*db_, config_.key, *grant,
|
||||
config_.lease.commit_timeout);
|
||||
}();
|
||||
if (!result) {
|
||||
interval_failure_ = result.error();
|
||||
if (result.error().kind == LeaseStoreError::unknown ||
|
||||
result.error().kind == LeaseStoreError::store) {
|
||||
if (result.error().kind == LeaseStoreError::unknown)
|
||||
++status_.unknown_outcomes;
|
||||
else
|
||||
++status_.store_failures;
|
||||
db_.reset();
|
||||
return;
|
||||
}
|
||||
authority_.fence();
|
||||
state_.reset();
|
||||
committed_.reset();
|
||||
admitted_interval_.reset();
|
||||
interval_request_.reset();
|
||||
status_.ready = false;
|
||||
status_.phase = result.error().kind == LeaseStoreError::fenced ? HostPhase::acquiring
|
||||
: HostPhase::quarantined;
|
||||
++status_.fences;
|
||||
return;
|
||||
}
|
||||
auto interval = *result;
|
||||
if (interval.has_value() &&
|
||||
(interval->source_commit != committed_->head.commit ||
|
||||
interval->source_manifest_sha256 != committed_->head.digest ||
|
||||
interval->begin_tick != committed_->head.end_tick || interval->controller_version != 1)) {
|
||||
authority_.fence();
|
||||
status_.ready = false;
|
||||
status_.phase = HostPhase::quarantined;
|
||||
return;
|
||||
}
|
||||
admitted_interval_ = interval;
|
||||
interval_request_.reset();
|
||||
interval_failure_.reset();
|
||||
interval_reload_ = false;
|
||||
}
|
||||
|
||||
std::expected<std::optional<persistence::CommittedCheckpoint>, CellularPublishFailure>
|
||||
CellularRegionHost::advance_unattended(std::uint64_t due_ticks) {
|
||||
if (!status_.ready || !population_mode() || !committed_.has_value() || pending_ ||
|
||||
!authority_.may_begin_commit(LeaseClock::now()) || !population_interval_safe(*state_))
|
||||
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
||||
if (!admitted_interval_.has_value()) {
|
||||
const auto remaining = ~std::uint64_t{0} - committed_->head.end_tick;
|
||||
const auto count = std::min({due_ticks, remaining, std::uint64_t{64}});
|
||||
if (count == 0 || (due_ticks < 20 && count != remaining))
|
||||
return std::optional<persistence::CommittedCheckpoint>{};
|
||||
interval_request_ = IntervalRequest{committed_->head, committed_->head.end_tick + count};
|
||||
reconcile_interval();
|
||||
refresh();
|
||||
if (!status_.ready || !admitted_interval_.has_value())
|
||||
return std::unexpected(
|
||||
CellularPublishFailure{CellularHostError::publication, {}, interval_failure_, {}});
|
||||
}
|
||||
const auto interval = admitted_interval_;
|
||||
if (!interval.has_value())
|
||||
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
|
||||
const CellularBatchHeader header{config_.key.world,
|
||||
config_.key.region,
|
||||
config_.frame,
|
||||
interval->source_commit,
|
||||
interval->source_manifest_sha256,
|
||||
interval->begin_tick,
|
||||
interval->end_tick};
|
||||
const auto inputs = unattended_population_inputs(*state_, header);
|
||||
if (!inputs) {
|
||||
authority_.fence();
|
||||
status_.ready = false;
|
||||
status_.phase = HostPhase::quarantined;
|
||||
return std::unexpected(
|
||||
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}});
|
||||
}
|
||||
auto published = publish(*inputs);
|
||||
if (!published)
|
||||
return std::unexpected(published.error());
|
||||
return std::optional{std::move(published->committed)};
|
||||
}
|
||||
} // namespace evo::server
|
||||
|
||||
Reference in New Issue
Block a user