M12.4: retain and settle owned population intervals
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m28s
Foundation / checks (tsan) (push) Failing after 18m22s
Foundation / checks (network) (push) Successful in 20m38s
Foundation / checks (client) (push) Failing after 21m6s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (postgres) (push) Canceled after 46m54s
Foundation / checks (checks) (push) Canceled after 47m3s

This commit is contained in:
dingo
2026-10-17 17:31:05 -04:00
parent 627411983f
commit 0a8962b57e
34 changed files with 1032 additions and 75 deletions
@@ -71,6 +71,8 @@ transition(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant&
auto state = restore_cellular_payload(*source, *previous);
if (!state)
return std::unexpected(failure(PublicationError::predecessor));
if (!server::population_interval_safe(**state))
return std::unexpected(failure(PublicationError::candidate));
auto population = server::CellularPopulation::create(std::move(*state), policy);
if (!population || (*population)->state().snapshot() != *source)
return std::unexpected(failure(PublicationError::candidate));
@@ -160,6 +162,8 @@ std::expected<CommittedCheckpoint, PopulationPublicationFailure> publish_populat
auto base = restore_population_payload(*source, *previous);
if (!base)
return std::unexpected(failure(PublicationError::predecessor));
if (!batch.conversion && !server::population_interval_safe((*base)->state()))
return std::unexpected(failure(PublicationError::candidate));
auto replay = server::replay_population_batch(**base, batch);
const auto payload = candidate.snapshot();
if (!replay || (*replay)->snapshot() != payload)
@@ -0,0 +1,127 @@
#include "evo/persistence/population_interval_store.hpp"
#include <charconv>
#include <string_view>
namespace evo::persistence {
namespace {
LeaseStoreFailure failure(const PgFailure& cause) {
const std::string_view state(cause.sqlstate.data(), 5);
auto kind = LeaseStoreError::store;
if (cause.kind == PgError::unknown)
kind = LeaseStoreError::unknown;
else if (cause.kind == PgError::config || cause.kind == PgError::bounds || state == "22023")
kind = LeaseStoreError::invalid;
else if (state == "EVL02")
kind = LeaseStoreError::fenced;
else if (state == "EVL03")
kind = LeaseStoreError::head_mismatch;
else if (state == "EVL04")
kind = LeaseStoreError::not_provisioned;
else if (state == "EVL05")
kind = LeaseStoreError::exhausted;
else if (state == "EVL06")
kind = LeaseStoreError::unsupported;
else if (state == "EVL07" || state == "EVL18")
kind = LeaseStoreError::commit_conflict;
return {kind, cause};
}
std::string hex(std::span<const std::uint8_t> bytes) {
constexpr std::string_view digits = "0123456789abcdef";
std::string out;
for (const auto b : bytes) {
out += digits[b >> 4U];
out += digits[b & 15U];
}
return out;
}
std::string uuid(const Uuid& id) {
const auto h = hex(id);
return h.substr(0, 8) + '-' + h.substr(8, 4) + '-' + h.substr(12, 4) + '-' + h.substr(16, 4) +
'-' + h.substr(20);
}
template <std::size_t N> std::optional<std::array<std::uint8_t, N>> unhex(std::string_view value) {
if (value.size() != N * 2)
return std::nullopt;
std::array<std::uint8_t, N> out{};
for (std::size_t i = 0; i < N; ++i) {
unsigned byte{};
const auto [end, error] =
std::from_chars(value.data() + i * 2, value.data() + i * 2 + 2, byte, 16);
if (error != std::errc{} || end != value.data() + i * 2 + 2 || byte > 255)
return std::nullopt;
out[i] = static_cast<std::uint8_t>(byte);
}
return out;
}
std::optional<std::uint64_t> number(std::string_view value) {
std::uint64_t out{};
const auto [end, error] = std::from_chars(value.data(), value.data() + value.size(), out);
if (error != std::errc{} || end != value.data() + value.size())
return std::nullopt;
return out;
}
std::optional<PopulationInterval> parse(const PgRows::value_type& row) {
if (row.size() != 5)
return std::nullopt;
const auto text = row[0].value_or("");
if (text.size() != 36 || text[8] != '-' || text[13] != '-' || text[18] != '-' ||
text[23] != '-')
return std::nullopt;
std::string compact;
for (const auto c : text)
if (c != '-')
compact += c;
const auto source = unhex<16>(compact);
const auto digest = unhex<32>(row[1].value_or(""));
const auto begin = number(row[2].value_or("")), end = number(row[3].value_or(""));
if (!source || !digest || !begin || !end || *end <= *begin || *end - *begin > 64 ||
*source == Uuid{} || *digest == Sha256Digest{} || row[4].value_or("") != "1")
return std::nullopt;
return PopulationInterval{*source, *digest, *begin, *end, 1};
}
constexpr auto columns = "source_commit::text,encode(source_manifest_sha256,'hex'),begin_tick::"
"text,end_tick::text,controller_version::text";
} // namespace
std::expected<PopulationInterval, LeaseStoreFailure>
admit_population_interval(Postgres& db, RegionKey key, const LeaseGrant& lease,
const RegionHead& source, std::uint64_t end_tick,
std::chrono::milliseconds timeout) {
const std::array<std::string, 7> parameters{
std::to_string(key.world), std::to_string(key.region), uuid(lease.holder),
std::to_string(lease.epoch), uuid(source.commit), "\\x" + hex(source.digest),
std::to_string(end_tick)};
auto rows = db.query(std::string("SELECT ") + columns +
" FROM evolution.admit_population_interval("
"$1::evolution.u64,$2::evolution.u64,$3::uuid,$4::evolution.u64,$5::"
"uuid,$6::bytea,$7::evolution.u64)",
parameters, timeout);
if (!rows)
return std::unexpected(failure(rows.error()));
const auto interval = rows->size() == 1 ? parse(rows->front()) : std::nullopt;
if (!interval || interval->source_commit != source.commit ||
interval->source_manifest_sha256 != source.digest ||
interval->begin_tick != source.end_tick || interval->end_tick != end_tick)
return std::unexpected(LeaseStoreFailure{LeaseStoreError::unknown, {}});
return *interval;
}
std::expected<std::optional<PopulationInterval>, LeaseStoreFailure>
load_population_interval(Postgres& db, RegionKey key, const LeaseGrant& lease,
std::chrono::milliseconds timeout) {
const std::array<std::string, 4> parameters{std::to_string(key.world),
std::to_string(key.region), uuid(lease.holder),
std::to_string(lease.epoch)};
auto rows = db.query(std::string("SELECT ") + columns +
" FROM evolution.load_population_interval("
"$1::evolution.u64,$2::evolution.u64,$3::uuid,$4::evolution.u64)",
parameters, timeout);
if (!rows)
return std::unexpected(failure(rows.error()));
if (rows->empty())
return std::optional<PopulationInterval>{};
const auto interval = rows->size() == 1 ? parse(rows->front()) : std::nullopt;
if (!interval)
return std::unexpected(LeaseStoreFailure{LeaseStoreError::store, {}});
return interval;
}
} // namespace evo::persistence
+133 -8
View File
@@ -199,15 +199,19 @@ void CellularRegionHost::acquire() {
}
if (!recover_state(*lease) || !committed_.has_value() || !state_)
return;
interval_reload_ = population_mode();
admitted_interval_.reset();
interval_request_.reset();
interval_failure_.reset();
queued_mutations_.clear();
mutations_uncertain_ = true;
pending_endings_.clear();
// Controlled cells cannot die in unattended profile 5; their earlier raw
// endings were drained before entry. Do not invent a new death commit.
if (!population_mode())
for (const auto& corpse : state_->corpses())
if (corpse.parent_life == 0)
pending_endings_.push_back({committed_->head.commit, corpse.life_id});
// Reconcile retained owned deaths after either profile's recovery, including
// older profile-5 records that admitted lingering cells. Existing endings
// are idempotent; no ending is inferred for a living suspended cell.
for (const auto& corpse : state_->corpses())
if (corpse.parent_life == 0)
pending_endings_.push_back({committed_->head.commit, corpse.life_id});
status_.phase = HostPhase::running;
status_.simulated_tick = committed_->head.end_tick;
status_.committed_tick = committed_->head.end_tick;
@@ -391,6 +395,7 @@ void CellularRegionHost::resolve() {
note_endings(*state_, *pending_->candidate, retried->head.commit);
state_ = std::move(pending_->candidate);
committed_ = std::move(*retried);
finish_interval();
for (const auto& action : pending_->actions)
finalized_actions_.push_back({action, committed_->head.sequence});
finalize_births(pending_->births);
@@ -406,7 +411,11 @@ void CellularRegionHost::resolve() {
void CellularRegionHost::refresh() {
const auto& grant = authority_.grant();
status_.ready = status_.phase == HostPhase::running && state_ && committed_ && grant &&
authority_.may_originate(LeaseClock::now());
authority_.may_originate(LeaseClock::now()) && !interval_reload_ &&
!interval_request_.has_value();
const auto& interval = admitted_interval_;
status_.ecological_pending_ticks =
interval.has_value() ? interval->end_tick - interval->begin_tick : 0;
status_.epoch = grant ? grant->epoch : 0;
status_.lease_remaining_ms =
status_.ready ? static_cast<std::uint64_t>(std::chrono::duration_cast<milliseconds>(
@@ -424,6 +433,8 @@ const HostStatus& CellularRegionHost::step() {
resolve();
else if (!state_)
acquire();
if (!pending_ && state_ && status_.phase == HostPhase::running)
reconcile_interval();
if (!pending_ && state_ && mutations_uncertain_ && status_.phase == HostPhase::running)
reload_mutations();
if (!pending_ && state_ && status_.phase == HostPhase::running)
@@ -839,6 +850,9 @@ void CellularRegionHost::drain() {
authority_.fence();
state_.reset();
committed_.reset();
admitted_interval_.reset();
interval_request_.reset();
interval_reload_ = false;
status_.phase = HostPhase::stopped;
status_.epoch = 0;
status_.lease_remaining_ms = 0;
@@ -891,6 +905,7 @@ CellularRegionHost::accept_population_publication(
note_endings(*state_, *candidate, result->head.commit);
state_ = std::move(candidate);
committed_ = *result;
finish_interval();
++status_.commits;
status_.committed_sequence = committed_->head.sequence;
status_.committed_tick = status_.simulated_tick = committed_->head.end_tick;
@@ -899,7 +914,8 @@ CellularRegionHost::accept_population_publication(
}
std::expected<void, CellularPublishFailure> CellularRegionHost::set_population_mode(bool enabled) {
if (!status_.ready || pending_ || mutations_uncertain_ || !state_ || !committed_.has_value() ||
if (!status_.ready || pending_ || mutations_uncertain_ || admitted_interval_.has_value() ||
interval_request_.has_value() || interval_reload_ || !state_ || !committed_.has_value() ||
!authority_.may_begin_commit(LeaseClock::now()) || !database())
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
if (population_mode() == enabled)
@@ -908,6 +924,8 @@ std::expected<void, CellularPublishFailure> CellularRegionHost::set_population_m
(!config_.population_policy || !queued_mutations_.empty() || !pending_endings_.empty() ||
state_->rules_version() != cellular_lives_foraging_rules))
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
if (enabled && !population_interval_safe(*state_))
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
const auto policy = config_.population_policy.value_or(CellularTierPolicy{});
std::array<std::uint8_t, 10> marker{};
marker[0] = enabled ? 1 : 0;
@@ -971,4 +989,111 @@ CellularRegionHost::publish_population(const CellularBatch& batch) {
return std::unexpected(accepted.error());
return persistence::PublishedCellularBatch{std::move(*accepted), {}};
}
void CellularRegionHost::finish_interval() {
const auto& interval = admitted_interval_;
if (interval.has_value() && committed_.has_value() &&
committed_->head.end_tick == interval->end_tick)
admitted_interval_.reset();
}
void CellularRegionHost::reconcile_interval() {
if (!interval_reload_ && !interval_request_.has_value())
return;
const auto grant = authority_.grant();
if (!population_mode() || !committed_.has_value() || !grant.has_value() ||
!authority_.may_begin_commit(LeaseClock::now()) || !database())
return;
auto result = [&]() -> std::expected<std::optional<persistence::PopulationInterval>,
persistence::LeaseStoreFailure> {
const auto& request = interval_request_;
if (request.has_value()) {
auto admitted = persistence::admit_population_interval(
*db_, config_.key, *grant, request->source, request->end_tick,
config_.lease.commit_timeout);
if (!admitted)
return std::unexpected(admitted.error());
return std::optional{*admitted};
}
return persistence::load_population_interval(*db_, config_.key, *grant,
config_.lease.commit_timeout);
}();
if (!result) {
interval_failure_ = result.error();
if (result.error().kind == LeaseStoreError::unknown ||
result.error().kind == LeaseStoreError::store) {
if (result.error().kind == LeaseStoreError::unknown)
++status_.unknown_outcomes;
else
++status_.store_failures;
db_.reset();
return;
}
authority_.fence();
state_.reset();
committed_.reset();
admitted_interval_.reset();
interval_request_.reset();
status_.ready = false;
status_.phase = result.error().kind == LeaseStoreError::fenced ? HostPhase::acquiring
: HostPhase::quarantined;
++status_.fences;
return;
}
auto interval = *result;
if (interval.has_value() &&
(interval->source_commit != committed_->head.commit ||
interval->source_manifest_sha256 != committed_->head.digest ||
interval->begin_tick != committed_->head.end_tick || interval->controller_version != 1)) {
authority_.fence();
status_.ready = false;
status_.phase = HostPhase::quarantined;
return;
}
admitted_interval_ = interval;
interval_request_.reset();
interval_failure_.reset();
interval_reload_ = false;
}
std::expected<std::optional<persistence::CommittedCheckpoint>, CellularPublishFailure>
CellularRegionHost::advance_unattended(std::uint64_t due_ticks) {
if (!status_.ready || !population_mode() || !committed_.has_value() || pending_ ||
!authority_.may_begin_commit(LeaseClock::now()) || !population_interval_safe(*state_))
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
if (!admitted_interval_.has_value()) {
const auto remaining = ~std::uint64_t{0} - committed_->head.end_tick;
const auto count = std::min({due_ticks, remaining, std::uint64_t{64}});
if (count == 0 || (due_ticks < 20 && count != remaining))
return std::optional<persistence::CommittedCheckpoint>{};
interval_request_ = IntervalRequest{committed_->head, committed_->head.end_tick + count};
reconcile_interval();
refresh();
if (!status_.ready || !admitted_interval_.has_value())
return std::unexpected(
CellularPublishFailure{CellularHostError::publication, {}, interval_failure_, {}});
}
const auto interval = admitted_interval_;
if (!interval.has_value())
return std::unexpected(CellularPublishFailure{CellularHostError::not_ready, {}, {}, {}});
const CellularBatchHeader header{config_.key.world,
config_.key.region,
config_.frame,
interval->source_commit,
interval->source_manifest_sha256,
interval->begin_tick,
interval->end_tick};
const auto inputs = unattended_population_inputs(*state_, header);
if (!inputs) {
authority_.fence();
status_.ready = false;
status_.phase = HostPhase::quarantined;
return std::unexpected(
CellularPublishFailure{CellularHostError::invalid_batch, {}, {}, {}});
}
auto published = publish(*inputs);
if (!published)
return std::unexpected(published.error());
return std::optional{std::move(published->committed)};
}
} // namespace evo::server
@@ -74,6 +74,13 @@ class Reader {
};
} // namespace
bool population_interval_safe(const CellularLives& state) noexcept {
return state.rules_version() == cellular_lives_foraging_rules &&
std::ranges::all_of(state.lives(), [](const auto& life) {
return life.account == LifeAccount{} || life.presence == CellularPresence::suspended;
});
}
std::expected<CellularBatch, CellularPopulationError>
unattended_population_inputs(const CellularLives& base, const CellularBatchHeader& header) {
if (header.world != base.world() || header.region != base.region() ||
@@ -178,6 +185,8 @@ execute_population_batch(const CellularPopulation& base, CellularPopulationBatch
std::expected<CellularPopulationExecution, CellularPopulationError>
prepare_population_batch(const CellularPopulation& base, CellularBatch biology) {
if (!population_interval_safe(base.state()))
return std::unexpected(CellularPopulationError::ineligible);
CellularPopulationBatch batch{biology.header, std::move(biology), {}, {}};
auto state = execute_population_batch(base, batch, true);
if (!state)
+12 -4
View File
@@ -16,7 +16,11 @@ bool cellular_unattended_tick_needed(const CellularLives& state) noexcept {
std::expected<CellularScheduleStatus, CellularScheduleError>
CellularTickSchedule::observe(std::uint64_t now_us, std::uint64_t epoch,
std::uint64_t committed_tick, bool needed, bool authoritative) {
std::uint64_t committed_tick, bool needed, bool authoritative,
std::uint64_t previously_admitted_ticks) {
if (previously_admitted_ticks > 64 ||
previously_admitted_ticks > std::numeric_limits<std::uint64_t>::max() - committed_tick)
return std::unexpected(CellularScheduleError::admission);
if (observed_ && now_us < last_us_)
return std::unexpected(CellularScheduleError::clock_regression);
if (authoritative && epoch == 0)
@@ -35,27 +39,30 @@ CellularTickSchedule::observe(std::uint64_t now_us, std::uint64_t epoch,
was_authoritative_ = authoritative;
status_.head_age_us = have_receipt_ ? now_us - receipt_us_ : 0;
status_.due = false;
status_.retry_ready = now_us >= retry_us_;
status_.debt = 0;
if (!authoritative || !needed) {
anchored_ = false;
retry_us_ = 0;
status_.retry_ready = true;
return status_;
}
if (!anchored_ || epoch != epoch_) {
anchored_ = true;
origin_us_ = now_us;
origin_tick_ = committed_tick;
origin_tick_ = committed_tick + previously_admitted_ticks;
epoch_ = epoch;
retry_us_ = 0;
}
head_ = committed_tick;
const auto elapsed_ticks = (now_us - origin_us_) / biology::cellular_motion_tick_microseconds;
const auto completed_ticks = committed_tick - origin_tick_;
const auto completed_ticks = committed_tick > origin_tick_ ? committed_tick - origin_tick_ : 0;
const auto remaining = std::numeric_limits<std::uint64_t>::max() - committed_tick;
status_.debt =
elapsed_ticks > completed_ticks ? std::min(elapsed_ticks - completed_ticks, remaining) : 0;
status_.max_debt = std::max(status_.max_debt, status_.debt);
status_.due = status_.debt > 0 && now_us >= retry_us_;
status_.retry_ready = now_us >= retry_us_;
status_.due = status_.debt > 0 && status_.retry_ready;
return status_;
}
void CellularTickSchedule::defer(std::uint64_t now_us) noexcept {
@@ -64,6 +71,7 @@ void CellularTickSchedule::defer(std::uint64_t now_us) noexcept {
? maximum
: now_us + biology::cellular_motion_tick_microseconds;
status_.due = false;
status_.retry_ready = false;
}
CellularScheduleStatus CellularTickSchedule::status() const noexcept {
return status_;
+2
View File
@@ -72,6 +72,8 @@ std::string render_metrics(const HostStatus& s, const OpsCounters& c) {
metric(out, "evo_region_checkpoint_age_milliseconds", "gauge",
"Time since the last confirmed commit or recovery", c.checkpoint_age_ms);
metric(out, "evo_region_tick_debt", "gauge", "Due ticks not yet simulated", s.tick_debt);
metric(out, "evo_region_ecological_pending_ticks", "gauge", "Durable admitted ecological ticks",
s.ecological_pending_ticks);
metric(out, "evo_region_tick_debt_max", "gauge", "Largest tick debt since recovery",
s.max_tick_debt);
metric(out, "evo_region_queued_inputs", "gauge", "Admitted inputs awaiting a tick", s.queued);