M12.4: retain and settle owned population intervals
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m28s
Foundation / checks (tsan) (push) Failing after 18m22s
Foundation / checks (network) (push) Successful in 20m38s
Foundation / checks (client) (push) Failing after 21m6s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (postgres) (push) Canceled after 46m54s
Foundation / checks (checks) (push) Canceled after 47m3s

This commit is contained in:
dingo
2026-10-17 17:31:05 -04:00
parent 627411983f
commit 0a8962b57e
34 changed files with 1032 additions and 75 deletions
+16 -2
View File
@@ -20,9 +20,9 @@ void pacing() {
require(sample.head_age_us == 500'000, "stalled head appeared fresh");
schedule.defer(500'000);
sample = checked(schedule.observe(549'999, 1, 100, true, true), "bounded retry");
require(!sample.due && sample.debt == 10, "retry consumed debt");
require(!sample.due && !sample.retry_ready && sample.debt == 10, "retry consumed debt");
sample = checked(schedule.observe(550'000, 1, 100, true, true), "retry due");
require(sample.due && sample.debt == 11, "retry lost ticks");
require(sample.due && sample.retry_ready && sample.debt == 11, "retry lost ticks");
for (std::uint64_t tick = 101; tick <= 111; ++tick) {
sample = checked(schedule.observe(550'000, 1, tick, true, true), "committed catchup");
require(sample.debt == 111 - tick, "confirmed head did not consume exact debt");
@@ -87,9 +87,23 @@ void historical_profile_is_not_opted_in() {
require(!state->lives().empty() && !cellular_unattended_tick_needed(*state),
"legacy profile without offline freeze was enabled");
}
void admitted_recovery_does_not_spend_new_wall_time() {
CellularTickSchedule schedule;
auto sample =
checked(schedule.observe(0, 2, 100, true, true, 64), "recovered admission anchor");
require(!sample.due && sample.retry_ready && sample.debt == 0, "recovery invented fresh debt");
sample = checked(schedule.observe(10'000, 2, 164, true, true), "recovered interval settled");
require(!sample.due && sample.debt == 0, "recorded prefix used new wall time");
sample = checked(schedule.observe(50'000, 2, 164, true, true), "fresh post-recovery time");
require(sample.due && sample.debt == 1, "recorded prefix delayed fresh work");
require(!schedule.observe(50'000, 2, 164, true, true, 65), "oversized admission accepted");
require(!schedule.observe(50'000, 2, ~std::uint64_t{0}, true, true, 1),
"admission wrapped tick");
}
} // namespace
int main() try {
pacing();
admitted_recovery_does_not_spend_new_wall_time();
offline_demand();
historical_profile_is_not_opted_in();
return 0;
+4 -2
View File
@@ -71,6 +71,7 @@ void exposition_is_bounded_and_well_formed() {
s.simulated_tick = 50;
s.committed_tick = 41;
s.lease_remaining_ms = 1234;
s.ecological_pending_ticks = 64;
OpsCounters c;
c.queue_capacity_inputs = 64;
c.queue_capacity_bytes = 65536;
@@ -91,10 +92,11 @@ void exposition_is_bounded_and_well_formed() {
CHECK(text.find("evo_region_phase{phase=\"running\"} 1\n") != std::string::npos);
CHECK(text.find("evo_region_ready 1\n") != std::string::npos);
CHECK(text.find("evo_region_commit_lag_ticks 9\n") != std::string::npos);
CHECK(text.find("evo_region_ecological_pending_ticks 64\n") != std::string::npos);
CHECK(text.find("evo_region_lease_remaining_milliseconds 1234\n") != std::string::npos);
CHECK(text.find("evo_region_dropped_total{reason=\"fenced\"} 5\n") != std::string::npos);
// Series count is fixed: 6 phases + 4 reasons + 22 unlabelled families.
CHECK(samples.size() == 6 + 4 + 22);
// Series count is fixed: 6 phases + 4 reasons + 23 unlabelled families.
CHECK(samples.size() == 6 + 4 + 23);
// Worst-case values keep the size bounded and never underflow the lag.
HostStatus big;
+40 -3
View File
@@ -17,6 +17,42 @@ CellularBatchHeader header(const CellularPopulation& state, std::uint64_t ticks)
return {s.world(), s.region(), s.frame(), filled<16>(1),
filled<32>(2), s.tick(), s.tick() + ticks};
}
Owner interval_source() {
auto source = cellular_descendants();
const auto stop = source->tick() + cellular_linger_ticks + 1;
while (source->tick() < stop) {
const CellularBatchHeader h{source->world(),
source->region(),
source->frame(),
filled<16>(1),
filled<32>(2),
source->tick(),
std::min(stop, source->tick() + 64)};
auto inputs = checked(unattended_population_inputs(*source, h), "suspension prefix inputs");
source = checked(replay_cellular_batch(*source, inputs), "suspension prefix replay");
}
require(population_interval_safe(*source), "controlled life did not finish lingering");
return source;
}
void lingering_is_not_new_population_work() {
auto active =
checked(CellularPopulation::create(cellular_descendants()), "legacy active source");
require(!population_interval_safe(active->state()), "active controlled source admitted");
auto biology = checked(unattended_population_inputs(active->state(), header(*active, 1)),
"legacy frozen tick");
require(!prepare_population_batch(*active, biology),
"new interval bypassed vulnerable lingering");
auto biological =
checked(replay_cellular_batch(active->state(), biology), "legacy recorded biology");
require(!population_interval_safe(*biological), "lingering controlled source admitted");
auto expected =
checked(CellularPopulation::create(std::move(biological)), "legacy population result");
// Existing profile-5 records retain their meaning. They can be verified and
// materialized to profile 4; new interval preparation rejects this state.
const CellularPopulationBatch legacy{biology.header, biology, {}, {{0, {}}}};
auto replayed = checked(replay_population_batch(*active, legacy), "retained legacy trace");
require(replayed->snapshot() == expected->snapshot(), "old recorded prefix changed meaning");
}
CellularBatch inputs_batch(const CellularPopulation& base) {
CellularBatch batch{header(base, 64), {}};
auto biological = restore(base.state());
@@ -37,8 +73,8 @@ CellularBatch inputs_batch(const CellularPopulation& base) {
return batch;
}
void traces() {
auto population = checked(
CellularPopulation::create(cellular_descendants(), {true, 1200, 1800}), "batch population");
auto population = checked(CellularPopulation::create(interval_source(), {true, 1200, 1800}),
"batch population");
std::uint64_t converted = 0;
for (unsigned interval = 0; interval < 6; ++interval) {
auto inputs = inputs_batch(*population);
@@ -83,7 +119,7 @@ void traces() {
"manual materialization changed biology");
}
void manifests() {
auto state = cellular_descendants();
auto state = interval_source();
auto population = checked(CellularPopulation::create(restore(*state), {true, 1200, 1800}),
"manifest population");
CheckpointManifest source;
@@ -151,6 +187,7 @@ void manifests() {
} // namespace
int main() try {
traces();
lingering_is_not_new_population_work();
manifests();
return 0;
} catch (const std::exception& error) {
+163 -3
View File
@@ -1,5 +1,6 @@
#include "evo/persistence/account_control_store.hpp"
#include "evo/persistence/population_checkpoint_store.hpp"
#include "evo/persistence/population_interval_store.hpp"
#include "evo/persistence/sha256.hpp"
#include "evo/server/cellular_region_host.hpp"
#include "fixtures/cellular_ecology.hpp"
@@ -157,6 +158,28 @@ void funded_source(Fixture& f) {
require(std::ranges::any_of(f.raw->lives(),
[](const auto& life) { return life.account == LifeAccount{}; }),
"durable fixture has no funded player-derived wildlife");
require(!migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build,
{true, 1200, 1800}, 5s),
"population entry bypassed vulnerable controlled life");
const auto suspended_at = f.raw->tick() + cellular_linger_ticks + 1;
while (f.raw->tick() < suspended_at) {
auto h = f.header(std::min(std::uint64_t{64}, suspended_at - f.raw->tick()));
auto prefix = take(unattended_population_inputs(*f.raw, h), "raw linger inputs");
f.raw = take(replay_cellular_batch(*f.raw, prefix), "raw linger biology");
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, prefix, *f.raw,
f.id(), f.build, 5s),
"raw linger publication")
.committed;
std::vector<std::uint64_t> endings;
for (const auto& corpse : f.raw->corpses())
if (corpse.parent_life == 0)
endings.push_back(corpse.life_id);
if (!endings.empty())
take(end_cellular_lives(*f.db, f.key, f.lease, f.head.head.commit, endings, 5s),
"raw linger endings");
}
require(population_interval_safe(*f.raw),
"population source retained a vulnerable controlled life");
}
CellularPopulationExecution unattended(Fixture& f) {
auto working = restore(f.population->state());
@@ -265,6 +288,9 @@ void conversions_and_retries(Fixture& f) {
}
void process_loss(Fixture& f, int point) {
auto next = unattended(f);
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, next.batch.header.end_tick,
5s),
"admit process-loss interval");
const auto predecessor = f.head.head;
const auto expected = next.state->snapshot();
const auto commit = f.id();
@@ -306,6 +332,9 @@ void process_loss(Fixture& f, int point) {
require((point == 2 && acknowledged == 1 && ack == 'A') || (point != 2 && acknowledged == 0),
"unexpected permanent acknowledgement");
recover(f);
const auto pending =
take(load_population_interval(*f.db, f.key, f.lease, 5s), "recover process interval");
require(pending.has_value() == (point == 0), "process loss changed interval settlement");
if (point == 0)
require(f.head.head == predecessor, "uncommitted process published a head");
else
@@ -314,6 +343,9 @@ void process_loss(Fixture& f, int point) {
}
void ambiguous_publication(Fixture& f, bool terminate) {
auto next = unattended(f);
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, next.batch.header.end_tick,
5s),
"admit ambiguous interval");
const auto expected = next.state->snapshot();
auto locker = connection(true);
take(locker->query("BEGIN", {}, 5s), "population lease row transaction");
@@ -352,6 +384,73 @@ void ambiguous_publication(Fixture& f, bool terminate) {
recover(f);
require(f.population->snapshot() == expected, "ambiguous publication changed retained sample");
}
void interval_contract(Fixture& f) {
require(!take(load_population_interval(*f.db, f.key, f.lease, 5s), "empty interval journal")
.has_value(),
"settled source retained pending work");
auto next = unattended(f);
const auto source = f.head.head;
const auto admitted =
take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
"durable interval admission");
require(take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
"duplicate interval admission") == admitted,
"admission retry changed boundary");
auto changed =
admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 63, 5s);
require(!changed && changed.error().kind == LeaseStoreError::commit_conflict,
"source admitted two different intervals");
const auto expected = next.state->snapshot();
require(!f.population->members().empty(), "interval fixture lost its exact cohort");
const auto genome = f.population->members().front().genome;
auto conversion =
take(prepare_population_conversion(*f.population, f.header(0), {genome, false}),
"conversion during admitted work");
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, conversion.batch,
*conversion.state, f.id(), f.build, 5s),
"conversion skipped an admitted interval");
require(
!materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
"profile exit skipped an admitted interval");
auto shortened_header = f.header(63);
auto shortened =
take(prepare_population_batch(
*f.population,
take(unattended_population_inputs(f.population->state(), shortened_header),
"shortened input")),
"shortened interval");
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, shortened.batch,
*shortened.state, f.id(), f.build, 5s),
"publication partially consumed an interval");
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release interval owner");
const auto epoch = f.lease.epoch;
f.lease =
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "replace interval owner");
require(f.lease.epoch > epoch && take(take(load_population_interval(*f.db, f.key, f.lease, 5s),
"new owner interval load"),
"retained admission") == admitted,
"replacement owner lost admitted work");
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, next.batch,
*next.state, f.id(), f.build, 5s),
"settle exact admitted interval");
recover(f);
require(f.population->snapshot() == expected &&
!take(load_population_interval(*f.db, f.key, f.lease, 5s), "settled interval load")
.has_value(),
"settlement lost work or retained spendable pending state");
require(take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
"settled admission retry") == admitted &&
!take(load_population_interval(*f.db, f.key, f.lease, 5s), "settled retry pending")
.has_value(),
"old admission retry reopened settled work");
require(!admit_population_interval(*f.db, f.key, f.lease, f.head.head,
f.head.head.end_tick + 65, 5s),
"oversized interval admitted");
const auto denied = f.db->query(
"SELECT * FROM evolution.publish_population_checkpoint_v44(''::bytea,''::bytea)", {}, 5s);
require(!denied && std::string(denied.error().sqlstate.data(), 5) == "42501",
"runtime bypassed interval guard");
}
void isolation_and_reverse(Fixture& f) {
auto interval = unattended(f);
const auto old_lease = f.lease;
@@ -409,6 +508,7 @@ void host_ownership_and_admission(Fixture& f) {
max_cellular_population_bytes),
"host objects");
CellularHostConfig config{f.key, 1, {}, f.build, {f.build}};
config.lease.commit_timeout = 300ms;
if (enabled)
config.population_policy = CellularTierPolicy{true, 1200, 1800};
return take(CellularRegionHost::create(
@@ -448,24 +548,83 @@ void host_ownership_and_admission(Fixture& f) {
// The fixture's old envelope may have a different membership history; biology is shared.
auto expected =
take(prepare_population_batch(*population, next.batch.biology), "host reference batch");
take(host->publish(next.batch.biology), "host population publication");
require(!take(host->advance_unattended(19), "collect bounded publication window").has_value() &&
host->state()->snapshot() == before,
"small local debt was admitted prematurely");
const auto published = take(host->advance_unattended(65), "host population publication");
require(published.has_value(), "host skipped admitted interval");
require(host->population_mode() &&
host->state()->snapshot() == expected.state->state().snapshot(),
"host population continuation differs");
const auto committed = take(host->committed(), "host continued head");
const auto continued = host->state()->snapshot();
const CellularBatchHeader recovered_header{f.key.world,
f.key.region,
1,
committed.head.commit,
committed.head.digest,
committed.head.end_tick,
committed.head.end_tick + 64};
auto recovered_expected = take(
prepare_population_batch(
*expected.state, take(unattended_population_inputs(*host->state(), recovered_header),
"recovered interval inputs")),
"recovered interval reference");
take(admit_population_interval(*f.db, f.key, take(host->authority(), "host interval authority"),
committed.head, committed.head.end_tick + 64, 5s),
"admit before host loss");
host->drain();
host = make_host(false);
require(host->step().ready && host->population_mode() &&
take(host->committed(), "host recovered head").head == committed.head &&
host->state()->snapshot() == continued,
"restarted host failed to recover population head");
require(host->status().ecological_pending_ticks == 64 && !host->set_population_mode(false),
"restarted host hid or skipped admitted work");
require(take(host->advance_unattended(0), "resume admitted work with zero new due time")
.has_value() &&
host->state()->snapshot() == recovered_expected.state->state().snapshot() &&
host->status().ecological_pending_ticks == 0,
"recovered interval did not settle exactly");
const auto recovered_head = take(host->committed(), "recovered interval head");
const auto recovered_bytes = host->state()->snapshot();
const CellularBatchHeader unknown_header{f.key.world,
f.key.region,
1,
recovered_head.head.commit,
recovered_head.head.digest,
recovered_head.head.end_tick,
recovered_head.head.end_tick + 64};
auto unknown_expected = take(
prepare_population_batch(*recovered_expected.state,
take(unattended_population_inputs(*host->state(), unknown_header),
"uncertain admission inputs")),
"uncertain admission reference");
auto locker = connection(true);
take(locker->query("BEGIN", {}, 5s), "host admission lock transaction");
take(
locker->query(
"SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE",
{}, 5s),
"hold host admission reply");
require(!host->advance_unattended(64) && !host->status().ready &&
host->state()->snapshot() == recovered_bytes,
"uncertain admission allowed simulation/output");
take(locker->query("ROLLBACK", {}, 5s), "release host admission reply");
for (unsigned attempt = 0; attempt < 200 && !host->step().ready; ++attempt)
std::this_thread::sleep_for(25ms);
require(host->status().ready && host->status().ecological_pending_ticks == 64,
"unknown admission failed exact reconciliation");
require(take(host->advance_unattended(0), "finish reconciled admission").has_value() &&
host->state()->snapshot() == unknown_expected.state->state().snapshot(),
"admission retry chose different work");
const auto final_bytes = host->state()->snapshot();
Uuid account{};
account.back() = 1;
// A returning player's operation completes the durable reverse transition before lookup.
[[maybe_unused]] const auto queried = host->query_primitive_life(account, 1, 1);
require(!host->population_mode() && host->state()->snapshot() == continued &&
host->state()->tick() == tick + 64,
require(!host->population_mode() && host->state()->snapshot() == final_bytes &&
host->state()->tick() == tick + 192,
"player operation did not materialize first");
host->drain();
}
@@ -474,6 +633,7 @@ void run_population_publication_tests() {
Fixture fixture;
funded_source(fixture);
conversions_and_retries(fixture);
interval_contract(fixture);
ambiguous_publication(fixture, false);
ambiguous_publication(fixture, true);
process_loss(fixture, 0);
+2 -1
View File
@@ -210,6 +210,7 @@ void upgrade_across_migration_and_barrier() {
" -v migration_sha_42=$(s 0042_cellular_v8_reproduction_bundle.sql)"
" -v migration_sha_43=$(s 0043_cellular_v9_foraging_bundle.sql)"
" -v migration_sha_44=$(s 0044_exact_population_publication.sql)"
" -v migration_sha_45=$(s 0045_owned_population_intervals.sql)"
" -v runtime_role=evo_runtime_test -f migrations/apply.sql"
" > /dev/null 2>&1") != 0);
CHECK(history(*owner_db) == "1,2,3,4,5,6,7");
@@ -226,7 +227,7 @@ void upgrade_across_migration_and_barrier() {
" EVO_PG_RUNTIME_ROLE=evo_runtime_test bash \"$EVO_SOURCE_ROOT/"
"tools/migrate-postgres.sh\" > /dev/null") == 0);
CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,"
"25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44");
"25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45");
CHECK(scalar(*owner_db, "SELECT (c.condeferrable AND c.condeferred AND "
"c.confrelid = 'evolution.region_commits'::regclass)::text "
"FROM pg_catalog.pg_constraint c WHERE c.conrelid = "