M12.4: retain and settle owned population intervals
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m28s
Foundation / checks (tsan) (push) Failing after 18m22s
Foundation / checks (network) (push) Successful in 20m38s
Foundation / checks (client) (push) Failing after 21m6s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (postgres) (push) Canceled after 46m54s
Foundation / checks (checks) (push) Canceled after 47m3s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m28s
Foundation / checks (tsan) (push) Failing after 18m22s
Foundation / checks (network) (push) Successful in 20m38s
Foundation / checks (client) (push) Failing after 21m6s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (postgres) (push) Canceled after 46m54s
Foundation / checks (checks) (push) Canceled after 47m3s
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
#include "evo/persistence/account_control_store.hpp"
|
||||
#include "evo/persistence/population_checkpoint_store.hpp"
|
||||
#include "evo/persistence/population_interval_store.hpp"
|
||||
#include "evo/persistence/sha256.hpp"
|
||||
#include "evo/server/cellular_region_host.hpp"
|
||||
#include "fixtures/cellular_ecology.hpp"
|
||||
@@ -157,6 +158,28 @@ void funded_source(Fixture& f) {
|
||||
require(std::ranges::any_of(f.raw->lives(),
|
||||
[](const auto& life) { return life.account == LifeAccount{}; }),
|
||||
"durable fixture has no funded player-derived wildlife");
|
||||
require(!migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build,
|
||||
{true, 1200, 1800}, 5s),
|
||||
"population entry bypassed vulnerable controlled life");
|
||||
const auto suspended_at = f.raw->tick() + cellular_linger_ticks + 1;
|
||||
while (f.raw->tick() < suspended_at) {
|
||||
auto h = f.header(std::min(std::uint64_t{64}, suspended_at - f.raw->tick()));
|
||||
auto prefix = take(unattended_population_inputs(*f.raw, h), "raw linger inputs");
|
||||
f.raw = take(replay_cellular_batch(*f.raw, prefix), "raw linger biology");
|
||||
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, prefix, *f.raw,
|
||||
f.id(), f.build, 5s),
|
||||
"raw linger publication")
|
||||
.committed;
|
||||
std::vector<std::uint64_t> endings;
|
||||
for (const auto& corpse : f.raw->corpses())
|
||||
if (corpse.parent_life == 0)
|
||||
endings.push_back(corpse.life_id);
|
||||
if (!endings.empty())
|
||||
take(end_cellular_lives(*f.db, f.key, f.lease, f.head.head.commit, endings, 5s),
|
||||
"raw linger endings");
|
||||
}
|
||||
require(population_interval_safe(*f.raw),
|
||||
"population source retained a vulnerable controlled life");
|
||||
}
|
||||
CellularPopulationExecution unattended(Fixture& f) {
|
||||
auto working = restore(f.population->state());
|
||||
@@ -265,6 +288,9 @@ void conversions_and_retries(Fixture& f) {
|
||||
}
|
||||
void process_loss(Fixture& f, int point) {
|
||||
auto next = unattended(f);
|
||||
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, next.batch.header.end_tick,
|
||||
5s),
|
||||
"admit process-loss interval");
|
||||
const auto predecessor = f.head.head;
|
||||
const auto expected = next.state->snapshot();
|
||||
const auto commit = f.id();
|
||||
@@ -306,6 +332,9 @@ void process_loss(Fixture& f, int point) {
|
||||
require((point == 2 && acknowledged == 1 && ack == 'A') || (point != 2 && acknowledged == 0),
|
||||
"unexpected permanent acknowledgement");
|
||||
recover(f);
|
||||
const auto pending =
|
||||
take(load_population_interval(*f.db, f.key, f.lease, 5s), "recover process interval");
|
||||
require(pending.has_value() == (point == 0), "process loss changed interval settlement");
|
||||
if (point == 0)
|
||||
require(f.head.head == predecessor, "uncommitted process published a head");
|
||||
else
|
||||
@@ -314,6 +343,9 @@ void process_loss(Fixture& f, int point) {
|
||||
}
|
||||
void ambiguous_publication(Fixture& f, bool terminate) {
|
||||
auto next = unattended(f);
|
||||
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head, next.batch.header.end_tick,
|
||||
5s),
|
||||
"admit ambiguous interval");
|
||||
const auto expected = next.state->snapshot();
|
||||
auto locker = connection(true);
|
||||
take(locker->query("BEGIN", {}, 5s), "population lease row transaction");
|
||||
@@ -352,6 +384,73 @@ void ambiguous_publication(Fixture& f, bool terminate) {
|
||||
recover(f);
|
||||
require(f.population->snapshot() == expected, "ambiguous publication changed retained sample");
|
||||
}
|
||||
void interval_contract(Fixture& f) {
|
||||
require(!take(load_population_interval(*f.db, f.key, f.lease, 5s), "empty interval journal")
|
||||
.has_value(),
|
||||
"settled source retained pending work");
|
||||
auto next = unattended(f);
|
||||
const auto source = f.head.head;
|
||||
const auto admitted =
|
||||
take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
|
||||
"durable interval admission");
|
||||
require(take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
|
||||
"duplicate interval admission") == admitted,
|
||||
"admission retry changed boundary");
|
||||
auto changed =
|
||||
admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 63, 5s);
|
||||
require(!changed && changed.error().kind == LeaseStoreError::commit_conflict,
|
||||
"source admitted two different intervals");
|
||||
const auto expected = next.state->snapshot();
|
||||
require(!f.population->members().empty(), "interval fixture lost its exact cohort");
|
||||
const auto genome = f.population->members().front().genome;
|
||||
auto conversion =
|
||||
take(prepare_population_conversion(*f.population, f.header(0), {genome, false}),
|
||||
"conversion during admitted work");
|
||||
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, conversion.batch,
|
||||
*conversion.state, f.id(), f.build, 5s),
|
||||
"conversion skipped an admitted interval");
|
||||
require(
|
||||
!materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
|
||||
"profile exit skipped an admitted interval");
|
||||
auto shortened_header = f.header(63);
|
||||
auto shortened =
|
||||
take(prepare_population_batch(
|
||||
*f.population,
|
||||
take(unattended_population_inputs(f.population->state(), shortened_header),
|
||||
"shortened input")),
|
||||
"shortened interval");
|
||||
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, shortened.batch,
|
||||
*shortened.state, f.id(), f.build, 5s),
|
||||
"publication partially consumed an interval");
|
||||
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release interval owner");
|
||||
const auto epoch = f.lease.epoch;
|
||||
f.lease =
|
||||
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "replace interval owner");
|
||||
require(f.lease.epoch > epoch && take(take(load_population_interval(*f.db, f.key, f.lease, 5s),
|
||||
"new owner interval load"),
|
||||
"retained admission") == admitted,
|
||||
"replacement owner lost admitted work");
|
||||
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, next.batch,
|
||||
*next.state, f.id(), f.build, 5s),
|
||||
"settle exact admitted interval");
|
||||
recover(f);
|
||||
require(f.population->snapshot() == expected &&
|
||||
!take(load_population_interval(*f.db, f.key, f.lease, 5s), "settled interval load")
|
||||
.has_value(),
|
||||
"settlement lost work or retained spendable pending state");
|
||||
require(take(admit_population_interval(*f.db, f.key, f.lease, source, source.end_tick + 64, 5s),
|
||||
"settled admission retry") == admitted &&
|
||||
!take(load_population_interval(*f.db, f.key, f.lease, 5s), "settled retry pending")
|
||||
.has_value(),
|
||||
"old admission retry reopened settled work");
|
||||
require(!admit_population_interval(*f.db, f.key, f.lease, f.head.head,
|
||||
f.head.head.end_tick + 65, 5s),
|
||||
"oversized interval admitted");
|
||||
const auto denied = f.db->query(
|
||||
"SELECT * FROM evolution.publish_population_checkpoint_v44(''::bytea,''::bytea)", {}, 5s);
|
||||
require(!denied && std::string(denied.error().sqlstate.data(), 5) == "42501",
|
||||
"runtime bypassed interval guard");
|
||||
}
|
||||
void isolation_and_reverse(Fixture& f) {
|
||||
auto interval = unattended(f);
|
||||
const auto old_lease = f.lease;
|
||||
@@ -409,6 +508,7 @@ void host_ownership_and_admission(Fixture& f) {
|
||||
max_cellular_population_bytes),
|
||||
"host objects");
|
||||
CellularHostConfig config{f.key, 1, {}, f.build, {f.build}};
|
||||
config.lease.commit_timeout = 300ms;
|
||||
if (enabled)
|
||||
config.population_policy = CellularTierPolicy{true, 1200, 1800};
|
||||
return take(CellularRegionHost::create(
|
||||
@@ -448,24 +548,83 @@ void host_ownership_and_admission(Fixture& f) {
|
||||
// The fixture's old envelope may have a different membership history; biology is shared.
|
||||
auto expected =
|
||||
take(prepare_population_batch(*population, next.batch.biology), "host reference batch");
|
||||
take(host->publish(next.batch.biology), "host population publication");
|
||||
require(!take(host->advance_unattended(19), "collect bounded publication window").has_value() &&
|
||||
host->state()->snapshot() == before,
|
||||
"small local debt was admitted prematurely");
|
||||
const auto published = take(host->advance_unattended(65), "host population publication");
|
||||
require(published.has_value(), "host skipped admitted interval");
|
||||
require(host->population_mode() &&
|
||||
host->state()->snapshot() == expected.state->state().snapshot(),
|
||||
"host population continuation differs");
|
||||
const auto committed = take(host->committed(), "host continued head");
|
||||
const auto continued = host->state()->snapshot();
|
||||
const CellularBatchHeader recovered_header{f.key.world,
|
||||
f.key.region,
|
||||
1,
|
||||
committed.head.commit,
|
||||
committed.head.digest,
|
||||
committed.head.end_tick,
|
||||
committed.head.end_tick + 64};
|
||||
auto recovered_expected = take(
|
||||
prepare_population_batch(
|
||||
*expected.state, take(unattended_population_inputs(*host->state(), recovered_header),
|
||||
"recovered interval inputs")),
|
||||
"recovered interval reference");
|
||||
take(admit_population_interval(*f.db, f.key, take(host->authority(), "host interval authority"),
|
||||
committed.head, committed.head.end_tick + 64, 5s),
|
||||
"admit before host loss");
|
||||
host->drain();
|
||||
host = make_host(false);
|
||||
require(host->step().ready && host->population_mode() &&
|
||||
take(host->committed(), "host recovered head").head == committed.head &&
|
||||
host->state()->snapshot() == continued,
|
||||
"restarted host failed to recover population head");
|
||||
require(host->status().ecological_pending_ticks == 64 && !host->set_population_mode(false),
|
||||
"restarted host hid or skipped admitted work");
|
||||
require(take(host->advance_unattended(0), "resume admitted work with zero new due time")
|
||||
.has_value() &&
|
||||
host->state()->snapshot() == recovered_expected.state->state().snapshot() &&
|
||||
host->status().ecological_pending_ticks == 0,
|
||||
"recovered interval did not settle exactly");
|
||||
const auto recovered_head = take(host->committed(), "recovered interval head");
|
||||
const auto recovered_bytes = host->state()->snapshot();
|
||||
const CellularBatchHeader unknown_header{f.key.world,
|
||||
f.key.region,
|
||||
1,
|
||||
recovered_head.head.commit,
|
||||
recovered_head.head.digest,
|
||||
recovered_head.head.end_tick,
|
||||
recovered_head.head.end_tick + 64};
|
||||
auto unknown_expected = take(
|
||||
prepare_population_batch(*recovered_expected.state,
|
||||
take(unattended_population_inputs(*host->state(), unknown_header),
|
||||
"uncertain admission inputs")),
|
||||
"uncertain admission reference");
|
||||
auto locker = connection(true);
|
||||
take(locker->query("BEGIN", {}, 5s), "host admission lock transaction");
|
||||
take(
|
||||
locker->query(
|
||||
"SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND region_id=1 FOR UPDATE",
|
||||
{}, 5s),
|
||||
"hold host admission reply");
|
||||
require(!host->advance_unattended(64) && !host->status().ready &&
|
||||
host->state()->snapshot() == recovered_bytes,
|
||||
"uncertain admission allowed simulation/output");
|
||||
take(locker->query("ROLLBACK", {}, 5s), "release host admission reply");
|
||||
for (unsigned attempt = 0; attempt < 200 && !host->step().ready; ++attempt)
|
||||
std::this_thread::sleep_for(25ms);
|
||||
require(host->status().ready && host->status().ecological_pending_ticks == 64,
|
||||
"unknown admission failed exact reconciliation");
|
||||
require(take(host->advance_unattended(0), "finish reconciled admission").has_value() &&
|
||||
host->state()->snapshot() == unknown_expected.state->state().snapshot(),
|
||||
"admission retry chose different work");
|
||||
const auto final_bytes = host->state()->snapshot();
|
||||
Uuid account{};
|
||||
account.back() = 1;
|
||||
// A returning player's operation completes the durable reverse transition before lookup.
|
||||
[[maybe_unused]] const auto queried = host->query_primitive_life(account, 1, 1);
|
||||
require(!host->population_mode() && host->state()->snapshot() == continued &&
|
||||
host->state()->tick() == tick + 64,
|
||||
require(!host->population_mode() && host->state()->snapshot() == final_bytes &&
|
||||
host->state()->tick() == tick + 192,
|
||||
"player operation did not materialize first");
|
||||
host->drain();
|
||||
}
|
||||
@@ -474,6 +633,7 @@ void run_population_publication_tests() {
|
||||
Fixture fixture;
|
||||
funded_source(fixture);
|
||||
conversions_and_retries(fixture);
|
||||
interval_contract(fixture);
|
||||
ambiguous_publication(fixture, false);
|
||||
ambiguous_publication(fixture, true);
|
||||
process_loss(fixture, 0);
|
||||
|
||||
Reference in New Issue
Block a user