M12.3: publish and recover durable exact populations
Foundation / checks (ecs) (push) Successful in 2m41s
Foundation / checks (tsan) (push) Failing after 18m12s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (network) (push) Canceled after 20m41s
Foundation / checks (postgres) (push) Canceled after 20m43s
Foundation / checks (checks) (push) Canceled after 20m49s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (client) (push) Failing after 17m4s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / checks (ecs) (push) Successful in 2m41s
Foundation / checks (tsan) (push) Failing after 18m12s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (network) (push) Canceled after 20m41s
Foundation / checks (postgres) (push) Canceled after 20m43s
Foundation / checks (checks) (push) Canceled after 20m49s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (client) (push) Failing after 17m4s
Foundation / postgres-tools (push) Canceled after 0s
This commit is contained in:
@@ -12,23 +12,6 @@ using namespace evo::server;
|
||||
using namespace evo::fixture;
|
||||
using Population = std::unique_ptr<CellularPopulation>;
|
||||
|
||||
Owner descendants() {
|
||||
auto state = opening(1, true);
|
||||
std::set<std::uint64_t> mutated;
|
||||
for (unsigned step = 0; step < 8192; ++step) {
|
||||
const auto input = inputs(*state, 1, mutated);
|
||||
const auto executed =
|
||||
checked(state->advance_tick_with_mutations(input.lives, input.mutations),
|
||||
"descendant fixture tick");
|
||||
for (const auto& mutation : executed.mutations)
|
||||
if (mutation.result == CellularMutationResult::applied)
|
||||
mutated.insert(mutation.life_id);
|
||||
}
|
||||
require(std::ranges::any_of(state->lives(),
|
||||
[](const auto& life) { return life.account == LifeAccount{}; }),
|
||||
"fixture has no funded descendant");
|
||||
return state;
|
||||
}
|
||||
Population restored(std::span<const std::uint8_t> bytes, const CellularLives& state) {
|
||||
return checked(CellularPopulation::restore(bytes, state.world(), state.region(), state.frame(),
|
||||
state.tick()),
|
||||
@@ -64,7 +47,8 @@ void refresh(std::vector<std::uint8_t>& bytes) {
|
||||
put(bytes, bytes.size() - 8, hash);
|
||||
}
|
||||
void conversions_and_codec() {
|
||||
auto population = checked(CellularPopulation::create(descendants()), "population create");
|
||||
auto population =
|
||||
checked(CellularPopulation::create(cellular_descendants()), "population create");
|
||||
const auto genome = wildlife_genome(*population);
|
||||
const auto before = population->snapshot();
|
||||
const auto biology = population->state().snapshot();
|
||||
@@ -129,7 +113,7 @@ void conversions_and_codec() {
|
||||
}
|
||||
|
||||
void automatic_and_restart() {
|
||||
auto initial = descendants();
|
||||
auto initial = cellular_descendants();
|
||||
auto policy = checked(CellularPopulation::create(restore(*initial), {true, 1200, 1800}),
|
||||
"automatic owner");
|
||||
auto exact = checked(CellularPopulation::create(restore(*initial)), "individual control");
|
||||
@@ -170,8 +154,9 @@ void write(const char* path, const CellularPopulation& population) {
|
||||
require(!stream.fail(), "test checkpoint write failed");
|
||||
}
|
||||
void checkpoint(const char* path, const char* expected) {
|
||||
auto population = checked(CellularPopulation::create(descendants(), {true, 1200, 1800}),
|
||||
"process checkpoint owner");
|
||||
auto population =
|
||||
checked(CellularPopulation::create(cellular_descendants(), {true, 1200, 1800}),
|
||||
"process checkpoint owner");
|
||||
for (unsigned tick = 0; tick < 250; ++tick)
|
||||
frozen_tick(*population);
|
||||
require(!population->members().empty(), "process checkpoint has no cohort");
|
||||
|
||||
Vendored
+21
-1
@@ -44,7 +44,8 @@ inline Owner restore(const CellularLives& state) {
|
||||
"reference restore");
|
||||
}
|
||||
|
||||
inline Owner opening(std::uint64_t seed, bool rich, std::uint32_t count = 4) {
|
||||
inline Owner opening(std::uint64_t seed, bool rich, std::uint32_t count = 4,
|
||||
bool populated = true) {
|
||||
require(count > 0 && count <= biology::max_cellular_tick_lives, "fixture population bound");
|
||||
const std::uint32_t side = count == 4 ? 6'000 : 48'000;
|
||||
const auto world = 200 + seed;
|
||||
@@ -106,6 +107,8 @@ inline Owner opening(std::uint64_t seed, bool rich, std::uint32_t count = 4) {
|
||||
checked(CellularLives::create(*primitive, std::move(habitat), {{side, side}, 0, 293'150},
|
||||
CellularLivesProfile::foraging),
|
||||
"reference cellular owner");
|
||||
if (!populated)
|
||||
return state;
|
||||
require(state->advance_tick(std::span<const CellularLifeIntent>{}).has_value(),
|
||||
"reference genesis tick");
|
||||
for (std::uint64_t index = 0; index < count; ++index) {
|
||||
@@ -177,4 +180,21 @@ inline CellularTickIntents inputs(const CellularLives& state, std::uint64_t seed
|
||||
return tick;
|
||||
}
|
||||
|
||||
inline Owner cellular_descendants() {
|
||||
auto state = opening(1, true);
|
||||
std::set<std::uint64_t> mutated;
|
||||
for (unsigned step = 0; step < 8192; ++step) {
|
||||
const auto input = inputs(*state, 1, mutated);
|
||||
const auto executed =
|
||||
checked(state->advance_tick_with_mutations(input.lives, input.mutations),
|
||||
"descendant fixture tick");
|
||||
for (const auto& mutation : executed.mutations)
|
||||
if (mutation.result == CellularMutationResult::applied)
|
||||
mutated.insert(mutation.life_id);
|
||||
}
|
||||
require(std::ranges::any_of(state->lives(),
|
||||
[](const auto& life) { return life.account == LifeAccount{}; }),
|
||||
"fixture has no funded descendant");
|
||||
return state;
|
||||
}
|
||||
} // namespace evo::fixture
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
#include "evo/persistence/region_checkpoint.hpp"
|
||||
#include "evo/server/lifecycle/cellular_population_batch.hpp"
|
||||
#include "fixtures/cellular_ecology.hpp"
|
||||
|
||||
namespace {
|
||||
using namespace evo;
|
||||
using namespace evo::server;
|
||||
using namespace evo::persistence;
|
||||
using namespace evo::fixture;
|
||||
template <std::size_t N> std::array<std::uint8_t, N> filled(std::uint8_t byte) {
|
||||
std::array<std::uint8_t, N> out{};
|
||||
out.fill(byte);
|
||||
return out;
|
||||
}
|
||||
CellularBatchHeader header(const CellularPopulation& state, std::uint64_t ticks) {
|
||||
const auto& s = state.state();
|
||||
return {s.world(), s.region(), s.frame(), filled<16>(1),
|
||||
filled<32>(2), s.tick(), s.tick() + ticks};
|
||||
}
|
||||
CellularBatch inputs_batch(const CellularPopulation& base) {
|
||||
CellularBatch batch{header(base, 64), {}};
|
||||
auto biological = restore(base.state());
|
||||
for (unsigned tick = 0; tick < 64; ++tick) {
|
||||
auto inputs =
|
||||
checked(wildlife_intents(*biological, biological->tick() + 1), "batch wildlife");
|
||||
for (const auto& life : biological->lives()) {
|
||||
if (life.account == LifeAccount{})
|
||||
continue;
|
||||
CellularLifeIntent frozen{life.identity.life, {}};
|
||||
frozen.intent.frozen = true;
|
||||
inputs.push_back(frozen);
|
||||
}
|
||||
std::ranges::sort(inputs, {}, &CellularLifeIntent::life_id);
|
||||
batch.ticks.push_back({inputs, {}, {}});
|
||||
checked(biological->advance_tick(inputs), "build biological input fixture");
|
||||
}
|
||||
return batch;
|
||||
}
|
||||
void traces() {
|
||||
auto population = checked(
|
||||
CellularPopulation::create(cellular_descendants(), {true, 1200, 1800}), "batch population");
|
||||
std::uint64_t converted = 0;
|
||||
for (unsigned interval = 0; interval < 6; ++interval) {
|
||||
auto inputs = inputs_batch(*population);
|
||||
require(checked(unattended_population_inputs(population->state(), inputs.header),
|
||||
"unattended worker batch") == inputs,
|
||||
"worker interval changed frozen/wildlife input contract");
|
||||
const auto predecessor = population->snapshot();
|
||||
auto execution = checked(prepare_population_batch(*population, inputs), "prepare trace");
|
||||
const auto bytes = checked(encode_population_batch(execution.batch), "encode trace");
|
||||
const auto decoded = checked(decode_population_batch(bytes), "decode trace");
|
||||
require(decoded == execution.batch, "population trace round trip");
|
||||
auto replay = checked(replay_population_batch(*population, decoded), "replay trace");
|
||||
require(replay->snapshot() == execution.state->snapshot() &&
|
||||
population->snapshot() == predecessor,
|
||||
"trace changed predecessor or replay result");
|
||||
auto altered = decoded;
|
||||
++altered.boundaries.back().sequence;
|
||||
require(!replay_population_batch(*population, altered), "altered sequence admitted");
|
||||
altered = decoded;
|
||||
altered.biology.ticks.front().lives.front().intent.frozen = false;
|
||||
require(!replay_population_batch(*population, altered),
|
||||
"unfrozen controlled step admitted");
|
||||
for (std::size_t length = 0; length < bytes.size(); length += 113)
|
||||
require(!decode_population_batch(std::span{bytes}.first(length)),
|
||||
"truncated trace admitted");
|
||||
auto extended = bytes;
|
||||
extended.push_back(0);
|
||||
require(!decode_population_batch(extended), "appended trace bytes admitted");
|
||||
converted += replay->members().size();
|
||||
population = std::move(replay);
|
||||
}
|
||||
require(converted > 0 && !population->members().empty(), "batch test has no cohorts");
|
||||
const auto genome = population->members().front().genome;
|
||||
const auto before = population->state().snapshot();
|
||||
auto conversion =
|
||||
checked(prepare_population_conversion(*population, header(*population, 0), {genome, false}),
|
||||
"manual trace");
|
||||
const auto encoded = checked(encode_population_batch(conversion.batch), "manual encode");
|
||||
const auto decoded = checked(decode_population_batch(encoded), "manual decode");
|
||||
auto replay = checked(replay_population_batch(*population, decoded), "manual replay");
|
||||
require(replay->state().snapshot() == before && replay->members().empty(),
|
||||
"manual materialization changed biology");
|
||||
}
|
||||
void manifests() {
|
||||
auto state = cellular_descendants();
|
||||
auto population = checked(CellularPopulation::create(restore(*state), {true, 1200, 1800}),
|
||||
"manifest population");
|
||||
CheckpointManifest source;
|
||||
source.profile = CheckpointProfile::cellular_lives_v1;
|
||||
source.cellular_versions = cellular_v9_versions;
|
||||
source.history_profile = cellular_revision_history_profile;
|
||||
source.key = {state->world(), state->region()};
|
||||
source.frame = state->frame();
|
||||
source.tick_microseconds = 50000;
|
||||
source.sequence = 2;
|
||||
source.commit = filled<16>(1);
|
||||
source.predecessor_commit = filled<16>(2);
|
||||
source.predecessor_digest = filled<32>(3);
|
||||
source.holder = filled<16>(4);
|
||||
source.epoch = 1;
|
||||
source.begin_tick = state->tick() - 1;
|
||||
source.end_tick = state->tick();
|
||||
source.build = filled<32>(5);
|
||||
source.payload_bytes = state->snapshot().size();
|
||||
source.payload_sha256 = filled<32>(6);
|
||||
source.batch = {128, filled<32>(7), 0, {}};
|
||||
require(encode_manifest(source).has_value(), "source manifest fixture");
|
||||
LeaseGrant lease{filled<16>(4), 1, state->frame(), std::chrono::milliseconds{20000}};
|
||||
auto next = checked(population_profile_transition_manifest(
|
||||
source, filled<32>(8), filled<16>(9), lease, source.build,
|
||||
population->snapshot().size(), filled<32>(10)),
|
||||
"profile transition");
|
||||
require(next.profile == CheckpointProfile::cellular_population_v1 && supported(next),
|
||||
"wrong profile");
|
||||
const auto encoded = checked(encode_manifest(next), "population manifest");
|
||||
require(checked(decode_manifest(encoded), "manifest decode") == next,
|
||||
"manifest profile changed");
|
||||
auto restored_population =
|
||||
checked(restore_population_payload(population->snapshot(), next), "profile payload");
|
||||
require(restored_population->snapshot() == population->snapshot(),
|
||||
"population payload changed");
|
||||
require(!restore_cellular_payload(population->snapshot(), next),
|
||||
"population mislabeled as cellular");
|
||||
auto back = checked(population_profile_transition_manifest(
|
||||
next, filled<32>(11), filled<16>(12), lease, source.build,
|
||||
state->snapshot().size(), filled<32>(13)),
|
||||
"reverse transition");
|
||||
require(back.end_tick == source.end_tick && back.profile == source.profile &&
|
||||
checked(restore_cellular_payload(state->snapshot(), back), "unwrapped payload")
|
||||
->snapshot() == state->snapshot(),
|
||||
"profile transition changed biology/time");
|
||||
auto execution = checked(prepare_population_batch(*population, inputs_batch(*population)),
|
||||
"manifest tick fixture");
|
||||
const auto trace = checked(encode_population_batch(execution.batch), "manifest trace fixture");
|
||||
const auto advancing = population_successor_manifest(
|
||||
next, filled<32>(11), filled<16>(12), lease, execution.state->state().tick(), source.build,
|
||||
execution.state->snapshot().size(), filled<32>(13), {trace.size(), filled<32>(14), 0, {}});
|
||||
require(advancing.has_value() &&
|
||||
restore_population_payload(execution.state->snapshot(), *advancing).has_value(),
|
||||
"population successor refused");
|
||||
require(!clone_manifest(next, filled<32>(1), {9999, 1}, filled<16>(3), 100, filled<32>(4), {}),
|
||||
"population clone admitted without identity remapping");
|
||||
auto malformed = next;
|
||||
malformed.sequence = 0;
|
||||
require(!encode_manifest(malformed), "population genesis admitted");
|
||||
malformed = next;
|
||||
malformed.cellular_versions = cellular_v8_versions;
|
||||
require(!supported(malformed), "unsupported nested rules admitted");
|
||||
}
|
||||
} // namespace
|
||||
int main() try {
|
||||
traces();
|
||||
manifests();
|
||||
return 0;
|
||||
} catch (const std::exception& error) {
|
||||
std::fprintf(stderr, "%s\n", error.what());
|
||||
return 1;
|
||||
}
|
||||
@@ -0,0 +1,486 @@
|
||||
#include "evo/persistence/account_control_store.hpp"
|
||||
#include "evo/persistence/population_checkpoint_store.hpp"
|
||||
#include "evo/persistence/sha256.hpp"
|
||||
#include "evo/server/cellular_region_host.hpp"
|
||||
#include "fixtures/cellular_ecology.hpp"
|
||||
|
||||
#include <csignal>
|
||||
#include <cstdlib>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <stdexcept>
|
||||
#include <sys/wait.h>
|
||||
#include <thread>
|
||||
#include <type_traits>
|
||||
#include <unistd.h>
|
||||
|
||||
void run_population_publication_tests();
|
||||
namespace {
|
||||
using namespace evo;
|
||||
using namespace evo::server;
|
||||
using namespace evo::persistence;
|
||||
using namespace evo::fixture;
|
||||
using namespace std::chrono_literals;
|
||||
template <typename T, typename E> T take(std::expected<T, E> value, const char* where) {
|
||||
if (!value) {
|
||||
std::string message{where};
|
||||
if constexpr (requires { value.error().cause.sqlstate; })
|
||||
message += " SQLSTATE=" + std::string(value.error().cause.sqlstate.data(), 5);
|
||||
if constexpr (requires { value.error().store.cause.sqlstate; })
|
||||
message += " SQLSTATE=" + std::string(value.error().store.cause.sqlstate.data(), 5);
|
||||
throw std::runtime_error(message);
|
||||
}
|
||||
if constexpr (!std::is_void_v<T>)
|
||||
return std::move(*value);
|
||||
}
|
||||
template <typename T> T take(std::optional<T> value, const char* where) {
|
||||
if (!value.has_value())
|
||||
throw std::runtime_error(where);
|
||||
return std::move(*value);
|
||||
}
|
||||
std::string environment(const char* name) {
|
||||
const auto* value = std::getenv(name);
|
||||
require(value != nullptr, "missing private test environment");
|
||||
return value;
|
||||
}
|
||||
std::unique_ptr<Postgres> connection(bool owner = false) {
|
||||
return take(Postgres::connect({environment("PGHOST"),
|
||||
{},
|
||||
environment("PGPORT"),
|
||||
environment("PGDATABASE"),
|
||||
environment(owner ? "PGUSER" : "EVO_PG_RUNTIME_USER"),
|
||||
{},
|
||||
{}},
|
||||
5s),
|
||||
"population connection");
|
||||
}
|
||||
Uuid identity(unsigned number) {
|
||||
Uuid id{};
|
||||
id[0] = 0xDA;
|
||||
id[1] = 0x12;
|
||||
id[14] = static_cast<std::uint8_t>(number >> 8U);
|
||||
id[15] = static_cast<std::uint8_t>(number);
|
||||
return id;
|
||||
}
|
||||
std::string hexadecimal(std::span<const std::uint8_t> bytes, bool prefix = true) {
|
||||
std::string out = prefix ? "\\x" : "";
|
||||
constexpr char digits[] = "0123456789abcdef";
|
||||
for (const auto b : bytes) {
|
||||
out.push_back(digits[b >> 4U]);
|
||||
out.push_back(digits[b & 15U]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
struct Fixture {
|
||||
RegionKey key{30201, 1};
|
||||
std::unique_ptr<Postgres> owner = connection(true), db = connection();
|
||||
std::unique_ptr<FilesystemObjectStore> objects;
|
||||
LeaseGrant lease;
|
||||
Sha256Digest build{};
|
||||
CommittedCheckpoint head;
|
||||
std::unique_ptr<CellularLives> raw;
|
||||
std::unique_ptr<CellularPopulation> population;
|
||||
unsigned next_id = 1;
|
||||
Uuid id() {
|
||||
return identity(next_id++);
|
||||
}
|
||||
CellularBatchHeader header(std::uint64_t count) const {
|
||||
return {key.world,
|
||||
key.region,
|
||||
1,
|
||||
head.head.commit,
|
||||
head.head.digest,
|
||||
head.head.end_tick,
|
||||
head.head.end_tick + count};
|
||||
}
|
||||
};
|
||||
void funded_source(Fixture& f) {
|
||||
f.build.fill(0xDA);
|
||||
const std::array<std::string, 1> build{hexadecimal(f.build)};
|
||||
take(f.owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES($1::bytea) "
|
||||
"ON CONFLICT DO NOTHING",
|
||||
build, 5s),
|
||||
"retain test build");
|
||||
take(f.owner->query("INSERT INTO evolution.life_allocators VALUES(30201,100)", {}, 5s),
|
||||
"test allocator");
|
||||
f.objects = take(
|
||||
FilesystemObjectStore::open(environment("PGHOST"), f.key, max_cellular_population_bytes),
|
||||
"population object namespace");
|
||||
f.raw = opening(30001, true, 4, false);
|
||||
f.head =
|
||||
take(provision_cellular_region(*f.owner, *f.objects, f.key, 1, *f.raw, f.id(), f.build, 5s),
|
||||
"empty cellular genesis");
|
||||
f.lease =
|
||||
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "population test lease");
|
||||
CellularBatch birth{f.header(1), {{}}};
|
||||
for (std::uint8_t index = 1; index <= 4; ++index) {
|
||||
Uuid account{};
|
||||
account.back() = index;
|
||||
const auto control =
|
||||
take(claim_account_control(*f.db, f.key, f.lease, account, 1, index, 60000ms, 5s),
|
||||
"founder account control");
|
||||
const auto reserved = take(
|
||||
reserve_primitive_life(*f.db, f.key, f.lease, account, control.generation, 1, 1, 5s),
|
||||
"funded founder reservation");
|
||||
birth.ticks.front().births.push_back({account, 1, control.generation, reserved.identity});
|
||||
}
|
||||
f.raw = take(replay_cellular_batch(*f.raw, birth), "funded founder replay");
|
||||
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, birth, *f.raw, f.id(),
|
||||
f.build, 5s),
|
||||
"funded founder publication")
|
||||
.committed;
|
||||
while (f.raw->next_bud_sequence() == 1 && f.raw->tick() < 20000) {
|
||||
CellularBatch batch{f.header(1), {}};
|
||||
for (unsigned tick = 0; tick < 64; ++tick) {
|
||||
auto intents =
|
||||
take(wildlife_intents(*f.raw, f.raw->tick() + 1), "funded source wildlife");
|
||||
for (const auto& life : f.raw->lives()) {
|
||||
if (life.account == LifeAccount{})
|
||||
continue;
|
||||
CellularLifeIntent intent{life.identity.life, {}};
|
||||
intent.intent.bud = (f.raw->tick() + life.identity.life) % 32 == 0;
|
||||
intents.push_back(intent);
|
||||
}
|
||||
std::ranges::sort(intents, {}, &CellularLifeIntent::life_id);
|
||||
take(f.raw->advance_tick(intents), "funded source biology");
|
||||
batch.ticks.push_back({intents, {}, {}});
|
||||
if (f.raw->next_bud_sequence() > 1)
|
||||
break;
|
||||
}
|
||||
batch.header.end_tick = f.raw->tick();
|
||||
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, batch, *f.raw,
|
||||
f.id(), f.build, 5s),
|
||||
"funded source batch")
|
||||
.committed;
|
||||
take(renew_region_lease(*f.db, f.key, f.lease, 60000ms, 5s), "source lease renewal");
|
||||
}
|
||||
require(std::ranges::any_of(f.raw->lives(),
|
||||
[](const auto& life) { return life.account == LifeAccount{}; }),
|
||||
"durable fixture has no funded player-derived wildlife");
|
||||
}
|
||||
CellularPopulationExecution unattended(Fixture& f) {
|
||||
auto working = restore(f.population->state());
|
||||
CellularBatch batch{f.header(64), {}};
|
||||
for (unsigned tick = 0; tick < 64; ++tick) {
|
||||
auto intents = take(wildlife_intents(*working, working->tick() + 1), "unattended wildlife");
|
||||
for (const auto& life : working->lives()) {
|
||||
if (life.account == LifeAccount{})
|
||||
continue;
|
||||
CellularLifeIntent frozen{life.identity.life, {}};
|
||||
frozen.intent.frozen = true;
|
||||
intents.push_back(frozen);
|
||||
}
|
||||
std::ranges::sort(intents, {}, &CellularLifeIntent::life_id);
|
||||
take(working->advance_tick(intents), "unattended fixture step");
|
||||
batch.ticks.push_back({intents, {}, {}});
|
||||
}
|
||||
return take(prepare_population_batch(*f.population, std::move(batch)),
|
||||
"population recorded interval");
|
||||
}
|
||||
void recover(Fixture& f) {
|
||||
const std::array<Sha256Digest, 1> builds{f.build};
|
||||
auto db = connection();
|
||||
auto recovered =
|
||||
take(recover_population_checkpoint(*db, *f.objects, f.key, f.lease, builds, 5s),
|
||||
"population durable recovery");
|
||||
f.head = std::move(recovered.committed);
|
||||
f.population = std::move(recovered.state);
|
||||
}
|
||||
void conversions_and_retries(Fixture& f) {
|
||||
const auto biology = f.raw->snapshot();
|
||||
Uuid account{};
|
||||
account.back() = 1;
|
||||
const auto input =
|
||||
encode_cellular_mutation_input({account, 9991, f.head.head.end_tick + 1, {100, 1, 1, 1}});
|
||||
take(f.db->query("BEGIN", {}, 5s), "reservation guard transaction");
|
||||
take(reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 9991, input, 5s),
|
||||
"reserve pending player choice");
|
||||
const auto blocked = migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
|
||||
f.build, {true, 1200, 1800}, 5s);
|
||||
require(!blocked && std::string(blocked.error().store.cause.sqlstate.data(), 5) == "EVL18",
|
||||
"population entry stranded a reserved player choice");
|
||||
take(f.db->query("ROLLBACK", {}, 5s), "rollback guard fixture");
|
||||
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
|
||||
f.build, {true, 1200, 1800}, 5s),
|
||||
"initial population transition");
|
||||
recover(f);
|
||||
require(f.population->state().snapshot() == biology && f.population->members().empty(),
|
||||
"transition changed biology");
|
||||
const auto refused =
|
||||
reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 9991, input, 5s);
|
||||
require(!refused && std::string(refused.error().cause.sqlstate.data(), 5) == "EVL06",
|
||||
"population profile admitted a player mutation reservation");
|
||||
const auto wildlife = std::ranges::find_if(f.population->state().lives(), [](const auto& life) {
|
||||
return life.account == LifeAccount{};
|
||||
});
|
||||
require(wildlife != f.population->state().lives().end(), "missing durable descendant");
|
||||
const auto genome = wildlife->identity.genome;
|
||||
auto conversion =
|
||||
take(prepare_population_conversion(*f.population, f.header(0), {genome, true}),
|
||||
"collapse sample");
|
||||
auto bad = conversion.batch;
|
||||
++bad.boundaries.front().sequence;
|
||||
require(!publish_population_batch(*f.db, *f.objects, f.lease, f.head, bad, *conversion.state,
|
||||
f.id(), f.build, 5s),
|
||||
"altered sample published");
|
||||
const auto commit = f.id();
|
||||
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, conversion.batch,
|
||||
*conversion.state, commit, f.build, 5s),
|
||||
"durable collapse");
|
||||
const PopulationPreparedCommit prepared{
|
||||
f.key, commit, f.head.manifest,
|
||||
take(encode_population_batch(conversion.batch), "prepared trace")};
|
||||
require(take(retry_population_prepared(*f.db, prepared, 5s), "idempotent retry").head ==
|
||||
f.head.head,
|
||||
"retry advanced head");
|
||||
require(take(resolve_population_prepared(*f.db, prepared, 5s), "read-only resolve").status ==
|
||||
Resolution::committed,
|
||||
"commit unresolved");
|
||||
auto changed = prepared;
|
||||
auto changed_manifest = take(decode_manifest(changed.manifest), "conflict manifest");
|
||||
changed_manifest.payload_sha256[0] ^= 1;
|
||||
changed.manifest = take(encode_manifest(changed_manifest), "conflict encoding");
|
||||
require(take(resolve_population_prepared(*f.db, changed, 5s), "conflicting resolve").status ==
|
||||
Resolution::conflict,
|
||||
"changed commit was accepted");
|
||||
require(!publish_region_checkpoint(*f.db, f.head.manifest, {}, 5s),
|
||||
"generic publisher bypassed profile gate");
|
||||
const auto expected = conversion.state->snapshot();
|
||||
recover(f);
|
||||
require(f.population->snapshot() == expected && !f.population->members().empty(),
|
||||
"committed sample not recovered");
|
||||
require(f.population->state().snapshot() == biology, "collapse healed/aged/moved biology");
|
||||
for (unsigned oscillation = 0; oscillation < 8; ++oscillation) {
|
||||
const bool collapse = oscillation % 2 != 0;
|
||||
auto next =
|
||||
take(prepare_population_conversion(*f.population, f.header(0), {genome, collapse}),
|
||||
"conversion oscillation");
|
||||
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, next.batch,
|
||||
*next.state, f.id(), f.build, 5s),
|
||||
"publish oscillation");
|
||||
recover(f);
|
||||
require(f.population->state().snapshot() == biology,
|
||||
"durable conversion changed exact state");
|
||||
}
|
||||
}
|
||||
void process_loss(Fixture& f, int point) {
|
||||
auto next = unattended(f);
|
||||
const auto predecessor = f.head.head;
|
||||
const auto expected = next.state->snapshot();
|
||||
const auto commit = f.id();
|
||||
int channel[2]{};
|
||||
require(::pipe(channel) == 0, "process acknowledgement pipe");
|
||||
const auto child = ::fork();
|
||||
require(child >= 0, "population fork");
|
||||
if (child == 0) {
|
||||
::close(channel[0]);
|
||||
try {
|
||||
auto db = connection();
|
||||
PublishHooks hooks;
|
||||
if (point == 0)
|
||||
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
|
||||
const auto published =
|
||||
publish_population_batch(*db, *f.objects, f.lease, f.head, next.batch, *next.state,
|
||||
commit, f.build, 5s, hooks);
|
||||
if (!published)
|
||||
::_exit(20);
|
||||
if (point == 2) {
|
||||
const char ack = 'A';
|
||||
if (::write(channel[1], &ack, 1) != 1)
|
||||
::_exit(21);
|
||||
}
|
||||
::kill(::getpid(), SIGKILL);
|
||||
} catch (...) {
|
||||
::_exit(22);
|
||||
}
|
||||
::_exit(23);
|
||||
}
|
||||
::close(channel[1]);
|
||||
char ack{};
|
||||
const auto acknowledged = ::read(channel[0], &ack, 1);
|
||||
::close(channel[0]);
|
||||
int status{};
|
||||
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
|
||||
WTERMSIG(status) == SIGKILL,
|
||||
"child missed population crash boundary");
|
||||
require((point == 2 && acknowledged == 1 && ack == 'A') || (point != 2 && acknowledged == 0),
|
||||
"unexpected permanent acknowledgement");
|
||||
recover(f);
|
||||
if (point == 0)
|
||||
require(f.head.head == predecessor, "uncommitted process published a head");
|
||||
else
|
||||
require(f.head.head.commit == commit && f.population->snapshot() == expected,
|
||||
"committed process loss changed state");
|
||||
}
|
||||
void ambiguous_publication(Fixture& f, bool terminate) {
|
||||
auto next = unattended(f);
|
||||
const auto expected = next.state->snapshot();
|
||||
auto locker = connection(true);
|
||||
take(locker->query("BEGIN", {}, 5s), "population lease row transaction");
|
||||
const std::array<std::string, 2> key{std::to_string(f.key.world), std::to_string(f.key.region)};
|
||||
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=$1::evolution.u64 "
|
||||
"AND region_id=$2::evolution.u64 FOR UPDATE",
|
||||
key, 5s),
|
||||
"hold population publication");
|
||||
auto timed = connection();
|
||||
auto unknown = publish_population_batch(*timed, *f.objects, f.lease, f.head, next.batch,
|
||||
*next.state, f.id(), f.build, 300ms);
|
||||
require(!unknown && unknown.error().kind == PublicationError::unknown,
|
||||
"population publication failed to retain ambiguous request");
|
||||
if (terminate) {
|
||||
const std::array<std::string, 1> user{environment("EVO_PG_RUNTIME_USER")};
|
||||
take(locker->query("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE "
|
||||
"usename=$1 AND wait_event_type='Lock'",
|
||||
user, 5s),
|
||||
"terminate uncommitted population writer");
|
||||
}
|
||||
take(locker->query("ROLLBACK", {}, 5s), "release population publication");
|
||||
auto resolver = connection();
|
||||
bool resolved = false;
|
||||
for (unsigned attempt = 0; attempt < 200; ++attempt) {
|
||||
auto result = take(resolve_population_prepared(*resolver, unknown.error().prepared, 5s),
|
||||
"resolve ambiguous population publication");
|
||||
if (result.status == (terminate ? Resolution::absent : Resolution::committed)) {
|
||||
resolved = true;
|
||||
break;
|
||||
}
|
||||
std::this_thread::sleep_for(25ms);
|
||||
}
|
||||
require(resolved, "ambiguous population publication did not resolve");
|
||||
f.head = take(retry_population_prepared(*resolver, unknown.error().prepared, 5s),
|
||||
"retry exact population request");
|
||||
recover(f);
|
||||
require(f.population->snapshot() == expected, "ambiguous publication changed retained sample");
|
||||
}
|
||||
void isolation_and_reverse(Fixture& f) {
|
||||
auto interval = unattended(f);
|
||||
const auto old_lease = f.lease;
|
||||
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release old authority");
|
||||
auto new_db = connection();
|
||||
f.lease = take(acquire_region_lease(*new_db, f.key, f.build, 60000ms, 5s),
|
||||
"new population authority");
|
||||
require(f.lease.epoch > old_lease.epoch, "authority epoch did not advance");
|
||||
const auto old_head = f.head.head;
|
||||
auto stale = publish_population_batch(*f.db, *f.objects, old_lease, f.head, interval.batch,
|
||||
*interval.state, f.id(), f.build, 5s);
|
||||
require(!stale && stale.error().store.kind == LeaseStoreError::fenced,
|
||||
"stale population writer published");
|
||||
recover(f);
|
||||
require(f.head.head == old_head, "fenced writer changed head");
|
||||
const auto expected = f.population->state().snapshot();
|
||||
f.head = take(
|
||||
materialize_population_profile(*new_db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
|
||||
"return to cellular profile");
|
||||
const std::array<Sha256Digest, 1> builds{f.build};
|
||||
auto recovered =
|
||||
take(recover_cellular_checkpoint(*new_db, *f.objects, f.key, f.lease, builds, 5s),
|
||||
"cellular recovery after materialization");
|
||||
require(recovered.state->snapshot() == expected &&
|
||||
recovered.manifest.profile == CheckpointProfile::cellular_lives_v1,
|
||||
"reverse transition changed biology or time");
|
||||
f.raw = std::move(recovered.state);
|
||||
}
|
||||
void missing_and_corrupt_objects(Fixture& f) {
|
||||
const auto manifest = take(decode_manifest(f.head.manifest), "object test manifest");
|
||||
const auto path = f.objects->directory() / hexadecimal(manifest.payload_sha256, false);
|
||||
const auto backup = path.string() + ".population-test";
|
||||
const std::array<Sha256Digest, 1> builds{f.build};
|
||||
std::filesystem::rename(path, backup);
|
||||
auto missing = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
|
||||
std::filesystem::rename(backup, path);
|
||||
require(!missing && missing.error().kind == RecoveryError::missing_payload,
|
||||
"missing population payload did not fail closed");
|
||||
std::filesystem::rename(path, backup);
|
||||
{
|
||||
std::ofstream corrupt(path, std::ios::binary);
|
||||
corrupt << 'X';
|
||||
}
|
||||
auto corrupt = recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
|
||||
std::filesystem::remove(path);
|
||||
std::filesystem::rename(backup, path);
|
||||
require(!corrupt && corrupt.error().kind == RecoveryError::corrupt_payload,
|
||||
"corrupt population payload did not fail closed");
|
||||
recover(f);
|
||||
}
|
||||
void host_ownership_and_admission(Fixture& f) {
|
||||
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture host authority");
|
||||
const auto make_host = [&](bool enabled) {
|
||||
auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), f.key,
|
||||
max_cellular_population_bytes),
|
||||
"host objects");
|
||||
CellularHostConfig config{f.key, 1, {}, f.build, {f.build}};
|
||||
if (enabled)
|
||||
config.population_policy = CellularTierPolicy{true, 1200, 1800};
|
||||
return take(CellularRegionHost::create(
|
||||
std::move(config),
|
||||
[] {
|
||||
return Postgres::connect({environment("PGHOST"),
|
||||
{},
|
||||
environment("PGPORT"),
|
||||
environment("PGDATABASE"),
|
||||
environment("EVO_PG_RUNTIME_USER"),
|
||||
{},
|
||||
{}},
|
||||
5s);
|
||||
},
|
||||
std::move(objects)),
|
||||
"population host");
|
||||
};
|
||||
auto host = make_host(false);
|
||||
require(host->step().ready && !host->population_mode(), "raw host recovery");
|
||||
require(!host->set_population_mode(true), "default host enabled population entry");
|
||||
host->drain();
|
||||
host = make_host(true);
|
||||
require(host->step().ready, "opt-in host recovery");
|
||||
const auto before = host->state()->snapshot();
|
||||
const auto tick = host->state()->tick();
|
||||
take(host->set_population_mode(true), "host population entry");
|
||||
require(host->population_mode() && host->state()->snapshot() == before,
|
||||
"host entry changed biology");
|
||||
auto population =
|
||||
take(CellularPopulation::create(
|
||||
take(CellularLives::restore(before, f.key.world, f.key.region, 1, tick),
|
||||
"host reference restore"),
|
||||
{true, 1200, 1800}),
|
||||
"host reference population");
|
||||
f.head = take(host->committed(), "host population head");
|
||||
auto next = unattended(f); // uses the same deterministic frozen/wildlife input contract.
|
||||
// The fixture's old envelope may have a different membership history; biology is shared.
|
||||
auto expected =
|
||||
take(prepare_population_batch(*population, next.batch.biology), "host reference batch");
|
||||
take(host->publish(next.batch.biology), "host population publication");
|
||||
require(host->population_mode() &&
|
||||
host->state()->snapshot() == expected.state->state().snapshot(),
|
||||
"host population continuation differs");
|
||||
const auto committed = take(host->committed(), "host continued head");
|
||||
const auto continued = host->state()->snapshot();
|
||||
host->drain();
|
||||
host = make_host(false);
|
||||
require(host->step().ready && host->population_mode() &&
|
||||
take(host->committed(), "host recovered head").head == committed.head &&
|
||||
host->state()->snapshot() == continued,
|
||||
"restarted host failed to recover population head");
|
||||
Uuid account{};
|
||||
account.back() = 1;
|
||||
// A returning player's operation completes the durable reverse transition before lookup.
|
||||
[[maybe_unused]] const auto queried = host->query_primitive_life(account, 1, 1);
|
||||
require(!host->population_mode() && host->state()->snapshot() == continued &&
|
||||
host->state()->tick() == tick + 64,
|
||||
"player operation did not materialize first");
|
||||
host->drain();
|
||||
}
|
||||
} // namespace
|
||||
void run_population_publication_tests() {
|
||||
Fixture fixture;
|
||||
funded_source(fixture);
|
||||
conversions_and_retries(fixture);
|
||||
ambiguous_publication(fixture, false);
|
||||
ambiguous_publication(fixture, true);
|
||||
process_loss(fixture, 0);
|
||||
process_loss(fixture, 1);
|
||||
process_loss(fixture, 2);
|
||||
missing_and_corrupt_objects(fixture);
|
||||
isolation_and_reverse(fixture);
|
||||
host_ownership_and_admission(fixture);
|
||||
std::puts("population publication, exact recovery, process loss and fencing passed");
|
||||
}
|
||||
@@ -9,6 +9,7 @@ void run_account_control_store_tests();
|
||||
void run_life_reservation_store_tests();
|
||||
void run_primitive_publication_tests();
|
||||
void run_cellular_publication_tests();
|
||||
void run_population_publication_tests();
|
||||
void run_probe_handoff_store_tests();
|
||||
void run_region_action_tests();
|
||||
void run_region_checkpoint_store_tests();
|
||||
@@ -125,6 +126,10 @@ void run() {
|
||||
} // namespace
|
||||
int main(int argc, char** argv) {
|
||||
try {
|
||||
if (argc == 2 && std::string_view(argv[1]) == "--population-only") {
|
||||
run_population_publication_tests();
|
||||
return 0;
|
||||
}
|
||||
if (argc == 2 && std::string_view(argv[1]) == "--life-reservations-only") {
|
||||
run_life_reservation_store_tests();
|
||||
run_primitive_publication_tests();
|
||||
@@ -140,6 +145,7 @@ int main(int argc, char** argv) {
|
||||
run_life_reservation_store_tests();
|
||||
run_primitive_publication_tests();
|
||||
run_cellular_publication_tests();
|
||||
run_population_publication_tests();
|
||||
run_probe_handoff_store_tests();
|
||||
run_region_action_tests();
|
||||
std::puts("region lease fencing integration passed");
|
||||
|
||||
@@ -196,6 +196,20 @@ void upgrade_across_migration_and_barrier() {
|
||||
" -v migration_sha_28=$(s 0028_cellular_v3_trace_bounds.sql)"
|
||||
" -v migration_sha_29=$(s 0029_cellular_mutation_reservations.sql)"
|
||||
" -v migration_sha_30=$(s 0030_cellular_birth_publication_binding.sql)"
|
||||
" -v migration_sha_31=$(s 0031_fix_cellular_birth_publication_record.sql)"
|
||||
" -v migration_sha_32=$(s 0032_fix_cellular_v3_manifest_normalization.sql)"
|
||||
" -v migration_sha_33=$(s 0033_fix_cellular_action_tick_offset.sql)"
|
||||
" -v migration_sha_34=$(s 0034_cellular_v4_engulf_bundle.sql)"
|
||||
" -v migration_sha_35=$(s 0035_cellular_mutation_publication.sql)"
|
||||
" -v migration_sha_36=$(s 0036_cellular_v5_budding_bundle.sql)"
|
||||
" -v migration_sha_37=$(s 0037_cellular_life_endings.sql)"
|
||||
" -v migration_sha_38=$(s 0038_cellular_v6_presence_bundle.sql)"
|
||||
" -v migration_sha_39=$(s 0039_account_identities.sql)"
|
||||
" -v migration_sha_40=$(s 0040_cellular_mutation_decline.sql)"
|
||||
" -v migration_sha_41=$(s 0041_cellular_v7_balance_bundle.sql)"
|
||||
" -v migration_sha_42=$(s 0042_cellular_v8_reproduction_bundle.sql)"
|
||||
" -v migration_sha_43=$(s 0043_cellular_v9_foraging_bundle.sql)"
|
||||
" -v migration_sha_44=$(s 0044_exact_population_publication.sql)"
|
||||
" -v runtime_role=evo_runtime_test -f migrations/apply.sql"
|
||||
" > /dev/null 2>&1") != 0);
|
||||
CHECK(history(*owner_db) == "1,2,3,4,5,6,7");
|
||||
@@ -212,7 +226,7 @@ void upgrade_across_migration_and_barrier() {
|
||||
" EVO_PG_RUNTIME_ROLE=evo_runtime_test bash \"$EVO_SOURCE_ROOT/"
|
||||
"tools/migrate-postgres.sh\" > /dev/null") == 0);
|
||||
CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,"
|
||||
"25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43");
|
||||
"25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44");
|
||||
CHECK(scalar(*owner_db, "SELECT (c.condeferrable AND c.condeferred AND "
|
||||
"c.confrelid = 'evolution.region_commits'::regclass)::text "
|
||||
"FROM pg_catalog.pg_constraint c WHERE c.conrelid = "
|
||||
|
||||
Reference in New Issue
Block a user