feat(persistence): admit durable retained offspring plans
Foundation / checks (tsan) (push) Canceled after 0s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (client) (push) Canceled after 1m11s
Foundation / checks (checks) (push) Canceled after 1m13s
Foundation / checks (postgres) (push) Canceled after 1m8s
Foundation / checks (ecs) (push) Canceled after 1m13s
Foundation / checks (network) (push) Canceled after 1m14s
Foundation / checks (deploy) (push) Canceled after 1m18s
Foundation / checks (tsan) (push) Canceled after 0s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (client) (push) Canceled after 1m11s
Foundation / checks (checks) (push) Canceled after 1m13s
Foundation / checks (postgres) (push) Canceled after 1m8s
Foundation / checks (ecs) (push) Canceled after 1m13s
Foundation / checks (network) (push) Canceled after 1m14s
Foundation / checks (deploy) (push) Canceled after 1m18s
This commit is contained in:
@@ -24,6 +24,7 @@
|
||||
|
||||
void run_population_publication_tests();
|
||||
namespace {
|
||||
void retained_plan_admission();
|
||||
using namespace evo;
|
||||
using namespace evo::server;
|
||||
using namespace evo::persistence;
|
||||
@@ -3517,6 +3518,7 @@ void host_ownership_and_admission(Fixture& f) {
|
||||
}
|
||||
} // namespace
|
||||
void run_population_publication_tests() {
|
||||
retained_plan_admission();
|
||||
Fixture fixture;
|
||||
funded_source(fixture);
|
||||
history_pages(fixture);
|
||||
@@ -3594,3 +3596,315 @@ void run_population_publication_tests() {
|
||||
std::puts("rules-9 durable migration and post-settlement funded growth passed");
|
||||
std::puts("population publication, exact recovery, process loss and fencing passed");
|
||||
}
|
||||
|
||||
namespace {
|
||||
void retained_plan_admission() {
|
||||
Fixture f;
|
||||
f.key = {30205, 1};
|
||||
f.next_id = 55000;
|
||||
funded_source(f);
|
||||
const std::array<Sha256Digest, 1> builds{f.build};
|
||||
const auto reload = [&] {
|
||||
auto db = connection();
|
||||
auto loaded = take(recover_cellular_checkpoint(*db, *f.objects, f.key, f.lease, builds, 5s),
|
||||
"plan recovery");
|
||||
f.head = std::move(loaded.committed);
|
||||
f.raw = std::move(loaded.state);
|
||||
};
|
||||
auto skipped = take(cellular_boundary_upgrade_manifest(
|
||||
take(decode_manifest(f.head.manifest), "plan source"),
|
||||
f.head.head.digest, f.id(), f.lease, f.build, f.raw->snapshot().size(),
|
||||
take(sha256(f.raw->snapshot()), "plan source hash")),
|
||||
"skipped plan fixture");
|
||||
skipped.cellular_versions = cellular_v12_versions;
|
||||
require(!publish_cellular_profile_migration(*f.db,
|
||||
take(encode_manifest(skipped), "skip encode"), 5s),
|
||||
"SQL plan upgrade skipped prerequisites");
|
||||
f.head = take(
|
||||
migrate_cellular_boundary_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
|
||||
"plan boundary prerequisite");
|
||||
f.head = take(migrate_cellular_maturation_profile(*f.db, *f.objects, f.lease, f.head, f.id(),
|
||||
f.build, 5s),
|
||||
"plan maturation prerequisite");
|
||||
reload();
|
||||
const auto before = f.head;
|
||||
const auto source = take(decode_manifest(before.manifest), "retained plan source");
|
||||
const auto expected = take(f.raw->inheritance_upgrade_candidate(), "plan oracle")->snapshot();
|
||||
Uuid account{};
|
||||
account.back() = 1;
|
||||
const auto pending_input =
|
||||
encode_cellular_mutation_input({account, 57001, before.head.end_tick + 1, {100, 1, 1, 0}});
|
||||
take(f.db->query("BEGIN", {}, 5s), "pending plan transaction");
|
||||
take(reserve_cellular_mutation(*f.db, f.key, f.lease, account, 1, 57001, pending_input, 5s),
|
||||
"pending plan reservation");
|
||||
const auto pending = migrate_cellular_inheritance_profile(*f.db, *f.objects, f.lease, before,
|
||||
f.id(), f.build, 5s);
|
||||
require(!pending && std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18",
|
||||
"upgrade reinterpreted pending choice");
|
||||
take(f.db->query("ROLLBACK", {}, 5s), "discard pending plan fixture");
|
||||
// Valid same-cut biology with equal stock must still fail exact upgrade replay.
|
||||
auto false_payload = expected;
|
||||
const auto put = [](std::vector<std::uint8_t>& bytes, std::size_t offset, std::uint64_t value) {
|
||||
for (unsigned i = 0; i < 8; ++i)
|
||||
bytes.at(offset + i) = static_cast<std::uint8_t>(value >> (8U * i));
|
||||
};
|
||||
put(false_payload, 316 + 4 * 8 + 72, f.raw->lives().front().position.x_um + 1);
|
||||
std::uint64_t fnv = 14695981039346656037ULL;
|
||||
for (const auto byte : std::span{false_payload}.first(false_payload.size() - 8))
|
||||
fnv = (fnv ^ byte) * 1099511628211ULL;
|
||||
put(false_payload, false_payload.size() - 8, fnv);
|
||||
const auto false_object = take(f.objects->put(false_payload), "false plan object");
|
||||
const auto false_manifest = take(
|
||||
cellular_inheritance_upgrade_manifest(source, before.head.digest, f.id(), f.lease, f.build,
|
||||
false_object.bytes, false_object.sha256),
|
||||
"false plan manifest");
|
||||
require(take(restore_cellular_payload(false_payload, false_manifest), "valid false plan state")
|
||||
->current_total() == f.raw->current_total() &&
|
||||
false_payload != expected,
|
||||
"false plan fixture changed inventory");
|
||||
take(f.db->query("BEGIN", {}, 5s), "false plan transaction");
|
||||
take(publish_cellular_profile_migration(
|
||||
*f.db, take(encode_manifest(false_manifest), "false plan bytes"), 5s),
|
||||
"false plan metadata");
|
||||
const auto false_recovery =
|
||||
recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
|
||||
require(!false_recovery && false_recovery.error().kind == RecoveryError::history,
|
||||
"false plan upgrade recovered by stock equality");
|
||||
take(f.db->query("ROLLBACK", {}, 5s), "rollback false plan metadata");
|
||||
take(
|
||||
f.owner->query(
|
||||
"CREATE FUNCTION evolution.test_refuse_plan55() RETURNS trigger LANGUAGE plpgsql AS $$ "
|
||||
"BEGIN IF NEW.world_id=30205 THEN RAISE EXCEPTION 'plan55 fault' USING "
|
||||
"ERRCODE='40001'; "
|
||||
"END IF; RETURN NEW; END $$",
|
||||
{}, 5s),
|
||||
"plan fault function");
|
||||
take(f.owner->query(
|
||||
"CREATE TRIGGER test_refuse_plan55 BEFORE INSERT ON evolution.region_commits "
|
||||
"FOR EACH ROW EXECUTE FUNCTION evolution.test_refuse_plan55()",
|
||||
{}, 5s),
|
||||
"plan fault trigger");
|
||||
const auto rejected = migrate_cellular_inheritance_profile(*f.db, *f.objects, f.lease, before,
|
||||
f.id(), f.build, 5s);
|
||||
take(f.owner->query("DROP TRIGGER test_refuse_plan55 ON evolution.region_commits", {}, 5s),
|
||||
"drop plan fault trigger");
|
||||
take(f.owner->query("DROP FUNCTION evolution.test_refuse_plan55()", {}, 5s),
|
||||
"drop plan fault function");
|
||||
require(!rejected && take(load_region_checkpoint(*f.db, f.key, 5s), "plan failed head").head ==
|
||||
before.head,
|
||||
"failed plan result advanced head");
|
||||
const auto evidence = take(
|
||||
f.owner->query(
|
||||
"SELECT count(*)::text FROM evolution.cellular_profile_migrations WHERE world_id=30205 "
|
||||
"AND "
|
||||
"target_versions='"
|
||||
"\\x0a0000000800000002000000010000000100000004000000030000000200000001000000'::bytea",
|
||||
{}, 5s),
|
||||
"failed plan evidence");
|
||||
require(evidence.front().front() == "0", "failed plan result retained upgrade journal");
|
||||
const auto killed_id = f.id();
|
||||
const auto child = ::fork();
|
||||
require(child >= 0, "plan process fork");
|
||||
if (child == 0) {
|
||||
auto db = connection();
|
||||
PublishHooks hooks;
|
||||
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
|
||||
[[maybe_unused]] const auto result = migrate_cellular_inheritance_profile(
|
||||
*db, *f.objects, f.lease, before, killed_id, f.build, 5s, hooks);
|
||||
::_exit(32);
|
||||
}
|
||||
int status{};
|
||||
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
|
||||
WTERMSIG(status) == SIGKILL &&
|
||||
take(load_region_checkpoint(*f.db, f.key, 5s), "killed plan head").head ==
|
||||
before.head,
|
||||
"pre-transaction process loss published plan upgrade");
|
||||
auto locker = connection(true);
|
||||
take(locker->query("BEGIN", {}, 5s), "uncertain plan transaction");
|
||||
take(
|
||||
locker->query(
|
||||
"SELECT 1 FROM evolution.region_leases WHERE world_id=30205 AND region_id=1 FOR UPDATE",
|
||||
{}, 5s),
|
||||
"hold plan lease lock");
|
||||
auto timed = connection();
|
||||
const auto unknown = migrate_cellular_inheritance_profile(*timed, *f.objects, f.lease, before,
|
||||
f.id(), f.build, 300ms);
|
||||
if (unknown)
|
||||
throw std::runtime_error("plan lock did not cause ambiguous publication");
|
||||
require(unknown.error().kind == PublicationError::unknown &&
|
||||
!unknown.error().prepared.manifest.empty(),
|
||||
"ambiguous plan request lost exact bytes");
|
||||
take(locker->query("ROLLBACK", {}, 5s), "release plan lease lock");
|
||||
const auto prepared = unknown.error().prepared;
|
||||
auto writer = connection();
|
||||
f.head = take(retry_cellular_profile_migration(*writer, prepared, 5s), "retry plan request");
|
||||
require(take(retry_cellular_profile_migration(*writer, prepared, 5s), "duplicate plan request")
|
||||
.head == f.head.head,
|
||||
"duplicate plan request advanced head");
|
||||
auto conflict = prepared;
|
||||
auto changed = take(decode_manifest(conflict.manifest), "changed plan retry");
|
||||
++changed.build[0];
|
||||
conflict.manifest = take(encode_manifest(changed), "changed plan bytes");
|
||||
require(!retry_cellular_profile_migration(*writer, conflict, 5s),
|
||||
"changed plan retry accepted");
|
||||
reload();
|
||||
require(f.raw->snapshot() == expected && f.head.head.end_tick == before.head.end_tick,
|
||||
"durable plan upgrade changed biology or time");
|
||||
require(!migrate_cellular_inheritance_profile(*f.db, *f.objects, f.lease, f.head, f.id(),
|
||||
f.build, 5s) &&
|
||||
!migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(), f.build,
|
||||
{true, 1200, 1800}, 5s),
|
||||
"repeated plan upgrade or population bypass");
|
||||
const auto path = f.objects->directory() / hexadecimal(source.payload_sha256, false);
|
||||
const auto backup = path.string() + ".plan55-test";
|
||||
std::filesystem::rename(path, backup);
|
||||
const auto missing = recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
|
||||
{
|
||||
std::ofstream corrupt(path, std::ios::binary);
|
||||
corrupt << 'X';
|
||||
}
|
||||
const auto corrupt = recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
|
||||
std::filesystem::remove(path);
|
||||
std::filesystem::rename(backup, path);
|
||||
require(!missing && !corrupt, "unusable retained source allowed fallback");
|
||||
reload();
|
||||
const auto step = [&](unsigned count, bool bud) {
|
||||
CellularBatch batch{f.header(count), {}};
|
||||
batch.format_version = cellular_batch_inheritance_format;
|
||||
for (unsigned tick = 0; tick < count; ++tick) {
|
||||
auto inputs = take(wildlife_intents(*f.raw, f.raw->tick() + 1), "plan wildlife");
|
||||
for (const auto& life : f.raw->lives()) {
|
||||
if (life.account == LifeAccount{})
|
||||
continue;
|
||||
CellularLifeIntent input{life.identity.life, {}};
|
||||
input.intent.frozen = life.identity.life != 100;
|
||||
input.intent.bud = life.identity.life == 100 && bud;
|
||||
inputs.push_back(input);
|
||||
}
|
||||
std::ranges::sort(inputs, {}, &CellularLifeIntent::life_id);
|
||||
take(f.raw->advance_tick(inputs), "plan biological tick");
|
||||
batch.ticks.push_back({std::move(inputs), {}, {}});
|
||||
}
|
||||
f.head = take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, batch, *f.raw,
|
||||
f.id(), f.build, 5s),
|
||||
"plan trace-4 publication")
|
||||
.committed;
|
||||
take(renew_region_lease(*f.db, f.key, f.lease, 60000ms, 5s), "plan writer renewal");
|
||||
};
|
||||
step(1, false);
|
||||
for (unsigned tries = 0; tries < 1000 && !f.raw->lives().front().mutation.offer; ++tries)
|
||||
step(64, false);
|
||||
const auto parent = f.raw->lives().front();
|
||||
const auto offer = parent.mutation.offer;
|
||||
if (!offer)
|
||||
throw std::runtime_error("durable inherited offer missing");
|
||||
require(parent.identity.life == 100, "durable plan actor died");
|
||||
const auto control = take(
|
||||
claim_account_control(*f.db, f.key, f.lease, account, 2, 101, 60000ms, 5s), "plan control");
|
||||
CellularMutationActionRecord action{account,
|
||||
57002,
|
||||
f.raw->tick() + 1,
|
||||
{100, parent.mutation.event_index,
|
||||
parent.mutation.offer_tick,
|
||||
offer->candidates.front().operator_id}};
|
||||
const auto reservation = take(
|
||||
reserve_cellular_mutation(*f.db, f.key, f.lease, account, control.generation,
|
||||
action.action_id, encode_cellular_mutation_input(action), 5s),
|
||||
"plan selection reservation");
|
||||
CellularBatch chosen{f.header(1), {}};
|
||||
chosen.format_version = cellular_batch_inheritance_format;
|
||||
auto inputs = take(wildlife_intents(*f.raw, f.raw->tick() + 1), "selected plan wildlife");
|
||||
for (const auto& life : f.raw->lives()) {
|
||||
if (life.account == LifeAccount{})
|
||||
continue;
|
||||
CellularLifeIntent input{life.identity.life, {}};
|
||||
input.intent.frozen = life.identity.life != 100;
|
||||
inputs.push_back(input);
|
||||
}
|
||||
std::ranges::sort(inputs, {}, &CellularLifeIntent::life_id);
|
||||
chosen.ticks.push_back({inputs,
|
||||
{},
|
||||
{{100, action.input.event_index, action.input.offer_tick,
|
||||
reservation.successor_identity, action.input.operator_id}}});
|
||||
const auto executed =
|
||||
take(f.raw->advance_tick_with_mutations(inputs, chosen.ticks.front().mutations),
|
||||
"select durable plan");
|
||||
require(executed.mutations.front().result == CellularMutationResult::offspring_plan_applied &&
|
||||
f.raw->lives().front().identity.genome == parent.identity.genome,
|
||||
"plan changed expressed genome");
|
||||
const auto trace_bytes = take(encode_cellular_batch(chosen), "selected trace bytes");
|
||||
const auto block =
|
||||
take(encode_cellular_mutation_action_block(std::span{&action, 1}, chosen.header.begin_tick,
|
||||
chosen.header.end_tick),
|
||||
"selected mutation block");
|
||||
const auto candidate_manifest =
|
||||
take(successor_manifest(
|
||||
take(decode_manifest(f.head.manifest), "selected predecessor manifest"),
|
||||
f.head.head.digest, f.id(), f.lease, f.raw->tick(), f.build,
|
||||
f.raw->snapshot().size(), take(sha256(f.raw->snapshot()), "selected payload hash"),
|
||||
{trace_bytes.size(), take(sha256(trace_bytes), "selected trace hash"), 0, {}}),
|
||||
"selected metadata fixture");
|
||||
for (const std::uint8_t bad_result :
|
||||
{std::uint8_t{0}, std::uint8_t{2}, std::uint8_t{3}, std::uint8_t{5}})
|
||||
require(!publish_cellular_mutation_checkpoint(
|
||||
*f.db, take(encode_manifest(candidate_manifest), "bad-result manifest"), {}, {},
|
||||
trace_bytes, block, std::span{&bad_result, 1}, 5s),
|
||||
"SQL admitted a foreign inherited result meaning");
|
||||
auto old_trace = chosen;
|
||||
old_trace.format_version = cellular_batch_format;
|
||||
const auto old_bytes = take(encode_cellular_batch(old_trace), "wrong-version trace");
|
||||
auto bad_pair = candidate_manifest;
|
||||
bad_pair.batch.trace_sha256 = take(sha256(old_bytes), "wrong-version trace hash");
|
||||
const std::uint8_t applied = 4;
|
||||
require(!publish_cellular_mutation_checkpoint(
|
||||
*f.db, take(encode_manifest(bad_pair), "wrong-pair manifest"), {}, {}, old_bytes,
|
||||
block, std::span{&applied, 1}, 5s) &&
|
||||
take(load_region_checkpoint(*f.db, f.key, 5s), "failed result head").head ==
|
||||
f.head.head,
|
||||
"SQL mixed trace versions or retained a rejected result");
|
||||
auto altered_trace = trace_bytes;
|
||||
altered_trace.at(cellular_batch_header_bytes + cellular_batch_tick_header_bytes + 8) = 1;
|
||||
require(!publish_cellular_mutation_checkpoint(
|
||||
*f.db, take(encode_manifest(candidate_manifest), "trace digest fixture"), {}, {},
|
||||
altered_trace, block, std::span{&applied, 1}, 5s),
|
||||
"parser normalization bypassed the actual trace digest");
|
||||
const auto selected =
|
||||
take(publish_cellular_batch(*f.db, *f.objects, f.lease, f.head, chosen, *f.raw, f.id(),
|
||||
f.build, 5s, {}, {}, std::span{&action, 1}),
|
||||
"durable plan publication");
|
||||
f.head = selected.committed;
|
||||
require(selected.mutation_results ==
|
||||
std::vector{CellularMutationResult::offspring_plan_applied},
|
||||
"plan result lost meaning");
|
||||
const auto selected_bytes = f.raw->snapshot();
|
||||
reload();
|
||||
require(f.raw->snapshot() == selected_bytes, "selected plan did not recover exactly");
|
||||
std::size_t born = 0;
|
||||
for (unsigned tries = 0; tries < 500 && born < 2; ++tries) {
|
||||
step(64, true);
|
||||
born =
|
||||
static_cast<std::size_t>(std::ranges::count_if(f.raw->lives(), [&](const auto& life) {
|
||||
return life.parent_life == 100 &&
|
||||
life.identity.genome == reservation.successor_identity;
|
||||
}));
|
||||
for (const auto& corpse : f.raw->corpses())
|
||||
if (corpse.parent_life == 100 &&
|
||||
take(f.raw->archive().organism(corpse.organism), "inherited dead sibling").genome ==
|
||||
reservation.successor_identity)
|
||||
++born;
|
||||
}
|
||||
require(born >= 2, "durable parent never repeated its selected offspring plan");
|
||||
const auto final_bytes = f.raw->snapshot();
|
||||
const auto stale = f.lease;
|
||||
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release plan writer");
|
||||
f.lease =
|
||||
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "replacement plan writer");
|
||||
require(!retry_cellular_profile_migration(*f.db, prepared, 5s) &&
|
||||
!migrate_cellular_inheritance_profile(*f.db, *f.objects, stale, before, f.id(),
|
||||
f.build, 5s),
|
||||
"stale plan writer escaped fence");
|
||||
reload();
|
||||
require(f.raw->snapshot() == final_bytes, "replacement changed inherited family");
|
||||
std::puts("retained-plan upgrade, choices, births, exact recovery and faults passed");
|
||||
}
|
||||
} // namespace
|
||||
|
||||
Reference in New Issue
Block a user