M12.4: admit and replay durable boundary-accounting upgrades
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m23s
Foundation / checks (network) (push) Successful in 21m4s
Foundation / checks (client) (push) Successful in 22m15s
Foundation / checks (tsan) (push) Successful in 23m3s
Foundation / checks (postgres) (push) Successful in 51m33s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 55m18s

This commit is contained in:
dingo
2026-10-18 08:13:36 -04:00
parent 2416735e2b
commit 1b5c0cdf7c
21 changed files with 812 additions and 98 deletions
+208 -5
View File
@@ -290,12 +290,14 @@ void conversions_and_retries(Fixture& f) {
take(encode_manifest(boundary_manifest), "boundary candidate encoding"),
{}};
auto rejected_boundary = retry_population_prepared(*f.db, boundary, 5s);
require(!rejected_boundary && rejected_boundary.error().kind == PublicationError::candidate &&
!resolve_population_prepared(*f.db, boundary, 5s),
"portable bundle 10 bypassed prepared publication admission");
require(!rejected_boundary && rejected_boundary.error().kind == PublicationError::rejected &&
take(resolve_population_prepared(*f.db, boundary, 5s),
"upgrade through wrong journal resolution")
.status == Resolution::absent,
"accounting upgrade bypassed its dedicated version journal");
auto rejected_upgrade = publish_population_profile_transition(*f.db, boundary.manifest, 5s);
require(!rejected_upgrade && rejected_upgrade.error().kind == LeaseStoreError::invalid,
"schema 45 admitted portable accounting upgrade without dedicated evidence");
require(!rejected_upgrade,
"representation publisher admitted accounting upgrade without dedicated evidence");
require(!publish_region_checkpoint(*f.db, boundary.manifest, {}, 5s),
"generic publisher admitted boundary upgrade");
require(take(load_region_checkpoint(*f.db, f.key, 5s), "head after future rejection").head ==
@@ -543,6 +545,206 @@ void missing_and_corrupt_objects(Fixture& f) {
"corrupt population payload did not fail closed");
recover(f);
}
void boundary_upgrades(Fixture& f) {
const std::array<Sha256Digest, 1> builds{f.build};
const auto original = f.head;
const auto expected_raw =
take(f.raw->migration_upgrade_candidate(), "raw upgrade oracle")->snapshot();
// Rehearse the raw upgrade, recovery and ordinary trace-3 publication in one
// transaction, then roll it back to retain a real bundle-9 source for profile 5.
take(f.db->query("BEGIN", {}, 5s), "raw upgrade rollback rehearsal");
auto raw_head = take(
migrate_cellular_boundary_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
"raw accounting upgrade");
auto raw = take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s),
"raw accounting recovery");
require(raw.state->snapshot() == expected_raw &&
raw.manifest.cellular_versions == cellular_v10_versions &&
raw_head.head.end_tick == original.head.end_tick,
"raw accounting upgrade changed biology or time");
const CellularBatchHeader h{f.key.world,
f.key.region,
1,
raw_head.head.commit,
raw_head.head.digest,
raw_head.head.end_tick,
raw_head.head.end_tick + 1};
const auto inputs = take(unattended_population_inputs(*raw.state, h), "upgraded raw inputs");
auto advanced = take(replay_cellular_batch(*raw.state, inputs), "upgraded raw oracle");
take(publish_cellular_batch(*f.db, *f.objects, f.lease, raw_head, inputs, *advanced, f.id(),
f.build, 5s),
"upgraded raw ordinary publication");
require(take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s),
"upgraded raw ordinary recovery")
.state->snapshot() == advanced->snapshot(),
"raw accounting continuation differs");
take(f.db->query("ROLLBACK", {}, 5s), "roll back raw upgrade rehearsal");
require(take(load_region_checkpoint(*f.db, f.key, 5s), "raw rollback head").head ==
original.head,
"rolled-back upgrade moved the durable head");
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
f.build, {true, 1200, 1800}, 5s),
"population accounting source entry");
recover(f);
const auto member = std::ranges::find_if(f.population->state().lives(), [](const auto& life) {
return life.account == LifeAccount{};
});
require(member != f.population->state().lives().end(), "accounting source has no wildlife");
auto collapsed = take(
prepare_population_conversion(*f.population, f.header(0), {member->identity.genome, true}),
"accounting source cohort");
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, collapsed.batch,
*collapsed.state, f.id(), f.build, 5s),
"accounting source membership publication");
recover(f);
require(!f.population->members().empty() && f.population->conversion_sequence() != 0,
"accounting source lost cohort history");
auto interval = unattended(f);
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head,
interval.batch.header.end_tick, 5s),
"upgrade pending interval");
auto pending =
migrate_cellular_boundary_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s);
require(!pending && std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18",
"accounting upgrade discarded admitted work");
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, interval.batch,
*interval.state, f.id(), f.build, 5s),
"settle pre-upgrade work");
recover(f);
const auto before = f.head;
const auto source = take(decode_manifest(before.manifest), "accounting source manifest");
const auto upgraded_population =
take(f.population->migration_upgrade_candidate(), "cohort upgrade oracle");
const auto expected = upgraded_population->snapshot();
const auto altered =
take(CellularPopulation::create(restore(upgraded_population->state()), {false, 1200, 1800}),
"valid unequal upgrade target");
require(altered->state().current_total() == upgraded_population->state().current_total() &&
altered->snapshot() != expected,
"unequal upgrade fixture must retain biological totals");
const auto altered_object = take(f.objects->put(altered->snapshot()), "unequal upgrade object");
const auto altered_manifest = take(
cellular_boundary_upgrade_manifest(source, before.head.digest, f.id(), f.lease, f.build,
altered_object.bytes, altered_object.sha256),
"unequal upgrade manifest");
take(f.db->query("BEGIN", {}, 5s), "unequal upgrade recovery rehearsal");
take(publish_cellular_profile_migration(
*f.db, take(encode_manifest(altered_manifest), "unequal upgrade encoding"), 5s),
"metadata-only unequal upgrade publication");
const auto unequal =
recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
take(f.db->query("ROLLBACK", {}, 5s), "undo unequal upgrade rehearsal");
require(!unequal && unequal.error().kind == RecoveryError::history &&
take(load_region_checkpoint(*f.db, f.key, 5s), "head after unequal upgrade").head ==
before.head,
"upgrade recovery compared only aggregate stock or rewrote the retained source");
const auto child = ::fork();
require(child >= 0, "upgrade pre-transaction fork");
if (child == 0) {
try {
auto db = connection();
PublishHooks hooks;
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
[[maybe_unused]] const auto result = migrate_cellular_boundary_profile(
*db, *f.objects, f.lease, before, identity(60000), f.build, 5s, hooks);
} catch (...) {
::_exit(31);
}
::_exit(32);
}
int status{};
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
WTERMSIG(status) == SIGKILL,
"upgrade child missed pre-transaction kill");
recover(f);
require(f.head.head == before.head, "unpublished upgrade became authoritative");
auto locker = connection(true);
take(locker->query("BEGIN", {}, 5s), "accounting ambiguity lock transaction");
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 "
"AND region_id=1 FOR UPDATE",
{}, 5s),
"hold accounting reply");
auto timed = connection();
auto unknown = migrate_cellular_boundary_profile(*timed, *f.objects, f.lease, before, f.id(),
f.build, 300ms);
require(!unknown && unknown.error().kind == PublicationError::unknown &&
!unknown.error().prepared.manifest.empty(),
"uncertain accounting upgrade lost its exact request");
take(locker->query("ROLLBACK", {}, 5s), "release accounting reply");
auto resolver = connection();
f.head = take(retry_cellular_profile_migration(*resolver, unknown.error().prepared, 5s),
"retry exact accounting upgrade");
require(take(retry_cellular_profile_migration(*resolver, unknown.error().prepared, 5s),
"duplicate accounting upgrade")
.head == f.head.head,
"duplicate accounting upgrade advanced head");
recover(f);
require(f.population->snapshot() == expected && f.head.head.end_tick == before.head.end_tick &&
take(decode_manifest(f.head.manifest), "upgraded population manifest")
.cellular_versions == cellular_v10_versions,
"accounting upgrade changed biology, cohort history or time");
auto changed = unknown.error().prepared;
auto changed_manifest = take(decode_manifest(changed.manifest), "changed accounting manifest");
changed_manifest.payload_sha256[0] ^= 1;
changed.manifest = take(encode_manifest(changed_manifest), "changed accounting bytes");
auto conflict = retry_cellular_profile_migration(*resolver, changed, 5s);
require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict,
"accounting retry accepted changed bytes");
require(
!migrate_cellular_boundary_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
"accounting upgrade applied twice");
const auto path = f.objects->directory() / hexadecimal(source.payload_sha256, false);
const auto backup = path.string() + ".upgrade-test";
std::filesystem::rename(path, backup);
const auto missing =
recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
std::filesystem::rename(backup, path);
require(!missing && missing.error().kind == RecoveryError::tail,
"accounting recovery trusted aggregate stock without its retained source");
missing_and_corrupt_objects(f);
const auto stale = f.lease;
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release accounting writer");
f.lease =
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "replace accounting writer");
auto stale_retry = retry_cellular_profile_migration(*f.db, unknown.error().prepared, 5s);
require(!stale_retry && stale_retry.error().store.kind == LeaseStoreError::fenced &&
!recover_population_checkpoint(*f.db, *f.objects, f.key, stale, builds, 5s),
"old accounting receipt granted current authority");
recover(f);
require(f.population->snapshot() == expected, "new epoch changed accounting upgrade");
auto future_interval = unattended(f);
auto old_trace = future_interval.batch;
old_trace.format_version = 1;
const auto old_bytes = take(encode_population_batch(old_trace), "wrong old trace fixture");
const auto upgraded_manifest = take(decode_manifest(f.head.manifest), "new accounting source");
auto wrong_trace_manifest =
take(population_successor_manifest(
upgraded_manifest, f.head.head.digest, f.id(), f.lease, old_trace.header.end_tick,
f.build, future_interval.state->snapshot().size(),
take(sha256(future_interval.state->snapshot()), "new payload SHA"),
{old_bytes.size(), take(sha256(old_bytes), "old trace SHA"), 0, {}}),
"wrong old trace manifest fixture");
const PopulationPreparedCommit wrong_trace{
f.key, wrong_trace_manifest.commit,
take(encode_manifest(wrong_trace_manifest), "wrong old trace manifest"), old_bytes};
auto refused_old = retry_population_prepared(*f.db, wrong_trace, 5s);
auto refused_old_sql =
publish_population_checkpoint(*f.db, wrong_trace.manifest, old_bytes, 5s);
require(!refused_old && refused_old.error().kind == PublicationError::candidate &&
!refused_old_sql && refused_old_sql.error().kind == LeaseStoreError::invalid &&
take(load_region_checkpoint(*f.db, f.key, 5s), "head after old trace").head ==
f.head.head,
"bundle 10 reinterpreted retained population trace 1");
// Exercise the existing publication/process-loss matrix with batch 2 as well.
process_loss(f, 0);
process_loss(f, 1);
process_loss(f, 2);
isolation_and_reverse(f);
require(f.raw->format_version() == cellular_lives_migration_format,
"reverse transition downgraded accounting owner");
}
void host_ownership_and_admission(Fixture& f) {
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture host authority");
const auto make_host = [&](bool enabled) {
@@ -683,6 +885,7 @@ void run_population_publication_tests() {
process_loss(fixture, 2);
missing_and_corrupt_objects(fixture);
isolation_and_reverse(fixture);
boundary_upgrades(fixture);
host_ownership_and_admission(fixture);
std::puts("population publication, exact recovery, process loss and fencing passed");
}