M12.4: admit and replay durable boundary-accounting upgrades
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m23s
Foundation / checks (network) (push) Successful in 21m4s
Foundation / checks (client) (push) Successful in 22m15s
Foundation / checks (tsan) (push) Successful in 23m3s
Foundation / checks (postgres) (push) Successful in 51m33s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 55m18s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m23s
Foundation / checks (network) (push) Successful in 21m4s
Foundation / checks (client) (push) Successful in 22m15s
Foundation / checks (tsan) (push) Successful in 23m3s
Foundation / checks (postgres) (push) Successful in 51m33s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 55m18s
This commit is contained in:
@@ -290,12 +290,14 @@ void conversions_and_retries(Fixture& f) {
|
||||
take(encode_manifest(boundary_manifest), "boundary candidate encoding"),
|
||||
{}};
|
||||
auto rejected_boundary = retry_population_prepared(*f.db, boundary, 5s);
|
||||
require(!rejected_boundary && rejected_boundary.error().kind == PublicationError::candidate &&
|
||||
!resolve_population_prepared(*f.db, boundary, 5s),
|
||||
"portable bundle 10 bypassed prepared publication admission");
|
||||
require(!rejected_boundary && rejected_boundary.error().kind == PublicationError::rejected &&
|
||||
take(resolve_population_prepared(*f.db, boundary, 5s),
|
||||
"upgrade through wrong journal resolution")
|
||||
.status == Resolution::absent,
|
||||
"accounting upgrade bypassed its dedicated version journal");
|
||||
auto rejected_upgrade = publish_population_profile_transition(*f.db, boundary.manifest, 5s);
|
||||
require(!rejected_upgrade && rejected_upgrade.error().kind == LeaseStoreError::invalid,
|
||||
"schema 45 admitted portable accounting upgrade without dedicated evidence");
|
||||
require(!rejected_upgrade,
|
||||
"representation publisher admitted accounting upgrade without dedicated evidence");
|
||||
require(!publish_region_checkpoint(*f.db, boundary.manifest, {}, 5s),
|
||||
"generic publisher admitted boundary upgrade");
|
||||
require(take(load_region_checkpoint(*f.db, f.key, 5s), "head after future rejection").head ==
|
||||
@@ -543,6 +545,206 @@ void missing_and_corrupt_objects(Fixture& f) {
|
||||
"corrupt population payload did not fail closed");
|
||||
recover(f);
|
||||
}
|
||||
void boundary_upgrades(Fixture& f) {
|
||||
const std::array<Sha256Digest, 1> builds{f.build};
|
||||
const auto original = f.head;
|
||||
const auto expected_raw =
|
||||
take(f.raw->migration_upgrade_candidate(), "raw upgrade oracle")->snapshot();
|
||||
// Rehearse the raw upgrade, recovery and ordinary trace-3 publication in one
|
||||
// transaction, then roll it back to retain a real bundle-9 source for profile 5.
|
||||
take(f.db->query("BEGIN", {}, 5s), "raw upgrade rollback rehearsal");
|
||||
auto raw_head = take(
|
||||
migrate_cellular_boundary_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
|
||||
"raw accounting upgrade");
|
||||
auto raw = take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s),
|
||||
"raw accounting recovery");
|
||||
require(raw.state->snapshot() == expected_raw &&
|
||||
raw.manifest.cellular_versions == cellular_v10_versions &&
|
||||
raw_head.head.end_tick == original.head.end_tick,
|
||||
"raw accounting upgrade changed biology or time");
|
||||
const CellularBatchHeader h{f.key.world,
|
||||
f.key.region,
|
||||
1,
|
||||
raw_head.head.commit,
|
||||
raw_head.head.digest,
|
||||
raw_head.head.end_tick,
|
||||
raw_head.head.end_tick + 1};
|
||||
const auto inputs = take(unattended_population_inputs(*raw.state, h), "upgraded raw inputs");
|
||||
auto advanced = take(replay_cellular_batch(*raw.state, inputs), "upgraded raw oracle");
|
||||
take(publish_cellular_batch(*f.db, *f.objects, f.lease, raw_head, inputs, *advanced, f.id(),
|
||||
f.build, 5s),
|
||||
"upgraded raw ordinary publication");
|
||||
require(take(recover_cellular_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s),
|
||||
"upgraded raw ordinary recovery")
|
||||
.state->snapshot() == advanced->snapshot(),
|
||||
"raw accounting continuation differs");
|
||||
take(f.db->query("ROLLBACK", {}, 5s), "roll back raw upgrade rehearsal");
|
||||
require(take(load_region_checkpoint(*f.db, f.key, 5s), "raw rollback head").head ==
|
||||
original.head,
|
||||
"rolled-back upgrade moved the durable head");
|
||||
|
||||
f.head = take(migrate_cellular_to_population(*f.db, *f.objects, f.lease, f.head, f.id(),
|
||||
f.build, {true, 1200, 1800}, 5s),
|
||||
"population accounting source entry");
|
||||
recover(f);
|
||||
const auto member = std::ranges::find_if(f.population->state().lives(), [](const auto& life) {
|
||||
return life.account == LifeAccount{};
|
||||
});
|
||||
require(member != f.population->state().lives().end(), "accounting source has no wildlife");
|
||||
auto collapsed = take(
|
||||
prepare_population_conversion(*f.population, f.header(0), {member->identity.genome, true}),
|
||||
"accounting source cohort");
|
||||
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, collapsed.batch,
|
||||
*collapsed.state, f.id(), f.build, 5s),
|
||||
"accounting source membership publication");
|
||||
recover(f);
|
||||
require(!f.population->members().empty() && f.population->conversion_sequence() != 0,
|
||||
"accounting source lost cohort history");
|
||||
auto interval = unattended(f);
|
||||
take(admit_population_interval(*f.db, f.key, f.lease, f.head.head,
|
||||
interval.batch.header.end_tick, 5s),
|
||||
"upgrade pending interval");
|
||||
auto pending =
|
||||
migrate_cellular_boundary_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s);
|
||||
require(!pending && std::string(pending.error().store.cause.sqlstate.data(), 5) == "EVL18",
|
||||
"accounting upgrade discarded admitted work");
|
||||
f.head = take(publish_population_batch(*f.db, *f.objects, f.lease, f.head, interval.batch,
|
||||
*interval.state, f.id(), f.build, 5s),
|
||||
"settle pre-upgrade work");
|
||||
recover(f);
|
||||
const auto before = f.head;
|
||||
const auto source = take(decode_manifest(before.manifest), "accounting source manifest");
|
||||
const auto upgraded_population =
|
||||
take(f.population->migration_upgrade_candidate(), "cohort upgrade oracle");
|
||||
const auto expected = upgraded_population->snapshot();
|
||||
const auto altered =
|
||||
take(CellularPopulation::create(restore(upgraded_population->state()), {false, 1200, 1800}),
|
||||
"valid unequal upgrade target");
|
||||
require(altered->state().current_total() == upgraded_population->state().current_total() &&
|
||||
altered->snapshot() != expected,
|
||||
"unequal upgrade fixture must retain biological totals");
|
||||
const auto altered_object = take(f.objects->put(altered->snapshot()), "unequal upgrade object");
|
||||
const auto altered_manifest = take(
|
||||
cellular_boundary_upgrade_manifest(source, before.head.digest, f.id(), f.lease, f.build,
|
||||
altered_object.bytes, altered_object.sha256),
|
||||
"unequal upgrade manifest");
|
||||
take(f.db->query("BEGIN", {}, 5s), "unequal upgrade recovery rehearsal");
|
||||
take(publish_cellular_profile_migration(
|
||||
*f.db, take(encode_manifest(altered_manifest), "unequal upgrade encoding"), 5s),
|
||||
"metadata-only unequal upgrade publication");
|
||||
const auto unequal =
|
||||
recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
|
||||
take(f.db->query("ROLLBACK", {}, 5s), "undo unequal upgrade rehearsal");
|
||||
require(!unequal && unequal.error().kind == RecoveryError::history &&
|
||||
take(load_region_checkpoint(*f.db, f.key, 5s), "head after unequal upgrade").head ==
|
||||
before.head,
|
||||
"upgrade recovery compared only aggregate stock or rewrote the retained source");
|
||||
const auto child = ::fork();
|
||||
require(child >= 0, "upgrade pre-transaction fork");
|
||||
if (child == 0) {
|
||||
try {
|
||||
auto db = connection();
|
||||
PublishHooks hooks;
|
||||
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
|
||||
[[maybe_unused]] const auto result = migrate_cellular_boundary_profile(
|
||||
*db, *f.objects, f.lease, before, identity(60000), f.build, 5s, hooks);
|
||||
} catch (...) {
|
||||
::_exit(31);
|
||||
}
|
||||
::_exit(32);
|
||||
}
|
||||
int status{};
|
||||
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
|
||||
WTERMSIG(status) == SIGKILL,
|
||||
"upgrade child missed pre-transaction kill");
|
||||
recover(f);
|
||||
require(f.head.head == before.head, "unpublished upgrade became authoritative");
|
||||
|
||||
auto locker = connection(true);
|
||||
take(locker->query("BEGIN", {}, 5s), "accounting ambiguity lock transaction");
|
||||
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 "
|
||||
"AND region_id=1 FOR UPDATE",
|
||||
{}, 5s),
|
||||
"hold accounting reply");
|
||||
auto timed = connection();
|
||||
auto unknown = migrate_cellular_boundary_profile(*timed, *f.objects, f.lease, before, f.id(),
|
||||
f.build, 300ms);
|
||||
require(!unknown && unknown.error().kind == PublicationError::unknown &&
|
||||
!unknown.error().prepared.manifest.empty(),
|
||||
"uncertain accounting upgrade lost its exact request");
|
||||
take(locker->query("ROLLBACK", {}, 5s), "release accounting reply");
|
||||
auto resolver = connection();
|
||||
f.head = take(retry_cellular_profile_migration(*resolver, unknown.error().prepared, 5s),
|
||||
"retry exact accounting upgrade");
|
||||
require(take(retry_cellular_profile_migration(*resolver, unknown.error().prepared, 5s),
|
||||
"duplicate accounting upgrade")
|
||||
.head == f.head.head,
|
||||
"duplicate accounting upgrade advanced head");
|
||||
recover(f);
|
||||
require(f.population->snapshot() == expected && f.head.head.end_tick == before.head.end_tick &&
|
||||
take(decode_manifest(f.head.manifest), "upgraded population manifest")
|
||||
.cellular_versions == cellular_v10_versions,
|
||||
"accounting upgrade changed biology, cohort history or time");
|
||||
auto changed = unknown.error().prepared;
|
||||
auto changed_manifest = take(decode_manifest(changed.manifest), "changed accounting manifest");
|
||||
changed_manifest.payload_sha256[0] ^= 1;
|
||||
changed.manifest = take(encode_manifest(changed_manifest), "changed accounting bytes");
|
||||
auto conflict = retry_cellular_profile_migration(*resolver, changed, 5s);
|
||||
require(!conflict && conflict.error().store.kind == LeaseStoreError::commit_conflict,
|
||||
"accounting retry accepted changed bytes");
|
||||
require(
|
||||
!migrate_cellular_boundary_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
|
||||
"accounting upgrade applied twice");
|
||||
const auto path = f.objects->directory() / hexadecimal(source.payload_sha256, false);
|
||||
const auto backup = path.string() + ".upgrade-test";
|
||||
std::filesystem::rename(path, backup);
|
||||
const auto missing =
|
||||
recover_population_checkpoint(*f.db, *f.objects, f.key, f.lease, builds, 5s);
|
||||
std::filesystem::rename(backup, path);
|
||||
require(!missing && missing.error().kind == RecoveryError::tail,
|
||||
"accounting recovery trusted aggregate stock without its retained source");
|
||||
missing_and_corrupt_objects(f);
|
||||
const auto stale = f.lease;
|
||||
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release accounting writer");
|
||||
f.lease =
|
||||
take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s), "replace accounting writer");
|
||||
auto stale_retry = retry_cellular_profile_migration(*f.db, unknown.error().prepared, 5s);
|
||||
require(!stale_retry && stale_retry.error().store.kind == LeaseStoreError::fenced &&
|
||||
!recover_population_checkpoint(*f.db, *f.objects, f.key, stale, builds, 5s),
|
||||
"old accounting receipt granted current authority");
|
||||
recover(f);
|
||||
require(f.population->snapshot() == expected, "new epoch changed accounting upgrade");
|
||||
auto future_interval = unattended(f);
|
||||
auto old_trace = future_interval.batch;
|
||||
old_trace.format_version = 1;
|
||||
const auto old_bytes = take(encode_population_batch(old_trace), "wrong old trace fixture");
|
||||
const auto upgraded_manifest = take(decode_manifest(f.head.manifest), "new accounting source");
|
||||
auto wrong_trace_manifest =
|
||||
take(population_successor_manifest(
|
||||
upgraded_manifest, f.head.head.digest, f.id(), f.lease, old_trace.header.end_tick,
|
||||
f.build, future_interval.state->snapshot().size(),
|
||||
take(sha256(future_interval.state->snapshot()), "new payload SHA"),
|
||||
{old_bytes.size(), take(sha256(old_bytes), "old trace SHA"), 0, {}}),
|
||||
"wrong old trace manifest fixture");
|
||||
const PopulationPreparedCommit wrong_trace{
|
||||
f.key, wrong_trace_manifest.commit,
|
||||
take(encode_manifest(wrong_trace_manifest), "wrong old trace manifest"), old_bytes};
|
||||
auto refused_old = retry_population_prepared(*f.db, wrong_trace, 5s);
|
||||
auto refused_old_sql =
|
||||
publish_population_checkpoint(*f.db, wrong_trace.manifest, old_bytes, 5s);
|
||||
require(!refused_old && refused_old.error().kind == PublicationError::candidate &&
|
||||
!refused_old_sql && refused_old_sql.error().kind == LeaseStoreError::invalid &&
|
||||
take(load_region_checkpoint(*f.db, f.key, 5s), "head after old trace").head ==
|
||||
f.head.head,
|
||||
"bundle 10 reinterpreted retained population trace 1");
|
||||
// Exercise the existing publication/process-loss matrix with batch 2 as well.
|
||||
process_loss(f, 0);
|
||||
process_loss(f, 1);
|
||||
process_loss(f, 2);
|
||||
isolation_and_reverse(f);
|
||||
require(f.raw->format_version() == cellular_lives_migration_format,
|
||||
"reverse transition downgraded accounting owner");
|
||||
}
|
||||
void host_ownership_and_admission(Fixture& f) {
|
||||
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture host authority");
|
||||
const auto make_host = [&](bool enabled) {
|
||||
@@ -683,6 +885,7 @@ void run_population_publication_tests() {
|
||||
process_loss(fixture, 2);
|
||||
missing_and_corrupt_objects(fixture);
|
||||
isolation_and_reverse(fixture);
|
||||
boundary_upgrades(fixture);
|
||||
host_ownership_and_admission(fixture);
|
||||
std::puts("population publication, exact recovery, process loss and fencing passed");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user