M12.4: retain read-only destination migration stages
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m25s
Foundation / checks (network) (push) Successful in 20m44s
Foundation / checks (client) (push) Successful in 21m57s
Foundation / checks (tsan) (push) Successful in 22m46s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 30m2s
Foundation / checks (postgres) (push) Canceled after 29m56s

This commit is contained in:
dingo
2026-10-18 09:32:50 -04:00
parent c625a4cb68
commit 0515702964
21 changed files with 1023 additions and 21 deletions
+319
View File
@@ -5,6 +5,7 @@
#include "evo/server/cellular_region_host.hpp"
#include "fixtures/cellular_ecology.hpp"
#include <barrier>
#include <csignal>
#include <cstdlib>
#include <filesystem>
@@ -868,6 +869,155 @@ void migration_freezes(Fixture& f) {
!take(retry_cellular_migration_freeze(*f.db, f.lease, prepared, 5s), "duplicate freeze")
.aborted,
"duplicate freeze lost its state");
const auto stage = [&](const CellularMigrationTrace& trace,
std::chrono::milliseconds timeout = 5s) {
return stage_cellular_migration(*f.db, *f.objects, *destination_objects,
destination_lease, f.head, destination_head, trace,
timeout);
};
// The optimistic destination cut must still be exact at first staging.
take(f.db->query("BEGIN", {}, 5s), "move destination before stage transaction");
if (representation == 0)
take(migrate_cellular_to_population(*f.db, *destination_objects, destination_lease,
destination_head, f.id(), f.build, {}, 5s),
"temporarily move destination profile");
else
take(materialize_population_profile(*f.db, *destination_objects, destination_lease,
destination_head, f.id(), f.build, 5s),
"temporarily move population destination");
const auto moved = stage(frozen.trace);
take(f.db->query("ROLLBACK", {}, 5s), "rollback moved destination fixture");
require(!moved && moved.error().store.kind == LeaseStoreError::head_mismatch,
"stage silently rebased a moved destination");
if (representation == 1) {
take(f.db->query("BEGIN", {}, 5s), "stage pending interval transaction");
take(admit_population_interval(*f.db, destination_key, destination_lease,
destination_head.head,
destination_head.head.end_tick + 20, 5s),
"destination admitted work before staging");
const auto pending = stage(frozen.trace);
take(f.db->query("ROLLBACK", {}, 5s), "rollback destination interval fixture");
require(!pending && pending.error().store.kind == LeaseStoreError::commit_conflict,
"stage stranded destination interval work");
}
const auto staged = take(stage(frozen.trace), "durable destination staging");
const CellularMigrationStagePrepared stage_request{destination_key, prepared.trace,
staged.ghost};
require(
!staged.freeze.aborted &&
take(take(load_cellular_migration_stage(*f.db, *destination_objects,
destination_key, destination_lease, 5s),
"recover destination ghost"),
"missing staged destination")
.ghost == staged.ghost &&
take(load_region_checkpoint(*f.db, destination_key, 5s),
"staged destination still owns original head")
.head == destination_before.head &&
take(retry_cellular_migration_stage(*f.db, destination_lease, stage_request, 5s),
"duplicate stage")
.ghost == staged.ghost,
"staging activated a ghost or lost exact replay");
auto changed_stage = stage_request;
++changed_stage.ghost.bytes;
const auto changed =
retry_cellular_migration_stage(*f.db, destination_lease, changed_stage, 5s);
require(!changed && changed.error().store.kind == LeaseStoreError::commit_conflict,
"stage accepted conflicting immutable ghost bytes");
auto reverse = frozen.trace;
reverse.request.request = f.id();
std::swap(reverse.request.source, reverse.request.destination);
const auto overlapping = retry_cellular_migration_freeze(
*f.db, destination_lease,
{destination_key, take(encode_migration_trace(reverse), "overlapping source trace")},
5s);
require(!overlapping && overlapping.error().store.kind == LeaseStoreError::commit_conflict,
"staged destination simultaneously prepared an outgoing workflow");
take(f.db->query("BEGIN", {}, 5s), "staged destination life reservation transaction");
Uuid new_account{};
new_account.back() = 5;
const auto staged_control =
take(claim_account_control(*f.db, destination_key, destination_lease, new_account, 1,
9950, 60000ms, 5s),
"staged destination reservation current control");
const auto staged_life =
reserve_primitive_life(*f.db, destination_key, destination_lease, new_account,
staged_control.generation, 1, 1, 5s);
take(f.db->query("ROLLBACK", {}, 5s), "rollback staged life reservation fixture");
require(!staged_life &&
std::string(staged_life.error().cause.sqlstate.data(), 5) == "EVL18",
"staged destination admitted a new player life reservation");
if (representation == 0) {
CellularBatch inputs{{f.key.world, 2, 1, destination_head.head.commit,
destination_head.head.digest, destination->tick(),
destination->tick() + 1},
{{}}};
const auto advanced = take(replay_cellular_batch(*destination, inputs),
"detached staged destination future");
const auto denied =
publish_cellular_batch(*f.db, *destination_objects, destination_lease,
destination_head, inputs, *advanced, f.id(), f.build, 5s);
require(!denied &&
std::string(denied.error().store.cause.sqlstate.data(), 5) == "EVL18",
"staged destination published an ordinary future");
} else {
require(!admit_population_interval(*f.db, destination_key, destination_lease,
destination_head.head,
destination_head.head.end_tick + 20, 5s) &&
!materialize_population_profile(*f.db, *destination_objects,
destination_lease, destination_head, f.id(),
f.build, 5s),
"staged destination progressed or changed profile");
}
const auto ghost_path =
destination_objects->directory() / hexadecimal(staged.ghost.sha256, false);
const auto ghost_backup = ghost_path.string() + ".stage-test";
std::filesystem::rename(ghost_path, ghost_backup);
const auto missing_ghost = load_cellular_migration_stage(
*f.db, *destination_objects, destination_key, destination_lease, 5s);
std::filesystem::rename(ghost_backup, ghost_path);
require(!missing_ghost && missing_ghost.error().kind == RecoveryError::tail,
"missing ghost fell back to the active destination");
char ghost_first{};
{
std::fstream file(ghost_path, std::ios::binary | std::ios::in | std::ios::out);
file.get(ghost_first);
file.seekp(0);
file.put(static_cast<char>(static_cast<unsigned char>(ghost_first) ^ 1U));
file.flush();
require(file.good(), "same-length ghost corruption fixture");
}
const auto corrupt_ghost = load_cellular_migration_stage(
*f.db, *destination_objects, destination_key, destination_lease, 5s);
{
std::fstream file(ghost_path, std::ios::binary | std::ios::in | std::ios::out);
file.put(ghost_first);
file.flush();
require(file.good(), "restore corrupt ghost fixture");
}
require(!corrupt_ghost && corrupt_ghost.error().kind == RecoveryError::tail,
"same-length corrupt ghost bypassed digest verification");
const auto source_manifest = take(decode_manifest(f.head.manifest), "stage source capsule");
const auto source_capsule_path =
destination_objects->directory() / hexadecimal(source_manifest.payload_sha256, false);
const auto source_capsule_backup = source_capsule_path.string() + ".stage-test";
std::filesystem::rename(source_capsule_path, source_capsule_backup);
const auto missing_source = load_cellular_migration_stage(
*f.db, *destination_objects, destination_key, destination_lease, 5s);
std::filesystem::rename(source_capsule_backup, source_capsule_path);
require(!missing_source && missing_source.error().kind == RecoveryError::tail,
"destination recovery used the live source instead of its retained capsule");
const auto old_destination_lease = destination_lease;
take(release_region_lease(*f.db, destination_key, destination_lease, 5s),
"release staged destination owner");
destination_lease = take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s),
"replacement staged destination owner");
require(
take(load_cellular_migration_stage(*f.db, *destination_objects, destination_key,
destination_lease, 5s),
"new epoch ghost replay")
.has_value() &&
!retry_cellular_migration_stage(*f.db, old_destination_lease, stage_request, 5s),
"replacement activated the ghost or stale destination staged it");
auto conflict_trace = frozen.trace;
++conflict_trace.transferred.energy.value;
auto conflict = retry_cellular_migration_freeze(
@@ -960,6 +1110,15 @@ void migration_freezes(Fixture& f) {
"aborted pointer load")
.has_value(),
"abort was not durable or preparation retry reopened it");
require(
!take(load_cellular_migration_stage(*f.db, *destination_objects, destination_key,
destination_lease, 5s),
"aborted destination pointer")
.has_value() &&
take(retry_cellular_migration_stage(*f.db, destination_lease, stage_request, 5s),
"retry aborted stage")
.freeze.aborted,
"source abort stranded the destination or stage retry reopened its pointer");
require(take(load_region_checkpoint(*f.db, f.key, 5s), "aborted source head").head ==
source_before.head &&
f.raw->snapshot() == source_bytes,
@@ -1004,6 +1163,76 @@ void migration_freezes(Fixture& f) {
5s),
"abort reconciled freeze");
const auto false_stage_freeze = take(prepare(request_for()), "false ghost preparation");
const auto original_destination =
take(decode_manifest(destination_head.manifest), "false ghost existing object");
const CellularMigrationStagePrepared false_ghost{
destination_key,
take(encode_migration_trace(false_stage_freeze.trace), "false ghost trace"),
{original_destination.payload_sha256, original_destination.payload_bytes}};
take(retry_cellular_migration_stage(*f.db, destination_lease, false_ghost, 5s),
"metadata-only false ghost stage");
const auto false_ghost_replay = load_cellular_migration_stage(
*f.db, *destination_objects, destination_key, destination_lease, 5s);
require(!false_ghost_replay && false_ghost_replay.error().kind == RecoveryError::history,
"stage recovery accepted a real object with the wrong complete biology");
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
false_stage_freeze.trace.request.request, 5s),
"abort false ghost evidence");
const auto uncertain_stage_freeze =
take(prepare(request_for()), "unknown stage preparation");
take(locker->query("BEGIN", {}, 5s), "stage ambiguity lock");
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND "
"region_id=2 FOR UPDATE",
{}, 5s),
"hold stage reply");
auto stage_timed = connection();
const auto stage_unknown = stage_cellular_migration(
*stage_timed, *f.objects, *destination_objects, destination_lease, f.head,
destination_head, uncertain_stage_freeze.trace, 300ms);
require(!stage_unknown && stage_unknown.error().kind == PublicationError::unknown &&
!stage_unknown.error().prepared.trace.empty(),
"stage ambiguity lost exact bytes");
take(locker->query("ROLLBACK", {}, 5s), "release stage ambiguity");
take(retry_cellular_migration_stage(*f.db, destination_lease,
stage_unknown.error().prepared, 5s),
"resolve unknown stage");
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
uncertain_stage_freeze.trace.request.request, 5s),
"abort reconciled stage");
for (unsigned race = 0; race < 4; ++race) {
const auto race_freeze = take(prepare(request_for()), "stage race preparation");
auto stage_db = connection(), abort_db = connection();
std::barrier start(2);
std::optional<std::expected<CellularMigrationStage, CellularMigrationStageFailure>>
staged_race;
std::optional<std::expected<CellularMigrationFreeze, LeaseStoreFailure>> aborted_race;
std::thread staging([&] {
start.arrive_and_wait();
staged_race = stage_cellular_migration(*stage_db, *f.objects, *destination_objects,
destination_lease, f.head, destination_head,
race_freeze.trace, 5s);
});
std::thread aborting([&] {
start.arrive_and_wait();
aborted_race = abort_cellular_migration_freeze(
*abort_db, f.key, f.lease, race_freeze.trace.request.request, 5s);
});
staging.join();
aborting.join();
require(aborted_race.has_value() && aborted_race->has_value() &&
staged_race.has_value() &&
(staged_race->has_value() ||
staged_race->error().store.kind == LeaseStoreError::commit_conflict) &&
!take(load_cellular_migration_stage(*f.db, *destination_objects,
destination_key, destination_lease, 5s),
"stage race final pointer")
.has_value(),
"stage/abort race left an active ghost or stranded pause");
}
// Both sides of preparation's transaction boundary, with real process death.
for (unsigned point = 0; point < 2; ++point) {
const auto killed_request = request_for();
@@ -1037,6 +1266,38 @@ void migration_freezes(Fixture& f) {
5s),
"abort killed preparation");
}
for (unsigned point = 0; point < 2; ++point) {
const auto killed = take(prepare(request_for()), "stage process-loss preparation");
const auto child = ::fork();
require(child >= 0, "stage publisher fork");
if (child == 0) {
try {
auto db = connection();
PublishHooks hooks;
if (point == 0)
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
if (!stage_cellular_migration(*db, *f.objects, *destination_objects,
destination_lease, f.head, destination_head,
killed.trace, 5s, hooks))
::_exit(44);
::kill(::getpid(), SIGKILL);
} catch (...) {
::_exit(45);
}
::_exit(46);
}
int status{};
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
WTERMSIG(status) == SIGKILL &&
take(load_cellular_migration_stage(*f.db, *destination_objects,
destination_key, destination_lease, 5s),
"recover killed stage")
.has_value() == (point == 1),
"stage process loss changed ownership or lost the durable pause");
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
killed.trace.request.request, 5s),
"abort killed stage");
}
}
// The owning host must retain the freeze through drain/recreation with entry off.
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture to frozen host");
@@ -1107,6 +1368,64 @@ void migration_freezes(Fixture& f) {
host->drain();
f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s),
"return freeze fixture authority");
const auto destination_host_freeze =
take(prepare_cellular_migration_freeze(*f.db, *f.objects, *destination_objects, f.lease,
f.head, destination_head, request_for(), 5s),
"destination host source preparation");
take(release_region_lease(*f.db, destination_key, destination_lease, 5s),
"release destination to owning host");
const auto make_destination_host = [&] {
auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), destination_key,
max_cellular_population_bytes),
"destination host objects");
CellularHostConfig config{destination_key, 1, {}, f.build, {f.build}};
config.lease.commit_timeout = 300ms;
return take(
CellularRegionHost::create(
config,
[] { return std::expected<std::unique_ptr<Postgres>, PgFailure>{connection()}; },
std::move(objects)),
"destination staging host");
};
auto destination_host = make_destination_host();
require(destination_host->step().ready, "destination host before staging");
const auto destination_bytes = destination_host->state()->snapshot();
take(locker->query("BEGIN", {}, 5s), "host stage ambiguity transaction");
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND "
"region_id=1 FOR UPDATE",
{}, 5s),
"hold owning destination stage");
const auto host_stage_unknown =
destination_host->stage_migration(f.head, *f.objects, destination_host_freeze.trace);
require(!host_stage_unknown && !destination_host->status().ready &&
destination_host->state()->snapshot() == destination_bytes,
"owning destination advanced during ambiguous staging");
take(locker->query("ROLLBACK", {}, 5s), "release owning stage reply");
for (unsigned attempt = 0; attempt < 200 && !destination_host->status().migration_frozen;
++attempt) {
destination_host->step();
std::this_thread::sleep_for(25ms);
}
require(destination_host->status().migration_frozen && !destination_host->status().ready &&
!destination_host->advance_unattended(64),
"owning destination activated its ghost");
destination_host->drain();
destination_host = make_destination_host();
require(!destination_host->step().ready && destination_host->status().migration_frozen &&
destination_host->state()->snapshot() == destination_bytes &&
!destination_host->authority().has_value(),
"replacement destination lost staging or adopted the uncommitted ghost");
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
destination_host_freeze.trace.request.request, 5s),
"source-authorized abort releases destination host");
for (unsigned attempt = 0; attempt < 200 && !destination_host->step().ready; ++attempt)
std::this_thread::sleep_for(25ms);
require(destination_host->status().ready && !destination_host->status().migration_frozen &&
destination_host->state()->snapshot() == destination_bytes,
"destination failed to resume its unchanged original state after source abort");
destination_host->drain();
destination_lease = take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s),
"return staged destination fixture authority");
f.head = take(
materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
"resume frozen population as raw owner");