M12.4: retain read-only destination migration stages
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m25s
Foundation / checks (network) (push) Successful in 20m44s
Foundation / checks (client) (push) Successful in 21m57s
Foundation / checks (tsan) (push) Successful in 22m46s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 30m2s
Foundation / checks (postgres) (push) Canceled after 29m56s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 2m25s
Foundation / checks (network) (push) Successful in 20m44s
Foundation / checks (client) (push) Successful in 21m57s
Foundation / checks (tsan) (push) Successful in 22m46s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 30m2s
Foundation / checks (postgres) (push) Canceled after 29m56s
This commit is contained in:
@@ -5,6 +5,7 @@
|
||||
#include "evo/server/cellular_region_host.hpp"
|
||||
#include "fixtures/cellular_ecology.hpp"
|
||||
|
||||
#include <barrier>
|
||||
#include <csignal>
|
||||
#include <cstdlib>
|
||||
#include <filesystem>
|
||||
@@ -868,6 +869,155 @@ void migration_freezes(Fixture& f) {
|
||||
!take(retry_cellular_migration_freeze(*f.db, f.lease, prepared, 5s), "duplicate freeze")
|
||||
.aborted,
|
||||
"duplicate freeze lost its state");
|
||||
const auto stage = [&](const CellularMigrationTrace& trace,
|
||||
std::chrono::milliseconds timeout = 5s) {
|
||||
return stage_cellular_migration(*f.db, *f.objects, *destination_objects,
|
||||
destination_lease, f.head, destination_head, trace,
|
||||
timeout);
|
||||
};
|
||||
// The optimistic destination cut must still be exact at first staging.
|
||||
take(f.db->query("BEGIN", {}, 5s), "move destination before stage transaction");
|
||||
if (representation == 0)
|
||||
take(migrate_cellular_to_population(*f.db, *destination_objects, destination_lease,
|
||||
destination_head, f.id(), f.build, {}, 5s),
|
||||
"temporarily move destination profile");
|
||||
else
|
||||
take(materialize_population_profile(*f.db, *destination_objects, destination_lease,
|
||||
destination_head, f.id(), f.build, 5s),
|
||||
"temporarily move population destination");
|
||||
const auto moved = stage(frozen.trace);
|
||||
take(f.db->query("ROLLBACK", {}, 5s), "rollback moved destination fixture");
|
||||
require(!moved && moved.error().store.kind == LeaseStoreError::head_mismatch,
|
||||
"stage silently rebased a moved destination");
|
||||
if (representation == 1) {
|
||||
take(f.db->query("BEGIN", {}, 5s), "stage pending interval transaction");
|
||||
take(admit_population_interval(*f.db, destination_key, destination_lease,
|
||||
destination_head.head,
|
||||
destination_head.head.end_tick + 20, 5s),
|
||||
"destination admitted work before staging");
|
||||
const auto pending = stage(frozen.trace);
|
||||
take(f.db->query("ROLLBACK", {}, 5s), "rollback destination interval fixture");
|
||||
require(!pending && pending.error().store.kind == LeaseStoreError::commit_conflict,
|
||||
"stage stranded destination interval work");
|
||||
}
|
||||
const auto staged = take(stage(frozen.trace), "durable destination staging");
|
||||
const CellularMigrationStagePrepared stage_request{destination_key, prepared.trace,
|
||||
staged.ghost};
|
||||
require(
|
||||
!staged.freeze.aborted &&
|
||||
take(take(load_cellular_migration_stage(*f.db, *destination_objects,
|
||||
destination_key, destination_lease, 5s),
|
||||
"recover destination ghost"),
|
||||
"missing staged destination")
|
||||
.ghost == staged.ghost &&
|
||||
take(load_region_checkpoint(*f.db, destination_key, 5s),
|
||||
"staged destination still owns original head")
|
||||
.head == destination_before.head &&
|
||||
take(retry_cellular_migration_stage(*f.db, destination_lease, stage_request, 5s),
|
||||
"duplicate stage")
|
||||
.ghost == staged.ghost,
|
||||
"staging activated a ghost or lost exact replay");
|
||||
auto changed_stage = stage_request;
|
||||
++changed_stage.ghost.bytes;
|
||||
const auto changed =
|
||||
retry_cellular_migration_stage(*f.db, destination_lease, changed_stage, 5s);
|
||||
require(!changed && changed.error().store.kind == LeaseStoreError::commit_conflict,
|
||||
"stage accepted conflicting immutable ghost bytes");
|
||||
auto reverse = frozen.trace;
|
||||
reverse.request.request = f.id();
|
||||
std::swap(reverse.request.source, reverse.request.destination);
|
||||
const auto overlapping = retry_cellular_migration_freeze(
|
||||
*f.db, destination_lease,
|
||||
{destination_key, take(encode_migration_trace(reverse), "overlapping source trace")},
|
||||
5s);
|
||||
require(!overlapping && overlapping.error().store.kind == LeaseStoreError::commit_conflict,
|
||||
"staged destination simultaneously prepared an outgoing workflow");
|
||||
take(f.db->query("BEGIN", {}, 5s), "staged destination life reservation transaction");
|
||||
Uuid new_account{};
|
||||
new_account.back() = 5;
|
||||
const auto staged_control =
|
||||
take(claim_account_control(*f.db, destination_key, destination_lease, new_account, 1,
|
||||
9950, 60000ms, 5s),
|
||||
"staged destination reservation current control");
|
||||
const auto staged_life =
|
||||
reserve_primitive_life(*f.db, destination_key, destination_lease, new_account,
|
||||
staged_control.generation, 1, 1, 5s);
|
||||
take(f.db->query("ROLLBACK", {}, 5s), "rollback staged life reservation fixture");
|
||||
require(!staged_life &&
|
||||
std::string(staged_life.error().cause.sqlstate.data(), 5) == "EVL18",
|
||||
"staged destination admitted a new player life reservation");
|
||||
if (representation == 0) {
|
||||
CellularBatch inputs{{f.key.world, 2, 1, destination_head.head.commit,
|
||||
destination_head.head.digest, destination->tick(),
|
||||
destination->tick() + 1},
|
||||
{{}}};
|
||||
const auto advanced = take(replay_cellular_batch(*destination, inputs),
|
||||
"detached staged destination future");
|
||||
const auto denied =
|
||||
publish_cellular_batch(*f.db, *destination_objects, destination_lease,
|
||||
destination_head, inputs, *advanced, f.id(), f.build, 5s);
|
||||
require(!denied &&
|
||||
std::string(denied.error().store.cause.sqlstate.data(), 5) == "EVL18",
|
||||
"staged destination published an ordinary future");
|
||||
} else {
|
||||
require(!admit_population_interval(*f.db, destination_key, destination_lease,
|
||||
destination_head.head,
|
||||
destination_head.head.end_tick + 20, 5s) &&
|
||||
!materialize_population_profile(*f.db, *destination_objects,
|
||||
destination_lease, destination_head, f.id(),
|
||||
f.build, 5s),
|
||||
"staged destination progressed or changed profile");
|
||||
}
|
||||
const auto ghost_path =
|
||||
destination_objects->directory() / hexadecimal(staged.ghost.sha256, false);
|
||||
const auto ghost_backup = ghost_path.string() + ".stage-test";
|
||||
std::filesystem::rename(ghost_path, ghost_backup);
|
||||
const auto missing_ghost = load_cellular_migration_stage(
|
||||
*f.db, *destination_objects, destination_key, destination_lease, 5s);
|
||||
std::filesystem::rename(ghost_backup, ghost_path);
|
||||
require(!missing_ghost && missing_ghost.error().kind == RecoveryError::tail,
|
||||
"missing ghost fell back to the active destination");
|
||||
char ghost_first{};
|
||||
{
|
||||
std::fstream file(ghost_path, std::ios::binary | std::ios::in | std::ios::out);
|
||||
file.get(ghost_first);
|
||||
file.seekp(0);
|
||||
file.put(static_cast<char>(static_cast<unsigned char>(ghost_first) ^ 1U));
|
||||
file.flush();
|
||||
require(file.good(), "same-length ghost corruption fixture");
|
||||
}
|
||||
const auto corrupt_ghost = load_cellular_migration_stage(
|
||||
*f.db, *destination_objects, destination_key, destination_lease, 5s);
|
||||
{
|
||||
std::fstream file(ghost_path, std::ios::binary | std::ios::in | std::ios::out);
|
||||
file.put(ghost_first);
|
||||
file.flush();
|
||||
require(file.good(), "restore corrupt ghost fixture");
|
||||
}
|
||||
require(!corrupt_ghost && corrupt_ghost.error().kind == RecoveryError::tail,
|
||||
"same-length corrupt ghost bypassed digest verification");
|
||||
const auto source_manifest = take(decode_manifest(f.head.manifest), "stage source capsule");
|
||||
const auto source_capsule_path =
|
||||
destination_objects->directory() / hexadecimal(source_manifest.payload_sha256, false);
|
||||
const auto source_capsule_backup = source_capsule_path.string() + ".stage-test";
|
||||
std::filesystem::rename(source_capsule_path, source_capsule_backup);
|
||||
const auto missing_source = load_cellular_migration_stage(
|
||||
*f.db, *destination_objects, destination_key, destination_lease, 5s);
|
||||
std::filesystem::rename(source_capsule_backup, source_capsule_path);
|
||||
require(!missing_source && missing_source.error().kind == RecoveryError::tail,
|
||||
"destination recovery used the live source instead of its retained capsule");
|
||||
const auto old_destination_lease = destination_lease;
|
||||
take(release_region_lease(*f.db, destination_key, destination_lease, 5s),
|
||||
"release staged destination owner");
|
||||
destination_lease = take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s),
|
||||
"replacement staged destination owner");
|
||||
require(
|
||||
take(load_cellular_migration_stage(*f.db, *destination_objects, destination_key,
|
||||
destination_lease, 5s),
|
||||
"new epoch ghost replay")
|
||||
.has_value() &&
|
||||
!retry_cellular_migration_stage(*f.db, old_destination_lease, stage_request, 5s),
|
||||
"replacement activated the ghost or stale destination staged it");
|
||||
auto conflict_trace = frozen.trace;
|
||||
++conflict_trace.transferred.energy.value;
|
||||
auto conflict = retry_cellular_migration_freeze(
|
||||
@@ -960,6 +1110,15 @@ void migration_freezes(Fixture& f) {
|
||||
"aborted pointer load")
|
||||
.has_value(),
|
||||
"abort was not durable or preparation retry reopened it");
|
||||
require(
|
||||
!take(load_cellular_migration_stage(*f.db, *destination_objects, destination_key,
|
||||
destination_lease, 5s),
|
||||
"aborted destination pointer")
|
||||
.has_value() &&
|
||||
take(retry_cellular_migration_stage(*f.db, destination_lease, stage_request, 5s),
|
||||
"retry aborted stage")
|
||||
.freeze.aborted,
|
||||
"source abort stranded the destination or stage retry reopened its pointer");
|
||||
require(take(load_region_checkpoint(*f.db, f.key, 5s), "aborted source head").head ==
|
||||
source_before.head &&
|
||||
f.raw->snapshot() == source_bytes,
|
||||
@@ -1004,6 +1163,76 @@ void migration_freezes(Fixture& f) {
|
||||
5s),
|
||||
"abort reconciled freeze");
|
||||
|
||||
const auto false_stage_freeze = take(prepare(request_for()), "false ghost preparation");
|
||||
const auto original_destination =
|
||||
take(decode_manifest(destination_head.manifest), "false ghost existing object");
|
||||
const CellularMigrationStagePrepared false_ghost{
|
||||
destination_key,
|
||||
take(encode_migration_trace(false_stage_freeze.trace), "false ghost trace"),
|
||||
{original_destination.payload_sha256, original_destination.payload_bytes}};
|
||||
take(retry_cellular_migration_stage(*f.db, destination_lease, false_ghost, 5s),
|
||||
"metadata-only false ghost stage");
|
||||
const auto false_ghost_replay = load_cellular_migration_stage(
|
||||
*f.db, *destination_objects, destination_key, destination_lease, 5s);
|
||||
require(!false_ghost_replay && false_ghost_replay.error().kind == RecoveryError::history,
|
||||
"stage recovery accepted a real object with the wrong complete biology");
|
||||
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
|
||||
false_stage_freeze.trace.request.request, 5s),
|
||||
"abort false ghost evidence");
|
||||
|
||||
const auto uncertain_stage_freeze =
|
||||
take(prepare(request_for()), "unknown stage preparation");
|
||||
take(locker->query("BEGIN", {}, 5s), "stage ambiguity lock");
|
||||
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND "
|
||||
"region_id=2 FOR UPDATE",
|
||||
{}, 5s),
|
||||
"hold stage reply");
|
||||
auto stage_timed = connection();
|
||||
const auto stage_unknown = stage_cellular_migration(
|
||||
*stage_timed, *f.objects, *destination_objects, destination_lease, f.head,
|
||||
destination_head, uncertain_stage_freeze.trace, 300ms);
|
||||
require(!stage_unknown && stage_unknown.error().kind == PublicationError::unknown &&
|
||||
!stage_unknown.error().prepared.trace.empty(),
|
||||
"stage ambiguity lost exact bytes");
|
||||
take(locker->query("ROLLBACK", {}, 5s), "release stage ambiguity");
|
||||
take(retry_cellular_migration_stage(*f.db, destination_lease,
|
||||
stage_unknown.error().prepared, 5s),
|
||||
"resolve unknown stage");
|
||||
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
|
||||
uncertain_stage_freeze.trace.request.request, 5s),
|
||||
"abort reconciled stage");
|
||||
|
||||
for (unsigned race = 0; race < 4; ++race) {
|
||||
const auto race_freeze = take(prepare(request_for()), "stage race preparation");
|
||||
auto stage_db = connection(), abort_db = connection();
|
||||
std::barrier start(2);
|
||||
std::optional<std::expected<CellularMigrationStage, CellularMigrationStageFailure>>
|
||||
staged_race;
|
||||
std::optional<std::expected<CellularMigrationFreeze, LeaseStoreFailure>> aborted_race;
|
||||
std::thread staging([&] {
|
||||
start.arrive_and_wait();
|
||||
staged_race = stage_cellular_migration(*stage_db, *f.objects, *destination_objects,
|
||||
destination_lease, f.head, destination_head,
|
||||
race_freeze.trace, 5s);
|
||||
});
|
||||
std::thread aborting([&] {
|
||||
start.arrive_and_wait();
|
||||
aborted_race = abort_cellular_migration_freeze(
|
||||
*abort_db, f.key, f.lease, race_freeze.trace.request.request, 5s);
|
||||
});
|
||||
staging.join();
|
||||
aborting.join();
|
||||
require(aborted_race.has_value() && aborted_race->has_value() &&
|
||||
staged_race.has_value() &&
|
||||
(staged_race->has_value() ||
|
||||
staged_race->error().store.kind == LeaseStoreError::commit_conflict) &&
|
||||
!take(load_cellular_migration_stage(*f.db, *destination_objects,
|
||||
destination_key, destination_lease, 5s),
|
||||
"stage race final pointer")
|
||||
.has_value(),
|
||||
"stage/abort race left an active ghost or stranded pause");
|
||||
}
|
||||
|
||||
// Both sides of preparation's transaction boundary, with real process death.
|
||||
for (unsigned point = 0; point < 2; ++point) {
|
||||
const auto killed_request = request_for();
|
||||
@@ -1037,6 +1266,38 @@ void migration_freezes(Fixture& f) {
|
||||
5s),
|
||||
"abort killed preparation");
|
||||
}
|
||||
for (unsigned point = 0; point < 2; ++point) {
|
||||
const auto killed = take(prepare(request_for()), "stage process-loss preparation");
|
||||
const auto child = ::fork();
|
||||
require(child >= 0, "stage publisher fork");
|
||||
if (child == 0) {
|
||||
try {
|
||||
auto db = connection();
|
||||
PublishHooks hooks;
|
||||
if (point == 0)
|
||||
hooks.before_transaction = [] { ::kill(::getpid(), SIGKILL); };
|
||||
if (!stage_cellular_migration(*db, *f.objects, *destination_objects,
|
||||
destination_lease, f.head, destination_head,
|
||||
killed.trace, 5s, hooks))
|
||||
::_exit(44);
|
||||
::kill(::getpid(), SIGKILL);
|
||||
} catch (...) {
|
||||
::_exit(45);
|
||||
}
|
||||
::_exit(46);
|
||||
}
|
||||
int status{};
|
||||
require(::waitpid(child, &status, 0) == child && WIFSIGNALED(status) &&
|
||||
WTERMSIG(status) == SIGKILL &&
|
||||
take(load_cellular_migration_stage(*f.db, *destination_objects,
|
||||
destination_key, destination_lease, 5s),
|
||||
"recover killed stage")
|
||||
.has_value() == (point == 1),
|
||||
"stage process loss changed ownership or lost the durable pause");
|
||||
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
|
||||
killed.trace.request.request, 5s),
|
||||
"abort killed stage");
|
||||
}
|
||||
}
|
||||
// The owning host must retain the freeze through drain/recreation with entry off.
|
||||
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release fixture to frozen host");
|
||||
@@ -1107,6 +1368,64 @@ void migration_freezes(Fixture& f) {
|
||||
host->drain();
|
||||
f.lease = take(acquire_region_lease(*f.db, f.key, f.build, 60000ms, 5s),
|
||||
"return freeze fixture authority");
|
||||
const auto destination_host_freeze =
|
||||
take(prepare_cellular_migration_freeze(*f.db, *f.objects, *destination_objects, f.lease,
|
||||
f.head, destination_head, request_for(), 5s),
|
||||
"destination host source preparation");
|
||||
take(release_region_lease(*f.db, destination_key, destination_lease, 5s),
|
||||
"release destination to owning host");
|
||||
const auto make_destination_host = [&] {
|
||||
auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), destination_key,
|
||||
max_cellular_population_bytes),
|
||||
"destination host objects");
|
||||
CellularHostConfig config{destination_key, 1, {}, f.build, {f.build}};
|
||||
config.lease.commit_timeout = 300ms;
|
||||
return take(
|
||||
CellularRegionHost::create(
|
||||
config,
|
||||
[] { return std::expected<std::unique_ptr<Postgres>, PgFailure>{connection()}; },
|
||||
std::move(objects)),
|
||||
"destination staging host");
|
||||
};
|
||||
auto destination_host = make_destination_host();
|
||||
require(destination_host->step().ready, "destination host before staging");
|
||||
const auto destination_bytes = destination_host->state()->snapshot();
|
||||
take(locker->query("BEGIN", {}, 5s), "host stage ambiguity transaction");
|
||||
take(locker->query("SELECT 1 FROM evolution.region_leases WHERE world_id=30201 AND "
|
||||
"region_id=1 FOR UPDATE",
|
||||
{}, 5s),
|
||||
"hold owning destination stage");
|
||||
const auto host_stage_unknown =
|
||||
destination_host->stage_migration(f.head, *f.objects, destination_host_freeze.trace);
|
||||
require(!host_stage_unknown && !destination_host->status().ready &&
|
||||
destination_host->state()->snapshot() == destination_bytes,
|
||||
"owning destination advanced during ambiguous staging");
|
||||
take(locker->query("ROLLBACK", {}, 5s), "release owning stage reply");
|
||||
for (unsigned attempt = 0; attempt < 200 && !destination_host->status().migration_frozen;
|
||||
++attempt) {
|
||||
destination_host->step();
|
||||
std::this_thread::sleep_for(25ms);
|
||||
}
|
||||
require(destination_host->status().migration_frozen && !destination_host->status().ready &&
|
||||
!destination_host->advance_unattended(64),
|
||||
"owning destination activated its ghost");
|
||||
destination_host->drain();
|
||||
destination_host = make_destination_host();
|
||||
require(!destination_host->step().ready && destination_host->status().migration_frozen &&
|
||||
destination_host->state()->snapshot() == destination_bytes &&
|
||||
!destination_host->authority().has_value(),
|
||||
"replacement destination lost staging or adopted the uncommitted ghost");
|
||||
take(abort_cellular_migration_freeze(*f.db, f.key, f.lease,
|
||||
destination_host_freeze.trace.request.request, 5s),
|
||||
"source-authorized abort releases destination host");
|
||||
for (unsigned attempt = 0; attempt < 200 && !destination_host->step().ready; ++attempt)
|
||||
std::this_thread::sleep_for(25ms);
|
||||
require(destination_host->status().ready && !destination_host->status().migration_frozen &&
|
||||
destination_host->state()->snapshot() == destination_bytes,
|
||||
"destination failed to resume its unchanged original state after source abort");
|
||||
destination_host->drain();
|
||||
destination_lease = take(acquire_region_lease(*f.db, destination_key, f.build, 60000ms, 5s),
|
||||
"return staged destination fixture authority");
|
||||
f.head = take(
|
||||
materialize_population_profile(*f.db, *f.objects, f.lease, f.head, f.id(), f.build, 5s),
|
||||
"resume frozen population as raw owner");
|
||||
|
||||
Reference in New Issue
Block a user