M12.4: qualify migration ownership across TLS path loss
Foundation / checks (ecs) (push) Successful in 2m29s
Foundation / checks (deploy) (push) Successful in 4m8s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 18m9s
Foundation / checks (client) (push) Canceled after 18m8s
Foundation / checks (network) (push) Canceled after 18m7s
Foundation / checks (tsan) (push) Canceled after 15m39s
Foundation / checks (postgres) (push) Canceled after 18m13s

This commit is contained in:
dingo
2026-10-18 13:19:59 -04:00
parent 0f071f3f8b
commit e0a7c13f6b
16 changed files with 707 additions and 14 deletions
+264
View File
@@ -4,8 +4,10 @@
#include "evo/persistence/sha256.hpp"
#include "evo/server/cellular_region_host.hpp"
#include "fixtures/cellular_ecology.hpp"
#include "fixtures/postgres_relay.hpp"
#include <barrier>
#include <charconv>
#include <csignal>
#include <cstdlib>
#include <filesystem>
@@ -96,6 +98,73 @@ struct Fixture {
head.head.end_tick + count};
}
};
std::uint16_t migration_backend_port() {
const auto port = environment("EVO_MIGRATION_TCP_PORT");
unsigned value{};
const auto [end, error] = std::from_chars(port.data(), port.data() + port.size(), value);
require(error == std::errc{} && end == port.data() + port.size() && value > 0 && value <= 65535,
"invalid private migration TLS port");
return static_cast<std::uint16_t>(value);
}
PgOptions migration_tls_options(const test::PostgresRelay& relay) {
return {"localhost",
"127.0.0.1",
relay.port(),
environment("PGDATABASE"),
environment("EVO_PG_RUNTIME_USER"),
{},
environment("EVO_MIGRATION_CA")};
}
std::unique_ptr<Postgres> migration_tls_connection(const test::PostgresRelay& relay) {
auto db = take(Postgres::connect(migration_tls_options(relay), 2s),
"migration verify-full TLS connection");
take(db->query("SELECT 1", {}, 2s), "migration TLS warmup");
return db;
}
void migration_transport_commit(Fixture& f, const CellularMigrationCommitPrepared& prepared,
RegionKey sk, RegionKey dk, const LeaseGrant& sl,
const LeaseGrant& dl, const RegionHead& source_before,
const RegionHead& destination_before) {
{
test::PostgresRelay relay(migration_backend_port());
auto wrong = migration_tls_options(relay);
wrong.host = "untrusted.evolution.test";
require(!Postgres::connect(wrong, 1s), "relay bypassed TLS hostname verification");
}
{
test::PostgresRelay relay(migration_backend_port());
auto db = migration_tls_connection(relay);
relay.drop_requests(true);
const auto lost = retry_cellular_migration_commit(*db, sl, dl, prepared, 300ms);
require(
!lost && lost.error().kind == PublicationError::unknown &&
lost.error().prepared.source_manifest == prepared.source_manifest &&
lost.error().prepared.destination_manifest == prepared.destination_manifest &&
relay.dropped_requests() > 0 && relay.healthy() &&
take(load_region_checkpoint(*f.db, sk, 5s), "lost request source head").head ==
source_before &&
take(load_region_checkpoint(*f.db, dk, 5s), "lost request destination head").head ==
destination_before,
"a dropped TLS request changed ownership or lost the exact pair");
}
{
test::PostgresRelay relay(migration_backend_port());
auto db = migration_tls_connection(relay);
relay.drop_responses(true);
const auto lost = retry_cellular_migration_commit(*db, sl, dl, prepared, 500ms);
require(
!lost && lost.error().kind == PublicationError::unknown &&
lost.error().prepared.source_manifest == prepared.source_manifest &&
lost.error().prepared.destination_manifest == prepared.destination_manifest &&
relay.dropped_responses() > 0 && relay.healthy() &&
take(load_region_checkpoint(*f.db, sk, 5s), "lost reply source head").manifest ==
prepared.source_manifest &&
take(load_region_checkpoint(*f.db, dk, 5s), "lost reply destination head")
.manifest == prepared.destination_manifest,
"lost TLS receipt was treated as rollback or committed only one head");
}
std::puts("migration TLS request/reply loss and hostname verification passed");
}
void funded_source(Fixture& f) {
f.build.fill(0xDA);
const std::array<std::string, 1> build{hexadecimal(f.build)};
@@ -746,6 +815,196 @@ void boundary_upgrades(Fixture& f) {
require(f.raw->format_version() == cellular_lives_migration_format,
"reverse transition downgraded accounting owner");
}
void migration_owner_partitions(Fixture& f, RegionKey destination_key,
FilesystemObjectStore& destination_objects,
LeaseGrant& destination_lease,
CommittedCheckpoint& destination_head, std::uint64_t genome) {
if (!std::getenv("EVO_MIGRATION_TCP_PORT"))
return;
const std::array<Sha256Digest, 1> builds{f.build};
for (unsigned lost_source = 0; lost_source < 2; ++lost_source) {
test::PostgresRelay source_relay(migration_backend_port()),
destination_relay(migration_backend_port());
const auto make_host = [&](RegionKey key, const test::PostgresRelay& relay) {
auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), key,
max_cellular_population_bytes),
"partition owner objects");
CellularHostConfig config{key, 1, {}, f.build, {f.build}};
config.lease = {3000ms, 400ms, 300ms, 150ms, 250ms};
const auto options = migration_tls_options(relay);
return take(CellularRegionHost::create(
config, [options] { return Postgres::connect(options, 500ms); },
std::move(objects)),
"partition owner host");
};
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release source to TLS owner");
take(release_region_lease(*f.db, destination_key, destination_lease, 5s),
"release destination to TLS owner");
auto source_host = make_host(f.key, source_relay),
destination_host = make_host(destination_key, destination_relay);
require(source_host->step().ready && destination_host->step().ready,
"TLS owners before freeze");
const auto sl = take(source_host->authority(), "TLS source fence"),
dl = take(destination_host->authority(), "TLS destination fence");
const auto source_bytes = source_host->state()->snapshot(),
destination_bytes = destination_host->state()->snapshot();
const auto source_before = f.head, destination_before = destination_head;
const CellularMigrationRequest request{
f.id(),
{f.key.region, f.head.head.commit, f.id(), f.head.head.digest},
{destination_key.region, destination_head.head.commit, f.id(),
destination_head.head.digest},
genome};
source_relay.drop_responses(true);
const auto unknown_prepare =
source_host->prepare_migration_freeze(destination_head, destination_objects, request);
require(!unknown_prepare &&
unknown_prepare.error().publication == PublicationError::unknown &&
!source_host->status().ready && source_relay.dropped_responses() > 0,
"lost preparation receipt did not pause source ownership");
const auto frozen =
take(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, sl, 5s),
"independent committed preparation observation"),
"missing preparation after TLS reply loss");
source_relay.drop_responses(false);
const auto reconcile_source_until = LeaseClock::now() + 2s;
while (!source_host->status().migration_frozen &&
LeaseClock::now() < reconcile_source_until) {
source_host->step();
destination_host->step();
std::this_thread::sleep_for(25ms);
}
require(source_host->status().migration_frozen && !source_host->status().ready,
"source lost the exact prepared request after TLS reconnect");
destination_relay.drop_responses(true);
const auto unknown_stage =
destination_host->stage_migration(f.head, *f.objects, frozen.trace);
require(!unknown_stage && unknown_stage.error().publication == PublicationError::unknown &&
!destination_host->status().ready &&
destination_relay.dropped_responses() > 0 &&
take(load_cellular_migration_stage(*f.db, destination_objects, destination_key,
dl, 5s),
"independent committed stage observation")
.has_value(),
"lost stage receipt activated the ghost or lost retained work");
destination_relay.drop_responses(false);
const auto reconcile_destination_until = LeaseClock::now() + 2s;
while (!destination_host->status().migration_frozen &&
LeaseClock::now() < reconcile_destination_until) {
source_host->step();
destination_host->step();
std::this_thread::sleep_for(25ms);
}
require(source_host->status().migration_frozen &&
destination_host->status().migration_frozen,
"TLS owners failed exact stage reconciliation");
auto& isolated = lost_source ? source_host : destination_host;
auto& healthy = lost_source ? destination_host : source_host;
auto& relay = lost_source ? source_relay : destination_relay;
const auto isolated_key = lost_source ? f.key : destination_key;
auto& replacement = lost_source ? f.lease : destination_lease;
const auto healthy_key = lost_source ? destination_key : f.key;
auto& healthy_lease = lost_source ? destination_lease : f.lease;
relay.drop_requests(true);
const auto expire_until = LeaseClock::now() + 3500ms;
while (LeaseClock::now() < expire_until) {
isolated->step();
healthy->step();
std::this_thread::sleep_for(25ms);
}
require(!isolated->status().ready && !isolated->authority().has_value() &&
!isolated->advance_unattended(64) && relay.dropped_requests() > 0 &&
healthy->status().phase == HostPhase::running &&
healthy->status().epoch == (lost_source ? dl.epoch : sl.epoch) &&
healthy->status().migration_frozen,
"isolated owner retained authority or healthy peer released a frozen ghost");
replacement = take(acquire_region_lease(*f.db, isolated_key, f.build, 60000ms, 5s),
"replacement after actual TLS partition expiry");
require(replacement.epoch > (lost_source ? sl.epoch : dl.epoch),
"replacement reused partitioned epoch");
if (lost_source)
require(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, replacement, 5s),
"replacement source retains freeze")
.has_value(),
"source lease expiry implicitly aborted");
else
require(take(load_cellular_migration_stage(*f.db, destination_objects, destination_key,
replacement, 5s),
"replacement destination retains stage")
.has_value(),
"destination lease expiry activated ghost");
const auto stale =
publish_cellular_migration(*f.db, *f.objects, destination_objects, sl, dl, f.head,
destination_head, frozen.trace, f.build, 5s);
require(!stale && stale.error().store.kind == LeaseStoreError::fenced &&
take(load_region_checkpoint(*f.db, f.key, 5s), "partition source head").head ==
source_before.head &&
take(load_region_checkpoint(*f.db, destination_key, 5s),
"partition destination head")
.head == destination_before.head,
"stale partitioned fence published a migration half");
healthy->step();
const auto current_source = lost_source ? replacement : sl;
if (lost_source) {
test::PostgresRelay abort_relay(migration_backend_port());
auto db = migration_tls_connection(abort_relay);
abort_relay.drop_responses(true);
const auto unknown =
abort_cellular_migration_freeze(*db, f.key, current_source, request.request, 300ms);
require(!unknown && unknown.error().kind == LeaseStoreError::unknown &&
abort_relay.dropped_responses() > 0 && abort_relay.healthy(),
"lost replacement abort receipt was not unknown");
take(abort_cellular_migration_freeze(*f.db, f.key, current_source, request.request, 5s),
"exact replacement abort reconciliation");
} else {
source_relay.drop_responses(true);
const auto unknown = source_host->abort_migration_freeze(request.request);
require(!unknown && !source_host->status().ready &&
source_relay.dropped_responses() > 0,
"lost owning-source abort receipt resumed the region");
source_relay.drop_responses(false);
}
require(!take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, current_source, 5s),
"independent abort source observation")
.has_value(),
"lost abort receipt reopened source stock");
const auto resume_until = LeaseClock::now() + 2s;
while (!healthy->step().ready && LeaseClock::now() < resume_until)
std::this_thread::sleep_for(25ms);
require(healthy->status().ready && healthy->state()->snapshot() ==
(lost_source ? destination_bytes : source_bytes),
"healthy owner resumed different stock after abort");
relay.drop_requests(false);
isolated->step();
require(!isolated->status().ready && !isolated->advance_unattended(64),
"healed stale owner resumed despite a replacement fence");
const auto isolated_manifest =
take(decode_manifest(lost_source ? f.head.manifest : destination_head.manifest),
"isolated head profile");
const auto& objects = lost_source ? *f.objects : destination_objects;
std::vector<std::uint8_t> recovered;
if (isolated_manifest.profile == CheckpointProfile::cellular_population_v1)
recovered = take(recover_population_checkpoint(*f.db, objects, isolated_key,
replacement, builds, 5s),
"replacement population replay")
.state->state()
.snapshot();
else
recovered = take(recover_cellular_checkpoint(*f.db, objects, isolated_key, replacement,
builds, 5s),
"replacement raw replay")
.state->snapshot();
require(recovered == (lost_source ? source_bytes : destination_bytes) &&
source_relay.healthy() && destination_relay.healthy(),
"replacement or relay lost exact stock");
source_host->drain();
destination_host->drain();
healthy_lease = take(acquire_region_lease(*f.db, healthy_key, f.build, 60000ms, 5s),
"return healthy TLS owner authority");
}
std::puts("migration independent TLS owner partitions, expiry, replacement and heal passed");
}
void migration_publication(Fixture& f, const RegionKey destination_key,
FilesystemObjectStore& destination_objects,
LeaseGrant& destination_lease, CommittedCheckpoint& destination_head) {
@@ -920,6 +1179,9 @@ void migration_publication(Fixture& f, const RegionKey destination_key,
take(load_region_checkpoint(*f.db, dk, 5s), "killed pair destination").head ==
destination_before.head,
"pre-transaction paired death changed a head");
if (direction == 0 && std::getenv("EVO_MIGRATION_TCP_PORT"))
migration_transport_commit(f, prepared, sk, dk, sl, dl, source_before.head,
destination_before.head);
if (direction == 0) {
auto locker = connection(true), timed = connection();
take(locker->query("BEGIN", {}, 5s), "pair ambiguity transaction");
@@ -1025,6 +1287,8 @@ void migration_publication(Fixture& f, const RegionKey destination_key,
dl = take(acquire_region_lease(*f.db, dk, f.build, 60000ms, 5s),
"return paired destination authority");
}
migration_owner_partitions(f, destination_key, destination_objects, destination_lease,
destination_head, genome);
}
// Race the actual two-head transaction against abort on independent connections.
const auto prepare_pair = [&](bool reverse) {