M12.4: qualify migration ownership across TLS path loss
Foundation / checks (ecs) (push) Successful in 2m29s
Foundation / checks (deploy) (push) Successful in 4m8s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 18m9s
Foundation / checks (client) (push) Canceled after 18m8s
Foundation / checks (network) (push) Canceled after 18m7s
Foundation / checks (tsan) (push) Canceled after 15m39s
Foundation / checks (postgres) (push) Canceled after 18m13s
Foundation / checks (ecs) (push) Successful in 2m29s
Foundation / checks (deploy) (push) Successful in 4m8s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 18m9s
Foundation / checks (client) (push) Canceled after 18m8s
Foundation / checks (network) (push) Canceled after 18m7s
Foundation / checks (tsan) (push) Canceled after 15m39s
Foundation / checks (postgres) (push) Canceled after 18m13s
This commit is contained in:
Vendored
+175
@@ -0,0 +1,175 @@
|
||||
#include "postgres_relay.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <arpa/inet.h>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <cerrno>
|
||||
#include <fcntl.h>
|
||||
#include <poll.h>
|
||||
#include <stdexcept>
|
||||
#include <sys/socket.h>
|
||||
#include <thread>
|
||||
#include <unistd.h>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace evo::test {
|
||||
namespace {
|
||||
struct Socket {
|
||||
int fd{-1};
|
||||
Socket() = default;
|
||||
explicit Socket(int value) : fd(value) {}
|
||||
~Socket() {
|
||||
if (fd >= 0)
|
||||
::close(fd);
|
||||
}
|
||||
Socket(const Socket&) = delete;
|
||||
Socket& operator=(const Socket&) = delete;
|
||||
Socket(Socket&& other) noexcept : fd(std::exchange(other.fd, -1)) {}
|
||||
Socket& operator=(Socket&& other) noexcept {
|
||||
if (fd >= 0)
|
||||
::close(fd);
|
||||
fd = std::exchange(other.fd, -1);
|
||||
return *this;
|
||||
}
|
||||
};
|
||||
sockaddr_in loopback(std::uint16_t port) {
|
||||
sockaddr_in address{};
|
||||
address.sin_family = AF_INET;
|
||||
address.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
|
||||
address.sin_port = htons(port);
|
||||
return address;
|
||||
}
|
||||
bool nonblocking(int fd) {
|
||||
const int flags = ::fcntl(fd, F_GETFL);
|
||||
return flags >= 0 && ::fcntl(fd, F_SETFL, flags | O_NONBLOCK) == 0;
|
||||
}
|
||||
bool receive(int fd, std::vector<char>& queued, bool drop, std::atomic<std::uint64_t>& discarded) {
|
||||
if (queued.size() >= 65536)
|
||||
return true;
|
||||
std::array<char, 8192> bytes{};
|
||||
const auto count = ::recv(fd, bytes.data(), std::min(bytes.size(), 65536 - queued.size()), 0);
|
||||
if (count < 0)
|
||||
return errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR;
|
||||
if (count == 0)
|
||||
return false;
|
||||
if (drop)
|
||||
discarded.fetch_add(static_cast<std::uint64_t>(count));
|
||||
else
|
||||
queued.insert(queued.end(), bytes.begin(), bytes.begin() + count);
|
||||
return true;
|
||||
}
|
||||
bool transmit(int fd, std::vector<char>& queued) {
|
||||
if (queued.empty())
|
||||
return true;
|
||||
const auto count = ::send(fd, queued.data(), queued.size(), MSG_NOSIGNAL);
|
||||
if (count < 0)
|
||||
return errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR;
|
||||
if (count == 0)
|
||||
return false;
|
||||
queued.erase(queued.begin(), queued.begin() + count);
|
||||
return true;
|
||||
}
|
||||
} // namespace
|
||||
struct PostgresRelay::Impl {
|
||||
Socket listener;
|
||||
std::uint16_t port{}, backend{};
|
||||
std::atomic<bool> requests{}, responses{}, failed{};
|
||||
std::atomic<std::uint64_t> request_bytes{}, response_bytes{};
|
||||
std::jthread worker;
|
||||
explicit Impl(std::uint16_t target)
|
||||
: listener(::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0)), backend(target) {
|
||||
auto address = loopback(0);
|
||||
socklen_t size = sizeof(address);
|
||||
if (target == 0 || listener.fd < 0 ||
|
||||
::bind(listener.fd, reinterpret_cast<const sockaddr*>(&address), sizeof(address)) !=
|
||||
0 ||
|
||||
::listen(listener.fd, 8) != 0 || !nonblocking(listener.fd) ||
|
||||
::getsockname(listener.fd, reinterpret_cast<sockaddr*>(&address), &size) != 0)
|
||||
throw std::runtime_error("private PostgreSQL relay listener");
|
||||
port = ntohs(address.sin_port);
|
||||
worker = std::jthread([this](const std::stop_token& stop) {
|
||||
try {
|
||||
run(stop);
|
||||
} catch (...) {
|
||||
failed = true;
|
||||
}
|
||||
});
|
||||
}
|
||||
void run(const std::stop_token& stop) {
|
||||
Socket client, server;
|
||||
std::vector<char> upstream, downstream;
|
||||
while (!stop.stop_requested()) {
|
||||
if (requests.load()) {
|
||||
request_bytes.fetch_add(upstream.size());
|
||||
upstream.clear();
|
||||
}
|
||||
if (responses.load()) {
|
||||
response_bytes.fetch_add(downstream.size());
|
||||
downstream.clear();
|
||||
}
|
||||
std::array<pollfd, 3> fds{
|
||||
{{listener.fd, static_cast<short>(client.fd < 0 ? POLLIN : 0), 0},
|
||||
{client.fd, static_cast<short>(POLLIN | (downstream.empty() ? 0 : POLLOUT)), 0},
|
||||
{server.fd, static_cast<short>(POLLIN | (upstream.empty() ? 0 : POLLOUT)), 0}}};
|
||||
const auto count = ::poll(fds.data(), fds.size(), 10);
|
||||
if (count < 0) {
|
||||
if (errno == EINTR)
|
||||
continue;
|
||||
failed = true;
|
||||
return;
|
||||
}
|
||||
if (fds[0].revents & POLLIN) {
|
||||
client = Socket(::accept4(listener.fd, nullptr, nullptr, SOCK_CLOEXEC));
|
||||
server = Socket(::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0));
|
||||
const auto target = loopback(backend);
|
||||
if (client.fd < 0 || server.fd < 0 ||
|
||||
::connect(server.fd, reinterpret_cast<const sockaddr*>(&target),
|
||||
sizeof(target)) != 0 ||
|
||||
!nonblocking(client.fd) || !nonblocking(server.fd)) {
|
||||
failed = true;
|
||||
return;
|
||||
}
|
||||
}
|
||||
bool alive = true;
|
||||
if (fds[1].revents & POLLIN)
|
||||
alive = receive(client.fd, upstream, requests.load(), request_bytes);
|
||||
if (alive && (fds[2].revents & POLLIN))
|
||||
alive = receive(server.fd, downstream, responses.load(), response_bytes);
|
||||
if (alive && (fds[1].revents & POLLOUT))
|
||||
alive = transmit(client.fd, downstream);
|
||||
if (alive && (fds[2].revents & POLLOUT))
|
||||
alive = transmit(server.fd, upstream);
|
||||
if (!alive || (fds[1].revents & (POLLERR | POLLHUP | POLLNVAL)) ||
|
||||
(fds[2].revents & (POLLERR | POLLHUP | POLLNVAL))) {
|
||||
client = Socket{};
|
||||
server = Socket{};
|
||||
upstream.clear();
|
||||
downstream.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
PostgresRelay::PostgresRelay(std::uint16_t backend_port)
|
||||
: impl_(std::make_unique<Impl>(backend_port)) {}
|
||||
PostgresRelay::~PostgresRelay() = default;
|
||||
std::string PostgresRelay::port() const {
|
||||
return std::to_string(impl_->port);
|
||||
}
|
||||
void PostgresRelay::drop_requests(bool value) {
|
||||
impl_->requests = value;
|
||||
}
|
||||
void PostgresRelay::drop_responses(bool value) {
|
||||
impl_->responses = value;
|
||||
}
|
||||
std::uint64_t PostgresRelay::dropped_requests() const {
|
||||
return impl_->request_bytes.load();
|
||||
}
|
||||
std::uint64_t PostgresRelay::dropped_responses() const {
|
||||
return impl_->response_bytes.load();
|
||||
}
|
||||
bool PostgresRelay::healthy() const {
|
||||
return !impl_->failed.load();
|
||||
}
|
||||
} // namespace evo::test
|
||||
Vendored
+26
@@ -0,0 +1,26 @@
|
||||
#pragma once
|
||||
|
||||
// Test-only opaque TCP relay. TLS remains end-to-end between libpq and PostgreSQL.
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
namespace evo::test {
|
||||
class PostgresRelay {
|
||||
public:
|
||||
explicit PostgresRelay(std::uint16_t backend_port);
|
||||
~PostgresRelay();
|
||||
PostgresRelay(const PostgresRelay&) = delete;
|
||||
PostgresRelay& operator=(const PostgresRelay&) = delete;
|
||||
[[nodiscard]] std::string port() const;
|
||||
void drop_requests(bool);
|
||||
void drop_responses(bool);
|
||||
[[nodiscard]] std::uint64_t dropped_requests() const;
|
||||
[[nodiscard]] std::uint64_t dropped_responses() const;
|
||||
[[nodiscard]] bool healthy() const;
|
||||
|
||||
private:
|
||||
struct Impl;
|
||||
std::unique_ptr<Impl> impl_;
|
||||
};
|
||||
} // namespace evo::test
|
||||
@@ -4,8 +4,10 @@
|
||||
#include "evo/persistence/sha256.hpp"
|
||||
#include "evo/server/cellular_region_host.hpp"
|
||||
#include "fixtures/cellular_ecology.hpp"
|
||||
#include "fixtures/postgres_relay.hpp"
|
||||
|
||||
#include <barrier>
|
||||
#include <charconv>
|
||||
#include <csignal>
|
||||
#include <cstdlib>
|
||||
#include <filesystem>
|
||||
@@ -96,6 +98,73 @@ struct Fixture {
|
||||
head.head.end_tick + count};
|
||||
}
|
||||
};
|
||||
std::uint16_t migration_backend_port() {
|
||||
const auto port = environment("EVO_MIGRATION_TCP_PORT");
|
||||
unsigned value{};
|
||||
const auto [end, error] = std::from_chars(port.data(), port.data() + port.size(), value);
|
||||
require(error == std::errc{} && end == port.data() + port.size() && value > 0 && value <= 65535,
|
||||
"invalid private migration TLS port");
|
||||
return static_cast<std::uint16_t>(value);
|
||||
}
|
||||
PgOptions migration_tls_options(const test::PostgresRelay& relay) {
|
||||
return {"localhost",
|
||||
"127.0.0.1",
|
||||
relay.port(),
|
||||
environment("PGDATABASE"),
|
||||
environment("EVO_PG_RUNTIME_USER"),
|
||||
{},
|
||||
environment("EVO_MIGRATION_CA")};
|
||||
}
|
||||
std::unique_ptr<Postgres> migration_tls_connection(const test::PostgresRelay& relay) {
|
||||
auto db = take(Postgres::connect(migration_tls_options(relay), 2s),
|
||||
"migration verify-full TLS connection");
|
||||
take(db->query("SELECT 1", {}, 2s), "migration TLS warmup");
|
||||
return db;
|
||||
}
|
||||
void migration_transport_commit(Fixture& f, const CellularMigrationCommitPrepared& prepared,
|
||||
RegionKey sk, RegionKey dk, const LeaseGrant& sl,
|
||||
const LeaseGrant& dl, const RegionHead& source_before,
|
||||
const RegionHead& destination_before) {
|
||||
{
|
||||
test::PostgresRelay relay(migration_backend_port());
|
||||
auto wrong = migration_tls_options(relay);
|
||||
wrong.host = "untrusted.evolution.test";
|
||||
require(!Postgres::connect(wrong, 1s), "relay bypassed TLS hostname verification");
|
||||
}
|
||||
{
|
||||
test::PostgresRelay relay(migration_backend_port());
|
||||
auto db = migration_tls_connection(relay);
|
||||
relay.drop_requests(true);
|
||||
const auto lost = retry_cellular_migration_commit(*db, sl, dl, prepared, 300ms);
|
||||
require(
|
||||
!lost && lost.error().kind == PublicationError::unknown &&
|
||||
lost.error().prepared.source_manifest == prepared.source_manifest &&
|
||||
lost.error().prepared.destination_manifest == prepared.destination_manifest &&
|
||||
relay.dropped_requests() > 0 && relay.healthy() &&
|
||||
take(load_region_checkpoint(*f.db, sk, 5s), "lost request source head").head ==
|
||||
source_before &&
|
||||
take(load_region_checkpoint(*f.db, dk, 5s), "lost request destination head").head ==
|
||||
destination_before,
|
||||
"a dropped TLS request changed ownership or lost the exact pair");
|
||||
}
|
||||
{
|
||||
test::PostgresRelay relay(migration_backend_port());
|
||||
auto db = migration_tls_connection(relay);
|
||||
relay.drop_responses(true);
|
||||
const auto lost = retry_cellular_migration_commit(*db, sl, dl, prepared, 500ms);
|
||||
require(
|
||||
!lost && lost.error().kind == PublicationError::unknown &&
|
||||
lost.error().prepared.source_manifest == prepared.source_manifest &&
|
||||
lost.error().prepared.destination_manifest == prepared.destination_manifest &&
|
||||
relay.dropped_responses() > 0 && relay.healthy() &&
|
||||
take(load_region_checkpoint(*f.db, sk, 5s), "lost reply source head").manifest ==
|
||||
prepared.source_manifest &&
|
||||
take(load_region_checkpoint(*f.db, dk, 5s), "lost reply destination head")
|
||||
.manifest == prepared.destination_manifest,
|
||||
"lost TLS receipt was treated as rollback or committed only one head");
|
||||
}
|
||||
std::puts("migration TLS request/reply loss and hostname verification passed");
|
||||
}
|
||||
void funded_source(Fixture& f) {
|
||||
f.build.fill(0xDA);
|
||||
const std::array<std::string, 1> build{hexadecimal(f.build)};
|
||||
@@ -746,6 +815,196 @@ void boundary_upgrades(Fixture& f) {
|
||||
require(f.raw->format_version() == cellular_lives_migration_format,
|
||||
"reverse transition downgraded accounting owner");
|
||||
}
|
||||
void migration_owner_partitions(Fixture& f, RegionKey destination_key,
|
||||
FilesystemObjectStore& destination_objects,
|
||||
LeaseGrant& destination_lease,
|
||||
CommittedCheckpoint& destination_head, std::uint64_t genome) {
|
||||
if (!std::getenv("EVO_MIGRATION_TCP_PORT"))
|
||||
return;
|
||||
const std::array<Sha256Digest, 1> builds{f.build};
|
||||
for (unsigned lost_source = 0; lost_source < 2; ++lost_source) {
|
||||
test::PostgresRelay source_relay(migration_backend_port()),
|
||||
destination_relay(migration_backend_port());
|
||||
const auto make_host = [&](RegionKey key, const test::PostgresRelay& relay) {
|
||||
auto objects = take(FilesystemObjectStore::open(environment("PGHOST"), key,
|
||||
max_cellular_population_bytes),
|
||||
"partition owner objects");
|
||||
CellularHostConfig config{key, 1, {}, f.build, {f.build}};
|
||||
config.lease = {3000ms, 400ms, 300ms, 150ms, 250ms};
|
||||
const auto options = migration_tls_options(relay);
|
||||
return take(CellularRegionHost::create(
|
||||
config, [options] { return Postgres::connect(options, 500ms); },
|
||||
std::move(objects)),
|
||||
"partition owner host");
|
||||
};
|
||||
take(release_region_lease(*f.db, f.key, f.lease, 5s), "release source to TLS owner");
|
||||
take(release_region_lease(*f.db, destination_key, destination_lease, 5s),
|
||||
"release destination to TLS owner");
|
||||
auto source_host = make_host(f.key, source_relay),
|
||||
destination_host = make_host(destination_key, destination_relay);
|
||||
require(source_host->step().ready && destination_host->step().ready,
|
||||
"TLS owners before freeze");
|
||||
const auto sl = take(source_host->authority(), "TLS source fence"),
|
||||
dl = take(destination_host->authority(), "TLS destination fence");
|
||||
const auto source_bytes = source_host->state()->snapshot(),
|
||||
destination_bytes = destination_host->state()->snapshot();
|
||||
const auto source_before = f.head, destination_before = destination_head;
|
||||
const CellularMigrationRequest request{
|
||||
f.id(),
|
||||
{f.key.region, f.head.head.commit, f.id(), f.head.head.digest},
|
||||
{destination_key.region, destination_head.head.commit, f.id(),
|
||||
destination_head.head.digest},
|
||||
genome};
|
||||
source_relay.drop_responses(true);
|
||||
const auto unknown_prepare =
|
||||
source_host->prepare_migration_freeze(destination_head, destination_objects, request);
|
||||
require(!unknown_prepare &&
|
||||
unknown_prepare.error().publication == PublicationError::unknown &&
|
||||
!source_host->status().ready && source_relay.dropped_responses() > 0,
|
||||
"lost preparation receipt did not pause source ownership");
|
||||
const auto frozen =
|
||||
take(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, sl, 5s),
|
||||
"independent committed preparation observation"),
|
||||
"missing preparation after TLS reply loss");
|
||||
source_relay.drop_responses(false);
|
||||
const auto reconcile_source_until = LeaseClock::now() + 2s;
|
||||
while (!source_host->status().migration_frozen &&
|
||||
LeaseClock::now() < reconcile_source_until) {
|
||||
source_host->step();
|
||||
destination_host->step();
|
||||
std::this_thread::sleep_for(25ms);
|
||||
}
|
||||
require(source_host->status().migration_frozen && !source_host->status().ready,
|
||||
"source lost the exact prepared request after TLS reconnect");
|
||||
destination_relay.drop_responses(true);
|
||||
const auto unknown_stage =
|
||||
destination_host->stage_migration(f.head, *f.objects, frozen.trace);
|
||||
require(!unknown_stage && unknown_stage.error().publication == PublicationError::unknown &&
|
||||
!destination_host->status().ready &&
|
||||
destination_relay.dropped_responses() > 0 &&
|
||||
take(load_cellular_migration_stage(*f.db, destination_objects, destination_key,
|
||||
dl, 5s),
|
||||
"independent committed stage observation")
|
||||
.has_value(),
|
||||
"lost stage receipt activated the ghost or lost retained work");
|
||||
destination_relay.drop_responses(false);
|
||||
const auto reconcile_destination_until = LeaseClock::now() + 2s;
|
||||
while (!destination_host->status().migration_frozen &&
|
||||
LeaseClock::now() < reconcile_destination_until) {
|
||||
source_host->step();
|
||||
destination_host->step();
|
||||
std::this_thread::sleep_for(25ms);
|
||||
}
|
||||
require(source_host->status().migration_frozen &&
|
||||
destination_host->status().migration_frozen,
|
||||
"TLS owners failed exact stage reconciliation");
|
||||
auto& isolated = lost_source ? source_host : destination_host;
|
||||
auto& healthy = lost_source ? destination_host : source_host;
|
||||
auto& relay = lost_source ? source_relay : destination_relay;
|
||||
const auto isolated_key = lost_source ? f.key : destination_key;
|
||||
auto& replacement = lost_source ? f.lease : destination_lease;
|
||||
const auto healthy_key = lost_source ? destination_key : f.key;
|
||||
auto& healthy_lease = lost_source ? destination_lease : f.lease;
|
||||
relay.drop_requests(true);
|
||||
const auto expire_until = LeaseClock::now() + 3500ms;
|
||||
while (LeaseClock::now() < expire_until) {
|
||||
isolated->step();
|
||||
healthy->step();
|
||||
std::this_thread::sleep_for(25ms);
|
||||
}
|
||||
require(!isolated->status().ready && !isolated->authority().has_value() &&
|
||||
!isolated->advance_unattended(64) && relay.dropped_requests() > 0 &&
|
||||
healthy->status().phase == HostPhase::running &&
|
||||
healthy->status().epoch == (lost_source ? dl.epoch : sl.epoch) &&
|
||||
healthy->status().migration_frozen,
|
||||
"isolated owner retained authority or healthy peer released a frozen ghost");
|
||||
replacement = take(acquire_region_lease(*f.db, isolated_key, f.build, 60000ms, 5s),
|
||||
"replacement after actual TLS partition expiry");
|
||||
require(replacement.epoch > (lost_source ? sl.epoch : dl.epoch),
|
||||
"replacement reused partitioned epoch");
|
||||
if (lost_source)
|
||||
require(take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, replacement, 5s),
|
||||
"replacement source retains freeze")
|
||||
.has_value(),
|
||||
"source lease expiry implicitly aborted");
|
||||
else
|
||||
require(take(load_cellular_migration_stage(*f.db, destination_objects, destination_key,
|
||||
replacement, 5s),
|
||||
"replacement destination retains stage")
|
||||
.has_value(),
|
||||
"destination lease expiry activated ghost");
|
||||
const auto stale =
|
||||
publish_cellular_migration(*f.db, *f.objects, destination_objects, sl, dl, f.head,
|
||||
destination_head, frozen.trace, f.build, 5s);
|
||||
require(!stale && stale.error().store.kind == LeaseStoreError::fenced &&
|
||||
take(load_region_checkpoint(*f.db, f.key, 5s), "partition source head").head ==
|
||||
source_before.head &&
|
||||
take(load_region_checkpoint(*f.db, destination_key, 5s),
|
||||
"partition destination head")
|
||||
.head == destination_before.head,
|
||||
"stale partitioned fence published a migration half");
|
||||
healthy->step();
|
||||
const auto current_source = lost_source ? replacement : sl;
|
||||
if (lost_source) {
|
||||
test::PostgresRelay abort_relay(migration_backend_port());
|
||||
auto db = migration_tls_connection(abort_relay);
|
||||
abort_relay.drop_responses(true);
|
||||
const auto unknown =
|
||||
abort_cellular_migration_freeze(*db, f.key, current_source, request.request, 300ms);
|
||||
require(!unknown && unknown.error().kind == LeaseStoreError::unknown &&
|
||||
abort_relay.dropped_responses() > 0 && abort_relay.healthy(),
|
||||
"lost replacement abort receipt was not unknown");
|
||||
take(abort_cellular_migration_freeze(*f.db, f.key, current_source, request.request, 5s),
|
||||
"exact replacement abort reconciliation");
|
||||
} else {
|
||||
source_relay.drop_responses(true);
|
||||
const auto unknown = source_host->abort_migration_freeze(request.request);
|
||||
require(!unknown && !source_host->status().ready &&
|
||||
source_relay.dropped_responses() > 0,
|
||||
"lost owning-source abort receipt resumed the region");
|
||||
source_relay.drop_responses(false);
|
||||
}
|
||||
require(!take(load_cellular_migration_freeze(*f.db, *f.objects, f.key, current_source, 5s),
|
||||
"independent abort source observation")
|
||||
.has_value(),
|
||||
"lost abort receipt reopened source stock");
|
||||
const auto resume_until = LeaseClock::now() + 2s;
|
||||
while (!healthy->step().ready && LeaseClock::now() < resume_until)
|
||||
std::this_thread::sleep_for(25ms);
|
||||
require(healthy->status().ready && healthy->state()->snapshot() ==
|
||||
(lost_source ? destination_bytes : source_bytes),
|
||||
"healthy owner resumed different stock after abort");
|
||||
relay.drop_requests(false);
|
||||
isolated->step();
|
||||
require(!isolated->status().ready && !isolated->advance_unattended(64),
|
||||
"healed stale owner resumed despite a replacement fence");
|
||||
const auto isolated_manifest =
|
||||
take(decode_manifest(lost_source ? f.head.manifest : destination_head.manifest),
|
||||
"isolated head profile");
|
||||
const auto& objects = lost_source ? *f.objects : destination_objects;
|
||||
std::vector<std::uint8_t> recovered;
|
||||
if (isolated_manifest.profile == CheckpointProfile::cellular_population_v1)
|
||||
recovered = take(recover_population_checkpoint(*f.db, objects, isolated_key,
|
||||
replacement, builds, 5s),
|
||||
"replacement population replay")
|
||||
.state->state()
|
||||
.snapshot();
|
||||
else
|
||||
recovered = take(recover_cellular_checkpoint(*f.db, objects, isolated_key, replacement,
|
||||
builds, 5s),
|
||||
"replacement raw replay")
|
||||
.state->snapshot();
|
||||
require(recovered == (lost_source ? source_bytes : destination_bytes) &&
|
||||
source_relay.healthy() && destination_relay.healthy(),
|
||||
"replacement or relay lost exact stock");
|
||||
source_host->drain();
|
||||
destination_host->drain();
|
||||
healthy_lease = take(acquire_region_lease(*f.db, healthy_key, f.build, 60000ms, 5s),
|
||||
"return healthy TLS owner authority");
|
||||
}
|
||||
std::puts("migration independent TLS owner partitions, expiry, replacement and heal passed");
|
||||
}
|
||||
|
||||
void migration_publication(Fixture& f, const RegionKey destination_key,
|
||||
FilesystemObjectStore& destination_objects,
|
||||
LeaseGrant& destination_lease, CommittedCheckpoint& destination_head) {
|
||||
@@ -920,6 +1179,9 @@ void migration_publication(Fixture& f, const RegionKey destination_key,
|
||||
take(load_region_checkpoint(*f.db, dk, 5s), "killed pair destination").head ==
|
||||
destination_before.head,
|
||||
"pre-transaction paired death changed a head");
|
||||
if (direction == 0 && std::getenv("EVO_MIGRATION_TCP_PORT"))
|
||||
migration_transport_commit(f, prepared, sk, dk, sl, dl, source_before.head,
|
||||
destination_before.head);
|
||||
if (direction == 0) {
|
||||
auto locker = connection(true), timed = connection();
|
||||
take(locker->query("BEGIN", {}, 5s), "pair ambiguity transaction");
|
||||
@@ -1025,6 +1287,8 @@ void migration_publication(Fixture& f, const RegionKey destination_key,
|
||||
dl = take(acquire_region_lease(*f.db, dk, f.build, 60000ms, 5s),
|
||||
"return paired destination authority");
|
||||
}
|
||||
migration_owner_partitions(f, destination_key, destination_objects, destination_lease,
|
||||
destination_head, genome);
|
||||
}
|
||||
// Race the actual two-head transaction against abort on independent connections.
|
||||
const auto prepare_pair = [&](bool reverse) {
|
||||
|
||||
Reference in New Issue
Block a user