Foundation / checks (deploy) (push) Successful in 3m20s
Foundation / checks (ecs) (push) Successful in 1m57s
Foundation / checks (client) (push) Failing after 10m4s
Foundation / checks (network) (push) Successful in 13m3s
Foundation / checks (tsan) (push) Successful in 7m53s
Foundation / checks (checks) (push) Successful in 32m18s
Foundation / checks (postgres) (push) Successful in 43m26s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
ADR-0096. Two real evo-server processes on PostgreSQL (regions 7/161 and 7/162, adjacent areas) run crossings and crossings back, SIGKILL of the source after prepare and of the destination after stage and after commit, duplicate and stale ticket retries, a client that never arrives and a destination that is down. After every scenario SQL checks one owner per probe, the conserved inventory and frozen-iff-open-transfer. The matrix found two recovery gaps, now fixed: - a source restarted after a prepare left the probe frozen for good (its abort deadline was in memory): schema v14 overdue_probe_transfers and a session-layer sweep abort open outgoing transfers past the ticket lifetime plus 10 s by the database clock; - a ticket replayed at a restarted destination after the probe moved on took control back: admission of a committed transfer now requires the record as that commit left it. M10.8 is complete. Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
520 lines
23 KiB
C++
520 lines
23 KiB
C++
// Real process tests for M10.8d (ADR-0096): two evo-server processes on PostgreSQL with
|
|
// adjacent marker areas, handoff keys and each other as neighbour; in-process
|
|
// RegionSessionClients follow offers and tickets. Regions 7/161 (A, cells 0..1) and 7/162
|
|
// (B, cells 2..3). Covers a crossing and a crossing back, SIGKILL of the source after
|
|
// prepare and of the destination after stage and after commit, duplicate and stale ticket
|
|
// retries, a client that never arrives (aborted by the restarted source's recovery sweep)
|
|
// and a destination that is down (aborted by the live source). After every scenario: one
|
|
// owning region per probe, the test inventory conserved, and a record frozen exactly while
|
|
// its one open transfer exists. Run by tools/ci/postgres-check.sh when the build has
|
|
// EVO_BUILD_QUIC.
|
|
#include "evo/client/region_session.hpp"
|
|
#include "evo/network/dev_credentials.hpp"
|
|
#include "evo/network/handoff_ticket.hpp"
|
|
#include "evo/network/session_grant.hpp"
|
|
#include "fixtures/server_process.hpp"
|
|
#include <algorithm>
|
|
#include <arpa/inet.h>
|
|
#include <array>
|
|
#include <chrono>
|
|
#include <fcntl.h>
|
|
#include <functional>
|
|
#include <netinet/in.h>
|
|
#include <optional>
|
|
#include <source_location>
|
|
#include <string>
|
|
#include <sys/socket.h>
|
|
#include <unistd.h>
|
|
#include <vector>
|
|
|
|
void run_region_handoff_process_tests();
|
|
|
|
namespace {
|
|
using namespace evo::test::process;
|
|
using namespace evo;
|
|
using namespace evo::network;
|
|
using client::RegionSessionClient;
|
|
using client::RegionSessionState;
|
|
using evo::test::region_identity;
|
|
void check(bool value, std::source_location at = std::source_location::current()) {
|
|
if (!value)
|
|
throw std::runtime_error("region handoff process test line " + std::to_string(at.line()));
|
|
}
|
|
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
|
template <typename T>
|
|
const T& must(const std::optional<T>& o,
|
|
std::source_location at = std::source_location::current()) {
|
|
if (!o)
|
|
throw std::runtime_error("region handoff process missing value line " +
|
|
std::to_string(at.line()));
|
|
return *o;
|
|
}
|
|
constexpr const char* name = "dev-region.evolution.test";
|
|
constexpr std::uint64_t region_a = 161, region_b = 162;
|
|
|
|
void write_file(const fs::path& path, std::span<const std::uint8_t> bytes) {
|
|
const int fd = ::open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0600);
|
|
expect(fd >= 0 &&
|
|
::write(fd, bytes.data(), bytes.size()) == static_cast<ssize_t>(bytes.size()) &&
|
|
::close(fd) == 0,
|
|
"write file");
|
|
}
|
|
void write_text(const fs::path& path, const std::string& text) {
|
|
write_file(path, {reinterpret_cast<const std::uint8_t*>(text.data()), text.size()});
|
|
}
|
|
AccountId account(std::uint8_t tag) {
|
|
AccountId a;
|
|
a.bytes.fill(tag);
|
|
return a;
|
|
}
|
|
template <std::size_t N> std::string hex(const std::array<std::uint8_t, N>& bytes) {
|
|
constexpr char digits[] = "0123456789abcdef";
|
|
std::string out;
|
|
for (const auto b : bytes) {
|
|
out.push_back(digits[b >> 4U]);
|
|
out.push_back(digits[b & 0xFU]);
|
|
}
|
|
return out;
|
|
}
|
|
std::uint64_t wall_seconds() {
|
|
return static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(
|
|
std::chrono::system_clock::now().time_since_epoch())
|
|
.count());
|
|
}
|
|
// A UDP port that was free a moment ago (each server must know its neighbour's port).
|
|
std::uint16_t free_port() {
|
|
const int fd = ::socket(AF_INET, SOCK_DGRAM | SOCK_CLOEXEC, 0);
|
|
expect(fd >= 0, "socket");
|
|
sockaddr_in a{};
|
|
a.sin_family = AF_INET;
|
|
a.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
|
|
socklen_t size = sizeof a;
|
|
const bool ok = ::bind(fd, reinterpret_cast<const sockaddr*>(&a), sizeof a) == 0 &&
|
|
::getsockname(fd, reinterpret_cast<sockaddr*>(&a), &size) == 0;
|
|
::close(fd);
|
|
expect(ok, "free port");
|
|
return ntohs(a.sin_port);
|
|
}
|
|
// A 32-byte scalar that imports as a key, written with its public key.
|
|
template <typename Key, std::size_t N>
|
|
void make_key(const fs::path& scalar_file, const fs::path& public_file,
|
|
const std::function<std::optional<Key>(std::span<const std::uint8_t>)>& import) {
|
|
for (int attempt = 0; attempt < 8; ++attempt) {
|
|
std::array<std::uint8_t, N> scalar{};
|
|
expect(random_bytes(scalar).has_value(), "random");
|
|
if (auto key = import(scalar)) {
|
|
write_file(scalar_file, scalar);
|
|
write_file(public_file, *key->public_key());
|
|
return;
|
|
}
|
|
}
|
|
expect(false, "key");
|
|
}
|
|
|
|
// A genesis commit byte for an all-equal commit id (ids are unique across the suite): one
|
|
// no other test names (later tests provision theirs by literal) and none earlier produced.
|
|
std::uint8_t unused_genesis() {
|
|
for (const unsigned b : {0x4BU, 0x4CU, 0x4DU, 0x8BU, 0x8CU, 0x8DU, 0x8EU}) {
|
|
std::string hex_id;
|
|
for (int i = 0; i < 16; ++i) {
|
|
hex_id.push_back("0123456789abcdef"[b >> 4U]);
|
|
hex_id.push_back("0123456789abcdef"[b & 0xFU]);
|
|
}
|
|
const std::array<std::string, 1> p{hex_id};
|
|
const auto r = owner()->query(
|
|
"SELECT count(*) FROM evolution.region_commits WHERE commit_id = $1::uuid", p, 5s);
|
|
expect(r.has_value() && r->size() == 1, "genesis lookup");
|
|
if ((*r)[0][0] == "0")
|
|
return static_cast<std::uint8_t>(b);
|
|
}
|
|
expect(false, "genesis byte");
|
|
return 0;
|
|
}
|
|
|
|
// Both regions: one CA and server certificate, one grant issuer, a handoff key each.
|
|
struct Pair {
|
|
ServerRegion a{provision(region_a, unused_genesis())}, b{provision(region_b, unused_genesis())};
|
|
fs::path files{a.root / "pair"};
|
|
std::string ca_pem, window;
|
|
std::optional<GrantSigningKey> issuer;
|
|
std::uint16_t port_a{free_port()}, port_b{free_port()};
|
|
Pair() {
|
|
fs::create_directory(files);
|
|
const auto made = generate_dev_credentials(DeploymentEnvironment::dev, name, 1,
|
|
std::chrono::system_clock::now());
|
|
expect(made.has_value(), "credentials");
|
|
ca_pem = made->authority_pem;
|
|
write_text(files / "ca.pem", made->authority_pem);
|
|
write_text(files / "server.pem", made->server_certificate_pem);
|
|
write_text(files / "server.key", made->server_key_pem);
|
|
make_key<GrantSigningKey, grant_private_key_bytes>(
|
|
files / "issuer.key", files / "issuer.pub", [](std::span<const std::uint8_t> s) {
|
|
auto k = GrantSigningKey::import(DeploymentEnvironment::dev, 1, s);
|
|
return k ? std::optional{std::move(*k)} : std::nullopt;
|
|
});
|
|
std::array<std::uint8_t, grant_private_key_bytes> scalar{};
|
|
const int fd = ::open((files / "issuer.key").c_str(), O_RDONLY | O_CLOEXEC);
|
|
expect(fd >= 0 &&
|
|
::read(fd, scalar.data(), scalar.size()) ==
|
|
static_cast<ssize_t>(scalar.size()) &&
|
|
::close(fd) == 0,
|
|
"issuer key");
|
|
auto key = GrantSigningKey::import(DeploymentEnvironment::dev, 1, scalar);
|
|
expect(key.has_value(), "issuer import");
|
|
issuer.emplace(std::move(*key));
|
|
for (const auto r : {region_a, region_b})
|
|
make_key<HandoffSigningKey, grant_private_key_bytes>(
|
|
files / ("handoff-" + std::to_string(r) + ".key"),
|
|
files / ("handoff-" + std::to_string(r) + ".pub"),
|
|
[r](std::span<const std::uint8_t> s) {
|
|
auto k = HandoffSigningKey::import(DeploymentEnvironment::dev,
|
|
static_cast<std::uint32_t>(r), s);
|
|
return k ? std::optional{std::move(*k)} : std::nullopt;
|
|
});
|
|
const auto now = wall_seconds();
|
|
window = "test-" + std::to_string(now) + ":" + std::to_string(now + 1200);
|
|
}
|
|
[[nodiscard]] std::vector<std::string> args(bool at_a) const {
|
|
const auto& own = at_a ? a : b;
|
|
const auto other = at_a ? region_b : region_a;
|
|
const std::string other_cells = at_a ? "2,0,3,1" : "0,0,1,1";
|
|
return server_args(
|
|
own, {"--synthetic-inputs",
|
|
"development-v1",
|
|
"--m10-listen",
|
|
"127.0.0.1:" + std::to_string(at_a ? port_a : port_b),
|
|
"--m10-cert",
|
|
(files / "server.pem").string(),
|
|
"--m10-key",
|
|
(files / "server.key").string(),
|
|
"--m10-issuer",
|
|
(files / "issuer.pub").string(),
|
|
"--m10-environment",
|
|
"dev",
|
|
"--session-window",
|
|
window,
|
|
"--m10-area",
|
|
at_a ? "0,0,1,1" : "2,0,3,1",
|
|
"--m10-handoff-key",
|
|
(files / ("handoff-" + std::to_string(own.key.region) + ".key")).string(),
|
|
"--m10-neighbour",
|
|
std::to_string(other) + "=" + other_cells +
|
|
"=127.0.0.1:" + std::to_string(at_a ? port_b : port_a) + "=" + name + "=" +
|
|
(files / ("handoff-" + std::to_string(other) + ".pub")).string()});
|
|
}
|
|
[[nodiscard]] WireGrant grant(const AccountId& who, std::uint64_t serial,
|
|
std::uint64_t region) const {
|
|
GrantClaims c;
|
|
c.grant_id = (region * 100'000) + serial;
|
|
c.account = who;
|
|
c.serial = serial;
|
|
c.world = {7, region_identity.frame};
|
|
c.region = region;
|
|
c.issued_at = wall_seconds();
|
|
c.expires_at = c.issued_at + 120;
|
|
const auto g = must(issuer).sign(c);
|
|
expect(g.has_value(), "grant");
|
|
return *g;
|
|
}
|
|
};
|
|
|
|
// A running evo-server, its output drained while waiting.
|
|
struct Server {
|
|
std::unique_ptr<Child> child;
|
|
explicit Server(const Pair& p, bool at_a) : child{std::make_unique<Child>(p.args(at_a))} {
|
|
const bool started =
|
|
child->wait_line("m10 listening ", 30s) && child->wait_line("ready 1", 30s) &&
|
|
child->wait_line("m10 motion area ", 30s) && child->printed("m10 handoff neighbours=1");
|
|
if (!started)
|
|
for (const auto& line : child->lines())
|
|
if (starts(line, "m10") || starts(line, "error") || starts(line, "usage"))
|
|
std::printf("server: %s\n", line.c_str());
|
|
CHECK(started);
|
|
}
|
|
void kill() {
|
|
child->signal(SIGKILL);
|
|
CHECK(child->wait_exit(30s) == 128 + SIGKILL);
|
|
}
|
|
};
|
|
|
|
struct Bench {
|
|
Pair pair;
|
|
std::unique_ptr<Server> a, b;
|
|
std::vector<AccountId> accounts; // Every probe this test created.
|
|
std::array<std::uint64_t, 256> serials{};
|
|
|
|
void pump() {
|
|
for (auto* s : {a.get(), b.get()})
|
|
if (s)
|
|
s->child->pump(1ms);
|
|
}
|
|
// Polls clients and live servers until `done`, at most `limit`.
|
|
bool until(const std::vector<RegionSessionClient*>& clients, const std::function<bool()>& done,
|
|
std::chrono::milliseconds limit = 20s) {
|
|
const auto end = Clock::now() + limit;
|
|
while (Clock::now() < end) {
|
|
for (auto* c : clients)
|
|
c->poll(Clock::now());
|
|
pump();
|
|
if (done())
|
|
return true;
|
|
}
|
|
return done();
|
|
}
|
|
std::unique_ptr<RegionSessionClient> dial(std::uint16_t port, const WireGrant& grant) {
|
|
client::RegionSessionClientConfig c;
|
|
c.transport.port = port;
|
|
c.transport.server_name = name;
|
|
c.transport.authority_pem = pair.ca_pem;
|
|
c.transport.authority_pem_file = (pair.files / "ca.pem").string();
|
|
c.grant = grant;
|
|
auto made = RegionSessionClient::connect(std::move(c), Clock::now());
|
|
CHECK(made.has_value());
|
|
return std::move(*made);
|
|
}
|
|
// A grant login (next serial for the account) at a region; waits for the outcome.
|
|
std::unique_ptr<RegionSessionClient> login(const AccountId& who, std::uint64_t region) {
|
|
if (std::ranges::find(accounts, who) == accounts.end())
|
|
accounts.push_back(who);
|
|
auto c = dial(region == region_a ? pair.port_a : pair.port_b,
|
|
pair.grant(who, ++serials.at(who.bytes[0]), region));
|
|
settle(*c);
|
|
return c;
|
|
}
|
|
// A ticket login at the offer's destination; waits for the outcome.
|
|
std::unique_ptr<RegionSessionClient> follow(const HandoffOffer& offer) {
|
|
const auto& to = offer.destination;
|
|
CHECK(to.server_name == name);
|
|
const auto port = to.region == region_a ? pair.port_a : pair.port_b;
|
|
CHECK(to.endpoint == "127.0.0.1:" + std::to_string(port));
|
|
auto c = dial(port, offer.ticket);
|
|
settle(*c);
|
|
return c;
|
|
}
|
|
void settle(RegionSessionClient& c) {
|
|
CHECK(until({&c}, [&] {
|
|
return c.state() == RegionSessionState::established ||
|
|
c.state() == RegionSessionState::closed;
|
|
}));
|
|
}
|
|
// Walks the client's probe east (+1) or west (-1) until it is handed off; the offer.
|
|
HandoffOffer cross(RegionSessionClient& c, int direction) {
|
|
CHECK(c.state() == RegionSessionState::established);
|
|
bool moving = false;
|
|
std::optional<HandoffOffer> offer;
|
|
CHECK(until({&c}, [&] {
|
|
for (const auto& r : c.take_intention_receipts())
|
|
moving = moving || r.status == AdmitStatus::admitted;
|
|
if (!moving && c.state() == RegionSessionState::established)
|
|
static_cast<void>(c.send_intention(c.server_tick() + 4,
|
|
static_cast<std::int16_t>(direction * 1000), 0));
|
|
if (!offer)
|
|
offer = c.take_handoff();
|
|
return offer.has_value() && c.state() == RegionSessionState::closed;
|
|
}));
|
|
CHECK(c.close_reason() == SessionCloseReason::handed_off);
|
|
return must(offer);
|
|
}
|
|
|
|
// SQL views of world 7.
|
|
static persistence::PgRows rows(const std::string& sql,
|
|
const std::vector<std::string>& params) {
|
|
auto r = owner()->query(sql, params, 5s);
|
|
CHECK(r.has_value());
|
|
return *r;
|
|
}
|
|
static std::string one(const std::string& sql, const std::vector<std::string>& params) {
|
|
const auto r = rows(sql, params);
|
|
CHECK(r.size() == 1 && r[0].size() == 1);
|
|
return r[0][0].value_or("");
|
|
}
|
|
// "owner:generation:frozen" of the account's probe.
|
|
static std::string probe(const AccountId& who) {
|
|
return one("SELECT region_id || ':' || generation || ':' || (transfer_id IS NOT NULL) "
|
|
"FROM evolution.probe_records WHERE world_id = 7 AND account_id = $1::uuid",
|
|
{hex(who.bytes)});
|
|
}
|
|
static std::string transfer(const TransferId& t) {
|
|
return one("SELECT state FROM evolution.probe_transfers WHERE world_id = 7 AND "
|
|
"transfer_id = $1::uuid",
|
|
{hex(t)});
|
|
}
|
|
static std::uint64_t control_generation(const AccountId& who) {
|
|
return std::stoull(one("SELECT generation FROM evolution.account_controls WHERE "
|
|
"world_id = 7 AND account_id = $1::uuid",
|
|
{hex(who.bytes)}));
|
|
}
|
|
// After every scenario: a single owning region (A or B) per probe, the inventory
|
|
// conserved, at most one open transfer per probe and the record frozen exactly under it.
|
|
void invariants() const {
|
|
std::string list;
|
|
for (const auto& who : accounts)
|
|
list += (list.empty() ? "'" : ",'") + hex(who.bytes) + "'::uuid";
|
|
const auto where = "r.world_id = 7 AND r.account_id IN (" + list + ")";
|
|
CHECK(one("SELECT count(*) || ':' || sum(r.inventory) FROM evolution.probe_records r "
|
|
"WHERE " +
|
|
where + " AND r.region_id IN (161, 162)",
|
|
{}) ==
|
|
std::to_string(accounts.size()) + ":" + std::to_string(accounts.size() * 1000));
|
|
CHECK(one("SELECT count(*) FROM evolution.probe_records r WHERE " + where +
|
|
" AND r.transfer_id IS DISTINCT FROM (SELECT t.transfer_id FROM "
|
|
"evolution.probe_transfers t WHERE t.world_id = 7 AND t.account_id = "
|
|
"r.account_id AND t.state IN ('prepared', 'staged'))",
|
|
{}) == "0");
|
|
CHECK(one("SELECT count(*) FROM evolution.probe_transfers t WHERE t.world_id = 7 AND "
|
|
"t.account_id IN (" +
|
|
list + ") AND t.inventory <> 1000",
|
|
{}) == "0");
|
|
}
|
|
// The region holding the account's durable control.
|
|
static std::string control_holder(const AccountId& who) {
|
|
return one("SELECT coalesce(region_id::text, '') FROM evolution.account_controls "
|
|
"WHERE world_id = 7 AND account_id = $1::uuid",
|
|
{hex(who.bytes)});
|
|
}
|
|
void kill(bool at_a) {
|
|
auto& s = at_a ? a : b;
|
|
s->kill();
|
|
s.reset();
|
|
}
|
|
void restart(bool at_a) {
|
|
const auto started = Clock::now();
|
|
(at_a ? a : b) = std::make_unique<Server>(pair, at_a); // Waits out the old lease.
|
|
std::printf("handoff process restart %c %lld ms\n", at_a ? 'A' : 'B',
|
|
static_cast<long long>(std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
Clock::now() - started)
|
|
.count()));
|
|
}
|
|
};
|
|
|
|
void handoffs() {
|
|
Bench w;
|
|
w.restart(true);
|
|
w.restart(false);
|
|
|
|
// 1. A crossing and a crossing back (generation 1 -> 2 -> 3), then a stale grant login
|
|
// at the old region is redirected.
|
|
const auto alice = account(0xA1);
|
|
auto a1 = w.login(alice, region_a);
|
|
CHECK(a1->state() == RegionSessionState::established && w.probe(alice) == "161:1:false");
|
|
const auto east = w.cross(*a1, 1);
|
|
CHECK(east.destination.region == region_b);
|
|
auto a2 = w.follow(east);
|
|
CHECK(a2->state() == RegionSessionState::established && w.probe(alice) == "162:2:false" &&
|
|
w.transfer(east.transfer) == "committed");
|
|
const auto west = w.cross(*a2, -1);
|
|
CHECK(west.destination.region == region_a);
|
|
auto a3 = w.follow(west);
|
|
CHECK(a3->state() == RegionSessionState::established && w.probe(alice) == "161:3:false");
|
|
auto stale = w.login(alice, region_b);
|
|
CHECK(stale->close_reason() == SessionCloseReason::moved && stale->redirect() &&
|
|
must(stale->redirect()).region == region_a);
|
|
w.invariants();
|
|
|
|
// 2. Destination SIGKILL after stage: the restarted destination stages again under its
|
|
// new epoch and commits when the ticket arrives. Then the probe moves on, and a replay of
|
|
// that ticket at a destination process that never saw it is refused durably.
|
|
const auto bob = account(0xB2);
|
|
auto b1 = w.login(bob, region_a);
|
|
const auto b_east = w.cross(*b1, 1);
|
|
const auto ticket_issued = wall_seconds();
|
|
CHECK(w.transfer(b_east.transfer) == "prepared" && w.probe(bob) == "161:1:true");
|
|
CHECK(w.one("SELECT transfer_state FROM evolution.stage_probe_transfer(7, 162, "
|
|
"(SELECT holder FROM evolution.region_leases WHERE world_id = 7 AND "
|
|
"region_id = 162), (SELECT epoch FROM evolution.region_leases WHERE "
|
|
"world_id = 7 AND region_id = 162), $1::uuid)",
|
|
{hex(b_east.transfer)}) == "staged"); // As B's process would, then B dies.
|
|
w.kill(false);
|
|
w.invariants();
|
|
w.restart(false);
|
|
auto b2 = w.follow(b_east);
|
|
CHECK(b2->state() == RegionSessionState::established && w.probe(bob) == "162:2:false" &&
|
|
w.transfer(b_east.transfer) == "committed");
|
|
const auto b_west = w.cross(*b2, -1);
|
|
auto b3 = w.follow(b_west);
|
|
CHECK(b3->state() == RegionSessionState::established && w.probe(bob) == "161:3:false");
|
|
|
|
// 3. Destination SIGKILL after commit (the client was admitted): a retry of the same
|
|
// ticket at the restarted destination converges (same commit, control moves again).
|
|
const auto carol = account(0xC3);
|
|
auto c1 = w.login(carol, region_a);
|
|
const auto c_east = w.cross(*c1, 1);
|
|
auto c2 = w.follow(c_east);
|
|
CHECK(c2->state() == RegionSessionState::established && w.probe(carol) == "162:2:false");
|
|
const auto control_before = w.control_generation(carol);
|
|
w.kill(false);
|
|
w.invariants();
|
|
w.restart(false);
|
|
// Bob's first ticket, replayed after his probe moved on (and before it expires).
|
|
auto replay = w.follow(b_east);
|
|
std::printf("handoff process replay at %llu s of the ticket's lifetime\n",
|
|
static_cast<unsigned long long>(wall_seconds() - ticket_issued));
|
|
CHECK(replay->state() == RegionSessionState::closed &&
|
|
replay->close_reason() == SessionCloseReason::unauthenticated &&
|
|
w.probe(bob) == "161:3:false" && w.control_holder(bob) == "161");
|
|
auto c3 = w.follow(c_east);
|
|
CHECK(c3->state() == RegionSessionState::established && w.probe(carol) == "162:2:false" &&
|
|
w.transfer(c_east.transfer) == "committed" &&
|
|
w.control_generation(carol) > control_before);
|
|
// A duplicate in the same process is refused locally (the ticket is used).
|
|
auto again = w.follow(c_east);
|
|
CHECK(again->close_reason() == SessionCloseReason::stale_grant &&
|
|
w.control_holder(carol) == "162");
|
|
w.invariants();
|
|
|
|
// 4. Source SIGKILL after prepare: Dave's ticket still commits at the destination; Erin
|
|
// never arrives, and her frozen probe is aborted by the restarted source's recovery
|
|
// sweep (the dead process's in-memory deadline is gone).
|
|
const auto dave = account(0xD4), erin = account(0xE5);
|
|
auto d1 = w.login(dave, region_a);
|
|
auto e1 = w.login(erin, region_a);
|
|
const auto d_east = w.cross(*d1, 1);
|
|
const auto e_east = w.cross(*e1, 1);
|
|
w.kill(true);
|
|
auto d2 = w.follow(d_east);
|
|
CHECK(d2->state() == RegionSessionState::established && w.probe(dave) == "162:2:false");
|
|
w.invariants();
|
|
|
|
// 5. A destination that is down: Frank's ticket cannot be used; the live source aborts
|
|
// after the ticket's lifetime. Erin's wait (above) overlaps.
|
|
w.kill(false);
|
|
w.restart(true);
|
|
auto e2 = w.login(erin, region_a); // Frozen under the open transfer: retry later.
|
|
CHECK(e2->close_reason() == SessionCloseReason::capacity && w.probe(erin) == "161:1:true");
|
|
const auto frank = account(0xF6);
|
|
auto f1 = w.login(frank, region_a);
|
|
const auto f_east = w.cross(*f1, 1);
|
|
auto f2 = w.dial(w.pair.port_b, f_east.ticket);
|
|
w.settle(*f2);
|
|
CHECK(f2->state() == RegionSessionState::closed && !f2->welcome());
|
|
w.invariants();
|
|
CHECK(w.until(
|
|
{},
|
|
[&] {
|
|
return w.transfer(e_east.transfer) == "aborted" &&
|
|
w.transfer(f_east.transfer) == "aborted";
|
|
},
|
|
90s));
|
|
CHECK(w.probe(erin) == "161:1:false" && w.probe(frank) == "161:1:false");
|
|
auto e3 = w.login(erin, region_a);
|
|
auto f3 = w.login(frank, region_a);
|
|
CHECK(e3->state() == RegionSessionState::established &&
|
|
f3->state() == RegionSessionState::established);
|
|
w.restart(false);
|
|
auto late = w.follow(f_east); // Expired by now, and its transfer aborted.
|
|
CHECK(late->close_reason() == SessionCloseReason::expired && w.probe(frank) == "161:1:false");
|
|
w.invariants();
|
|
for (auto* s : {w.a.get(), w.b.get()}) {
|
|
s->child->signal(SIGTERM);
|
|
CHECK(s->child->wait_exit(30s) == 0);
|
|
}
|
|
// Output is flushed at exit. This B process only refused (expired).
|
|
CHECK(w.a->child->printed("m10 opened ") && w.b->child->printed("m10 refused "));
|
|
}
|
|
} // namespace
|
|
|
|
void run_region_handoff_process_tests() {
|
|
handoffs();
|
|
std::puts("region handoff process integration passed");
|
|
}
|