// Real process tests for M10.8d (ADR-0096): two evo-server processes on PostgreSQL with // adjacent marker areas, handoff keys and each other as neighbour; in-process // RegionSessionClients follow offers and tickets. Regions 7/161 (A, cells 0..1) and 7/162 // (B, cells 2..3). Covers a crossing and a crossing back, SIGKILL of the source after // prepare and of the destination after stage and after commit, duplicate and stale ticket // retries, a client that never arrives (aborted by the restarted source's recovery sweep) // and a destination that is down (aborted by the live source). After every scenario: one // owning region per probe, the test inventory conserved, and a record frozen exactly while // its one open transfer exists. Run by tools/ci/postgres-check.sh when the build has // EVO_BUILD_QUIC. #include "evo/client/region_session.hpp" #include "evo/network/dev_credentials.hpp" #include "evo/network/handoff_ticket.hpp" #include "evo/network/session_grant.hpp" #include "fixtures/server_process.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include void run_region_handoff_process_tests(); namespace { using namespace evo::test::process; using namespace evo; using namespace evo::network; using client::RegionSessionClient; using client::RegionSessionState; using evo::test::region_identity; void check(bool value, std::source_location at = std::source_location::current()) { if (!value) throw std::runtime_error("region handoff process test line " + std::to_string(at.line())); } #define CHECK(...) check(static_cast(__VA_ARGS__)) template const T& must(const std::optional& o, std::source_location at = std::source_location::current()) { if (!o) throw std::runtime_error("region handoff process missing value line " + std::to_string(at.line())); return *o; } constexpr const char* name = "dev-region.evolution.test"; constexpr std::uint64_t region_a = 161, region_b = 162; void write_file(const fs::path& path, std::span bytes) { const int fd = ::open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0600); expect(fd >= 0 && ::write(fd, bytes.data(), bytes.size()) == static_cast(bytes.size()) && ::close(fd) == 0, "write file"); } void write_text(const fs::path& path, const std::string& text) { write_file(path, {reinterpret_cast(text.data()), text.size()}); } AccountId account(std::uint8_t tag) { AccountId a; a.bytes.fill(tag); return a; } template std::string hex(const std::array& bytes) { constexpr char digits[] = "0123456789abcdef"; std::string out; for (const auto b : bytes) { out.push_back(digits[b >> 4U]); out.push_back(digits[b & 0xFU]); } return out; } std::uint64_t wall_seconds() { return static_cast(std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count()); } // A UDP port that was free a moment ago (each server must know its neighbour's port). std::uint16_t free_port() { const int fd = ::socket(AF_INET, SOCK_DGRAM | SOCK_CLOEXEC, 0); expect(fd >= 0, "socket"); sockaddr_in a{}; a.sin_family = AF_INET; a.sin_addr.s_addr = htonl(INADDR_LOOPBACK); socklen_t size = sizeof a; const bool ok = ::bind(fd, reinterpret_cast(&a), sizeof a) == 0 && ::getsockname(fd, reinterpret_cast(&a), &size) == 0; ::close(fd); expect(ok, "free port"); return ntohs(a.sin_port); } // A 32-byte scalar that imports as a key, written with its public key. template void make_key(const fs::path& scalar_file, const fs::path& public_file, const std::function(std::span)>& import) { for (int attempt = 0; attempt < 8; ++attempt) { std::array scalar{}; expect(random_bytes(scalar).has_value(), "random"); if (auto key = import(scalar)) { write_file(scalar_file, scalar); write_file(public_file, *key->public_key()); return; } } expect(false, "key"); } // A genesis commit byte for an all-equal commit id (ids are unique across the suite): one // no other test names (later tests provision theirs by literal) and none earlier produced. std::uint8_t unused_genesis() { for (const unsigned b : {0x4BU, 0x4CU, 0x4DU, 0x8BU, 0x8CU, 0x8DU, 0x8EU}) { std::string hex_id; for (int i = 0; i < 16; ++i) { hex_id.push_back("0123456789abcdef"[b >> 4U]); hex_id.push_back("0123456789abcdef"[b & 0xFU]); } const std::array p{hex_id}; const auto r = owner()->query( "SELECT count(*) FROM evolution.region_commits WHERE commit_id = $1::uuid", p, 5s); expect(r.has_value() && r->size() == 1, "genesis lookup"); if ((*r)[0][0] == "0") return static_cast(b); } expect(false, "genesis byte"); return 0; } // Both regions: one CA and server certificate, one grant issuer, a handoff key each. struct Pair { ServerRegion a{provision(region_a, unused_genesis())}, b{provision(region_b, unused_genesis())}; fs::path files{a.root / "pair"}; std::string ca_pem, window; std::optional issuer; std::uint16_t port_a{free_port()}, port_b{free_port()}; Pair() { fs::create_directory(files); const auto made = generate_dev_credentials(DeploymentEnvironment::dev, name, 1, std::chrono::system_clock::now()); expect(made.has_value(), "credentials"); ca_pem = made->authority_pem; write_text(files / "ca.pem", made->authority_pem); write_text(files / "server.pem", made->server_certificate_pem); write_text(files / "server.key", made->server_key_pem); make_key( files / "issuer.key", files / "issuer.pub", [](std::span s) { auto k = GrantSigningKey::import(DeploymentEnvironment::dev, 1, s); return k ? std::optional{std::move(*k)} : std::nullopt; }); std::array scalar{}; const int fd = ::open((files / "issuer.key").c_str(), O_RDONLY | O_CLOEXEC); expect(fd >= 0 && ::read(fd, scalar.data(), scalar.size()) == static_cast(scalar.size()) && ::close(fd) == 0, "issuer key"); auto key = GrantSigningKey::import(DeploymentEnvironment::dev, 1, scalar); expect(key.has_value(), "issuer import"); issuer.emplace(std::move(*key)); for (const auto r : {region_a, region_b}) make_key( files / ("handoff-" + std::to_string(r) + ".key"), files / ("handoff-" + std::to_string(r) + ".pub"), [r](std::span s) { auto k = HandoffSigningKey::import(DeploymentEnvironment::dev, static_cast(r), s); return k ? std::optional{std::move(*k)} : std::nullopt; }); const auto now = wall_seconds(); window = "test-" + std::to_string(now) + ":" + std::to_string(now + 1200); } [[nodiscard]] std::vector args(bool at_a) const { const auto& own = at_a ? a : b; const auto other = at_a ? region_b : region_a; const std::string other_cells = at_a ? "2,0,3,1" : "0,0,1,1"; return server_args( own, {"--synthetic-inputs", "development-v1", "--m10-listen", "127.0.0.1:" + std::to_string(at_a ? port_a : port_b), "--m10-cert", (files / "server.pem").string(), "--m10-key", (files / "server.key").string(), "--m10-issuer", (files / "issuer.pub").string(), "--m10-environment", "dev", "--m10-load-governor", "off", "--session-window", window, "--m10-area", at_a ? "0,0,1,1" : "2,0,3,1", "--m10-handoff-key", (files / ("handoff-" + std::to_string(own.key.region) + ".key")).string(), "--m10-neighbour", std::to_string(other) + "=" + other_cells + "=127.0.0.1:" + std::to_string(at_a ? port_b : port_a) + "=" + name + "=" + (files / ("handoff-" + std::to_string(other) + ".pub")).string()}); } [[nodiscard]] WireGrant grant(const AccountId& who, std::uint64_t serial, std::uint64_t region) const { GrantClaims c; c.grant_id = (region * 100'000) + serial; c.account = who; c.serial = serial; c.world = {7, region_identity.frame}; c.region = region; c.issued_at = wall_seconds(); c.expires_at = c.issued_at + 120; const auto g = must(issuer).sign(c); expect(g.has_value(), "grant"); return *g; } }; // A running evo-server, its output drained while waiting. struct Server { std::unique_ptr child; explicit Server(const Pair& p, bool at_a) : child{std::make_unique(p.args(at_a))} { const bool started = child->wait_line("m10 listening ", 30s) && child->wait_line("ready 1", 30s) && child->wait_line("m10 motion area ", 30s) && child->printed("m10 handoff neighbours=1"); if (!started) for (const auto& line : child->lines()) if (starts(line, "m10") || starts(line, "error") || starts(line, "usage")) std::printf("server: %s\n", line.c_str()); CHECK(started); } void kill() { child->signal(SIGKILL); CHECK(child->wait_exit(30s) == 128 + SIGKILL); } }; struct Bench { Pair pair; std::unique_ptr a, b; std::vector accounts; // Every probe this test created. std::array serials{}; void pump() { for (auto* s : {a.get(), b.get()}) if (s) s->child->pump(1ms); } // Polls clients and live servers until `done`, at most `limit`. bool until(const std::vector& clients, const std::function& done, std::chrono::milliseconds limit = 20s) { const auto end = Clock::now() + limit; while (Clock::now() < end) { for (auto* c : clients) c->poll(Clock::now()); pump(); if (done()) return true; } return done(); } std::unique_ptr dial(std::uint16_t port, const WireGrant& grant) { client::RegionSessionClientConfig c; c.transport.port = port; c.transport.server_name = name; c.transport.authority_pem = pair.ca_pem; c.transport.authority_pem_file = (pair.files / "ca.pem").string(); c.grant = grant; auto made = RegionSessionClient::connect(std::move(c), Clock::now()); CHECK(made.has_value()); return std::move(*made); } // A grant login (next serial for the account) at a region; waits for the outcome. std::unique_ptr login(const AccountId& who, std::uint64_t region) { if (std::ranges::find(accounts, who) == accounts.end()) accounts.push_back(who); auto c = dial(region == region_a ? pair.port_a : pair.port_b, pair.grant(who, ++serials.at(who.bytes[0]), region)); settle(*c); return c; } // A ticket login at the offer's destination; waits for the outcome. std::unique_ptr follow(const HandoffOffer& offer) { const auto& to = offer.destination; CHECK(to.server_name == name); const auto port = to.region == region_a ? pair.port_a : pair.port_b; CHECK(to.endpoint == "127.0.0.1:" + std::to_string(port)); auto c = dial(port, offer.ticket); settle(*c); return c; } void settle(RegionSessionClient& c) { CHECK(until({&c}, [&] { return c.state() == RegionSessionState::established || c.state() == RegionSessionState::closed; })); } // Walks the client's probe east (+1) or west (-1) until it is handed off; the offer. HandoffOffer cross(RegionSessionClient& c, int direction) { CHECK(c.state() == RegionSessionState::established); bool moving = false; std::optional offer; CHECK(until({&c}, [&] { for (const auto& r : c.take_intention_receipts()) moving = moving || r.status == AdmitStatus::admitted; if (!moving && c.state() == RegionSessionState::established) static_cast(c.send_intention(c.server_tick() + 4, static_cast(direction * 1000), 0)); if (!offer) offer = c.take_handoff(); return offer.has_value() && c.state() == RegionSessionState::closed; })); CHECK(c.close_reason() == SessionCloseReason::handed_off); return must(offer); } // SQL views of world 7. static persistence::PgRows rows(const std::string& sql, const std::vector& params) { auto r = owner()->query(sql, params, 5s); CHECK(r.has_value()); return *r; } static std::string one(const std::string& sql, const std::vector& params) { const auto r = rows(sql, params); CHECK(r.size() == 1 && r[0].size() == 1); return r[0][0].value_or(""); } // "owner:generation:frozen" of the account's probe. static std::string probe(const AccountId& who) { return one("SELECT region_id || ':' || generation || ':' || (transfer_id IS NOT NULL) " "FROM evolution.probe_records WHERE world_id = 7 AND account_id = $1::uuid", {hex(who.bytes)}); } static std::string transfer(const TransferId& t) { return one("SELECT state FROM evolution.probe_transfers WHERE world_id = 7 AND " "transfer_id = $1::uuid", {hex(t)}); } static std::uint64_t control_generation(const AccountId& who) { return std::stoull(one("SELECT generation FROM evolution.account_controls WHERE " "world_id = 7 AND account_id = $1::uuid", {hex(who.bytes)})); } // After every scenario: a single owning region (A or B) per probe, the inventory // conserved, at most one open transfer per probe and the record frozen exactly under it. void invariants() const { std::string list; for (const auto& who : accounts) list += (list.empty() ? "'" : ",'") + hex(who.bytes) + "'::uuid"; const auto where = "r.world_id = 7 AND r.account_id IN (" + list + ")"; CHECK(one("SELECT count(*) || ':' || sum(r.inventory) FROM evolution.probe_records r " "WHERE " + where + " AND r.region_id IN (161, 162)", {}) == std::to_string(accounts.size()) + ":" + std::to_string(accounts.size() * 1000)); CHECK(one("SELECT count(*) FROM evolution.probe_records r WHERE " + where + " AND r.transfer_id IS DISTINCT FROM (SELECT t.transfer_id FROM " "evolution.probe_transfers t WHERE t.world_id = 7 AND t.account_id = " "r.account_id AND t.state IN ('prepared', 'staged'))", {}) == "0"); CHECK(one("SELECT count(*) FROM evolution.probe_transfers t WHERE t.world_id = 7 AND " "t.account_id IN (" + list + ") AND t.inventory <> 1000", {}) == "0"); } // The region holding the account's durable control. static std::string control_holder(const AccountId& who) { return one("SELECT coalesce(region_id::text, '') FROM evolution.account_controls " "WHERE world_id = 7 AND account_id = $1::uuid", {hex(who.bytes)}); } void kill(bool at_a) { auto& s = at_a ? a : b; s->kill(); s.reset(); } void restart(bool at_a) { const auto started = Clock::now(); (at_a ? a : b) = std::make_unique(pair, at_a); // Waits out the old lease. std::printf("handoff process restart %c %lld ms\n", at_a ? 'A' : 'B', static_cast(std::chrono::duration_cast( Clock::now() - started) .count())); } }; void handoffs() { Bench w; w.restart(true); w.restart(false); // 1. A crossing and a crossing back (generation 1 -> 2 -> 3), then a stale grant login // at the old region is redirected. const auto alice = account(0xA1); auto a1 = w.login(alice, region_a); CHECK(a1->state() == RegionSessionState::established && w.probe(alice) == "161:1:false"); const auto east = w.cross(*a1, 1); CHECK(east.destination.region == region_b); auto a2 = w.follow(east); CHECK(a2->state() == RegionSessionState::established && w.probe(alice) == "162:2:false" && w.transfer(east.transfer) == "committed"); const auto west = w.cross(*a2, -1); CHECK(west.destination.region == region_a); auto a3 = w.follow(west); CHECK(a3->state() == RegionSessionState::established && w.probe(alice) == "161:3:false"); auto stale = w.login(alice, region_b); CHECK(stale->close_reason() == SessionCloseReason::moved && stale->redirect() && must(stale->redirect()).region == region_a); w.invariants(); // 2. Destination SIGKILL after stage: the restarted destination stages again under its // new epoch and commits when the ticket arrives. Then the probe moves on, and a replay of // that ticket at a destination process that never saw it is refused durably. const auto bob = account(0xB2); auto b1 = w.login(bob, region_a); const auto b_east = w.cross(*b1, 1); const auto ticket_issued = wall_seconds(); CHECK(w.transfer(b_east.transfer) == "prepared" && w.probe(bob) == "161:1:true"); CHECK(w.one("SELECT transfer_state FROM evolution.stage_probe_transfer(7, 162, " "(SELECT holder FROM evolution.region_leases WHERE world_id = 7 AND " "region_id = 162), (SELECT epoch FROM evolution.region_leases WHERE " "world_id = 7 AND region_id = 162), $1::uuid)", {hex(b_east.transfer)}) == "staged"); // As B's process would, then B dies. w.kill(false); w.invariants(); w.restart(false); auto b2 = w.follow(b_east); CHECK(b2->state() == RegionSessionState::established && w.probe(bob) == "162:2:false" && w.transfer(b_east.transfer) == "committed"); const auto b_west = w.cross(*b2, -1); auto b3 = w.follow(b_west); CHECK(b3->state() == RegionSessionState::established && w.probe(bob) == "161:3:false"); // 3. Destination SIGKILL after commit (the client was admitted): a retry of the same // ticket at the restarted destination converges (same commit, control moves again). const auto carol = account(0xC3); auto c1 = w.login(carol, region_a); const auto c_east = w.cross(*c1, 1); auto c2 = w.follow(c_east); CHECK(c2->state() == RegionSessionState::established && w.probe(carol) == "162:2:false"); const auto control_before = w.control_generation(carol); w.kill(false); w.invariants(); w.restart(false); // Bob's first ticket, replayed after his probe moved on (and before it expires). auto replay = w.follow(b_east); std::printf("handoff process replay at %llu s of the ticket's lifetime\n", static_cast(wall_seconds() - ticket_issued)); CHECK(replay->state() == RegionSessionState::closed && replay->close_reason() == SessionCloseReason::unauthenticated && w.probe(bob) == "161:3:false" && w.control_holder(bob) == "161"); auto c3 = w.follow(c_east); CHECK(c3->state() == RegionSessionState::established && w.probe(carol) == "162:2:false" && w.transfer(c_east.transfer) == "committed" && w.control_generation(carol) > control_before); // A duplicate in the same process is refused locally (the ticket is used). auto again = w.follow(c_east); CHECK(again->close_reason() == SessionCloseReason::stale_grant && w.control_holder(carol) == "162"); w.invariants(); // 4. Source SIGKILL after prepare: Dave's ticket still commits at the destination; Erin // never arrives, and her frozen probe is aborted by the restarted source's recovery // sweep (the dead process's in-memory deadline is gone). const auto dave = account(0xD4), erin = account(0xE5); auto d1 = w.login(dave, region_a); auto e1 = w.login(erin, region_a); const auto d_east = w.cross(*d1, 1); const auto e_east = w.cross(*e1, 1); w.kill(true); auto d2 = w.follow(d_east); CHECK(d2->state() == RegionSessionState::established && w.probe(dave) == "162:2:false"); w.invariants(); // 5. A destination that is down: Frank's ticket cannot be used; the live source aborts // after the ticket's lifetime. Erin's wait (above) overlaps. w.kill(false); w.restart(true); auto e2 = w.login(erin, region_a); // Frozen under the open transfer: retry later. CHECK(e2->close_reason() == SessionCloseReason::capacity && w.probe(erin) == "161:1:true"); const auto frank = account(0xF6); auto f1 = w.login(frank, region_a); const auto f_east = w.cross(*f1, 1); auto f2 = w.dial(w.pair.port_b, f_east.ticket); w.settle(*f2); CHECK(f2->state() == RegionSessionState::closed && !f2->welcome()); w.invariants(); CHECK(w.until( {}, [&] { return w.transfer(e_east.transfer) == "aborted" && w.transfer(f_east.transfer) == "aborted"; }, 90s)); CHECK(w.probe(erin) == "161:1:false" && w.probe(frank) == "161:1:false"); auto e3 = w.login(erin, region_a); auto f3 = w.login(frank, region_a); CHECK(e3->state() == RegionSessionState::established && f3->state() == RegionSessionState::established); w.restart(false); auto late = w.follow(f_east); // Expired by now, and its transfer aborted. CHECK(late->close_reason() == SessionCloseReason::expired && w.probe(frank) == "161:1:false"); w.invariants(); for (auto* s : {w.a.get(), w.b.get()}) { s->child->signal(SIGTERM); CHECK(s->child->wait_exit(30s) == 0); } // Output is flushed at exit. This B process only refused (expired). CHECK(w.a->child->printed("m10 opened ") && w.b->child->printed("m10 refused ")); } } // namespace void run_region_handoff_process_tests() { handoffs(); std::puts("region handoff process integration passed"); }