Files
aevolution/tests/region_handoff_process_tests.cpp
dingoandClaude Opus 5.5 66efe87c90
Foundation / checks (deploy) (push) Successful in 3m15s
Foundation / checks (ecs) (push) Successful in 1m57s
Foundation / checks (client) (push) Successful in 12m33s
Foundation / checks (network) (push) Successful in 13m19s
Foundation / checks (tsan) (push) Successful in 8m5s
Foundation / checks (checks) (push) Successful in 32m42s
Foundation / checks (postgres) (push) Successful in 43m45s
Foundation / postgres-tools (push) Successful in 20s
Foundation / server-image (push) Successful in 24s
Foundation / image (push) Successful in 2m31s
Fix CI: run M10 process tests with the load governor off
checks (postgres) failed on both M10.9 pushes: on the shared CI runner one
session measured 22 ms p95 per tick, so the governor correctly refused
and the test's assumption about host speed failed (server-image was
skipped as a result). evo-server gains --m10-load-governor on|off (dev
only): off keeps measuring and exposing the work but never refuses or
sheds. The M10 session, load-smoke and two-process handoff tests use it;
the governor stays covered by its unit tests. Refreshes the dev session
window.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-07 11:58:37 -04:00

522 lines
24 KiB
C++

// Real process tests for M10.8d (ADR-0096): two evo-server processes on PostgreSQL with
// adjacent marker areas, handoff keys and each other as neighbour; in-process
// RegionSessionClients follow offers and tickets. Regions 7/161 (A, cells 0..1) and 7/162
// (B, cells 2..3). Covers a crossing and a crossing back, SIGKILL of the source after
// prepare and of the destination after stage and after commit, duplicate and stale ticket
// retries, a client that never arrives (aborted by the restarted source's recovery sweep)
// and a destination that is down (aborted by the live source). After every scenario: one
// owning region per probe, the test inventory conserved, and a record frozen exactly while
// its one open transfer exists. Run by tools/ci/postgres-check.sh when the build has
// EVO_BUILD_QUIC.
#include "evo/client/region_session.hpp"
#include "evo/network/dev_credentials.hpp"
#include "evo/network/handoff_ticket.hpp"
#include "evo/network/session_grant.hpp"
#include "fixtures/server_process.hpp"
#include <algorithm>
#include <arpa/inet.h>
#include <array>
#include <chrono>
#include <fcntl.h>
#include <functional>
#include <netinet/in.h>
#include <optional>
#include <source_location>
#include <string>
#include <sys/socket.h>
#include <unistd.h>
#include <vector>
void run_region_handoff_process_tests();
namespace {
using namespace evo::test::process;
using namespace evo;
using namespace evo::network;
using client::RegionSessionClient;
using client::RegionSessionState;
using evo::test::region_identity;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region handoff process test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
template <typename T>
const T& must(const std::optional<T>& o,
std::source_location at = std::source_location::current()) {
if (!o)
throw std::runtime_error("region handoff process missing value line " +
std::to_string(at.line()));
return *o;
}
constexpr const char* name = "dev-region.evolution.test";
constexpr std::uint64_t region_a = 161, region_b = 162;
void write_file(const fs::path& path, std::span<const std::uint8_t> bytes) {
const int fd = ::open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0600);
expect(fd >= 0 &&
::write(fd, bytes.data(), bytes.size()) == static_cast<ssize_t>(bytes.size()) &&
::close(fd) == 0,
"write file");
}
void write_text(const fs::path& path, const std::string& text) {
write_file(path, {reinterpret_cast<const std::uint8_t*>(text.data()), text.size()});
}
AccountId account(std::uint8_t tag) {
AccountId a;
a.bytes.fill(tag);
return a;
}
template <std::size_t N> std::string hex(const std::array<std::uint8_t, N>& bytes) {
constexpr char digits[] = "0123456789abcdef";
std::string out;
for (const auto b : bytes) {
out.push_back(digits[b >> 4U]);
out.push_back(digits[b & 0xFU]);
}
return out;
}
std::uint64_t wall_seconds() {
return static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count());
}
// A UDP port that was free a moment ago (each server must know its neighbour's port).
std::uint16_t free_port() {
const int fd = ::socket(AF_INET, SOCK_DGRAM | SOCK_CLOEXEC, 0);
expect(fd >= 0, "socket");
sockaddr_in a{};
a.sin_family = AF_INET;
a.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
socklen_t size = sizeof a;
const bool ok = ::bind(fd, reinterpret_cast<const sockaddr*>(&a), sizeof a) == 0 &&
::getsockname(fd, reinterpret_cast<sockaddr*>(&a), &size) == 0;
::close(fd);
expect(ok, "free port");
return ntohs(a.sin_port);
}
// A 32-byte scalar that imports as a key, written with its public key.
template <typename Key, std::size_t N>
void make_key(const fs::path& scalar_file, const fs::path& public_file,
const std::function<std::optional<Key>(std::span<const std::uint8_t>)>& import) {
for (int attempt = 0; attempt < 8; ++attempt) {
std::array<std::uint8_t, N> scalar{};
expect(random_bytes(scalar).has_value(), "random");
if (auto key = import(scalar)) {
write_file(scalar_file, scalar);
write_file(public_file, *key->public_key());
return;
}
}
expect(false, "key");
}
// A genesis commit byte for an all-equal commit id (ids are unique across the suite): one
// no other test names (later tests provision theirs by literal) and none earlier produced.
std::uint8_t unused_genesis() {
for (const unsigned b : {0x4BU, 0x4CU, 0x4DU, 0x8BU, 0x8CU, 0x8DU, 0x8EU}) {
std::string hex_id;
for (int i = 0; i < 16; ++i) {
hex_id.push_back("0123456789abcdef"[b >> 4U]);
hex_id.push_back("0123456789abcdef"[b & 0xFU]);
}
const std::array<std::string, 1> p{hex_id};
const auto r = owner()->query(
"SELECT count(*) FROM evolution.region_commits WHERE commit_id = $1::uuid", p, 5s);
expect(r.has_value() && r->size() == 1, "genesis lookup");
if ((*r)[0][0] == "0")
return static_cast<std::uint8_t>(b);
}
expect(false, "genesis byte");
return 0;
}
// Both regions: one CA and server certificate, one grant issuer, a handoff key each.
struct Pair {
ServerRegion a{provision(region_a, unused_genesis())}, b{provision(region_b, unused_genesis())};
fs::path files{a.root / "pair"};
std::string ca_pem, window;
std::optional<GrantSigningKey> issuer;
std::uint16_t port_a{free_port()}, port_b{free_port()};
Pair() {
fs::create_directory(files);
const auto made = generate_dev_credentials(DeploymentEnvironment::dev, name, 1,
std::chrono::system_clock::now());
expect(made.has_value(), "credentials");
ca_pem = made->authority_pem;
write_text(files / "ca.pem", made->authority_pem);
write_text(files / "server.pem", made->server_certificate_pem);
write_text(files / "server.key", made->server_key_pem);
make_key<GrantSigningKey, grant_private_key_bytes>(
files / "issuer.key", files / "issuer.pub", [](std::span<const std::uint8_t> s) {
auto k = GrantSigningKey::import(DeploymentEnvironment::dev, 1, s);
return k ? std::optional{std::move(*k)} : std::nullopt;
});
std::array<std::uint8_t, grant_private_key_bytes> scalar{};
const int fd = ::open((files / "issuer.key").c_str(), O_RDONLY | O_CLOEXEC);
expect(fd >= 0 &&
::read(fd, scalar.data(), scalar.size()) ==
static_cast<ssize_t>(scalar.size()) &&
::close(fd) == 0,
"issuer key");
auto key = GrantSigningKey::import(DeploymentEnvironment::dev, 1, scalar);
expect(key.has_value(), "issuer import");
issuer.emplace(std::move(*key));
for (const auto r : {region_a, region_b})
make_key<HandoffSigningKey, grant_private_key_bytes>(
files / ("handoff-" + std::to_string(r) + ".key"),
files / ("handoff-" + std::to_string(r) + ".pub"),
[r](std::span<const std::uint8_t> s) {
auto k = HandoffSigningKey::import(DeploymentEnvironment::dev,
static_cast<std::uint32_t>(r), s);
return k ? std::optional{std::move(*k)} : std::nullopt;
});
const auto now = wall_seconds();
window = "test-" + std::to_string(now) + ":" + std::to_string(now + 1200);
}
[[nodiscard]] std::vector<std::string> args(bool at_a) const {
const auto& own = at_a ? a : b;
const auto other = at_a ? region_b : region_a;
const std::string other_cells = at_a ? "2,0,3,1" : "0,0,1,1";
return server_args(
own, {"--synthetic-inputs",
"development-v1",
"--m10-listen",
"127.0.0.1:" + std::to_string(at_a ? port_a : port_b),
"--m10-cert",
(files / "server.pem").string(),
"--m10-key",
(files / "server.key").string(),
"--m10-issuer",
(files / "issuer.pub").string(),
"--m10-environment",
"dev",
"--m10-load-governor",
"off",
"--session-window",
window,
"--m10-area",
at_a ? "0,0,1,1" : "2,0,3,1",
"--m10-handoff-key",
(files / ("handoff-" + std::to_string(own.key.region) + ".key")).string(),
"--m10-neighbour",
std::to_string(other) + "=" + other_cells +
"=127.0.0.1:" + std::to_string(at_a ? port_b : port_a) + "=" + name + "=" +
(files / ("handoff-" + std::to_string(other) + ".pub")).string()});
}
[[nodiscard]] WireGrant grant(const AccountId& who, std::uint64_t serial,
std::uint64_t region) const {
GrantClaims c;
c.grant_id = (region * 100'000) + serial;
c.account = who;
c.serial = serial;
c.world = {7, region_identity.frame};
c.region = region;
c.issued_at = wall_seconds();
c.expires_at = c.issued_at + 120;
const auto g = must(issuer).sign(c);
expect(g.has_value(), "grant");
return *g;
}
};
// A running evo-server, its output drained while waiting.
struct Server {
std::unique_ptr<Child> child;
explicit Server(const Pair& p, bool at_a) : child{std::make_unique<Child>(p.args(at_a))} {
const bool started =
child->wait_line("m10 listening ", 30s) && child->wait_line("ready 1", 30s) &&
child->wait_line("m10 motion area ", 30s) && child->printed("m10 handoff neighbours=1");
if (!started)
for (const auto& line : child->lines())
if (starts(line, "m10") || starts(line, "error") || starts(line, "usage"))
std::printf("server: %s\n", line.c_str());
CHECK(started);
}
void kill() {
child->signal(SIGKILL);
CHECK(child->wait_exit(30s) == 128 + SIGKILL);
}
};
struct Bench {
Pair pair;
std::unique_ptr<Server> a, b;
std::vector<AccountId> accounts; // Every probe this test created.
std::array<std::uint64_t, 256> serials{};
void pump() {
for (auto* s : {a.get(), b.get()})
if (s)
s->child->pump(1ms);
}
// Polls clients and live servers until `done`, at most `limit`.
bool until(const std::vector<RegionSessionClient*>& clients, const std::function<bool()>& done,
std::chrono::milliseconds limit = 20s) {
const auto end = Clock::now() + limit;
while (Clock::now() < end) {
for (auto* c : clients)
c->poll(Clock::now());
pump();
if (done())
return true;
}
return done();
}
std::unique_ptr<RegionSessionClient> dial(std::uint16_t port, const WireGrant& grant) {
client::RegionSessionClientConfig c;
c.transport.port = port;
c.transport.server_name = name;
c.transport.authority_pem = pair.ca_pem;
c.transport.authority_pem_file = (pair.files / "ca.pem").string();
c.grant = grant;
auto made = RegionSessionClient::connect(std::move(c), Clock::now());
CHECK(made.has_value());
return std::move(*made);
}
// A grant login (next serial for the account) at a region; waits for the outcome.
std::unique_ptr<RegionSessionClient> login(const AccountId& who, std::uint64_t region) {
if (std::ranges::find(accounts, who) == accounts.end())
accounts.push_back(who);
auto c = dial(region == region_a ? pair.port_a : pair.port_b,
pair.grant(who, ++serials.at(who.bytes[0]), region));
settle(*c);
return c;
}
// A ticket login at the offer's destination; waits for the outcome.
std::unique_ptr<RegionSessionClient> follow(const HandoffOffer& offer) {
const auto& to = offer.destination;
CHECK(to.server_name == name);
const auto port = to.region == region_a ? pair.port_a : pair.port_b;
CHECK(to.endpoint == "127.0.0.1:" + std::to_string(port));
auto c = dial(port, offer.ticket);
settle(*c);
return c;
}
void settle(RegionSessionClient& c) {
CHECK(until({&c}, [&] {
return c.state() == RegionSessionState::established ||
c.state() == RegionSessionState::closed;
}));
}
// Walks the client's probe east (+1) or west (-1) until it is handed off; the offer.
HandoffOffer cross(RegionSessionClient& c, int direction) {
CHECK(c.state() == RegionSessionState::established);
bool moving = false;
std::optional<HandoffOffer> offer;
CHECK(until({&c}, [&] {
for (const auto& r : c.take_intention_receipts())
moving = moving || r.status == AdmitStatus::admitted;
if (!moving && c.state() == RegionSessionState::established)
static_cast<void>(c.send_intention(c.server_tick() + 4,
static_cast<std::int16_t>(direction * 1000), 0));
if (!offer)
offer = c.take_handoff();
return offer.has_value() && c.state() == RegionSessionState::closed;
}));
CHECK(c.close_reason() == SessionCloseReason::handed_off);
return must(offer);
}
// SQL views of world 7.
static persistence::PgRows rows(const std::string& sql,
const std::vector<std::string>& params) {
auto r = owner()->query(sql, params, 5s);
CHECK(r.has_value());
return *r;
}
static std::string one(const std::string& sql, const std::vector<std::string>& params) {
const auto r = rows(sql, params);
CHECK(r.size() == 1 && r[0].size() == 1);
return r[0][0].value_or("");
}
// "owner:generation:frozen" of the account's probe.
static std::string probe(const AccountId& who) {
return one("SELECT region_id || ':' || generation || ':' || (transfer_id IS NOT NULL) "
"FROM evolution.probe_records WHERE world_id = 7 AND account_id = $1::uuid",
{hex(who.bytes)});
}
static std::string transfer(const TransferId& t) {
return one("SELECT state FROM evolution.probe_transfers WHERE world_id = 7 AND "
"transfer_id = $1::uuid",
{hex(t)});
}
static std::uint64_t control_generation(const AccountId& who) {
return std::stoull(one("SELECT generation FROM evolution.account_controls WHERE "
"world_id = 7 AND account_id = $1::uuid",
{hex(who.bytes)}));
}
// After every scenario: a single owning region (A or B) per probe, the inventory
// conserved, at most one open transfer per probe and the record frozen exactly under it.
void invariants() const {
std::string list;
for (const auto& who : accounts)
list += (list.empty() ? "'" : ",'") + hex(who.bytes) + "'::uuid";
const auto where = "r.world_id = 7 AND r.account_id IN (" + list + ")";
CHECK(one("SELECT count(*) || ':' || sum(r.inventory) FROM evolution.probe_records r "
"WHERE " +
where + " AND r.region_id IN (161, 162)",
{}) ==
std::to_string(accounts.size()) + ":" + std::to_string(accounts.size() * 1000));
CHECK(one("SELECT count(*) FROM evolution.probe_records r WHERE " + where +
" AND r.transfer_id IS DISTINCT FROM (SELECT t.transfer_id FROM "
"evolution.probe_transfers t WHERE t.world_id = 7 AND t.account_id = "
"r.account_id AND t.state IN ('prepared', 'staged'))",
{}) == "0");
CHECK(one("SELECT count(*) FROM evolution.probe_transfers t WHERE t.world_id = 7 AND "
"t.account_id IN (" +
list + ") AND t.inventory <> 1000",
{}) == "0");
}
// The region holding the account's durable control.
static std::string control_holder(const AccountId& who) {
return one("SELECT coalesce(region_id::text, '') FROM evolution.account_controls "
"WHERE world_id = 7 AND account_id = $1::uuid",
{hex(who.bytes)});
}
void kill(bool at_a) {
auto& s = at_a ? a : b;
s->kill();
s.reset();
}
void restart(bool at_a) {
const auto started = Clock::now();
(at_a ? a : b) = std::make_unique<Server>(pair, at_a); // Waits out the old lease.
std::printf("handoff process restart %c %lld ms\n", at_a ? 'A' : 'B',
static_cast<long long>(std::chrono::duration_cast<std::chrono::milliseconds>(
Clock::now() - started)
.count()));
}
};
void handoffs() {
Bench w;
w.restart(true);
w.restart(false);
// 1. A crossing and a crossing back (generation 1 -> 2 -> 3), then a stale grant login
// at the old region is redirected.
const auto alice = account(0xA1);
auto a1 = w.login(alice, region_a);
CHECK(a1->state() == RegionSessionState::established && w.probe(alice) == "161:1:false");
const auto east = w.cross(*a1, 1);
CHECK(east.destination.region == region_b);
auto a2 = w.follow(east);
CHECK(a2->state() == RegionSessionState::established && w.probe(alice) == "162:2:false" &&
w.transfer(east.transfer) == "committed");
const auto west = w.cross(*a2, -1);
CHECK(west.destination.region == region_a);
auto a3 = w.follow(west);
CHECK(a3->state() == RegionSessionState::established && w.probe(alice) == "161:3:false");
auto stale = w.login(alice, region_b);
CHECK(stale->close_reason() == SessionCloseReason::moved && stale->redirect() &&
must(stale->redirect()).region == region_a);
w.invariants();
// 2. Destination SIGKILL after stage: the restarted destination stages again under its
// new epoch and commits when the ticket arrives. Then the probe moves on, and a replay of
// that ticket at a destination process that never saw it is refused durably.
const auto bob = account(0xB2);
auto b1 = w.login(bob, region_a);
const auto b_east = w.cross(*b1, 1);
const auto ticket_issued = wall_seconds();
CHECK(w.transfer(b_east.transfer) == "prepared" && w.probe(bob) == "161:1:true");
CHECK(w.one("SELECT transfer_state FROM evolution.stage_probe_transfer(7, 162, "
"(SELECT holder FROM evolution.region_leases WHERE world_id = 7 AND "
"region_id = 162), (SELECT epoch FROM evolution.region_leases WHERE "
"world_id = 7 AND region_id = 162), $1::uuid)",
{hex(b_east.transfer)}) == "staged"); // As B's process would, then B dies.
w.kill(false);
w.invariants();
w.restart(false);
auto b2 = w.follow(b_east);
CHECK(b2->state() == RegionSessionState::established && w.probe(bob) == "162:2:false" &&
w.transfer(b_east.transfer) == "committed");
const auto b_west = w.cross(*b2, -1);
auto b3 = w.follow(b_west);
CHECK(b3->state() == RegionSessionState::established && w.probe(bob) == "161:3:false");
// 3. Destination SIGKILL after commit (the client was admitted): a retry of the same
// ticket at the restarted destination converges (same commit, control moves again).
const auto carol = account(0xC3);
auto c1 = w.login(carol, region_a);
const auto c_east = w.cross(*c1, 1);
auto c2 = w.follow(c_east);
CHECK(c2->state() == RegionSessionState::established && w.probe(carol) == "162:2:false");
const auto control_before = w.control_generation(carol);
w.kill(false);
w.invariants();
w.restart(false);
// Bob's first ticket, replayed after his probe moved on (and before it expires).
auto replay = w.follow(b_east);
std::printf("handoff process replay at %llu s of the ticket's lifetime\n",
static_cast<unsigned long long>(wall_seconds() - ticket_issued));
CHECK(replay->state() == RegionSessionState::closed &&
replay->close_reason() == SessionCloseReason::unauthenticated &&
w.probe(bob) == "161:3:false" && w.control_holder(bob) == "161");
auto c3 = w.follow(c_east);
CHECK(c3->state() == RegionSessionState::established && w.probe(carol) == "162:2:false" &&
w.transfer(c_east.transfer) == "committed" &&
w.control_generation(carol) > control_before);
// A duplicate in the same process is refused locally (the ticket is used).
auto again = w.follow(c_east);
CHECK(again->close_reason() == SessionCloseReason::stale_grant &&
w.control_holder(carol) == "162");
w.invariants();
// 4. Source SIGKILL after prepare: Dave's ticket still commits at the destination; Erin
// never arrives, and her frozen probe is aborted by the restarted source's recovery
// sweep (the dead process's in-memory deadline is gone).
const auto dave = account(0xD4), erin = account(0xE5);
auto d1 = w.login(dave, region_a);
auto e1 = w.login(erin, region_a);
const auto d_east = w.cross(*d1, 1);
const auto e_east = w.cross(*e1, 1);
w.kill(true);
auto d2 = w.follow(d_east);
CHECK(d2->state() == RegionSessionState::established && w.probe(dave) == "162:2:false");
w.invariants();
// 5. A destination that is down: Frank's ticket cannot be used; the live source aborts
// after the ticket's lifetime. Erin's wait (above) overlaps.
w.kill(false);
w.restart(true);
auto e2 = w.login(erin, region_a); // Frozen under the open transfer: retry later.
CHECK(e2->close_reason() == SessionCloseReason::capacity && w.probe(erin) == "161:1:true");
const auto frank = account(0xF6);
auto f1 = w.login(frank, region_a);
const auto f_east = w.cross(*f1, 1);
auto f2 = w.dial(w.pair.port_b, f_east.ticket);
w.settle(*f2);
CHECK(f2->state() == RegionSessionState::closed && !f2->welcome());
w.invariants();
CHECK(w.until(
{},
[&] {
return w.transfer(e_east.transfer) == "aborted" &&
w.transfer(f_east.transfer) == "aborted";
},
90s));
CHECK(w.probe(erin) == "161:1:false" && w.probe(frank) == "161:1:false");
auto e3 = w.login(erin, region_a);
auto f3 = w.login(frank, region_a);
CHECK(e3->state() == RegionSessionState::established &&
f3->state() == RegionSessionState::established);
w.restart(false);
auto late = w.follow(f_east); // Expired by now, and its transfer aborted.
CHECK(late->close_reason() == SessionCloseReason::expired && w.probe(frank) == "161:1:false");
w.invariants();
for (auto* s : {w.a.get(), w.b.get()}) {
s->child->signal(SIGTERM);
CHECK(s->child->wait_exit(30s) == 0);
}
// Output is flushed at exit. This B process only refused (expired).
CHECK(w.a->child->printed("m10 opened ") && w.b->child->printed("m10 refused "));
}
} // namespace
void run_region_handoff_process_tests() {
handoffs();
std::puts("region handoff process integration passed");
}