Files
aevolution/tests/region_m10_session_tests.cpp
T
dingo e6ba9bd1a9
Foundation / checks (deploy) (push) Successful in 6s
Foundation / checks (checks) (push) Failing after 40s
Foundation / checks (postgres) (push) Canceled after 0s
Foundation / checks (tsan) (push) Canceled after 0s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (client) (push) Canceled after 1m12s
Foundation / checks (network) (push) Canceled after 37s
Foundation / checks (ecs) (push) Canceled after 1m9s
Test life owner routing across region restart
2026-10-11 01:34:46 -04:00

900 lines
41 KiB
C++

// Real process tests for M10.4c (ADR-0091, ADR-0092): evo-server with PostgreSQL, synthetic
// ticks and the QUIC session listener; in-process RegionSessionClients and the headless
// probe. Intentions get receipts and the simulated watermark; actions get durable outcomes
// that survive SIGKILL after their commit, and resolve as "no record" (then commit once on
// retry) after SIGKILL before it. Region 7/151. Run by tools/ci/postgres-check.sh
// when the build has EVO_BUILD_QUIC.
#include "evo/biology/founder.hpp"
#include "evo/client/region_session.hpp"
#include "evo/network/dev_credentials.hpp"
#include "evo/network/session_grant.hpp"
#include "evo/persistence/checkpoint_store.hpp"
#include "evo/server/lifecycle/primitive_lives.hpp"
#include "fixtures/server_process.hpp"
#include <algorithm>
#include <arpa/inet.h>
#include <array>
#include <chrono>
#include <fcntl.h>
#include <functional>
#include <netinet/in.h>
#include <optional>
#include <source_location>
#include <sstream>
#include <string>
#include <sys/socket.h>
#include <thread>
#include <unistd.h>
void run_region_m10_session_tests();
namespace {
using namespace evo::test::process;
using namespace evo;
using namespace evo::network;
using client::RegionSessionClient;
using client::RegionSessionState;
using evo::test::region_identity;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region m10 session test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
template <typename T>
const T& must(const std::optional<T>& o,
std::source_location at = std::source_location::current()) {
if (!o)
throw std::runtime_error("region m10 session missing value line " +
std::to_string(at.line()));
return *o;
}
constexpr const char* name = "dev-region.evolution.test";
void write_file(const fs::path& path, std::span<const std::uint8_t> bytes) {
const int fd = ::open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0600);
expect(fd >= 0 &&
::write(fd, bytes.data(), bytes.size()) == static_cast<ssize_t>(bytes.size()) &&
::close(fd) == 0,
"write file");
}
void write_text(const fs::path& path, const std::string& text) {
write_file(path, {reinterpret_cast<const std::uint8_t*>(text.data()), text.size()});
}
AccountId account(std::uint8_t tag) {
AccountId a;
a.bytes.fill(tag);
return a;
}
std::string hex(const AccountId& a) {
constexpr char digits[] = "0123456789abcdef";
std::string out;
for (const auto b : a.bytes) {
out.push_back(digits[b >> 4U]);
out.push_back(digits[b & 0xFU]);
}
return out;
}
// GET /metrics from the ops listener; the value of an unlabelled series (ADR-0097 adds the
// evo_m10_* families when the listener is enabled).
std::optional<std::uint64_t> metric(std::uint16_t port, const std::string& series) {
const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0);
expect(fd >= 0, "socket");
sockaddr_in a{};
a.sin_family = AF_INET;
a.sin_port = htons(port);
a.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
timeval timeout{3, 0};
::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout);
std::string raw;
if (::connect(fd, reinterpret_cast<const sockaddr*>(&a), sizeof a) == 0) {
const std::string request = "GET /metrics HTTP/1.1\r\nHost: test\r\n\r\n";
if (::send(fd, request.data(), request.size(), MSG_NOSIGNAL) ==
static_cast<ssize_t>(request.size())) {
std::array<char, 4096> chunk{};
for (ssize_t n = 0; (n = ::recv(fd, chunk.data(), chunk.size(), 0)) > 0;)
raw.append(chunk.data(), static_cast<std::size_t>(n));
}
}
::close(fd);
const auto at = raw.find("\n" + series + " ");
if (at == std::string::npos)
return std::nullopt;
return std::stoull(raw.substr(at + series.size() + 2));
}
// A UDP port that was free a moment ago.
std::uint16_t free_port() {
const int fd = ::socket(AF_INET, SOCK_DGRAM | SOCK_CLOEXEC, 0);
expect(fd >= 0, "socket");
sockaddr_in a{};
a.sin_family = AF_INET;
a.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
socklen_t size = sizeof a;
const bool ok = ::bind(fd, reinterpret_cast<const sockaddr*>(&a), sizeof a) == 0 &&
::getsockname(fd, reinterpret_cast<sockaddr*>(&a), &size) == 0;
::close(fd);
expect(ok, "free port");
return ntohs(a.sin_port);
}
// One region's credentials, issuer key and server process arguments.
struct Setup {
ServerRegion region;
std::string ca_pem;
std::string listen{"127.0.0.1:0"}; // A fixed port lets clients reconnect after a restart.
std::optional<GrantSigningKey> issuer;
explicit Setup(std::uint64_t r, std::uint8_t genesis) : region{provision(r, genesis)} {
const auto made = generate_dev_credentials(DeploymentEnvironment::dev, name, 1,
std::chrono::system_clock::now());
expect(made.has_value(), "credentials");
ca_pem = made->authority_pem;
write_text(region.root / "ca.pem", made->authority_pem);
write_text(region.root / "server.pem", made->server_certificate_pem);
write_text(region.root / "server.key", made->server_key_pem);
for (int attempt = 0; attempt < 8 && !issuer; ++attempt) {
std::array<std::uint8_t, grant_private_key_bytes> scalar{};
expect(random_bytes(scalar).has_value(), "random");
if (auto key = GrantSigningKey::import(DeploymentEnvironment::dev, 1, scalar)) {
write_file(region.root / "issuer.key", scalar);
write_file(region.root / "issuer.pub", *key->public_key());
issuer.emplace(std::move(*key));
}
}
expect(issuer.has_value(), "issuer key");
}
[[nodiscard]] std::vector<std::string> args() const {
return server_args(
region, {"--synthetic-inputs",
"development-v1",
"--m10-load-governor",
"off",
"--ops-listen",
"127.0.0.1:0",
"--m10-listen",
listen,
"--m10-cert",
(region.root / "server.pem").string(),
"--m10-key",
(region.root / "server.key").string(),
"--m10-issuer",
(region.root / "issuer.pub").string(),
"--m10-environment",
"dev",
"--m10-max-grant-lifetime",
"900",
"--session-window",
"test-" +
std::to_string(static_cast<std::uint64_t>(
std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count())) +
":" +
std::to_string(static_cast<std::uint64_t>(
std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count()) +
1200)});
}
[[nodiscard]] WireGrant grant(const AccountId& who, std::uint64_t serial) const {
GrantClaims c;
c.grant_id = 9000 + serial;
c.account = who;
c.serial = serial;
c.world = {region.key.world, region_identity.frame};
c.region = region.key.region;
c.issued_at =
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count());
c.expires_at = c.issued_at + 120;
const auto g = must(issuer).sign(c);
expect(g.has_value(), "grant");
return *g;
}
};
// A running evo-server and its QUIC port, its output drained while waiting.
struct Server {
std::unique_ptr<Child> child;
std::uint16_t port{}, ops{};
explicit Server(const Setup& s) : child{std::make_unique<Child>(s.args())} {
CHECK(child->wait_line("m10 listening ", 30s));
for (const auto& l : child->lines()) {
if (starts(l, "m10 listening "))
port = static_cast<std::uint16_t>(std::stoul(l.substr(14)));
if (starts(l, "ops listening "))
ops = static_cast<std::uint16_t>(std::stoul(l.substr(14)));
}
CHECK(ops != 0);
CHECK(port != 0 && child->wait_line("ready 1", 30s));
}
void pump() {
child->pump(1ms);
}
void kill() {
child->signal(SIGKILL);
CHECK(child->wait_exit(30s) == 128 + SIGKILL);
}
};
std::unique_ptr<RegionSessionClient> connect(const Setup& s, Server& server,
const WireGrant& grant) {
client::RegionSessionClientConfig c;
c.transport.port = server.port;
c.transport.server_name = name;
c.transport.authority_pem = s.ca_pem;
c.transport.authority_pem_file = (s.region.root / "ca.pem").string();
c.grant = grant;
auto made = RegionSessionClient::connect(std::move(c), std::chrono::steady_clock::now());
CHECK(made.has_value());
return std::move(*made);
}
// Polls client and server until `done`, at most `limit`.
bool until(Server& server, RegionSessionClient& c, const std::function<bool()>& done,
std::chrono::milliseconds limit = 20s) {
const auto end = Clock::now() + limit;
while (Clock::now() < end) {
c.poll(Clock::now());
server.pump();
if (done())
return true;
}
return done();
}
bool established(Server& server, RegionSessionClient& c) {
return until(server, c,
[&] {
return c.state() == RegionSessionState::established ||
c.state() == RegionSessionState::closed;
}) &&
c.state() == RegionSessionState::established;
}
std::optional<ActionOutcome> outcome_of(Server& server, RegionSessionClient& c, std::uint64_t id,
bool final_only) {
std::optional<ActionOutcome> found;
until(server, c, [&] {
for (const auto& o : c.take_action_outcomes())
if (o.action_id == id &&
(!final_only || o.result == ActionResult::committed_no_effect ||
o.result == ActionResult::not_committed))
found = o;
return found.has_value();
});
return found;
}
struct PrimitiveSetup {
ServerRegion region;
std::string ca_pem;
std::optional<GrantSigningKey> issuer;
explicit PrimitiveSetup(std::uint64_t r)
: region{{7, r},
fs::path(env("PGHOST")) / ("server-primitive-session-" + std::to_string(r))} {
fs::remove_all(region.root);
fs::create_directories(region.root);
fs::permissions(region.root, fs::perms::owner_all, fs::perm_options::replace);
auto db = owner();
CHECK(db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('b1',32),'hex')) ON CONFLICT DO NOTHING",
{}, 5s));
const auto endowed = biology::primitive_endowment();
CHECK(endowed.has_value());
auto funding = endowed->body;
funding.energy.value += endowed->reserve.energy.value + endowed->construction.value;
auto opening = server::PrimitiveLives::create(
{region.key.world, region.key.region, region_identity.frame, 1, funding, {{0, 0}}});
CHECK(opening.has_value());
auto objects = persistence::FilesystemObjectStore::open(region.root, region.key,
server::max_primitive_lives_bytes);
CHECK(objects.has_value());
persistence::Sha256Digest build{};
build.fill(0xB1);
persistence::Uuid commit{};
commit.fill(static_cast<std::uint8_t>(r));
CHECK(persistence::provision_primitive_region(
*db, **objects, region.key, region_identity.frame, **opening, commit, build, 5s));
const std::array<std::string, 2> allocator_args{std::to_string(region.key.world), "1"};
CHECK(db->query(
"INSERT INTO evolution.life_allocators VALUES($1::evolution.u64, $2::evolution.u64) "
"ON CONFLICT DO NOTHING",
allocator_args, 5s));
const auto credentials = generate_dev_credentials(DeploymentEnvironment::dev, name, 1,
std::chrono::system_clock::now());
CHECK(credentials.has_value());
ca_pem = credentials->authority_pem;
write_text(region.root / "ca.pem", credentials->authority_pem);
write_text(region.root / "server.pem", credentials->server_certificate_pem);
write_text(region.root / "server.key", credentials->server_key_pem);
for (int attempt = 0; attempt < 8 && !issuer; ++attempt) {
std::array<std::uint8_t, grant_private_key_bytes> scalar{};
CHECK(random_bytes(scalar));
if (auto key = GrantSigningKey::import(DeploymentEnvironment::dev, 1, scalar)) {
if (const auto public_key = key->public_key()) {
write_file(region.root / "issuer.pub", *public_key);
issuer.emplace(std::move(*key));
}
}
}
CHECK(issuer.has_value());
}
WireGrant grant(const AccountId& who, std::uint64_t serial) const {
GrantClaims claims;
claims.grant_id = 19000 + serial;
claims.account = who;
claims.serial = serial;
claims.world = {region.key.world, region_identity.frame};
claims.region = region.key.region;
claims.issued_at =
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count());
claims.expires_at = claims.issued_at + 120;
return must(issuer).sign(claims).value();
}
std::vector<std::string> args(std::uint16_t port = 0,
const std::vector<std::string>& routes = {}) const {
auto result = std::vector<std::string>{
env("EVO_SERVER_BIN"),
"--profile",
"primitive",
"--world",
std::to_string(region.key.world),
"--region",
std::to_string(region.key.region),
"--frame",
std::to_string(region_identity.frame),
"--pg-host",
env("PGHOST"),
"--pg-port",
env("PGPORT"),
"--pg-database",
env("PGDATABASE"),
"--pg-user",
env("EVO_PG_RUNTIME_USER"),
"--objects",
region.root.string(),
"--build",
"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1",
"--lease-profile",
"fast",
"--ready-file",
(region.root / "ready").string(),
"--primitive-listen",
"127.0.0.1:" + std::to_string(port),
"--primitive-cert",
(region.root / "server.pem").string(),
"--primitive-key",
(region.root / "server.key").string(),
"--primitive-issuer",
(region.root / "issuer.pub").string(),
"--primitive-environment",
"dev"};
for (const auto& route : routes) {
result.push_back("--life-route");
result.push_back(route);
}
return result;
}
};
struct PrimitiveServer {
std::unique_ptr<Child> child;
std::uint16_t port{};
explicit PrimitiveServer(const PrimitiveSetup& setup, std::uint16_t fixed_port = 0,
const std::vector<std::string>& routes = {})
: child{std::make_unique<Child>(setup.args(fixed_port, routes))} {
CHECK(child->wait_line("primitive listening ", 30s));
for (const auto& line : child->lines())
if (starts(line, "primitive listening "))
port = static_cast<std::uint16_t>(std::stoul(line.substr(19)));
CHECK(port != 0 && child->wait_line("ready 1", 30s));
}
void pump() {
child->pump(1ms);
}
};
std::unique_ptr<RegionSessionClient>
connect_primitive(const PrimitiveSetup& setup, PrimitiveServer& server, const WireGrant& grant) {
client::RegionSessionClientConfig config;
config.transport.port = server.port;
config.transport.server_name = name;
config.transport.authority_pem = setup.ca_pem;
config.transport.authority_pem_file = (setup.region.root / "ca.pem").string();
config.grant = grant;
config.protocol_version = session_protocol_v5;
auto made = RegionSessionClient::connect(std::move(config), Clock::now());
CHECK(made.has_value());
return std::move(*made);
}
bool until_primitive(PrimitiveServer& server, RegionSessionClient& client,
const std::function<bool()>& done, std::chrono::milliseconds limit = 20s) {
const auto end = Clock::now() + limit;
while (Clock::now() < end) {
client.poll(Clock::now());
server.pump();
if (done())
return true;
}
return done();
}
void primitive_life_v5_process_round_trip() {
PrimitiveSetup setup{35};
const auto who = account(0xA7);
auto server = std::make_unique<PrimitiveServer>(setup);
auto client = connect_primitive(setup, *server, setup.grant(who, 1));
CHECK(until_primitive(*server, *client,
[&] { return client->state() == RegionSessionState::established; }));
CHECK(client->request_primitive_life(501));
std::vector<PrimitiveLifeReceipt> receipts;
std::vector<PrimitiveLifeOutcome> outcomes;
CHECK(until_primitive(*server, *client, [&] {
auto next_receipts = client->take_primitive_life_receipts();
auto next_outcomes = client->take_primitive_life_outcomes();
receipts.insert(receipts.end(), next_receipts.begin(), next_receipts.end());
outcomes.insert(outcomes.end(), next_outcomes.begin(), next_outcomes.end());
return !receipts.empty() && !outcomes.empty();
}));
CHECK(receipts.front().status == AdmitStatus::admitted &&
outcomes.front().result == PrimitiveLifeResult::born &&
outcomes.front().life_identity != 0 && outcomes.front().commit_sequence != 0);
const auto born = outcomes.front();
// A second region sees the durable life owner and redirects before its ledger can
// move account control. The route catalog includes both regions, independently of
// the probe-neighbour configuration.
PrimitiveSetup destination_setup{36};
const auto destination_port = free_port();
const std::vector<std::string> routes{
"35=127.0.0.1:" + std::to_string(server->port) + "=" + name,
"36=127.0.0.1:" + std::to_string(destination_port) + "=" + name};
auto destination =
std::make_unique<PrimitiveServer>(destination_setup, destination_port, routes);
auto redirected =
connect_primitive(destination_setup, *destination, destination_setup.grant(who, 2));
CHECK(until_primitive(*destination, *redirected,
[&] { return redirected->state() == RegionSessionState::closed; }));
CHECK(redirected->close_reason() == SessionCloseReason::moved && redirected->redirect());
CHECK(must(redirected->redirect()) ==
RegionAddress{35, "127.0.0.1:" + std::to_string(server->port), name});
const std::array<std::string, 2> account_key{"7", hex(who)};
const auto control = owner()->query(
"SELECT region_id FROM evolution.account_controls WHERE world_id=$1::evolution.u64 "
"AND account_id=$2::uuid",
account_key, 5s);
CHECK(control && control->size() == 1 && (*control)[0][0] == "35");
destination->child->signal(SIGTERM);
CHECK(destination->child->wait_exit(30s) == 0);
// Kill the owner process without cleanup. Its recovered reservation remains the route
// target after the lease expires and the region host restarts.
server->child->signal(SIGKILL);
CHECK(server->child->wait_exit(30s) == 128 + SIGKILL);
server = std::make_unique<PrimitiveServer>(setup, server->port);
auto resumed = connect_primitive(setup, *server, setup.grant(who, 2));
CHECK(until_primitive(*server, *resumed,
[&] { return resumed->state() == RegionSessionState::established; }));
CHECK(resumed->query_primitive_life(501));
outcomes.clear();
CHECK(until_primitive(*server, *resumed, [&] {
outcomes = resumed->take_primitive_life_outcomes();
return !outcomes.empty();
}));
CHECK(outcomes.front() == born);
destination = std::make_unique<PrimitiveServer>(destination_setup, destination_port, routes);
auto redirected_after_restart =
connect_primitive(destination_setup, *destination, destination_setup.grant(who, 3));
CHECK(until_primitive(*destination, *redirected_after_restart, [&] {
return redirected_after_restart->state() == RegionSessionState::closed;
}));
CHECK(redirected_after_restart->close_reason() == SessionCloseReason::moved &&
redirected_after_restart->redirect() &&
must(redirected_after_restart->redirect()).region == 35);
destination->child->signal(SIGTERM);
CHECK(destination->child->wait_exit(30s) == 0);
resumed->close();
server->child->signal(SIGTERM);
CHECK(server->child->wait_exit(30s) == 0);
}
std::optional<ActionReceipt> receipt_of(Server& server, RegionSessionClient& c, std::uint64_t id) {
std::optional<ActionReceipt> found;
until(server, c, [&] {
for (const auto& r : c.take_action_receipts())
if (r.action_id == id)
found = r;
return found.has_value();
});
return found;
}
// "tick:commit" of the durable record of (account, id) in world 7, or empty.
std::string record(const AccountId& who, std::uint64_t id) {
const std::array<std::string, 2> p{hex(who), std::to_string(id)};
auto rows =
owner()->query("SELECT tick || ':' || commit_sequence FROM evolution.region_actions "
"WHERE world_id = 7 AND account_id = $1::uuid AND "
"action_id = $2::evolution.u64",
p, 5s);
CHECK(rows.has_value());
return rows->empty() ? std::string{} : (*rows)[0][0].value_or("");
}
void end_to_end() {
Setup s{151, 0xC5};
const auto alice = account(0x51), bob = account(0x52);
auto server = std::make_unique<Server>(s);
auto a = connect(s, *server, s.grant(alice, 1));
CHECK(established(*server, *a));
// Snapshots of the real region (ADR-0093): the default interest is the first habitat
// site with radius 7; every cell lies in one slice and one 15 x 15 square.
const auto first_tick = a->server_tick();
CHECK(until(*server, *a, [&] { return a->view() && !a->view()->environment.empty(); }));
const auto shown = a->environment();
CHECK(shown && !shown->cells().empty() && shown->cells().size() <= 225);
const auto [min_x, max_x] =
std::ranges::minmax(shown->cells(), {}, [](const auto& c) { return c.voxel.x; });
const auto [min_y, max_y] =
std::ranges::minmax(shown->cells(), {}, [](const auto& c) { return c.voxel.y; });
CHECK(max_x.voxel.x - min_x.voxel.x <= 14 && max_y.voxel.y - min_y.voxel.y <= 14);
CHECK(std::ranges::all_of(shown->cells(),
[&](const auto& c) { return c.voxel.z == shown->slice_z(); }));
CHECK(until(*server, *a, [&] { return a->server_tick() >= first_tick + 40; }));
// M10.9a (ADR-0097): the listener's load is exposed. These servers run with
// --m10-load-governor off: their host's speed is unknown (a shared CI runner measured
// 22 ms p95 for one session), so work is measured but nothing is refused or shed.
const auto sessions = metric(server->ops, "evo_m10_sessions");
const auto p95 = metric(server->ops, "evo_m10_tick_work_p95_microseconds");
std::printf("m10 load report sessions=%llu p95_us=%llu p99_us=%llu session_p99_us=%llu\n",
static_cast<unsigned long long>(sessions.value_or(0)),
static_cast<unsigned long long>(p95.value_or(0)),
static_cast<unsigned long long>(
metric(server->ops, "evo_m10_tick_work_p99_microseconds").value_or(0)),
static_cast<unsigned long long>(
metric(server->ops, "evo_m10_session_work_p99_microseconds").value_or(0)));
CHECK(sessions == 1u && p95.value_or(0) > 0 && metric(server->ops, "evo_m10_refusing") == 0u &&
metric(server->ops, "evo_m10_shed_level") == 0u &&
metric(server->ops, "evo_m10_refused_logins_total") == 0u);
std::printf("m10 snapshot report cells=%zu ticks=%llu chunks=%llu bytes=%llu\n",
shown->cells().size(),
static_cast<unsigned long long>(a->server_tick() - first_tick),
static_cast<unsigned long long>(a->snapshot_chunks()),
static_cast<unsigned long long>(a->snapshot_bytes()));
// Intentions: receipts, then the watermark.
// Datagrams may be refused until the QUIC datagram extension is negotiated: retry.
std::optional<std::uint64_t> sequence;
CHECK(until(*server, *a, [&] {
return a->server_tick() > 0 &&
(sequence = a->send_intention(a->server_tick() + 2, 100, 0)).has_value();
}));
CHECK(sequence == 1u);
std::vector<IntentionReceipt> receipts;
CHECK(until(*server, *a, [&] { return !(receipts = a->take_intention_receipts()).empty(); }));
CHECK(receipts[0].sequence == 1 && receipts[0].status == AdmitStatus::admitted);
CHECK(until(*server, *a, [&] { return a->simulated() && a->simulated()->sequence >= 1; }));
// Action: receipt, then a final outcome naming the durable commit.
CHECK(a->send_action(1, ActionKind::probe, a->server_tick() + 3));
const auto r1 = receipt_of(*server, *a, 1);
CHECK(r1 && must(r1).status == AdmitStatus::admitted);
const auto o1 = outcome_of(*server, *a, 1, true);
CHECK(o1 && must(o1).result == ActionResult::committed_no_effect &&
must(o1).tick == must(r1).tick && must(o1).commit_sequence > 0);
CHECK(record(alice, 1) ==
std::to_string(must(o1).tick) + ":" + std::to_string(must(o1).commit_sequence));
// SIGKILL after the commit (the record exists), whether or not the outcome was sent.
CHECK(a->send_action(2, ActionKind::probe, a->server_tick() + 3));
const auto r2 = receipt_of(*server, *a, 2);
CHECK(r2 && must(r2).status == AdmitStatus::admitted);
const bool recorded = until(*server, *a, [&] { return !record(alice, 2).empty(); });
if (!recorded)
for (const auto& line : server->child->lines())
if (starts(line, "m10") || starts(line, "phase") || starts(line, "dropped"))
std::printf("server: %s\n", line.c_str());
CHECK(recorded);
server->kill();
server = std::make_unique<Server>(s); // Waits out the old lease.
auto b = connect(s, *server, s.grant(alice, 2));
CHECK(established(*server, *b));
CHECK(b->query_action(2));
const auto o2 = outcome_of(*server, *b, 2, false);
CHECK(o2 && must(o2).result == ActionResult::committed_no_effect &&
record(alice, 2) ==
std::to_string(must(o2).tick) + ":" + std::to_string(must(o2).commit_sequence));
// A retry of a committed action on the new process: duplicate, same outcome.
CHECK(b->send_action(1, ActionKind::probe, b->server_tick() + 3));
const auto d1 = receipt_of(*server, *b, 1);
CHECK(d1 && must(d1).status == AdmitStatus::duplicate && must(d1).tick == must(o1).tick);
CHECK(outcome_of(*server, *b, 1, true) == o1);
// SIGKILL before the commit: the action's tick has not run when the process dies.
CHECK(b->send_action(3, ActionKind::probe, b->server_tick() + 8));
const auto r3 = receipt_of(*server, *b, 3);
CHECK(r3 && must(r3).status == AdmitStatus::admitted);
server->kill();
CHECK(record(alice, 3).empty());
server = std::make_unique<Server>(s);
auto c = connect(s, *server, s.grant(alice, 3));
CHECK(established(*server, *c));
CHECK(c->query_action(3));
const auto unknown = outcome_of(*server, *c, 3, false);
CHECK(unknown && unknown->result == ActionResult::unknown); // No record: never committed.
CHECK(c->send_action(3, ActionKind::probe, c->server_tick() + 3)); // Safe to retry.
CHECK(must(receipt_of(*server, *c, 3)).status == AdmitStatus::admitted);
const auto o3 = outcome_of(*server, *c, 3, true);
CHECK(o3 && must(o3).result == ActionResult::committed_no_effect && !record(alice, 3).empty());
// The headless probe: login, intentions, actions and a query, then a clean exit. The
// probe does not retry, so make sure the region accepts logins (governor off: at once).
CHECK(until(*server, *c, [&] { return metric(server->ops, "evo_m10_refusing") == 0u; }));
const auto probe = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-session-probe";
Child p{{probe.string(), "run",
"--connect", "127.0.0.1:" + std::to_string(server->port),
"--server-name", name,
"--ca", (s.region.root / "ca.pem").string(),
"--issuer-key", (s.region.root / "issuer.key").string(),
"--world", "7",
"--frame", std::to_string(region_identity.frame),
"--region", "151",
"--account", hex(bob),
"--serial", "1",
"--lifetime", "900", // Over the default 300 s: allowed by the server flag.
"--intents", "3",
"--actions", "2",
"--action-base", "10",
"--query", "10",
"--snapshots", "8"}};
int code = -1;
for (const auto end = Clock::now() + 30s; Clock::now() < end && code < 0;) {
server->pump();
p.pump(5ms);
if (p.printed("done") || p.printed("timeout") || p.printed("closed"))
code = p.wait_exit(10s);
}
const bool probed = code == 0 && p.printed("welcome ") && p.printed("intention 1 0") &&
p.printed("outcome 10 1 ") && p.printed("outcome 11 1 ") &&
p.printed("snapshot ") && p.printed("snapshot-bytes ");
if (!probed) {
for (const auto& line : p.lines())
if (!starts(line, "snapshot ") && !starts(line, "marker "))
std::printf("probe: %s\n", line.c_str());
for (const auto& line : server->child->lines())
if (starts(line, "m10 load") || starts(line, "m10 refused") || starts(line, "phase"))
std::printf("server: %s\n", line.c_str());
}
CHECK(probed);
CHECK(!record(bob, 10).empty() && !record(bob, 11).empty());
// The probe's marker (ADR-0094): its intentions (100, 0) were applied and moved it +x.
std::vector<std::array<long long, 4>> marks; // tick, x, y, applied
for (const auto& line : p.lines())
if (line.starts_with("marker ")) {
std::array<long long, 4> m{};
std::istringstream in{line.substr(7)};
in >> m[0] >> m[1] >> m[2] >> m[3];
if (in)
marks.push_back(m);
}
CHECK(marks.size() >= 2 && marks.back()[3] >= 1 && marks.back()[1] > marks.front()[1] &&
marks.back()[2] == marks.front()[2]);
// M10.9 (ADR-0097): a login from an existing bundle (as a dev renewal writes one), with
// no issuer key: grant 1 of a two-grant bundle for Dana.
const auto dana = account(0x53);
const auto bundle = s.region.root / "dana-grants.bin";
Child g{{probe.string(), "grants", "--issuer-key", (s.region.root / "issuer.key").string(),
"--world", "7", "--frame", std::to_string(region_identity.frame), "--region", "151",
"--account", hex(dana), "--count", "2", "--lifetime", "900", "--out",
bundle.string()}};
CHECK(g.wait_exit(30s) == 0);
Child q{{probe.string(), "run", "--connect", "127.0.0.1:" + std::to_string(server->port),
"--server-name", name, "--ca", (s.region.root / "ca.pem").string(), "--grants",
bundle.string(), "--grant", "1", "--intents", "2", "--snapshots", "2"}};
code = -1;
for (const auto end = Clock::now() + 30s; Clock::now() < end && code < 0;) {
server->pump();
q.pump(5ms);
if (q.printed("done") || q.printed("timeout") || q.printed("closed"))
code = q.wait_exit(10s);
}
CHECK(code == 0 && q.printed("grant 1 of 2") && q.printed("welcome ") &&
q.printed("intention 1 0"));
server->child->signal(SIGTERM);
CHECK(server->child->wait_exit(30s) == 0);
CHECK(server->child->printed("m10 opened ") && server->child->printed("m10 committed ") &&
server->child->printed("m10 motion area ") &&
server->child->printed("m10 load-report sessions="));
}
// M10.9b (ADR-0097): evo-load against a real evo-server, two steps with the region's
// metrics. A smoke test of the tool and the server under concurrent sessions, not
// qualification (only evolution-dev runs count): it checks that every bot logs in, that
// every step measures responses, outcomes and snapshots for its bots, and that the report
// is complete. The lines are kept in the log as a development reference.
void load_smoke() {
Setup s{152, 0xC6};
Server server{s};
const auto tool = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-load";
// M10.9d: the bots log in from a bundle (as the dev drill does): 8 accounts x 2 grants.
const auto probe = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-session-probe";
const auto bundle = s.region.root / "load-grants.bin";
Child g{{probe.string(), "grants", "--issuer-key", (s.region.root / "issuer.key").string(),
"--world", "7", "--frame", std::to_string(region_identity.frame),
"--region", "152", "--account", "0000000000000000000000000000b000",
"--accounts", "8", "--count", "2",
"--lifetime", "900", "--out", bundle.string()}};
CHECK(g.wait_exit(30s) == 0 && g.printed("wrote 16 grants "));
Child load{{tool.string(),
"--connect",
"127.0.0.1:" + std::to_string(server.port),
"--server-name",
name,
"--ca",
(s.region.root / "ca.pem").string(),
"--grants",
bundle.string(),
"--grants-per-bot",
"2",
"--world",
"7",
"--frame",
std::to_string(region_identity.frame),
"--region",
"152",
"--steps",
"4,8",
"--hold",
"8",
"--warmup",
"3",
"--ramp-ms",
"50",
"--threads",
"2",
"--lifetime",
"900",
"--metrics",
"127.0.0.1:" + std::to_string(server.ops),
"--per-client"}};
int code = -1;
for (const auto end = Clock::now() + 120s; Clock::now() < end && code < 0;) {
server.pump();
load.pump(5ms);
if (load.printed("load result "))
code = load.wait_exit(30s);
}
for (const auto& line : load.lines())
if (starts(line, "load "))
std::printf("%s\n", line.c_str());
CHECK(code == 0);
std::size_t steps = 0;
for (const auto& line : load.lines()) {
if (!starts(line, "load step "))
continue;
++steps;
// Every bot logged in once: 4 in the first step, 4 more in the second.
CHECK((starts(line, "load step N=4 established=4 logins=4 ") ||
starts(line, "load step N=8 established=8 logins=4 ")) &&
line.find(" drops=0 login_failures=0 refused=0 ") != std::string::npos &&
line.find(" exhausted=0 hold_utc=") != std::string::npos);
}
const auto has = [&](std::string_view prefix) {
return std::ranges::count_if(load.lines(), [&](const std::string& l) {
return starts(l, std::string{prefix});
}) == 2;
};
CHECK(steps == 2 && has("load response_ms ") && has("load outcome_ms ") &&
has("load snapshots ") && has("load server ") && has("load verdict "));
CHECK(std::ranges::none_of(load.lines(), [](const std::string& l) {
return starts(l, "load response_ms") && l.find(" n=0 ") != std::string::npos;
}));
CHECK(std::ranges::count_if(
load.lines(), [](const std::string& l) { return starts(l, "load client "); }) == 12);
server.child->signal(SIGTERM);
CHECK(server.child->wait_exit(30s) == 0);
}
// M10.9c-2 (ADR-0097): region authority lost under load. 24 evo-load bots on a real
// evo-server; the server is SIGKILLed (its lease must expire before a successor runs), then
// later drained with SIGTERM (sessions close with shutdown). Each time the successor takes
// the same port under a new epoch and every bot logs in again within 15 s of it being ready,
// and the bots count the drops. Loopback only; the bound is a development reference.
void lease_loss_under_load() {
Setup s{153, 0xC7};
s.listen = "127.0.0.1:" + std::to_string(free_port());
auto server = std::make_unique<Server>(s);
constexpr std::uint64_t bots = 24;
const auto tool = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-load";
Child load{{tool.string(),
"--connect",
"127.0.0.1:" + std::to_string(server->port),
"--server-name",
name,
"--ca",
(s.region.root / "ca.pem").string(),
"--issuer-key",
(s.region.root / "issuer.key").string(),
"--world",
"7",
"--frame",
std::to_string(region_identity.frame),
"--region",
"153",
"--steps",
std::to_string(bots),
"--hold",
"100",
"--warmup",
"2",
"--ramp-ms",
"50",
"--threads",
"4",
"--lifetime",
"900"}};
const auto pump = [&] {
server->pump();
load.pump(5ms);
};
// Milliseconds until the region reports `n` sessions, or nothing within `limit`.
const auto sessions_after =
[&](std::chrono::milliseconds limit) -> std::optional<std::chrono::milliseconds> {
const auto started = Clock::now();
for (int i = 0; Clock::now() - started < limit; ++i) {
pump();
if (i % 20 == 0 && metric(server->ops, "evo_m10_sessions") == bots)
return std::chrono::duration_cast<std::chrono::milliseconds>(Clock::now() -
started);
}
return std::nullopt;
};
CHECK(sessions_after(60s).has_value());
// A successor on the same port; how long until it is ready and every bot is back.
const auto successor = [&](const char* how) {
const auto lost = Clock::now();
server = std::make_unique<Server>(s); // Waits out the old lease, then "ready 1".
const auto ready =
std::chrono::duration_cast<std::chrono::milliseconds>(Clock::now() - lost);
const auto back = sessions_after(15s);
std::printf("lease loss %s ready_ms=%lld all_back_ms=%lld\n", how,
static_cast<long long>(ready.count()),
static_cast<long long>(back ? back->count() : -1));
return back.has_value();
};
server->kill();
CHECK(successor("sigkill"));
server->child->signal(SIGTERM);
CHECK(server->child->wait_exit(30s) == 0);
CHECK(successor("sigterm"));
// The bots measured both losses within their hold.
int code = -1;
for (const auto end = Clock::now() + 180s; Clock::now() < end && code < 0;) {
pump();
if (load.printed("load result "))
code = load.wait_exit(30s);
}
std::string step;
for (const auto& line : load.lines()) {
if (starts(line, "load step ") || starts(line, "load response_ms") ||
starts(line, "load login_ms"))
std::printf("%s\n", line.c_str());
if (starts(line, "load step "))
step = line;
}
CHECK(code == 0 && starts(step, "load step N=24 established=24 "));
const auto drops_at = step.find(" drops=");
CHECK(drops_at != std::string::npos &&
std::stoull(step.substr(drops_at + 7)) >= 2 * bots); // Both losses dropped every bot.
server->child->signal(SIGTERM);
CHECK(server->child->wait_exit(30s) == 0);
}
} // namespace
void run_region_m10_session_tests() {
end_to_end();
load_smoke();
lease_loss_under_load();
primitive_life_v5_process_round_trip();
std::puts("region m10 session integration passed");
}