// Real process tests for M10.4c (ADR-0091, ADR-0092): evo-server with PostgreSQL, synthetic // ticks and the QUIC session listener; in-process RegionSessionClients and the headless // probe. Intentions get receipts and the simulated watermark; actions get durable outcomes // that survive SIGKILL after their commit, and resolve as "no record" (then commit once on // retry) after SIGKILL before it. Region 7/151. Run by tools/ci/postgres-check.sh // when the build has EVO_BUILD_QUIC. #include "evo/client/region_session.hpp" #include "evo/network/dev_credentials.hpp" #include "evo/network/session_grant.hpp" #include "fixtures/server_process.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include void run_region_m10_session_tests(); namespace { using namespace evo::test::process; using namespace evo; using namespace evo::network; using client::RegionSessionClient; using client::RegionSessionState; using evo::test::region_identity; void check(bool value, std::source_location at = std::source_location::current()) { if (!value) throw std::runtime_error("region m10 session test line " + std::to_string(at.line())); } #define CHECK(...) check(static_cast(__VA_ARGS__)) template const T& must(const std::optional& o, std::source_location at = std::source_location::current()) { if (!o) throw std::runtime_error("region m10 session missing value line " + std::to_string(at.line())); return *o; } constexpr const char* name = "dev-region.evolution.test"; void write_file(const fs::path& path, std::span bytes) { const int fd = ::open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0600); expect(fd >= 0 && ::write(fd, bytes.data(), bytes.size()) == static_cast(bytes.size()) && ::close(fd) == 0, "write file"); } void write_text(const fs::path& path, const std::string& text) { write_file(path, {reinterpret_cast(text.data()), text.size()}); } AccountId account(std::uint8_t tag) { AccountId a; a.bytes.fill(tag); return a; } std::string hex(const AccountId& a) { constexpr char digits[] = "0123456789abcdef"; std::string out; for (const auto b : a.bytes) { out.push_back(digits[b >> 4U]); out.push_back(digits[b & 0xFU]); } return out; } // GET /metrics from the ops listener; the value of an unlabelled series (ADR-0097 adds the // evo_m10_* families when the listener is enabled). std::optional metric(std::uint16_t port, const std::string& series) { const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); expect(fd >= 0, "socket"); sockaddr_in a{}; a.sin_family = AF_INET; a.sin_port = htons(port); a.sin_addr.s_addr = htonl(INADDR_LOOPBACK); timeval timeout{3, 0}; ::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout); std::string raw; if (::connect(fd, reinterpret_cast(&a), sizeof a) == 0) { const std::string request = "GET /metrics HTTP/1.1\r\nHost: test\r\n\r\n"; if (::send(fd, request.data(), request.size(), MSG_NOSIGNAL) == static_cast(request.size())) { std::array chunk{}; for (ssize_t n = 0; (n = ::recv(fd, chunk.data(), chunk.size(), 0)) > 0;) raw.append(chunk.data(), static_cast(n)); } } ::close(fd); const auto at = raw.find("\n" + series + " "); if (at == std::string::npos) return std::nullopt; return std::stoull(raw.substr(at + series.size() + 2)); } // A UDP port that was free a moment ago. std::uint16_t free_port() { const int fd = ::socket(AF_INET, SOCK_DGRAM | SOCK_CLOEXEC, 0); expect(fd >= 0, "socket"); sockaddr_in a{}; a.sin_family = AF_INET; a.sin_addr.s_addr = htonl(INADDR_LOOPBACK); socklen_t size = sizeof a; const bool ok = ::bind(fd, reinterpret_cast(&a), sizeof a) == 0 && ::getsockname(fd, reinterpret_cast(&a), &size) == 0; ::close(fd); expect(ok, "free port"); return ntohs(a.sin_port); } // One region's credentials, issuer key and server process arguments. struct Setup { ServerRegion region; std::string ca_pem; std::string listen{"127.0.0.1:0"}; // A fixed port lets clients reconnect after a restart. std::optional issuer; explicit Setup(std::uint64_t r, std::uint8_t genesis) : region{provision(r, genesis)} { const auto made = generate_dev_credentials(DeploymentEnvironment::dev, name, 1, std::chrono::system_clock::now()); expect(made.has_value(), "credentials"); ca_pem = made->authority_pem; write_text(region.root / "ca.pem", made->authority_pem); write_text(region.root / "server.pem", made->server_certificate_pem); write_text(region.root / "server.key", made->server_key_pem); for (int attempt = 0; attempt < 8 && !issuer; ++attempt) { std::array scalar{}; expect(random_bytes(scalar).has_value(), "random"); if (auto key = GrantSigningKey::import(DeploymentEnvironment::dev, 1, scalar)) { write_file(region.root / "issuer.key", scalar); write_file(region.root / "issuer.pub", *key->public_key()); issuer.emplace(std::move(*key)); } } expect(issuer.has_value(), "issuer key"); } [[nodiscard]] std::vector args() const { return server_args( region, {"--synthetic-inputs", "development-v1", "--m10-load-governor", "off", "--ops-listen", "127.0.0.1:0", "--m10-listen", listen, "--m10-cert", (region.root / "server.pem").string(), "--m10-key", (region.root / "server.key").string(), "--m10-issuer", (region.root / "issuer.pub").string(), "--m10-environment", "dev", "--m10-max-grant-lifetime", "900", "--session-window", "test-" + std::to_string(static_cast( std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count())) + ":" + std::to_string(static_cast( std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count()) + 1200)}); } [[nodiscard]] WireGrant grant(const AccountId& who, std::uint64_t serial) const { GrantClaims c; c.grant_id = 9000 + serial; c.account = who; c.serial = serial; c.world = {region.key.world, region_identity.frame}; c.region = region.key.region; c.issued_at = static_cast(std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count()); c.expires_at = c.issued_at + 120; const auto g = must(issuer).sign(c); expect(g.has_value(), "grant"); return *g; } }; // A running evo-server and its QUIC port, its output drained while waiting. struct Server { std::unique_ptr child; std::uint16_t port{}, ops{}; explicit Server(const Setup& s) : child{std::make_unique(s.args())} { CHECK(child->wait_line("m10 listening ", 30s)); for (const auto& l : child->lines()) { if (starts(l, "m10 listening ")) port = static_cast(std::stoul(l.substr(14))); if (starts(l, "ops listening ")) ops = static_cast(std::stoul(l.substr(14))); } CHECK(ops != 0); CHECK(port != 0 && child->wait_line("ready 1", 30s)); } void pump() { child->pump(1ms); } void kill() { child->signal(SIGKILL); CHECK(child->wait_exit(30s) == 128 + SIGKILL); } }; std::unique_ptr connect(const Setup& s, Server& server, const WireGrant& grant) { client::RegionSessionClientConfig c; c.transport.port = server.port; c.transport.server_name = name; c.transport.authority_pem = s.ca_pem; c.transport.authority_pem_file = (s.region.root / "ca.pem").string(); c.grant = grant; auto made = RegionSessionClient::connect(std::move(c), std::chrono::steady_clock::now()); CHECK(made.has_value()); return std::move(*made); } // Polls client and server until `done`, at most `limit`. bool until(Server& server, RegionSessionClient& c, const std::function& done, std::chrono::milliseconds limit = 20s) { const auto end = Clock::now() + limit; while (Clock::now() < end) { c.poll(Clock::now()); server.pump(); if (done()) return true; } return done(); } bool established(Server& server, RegionSessionClient& c) { return until(server, c, [&] { return c.state() == RegionSessionState::established || c.state() == RegionSessionState::closed; }) && c.state() == RegionSessionState::established; } std::optional outcome_of(Server& server, RegionSessionClient& c, std::uint64_t id, bool final_only) { std::optional found; until(server, c, [&] { for (const auto& o : c.take_action_outcomes()) if (o.action_id == id && (!final_only || o.result == ActionResult::committed_no_effect || o.result == ActionResult::not_committed)) found = o; return found.has_value(); }); return found; } std::optional receipt_of(Server& server, RegionSessionClient& c, std::uint64_t id) { std::optional found; until(server, c, [&] { for (const auto& r : c.take_action_receipts()) if (r.action_id == id) found = r; return found.has_value(); }); return found; } // "tick:commit" of the durable record of (account, id) in world 7, or empty. std::string record(const AccountId& who, std::uint64_t id) { const std::array p{hex(who), std::to_string(id)}; auto rows = owner()->query("SELECT tick || ':' || commit_sequence FROM evolution.region_actions " "WHERE world_id = 7 AND account_id = $1::uuid AND " "action_id = $2::evolution.u64", p, 5s); CHECK(rows.has_value()); return rows->empty() ? std::string{} : (*rows)[0][0].value_or(""); } void end_to_end() { Setup s{151, 0xC5}; const auto alice = account(0x51), bob = account(0x52); auto server = std::make_unique(s); auto a = connect(s, *server, s.grant(alice, 1)); CHECK(established(*server, *a)); // Snapshots of the real region (ADR-0093): the default interest is the first habitat // site with radius 7; every cell lies in one slice and one 15 x 15 square. const auto first_tick = a->server_tick(); CHECK(until(*server, *a, [&] { return a->view() && !a->view()->environment.empty(); })); const auto shown = a->environment(); CHECK(shown && !shown->cells().empty() && shown->cells().size() <= 225); const auto [min_x, max_x] = std::ranges::minmax(shown->cells(), {}, [](const auto& c) { return c.voxel.x; }); const auto [min_y, max_y] = std::ranges::minmax(shown->cells(), {}, [](const auto& c) { return c.voxel.y; }); CHECK(max_x.voxel.x - min_x.voxel.x <= 14 && max_y.voxel.y - min_y.voxel.y <= 14); CHECK(std::ranges::all_of(shown->cells(), [&](const auto& c) { return c.voxel.z == shown->slice_z(); })); CHECK(until(*server, *a, [&] { return a->server_tick() >= first_tick + 40; })); // M10.9a (ADR-0097): the listener's load is exposed. These servers run with // --m10-load-governor off: their host's speed is unknown (a shared CI runner measured // 22 ms p95 for one session), so work is measured but nothing is refused or shed. const auto sessions = metric(server->ops, "evo_m10_sessions"); const auto p95 = metric(server->ops, "evo_m10_tick_work_p95_microseconds"); std::printf("m10 load report sessions=%llu p95_us=%llu p99_us=%llu session_p99_us=%llu\n", static_cast(sessions.value_or(0)), static_cast(p95.value_or(0)), static_cast( metric(server->ops, "evo_m10_tick_work_p99_microseconds").value_or(0)), static_cast( metric(server->ops, "evo_m10_session_work_p99_microseconds").value_or(0))); CHECK(sessions == 1u && p95.value_or(0) > 0 && metric(server->ops, "evo_m10_refusing") == 0u && metric(server->ops, "evo_m10_shed_level") == 0u && metric(server->ops, "evo_m10_refused_logins_total") == 0u); std::printf("m10 snapshot report cells=%zu ticks=%llu chunks=%llu bytes=%llu\n", shown->cells().size(), static_cast(a->server_tick() - first_tick), static_cast(a->snapshot_chunks()), static_cast(a->snapshot_bytes())); // Intentions: receipts, then the watermark. // Datagrams may be refused until the QUIC datagram extension is negotiated: retry. std::optional sequence; CHECK(until(*server, *a, [&] { return a->server_tick() > 0 && (sequence = a->send_intention(a->server_tick() + 2, 100, 0)).has_value(); })); CHECK(sequence == 1u); std::vector receipts; CHECK(until(*server, *a, [&] { return !(receipts = a->take_intention_receipts()).empty(); })); CHECK(receipts[0].sequence == 1 && receipts[0].status == AdmitStatus::admitted); CHECK(until(*server, *a, [&] { return a->simulated() && a->simulated()->sequence >= 1; })); // Action: receipt, then a final outcome naming the durable commit. CHECK(a->send_action(1, ActionKind::probe, a->server_tick() + 3)); const auto r1 = receipt_of(*server, *a, 1); CHECK(r1 && must(r1).status == AdmitStatus::admitted); const auto o1 = outcome_of(*server, *a, 1, true); CHECK(o1 && must(o1).result == ActionResult::committed_no_effect && must(o1).tick == must(r1).tick && must(o1).commit_sequence > 0); CHECK(record(alice, 1) == std::to_string(must(o1).tick) + ":" + std::to_string(must(o1).commit_sequence)); // SIGKILL after the commit (the record exists), whether or not the outcome was sent. CHECK(a->send_action(2, ActionKind::probe, a->server_tick() + 3)); const auto r2 = receipt_of(*server, *a, 2); CHECK(r2 && must(r2).status == AdmitStatus::admitted); const bool recorded = until(*server, *a, [&] { return !record(alice, 2).empty(); }); if (!recorded) for (const auto& line : server->child->lines()) if (starts(line, "m10") || starts(line, "phase") || starts(line, "dropped")) std::printf("server: %s\n", line.c_str()); CHECK(recorded); server->kill(); server = std::make_unique(s); // Waits out the old lease. auto b = connect(s, *server, s.grant(alice, 2)); CHECK(established(*server, *b)); CHECK(b->query_action(2)); const auto o2 = outcome_of(*server, *b, 2, false); CHECK(o2 && must(o2).result == ActionResult::committed_no_effect && record(alice, 2) == std::to_string(must(o2).tick) + ":" + std::to_string(must(o2).commit_sequence)); // A retry of a committed action on the new process: duplicate, same outcome. CHECK(b->send_action(1, ActionKind::probe, b->server_tick() + 3)); const auto d1 = receipt_of(*server, *b, 1); CHECK(d1 && must(d1).status == AdmitStatus::duplicate && must(d1).tick == must(o1).tick); CHECK(outcome_of(*server, *b, 1, true) == o1); // SIGKILL before the commit: the action's tick has not run when the process dies. CHECK(b->send_action(3, ActionKind::probe, b->server_tick() + 8)); const auto r3 = receipt_of(*server, *b, 3); CHECK(r3 && must(r3).status == AdmitStatus::admitted); server->kill(); CHECK(record(alice, 3).empty()); server = std::make_unique(s); auto c = connect(s, *server, s.grant(alice, 3)); CHECK(established(*server, *c)); CHECK(c->query_action(3)); const auto unknown = outcome_of(*server, *c, 3, false); CHECK(unknown && unknown->result == ActionResult::unknown); // No record: never committed. CHECK(c->send_action(3, ActionKind::probe, c->server_tick() + 3)); // Safe to retry. CHECK(must(receipt_of(*server, *c, 3)).status == AdmitStatus::admitted); const auto o3 = outcome_of(*server, *c, 3, true); CHECK(o3 && must(o3).result == ActionResult::committed_no_effect && !record(alice, 3).empty()); // The headless probe: login, intentions, actions and a query, then a clean exit. The // probe does not retry, so make sure the region accepts logins (governor off: at once). CHECK(until(*server, *c, [&] { return metric(server->ops, "evo_m10_refusing") == 0u; })); const auto probe = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-session-probe"; Child p{{probe.string(), "run", "--connect", "127.0.0.1:" + std::to_string(server->port), "--server-name", name, "--ca", (s.region.root / "ca.pem").string(), "--issuer-key", (s.region.root / "issuer.key").string(), "--world", "7", "--frame", std::to_string(region_identity.frame), "--region", "151", "--account", hex(bob), "--serial", "1", "--lifetime", "900", // Over the default 300 s: allowed by the server flag. "--intents", "3", "--actions", "2", "--action-base", "10", "--query", "10", "--snapshots", "8"}}; int code = -1; for (const auto end = Clock::now() + 30s; Clock::now() < end && code < 0;) { server->pump(); p.pump(5ms); if (p.printed("done") || p.printed("timeout") || p.printed("closed")) code = p.wait_exit(10s); } const bool probed = code == 0 && p.printed("welcome ") && p.printed("intention 1 0") && p.printed("outcome 10 1 ") && p.printed("outcome 11 1 ") && p.printed("snapshot ") && p.printed("snapshot-bytes "); if (!probed) { for (const auto& line : p.lines()) if (!starts(line, "snapshot ") && !starts(line, "marker ")) std::printf("probe: %s\n", line.c_str()); for (const auto& line : server->child->lines()) if (starts(line, "m10 load") || starts(line, "m10 refused") || starts(line, "phase")) std::printf("server: %s\n", line.c_str()); } CHECK(probed); CHECK(!record(bob, 10).empty() && !record(bob, 11).empty()); // The probe's marker (ADR-0094): its intentions (100, 0) were applied and moved it +x. std::vector> marks; // tick, x, y, applied for (const auto& line : p.lines()) if (line.starts_with("marker ")) { std::array m{}; std::istringstream in{line.substr(7)}; in >> m[0] >> m[1] >> m[2] >> m[3]; if (in) marks.push_back(m); } CHECK(marks.size() >= 2 && marks.back()[3] >= 1 && marks.back()[1] > marks.front()[1] && marks.back()[2] == marks.front()[2]); // M10.9 (ADR-0097): a login from an existing bundle (as a dev renewal writes one), with // no issuer key: grant 1 of a two-grant bundle for Dana. const auto dana = account(0x53); const auto bundle = s.region.root / "dana-grants.bin"; Child g{{probe.string(), "grants", "--issuer-key", (s.region.root / "issuer.key").string(), "--world", "7", "--frame", std::to_string(region_identity.frame), "--region", "151", "--account", hex(dana), "--count", "2", "--lifetime", "900", "--out", bundle.string()}}; CHECK(g.wait_exit(30s) == 0); Child q{{probe.string(), "run", "--connect", "127.0.0.1:" + std::to_string(server->port), "--server-name", name, "--ca", (s.region.root / "ca.pem").string(), "--grants", bundle.string(), "--grant", "1", "--intents", "2", "--snapshots", "2"}}; code = -1; for (const auto end = Clock::now() + 30s; Clock::now() < end && code < 0;) { server->pump(); q.pump(5ms); if (q.printed("done") || q.printed("timeout") || q.printed("closed")) code = q.wait_exit(10s); } CHECK(code == 0 && q.printed("grant 1 of 2") && q.printed("welcome ") && q.printed("intention 1 0")); server->child->signal(SIGTERM); CHECK(server->child->wait_exit(30s) == 0); CHECK(server->child->printed("m10 opened ") && server->child->printed("m10 committed ") && server->child->printed("m10 motion area ") && server->child->printed("m10 load-report sessions=")); } // M10.9b (ADR-0097): evo-load against a real evo-server, two steps with the region's // metrics. A smoke test of the tool and the server under concurrent sessions, not // qualification (only evolution-dev runs count): it checks that every bot logs in, that // every step measures responses, outcomes and snapshots for its bots, and that the report // is complete. The lines are kept in the log as a development reference. void load_smoke() { Setup s{152, 0xC6}; Server server{s}; const auto tool = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-load"; // M10.9d: the bots log in from a bundle (as the dev drill does): 8 accounts x 2 grants. const auto probe = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-session-probe"; const auto bundle = s.region.root / "load-grants.bin"; Child g{{probe.string(), "grants", "--issuer-key", (s.region.root / "issuer.key").string(), "--world", "7", "--frame", std::to_string(region_identity.frame), "--region", "152", "--account", "0000000000000000000000000000b000", "--accounts", "8", "--count", "2", "--lifetime", "900", "--out", bundle.string()}}; CHECK(g.wait_exit(30s) == 0 && g.printed("wrote 16 grants ")); Child load{{tool.string(), "--connect", "127.0.0.1:" + std::to_string(server.port), "--server-name", name, "--ca", (s.region.root / "ca.pem").string(), "--grants", bundle.string(), "--grants-per-bot", "2", "--world", "7", "--frame", std::to_string(region_identity.frame), "--region", "152", "--steps", "4,8", "--hold", "8", "--warmup", "3", "--ramp-ms", "50", "--threads", "2", "--lifetime", "900", "--metrics", "127.0.0.1:" + std::to_string(server.ops), "--per-client"}}; int code = -1; for (const auto end = Clock::now() + 120s; Clock::now() < end && code < 0;) { server.pump(); load.pump(5ms); if (load.printed("load result ")) code = load.wait_exit(30s); } for (const auto& line : load.lines()) if (starts(line, "load ")) std::printf("%s\n", line.c_str()); CHECK(code == 0); std::size_t steps = 0; for (const auto& line : load.lines()) { if (!starts(line, "load step ")) continue; ++steps; // Every bot logged in once: 4 in the first step, 4 more in the second. CHECK((starts(line, "load step N=4 established=4 logins=4 ") || starts(line, "load step N=8 established=8 logins=4 ")) && line.find(" drops=0 login_failures=0 refused=0 ") != std::string::npos && line.find(" exhausted=0 hold_utc=") != std::string::npos); } const auto has = [&](std::string_view prefix) { return std::ranges::count_if(load.lines(), [&](const std::string& l) { return starts(l, std::string{prefix}); }) == 2; }; CHECK(steps == 2 && has("load response_ms ") && has("load outcome_ms ") && has("load snapshots ") && has("load server ") && has("load verdict ")); CHECK(std::ranges::none_of(load.lines(), [](const std::string& l) { return starts(l, "load response_ms") && l.find(" n=0 ") != std::string::npos; })); CHECK(std::ranges::count_if( load.lines(), [](const std::string& l) { return starts(l, "load client "); }) == 12); server.child->signal(SIGTERM); CHECK(server.child->wait_exit(30s) == 0); } // M10.9c-2 (ADR-0097): region authority lost under load. 24 evo-load bots on a real // evo-server; the server is SIGKILLed (its lease must expire before a successor runs), then // later drained with SIGTERM (sessions close with shutdown). Each time the successor takes // the same port under a new epoch and every bot logs in again within 15 s of it being ready, // and the bots count the drops. Loopback only; the bound is a development reference. void lease_loss_under_load() { Setup s{153, 0xC7}; s.listen = "127.0.0.1:" + std::to_string(free_port()); auto server = std::make_unique(s); constexpr std::uint64_t bots = 24; const auto tool = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-load"; Child load{{tool.string(), "--connect", "127.0.0.1:" + std::to_string(server->port), "--server-name", name, "--ca", (s.region.root / "ca.pem").string(), "--issuer-key", (s.region.root / "issuer.key").string(), "--world", "7", "--frame", std::to_string(region_identity.frame), "--region", "153", "--steps", std::to_string(bots), "--hold", "100", "--warmup", "2", "--ramp-ms", "50", "--threads", "4", "--lifetime", "900"}}; const auto pump = [&] { server->pump(); load.pump(5ms); }; // Milliseconds until the region reports `n` sessions, or nothing within `limit`. const auto sessions_after = [&](std::chrono::milliseconds limit) -> std::optional { const auto started = Clock::now(); for (int i = 0; Clock::now() - started < limit; ++i) { pump(); if (i % 20 == 0 && metric(server->ops, "evo_m10_sessions") == bots) return std::chrono::duration_cast(Clock::now() - started); } return std::nullopt; }; CHECK(sessions_after(60s).has_value()); // A successor on the same port; how long until it is ready and every bot is back. const auto successor = [&](const char* how) { const auto lost = Clock::now(); server = std::make_unique(s); // Waits out the old lease, then "ready 1". const auto ready = std::chrono::duration_cast(Clock::now() - lost); const auto back = sessions_after(15s); std::printf("lease loss %s ready_ms=%lld all_back_ms=%lld\n", how, static_cast(ready.count()), static_cast(back ? back->count() : -1)); return back.has_value(); }; server->kill(); CHECK(successor("sigkill")); server->child->signal(SIGTERM); CHECK(server->child->wait_exit(30s) == 0); CHECK(successor("sigterm")); // The bots measured both losses within their hold. int code = -1; for (const auto end = Clock::now() + 180s; Clock::now() < end && code < 0;) { pump(); if (load.printed("load result ")) code = load.wait_exit(30s); } std::string step; for (const auto& line : load.lines()) { if (starts(line, "load step ") || starts(line, "load response_ms") || starts(line, "load login_ms")) std::printf("%s\n", line.c_str()); if (starts(line, "load step ")) step = line; } CHECK(code == 0 && starts(step, "load step N=24 established=24 ")); const auto drops_at = step.find(" drops="); CHECK(drops_at != std::string::npos && std::stoull(step.substr(drops_at + 7)) >= 2 * bots); // Both losses dropped every bot. server->child->signal(SIGTERM); CHECK(server->child->wait_exit(30s) == 0); } } // namespace void run_region_m10_session_tests() { end_to_end(); load_smoke(); lease_loss_under_load(); std::puts("region m10 session integration passed"); }