Add M10.9d load drill harness and serialize Mbed TLS use
Foundation / checks (deploy) (push) Successful in 3m24s
Foundation / checks (ecs) (push) Successful in 1m56s
Foundation / checks (client) (push) Successful in 12m18s
Foundation / checks (network) (push) Successful in 13m30s
Foundation / checks (tsan) (push) Successful in 8m0s
Foundation / checks (checks) (push) Successful in 32m39s
Foundation / checks (postgres) (push) Successful in 45m47s
Foundation / postgres-tools (push) Successful in 17s
Foundation / image (push) Successful in 2m30s
Foundation / server-image (push) Successful in 4m8s

ADR-0097. tools/ops/load-drill.sh (runbook docs/networking/M10_LOAD_DRILL.md)
runs the evolution-dev reference qualification: synthetic inputs on for
the run (the ADR-0073 drill-time override, restored on exit), bundle-mode
evo-load bots through the public route, and each step's hold matched with
the server's reports. evo-server logs "m10 load-report" every 10 s, since
the agent cannot reach the ops port. evo-load gains --grants FILE
--grants-per-bot C, exhausted=, hold_utc= and close_reasons=;
evo-session-probe grants gains --accounts N.

Finding: Mbed TLS/PSA are built without MBEDTLS_THREADING_C, and bots on
different threads verifying their first connections at once were refused
as untrusted (two per smoke run). One process-wide recursive crypto lock
now guards every network-library entry point that touches them; the
smoke went from 2/2 failing to 3/3 clean. quic-transport adds a
concurrent-connect test. Refreshes the dev session window.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-08 08:38:13 -04:00
co-authored by Claude Opus 5.5
parent 47b84bce74
commit ada77fd761
16 changed files with 609 additions and 70 deletions
+62
View File
@@ -5,12 +5,15 @@
#include "evo/network/dev_credentials.hpp"
#include "udp_relay.hpp"
#include <algorithm>
#include <array>
#include <atomic>
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <functional>
#include <latch>
#include <set>
#include <source_location>
#include <stdexcept>
@@ -558,8 +561,67 @@ void garbage_and_close() {
}
} // namespace
// M10.9d (ADR-0097): client endpoints on several threads connecting at the same moment all
// connect and pass the owned pin check. Mbed TLS runs without its threading layer; evo-load
// saw two concurrent first verifications refused as `untrusted` before the shared crypto
// lock. This test covers concurrent connects; it does not reliably reproduce that race (PSA
// is already initialised here by the credential generation), which evo-load's smoke does.
void concurrent_clients() {
const auto f = credentials("concurrent");
Limits limits;
limits.max_connections = 64; // 5 rounds x 8, whatever the server has not yet reaped.
auto server = serve(f, limits);
constexpr int threads = 8;
for (int round = 0; round < 5; ++round) {
std::atomic<int> connected{0}, finished{0};
std::array<std::atomic<int>, 8> causes{};
std::latch start{threads};
std::vector<std::thread> clients;
clients.reserve(threads);
for (int t = 0; t < threads; ++t)
clients.emplace_back([&] {
start.arrive_and_wait();
auto client = dial(f, server->port());
const auto end = std::chrono::steady_clock::now() + std::chrono::seconds{10};
bool done = false;
while (!done && std::chrono::steady_clock::now() < end) {
for (const auto& e : client->poll()) {
if (e.kind == EventKind::connected) {
++connected;
done = true;
} else if (e.kind == EventKind::closed) {
causes[static_cast<std::size_t>(e.cause)] += 1;
done = true;
}
}
std::this_thread::sleep_for(std::chrono::milliseconds{1});
}
++finished;
});
while (finished < threads) {
(void)server->poll();
std::this_thread::sleep_for(std::chrono::milliseconds{1});
}
for (auto& c : clients)
c.join();
if (connected != threads) {
std::printf("concurrent round %d connected=%d", round, connected.load());
for (std::size_t i = 0; i < causes.size(); ++i)
std::printf(" cause%zu=%d", i, causes.at(i).load());
std::printf("\n");
}
CHECK(connected == threads);
for (int i = 0; i < 200; ++i) { // Let the server see the closes.
(void)server->poll();
std::this_thread::sleep_for(std::chrono::milliseconds{1});
}
}
std::filesystem::remove_all(f.dir);
}
int main() try {
framing();
concurrent_clients();
channels();
pinning();
impaired();
+17 -4
View File
@@ -462,7 +462,8 @@ void end_to_end() {
server->child->signal(SIGTERM);
CHECK(server->child->wait_exit(30s) == 0);
CHECK(server->child->printed("m10 opened ") && server->child->printed("m10 committed ") &&
server->child->printed("m10 motion area "));
server->child->printed("m10 motion area ") &&
server->child->printed("m10 load-report sessions="));
}
// M10.9b (ADR-0097): evo-load against a real evo-server, two steps with the region's
// metrics. A smoke test of the tool and the server under concurrent sessions, not
@@ -473,6 +474,15 @@ void load_smoke() {
Setup s{152, 0xC6};
Server server{s};
const auto tool = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-load";
// M10.9d: the bots log in from a bundle (as the dev drill does): 8 accounts x 2 grants.
const auto probe = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-session-probe";
const auto bundle = s.region.root / "load-grants.bin";
Child g{{probe.string(), "grants", "--issuer-key", (s.region.root / "issuer.key").string(),
"--world", "7", "--frame", std::to_string(region_identity.frame),
"--region", "152", "--account", "0000000000000000000000000000b000",
"--accounts", "8", "--count", "2",
"--lifetime", "900", "--out", bundle.string()}};
CHECK(g.wait_exit(30s) == 0 && g.printed("wrote 16 grants "));
Child load{{tool.string(),
"--connect",
"127.0.0.1:" + std::to_string(server.port),
@@ -480,8 +490,10 @@ void load_smoke() {
name,
"--ca",
(s.region.root / "ca.pem").string(),
"--issuer-key",
(s.region.root / "issuer.key").string(),
"--grants",
bundle.string(),
"--grants-per-bot",
"2",
"--world",
"7",
"--frame",
@@ -522,7 +534,8 @@ void load_smoke() {
// Every bot logged in once: 4 in the first step, 4 more in the second.
CHECK((starts(line, "load step N=4 established=4 logins=4 ") ||
starts(line, "load step N=8 established=8 logins=4 ")) &&
line.find(" drops=0 login_failures=0 refused=0 ") != std::string::npos);
line.find(" drops=0 login_failures=0 refused=0 ") != std::string::npos &&
line.find(" exhausted=0 hold_utc=") != std::string::npos);
}
const auto has = [&](std::string_view prefix) {
return std::ranges::count_if(load.lines(), [&](const std::string& l) {