Add M10.9d load drill harness and serialize Mbed TLS use
Foundation / checks (deploy) (push) Successful in 3m24s
Foundation / checks (ecs) (push) Successful in 1m56s
Foundation / checks (client) (push) Successful in 12m18s
Foundation / checks (network) (push) Successful in 13m30s
Foundation / checks (tsan) (push) Successful in 8m0s
Foundation / checks (checks) (push) Successful in 32m39s
Foundation / checks (postgres) (push) Successful in 45m47s
Foundation / postgres-tools (push) Successful in 17s
Foundation / image (push) Successful in 2m30s
Foundation / server-image (push) Successful in 4m8s
Foundation / checks (deploy) (push) Successful in 3m24s
Foundation / checks (ecs) (push) Successful in 1m56s
Foundation / checks (client) (push) Successful in 12m18s
Foundation / checks (network) (push) Successful in 13m30s
Foundation / checks (tsan) (push) Successful in 8m0s
Foundation / checks (checks) (push) Successful in 32m39s
Foundation / checks (postgres) (push) Successful in 45m47s
Foundation / postgres-tools (push) Successful in 17s
Foundation / image (push) Successful in 2m30s
Foundation / server-image (push) Successful in 4m8s
ADR-0097. tools/ops/load-drill.sh (runbook docs/networking/M10_LOAD_DRILL.md) runs the evolution-dev reference qualification: synthetic inputs on for the run (the ADR-0073 drill-time override, restored on exit), bundle-mode evo-load bots through the public route, and each step's hold matched with the server's reports. evo-server logs "m10 load-report" every 10 s, since the agent cannot reach the ops port. evo-load gains --grants FILE --grants-per-bot C, exhausted=, hold_utc= and close_reasons=; evo-session-probe grants gains --accounts N. Finding: Mbed TLS/PSA are built without MBEDTLS_THREADING_C, and bots on different threads verifying their first connections at once were refused as untrusted (two per smoke run). One process-wide recursive crypto lock now guards every network-library entry point that touches them; the smoke went from 2/2 failing to 3/3 clean. quic-transport adds a concurrent-connect test. Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -5,12 +5,15 @@
|
||||
#include "evo/network/dev_credentials.hpp"
|
||||
#include "udp_relay.hpp"
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <functional>
|
||||
#include <latch>
|
||||
#include <set>
|
||||
#include <source_location>
|
||||
#include <stdexcept>
|
||||
@@ -558,8 +561,67 @@ void garbage_and_close() {
|
||||
}
|
||||
} // namespace
|
||||
|
||||
// M10.9d (ADR-0097): client endpoints on several threads connecting at the same moment all
|
||||
// connect and pass the owned pin check. Mbed TLS runs without its threading layer; evo-load
|
||||
// saw two concurrent first verifications refused as `untrusted` before the shared crypto
|
||||
// lock. This test covers concurrent connects; it does not reliably reproduce that race (PSA
|
||||
// is already initialised here by the credential generation), which evo-load's smoke does.
|
||||
void concurrent_clients() {
|
||||
const auto f = credentials("concurrent");
|
||||
Limits limits;
|
||||
limits.max_connections = 64; // 5 rounds x 8, whatever the server has not yet reaped.
|
||||
auto server = serve(f, limits);
|
||||
constexpr int threads = 8;
|
||||
for (int round = 0; round < 5; ++round) {
|
||||
std::atomic<int> connected{0}, finished{0};
|
||||
std::array<std::atomic<int>, 8> causes{};
|
||||
std::latch start{threads};
|
||||
std::vector<std::thread> clients;
|
||||
clients.reserve(threads);
|
||||
for (int t = 0; t < threads; ++t)
|
||||
clients.emplace_back([&] {
|
||||
start.arrive_and_wait();
|
||||
auto client = dial(f, server->port());
|
||||
const auto end = std::chrono::steady_clock::now() + std::chrono::seconds{10};
|
||||
bool done = false;
|
||||
while (!done && std::chrono::steady_clock::now() < end) {
|
||||
for (const auto& e : client->poll()) {
|
||||
if (e.kind == EventKind::connected) {
|
||||
++connected;
|
||||
done = true;
|
||||
} else if (e.kind == EventKind::closed) {
|
||||
causes[static_cast<std::size_t>(e.cause)] += 1;
|
||||
done = true;
|
||||
}
|
||||
}
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds{1});
|
||||
}
|
||||
++finished;
|
||||
});
|
||||
while (finished < threads) {
|
||||
(void)server->poll();
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds{1});
|
||||
}
|
||||
for (auto& c : clients)
|
||||
c.join();
|
||||
if (connected != threads) {
|
||||
std::printf("concurrent round %d connected=%d", round, connected.load());
|
||||
for (std::size_t i = 0; i < causes.size(); ++i)
|
||||
std::printf(" cause%zu=%d", i, causes.at(i).load());
|
||||
std::printf("\n");
|
||||
}
|
||||
CHECK(connected == threads);
|
||||
for (int i = 0; i < 200; ++i) { // Let the server see the closes.
|
||||
(void)server->poll();
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds{1});
|
||||
}
|
||||
}
|
||||
std::filesystem::remove_all(f.dir);
|
||||
}
|
||||
|
||||
int main() try {
|
||||
framing();
|
||||
concurrent_clients();
|
||||
channels();
|
||||
pinning();
|
||||
impaired();
|
||||
|
||||
@@ -462,7 +462,8 @@ void end_to_end() {
|
||||
server->child->signal(SIGTERM);
|
||||
CHECK(server->child->wait_exit(30s) == 0);
|
||||
CHECK(server->child->printed("m10 opened ") && server->child->printed("m10 committed ") &&
|
||||
server->child->printed("m10 motion area "));
|
||||
server->child->printed("m10 motion area ") &&
|
||||
server->child->printed("m10 load-report sessions="));
|
||||
}
|
||||
// M10.9b (ADR-0097): evo-load against a real evo-server, two steps with the region's
|
||||
// metrics. A smoke test of the tool and the server under concurrent sessions, not
|
||||
@@ -473,6 +474,15 @@ void load_smoke() {
|
||||
Setup s{152, 0xC6};
|
||||
Server server{s};
|
||||
const auto tool = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-load";
|
||||
// M10.9d: the bots log in from a bundle (as the dev drill does): 8 accounts x 2 grants.
|
||||
const auto probe = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-session-probe";
|
||||
const auto bundle = s.region.root / "load-grants.bin";
|
||||
Child g{{probe.string(), "grants", "--issuer-key", (s.region.root / "issuer.key").string(),
|
||||
"--world", "7", "--frame", std::to_string(region_identity.frame),
|
||||
"--region", "152", "--account", "0000000000000000000000000000b000",
|
||||
"--accounts", "8", "--count", "2",
|
||||
"--lifetime", "900", "--out", bundle.string()}};
|
||||
CHECK(g.wait_exit(30s) == 0 && g.printed("wrote 16 grants "));
|
||||
Child load{{tool.string(),
|
||||
"--connect",
|
||||
"127.0.0.1:" + std::to_string(server.port),
|
||||
@@ -480,8 +490,10 @@ void load_smoke() {
|
||||
name,
|
||||
"--ca",
|
||||
(s.region.root / "ca.pem").string(),
|
||||
"--issuer-key",
|
||||
(s.region.root / "issuer.key").string(),
|
||||
"--grants",
|
||||
bundle.string(),
|
||||
"--grants-per-bot",
|
||||
"2",
|
||||
"--world",
|
||||
"7",
|
||||
"--frame",
|
||||
@@ -522,7 +534,8 @@ void load_smoke() {
|
||||
// Every bot logged in once: 4 in the first step, 4 more in the second.
|
||||
CHECK((starts(line, "load step N=4 established=4 logins=4 ") ||
|
||||
starts(line, "load step N=8 established=8 logins=4 ")) &&
|
||||
line.find(" drops=0 login_failures=0 refused=0 ") != std::string::npos);
|
||||
line.find(" drops=0 login_failures=0 refused=0 ") != std::string::npos &&
|
||||
line.find(" exhausted=0 hold_utc=") != std::string::npos);
|
||||
}
|
||||
const auto has = [&](std::string_view prefix) {
|
||||
return std::ranges::count_if(load.lines(), [&](const std::string& l) {
|
||||
|
||||
Reference in New Issue
Block a user