Add M10.9d load drill harness and serialize Mbed TLS use
Foundation / checks (deploy) (push) Successful in 3m24s
Foundation / checks (ecs) (push) Successful in 1m56s
Foundation / checks (client) (push) Successful in 12m18s
Foundation / checks (network) (push) Successful in 13m30s
Foundation / checks (tsan) (push) Successful in 8m0s
Foundation / checks (checks) (push) Successful in 32m39s
Foundation / checks (postgres) (push) Successful in 45m47s
Foundation / postgres-tools (push) Successful in 17s
Foundation / image (push) Successful in 2m30s
Foundation / server-image (push) Successful in 4m8s

ADR-0097. tools/ops/load-drill.sh (runbook docs/networking/M10_LOAD_DRILL.md)
runs the evolution-dev reference qualification: synthetic inputs on for
the run (the ADR-0073 drill-time override, restored on exit), bundle-mode
evo-load bots through the public route, and each step's hold matched with
the server's reports. evo-server logs "m10 load-report" every 10 s, since
the agent cannot reach the ops port. evo-load gains --grants FILE
--grants-per-bot C, exhausted=, hold_utc= and close_reasons=;
evo-session-probe grants gains --accounts N.

Finding: Mbed TLS/PSA are built without MBEDTLS_THREADING_C, and bots on
different threads verifying their first connections at once were refused
as untrusted (two per smoke run). One process-wide recursive crypto lock
now guards every network-library entry point that touches them; the
smoke went from 2/2 failing to 3/3 clean. quic-transport adds a
concurrent-connect test. Refreshes the dev session window.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-08 08:38:13 -04:00
co-authored by Claude Opus 5.5
parent 47b84bce74
commit ada77fd761
16 changed files with 609 additions and 70 deletions
+107 -33
View File
@@ -2,7 +2,8 @@
// runs against the evolution-dev reference worker count as evidence (10.9d); local runs are
// smoke tests and development references.
//
// evo-load --connect HOST:PORT --server-name NAME --ca FILE --issuer-key FILE
// evo-load --connect HOST:PORT --server-name NAME --ca FILE
// (--issuer-key FILE | --grants FILE --grants-per-bot C)
// --world N --frame N --region N [--steps N[,N]...] [--hold S] [--warmup S]
// [--ramp-ms MS] [--threads T] [--seed X] [--lifetime S] [--metrics HOST:PORT]
// [--response-budget-ms MS] [--per-client] [--expect-established]
@@ -14,11 +15,17 @@
// probe marker; every 2 s it sends a probe action; every 10 s it moves its interest by a
// cell (a fresh baseline). A closed or refused bot logs in again after 2 s with the next
// serial.
// M10.9d: with --grants (a bundle from `evo-session-probe grants --accounts N --count C`,
// e.g. made where the dev issuer key lives), bot i logs in with grants i*C .. i*C+C-1 in
// order instead of signing; a bot that used all C stays out (counted as exhausted).
//
// Steps: the bot count rises to each step's N (one login every --ramp-ms), waits --warmup
// seconds, then measures for --hold seconds. Bots stay connected into the next step. Per
// step it prints:
// load step N=.. established=.. logins=.. drops=.. login_failures=.. refused=.. seconds=..
// exhausted=.. hold_utc=START..END (unix seconds of the hold, to match server samples)
// close_reasons=REASON:COUNT,... (session-v1 close reasons; without a session Close,
// 256 + the QUIC transport cause, or 255 when unknown)
// (drops: established sessions closed; login failures: closed before Welcome, including
// transport refusals at the connection limit; refused: closed with capacity or
// rate_limited, either way. Logins, drops and failures include the step's ramp and warmup;
@@ -111,7 +118,8 @@ double ms(std::uint64_t us) {
}
struct Options {
std::string host, server_name, ca, issuer_key, metrics_host;
std::string host, server_name, ca, issuer_key, grants_file, metrics_host;
std::uint64_t grants_per_bot{};
std::uint16_t port{}, metrics_port{};
std::uint64_t world{}, frame{}, region{}, seed{1}, lifetime{900};
std::vector<std::size_t> steps{20};
@@ -125,6 +133,7 @@ struct Samples {
gap_ticks;
std::uint64_t snapshots{}, bytes{}, intents{}, refused_intents{}, logins{}, drops{},
login_failures{}, refused{};
std::map<unsigned, std::uint64_t> closes; // By session close reason (255: no Close).
std::uint64_t connected_polls{}; // Polls while established (starvation is judged on these).
};
void append(std::vector<std::uint64_t>& to, const std::vector<std::uint64_t>& from) {
@@ -142,6 +151,8 @@ struct Bot {
client::InputLead lead;
std::uint64_t last_tick{}, last_view{}, last_bytes{}, next_action_id{1};
std::uint64_t applied{}; // Newest applied intention sequence seen in an own marker.
std::uint64_t used{}; // Bundle grants used (bundle mode).
std::atomic<bool> exhausted{false};
std::map<std::uint64_t, Clock::time_point> receipt_pending, response_pending, outcome_pending;
std::map<std::uint64_t, std::uint64_t> intent_tick;
std::int16_t vx{}, vy{};
@@ -153,9 +164,11 @@ struct Bot {
class Load {
public:
Load(Options o, GrantSigningKey key, std::string address)
: o_{std::move(o)}, key_{std::move(key)}, address_{std::move(address)},
authority_{read(o_.ca)} {
// Signs with `key`, or logs in from `bundle` (concatenated grants, grants_per_bot each).
Load(Options o, std::optional<GrantSigningKey> key, std::vector<SessionGrant> bundle,
std::string address)
: o_{std::move(o)}, key_{std::move(key)}, bundle_{std::move(bundle)},
address_{std::move(address)}, authority_{read(o_.ca)} {
const auto most = *std::ranges::max_element(o_.steps);
bots_.reserve(most);
for (std::size_t i = 0; i < most; ++i) {
@@ -199,6 +212,11 @@ class Load {
}
return out;
}
[[nodiscard]] std::size_t exhausted(std::size_t n) const {
return static_cast<std::size_t>(
std::count_if(bots_.begin(), bots_.begin() + static_cast<std::ptrdiff_t>(n),
[](const auto& b) { return b->exhausted.load(); }));
}
[[nodiscard]] std::size_t established(std::size_t n) const {
return static_cast<std::size_t>(
std::count_if(bots_.begin(), bots_.begin() + static_cast<std::ptrdiff_t>(n),
@@ -229,15 +247,26 @@ class Load {
}
}
void login(Bot& b, Clock::time_point now) {
GrantClaims claims;
claims.account = b.account;
claims.serial = ++b.serial;
claims.world = {o_.world, o_.frame};
claims.region = o_.region;
claims.issued_at = now_seconds();
claims.expires_at = claims.issued_at + o_.lifetime;
claims.grant_id = (b.serial << 16U) ^ (b.index + 1);
const auto grant = key_.sign(claims);
std::optional<SessionGrant> grant;
if (key_) {
GrantClaims claims;
claims.account = b.account;
claims.serial = ++b.serial;
claims.world = {o_.world, o_.frame};
claims.region = o_.region;
claims.issued_at = now_seconds();
claims.expires_at = claims.issued_at + o_.lifetime;
claims.grant_id = (b.serial << 16U) ^ (b.index + 1);
if (auto signed_grant = key_->sign(claims))
grant = *signed_grant;
} else {
if (b.used >= o_.grants_per_bot) {
b.exhausted = true; // Out of grants: stays out.
b.retry_at = Clock::time_point::max();
return;
}
grant = bundle_.at((b.index * o_.grants_per_bot) + b.used++);
}
client::RegionSessionClientConfig config;
config.transport.server_address = address_;
config.transport.port = o_.port;
@@ -278,6 +307,13 @@ class Load {
++b.s.drops;
else
++b.s.login_failures;
if (const auto reason = c.close_reason();
reason && *reason != SessionCloseReason::connection_lost)
++b.s.closes[static_cast<unsigned>(*reason)];
else if (const auto cause = c.transport_cause()) // No session Close: 256 + cause.
++b.s.closes[256U + static_cast<unsigned>(*cause)];
else
++b.s.closes[255U];
if (c.close_reason() == SessionCloseReason::capacity ||
c.close_reason() == SessionCloseReason::rate_limited)
++b.s.refused;
@@ -406,7 +442,8 @@ class Load {
}
Options o_;
GrantSigningKey key_;
std::optional<GrantSigningKey> key_;
std::vector<SessionGrant> bundle_;
std::string address_, authority_;
std::vector<std::unique_ptr<Bot>> bots_;
std::vector<std::mutex> locks_;
@@ -481,9 +518,9 @@ void line(const char* name, const std::vector<std::uint64_t>& pooled,
}
// One step's report; true when within budget.
bool report(const Options& o, std::size_t n, std::size_t established,
const std::vector<Samples>& all, double seconds, double cpu,
const std::map<std::string, std::uint64_t>& server) {
bool report(const Options& o, std::size_t n, std::size_t established, std::size_t exhausted,
std::pair<std::uint64_t, std::uint64_t> hold_utc, const std::vector<Samples>& all,
double seconds, double cpu, const std::map<std::string, std::uint64_t>& server) {
Samples pooled;
std::vector<std::uint64_t> response95, receipt95, outcome95, login95, correction95, rates, kib;
for (const auto& s : all) {
@@ -499,6 +536,8 @@ bool report(const Options& o, std::size_t n, std::size_t established,
pooled.logins += s.logins;
pooled.drops += s.drops;
pooled.login_failures += s.login_failures;
for (const auto& [why, count] : s.closes)
pooled.closes[why] += count;
pooled.refused += s.refused;
response95.push_back(percentile(s.response_us, 0.95));
receipt95.push_back(percentile(s.receipt_us, 0.95));
@@ -511,12 +550,18 @@ bool report(const Options& o, std::size_t n, std::size_t established,
kib.push_back(static_cast<std::uint64_t>(
std::lround(static_cast<double>(s.bytes) / 1024.0 / std::max(seconds, 0.001))));
}
std::string reasons;
for (const auto& [why, count] : pooled.closes)
reasons += (reasons.empty() ? "" : ",") + std::to_string(why) + ":" + std::to_string(count);
std::printf("load step N=%zu established=%zu logins=%llu drops=%llu login_failures=%llu "
"refused=%llu seconds=%.1f\n",
"refused=%llu seconds=%.1f exhausted=%zu hold_utc=%llu..%llu close_reasons=%s\n",
n, established, static_cast<unsigned long long>(pooled.logins),
static_cast<unsigned long long>(pooled.drops),
static_cast<unsigned long long>(pooled.login_failures),
static_cast<unsigned long long>(pooled.refused), seconds);
static_cast<unsigned long long>(pooled.refused), seconds, exhausted,
static_cast<unsigned long long>(hold_utc.first),
static_cast<unsigned long long>(hold_utc.second),
reasons.empty() ? "-" : reasons.c_str());
line("response", pooled.response_us, response95, 1000.0, "ms");
line("receipt", pooled.receipt_us, receipt95, 1000.0, "ms");
line("outcome", pooled.outcome_us, outcome95, 1000.0, "ms");
@@ -663,6 +708,10 @@ int main(int argc, char** argv) {
o.ca = value;
else if (flag == "--issuer-key")
o.issuer_key = value;
else if (flag == "--grants")
o.grants_file = value;
else if (n && flag == "--grants-per-bot" && *n >= 1 && *n <= 64)
o.grants_per_bot = *n;
else if (flag == "--steps") {
const auto s = steps_of(value);
ok = s.has_value();
@@ -676,7 +725,7 @@ int main(int argc, char** argv) {
o.region = *n;
else if (n && flag == "--seed" && *n != 0)
o.seed = *n;
else if (n && flag == "--lifetime" && *n >= 60 && *n <= 3600)
else if (n && flag == "--lifetime" && *n >= 60 && *n <= 14400)
o.lifetime = *n;
else if (n && flag == "--hold" && *n >= 1 && *n <= 3600)
o.hold_s = *n;
@@ -691,23 +740,44 @@ int main(int argc, char** argv) {
else
ok = false;
}
if (!ok || o.host.empty() || o.server_name.empty() || o.ca.empty() || o.issuer_key.empty() ||
o.world == 0 || o.frame == 0 || o.region == 0) {
const bool signs = !o.issuer_key.empty();
if (!ok || o.host.empty() || o.server_name.empty() || o.ca.empty() ||
signs == !o.grants_file.empty() || (!signs && o.grants_per_bot == 0) || o.world == 0 ||
o.frame == 0 || o.region == 0) {
std::fputs("usage: evo-load --connect HOST:PORT --server-name NAME --ca FILE "
"--issuer-key FILE --world N --frame N --region N [--steps N[,N]...] "
"(--issuer-key FILE | --grants FILE --grants-per-bot C) --world N "
"--frame N --region N [--steps N[,N]...] "
"[--hold S] [--warmup S] [--ramp-ms MS] [--threads T] [--seed X] "
"[--lifetime S] [--metrics HOST:PORT] [--response-budget-ms MS] "
"[--per-client] [--expect-established]\n",
stderr);
return 2;
}
const auto scalar = read(o.issuer_key);
auto key = GrantSigningKey::import(
DeploymentEnvironment::dev, 1,
std::span{reinterpret_cast<const std::uint8_t*>(scalar.data()), scalar.size()});
if (!key) {
std::puts("error issuer-key");
return 2;
std::optional<GrantSigningKey> key;
std::vector<SessionGrant> bundle;
if (signs) {
const auto scalar = read(o.issuer_key);
auto imported = GrantSigningKey::import(
DeploymentEnvironment::dev, 1,
std::span{reinterpret_cast<const std::uint8_t*>(scalar.data()), scalar.size()});
if (!imported) {
std::puts("error issuer-key");
return 2;
}
key.emplace(std::move(*imported));
} else {
const auto bytes = read(o.grants_file);
const auto needed = *std::ranges::max_element(o.steps) * o.grants_per_bot;
if (bytes.size() % session_grant_bytes != 0 ||
bytes.size() / session_grant_bytes < needed) {
std::printf("error grants (need %llu grants: bots x grants-per-bot)\n",
static_cast<unsigned long long>(needed));
return 2;
}
bundle.resize(bytes.size() / session_grant_bytes);
for (std::size_t i = 0; i < bundle.size(); ++i)
std::copy_n(bytes.begin() + static_cast<std::ptrdiff_t>(i * session_grant_bytes),
session_grant_bytes, reinterpret_cast<char*>(bundle[i].data()));
}
const auto address = quic::resolve_ipv4(o.host);
if (!address) {
@@ -723,7 +793,7 @@ int main(int argc, char** argv) {
bool smoke_ok = true;
std::optional<std::size_t> sustainable, saturated;
{
Load load{o, std::move(*key), *address};
Load load{o, std::move(key), std::move(bundle), *address};
for (const auto n : o.steps) {
load.activate(n);
std::this_thread::sleep_for(std::chrono::seconds{o.warmup_s});
@@ -731,6 +801,7 @@ int main(int argc, char** argv) {
// the warmup is discarded.
const auto warm = load.take(n);
const auto started = Clock::now();
const auto hold_start = now_seconds();
const auto cpu0 = cpu_seconds();
std::this_thread::sleep_for(std::chrono::seconds{o.hold_s});
auto samples = load.take(n);
@@ -739,12 +810,15 @@ int main(int argc, char** argv) {
samples[i].logins += warm[i].logins;
samples[i].drops += warm[i].drops;
samples[i].login_failures += warm[i].login_failures;
for (const auto& [why, count] : warm[i].closes)
samples[i].closes[why] += count;
samples[i].refused += warm[i].refused;
}
const auto seconds = std::chrono::duration<double>(Clock::now() - started).count();
const auto established = load.established(n);
const bool within =
report(o, n, established, samples, seconds, cpu_seconds() - cpu0, scrape(o));
report(o, n, established, load.exhausted(n), {hold_start, now_seconds()}, samples,
seconds, cpu_seconds() - cpu0, scrape(o));
if (within && !saturated)
sustainable = n;
if (!within && !saturated)