Add M10.9d load drill harness and serialize Mbed TLS use
Foundation / checks (deploy) (push) Successful in 3m24s
Foundation / checks (ecs) (push) Successful in 1m56s
Foundation / checks (client) (push) Successful in 12m18s
Foundation / checks (network) (push) Successful in 13m30s
Foundation / checks (tsan) (push) Successful in 8m0s
Foundation / checks (checks) (push) Successful in 32m39s
Foundation / checks (postgres) (push) Successful in 45m47s
Foundation / postgres-tools (push) Successful in 17s
Foundation / image (push) Successful in 2m30s
Foundation / server-image (push) Successful in 4m8s
Foundation / checks (deploy) (push) Successful in 3m24s
Foundation / checks (ecs) (push) Successful in 1m56s
Foundation / checks (client) (push) Successful in 12m18s
Foundation / checks (network) (push) Successful in 13m30s
Foundation / checks (tsan) (push) Successful in 8m0s
Foundation / checks (checks) (push) Successful in 32m39s
Foundation / checks (postgres) (push) Successful in 45m47s
Foundation / postgres-tools (push) Successful in 17s
Foundation / image (push) Successful in 2m30s
Foundation / server-image (push) Successful in 4m8s
ADR-0097. tools/ops/load-drill.sh (runbook docs/networking/M10_LOAD_DRILL.md) runs the evolution-dev reference qualification: synthetic inputs on for the run (the ADR-0073 drill-time override, restored on exit), bundle-mode evo-load bots through the public route, and each step's hold matched with the server's reports. evo-server logs "m10 load-report" every 10 s, since the agent cannot reach the ops port. evo-load gains --grants FILE --grants-per-bot C, exhausted=, hold_utc= and close_reasons=; evo-session-probe grants gains --accounts N. Finding: Mbed TLS/PSA are built without MBEDTLS_THREADING_C, and bots on different threads verifying their first connections at once were refused as untrusted (two per smoke run). One process-wide recursive crypto lock now guards every network-library entry point that touches them; the smoke went from 2/2 failing to 3/3 clean. quic-transport adds a concurrent-connect test. Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+107
-33
@@ -2,7 +2,8 @@
|
||||
// runs against the evolution-dev reference worker count as evidence (10.9d); local runs are
|
||||
// smoke tests and development references.
|
||||
//
|
||||
// evo-load --connect HOST:PORT --server-name NAME --ca FILE --issuer-key FILE
|
||||
// evo-load --connect HOST:PORT --server-name NAME --ca FILE
|
||||
// (--issuer-key FILE | --grants FILE --grants-per-bot C)
|
||||
// --world N --frame N --region N [--steps N[,N]...] [--hold S] [--warmup S]
|
||||
// [--ramp-ms MS] [--threads T] [--seed X] [--lifetime S] [--metrics HOST:PORT]
|
||||
// [--response-budget-ms MS] [--per-client] [--expect-established]
|
||||
@@ -14,11 +15,17 @@
|
||||
// probe marker; every 2 s it sends a probe action; every 10 s it moves its interest by a
|
||||
// cell (a fresh baseline). A closed or refused bot logs in again after 2 s with the next
|
||||
// serial.
|
||||
// M10.9d: with --grants (a bundle from `evo-session-probe grants --accounts N --count C`,
|
||||
// e.g. made where the dev issuer key lives), bot i logs in with grants i*C .. i*C+C-1 in
|
||||
// order instead of signing; a bot that used all C stays out (counted as exhausted).
|
||||
//
|
||||
// Steps: the bot count rises to each step's N (one login every --ramp-ms), waits --warmup
|
||||
// seconds, then measures for --hold seconds. Bots stay connected into the next step. Per
|
||||
// step it prints:
|
||||
// load step N=.. established=.. logins=.. drops=.. login_failures=.. refused=.. seconds=..
|
||||
// exhausted=.. hold_utc=START..END (unix seconds of the hold, to match server samples)
|
||||
// close_reasons=REASON:COUNT,... (session-v1 close reasons; without a session Close,
|
||||
// 256 + the QUIC transport cause, or 255 when unknown)
|
||||
// (drops: established sessions closed; login failures: closed before Welcome, including
|
||||
// transport refusals at the connection limit; refused: closed with capacity or
|
||||
// rate_limited, either way. Logins, drops and failures include the step's ramp and warmup;
|
||||
@@ -111,7 +118,8 @@ double ms(std::uint64_t us) {
|
||||
}
|
||||
|
||||
struct Options {
|
||||
std::string host, server_name, ca, issuer_key, metrics_host;
|
||||
std::string host, server_name, ca, issuer_key, grants_file, metrics_host;
|
||||
std::uint64_t grants_per_bot{};
|
||||
std::uint16_t port{}, metrics_port{};
|
||||
std::uint64_t world{}, frame{}, region{}, seed{1}, lifetime{900};
|
||||
std::vector<std::size_t> steps{20};
|
||||
@@ -125,6 +133,7 @@ struct Samples {
|
||||
gap_ticks;
|
||||
std::uint64_t snapshots{}, bytes{}, intents{}, refused_intents{}, logins{}, drops{},
|
||||
login_failures{}, refused{};
|
||||
std::map<unsigned, std::uint64_t> closes; // By session close reason (255: no Close).
|
||||
std::uint64_t connected_polls{}; // Polls while established (starvation is judged on these).
|
||||
};
|
||||
void append(std::vector<std::uint64_t>& to, const std::vector<std::uint64_t>& from) {
|
||||
@@ -142,6 +151,8 @@ struct Bot {
|
||||
client::InputLead lead;
|
||||
std::uint64_t last_tick{}, last_view{}, last_bytes{}, next_action_id{1};
|
||||
std::uint64_t applied{}; // Newest applied intention sequence seen in an own marker.
|
||||
std::uint64_t used{}; // Bundle grants used (bundle mode).
|
||||
std::atomic<bool> exhausted{false};
|
||||
std::map<std::uint64_t, Clock::time_point> receipt_pending, response_pending, outcome_pending;
|
||||
std::map<std::uint64_t, std::uint64_t> intent_tick;
|
||||
std::int16_t vx{}, vy{};
|
||||
@@ -153,9 +164,11 @@ struct Bot {
|
||||
|
||||
class Load {
|
||||
public:
|
||||
Load(Options o, GrantSigningKey key, std::string address)
|
||||
: o_{std::move(o)}, key_{std::move(key)}, address_{std::move(address)},
|
||||
authority_{read(o_.ca)} {
|
||||
// Signs with `key`, or logs in from `bundle` (concatenated grants, grants_per_bot each).
|
||||
Load(Options o, std::optional<GrantSigningKey> key, std::vector<SessionGrant> bundle,
|
||||
std::string address)
|
||||
: o_{std::move(o)}, key_{std::move(key)}, bundle_{std::move(bundle)},
|
||||
address_{std::move(address)}, authority_{read(o_.ca)} {
|
||||
const auto most = *std::ranges::max_element(o_.steps);
|
||||
bots_.reserve(most);
|
||||
for (std::size_t i = 0; i < most; ++i) {
|
||||
@@ -199,6 +212,11 @@ class Load {
|
||||
}
|
||||
return out;
|
||||
}
|
||||
[[nodiscard]] std::size_t exhausted(std::size_t n) const {
|
||||
return static_cast<std::size_t>(
|
||||
std::count_if(bots_.begin(), bots_.begin() + static_cast<std::ptrdiff_t>(n),
|
||||
[](const auto& b) { return b->exhausted.load(); }));
|
||||
}
|
||||
[[nodiscard]] std::size_t established(std::size_t n) const {
|
||||
return static_cast<std::size_t>(
|
||||
std::count_if(bots_.begin(), bots_.begin() + static_cast<std::ptrdiff_t>(n),
|
||||
@@ -229,15 +247,26 @@ class Load {
|
||||
}
|
||||
}
|
||||
void login(Bot& b, Clock::time_point now) {
|
||||
GrantClaims claims;
|
||||
claims.account = b.account;
|
||||
claims.serial = ++b.serial;
|
||||
claims.world = {o_.world, o_.frame};
|
||||
claims.region = o_.region;
|
||||
claims.issued_at = now_seconds();
|
||||
claims.expires_at = claims.issued_at + o_.lifetime;
|
||||
claims.grant_id = (b.serial << 16U) ^ (b.index + 1);
|
||||
const auto grant = key_.sign(claims);
|
||||
std::optional<SessionGrant> grant;
|
||||
if (key_) {
|
||||
GrantClaims claims;
|
||||
claims.account = b.account;
|
||||
claims.serial = ++b.serial;
|
||||
claims.world = {o_.world, o_.frame};
|
||||
claims.region = o_.region;
|
||||
claims.issued_at = now_seconds();
|
||||
claims.expires_at = claims.issued_at + o_.lifetime;
|
||||
claims.grant_id = (b.serial << 16U) ^ (b.index + 1);
|
||||
if (auto signed_grant = key_->sign(claims))
|
||||
grant = *signed_grant;
|
||||
} else {
|
||||
if (b.used >= o_.grants_per_bot) {
|
||||
b.exhausted = true; // Out of grants: stays out.
|
||||
b.retry_at = Clock::time_point::max();
|
||||
return;
|
||||
}
|
||||
grant = bundle_.at((b.index * o_.grants_per_bot) + b.used++);
|
||||
}
|
||||
client::RegionSessionClientConfig config;
|
||||
config.transport.server_address = address_;
|
||||
config.transport.port = o_.port;
|
||||
@@ -278,6 +307,13 @@ class Load {
|
||||
++b.s.drops;
|
||||
else
|
||||
++b.s.login_failures;
|
||||
if (const auto reason = c.close_reason();
|
||||
reason && *reason != SessionCloseReason::connection_lost)
|
||||
++b.s.closes[static_cast<unsigned>(*reason)];
|
||||
else if (const auto cause = c.transport_cause()) // No session Close: 256 + cause.
|
||||
++b.s.closes[256U + static_cast<unsigned>(*cause)];
|
||||
else
|
||||
++b.s.closes[255U];
|
||||
if (c.close_reason() == SessionCloseReason::capacity ||
|
||||
c.close_reason() == SessionCloseReason::rate_limited)
|
||||
++b.s.refused;
|
||||
@@ -406,7 +442,8 @@ class Load {
|
||||
}
|
||||
|
||||
Options o_;
|
||||
GrantSigningKey key_;
|
||||
std::optional<GrantSigningKey> key_;
|
||||
std::vector<SessionGrant> bundle_;
|
||||
std::string address_, authority_;
|
||||
std::vector<std::unique_ptr<Bot>> bots_;
|
||||
std::vector<std::mutex> locks_;
|
||||
@@ -481,9 +518,9 @@ void line(const char* name, const std::vector<std::uint64_t>& pooled,
|
||||
}
|
||||
|
||||
// One step's report; true when within budget.
|
||||
bool report(const Options& o, std::size_t n, std::size_t established,
|
||||
const std::vector<Samples>& all, double seconds, double cpu,
|
||||
const std::map<std::string, std::uint64_t>& server) {
|
||||
bool report(const Options& o, std::size_t n, std::size_t established, std::size_t exhausted,
|
||||
std::pair<std::uint64_t, std::uint64_t> hold_utc, const std::vector<Samples>& all,
|
||||
double seconds, double cpu, const std::map<std::string, std::uint64_t>& server) {
|
||||
Samples pooled;
|
||||
std::vector<std::uint64_t> response95, receipt95, outcome95, login95, correction95, rates, kib;
|
||||
for (const auto& s : all) {
|
||||
@@ -499,6 +536,8 @@ bool report(const Options& o, std::size_t n, std::size_t established,
|
||||
pooled.logins += s.logins;
|
||||
pooled.drops += s.drops;
|
||||
pooled.login_failures += s.login_failures;
|
||||
for (const auto& [why, count] : s.closes)
|
||||
pooled.closes[why] += count;
|
||||
pooled.refused += s.refused;
|
||||
response95.push_back(percentile(s.response_us, 0.95));
|
||||
receipt95.push_back(percentile(s.receipt_us, 0.95));
|
||||
@@ -511,12 +550,18 @@ bool report(const Options& o, std::size_t n, std::size_t established,
|
||||
kib.push_back(static_cast<std::uint64_t>(
|
||||
std::lround(static_cast<double>(s.bytes) / 1024.0 / std::max(seconds, 0.001))));
|
||||
}
|
||||
std::string reasons;
|
||||
for (const auto& [why, count] : pooled.closes)
|
||||
reasons += (reasons.empty() ? "" : ",") + std::to_string(why) + ":" + std::to_string(count);
|
||||
std::printf("load step N=%zu established=%zu logins=%llu drops=%llu login_failures=%llu "
|
||||
"refused=%llu seconds=%.1f\n",
|
||||
"refused=%llu seconds=%.1f exhausted=%zu hold_utc=%llu..%llu close_reasons=%s\n",
|
||||
n, established, static_cast<unsigned long long>(pooled.logins),
|
||||
static_cast<unsigned long long>(pooled.drops),
|
||||
static_cast<unsigned long long>(pooled.login_failures),
|
||||
static_cast<unsigned long long>(pooled.refused), seconds);
|
||||
static_cast<unsigned long long>(pooled.refused), seconds, exhausted,
|
||||
static_cast<unsigned long long>(hold_utc.first),
|
||||
static_cast<unsigned long long>(hold_utc.second),
|
||||
reasons.empty() ? "-" : reasons.c_str());
|
||||
line("response", pooled.response_us, response95, 1000.0, "ms");
|
||||
line("receipt", pooled.receipt_us, receipt95, 1000.0, "ms");
|
||||
line("outcome", pooled.outcome_us, outcome95, 1000.0, "ms");
|
||||
@@ -663,6 +708,10 @@ int main(int argc, char** argv) {
|
||||
o.ca = value;
|
||||
else if (flag == "--issuer-key")
|
||||
o.issuer_key = value;
|
||||
else if (flag == "--grants")
|
||||
o.grants_file = value;
|
||||
else if (n && flag == "--grants-per-bot" && *n >= 1 && *n <= 64)
|
||||
o.grants_per_bot = *n;
|
||||
else if (flag == "--steps") {
|
||||
const auto s = steps_of(value);
|
||||
ok = s.has_value();
|
||||
@@ -676,7 +725,7 @@ int main(int argc, char** argv) {
|
||||
o.region = *n;
|
||||
else if (n && flag == "--seed" && *n != 0)
|
||||
o.seed = *n;
|
||||
else if (n && flag == "--lifetime" && *n >= 60 && *n <= 3600)
|
||||
else if (n && flag == "--lifetime" && *n >= 60 && *n <= 14400)
|
||||
o.lifetime = *n;
|
||||
else if (n && flag == "--hold" && *n >= 1 && *n <= 3600)
|
||||
o.hold_s = *n;
|
||||
@@ -691,23 +740,44 @@ int main(int argc, char** argv) {
|
||||
else
|
||||
ok = false;
|
||||
}
|
||||
if (!ok || o.host.empty() || o.server_name.empty() || o.ca.empty() || o.issuer_key.empty() ||
|
||||
o.world == 0 || o.frame == 0 || o.region == 0) {
|
||||
const bool signs = !o.issuer_key.empty();
|
||||
if (!ok || o.host.empty() || o.server_name.empty() || o.ca.empty() ||
|
||||
signs == !o.grants_file.empty() || (!signs && o.grants_per_bot == 0) || o.world == 0 ||
|
||||
o.frame == 0 || o.region == 0) {
|
||||
std::fputs("usage: evo-load --connect HOST:PORT --server-name NAME --ca FILE "
|
||||
"--issuer-key FILE --world N --frame N --region N [--steps N[,N]...] "
|
||||
"(--issuer-key FILE | --grants FILE --grants-per-bot C) --world N "
|
||||
"--frame N --region N [--steps N[,N]...] "
|
||||
"[--hold S] [--warmup S] [--ramp-ms MS] [--threads T] [--seed X] "
|
||||
"[--lifetime S] [--metrics HOST:PORT] [--response-budget-ms MS] "
|
||||
"[--per-client] [--expect-established]\n",
|
||||
stderr);
|
||||
return 2;
|
||||
}
|
||||
const auto scalar = read(o.issuer_key);
|
||||
auto key = GrantSigningKey::import(
|
||||
DeploymentEnvironment::dev, 1,
|
||||
std::span{reinterpret_cast<const std::uint8_t*>(scalar.data()), scalar.size()});
|
||||
if (!key) {
|
||||
std::puts("error issuer-key");
|
||||
return 2;
|
||||
std::optional<GrantSigningKey> key;
|
||||
std::vector<SessionGrant> bundle;
|
||||
if (signs) {
|
||||
const auto scalar = read(o.issuer_key);
|
||||
auto imported = GrantSigningKey::import(
|
||||
DeploymentEnvironment::dev, 1,
|
||||
std::span{reinterpret_cast<const std::uint8_t*>(scalar.data()), scalar.size()});
|
||||
if (!imported) {
|
||||
std::puts("error issuer-key");
|
||||
return 2;
|
||||
}
|
||||
key.emplace(std::move(*imported));
|
||||
} else {
|
||||
const auto bytes = read(o.grants_file);
|
||||
const auto needed = *std::ranges::max_element(o.steps) * o.grants_per_bot;
|
||||
if (bytes.size() % session_grant_bytes != 0 ||
|
||||
bytes.size() / session_grant_bytes < needed) {
|
||||
std::printf("error grants (need %llu grants: bots x grants-per-bot)\n",
|
||||
static_cast<unsigned long long>(needed));
|
||||
return 2;
|
||||
}
|
||||
bundle.resize(bytes.size() / session_grant_bytes);
|
||||
for (std::size_t i = 0; i < bundle.size(); ++i)
|
||||
std::copy_n(bytes.begin() + static_cast<std::ptrdiff_t>(i * session_grant_bytes),
|
||||
session_grant_bytes, reinterpret_cast<char*>(bundle[i].data()));
|
||||
}
|
||||
const auto address = quic::resolve_ipv4(o.host);
|
||||
if (!address) {
|
||||
@@ -723,7 +793,7 @@ int main(int argc, char** argv) {
|
||||
bool smoke_ok = true;
|
||||
std::optional<std::size_t> sustainable, saturated;
|
||||
{
|
||||
Load load{o, std::move(*key), *address};
|
||||
Load load{o, std::move(key), std::move(bundle), *address};
|
||||
for (const auto n : o.steps) {
|
||||
load.activate(n);
|
||||
std::this_thread::sleep_for(std::chrono::seconds{o.warmup_s});
|
||||
@@ -731,6 +801,7 @@ int main(int argc, char** argv) {
|
||||
// the warmup is discarded.
|
||||
const auto warm = load.take(n);
|
||||
const auto started = Clock::now();
|
||||
const auto hold_start = now_seconds();
|
||||
const auto cpu0 = cpu_seconds();
|
||||
std::this_thread::sleep_for(std::chrono::seconds{o.hold_s});
|
||||
auto samples = load.take(n);
|
||||
@@ -739,12 +810,15 @@ int main(int argc, char** argv) {
|
||||
samples[i].logins += warm[i].logins;
|
||||
samples[i].drops += warm[i].drops;
|
||||
samples[i].login_failures += warm[i].login_failures;
|
||||
for (const auto& [why, count] : warm[i].closes)
|
||||
samples[i].closes[why] += count;
|
||||
samples[i].refused += warm[i].refused;
|
||||
}
|
||||
const auto seconds = std::chrono::duration<double>(Clock::now() - started).count();
|
||||
const auto established = load.established(n);
|
||||
const bool within =
|
||||
report(o, n, established, samples, seconds, cpu_seconds() - cpu0, scrape(o));
|
||||
report(o, n, established, load.exhausted(n), {hold_start, now_seconds()}, samples,
|
||||
seconds, cpu_seconds() - cpu0, scrape(o));
|
||||
if (within && !saturated)
|
||||
sustainable = n;
|
||||
if (!within && !saturated)
|
||||
|
||||
Reference in New Issue
Block a user