// M10.9b multi-client load generator (ADR-0097). Development and qualification tool: only // runs against the evolution-dev reference worker count as evidence (10.9d); local runs are // smoke tests and development references. // // evo-load --connect HOST:PORT --server-name NAME --ca FILE --issuer-key FILE // --world N --frame N --region N [--steps N[,N]...] [--hold S] [--warmup S] // [--ramp-ms MS] [--threads T] [--seed X] [--lifetime S] [--metrics HOST:PORT] // [--response-budget-ms MS] [--per-client] [--expect-established] // // Each simulated client (bot) is a RegionSessionClient with its own account (from --seed and // its index) logging in with a dev grant signed here (the issuer key never leaves this // process). Bots run on T worker threads (each owns a slice). Every server tick a bot sends // one intention for tick + its input lead (a random walk turning every 2 s) and predicts its // probe marker; every 2 s it sends a probe action; every 10 s it moves its interest by a // cell (a fresh baseline). A closed or refused bot logs in again after 2 s with the next // serial. // // Steps: the bot count rises to each step's N (one login every --ramp-ms), waits --warmup // seconds, then measures for --hold seconds. Bots stay connected into the next step. Per // step it prints: // load step N=.. established=.. logins=.. drops=.. login_failures=.. refused=.. seconds=.. // (drops: established sessions closed; login failures: closed before Welcome, including // transport refusals at the connection limit; refused: closed with capacity or // rate_limited, either way. Logins, drops and failures include the step's ramp and warmup; // everything else is the hold) // load response_ms p50/p95/p99/max=.. (input -> first own-marker snapshot applying it) // client_p95 min/med/max=.. // load receipt_ms ... (intention -> receipt) // load outcome_ms ... (action -> final outcome) // load login_ms ... (connect -> Welcome) // load correction_mm ... (prediction error per authoritative marker) // load snapshots hz min/med/max=.. gap_ticks p99/max=.. kib_s med/max=.. // intents=.. refused_intents=.. // load client_cpu_pct=.. (this process, all threads; Linux) // load server ... (with --metrics: the region's evo_m10_* gauges at step end) // load starved N [indices] (among bots connected during the hold: snapshot rate < 80 % of // the median, or a response p95 over both twice the median and the budget) // load verdict N=.. within_budget=0|1 [reasons] // With --per-client each bot's own line follows its step. --expect-established exits 1 // unless every step ended with all its bots established and every step saw snapshots and // final outcomes (the smoke test). #include "evo/client/motion_prediction.hpp" #include "evo/client/region_session.hpp" #include "evo/network/session_grant.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifndef _WIN32 #include #include #include #include #include #endif namespace { using namespace evo; using namespace evo::network; using Clock = std::chrono::steady_clock; using std::chrono::milliseconds; std::optional number(std::string_view v) { std::uint64_t n = 0; const auto [end, ec] = std::from_chars(v.data(), v.data() + v.size(), n); if (ec != std::errc{} || end != v.data() + v.size()) return std::nullopt; return n; } std::string read(const std::filesystem::path& path) { std::ifstream file{path, std::ios::binary}; return {std::istreambuf_iterator{file}, std::istreambuf_iterator{}}; } std::uint64_t now_seconds() { return static_cast(std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count()); } std::uint64_t micros(Clock::duration d) { return static_cast(std::max( 0, std::chrono::duration_cast(d).count())); } // Nearest rank over a copy. std::uint64_t percentile(std::vector v, double p) { if (v.empty()) return 0; std::ranges::sort(v); const auto rank = static_cast(std::ceil(p * static_cast(v.size()))); return v.at(std::clamp(rank, 1, v.size()) - 1); } double ms(std::uint64_t us) { return static_cast(us) / 1000.0; } struct Options { std::string host, server_name, ca, issuer_key, metrics_host; std::uint16_t port{}, metrics_port{}; std::uint64_t world{}, frame{}, region{}, seed{1}, lifetime{900}; std::vector steps{20}; std::uint64_t hold_s{30}, warmup_s{5}, ramp_ms{100}, threads{4}, response_budget_ms{150}; bool per_client{}, expect_established{}; }; // What one bot measured since the last take. struct Samples { std::vector response_us, receipt_us, outcome_us, login_us, correction_um, gap_ticks; std::uint64_t snapshots{}, bytes{}, intents{}, refused_intents{}, logins{}, drops{}, login_failures{}, refused{}; std::uint64_t connected_polls{}; // Polls while established (starvation is judged on these). }; void append(std::vector& to, const std::vector& from) { to.insert(to.end(), from.begin(), from.end()); } struct Bot { std::size_t index{}; AccountId account; std::uint64_t serial{}; std::unique_ptr c; std::atomic active{false}, established{false}; Clock::time_point connect_at, retry_at, next_action, next_interest, next_turn; client::MotionPredictor predictor; client::InputLead lead; std::uint64_t last_tick{}, last_view{}, last_bytes{}, next_action_id{1}; std::uint64_t applied{}; // Newest applied intention sequence seen in an own marker. std::map receipt_pending, response_pending, outcome_pending; std::map intent_tick; std::int16_t vx{}, vy{}; std::optional home; int interest_step{}; std::mt19937_64 rng; Samples s; }; class Load { public: Load(Options o, GrantSigningKey key, std::string address) : o_{std::move(o)}, key_{std::move(key)}, address_{std::move(address)}, authority_{read(o_.ca)} { const auto most = *std::ranges::max_element(o_.steps); bots_.reserve(most); for (std::size_t i = 0; i < most; ++i) { auto b = std::make_unique(); b->index = i; for (std::size_t k = 0; k < 8; ++k) { b->account.bytes.at(k) = static_cast((o_.seed >> (8U * k)) & 0xFFU); b->account.bytes.at(8 + k) = static_cast(((i + 1) >> (8U * k)) & 0xFFU); } b->serial = now_seconds() * 64; // Later runs supersede earlier ones. b->rng.seed(o_.seed * 1'000'003 + i); bots_.push_back(std::move(b)); } const auto threads = std::clamp(o_.threads, 1, 64); locks_ = std::vector(threads); for (std::size_t t = 0; t < threads; ++t) workers_.emplace_back([this, t, threads] { work(t, threads); }); } Load(const Load&) = delete; Load& operator=(const Load&) = delete; ~Load() { stop_ = true; for (auto& w : workers_) w.join(); } void activate(std::size_t n) { for (std::size_t i = 0; i < n && i < bots_.size(); ++i) { if (bots_[i]->active.exchange(true)) continue; std::this_thread::sleep_for(milliseconds{o_.ramp_ms}); } } // Every active bot's samples since the last take (cleared). std::vector take(std::size_t n) { std::vector out(n); for (std::size_t t = 0; t < locks_.size(); ++t) { const std::scoped_lock lock{locks_[t]}; for (std::size_t i = t; i < n; i += locks_.size()) out[i] = std::exchange(bots_[i]->s, Samples{}); } return out; } [[nodiscard]] std::size_t established(std::size_t n) const { return static_cast( std::count_if(bots_.begin(), bots_.begin() + static_cast(n), [](const auto& b) { return b->established.load(); })); } private: void work(std::size_t t, std::size_t threads) { while (!stop_) { { const std::scoped_lock lock{locks_[t]}; const auto now = Clock::now(); for (std::size_t i = t; i < bots_.size(); i += threads) if (bots_[i]->active) step(*bots_[i], now); } std::this_thread::sleep_for(milliseconds{2}); } const std::scoped_lock lock{locks_[t]}; for (std::size_t i = t; i < bots_.size(); i += threads) if (auto& c = bots_[i]->c; c && c->state() == client::RegionSessionState::established) c->close(); for (int k = 0; k < 50; ++k) { // Let the closes flush. for (std::size_t i = t; i < bots_.size(); i += threads) if (bots_[i]->c) bots_[i]->c->poll(Clock::now()); std::this_thread::sleep_for(milliseconds{2}); } } void login(Bot& b, Clock::time_point now) { GrantClaims claims; claims.account = b.account; claims.serial = ++b.serial; claims.world = {o_.world, o_.frame}; claims.region = o_.region; claims.issued_at = now_seconds(); claims.expires_at = claims.issued_at + o_.lifetime; claims.grant_id = (b.serial << 16U) ^ (b.index + 1); const auto grant = key_.sign(claims); client::RegionSessionClientConfig config; config.transport.server_address = address_; config.transport.port = o_.port; config.transport.server_name = o_.server_name; config.transport.authority_pem = authority_; config.transport.authority_pem_file = o_.ca; if (!grant) { b.retry_at = now + std::chrono::seconds{2}; return; } config.grant = *grant; auto made = client::RegionSessionClient::connect(std::move(config), now); if (!made) { b.retry_at = now + std::chrono::seconds{2}; return; } b.c = std::move(*made); b.connect_at = now; b.last_tick = b.last_view = b.last_bytes = b.applied = 0; b.receipt_pending.clear(); b.response_pending.clear(); b.outcome_pending.clear(); b.intent_tick.clear(); b.predictor.reset(); b.lead.reset(); b.home.reset(); } void step(Bot& b, Clock::time_point now) { if (!b.c) { if (now >= b.retry_at) login(b, now); return; } auto& c = *b.c; c.poll(now); if (c.state() == client::RegionSessionState::closed) { if (b.established.exchange(false)) ++b.s.drops; else ++b.s.login_failures; if (c.close_reason() == SessionCloseReason::capacity || c.close_reason() == SessionCloseReason::rate_limited) ++b.s.refused; b.c.reset(); b.retry_at = now + std::chrono::seconds{2} + milliseconds{std::uniform_int_distribution{0, 500}(b.rng)}; return; } if (c.state() != client::RegionSessionState::established) return; ++b.s.connected_polls; if (!b.established.exchange(true)) { ++b.s.logins; b.s.login_us.push_back(micros(now - b.connect_at)); b.next_action = now + milliseconds{std::uniform_int_distribution{0, 2000}(b.rng)}; b.next_interest = now + std::chrono::seconds{10}; } // Receipts: RTT, lead and refusals. for (const auto& r : c.take_intention_receipts()) { if (const auto it = b.receipt_pending.find(r.sequence); it != b.receipt_pending.end()) { b.s.receipt_us.push_back(micros(now - it->second)); b.receipt_pending.erase(it); } const auto tick = b.intent_tick.find(r.sequence); if (r.status == AdmitStatus::admitted) { if (tick != b.intent_tick.end()) b.lead.admitted(tick->second, r.server_tick); } else { ++b.s.refused_intents; b.predictor.rejected(r.sequence); b.response_pending.erase(r.sequence); if (r.status == AdmitStatus::outside_window) b.lead.too_late(); } if (tick != b.intent_tick.end()) b.intent_tick.erase(tick); } // Snapshots: cadence, own marker (responsiveness, prediction errors), bytes. if (const auto* v = c.view(); v && v->tick != b.last_view) { if (b.last_view != 0 && v->tick > b.last_view) b.s.gap_ticks.push_back(v->tick - b.last_view); b.last_view = v->tick; ++b.s.snapshots; if (!b.home && !v->environment.empty()) { std::int64_t x0 = v->environment.front().voxel.x, x1 = x0; std::int64_t y0 = v->environment.front().voxel.y, y1 = y0; for (const auto& cell : v->environment) { x0 = std::min(x0, cell.voxel.x); x1 = std::max(x1, cell.voxel.x); y0 = std::min(y0, cell.voxel.y); y1 = std::max(y1, cell.voxel.y); } b.home = SessionInterest{(x0 + x1) / 2, (y0 + y1) / 2, v->slice_z, 7}; } for (const auto& m : v->markers) { if (!m.own) continue; b.predictor.authoritative({v->tick, m.applied_sequence, m.state, m.area}); if (m.applied_sequence > b.applied) { b.applied = m.applied_sequence; while (!b.response_pending.empty() && b.response_pending.begin()->first <= b.applied) { b.s.response_us.push_back(micros(now - b.response_pending.begin()->second)); b.response_pending.erase(b.response_pending.begin()); } } } } for (const auto e : b.predictor.take_errors()) b.s.correction_um.push_back(e); const auto bytes = c.snapshot_bytes(); b.s.bytes += bytes - std::min(bytes, b.last_bytes); b.last_bytes = bytes; // Outcomes: final ones only. for (const auto& out : c.take_action_outcomes()) if (out.result == ActionResult::committed_no_effect || out.result == ActionResult::not_committed) if (const auto it = b.outcome_pending.find(out.action_id); it != b.outcome_pending.end()) { b.s.outcome_us.push_back(micros(now - it->second)); b.outcome_pending.erase(it); } (void)c.take_action_receipts(); // Once per server tick: an intention, a prediction, sometimes an action or interest. const auto tick = c.server_tick(); if (tick == 0 || tick == b.last_tick) return; b.last_tick = tick; if (now >= b.next_turn) { const double angle = std::uniform_real_distribution{0, 6.283185307}(b.rng); b.vx = static_cast(std::lround(700.0 * std::cos(angle))); b.vy = static_cast(std::lround(700.0 * std::sin(angle))); b.next_turn = now + std::chrono::seconds{2}; } const auto target = tick + b.lead.lead(); if (!b.predictor.full()) if (const auto seq = c.send_intention(target, b.vx, b.vy)) { ++b.s.intents; (void)b.predictor.input({*seq, target, {b.vx, b.vy}}); b.receipt_pending[*seq] = now; b.response_pending[*seq] = now; b.intent_tick[*seq] = target; for (auto* pending : {&b.receipt_pending, &b.response_pending}) while (pending->size() > 256) pending->erase(pending->begin()); while (b.intent_tick.size() > 256) b.intent_tick.erase(b.intent_tick.begin()); } (void)b.predictor.predict(target); (void)b.predictor.take_correction(); if (now >= b.next_action) { const auto id = b.next_action_id++; if (c.send_action(id, ActionKind::probe, tick + 3)) { b.outcome_pending[id] = now; while (b.outcome_pending.size() > 64) b.outcome_pending.erase(b.outcome_pending.begin()); } b.next_action = now + std::chrono::seconds{2}; } if (now >= b.next_interest && b.home) { auto in = *b.home; in.x += (b.interest_step++ % 2 == 0) ? 1 : 0; // Alternate: a fresh baseline each. (void)c.set_interest(in); b.next_interest = now + std::chrono::seconds{10}; } } Options o_; GrantSigningKey key_; std::string address_, authority_; std::vector> bots_; std::vector locks_; std::vector workers_; std::atomic stop_{false}; }; // The region's /metrics (evo_m10_* and evo_region_tick_debt), unlabelled series only. std::map scrape(const Options& o) { std::map out; #ifndef _WIN32 if (o.metrics_port == 0) return out; const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); if (fd < 0) return out; sockaddr_in a{}; a.sin_family = AF_INET; a.sin_port = htons(o.metrics_port); timeval timeout{3, 0}; ::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout); std::string raw; if (::inet_pton(AF_INET, o.metrics_host.c_str(), &a.sin_addr) == 1 && ::connect(fd, reinterpret_cast(&a), sizeof a) == 0) { const std::string request = "GET /metrics HTTP/1.1\r\nHost: load\r\n\r\n"; if (::send(fd, request.data(), request.size(), MSG_NOSIGNAL) == static_cast(request.size())) { std::array chunk{}; for (ssize_t n = 0; (n = ::recv(fd, chunk.data(), chunk.size(), 0)) > 0;) raw.append(chunk.data(), static_cast(n)); } } ::close(fd); std::size_t at = 0; while (at < raw.size()) { const auto end = std::min(raw.find('\n', at), raw.size()); const std::string_view line{raw.data() + at, end - at}; at = end + 1; if (!line.starts_with("evo_m10_") && !line.starts_with("evo_region_tick_debt")) continue; const auto space = line.find(' '); if (space == std::string_view::npos || line.find('{') != std::string_view::npos) continue; if (const auto v = number(line.substr(space + 1))) out[std::string{line.substr(0, space)}] = *v; } #else (void)o; #endif return out; } double cpu_seconds() { #ifndef _WIN32 rusage u{}; ::getrusage(RUSAGE_SELF, &u); return static_cast(u.ru_utime.tv_sec + u.ru_stime.tv_sec) + (static_cast(u.ru_utime.tv_usec + u.ru_stime.tv_usec) / 1e6); #else return 0; #endif } void line(const char* name, const std::vector& pooled, const std::vector& client_p95, double scale, const char* unit) { const auto f = [&](std::uint64_t v) { return static_cast(v) / scale; }; std::printf("load %s_%s p50/p95/p99/max=%.1f/%.1f/%.1f/%.1f n=%zu client_p95 " "min/med/max=%.1f/%.1f/%.1f\n", name, unit, f(percentile(pooled, 0.50)), f(percentile(pooled, 0.95)), f(percentile(pooled, 0.99)), f(percentile(pooled, 1.0)), pooled.size(), f(percentile(client_p95, 0.0)), f(percentile(client_p95, 0.5)), f(percentile(client_p95, 1.0))); } // One step's report; true when within budget. bool report(const Options& o, std::size_t n, std::size_t established, const std::vector& all, double seconds, double cpu, const std::map& server) { Samples pooled; std::vector response95, receipt95, outcome95, login95, correction95, rates, kib; for (const auto& s : all) { append(pooled.response_us, s.response_us); append(pooled.receipt_us, s.receipt_us); append(pooled.outcome_us, s.outcome_us); append(pooled.login_us, s.login_us); append(pooled.correction_um, s.correction_um); append(pooled.gap_ticks, s.gap_ticks); pooled.snapshots += s.snapshots; pooled.intents += s.intents; pooled.refused_intents += s.refused_intents; pooled.logins += s.logins; pooled.drops += s.drops; pooled.login_failures += s.login_failures; pooled.refused += s.refused; response95.push_back(percentile(s.response_us, 0.95)); receipt95.push_back(percentile(s.receipt_us, 0.95)); outcome95.push_back(percentile(s.outcome_us, 0.95)); if (!s.login_us.empty()) login95.push_back(percentile(s.login_us, 0.95)); correction95.push_back(percentile(s.correction_um, 0.95)); rates.push_back(static_cast( std::lround(1000.0 * static_cast(s.snapshots) / std::max(seconds, 0.001)))); kib.push_back(static_cast( std::lround(static_cast(s.bytes) / 1024.0 / std::max(seconds, 0.001)))); } std::printf("load step N=%zu established=%zu logins=%llu drops=%llu login_failures=%llu " "refused=%llu seconds=%.1f\n", n, established, static_cast(pooled.logins), static_cast(pooled.drops), static_cast(pooled.login_failures), static_cast(pooled.refused), seconds); line("response", pooled.response_us, response95, 1000.0, "ms"); line("receipt", pooled.receipt_us, receipt95, 1000.0, "ms"); line("outcome", pooled.outcome_us, outcome95, 1000.0, "ms"); line("login", pooled.login_us, login95, 1000.0, "ms"); line("correction", pooled.correction_um, correction95, 1000.0, "mm"); std::printf("load snapshots hz min/med/max=%.2f/%.2f/%.2f gap_ticks p99/max=%llu/%llu " "kib_s med/max=%llu/%llu intents=%llu refused_intents=%llu\n", static_cast(percentile(rates, 0.0)) / 1000.0, static_cast(percentile(rates, 0.5)) / 1000.0, static_cast(percentile(rates, 1.0)) / 1000.0, static_cast(percentile(pooled.gap_ticks, 0.99)), static_cast(percentile(pooled.gap_ticks, 1.0)), static_cast(percentile(kib, 0.5)), static_cast(percentile(kib, 1.0)), static_cast(pooled.intents), static_cast(pooled.refused_intents)); std::printf("load client_cpu_pct=%.0f\n", 100.0 * cpu / std::max(seconds, 0.001)); if (!server.empty()) { std::printf("load server"); for (const auto& [k, v] : server) std::printf(" %s=%llu", k.c_str(), static_cast(v)); std::printf("\n"); } // Starvation: throughput must not hide a client left behind. Judged among the bots that // were connected during the hold (the others are reported as not established). std::vector connected_rates, connected_response; for (std::size_t i = 0; i < all.size(); ++i) if (all[i].connected_polls > 0) { connected_rates.push_back(rates[i]); connected_response.push_back(response95[i]); } const auto median_rate = percentile(connected_rates, 0.5); const auto median_response = percentile(connected_response, 0.5); const auto budget_us = o.response_budget_ms * 1000; std::vector starved; for (std::size_t i = 0; i < all.size(); ++i) if (all[i].connected_polls > 0 && (rates[i] * 5 < median_rate * 4 || (response95[i] > 2 * median_response && response95[i] > budget_us))) starved.push_back(i); std::printf("load starved %zu", starved.size()); for (const auto i : starved) std::printf(" %zu", i); std::printf("\n"); std::vector why; if (established < n) why.emplace_back("not-established"); if (pooled.refused > 0 || pooled.login_failures > 0) why.emplace_back("refused"); if (pooled.drops > 0) why.emplace_back("drops"); if (!starved.empty()) why.emplace_back("starved"); if (pooled.response_us.empty() || percentile(pooled.response_us, 0.95) > budget_us) why.emplace_back("response"); if (pooled.outcome_us.empty() || percentile(pooled.outcome_us, 0.95) > 1'500'000) why.emplace_back("outcome"); const auto get = [&](const char* k) { const auto it = server.find(k); return it == server.end() ? 0 : it->second; }; if (get("evo_m10_tick_work_p99_microseconds") > 25'000) why.emplace_back("tick-work"); if (get("evo_m10_refusing") != 0 || get("evo_m10_shed_level") != 0) why.emplace_back("governor"); if (get("evo_region_tick_debt") != 0) why.emplace_back("tick-debt"); std::printf("load verdict N=%zu within_budget=%d", n, why.empty() ? 1 : 0); for (const auto& w : why) std::printf(" %s", w.c_str()); std::printf("\n"); if (o.per_client) for (std::size_t i = 0; i < all.size(); ++i) std::printf("load client %zu snapshots=%llu response_p95_ms=%.1f receipt_p95_ms=%.1f " "outcome_p95_ms=%.1f correction_p95_mm=%.1f drops=%llu " "login_failures=%llu\n", i, static_cast(all[i].snapshots), ms(response95[i]), ms(receipt95[i]), ms(outcome95[i]), static_cast(correction95[i]) / 1000.0, static_cast(all[i].drops), static_cast(all[i].login_failures)); std::fflush(stdout); return why.empty(); } std::optional> endpoint(std::string_view v) { const auto colon = v.rfind(':'); if (colon == std::string_view::npos || colon == 0) return std::nullopt; const auto port = number(v.substr(colon + 1)); if (!port || *port == 0 || *port > 65535) return std::nullopt; return std::pair{std::string{v.substr(0, colon)}, static_cast(*port)}; } std::optional> steps_of(std::string_view v) { std::vector out; while (!v.empty()) { const auto comma = v.find(','); const auto n = number(v.substr(0, comma)); if (!n || *n == 0 || *n > 256 || (!out.empty() && *n < out.back())) return std::nullopt; out.push_back(*n); v.remove_prefix(comma == std::string_view::npos ? v.size() : comma + 1); } if (out.empty()) return std::nullopt; return out; } } // namespace int main(int argc, char** argv) { const std::vector args(argv + 1, argv + argc); Options o; bool ok = true; for (std::size_t i = 0; i < args.size() && ok; ++i) { const auto flag = args[i]; if (flag == "--per-client") { o.per_client = true; continue; } if (flag == "--expect-established") { o.expect_established = true; continue; } if (i + 1 >= args.size()) { ok = false; break; } const auto value = args[++i]; const auto n = number(value); if (flag == "--connect") { const auto e = endpoint(value); ok = e.has_value(); if (e) std::tie(o.host, o.port) = *e; } else if (flag == "--metrics") { const auto e = endpoint(value); ok = e.has_value(); if (e) std::tie(o.metrics_host, o.metrics_port) = *e; } else if (flag == "--server-name") o.server_name = value; else if (flag == "--ca") o.ca = value; else if (flag == "--issuer-key") o.issuer_key = value; else if (flag == "--steps") { const auto s = steps_of(value); ok = s.has_value(); if (s) o.steps = *s; } else if (n && flag == "--world") o.world = *n; else if (n && flag == "--frame") o.frame = *n; else if (n && flag == "--region") o.region = *n; else if (n && flag == "--seed" && *n != 0) o.seed = *n; else if (n && flag == "--lifetime" && *n >= 60 && *n <= 3600) o.lifetime = *n; else if (n && flag == "--hold" && *n >= 1 && *n <= 3600) o.hold_s = *n; else if (n && flag == "--warmup" && *n <= 600) o.warmup_s = *n; else if (n && flag == "--ramp-ms" && *n <= 10'000) o.ramp_ms = *n; else if (n && flag == "--threads" && *n >= 1 && *n <= 64) o.threads = *n; else if (n && flag == "--response-budget-ms" && *n >= 1) o.response_budget_ms = *n; else ok = false; } if (!ok || o.host.empty() || o.server_name.empty() || o.ca.empty() || o.issuer_key.empty() || o.world == 0 || o.frame == 0 || o.region == 0) { std::fputs("usage: evo-load --connect HOST:PORT --server-name NAME --ca FILE " "--issuer-key FILE --world N --frame N --region N [--steps N[,N]...] " "[--hold S] [--warmup S] [--ramp-ms MS] [--threads T] [--seed X] " "[--lifetime S] [--metrics HOST:PORT] [--response-budget-ms MS] " "[--per-client] [--expect-established]\n", stderr); return 2; } const auto scalar = read(o.issuer_key); auto key = GrantSigningKey::import( DeploymentEnvironment::dev, 1, std::span{reinterpret_cast(scalar.data()), scalar.size()}); if (!key) { std::puts("error issuer-key"); return 2; } const auto address = quic::resolve_ipv4(o.host); if (!address) { std::puts("error resolve"); return 2; } std::printf("load start steps=%zu hold=%llu warmup=%llu threads=%llu budget_ms=%llu\n", o.steps.size(), static_cast(o.hold_s), static_cast(o.warmup_s), static_cast(o.threads), static_cast(o.response_budget_ms)); std::fflush(stdout); bool smoke_ok = true; std::optional sustainable, saturated; { Load load{o, std::move(*key), *address}; for (const auto n : o.steps) { load.activate(n); std::this_thread::sleep_for(std::chrono::seconds{o.warmup_s}); // The ramp's and warmup's logins (and closes) belong to this step; the rest of // the warmup is discarded. const auto warm = load.take(n); const auto started = Clock::now(); const auto cpu0 = cpu_seconds(); std::this_thread::sleep_for(std::chrono::seconds{o.hold_s}); auto samples = load.take(n); for (std::size_t i = 0; i < n; ++i) { append(samples[i].login_us, warm[i].login_us); samples[i].logins += warm[i].logins; samples[i].drops += warm[i].drops; samples[i].login_failures += warm[i].login_failures; samples[i].refused += warm[i].refused; } const auto seconds = std::chrono::duration(Clock::now() - started).count(); const auto established = load.established(n); const bool within = report(o, n, established, samples, seconds, cpu_seconds() - cpu0, scrape(o)); if (within && !saturated) sustainable = n; if (!within && !saturated) saturated = n; std::uint64_t snapshots = 0, outcomes = 0; for (const auto& s : samples) { snapshots += s.snapshots; outcomes += s.outcome_us.size(); } smoke_ok = smoke_ok && established == n && snapshots > 0 && outcomes > 0; } } std::printf("load result sustainable=%zu saturated_at=%zu\n", sustainable.value_or(0), saturated.value_or(0)); std::fflush(stdout); return o.expect_established && !smoke_ok ? 1 : 0; }