Files
dingoandClaude Opus 5.5 9f9a93d4c8
Foundation / checks (client) (push) Successful in 14s
Foundation / checks (checks) (push) Successful in 16s
Foundation / checks (deploy) (push) Successful in 15s
Foundation / checks (network) (push) Successful in 18s
Foundation / checks (postgres) (push) Successful in 1m34s
Foundation / checks (ecs) (push) Successful in 1m53s
Foundation / checks (tsan) (push) Successful in 5m44s
Foundation / server-image (push) Successful in 1m0s
Foundation / postgres-tools (push) Successful in 1m38s
Foundation / image (push) Successful in 2m34s
M10.9d: watermark coalescing, budgets v2, evo-load action ids
- IntentLimits::watermark_coalesce (evo-server: every 4, coalesce 4); ADR-0092 amendment.
- m10 load-report splits poll/replicate and commit-tick/other-tick maxima.
- Budgets v2: work p95 <= 25 ms over all ticks, commit ticks <= 50 ms (ADR-0097);
  moving the commit off the tick is a recorded follow-up.
- evo-load: time-based action ids, duplicate_actions, receipts before outcomes;
  drill ramps at 500 ms and judges by v2.
- Refresh the dev session window.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 18:00:41 -04:00

861 lines
39 KiB
C++

// M10.9b multi-client load generator (ADR-0097). Development and qualification tool: only
// runs against the evolution-dev reference worker count as evidence (10.9d); local runs are
// smoke tests and development references.
//
// evo-load --connect HOST:PORT --server-name NAME --ca FILE
// (--issuer-key FILE | --grants FILE --grants-per-bot C)
// --world N --frame N --region N [--steps N[,N]...] [--hold S] [--warmup S]
// [--ramp-ms MS] [--threads T] [--seed X] [--lifetime S] [--metrics HOST:PORT]
// [--response-budget-ms MS] [--per-client] [--expect-established]
//
// Each simulated client (bot) is a RegionSessionClient with its own account (from --seed and
// its index) logging in with a dev grant signed here (the issuer key never leaves this
// process). Bots run on T worker threads (each owns a slice). Every server tick a bot sends
// one intention for tick + its input lead (a random walk turning every 2 s) and predicts its
// probe marker; every 2 s it sends a probe action; every 10 s it moves its interest by a
// cell (a fresh baseline). A closed bot logs in again after 2 s with the next serial; one
// refused for load (capacity, rate_limited) waits 10 s and, in bundle mode, reuses the same
// grant (the region refused it before using it).
// M10.9d: with --grants (a bundle from `evo-session-probe grants --accounts N --count C`,
// e.g. made where the dev issuer key lives), bot i logs in with grants i*C .. i*C+C-1 in
// order instead of signing; a bot that used all C stays out (counted as exhausted).
//
// Steps: the bot count rises to each step's N (one login every --ramp-ms), waits --warmup
// seconds, then measures for --hold seconds. Bots stay connected into the next step. Per
// step it prints:
// load step N=.. established=.. logins=.. drops=.. login_failures=.. refused=.. seconds=..
// exhausted=.. hold_utc=START..END (unix seconds of the hold, to match server samples)
// close_reasons=REASON:COUNT,... (session-v1 close reasons; without a session Close,
// 256 + the QUIC transport cause, or 255 when unknown)
// (drops: established sessions closed; login failures: closed before Welcome, including
// transport refusals at the connection limit; refused: closed with capacity or
// rate_limited, either way. Logins, drops and failures include the step's ramp and warmup;
// everything else is the hold)
// load response_ms p50/p95/p99/max=.. (input -> first own-marker snapshot applying it)
// client_p95 min/med/max=..
// load receipt_ms ... (intention -> receipt)
// load outcome_ms ... (action -> final outcome)
// load login_ms ... (connect -> Welcome)
// load correction_mm ... (prediction error per authoritative marker)
// load snapshots hz min/med/max=.. gap_ticks p99/max=.. kib_s med/max=..
// intents=.. refused_intents=..
// load client_cpu_pct=.. (this process, all threads; Linux)
// load server ... (with --metrics: the region's evo_m10_* gauges at step end)
// load starved N [indices] (among bots connected during the hold: snapshot rate < 80 % of
// the median, or a response p95 over both twice the median and the budget)
// load verdict N=.. within_budget=0|1 [reasons]
// With --per-client each bot's own line follows its step. --expect-established exits 1
// unless every step ended with all its bots established and every step saw snapshots and
// final outcomes (the smoke test).
#include "evo/client/motion_prediction.hpp"
#include "evo/client/region_session.hpp"
#include "evo/network/session_grant.hpp"
#include <algorithm>
#include <array>
#include <atomic>
#include <charconv>
#include <chrono>
#include <cmath>
#include <cstdio>
#include <ctime>
#include <filesystem>
#include <fstream>
#include <iterator>
#include <map>
#include <memory>
#include <mutex>
#include <optional>
#include <random>
#include <string>
#include <string_view>
#include <thread>
#include <tuple>
#include <utility>
#include <vector>
#ifndef _WIN32
#include <arpa/inet.h>
#include <netinet/in.h>
#include <sys/resource.h>
#include <sys/socket.h>
#include <unistd.h>
#endif
namespace {
using namespace evo;
using namespace evo::network;
using Clock = std::chrono::steady_clock;
using std::chrono::milliseconds;
std::optional<std::uint64_t> number(std::string_view v) {
std::uint64_t n = 0;
const auto [end, ec] = std::from_chars(v.data(), v.data() + v.size(), n);
if (ec != std::errc{} || end != v.data() + v.size())
return std::nullopt;
return n;
}
std::string read(const std::filesystem::path& path) {
std::ifstream file{path, std::ios::binary};
return {std::istreambuf_iterator<char>{file}, std::istreambuf_iterator<char>{}};
}
std::uint64_t now_seconds() {
return static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count());
}
std::uint64_t micros(Clock::duration d) {
return static_cast<std::uint64_t>(std::max<std::int64_t>(
0, std::chrono::duration_cast<std::chrono::microseconds>(d).count()));
}
// Nearest rank over a copy.
std::uint64_t percentile(std::vector<std::uint64_t> v, double p) {
if (v.empty())
return 0;
std::ranges::sort(v);
const auto rank = static_cast<std::size_t>(std::ceil(p * static_cast<double>(v.size())));
return v.at(std::clamp<std::size_t>(rank, 1, v.size()) - 1);
}
double ms(std::uint64_t us) {
return static_cast<double>(us) / 1000.0;
}
struct Options {
std::string host, server_name, ca, issuer_key, grants_file, metrics_host;
std::uint64_t grants_per_bot{};
std::uint16_t port{}, metrics_port{};
std::uint64_t world{}, frame{}, region{}, seed{1}, lifetime{900};
std::vector<std::size_t> steps{20};
std::uint64_t hold_s{30}, warmup_s{5}, ramp_ms{100}, threads{4}, response_budget_ms{150};
bool per_client{}, expect_established{};
};
// What one bot measured since the last take.
struct Samples {
std::vector<std::uint64_t> response_us, receipt_us, outcome_us, login_us, correction_um,
gap_ticks;
std::uint64_t duplicate_actions{}; // Receipts saying duplicate: no outcome latency.
std::uint64_t snapshots{}, bytes{}, intents{}, refused_intents{}, logins{}, drops{},
login_failures{}, refused{};
std::map<unsigned, std::uint64_t> closes; // By session close reason (255: no Close).
std::uint64_t connected_polls{}; // Polls while established (starvation is judged on these).
};
void append(std::vector<std::uint64_t>& to, const std::vector<std::uint64_t>& from) {
to.insert(to.end(), from.begin(), from.end());
}
struct Bot {
std::size_t index{};
AccountId account;
std::uint64_t serial{};
std::unique_ptr<client::RegionSessionClient> c;
std::atomic<bool> active{false}, established{false};
Clock::time_point connect_at, retry_at, next_action, next_interest, next_turn;
client::MotionPredictor predictor;
client::InputLead lead;
std::uint64_t last_tick{}, last_view{}, last_bytes{}, next_action_id{1};
std::uint64_t applied{}; // Newest applied intention sequence seen in an own marker.
std::uint64_t used{}; // Bundle grants used (bundle mode).
std::atomic<bool> exhausted{false};
std::map<std::uint64_t, Clock::time_point> receipt_pending, response_pending, outcome_pending;
std::map<std::uint64_t, std::uint64_t> intent_tick;
std::int16_t vx{}, vy{};
std::optional<SessionInterest> home;
int interest_step{};
std::mt19937_64 rng;
Samples s;
};
class Load {
public:
// Signs with `key`, or logs in from `bundle` (concatenated grants, grants_per_bot each).
Load(Options o, std::optional<GrantSigningKey> key, std::vector<SessionGrant> bundle,
std::string address)
: o_{std::move(o)}, key_{std::move(key)}, bundle_{std::move(bundle)},
address_{std::move(address)}, authority_{read(o_.ca)} {
const auto most = *std::ranges::max_element(o_.steps);
bots_.reserve(most);
for (std::size_t i = 0; i < most; ++i) {
auto b = std::make_unique<Bot>();
b->index = i;
for (std::size_t k = 0; k < 8; ++k) {
b->account.bytes.at(k) = static_cast<std::uint8_t>((o_.seed >> (8U * k)) & 0xFFU);
b->account.bytes.at(8 + k) =
static_cast<std::uint8_t>(((i + 1) >> (8U * k)) & 0xFFU);
}
b->serial = now_seconds() * 64; // Later runs supersede earlier ones.
// Action ids are unique per account across runs too: a reused id is a duplicate,
// answered at once with the recorded outcome (10.9d: the third dev drill's 16 ms
// "outcomes" were duplicates of an earlier run's actions).
b->next_action_id = now_seconds() * 1000;
b->rng.seed(o_.seed * 1'000'003 + i);
bots_.push_back(std::move(b));
}
const auto threads = std::clamp<std::size_t>(o_.threads, 1, 64);
locks_ = std::vector<std::mutex>(threads);
for (std::size_t t = 0; t < threads; ++t)
workers_.emplace_back([this, t, threads] { work(t, threads); });
}
Load(const Load&) = delete;
Load& operator=(const Load&) = delete;
~Load() {
stop_ = true;
for (auto& w : workers_)
w.join();
}
void activate(std::size_t n) {
for (std::size_t i = 0; i < n && i < bots_.size(); ++i) {
if (bots_[i]->active.exchange(true))
continue;
std::this_thread::sleep_for(milliseconds{o_.ramp_ms});
}
}
// Every active bot's samples since the last take (cleared).
std::vector<Samples> take(std::size_t n) {
std::vector<Samples> out(n);
for (std::size_t t = 0; t < locks_.size(); ++t) {
const std::scoped_lock lock{locks_[t]};
for (std::size_t i = t; i < n; i += locks_.size())
out[i] = std::exchange(bots_[i]->s, Samples{});
}
return out;
}
[[nodiscard]] std::size_t exhausted(std::size_t n) const {
return static_cast<std::size_t>(
std::count_if(bots_.begin(), bots_.begin() + static_cast<std::ptrdiff_t>(n),
[](const auto& b) { return b->exhausted.load(); }));
}
[[nodiscard]] std::size_t established(std::size_t n) const {
return static_cast<std::size_t>(
std::count_if(bots_.begin(), bots_.begin() + static_cast<std::ptrdiff_t>(n),
[](const auto& b) { return b->established.load(); }));
}
private:
void work(std::size_t t, std::size_t threads) {
while (!stop_) {
{
const std::scoped_lock lock{locks_[t]};
const auto now = Clock::now();
for (std::size_t i = t; i < bots_.size(); i += threads)
if (bots_[i]->active)
step(*bots_[i], now);
}
std::this_thread::sleep_for(milliseconds{2});
}
const std::scoped_lock lock{locks_[t]};
for (std::size_t i = t; i < bots_.size(); i += threads)
if (auto& c = bots_[i]->c; c && c->state() == client::RegionSessionState::established)
c->close();
for (int k = 0; k < 50; ++k) { // Let the closes flush.
for (std::size_t i = t; i < bots_.size(); i += threads)
if (bots_[i]->c)
bots_[i]->c->poll(Clock::now());
std::this_thread::sleep_for(milliseconds{2});
}
}
void login(Bot& b, Clock::time_point now) {
std::optional<SessionGrant> grant;
if (key_) {
GrantClaims claims;
claims.account = b.account;
claims.serial = ++b.serial;
claims.world = {o_.world, o_.frame};
claims.region = o_.region;
claims.issued_at = now_seconds();
claims.expires_at = claims.issued_at + o_.lifetime;
claims.grant_id = (b.serial << 16U) ^ (b.index + 1);
if (auto signed_grant = key_->sign(claims))
grant = *signed_grant;
} else {
if (b.used >= o_.grants_per_bot) {
b.exhausted = true; // Out of grants: stays out.
b.retry_at = Clock::time_point::max();
return;
}
grant = bundle_.at((b.index * o_.grants_per_bot) + b.used++);
}
client::RegionSessionClientConfig config;
config.transport.server_address = address_;
config.transport.port = o_.port;
config.transport.server_name = o_.server_name;
config.transport.authority_pem = authority_;
config.transport.authority_pem_file = o_.ca;
if (!grant) {
b.retry_at = now + std::chrono::seconds{2};
return;
}
config.grant = *grant;
auto made = client::RegionSessionClient::connect(std::move(config), now);
if (!made) {
b.retry_at = now + std::chrono::seconds{2};
return;
}
b.c = std::move(*made);
b.connect_at = now;
b.last_tick = b.last_view = b.last_bytes = b.applied = 0;
b.receipt_pending.clear();
b.response_pending.clear();
b.outcome_pending.clear();
b.intent_tick.clear();
b.predictor.reset();
b.lead.reset();
b.home.reset();
}
void step(Bot& b, Clock::time_point now) {
if (!b.c) {
if (now >= b.retry_at)
login(b, now);
return;
}
auto& c = *b.c;
c.poll(now);
if (c.state() == client::RegionSessionState::closed) {
if (b.established.exchange(false))
++b.s.drops;
else
++b.s.login_failures;
if (const auto reason = c.close_reason();
reason && *reason != SessionCloseReason::connection_lost)
++b.s.closes[static_cast<unsigned>(*reason)];
else if (const auto cause = c.transport_cause()) // No session Close: 256 + cause.
++b.s.closes[256U + static_cast<unsigned>(*cause)];
else
++b.s.closes[255U];
const bool refused = c.close_reason() == SessionCloseReason::capacity ||
c.close_reason() == SessionCloseReason::rate_limited;
auto backoff = std::chrono::seconds{2};
if (refused) {
++b.s.refused;
// Refused for load before the grant was used: the region is shedding, so back
// off longer, and a bundle grant stays usable for the next try (10.9d).
backoff = std::chrono::seconds{10};
if (!key_ && b.used > 0)
--b.used;
}
b.c.reset();
b.retry_at =
now + backoff + milliseconds{std::uniform_int_distribution<int>{0, 500}(b.rng)};
return;
}
if (c.state() != client::RegionSessionState::established)
return;
++b.s.connected_polls;
if (!b.established.exchange(true)) {
++b.s.logins;
b.s.login_us.push_back(micros(now - b.connect_at));
b.next_action = now + milliseconds{std::uniform_int_distribution<int>{0, 2000}(b.rng)};
b.next_interest = now + std::chrono::seconds{10};
}
// Receipts: RTT, lead and refusals.
for (const auto& r : c.take_intention_receipts()) {
if (const auto it = b.receipt_pending.find(r.sequence); it != b.receipt_pending.end()) {
b.s.receipt_us.push_back(micros(now - it->second));
b.receipt_pending.erase(it);
}
const auto tick = b.intent_tick.find(r.sequence);
if (r.status == AdmitStatus::admitted) {
if (tick != b.intent_tick.end())
b.lead.admitted(tick->second, r.server_tick);
} else {
++b.s.refused_intents;
b.predictor.rejected(r.sequence);
b.response_pending.erase(r.sequence);
if (r.status == AdmitStatus::outside_window)
b.lead.too_late();
}
if (tick != b.intent_tick.end())
b.intent_tick.erase(tick);
}
// Snapshots: cadence, own marker (responsiveness, prediction errors), bytes.
if (const auto* v = c.view(); v && v->tick != b.last_view) {
if (b.last_view != 0 && v->tick > b.last_view)
b.s.gap_ticks.push_back(v->tick - b.last_view);
b.last_view = v->tick;
++b.s.snapshots;
if (!b.home && !v->environment.empty()) {
std::int64_t x0 = v->environment.front().voxel.x, x1 = x0;
std::int64_t y0 = v->environment.front().voxel.y, y1 = y0;
for (const auto& cell : v->environment) {
x0 = std::min(x0, cell.voxel.x);
x1 = std::max(x1, cell.voxel.x);
y0 = std::min(y0, cell.voxel.y);
y1 = std::max(y1, cell.voxel.y);
}
b.home = SessionInterest{(x0 + x1) / 2, (y0 + y1) / 2, v->slice_z, 7};
}
for (const auto& m : v->markers) {
if (!m.own)
continue;
b.predictor.authoritative({v->tick, m.applied_sequence, m.state, m.area});
if (m.applied_sequence > b.applied) {
b.applied = m.applied_sequence;
while (!b.response_pending.empty() &&
b.response_pending.begin()->first <= b.applied) {
b.s.response_us.push_back(micros(now - b.response_pending.begin()->second));
b.response_pending.erase(b.response_pending.begin());
}
}
}
}
for (const auto e : b.predictor.take_errors())
b.s.correction_um.push_back(e);
const auto bytes = c.snapshot_bytes();
b.s.bytes += bytes - std::min(bytes, b.last_bytes);
b.last_bytes = bytes;
// Receipts first: a duplicate's recorded outcome is not a latency sample.
for (const auto& r : c.take_action_receipts())
if (r.status == AdmitStatus::duplicate) {
++b.s.duplicate_actions;
b.outcome_pending.erase(r.action_id);
}
// Outcomes: final ones only.
for (const auto& out : c.take_action_outcomes())
if (out.result == ActionResult::committed_no_effect ||
out.result == ActionResult::not_committed)
if (const auto it = b.outcome_pending.find(out.action_id);
it != b.outcome_pending.end()) {
b.s.outcome_us.push_back(micros(now - it->second));
b.outcome_pending.erase(it);
}
// Once per server tick: an intention, a prediction, sometimes an action or interest.
const auto tick = c.server_tick();
if (tick == 0 || tick == b.last_tick)
return;
b.last_tick = tick;
if (now >= b.next_turn) {
const double angle = std::uniform_real_distribution<double>{0, 6.283185307}(b.rng);
b.vx = static_cast<std::int16_t>(std::lround(700.0 * std::cos(angle)));
b.vy = static_cast<std::int16_t>(std::lround(700.0 * std::sin(angle)));
b.next_turn = now + std::chrono::seconds{2};
}
const auto target = tick + b.lead.lead();
if (!b.predictor.full())
if (const auto seq = c.send_intention(target, b.vx, b.vy)) {
++b.s.intents;
(void)b.predictor.input({*seq, target, {b.vx, b.vy}});
b.receipt_pending[*seq] = now;
b.response_pending[*seq] = now;
b.intent_tick[*seq] = target;
for (auto* pending : {&b.receipt_pending, &b.response_pending})
while (pending->size() > 256)
pending->erase(pending->begin());
while (b.intent_tick.size() > 256)
b.intent_tick.erase(b.intent_tick.begin());
}
(void)b.predictor.predict(target);
(void)b.predictor.take_correction();
if (now >= b.next_action) {
const auto id = b.next_action_id++;
if (c.send_action(id, ActionKind::probe, tick + 3)) {
b.outcome_pending[id] = now;
while (b.outcome_pending.size() > 64)
b.outcome_pending.erase(b.outcome_pending.begin());
}
b.next_action = now + std::chrono::seconds{2};
}
if (now >= b.next_interest && b.home) {
auto in = *b.home;
in.x += (b.interest_step++ % 2 == 0) ? 1 : 0; // Alternate: a fresh baseline each.
(void)c.set_interest(in);
b.next_interest = now + std::chrono::seconds{10};
}
}
Options o_;
std::optional<GrantSigningKey> key_;
std::vector<SessionGrant> bundle_;
std::string address_, authority_;
std::vector<std::unique_ptr<Bot>> bots_;
std::vector<std::mutex> locks_;
std::vector<std::thread> workers_;
std::atomic<bool> stop_{false};
};
// The region's /metrics (evo_m10_* and evo_region_tick_debt), unlabelled series only.
std::map<std::string, std::uint64_t> scrape(const Options& o) {
std::map<std::string, std::uint64_t> out;
#ifndef _WIN32
if (o.metrics_port == 0)
return out;
const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0);
if (fd < 0)
return out;
sockaddr_in a{};
a.sin_family = AF_INET;
a.sin_port = htons(o.metrics_port);
timeval timeout{3, 0};
::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout);
std::string raw;
if (::inet_pton(AF_INET, o.metrics_host.c_str(), &a.sin_addr) == 1 &&
::connect(fd, reinterpret_cast<const sockaddr*>(&a), sizeof a) == 0) {
const std::string request = "GET /metrics HTTP/1.1\r\nHost: load\r\n\r\n";
if (::send(fd, request.data(), request.size(), MSG_NOSIGNAL) ==
static_cast<ssize_t>(request.size())) {
std::array<char, 4096> chunk{};
for (ssize_t n = 0; (n = ::recv(fd, chunk.data(), chunk.size(), 0)) > 0;)
raw.append(chunk.data(), static_cast<std::size_t>(n));
}
}
::close(fd);
std::size_t at = 0;
while (at < raw.size()) {
const auto end = std::min(raw.find('\n', at), raw.size());
const std::string_view line{raw.data() + at, end - at};
at = end + 1;
if (!line.starts_with("evo_m10_") && !line.starts_with("evo_region_tick_debt"))
continue;
const auto space = line.find(' ');
if (space == std::string_view::npos || line.find('{') != std::string_view::npos)
continue;
if (const auto v = number(line.substr(space + 1)))
out[std::string{line.substr(0, space)}] = *v;
}
#else
(void)o;
#endif
return out;
}
double cpu_seconds() {
#ifndef _WIN32
rusage u{};
::getrusage(RUSAGE_SELF, &u);
return static_cast<double>(u.ru_utime.tv_sec + u.ru_stime.tv_sec) +
(static_cast<double>(u.ru_utime.tv_usec + u.ru_stime.tv_usec) / 1e6);
#else
return 0;
#endif
}
void line(const char* name, const std::vector<std::uint64_t>& pooled,
const std::vector<std::uint64_t>& client_p95, double scale, const char* unit) {
const auto f = [&](std::uint64_t v) { return static_cast<double>(v) / scale; };
std::printf("load %s_%s p50/p95/p99/max=%.1f/%.1f/%.1f/%.1f n=%zu client_p95 "
"min/med/max=%.1f/%.1f/%.1f\n",
name, unit, f(percentile(pooled, 0.50)), f(percentile(pooled, 0.95)),
f(percentile(pooled, 0.99)), f(percentile(pooled, 1.0)), pooled.size(),
f(percentile(client_p95, 0.0)), f(percentile(client_p95, 0.5)),
f(percentile(client_p95, 1.0)));
}
// One step's report; true when within budget.
bool report(const Options& o, std::size_t n, std::size_t established, std::size_t exhausted,
std::pair<std::uint64_t, std::uint64_t> hold_utc, const std::vector<Samples>& all,
double seconds, double cpu, const std::map<std::string, std::uint64_t>& server) {
Samples pooled;
std::vector<std::uint64_t> response95, receipt95, outcome95, login95, correction95, rates, kib;
for (const auto& s : all) {
append(pooled.response_us, s.response_us);
append(pooled.receipt_us, s.receipt_us);
append(pooled.outcome_us, s.outcome_us);
append(pooled.login_us, s.login_us);
append(pooled.correction_um, s.correction_um);
append(pooled.gap_ticks, s.gap_ticks);
pooled.snapshots += s.snapshots;
pooled.intents += s.intents;
pooled.refused_intents += s.refused_intents;
pooled.duplicate_actions += s.duplicate_actions;
pooled.logins += s.logins;
pooled.drops += s.drops;
pooled.login_failures += s.login_failures;
for (const auto& [why, count] : s.closes)
pooled.closes[why] += count;
pooled.refused += s.refused;
response95.push_back(percentile(s.response_us, 0.95));
receipt95.push_back(percentile(s.receipt_us, 0.95));
outcome95.push_back(percentile(s.outcome_us, 0.95));
if (!s.login_us.empty())
login95.push_back(percentile(s.login_us, 0.95));
correction95.push_back(percentile(s.correction_um, 0.95));
rates.push_back(static_cast<std::uint64_t>(
std::lround(1000.0 * static_cast<double>(s.snapshots) / std::max(seconds, 0.001))));
kib.push_back(static_cast<std::uint64_t>(
std::lround(static_cast<double>(s.bytes) / 1024.0 / std::max(seconds, 0.001))));
}
std::string reasons;
for (const auto& [why, count] : pooled.closes)
reasons += (reasons.empty() ? "" : ",") + std::to_string(why) + ":" + std::to_string(count);
std::printf("load step N=%zu established=%zu logins=%llu drops=%llu login_failures=%llu "
"refused=%llu seconds=%.1f exhausted=%zu hold_utc=%llu..%llu close_reasons=%s\n",
n, established, static_cast<unsigned long long>(pooled.logins),
static_cast<unsigned long long>(pooled.drops),
static_cast<unsigned long long>(pooled.login_failures),
static_cast<unsigned long long>(pooled.refused), seconds, exhausted,
static_cast<unsigned long long>(hold_utc.first),
static_cast<unsigned long long>(hold_utc.second),
reasons.empty() ? "-" : reasons.c_str());
line("response", pooled.response_us, response95, 1000.0, "ms");
line("receipt", pooled.receipt_us, receipt95, 1000.0, "ms");
line("outcome", pooled.outcome_us, outcome95, 1000.0, "ms");
line("login", pooled.login_us, login95, 1000.0, "ms");
line("correction", pooled.correction_um, correction95, 1000.0, "mm");
std::printf(
"load snapshots hz min/med/max=%.2f/%.2f/%.2f gap_ticks p99/max=%llu/%llu "
"kib_s med/max=%llu/%llu intents=%llu refused_intents=%llu duplicate_actions=%llu\n",
static_cast<double>(percentile(rates, 0.0)) / 1000.0,
static_cast<double>(percentile(rates, 0.5)) / 1000.0,
static_cast<double>(percentile(rates, 1.0)) / 1000.0,
static_cast<unsigned long long>(percentile(pooled.gap_ticks, 0.99)),
static_cast<unsigned long long>(percentile(pooled.gap_ticks, 1.0)),
static_cast<unsigned long long>(percentile(kib, 0.5)),
static_cast<unsigned long long>(percentile(kib, 1.0)),
static_cast<unsigned long long>(pooled.intents),
static_cast<unsigned long long>(pooled.refused_intents),
static_cast<unsigned long long>(pooled.duplicate_actions));
std::printf("load client_cpu_pct=%.0f\n", 100.0 * cpu / std::max(seconds, 0.001));
if (!server.empty()) {
std::printf("load server");
for (const auto& [k, v] : server)
std::printf(" %s=%llu", k.c_str(), static_cast<unsigned long long>(v));
std::printf("\n");
}
// Starvation: throughput must not hide a client left behind. Judged among the bots that
// were connected during the hold (the others are reported as not established).
std::vector<std::uint64_t> connected_rates, connected_response;
for (std::size_t i = 0; i < all.size(); ++i)
if (all[i].connected_polls > 0) {
connected_rates.push_back(rates[i]);
connected_response.push_back(response95[i]);
}
const auto median_rate = percentile(connected_rates, 0.5);
const auto median_response = percentile(connected_response, 0.5);
const auto budget_us = o.response_budget_ms * 1000;
std::vector<std::size_t> starved;
for (std::size_t i = 0; i < all.size(); ++i)
if (all[i].connected_polls > 0 &&
(rates[i] * 5 < median_rate * 4 ||
(response95[i] > 2 * median_response && response95[i] > budget_us)))
starved.push_back(i);
std::printf("load starved %zu", starved.size());
for (const auto i : starved)
std::printf(" %zu", i);
std::printf("\n");
std::vector<std::string> why;
if (established < n)
why.emplace_back("not-established");
if (pooled.refused > 0 || pooled.login_failures > 0)
why.emplace_back("refused");
if (pooled.drops > 0)
why.emplace_back("drops");
if (!starved.empty())
why.emplace_back("starved");
if (pooled.response_us.empty() || percentile(pooled.response_us, 0.95) > budget_us)
why.emplace_back("response");
if (pooled.outcome_us.empty() || percentile(pooled.outcome_us, 0.95) > 1'500'000)
why.emplace_back("outcome");
const auto get = [&](const char* k) {
const auto it = server.find(k);
return it == server.end() ? 0 : it->second;
};
if (get("evo_m10_tick_work_p99_microseconds") > 25'000)
why.emplace_back("tick-work");
if (get("evo_m10_refusing") != 0 || get("evo_m10_shed_level") != 0)
why.emplace_back("governor");
if (get("evo_region_tick_debt") != 0)
why.emplace_back("tick-debt");
std::printf("load verdict N=%zu within_budget=%d", n, why.empty() ? 1 : 0);
for (const auto& w : why)
std::printf(" %s", w.c_str());
std::printf("\n");
if (o.per_client)
for (std::size_t i = 0; i < all.size(); ++i)
std::printf("load client %zu snapshots=%llu response_p95_ms=%.1f receipt_p95_ms=%.1f "
"outcome_p95_ms=%.1f correction_p95_mm=%.1f drops=%llu "
"login_failures=%llu\n",
i, static_cast<unsigned long long>(all[i].snapshots), ms(response95[i]),
ms(receipt95[i]), ms(outcome95[i]),
static_cast<double>(correction95[i]) / 1000.0,
static_cast<unsigned long long>(all[i].drops),
static_cast<unsigned long long>(all[i].login_failures));
std::fflush(stdout);
return why.empty();
}
std::optional<std::pair<std::string, std::uint16_t>> endpoint(std::string_view v) {
const auto colon = v.rfind(':');
if (colon == std::string_view::npos || colon == 0)
return std::nullopt;
const auto port = number(v.substr(colon + 1));
if (!port || *port == 0 || *port > 65535)
return std::nullopt;
return std::pair{std::string{v.substr(0, colon)}, static_cast<std::uint16_t>(*port)};
}
std::optional<std::vector<std::size_t>> steps_of(std::string_view v) {
std::vector<std::size_t> out;
while (!v.empty()) {
const auto comma = v.find(',');
const auto n = number(v.substr(0, comma));
if (!n || *n == 0 || *n > 256 || (!out.empty() && *n < out.back()))
return std::nullopt;
out.push_back(*n);
v.remove_prefix(comma == std::string_view::npos ? v.size() : comma + 1);
}
if (out.empty())
return std::nullopt;
return out;
}
} // namespace
int main(int argc, char** argv) {
const std::vector<std::string_view> args(argv + 1, argv + argc);
Options o;
bool ok = true;
for (std::size_t i = 0; i < args.size() && ok; ++i) {
const auto flag = args[i];
if (flag == "--per-client") {
o.per_client = true;
continue;
}
if (flag == "--expect-established") {
o.expect_established = true;
continue;
}
if (i + 1 >= args.size()) {
ok = false;
break;
}
const auto value = args[++i];
const auto n = number(value);
if (flag == "--connect") {
const auto e = endpoint(value);
ok = e.has_value();
if (e)
std::tie(o.host, o.port) = *e;
} else if (flag == "--metrics") {
const auto e = endpoint(value);
ok = e.has_value();
if (e)
std::tie(o.metrics_host, o.metrics_port) = *e;
} else if (flag == "--server-name")
o.server_name = value;
else if (flag == "--ca")
o.ca = value;
else if (flag == "--issuer-key")
o.issuer_key = value;
else if (flag == "--grants")
o.grants_file = value;
else if (n && flag == "--grants-per-bot" && *n >= 1 && *n <= 64)
o.grants_per_bot = *n;
else if (flag == "--steps") {
const auto s = steps_of(value);
ok = s.has_value();
if (s)
o.steps = *s;
} else if (n && flag == "--world")
o.world = *n;
else if (n && flag == "--frame")
o.frame = *n;
else if (n && flag == "--region")
o.region = *n;
else if (n && flag == "--seed" && *n != 0)
o.seed = *n;
else if (n && flag == "--lifetime" && *n >= 60 && *n <= 14400)
o.lifetime = *n;
else if (n && flag == "--hold" && *n >= 1 && *n <= 3600)
o.hold_s = *n;
else if (n && flag == "--warmup" && *n <= 600)
o.warmup_s = *n;
else if (n && flag == "--ramp-ms" && *n <= 10'000)
o.ramp_ms = *n;
else if (n && flag == "--threads" && *n >= 1 && *n <= 64)
o.threads = *n;
else if (n && flag == "--response-budget-ms" && *n >= 1)
o.response_budget_ms = *n;
else
ok = false;
}
const bool signs = !o.issuer_key.empty();
if (!ok || o.host.empty() || o.server_name.empty() || o.ca.empty() ||
signs == !o.grants_file.empty() || (!signs && o.grants_per_bot == 0) || o.world == 0 ||
o.frame == 0 || o.region == 0) {
std::fputs("usage: evo-load --connect HOST:PORT --server-name NAME --ca FILE "
"(--issuer-key FILE | --grants FILE --grants-per-bot C) --world N "
"--frame N --region N [--steps N[,N]...] "
"[--hold S] [--warmup S] [--ramp-ms MS] [--threads T] [--seed X] "
"[--lifetime S] [--metrics HOST:PORT] [--response-budget-ms MS] "
"[--per-client] [--expect-established]\n",
stderr);
return 2;
}
std::optional<GrantSigningKey> key;
std::vector<SessionGrant> bundle;
if (signs) {
const auto scalar = read(o.issuer_key);
auto imported = GrantSigningKey::import(
DeploymentEnvironment::dev, 1,
std::span{reinterpret_cast<const std::uint8_t*>(scalar.data()), scalar.size()});
if (!imported) {
std::puts("error issuer-key");
return 2;
}
key.emplace(std::move(*imported));
} else {
const auto bytes = read(o.grants_file);
const auto needed = *std::ranges::max_element(o.steps) * o.grants_per_bot;
if (bytes.size() % session_grant_bytes != 0 ||
bytes.size() / session_grant_bytes < needed) {
std::printf("error grants (need %llu grants: bots x grants-per-bot)\n",
static_cast<unsigned long long>(needed));
return 2;
}
bundle.resize(bytes.size() / session_grant_bytes);
for (std::size_t i = 0; i < bundle.size(); ++i)
std::copy_n(bytes.begin() + static_cast<std::ptrdiff_t>(i * session_grant_bytes),
session_grant_bytes, reinterpret_cast<char*>(bundle[i].data()));
}
const auto address = quic::resolve_ipv4(o.host);
if (!address) {
std::puts("error resolve");
return 2;
}
std::printf("load start steps=%zu hold=%llu warmup=%llu threads=%llu budget_ms=%llu\n",
o.steps.size(), static_cast<unsigned long long>(o.hold_s),
static_cast<unsigned long long>(o.warmup_s),
static_cast<unsigned long long>(o.threads),
static_cast<unsigned long long>(o.response_budget_ms));
std::fflush(stdout);
bool smoke_ok = true;
std::optional<std::size_t> sustainable, saturated;
{
Load load{o, std::move(key), std::move(bundle), *address};
for (const auto n : o.steps) {
load.activate(n);
std::this_thread::sleep_for(std::chrono::seconds{o.warmup_s});
// The ramp's and warmup's logins (and closes) belong to this step; the rest of
// the warmup is discarded.
const auto warm = load.take(n);
const auto started = Clock::now();
const auto hold_start = now_seconds();
const auto cpu0 = cpu_seconds();
std::this_thread::sleep_for(std::chrono::seconds{o.hold_s});
auto samples = load.take(n);
for (std::size_t i = 0; i < n; ++i) {
append(samples[i].login_us, warm[i].login_us);
samples[i].logins += warm[i].logins;
samples[i].drops += warm[i].drops;
samples[i].login_failures += warm[i].login_failures;
for (const auto& [why, count] : warm[i].closes)
samples[i].closes[why] += count;
samples[i].refused += warm[i].refused;
}
const auto seconds = std::chrono::duration<double>(Clock::now() - started).count();
const auto established = load.established(n);
const bool within =
report(o, n, established, load.exhausted(n), {hold_start, now_seconds()}, samples,
seconds, cpu_seconds() - cpu0, scrape(o));
if (within && !saturated)
sustainable = n;
if (!within && !saturated)
saturated = n;
std::uint64_t snapshots = 0, outcomes = 0;
for (const auto& s : samples) {
snapshots += s.snapshots;
outcomes += s.outcome_us.size();
}
smoke_ok = smoke_ok && established == n && snapshots > 0 && outcomes > 0;
}
}
std::printf("load result sustainable=%zu saturated_at=%zu\n", sustainable.value_or(0),
saturated.value_or(0));
std::fflush(stdout);
return o.expect_established && !smoke_ok ? 1 : 0;
}