Foundation / checks (client) (push) Successful in 14s
Foundation / checks (checks) (push) Successful in 16s
Foundation / checks (deploy) (push) Successful in 15s
Foundation / checks (network) (push) Successful in 18s
Foundation / checks (postgres) (push) Successful in 1m34s
Foundation / checks (ecs) (push) Successful in 1m53s
Foundation / checks (tsan) (push) Successful in 5m44s
Foundation / server-image (push) Successful in 1m0s
Foundation / postgres-tools (push) Successful in 1m38s
Foundation / image (push) Successful in 2m34s
- IntentLimits::watermark_coalesce (evo-server: every 4, coalesce 4); ADR-0092 amendment. - m10 load-report splits poll/replicate and commit-tick/other-tick maxima. - Budgets v2: work p95 <= 25 ms over all ticks, commit ticks <= 50 ms (ADR-0097); moving the commit off the tick is a recorded follow-up. - evo-load: time-based action ids, duplicate_actions, receipts before outcomes; drill ramps at 500 ms and judges by v2. - Refresh the dev session window. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
861 lines
39 KiB
C++
861 lines
39 KiB
C++
// M10.9b multi-client load generator (ADR-0097). Development and qualification tool: only
|
|
// runs against the evolution-dev reference worker count as evidence (10.9d); local runs are
|
|
// smoke tests and development references.
|
|
//
|
|
// evo-load --connect HOST:PORT --server-name NAME --ca FILE
|
|
// (--issuer-key FILE | --grants FILE --grants-per-bot C)
|
|
// --world N --frame N --region N [--steps N[,N]...] [--hold S] [--warmup S]
|
|
// [--ramp-ms MS] [--threads T] [--seed X] [--lifetime S] [--metrics HOST:PORT]
|
|
// [--response-budget-ms MS] [--per-client] [--expect-established]
|
|
//
|
|
// Each simulated client (bot) is a RegionSessionClient with its own account (from --seed and
|
|
// its index) logging in with a dev grant signed here (the issuer key never leaves this
|
|
// process). Bots run on T worker threads (each owns a slice). Every server tick a bot sends
|
|
// one intention for tick + its input lead (a random walk turning every 2 s) and predicts its
|
|
// probe marker; every 2 s it sends a probe action; every 10 s it moves its interest by a
|
|
// cell (a fresh baseline). A closed bot logs in again after 2 s with the next serial; one
|
|
// refused for load (capacity, rate_limited) waits 10 s and, in bundle mode, reuses the same
|
|
// grant (the region refused it before using it).
|
|
// M10.9d: with --grants (a bundle from `evo-session-probe grants --accounts N --count C`,
|
|
// e.g. made where the dev issuer key lives), bot i logs in with grants i*C .. i*C+C-1 in
|
|
// order instead of signing; a bot that used all C stays out (counted as exhausted).
|
|
//
|
|
// Steps: the bot count rises to each step's N (one login every --ramp-ms), waits --warmup
|
|
// seconds, then measures for --hold seconds. Bots stay connected into the next step. Per
|
|
// step it prints:
|
|
// load step N=.. established=.. logins=.. drops=.. login_failures=.. refused=.. seconds=..
|
|
// exhausted=.. hold_utc=START..END (unix seconds of the hold, to match server samples)
|
|
// close_reasons=REASON:COUNT,... (session-v1 close reasons; without a session Close,
|
|
// 256 + the QUIC transport cause, or 255 when unknown)
|
|
// (drops: established sessions closed; login failures: closed before Welcome, including
|
|
// transport refusals at the connection limit; refused: closed with capacity or
|
|
// rate_limited, either way. Logins, drops and failures include the step's ramp and warmup;
|
|
// everything else is the hold)
|
|
// load response_ms p50/p95/p99/max=.. (input -> first own-marker snapshot applying it)
|
|
// client_p95 min/med/max=..
|
|
// load receipt_ms ... (intention -> receipt)
|
|
// load outcome_ms ... (action -> final outcome)
|
|
// load login_ms ... (connect -> Welcome)
|
|
// load correction_mm ... (prediction error per authoritative marker)
|
|
// load snapshots hz min/med/max=.. gap_ticks p99/max=.. kib_s med/max=..
|
|
// intents=.. refused_intents=..
|
|
// load client_cpu_pct=.. (this process, all threads; Linux)
|
|
// load server ... (with --metrics: the region's evo_m10_* gauges at step end)
|
|
// load starved N [indices] (among bots connected during the hold: snapshot rate < 80 % of
|
|
// the median, or a response p95 over both twice the median and the budget)
|
|
// load verdict N=.. within_budget=0|1 [reasons]
|
|
// With --per-client each bot's own line follows its step. --expect-established exits 1
|
|
// unless every step ended with all its bots established and every step saw snapshots and
|
|
// final outcomes (the smoke test).
|
|
#include "evo/client/motion_prediction.hpp"
|
|
#include "evo/client/region_session.hpp"
|
|
#include "evo/network/session_grant.hpp"
|
|
#include <algorithm>
|
|
#include <array>
|
|
#include <atomic>
|
|
#include <charconv>
|
|
#include <chrono>
|
|
#include <cmath>
|
|
#include <cstdio>
|
|
#include <ctime>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <iterator>
|
|
#include <map>
|
|
#include <memory>
|
|
#include <mutex>
|
|
#include <optional>
|
|
#include <random>
|
|
#include <string>
|
|
#include <string_view>
|
|
#include <thread>
|
|
#include <tuple>
|
|
#include <utility>
|
|
#include <vector>
|
|
#ifndef _WIN32
|
|
#include <arpa/inet.h>
|
|
#include <netinet/in.h>
|
|
#include <sys/resource.h>
|
|
#include <sys/socket.h>
|
|
#include <unistd.h>
|
|
#endif
|
|
|
|
namespace {
|
|
using namespace evo;
|
|
using namespace evo::network;
|
|
using Clock = std::chrono::steady_clock;
|
|
using std::chrono::milliseconds;
|
|
|
|
std::optional<std::uint64_t> number(std::string_view v) {
|
|
std::uint64_t n = 0;
|
|
const auto [end, ec] = std::from_chars(v.data(), v.data() + v.size(), n);
|
|
if (ec != std::errc{} || end != v.data() + v.size())
|
|
return std::nullopt;
|
|
return n;
|
|
}
|
|
std::string read(const std::filesystem::path& path) {
|
|
std::ifstream file{path, std::ios::binary};
|
|
return {std::istreambuf_iterator<char>{file}, std::istreambuf_iterator<char>{}};
|
|
}
|
|
std::uint64_t now_seconds() {
|
|
return static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(
|
|
std::chrono::system_clock::now().time_since_epoch())
|
|
.count());
|
|
}
|
|
std::uint64_t micros(Clock::duration d) {
|
|
return static_cast<std::uint64_t>(std::max<std::int64_t>(
|
|
0, std::chrono::duration_cast<std::chrono::microseconds>(d).count()));
|
|
}
|
|
// Nearest rank over a copy.
|
|
std::uint64_t percentile(std::vector<std::uint64_t> v, double p) {
|
|
if (v.empty())
|
|
return 0;
|
|
std::ranges::sort(v);
|
|
const auto rank = static_cast<std::size_t>(std::ceil(p * static_cast<double>(v.size())));
|
|
return v.at(std::clamp<std::size_t>(rank, 1, v.size()) - 1);
|
|
}
|
|
double ms(std::uint64_t us) {
|
|
return static_cast<double>(us) / 1000.0;
|
|
}
|
|
|
|
struct Options {
|
|
std::string host, server_name, ca, issuer_key, grants_file, metrics_host;
|
|
std::uint64_t grants_per_bot{};
|
|
std::uint16_t port{}, metrics_port{};
|
|
std::uint64_t world{}, frame{}, region{}, seed{1}, lifetime{900};
|
|
std::vector<std::size_t> steps{20};
|
|
std::uint64_t hold_s{30}, warmup_s{5}, ramp_ms{100}, threads{4}, response_budget_ms{150};
|
|
bool per_client{}, expect_established{};
|
|
};
|
|
|
|
// What one bot measured since the last take.
|
|
struct Samples {
|
|
std::vector<std::uint64_t> response_us, receipt_us, outcome_us, login_us, correction_um,
|
|
gap_ticks;
|
|
std::uint64_t duplicate_actions{}; // Receipts saying duplicate: no outcome latency.
|
|
std::uint64_t snapshots{}, bytes{}, intents{}, refused_intents{}, logins{}, drops{},
|
|
login_failures{}, refused{};
|
|
std::map<unsigned, std::uint64_t> closes; // By session close reason (255: no Close).
|
|
std::uint64_t connected_polls{}; // Polls while established (starvation is judged on these).
|
|
};
|
|
void append(std::vector<std::uint64_t>& to, const std::vector<std::uint64_t>& from) {
|
|
to.insert(to.end(), from.begin(), from.end());
|
|
}
|
|
|
|
struct Bot {
|
|
std::size_t index{};
|
|
AccountId account;
|
|
std::uint64_t serial{};
|
|
std::unique_ptr<client::RegionSessionClient> c;
|
|
std::atomic<bool> active{false}, established{false};
|
|
Clock::time_point connect_at, retry_at, next_action, next_interest, next_turn;
|
|
client::MotionPredictor predictor;
|
|
client::InputLead lead;
|
|
std::uint64_t last_tick{}, last_view{}, last_bytes{}, next_action_id{1};
|
|
std::uint64_t applied{}; // Newest applied intention sequence seen in an own marker.
|
|
std::uint64_t used{}; // Bundle grants used (bundle mode).
|
|
std::atomic<bool> exhausted{false};
|
|
std::map<std::uint64_t, Clock::time_point> receipt_pending, response_pending, outcome_pending;
|
|
std::map<std::uint64_t, std::uint64_t> intent_tick;
|
|
std::int16_t vx{}, vy{};
|
|
std::optional<SessionInterest> home;
|
|
int interest_step{};
|
|
std::mt19937_64 rng;
|
|
Samples s;
|
|
};
|
|
|
|
class Load {
|
|
public:
|
|
// Signs with `key`, or logs in from `bundle` (concatenated grants, grants_per_bot each).
|
|
Load(Options o, std::optional<GrantSigningKey> key, std::vector<SessionGrant> bundle,
|
|
std::string address)
|
|
: o_{std::move(o)}, key_{std::move(key)}, bundle_{std::move(bundle)},
|
|
address_{std::move(address)}, authority_{read(o_.ca)} {
|
|
const auto most = *std::ranges::max_element(o_.steps);
|
|
bots_.reserve(most);
|
|
for (std::size_t i = 0; i < most; ++i) {
|
|
auto b = std::make_unique<Bot>();
|
|
b->index = i;
|
|
for (std::size_t k = 0; k < 8; ++k) {
|
|
b->account.bytes.at(k) = static_cast<std::uint8_t>((o_.seed >> (8U * k)) & 0xFFU);
|
|
b->account.bytes.at(8 + k) =
|
|
static_cast<std::uint8_t>(((i + 1) >> (8U * k)) & 0xFFU);
|
|
}
|
|
b->serial = now_seconds() * 64; // Later runs supersede earlier ones.
|
|
// Action ids are unique per account across runs too: a reused id is a duplicate,
|
|
// answered at once with the recorded outcome (10.9d: the third dev drill's 16 ms
|
|
// "outcomes" were duplicates of an earlier run's actions).
|
|
b->next_action_id = now_seconds() * 1000;
|
|
b->rng.seed(o_.seed * 1'000'003 + i);
|
|
bots_.push_back(std::move(b));
|
|
}
|
|
const auto threads = std::clamp<std::size_t>(o_.threads, 1, 64);
|
|
locks_ = std::vector<std::mutex>(threads);
|
|
for (std::size_t t = 0; t < threads; ++t)
|
|
workers_.emplace_back([this, t, threads] { work(t, threads); });
|
|
}
|
|
Load(const Load&) = delete;
|
|
Load& operator=(const Load&) = delete;
|
|
~Load() {
|
|
stop_ = true;
|
|
for (auto& w : workers_)
|
|
w.join();
|
|
}
|
|
void activate(std::size_t n) {
|
|
for (std::size_t i = 0; i < n && i < bots_.size(); ++i) {
|
|
if (bots_[i]->active.exchange(true))
|
|
continue;
|
|
std::this_thread::sleep_for(milliseconds{o_.ramp_ms});
|
|
}
|
|
}
|
|
// Every active bot's samples since the last take (cleared).
|
|
std::vector<Samples> take(std::size_t n) {
|
|
std::vector<Samples> out(n);
|
|
for (std::size_t t = 0; t < locks_.size(); ++t) {
|
|
const std::scoped_lock lock{locks_[t]};
|
|
for (std::size_t i = t; i < n; i += locks_.size())
|
|
out[i] = std::exchange(bots_[i]->s, Samples{});
|
|
}
|
|
return out;
|
|
}
|
|
[[nodiscard]] std::size_t exhausted(std::size_t n) const {
|
|
return static_cast<std::size_t>(
|
|
std::count_if(bots_.begin(), bots_.begin() + static_cast<std::ptrdiff_t>(n),
|
|
[](const auto& b) { return b->exhausted.load(); }));
|
|
}
|
|
[[nodiscard]] std::size_t established(std::size_t n) const {
|
|
return static_cast<std::size_t>(
|
|
std::count_if(bots_.begin(), bots_.begin() + static_cast<std::ptrdiff_t>(n),
|
|
[](const auto& b) { return b->established.load(); }));
|
|
}
|
|
|
|
private:
|
|
void work(std::size_t t, std::size_t threads) {
|
|
while (!stop_) {
|
|
{
|
|
const std::scoped_lock lock{locks_[t]};
|
|
const auto now = Clock::now();
|
|
for (std::size_t i = t; i < bots_.size(); i += threads)
|
|
if (bots_[i]->active)
|
|
step(*bots_[i], now);
|
|
}
|
|
std::this_thread::sleep_for(milliseconds{2});
|
|
}
|
|
const std::scoped_lock lock{locks_[t]};
|
|
for (std::size_t i = t; i < bots_.size(); i += threads)
|
|
if (auto& c = bots_[i]->c; c && c->state() == client::RegionSessionState::established)
|
|
c->close();
|
|
for (int k = 0; k < 50; ++k) { // Let the closes flush.
|
|
for (std::size_t i = t; i < bots_.size(); i += threads)
|
|
if (bots_[i]->c)
|
|
bots_[i]->c->poll(Clock::now());
|
|
std::this_thread::sleep_for(milliseconds{2});
|
|
}
|
|
}
|
|
void login(Bot& b, Clock::time_point now) {
|
|
std::optional<SessionGrant> grant;
|
|
if (key_) {
|
|
GrantClaims claims;
|
|
claims.account = b.account;
|
|
claims.serial = ++b.serial;
|
|
claims.world = {o_.world, o_.frame};
|
|
claims.region = o_.region;
|
|
claims.issued_at = now_seconds();
|
|
claims.expires_at = claims.issued_at + o_.lifetime;
|
|
claims.grant_id = (b.serial << 16U) ^ (b.index + 1);
|
|
if (auto signed_grant = key_->sign(claims))
|
|
grant = *signed_grant;
|
|
} else {
|
|
if (b.used >= o_.grants_per_bot) {
|
|
b.exhausted = true; // Out of grants: stays out.
|
|
b.retry_at = Clock::time_point::max();
|
|
return;
|
|
}
|
|
grant = bundle_.at((b.index * o_.grants_per_bot) + b.used++);
|
|
}
|
|
client::RegionSessionClientConfig config;
|
|
config.transport.server_address = address_;
|
|
config.transport.port = o_.port;
|
|
config.transport.server_name = o_.server_name;
|
|
config.transport.authority_pem = authority_;
|
|
config.transport.authority_pem_file = o_.ca;
|
|
if (!grant) {
|
|
b.retry_at = now + std::chrono::seconds{2};
|
|
return;
|
|
}
|
|
config.grant = *grant;
|
|
auto made = client::RegionSessionClient::connect(std::move(config), now);
|
|
if (!made) {
|
|
b.retry_at = now + std::chrono::seconds{2};
|
|
return;
|
|
}
|
|
b.c = std::move(*made);
|
|
b.connect_at = now;
|
|
b.last_tick = b.last_view = b.last_bytes = b.applied = 0;
|
|
b.receipt_pending.clear();
|
|
b.response_pending.clear();
|
|
b.outcome_pending.clear();
|
|
b.intent_tick.clear();
|
|
b.predictor.reset();
|
|
b.lead.reset();
|
|
b.home.reset();
|
|
}
|
|
void step(Bot& b, Clock::time_point now) {
|
|
if (!b.c) {
|
|
if (now >= b.retry_at)
|
|
login(b, now);
|
|
return;
|
|
}
|
|
auto& c = *b.c;
|
|
c.poll(now);
|
|
if (c.state() == client::RegionSessionState::closed) {
|
|
if (b.established.exchange(false))
|
|
++b.s.drops;
|
|
else
|
|
++b.s.login_failures;
|
|
if (const auto reason = c.close_reason();
|
|
reason && *reason != SessionCloseReason::connection_lost)
|
|
++b.s.closes[static_cast<unsigned>(*reason)];
|
|
else if (const auto cause = c.transport_cause()) // No session Close: 256 + cause.
|
|
++b.s.closes[256U + static_cast<unsigned>(*cause)];
|
|
else
|
|
++b.s.closes[255U];
|
|
const bool refused = c.close_reason() == SessionCloseReason::capacity ||
|
|
c.close_reason() == SessionCloseReason::rate_limited;
|
|
auto backoff = std::chrono::seconds{2};
|
|
if (refused) {
|
|
++b.s.refused;
|
|
// Refused for load before the grant was used: the region is shedding, so back
|
|
// off longer, and a bundle grant stays usable for the next try (10.9d).
|
|
backoff = std::chrono::seconds{10};
|
|
if (!key_ && b.used > 0)
|
|
--b.used;
|
|
}
|
|
b.c.reset();
|
|
b.retry_at =
|
|
now + backoff + milliseconds{std::uniform_int_distribution<int>{0, 500}(b.rng)};
|
|
return;
|
|
}
|
|
if (c.state() != client::RegionSessionState::established)
|
|
return;
|
|
++b.s.connected_polls;
|
|
if (!b.established.exchange(true)) {
|
|
++b.s.logins;
|
|
b.s.login_us.push_back(micros(now - b.connect_at));
|
|
b.next_action = now + milliseconds{std::uniform_int_distribution<int>{0, 2000}(b.rng)};
|
|
b.next_interest = now + std::chrono::seconds{10};
|
|
}
|
|
// Receipts: RTT, lead and refusals.
|
|
for (const auto& r : c.take_intention_receipts()) {
|
|
if (const auto it = b.receipt_pending.find(r.sequence); it != b.receipt_pending.end()) {
|
|
b.s.receipt_us.push_back(micros(now - it->second));
|
|
b.receipt_pending.erase(it);
|
|
}
|
|
const auto tick = b.intent_tick.find(r.sequence);
|
|
if (r.status == AdmitStatus::admitted) {
|
|
if (tick != b.intent_tick.end())
|
|
b.lead.admitted(tick->second, r.server_tick);
|
|
} else {
|
|
++b.s.refused_intents;
|
|
b.predictor.rejected(r.sequence);
|
|
b.response_pending.erase(r.sequence);
|
|
if (r.status == AdmitStatus::outside_window)
|
|
b.lead.too_late();
|
|
}
|
|
if (tick != b.intent_tick.end())
|
|
b.intent_tick.erase(tick);
|
|
}
|
|
// Snapshots: cadence, own marker (responsiveness, prediction errors), bytes.
|
|
if (const auto* v = c.view(); v && v->tick != b.last_view) {
|
|
if (b.last_view != 0 && v->tick > b.last_view)
|
|
b.s.gap_ticks.push_back(v->tick - b.last_view);
|
|
b.last_view = v->tick;
|
|
++b.s.snapshots;
|
|
if (!b.home && !v->environment.empty()) {
|
|
std::int64_t x0 = v->environment.front().voxel.x, x1 = x0;
|
|
std::int64_t y0 = v->environment.front().voxel.y, y1 = y0;
|
|
for (const auto& cell : v->environment) {
|
|
x0 = std::min(x0, cell.voxel.x);
|
|
x1 = std::max(x1, cell.voxel.x);
|
|
y0 = std::min(y0, cell.voxel.y);
|
|
y1 = std::max(y1, cell.voxel.y);
|
|
}
|
|
b.home = SessionInterest{(x0 + x1) / 2, (y0 + y1) / 2, v->slice_z, 7};
|
|
}
|
|
for (const auto& m : v->markers) {
|
|
if (!m.own)
|
|
continue;
|
|
b.predictor.authoritative({v->tick, m.applied_sequence, m.state, m.area});
|
|
if (m.applied_sequence > b.applied) {
|
|
b.applied = m.applied_sequence;
|
|
while (!b.response_pending.empty() &&
|
|
b.response_pending.begin()->first <= b.applied) {
|
|
b.s.response_us.push_back(micros(now - b.response_pending.begin()->second));
|
|
b.response_pending.erase(b.response_pending.begin());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
for (const auto e : b.predictor.take_errors())
|
|
b.s.correction_um.push_back(e);
|
|
const auto bytes = c.snapshot_bytes();
|
|
b.s.bytes += bytes - std::min(bytes, b.last_bytes);
|
|
b.last_bytes = bytes;
|
|
// Receipts first: a duplicate's recorded outcome is not a latency sample.
|
|
for (const auto& r : c.take_action_receipts())
|
|
if (r.status == AdmitStatus::duplicate) {
|
|
++b.s.duplicate_actions;
|
|
b.outcome_pending.erase(r.action_id);
|
|
}
|
|
// Outcomes: final ones only.
|
|
for (const auto& out : c.take_action_outcomes())
|
|
if (out.result == ActionResult::committed_no_effect ||
|
|
out.result == ActionResult::not_committed)
|
|
if (const auto it = b.outcome_pending.find(out.action_id);
|
|
it != b.outcome_pending.end()) {
|
|
b.s.outcome_us.push_back(micros(now - it->second));
|
|
b.outcome_pending.erase(it);
|
|
}
|
|
// Once per server tick: an intention, a prediction, sometimes an action or interest.
|
|
const auto tick = c.server_tick();
|
|
if (tick == 0 || tick == b.last_tick)
|
|
return;
|
|
b.last_tick = tick;
|
|
if (now >= b.next_turn) {
|
|
const double angle = std::uniform_real_distribution<double>{0, 6.283185307}(b.rng);
|
|
b.vx = static_cast<std::int16_t>(std::lround(700.0 * std::cos(angle)));
|
|
b.vy = static_cast<std::int16_t>(std::lround(700.0 * std::sin(angle)));
|
|
b.next_turn = now + std::chrono::seconds{2};
|
|
}
|
|
const auto target = tick + b.lead.lead();
|
|
if (!b.predictor.full())
|
|
if (const auto seq = c.send_intention(target, b.vx, b.vy)) {
|
|
++b.s.intents;
|
|
(void)b.predictor.input({*seq, target, {b.vx, b.vy}});
|
|
b.receipt_pending[*seq] = now;
|
|
b.response_pending[*seq] = now;
|
|
b.intent_tick[*seq] = target;
|
|
for (auto* pending : {&b.receipt_pending, &b.response_pending})
|
|
while (pending->size() > 256)
|
|
pending->erase(pending->begin());
|
|
while (b.intent_tick.size() > 256)
|
|
b.intent_tick.erase(b.intent_tick.begin());
|
|
}
|
|
(void)b.predictor.predict(target);
|
|
(void)b.predictor.take_correction();
|
|
if (now >= b.next_action) {
|
|
const auto id = b.next_action_id++;
|
|
if (c.send_action(id, ActionKind::probe, tick + 3)) {
|
|
b.outcome_pending[id] = now;
|
|
while (b.outcome_pending.size() > 64)
|
|
b.outcome_pending.erase(b.outcome_pending.begin());
|
|
}
|
|
b.next_action = now + std::chrono::seconds{2};
|
|
}
|
|
if (now >= b.next_interest && b.home) {
|
|
auto in = *b.home;
|
|
in.x += (b.interest_step++ % 2 == 0) ? 1 : 0; // Alternate: a fresh baseline each.
|
|
(void)c.set_interest(in);
|
|
b.next_interest = now + std::chrono::seconds{10};
|
|
}
|
|
}
|
|
|
|
Options o_;
|
|
std::optional<GrantSigningKey> key_;
|
|
std::vector<SessionGrant> bundle_;
|
|
std::string address_, authority_;
|
|
std::vector<std::unique_ptr<Bot>> bots_;
|
|
std::vector<std::mutex> locks_;
|
|
std::vector<std::thread> workers_;
|
|
std::atomic<bool> stop_{false};
|
|
};
|
|
|
|
// The region's /metrics (evo_m10_* and evo_region_tick_debt), unlabelled series only.
|
|
std::map<std::string, std::uint64_t> scrape(const Options& o) {
|
|
std::map<std::string, std::uint64_t> out;
|
|
#ifndef _WIN32
|
|
if (o.metrics_port == 0)
|
|
return out;
|
|
const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0);
|
|
if (fd < 0)
|
|
return out;
|
|
sockaddr_in a{};
|
|
a.sin_family = AF_INET;
|
|
a.sin_port = htons(o.metrics_port);
|
|
timeval timeout{3, 0};
|
|
::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout);
|
|
std::string raw;
|
|
if (::inet_pton(AF_INET, o.metrics_host.c_str(), &a.sin_addr) == 1 &&
|
|
::connect(fd, reinterpret_cast<const sockaddr*>(&a), sizeof a) == 0) {
|
|
const std::string request = "GET /metrics HTTP/1.1\r\nHost: load\r\n\r\n";
|
|
if (::send(fd, request.data(), request.size(), MSG_NOSIGNAL) ==
|
|
static_cast<ssize_t>(request.size())) {
|
|
std::array<char, 4096> chunk{};
|
|
for (ssize_t n = 0; (n = ::recv(fd, chunk.data(), chunk.size(), 0)) > 0;)
|
|
raw.append(chunk.data(), static_cast<std::size_t>(n));
|
|
}
|
|
}
|
|
::close(fd);
|
|
std::size_t at = 0;
|
|
while (at < raw.size()) {
|
|
const auto end = std::min(raw.find('\n', at), raw.size());
|
|
const std::string_view line{raw.data() + at, end - at};
|
|
at = end + 1;
|
|
if (!line.starts_with("evo_m10_") && !line.starts_with("evo_region_tick_debt"))
|
|
continue;
|
|
const auto space = line.find(' ');
|
|
if (space == std::string_view::npos || line.find('{') != std::string_view::npos)
|
|
continue;
|
|
if (const auto v = number(line.substr(space + 1)))
|
|
out[std::string{line.substr(0, space)}] = *v;
|
|
}
|
|
#else
|
|
(void)o;
|
|
#endif
|
|
return out;
|
|
}
|
|
double cpu_seconds() {
|
|
#ifndef _WIN32
|
|
rusage u{};
|
|
::getrusage(RUSAGE_SELF, &u);
|
|
return static_cast<double>(u.ru_utime.tv_sec + u.ru_stime.tv_sec) +
|
|
(static_cast<double>(u.ru_utime.tv_usec + u.ru_stime.tv_usec) / 1e6);
|
|
#else
|
|
return 0;
|
|
#endif
|
|
}
|
|
|
|
void line(const char* name, const std::vector<std::uint64_t>& pooled,
|
|
const std::vector<std::uint64_t>& client_p95, double scale, const char* unit) {
|
|
const auto f = [&](std::uint64_t v) { return static_cast<double>(v) / scale; };
|
|
std::printf("load %s_%s p50/p95/p99/max=%.1f/%.1f/%.1f/%.1f n=%zu client_p95 "
|
|
"min/med/max=%.1f/%.1f/%.1f\n",
|
|
name, unit, f(percentile(pooled, 0.50)), f(percentile(pooled, 0.95)),
|
|
f(percentile(pooled, 0.99)), f(percentile(pooled, 1.0)), pooled.size(),
|
|
f(percentile(client_p95, 0.0)), f(percentile(client_p95, 0.5)),
|
|
f(percentile(client_p95, 1.0)));
|
|
}
|
|
|
|
// One step's report; true when within budget.
|
|
bool report(const Options& o, std::size_t n, std::size_t established, std::size_t exhausted,
|
|
std::pair<std::uint64_t, std::uint64_t> hold_utc, const std::vector<Samples>& all,
|
|
double seconds, double cpu, const std::map<std::string, std::uint64_t>& server) {
|
|
Samples pooled;
|
|
std::vector<std::uint64_t> response95, receipt95, outcome95, login95, correction95, rates, kib;
|
|
for (const auto& s : all) {
|
|
append(pooled.response_us, s.response_us);
|
|
append(pooled.receipt_us, s.receipt_us);
|
|
append(pooled.outcome_us, s.outcome_us);
|
|
append(pooled.login_us, s.login_us);
|
|
append(pooled.correction_um, s.correction_um);
|
|
append(pooled.gap_ticks, s.gap_ticks);
|
|
pooled.snapshots += s.snapshots;
|
|
pooled.intents += s.intents;
|
|
pooled.refused_intents += s.refused_intents;
|
|
pooled.duplicate_actions += s.duplicate_actions;
|
|
pooled.logins += s.logins;
|
|
pooled.drops += s.drops;
|
|
pooled.login_failures += s.login_failures;
|
|
for (const auto& [why, count] : s.closes)
|
|
pooled.closes[why] += count;
|
|
pooled.refused += s.refused;
|
|
response95.push_back(percentile(s.response_us, 0.95));
|
|
receipt95.push_back(percentile(s.receipt_us, 0.95));
|
|
outcome95.push_back(percentile(s.outcome_us, 0.95));
|
|
if (!s.login_us.empty())
|
|
login95.push_back(percentile(s.login_us, 0.95));
|
|
correction95.push_back(percentile(s.correction_um, 0.95));
|
|
rates.push_back(static_cast<std::uint64_t>(
|
|
std::lround(1000.0 * static_cast<double>(s.snapshots) / std::max(seconds, 0.001))));
|
|
kib.push_back(static_cast<std::uint64_t>(
|
|
std::lround(static_cast<double>(s.bytes) / 1024.0 / std::max(seconds, 0.001))));
|
|
}
|
|
std::string reasons;
|
|
for (const auto& [why, count] : pooled.closes)
|
|
reasons += (reasons.empty() ? "" : ",") + std::to_string(why) + ":" + std::to_string(count);
|
|
std::printf("load step N=%zu established=%zu logins=%llu drops=%llu login_failures=%llu "
|
|
"refused=%llu seconds=%.1f exhausted=%zu hold_utc=%llu..%llu close_reasons=%s\n",
|
|
n, established, static_cast<unsigned long long>(pooled.logins),
|
|
static_cast<unsigned long long>(pooled.drops),
|
|
static_cast<unsigned long long>(pooled.login_failures),
|
|
static_cast<unsigned long long>(pooled.refused), seconds, exhausted,
|
|
static_cast<unsigned long long>(hold_utc.first),
|
|
static_cast<unsigned long long>(hold_utc.second),
|
|
reasons.empty() ? "-" : reasons.c_str());
|
|
line("response", pooled.response_us, response95, 1000.0, "ms");
|
|
line("receipt", pooled.receipt_us, receipt95, 1000.0, "ms");
|
|
line("outcome", pooled.outcome_us, outcome95, 1000.0, "ms");
|
|
line("login", pooled.login_us, login95, 1000.0, "ms");
|
|
line("correction", pooled.correction_um, correction95, 1000.0, "mm");
|
|
std::printf(
|
|
"load snapshots hz min/med/max=%.2f/%.2f/%.2f gap_ticks p99/max=%llu/%llu "
|
|
"kib_s med/max=%llu/%llu intents=%llu refused_intents=%llu duplicate_actions=%llu\n",
|
|
static_cast<double>(percentile(rates, 0.0)) / 1000.0,
|
|
static_cast<double>(percentile(rates, 0.5)) / 1000.0,
|
|
static_cast<double>(percentile(rates, 1.0)) / 1000.0,
|
|
static_cast<unsigned long long>(percentile(pooled.gap_ticks, 0.99)),
|
|
static_cast<unsigned long long>(percentile(pooled.gap_ticks, 1.0)),
|
|
static_cast<unsigned long long>(percentile(kib, 0.5)),
|
|
static_cast<unsigned long long>(percentile(kib, 1.0)),
|
|
static_cast<unsigned long long>(pooled.intents),
|
|
static_cast<unsigned long long>(pooled.refused_intents),
|
|
static_cast<unsigned long long>(pooled.duplicate_actions));
|
|
std::printf("load client_cpu_pct=%.0f\n", 100.0 * cpu / std::max(seconds, 0.001));
|
|
if (!server.empty()) {
|
|
std::printf("load server");
|
|
for (const auto& [k, v] : server)
|
|
std::printf(" %s=%llu", k.c_str(), static_cast<unsigned long long>(v));
|
|
std::printf("\n");
|
|
}
|
|
// Starvation: throughput must not hide a client left behind. Judged among the bots that
|
|
// were connected during the hold (the others are reported as not established).
|
|
std::vector<std::uint64_t> connected_rates, connected_response;
|
|
for (std::size_t i = 0; i < all.size(); ++i)
|
|
if (all[i].connected_polls > 0) {
|
|
connected_rates.push_back(rates[i]);
|
|
connected_response.push_back(response95[i]);
|
|
}
|
|
const auto median_rate = percentile(connected_rates, 0.5);
|
|
const auto median_response = percentile(connected_response, 0.5);
|
|
const auto budget_us = o.response_budget_ms * 1000;
|
|
std::vector<std::size_t> starved;
|
|
for (std::size_t i = 0; i < all.size(); ++i)
|
|
if (all[i].connected_polls > 0 &&
|
|
(rates[i] * 5 < median_rate * 4 ||
|
|
(response95[i] > 2 * median_response && response95[i] > budget_us)))
|
|
starved.push_back(i);
|
|
std::printf("load starved %zu", starved.size());
|
|
for (const auto i : starved)
|
|
std::printf(" %zu", i);
|
|
std::printf("\n");
|
|
std::vector<std::string> why;
|
|
if (established < n)
|
|
why.emplace_back("not-established");
|
|
if (pooled.refused > 0 || pooled.login_failures > 0)
|
|
why.emplace_back("refused");
|
|
if (pooled.drops > 0)
|
|
why.emplace_back("drops");
|
|
if (!starved.empty())
|
|
why.emplace_back("starved");
|
|
if (pooled.response_us.empty() || percentile(pooled.response_us, 0.95) > budget_us)
|
|
why.emplace_back("response");
|
|
if (pooled.outcome_us.empty() || percentile(pooled.outcome_us, 0.95) > 1'500'000)
|
|
why.emplace_back("outcome");
|
|
const auto get = [&](const char* k) {
|
|
const auto it = server.find(k);
|
|
return it == server.end() ? 0 : it->second;
|
|
};
|
|
if (get("evo_m10_tick_work_p99_microseconds") > 25'000)
|
|
why.emplace_back("tick-work");
|
|
if (get("evo_m10_refusing") != 0 || get("evo_m10_shed_level") != 0)
|
|
why.emplace_back("governor");
|
|
if (get("evo_region_tick_debt") != 0)
|
|
why.emplace_back("tick-debt");
|
|
std::printf("load verdict N=%zu within_budget=%d", n, why.empty() ? 1 : 0);
|
|
for (const auto& w : why)
|
|
std::printf(" %s", w.c_str());
|
|
std::printf("\n");
|
|
if (o.per_client)
|
|
for (std::size_t i = 0; i < all.size(); ++i)
|
|
std::printf("load client %zu snapshots=%llu response_p95_ms=%.1f receipt_p95_ms=%.1f "
|
|
"outcome_p95_ms=%.1f correction_p95_mm=%.1f drops=%llu "
|
|
"login_failures=%llu\n",
|
|
i, static_cast<unsigned long long>(all[i].snapshots), ms(response95[i]),
|
|
ms(receipt95[i]), ms(outcome95[i]),
|
|
static_cast<double>(correction95[i]) / 1000.0,
|
|
static_cast<unsigned long long>(all[i].drops),
|
|
static_cast<unsigned long long>(all[i].login_failures));
|
|
std::fflush(stdout);
|
|
return why.empty();
|
|
}
|
|
|
|
std::optional<std::pair<std::string, std::uint16_t>> endpoint(std::string_view v) {
|
|
const auto colon = v.rfind(':');
|
|
if (colon == std::string_view::npos || colon == 0)
|
|
return std::nullopt;
|
|
const auto port = number(v.substr(colon + 1));
|
|
if (!port || *port == 0 || *port > 65535)
|
|
return std::nullopt;
|
|
return std::pair{std::string{v.substr(0, colon)}, static_cast<std::uint16_t>(*port)};
|
|
}
|
|
std::optional<std::vector<std::size_t>> steps_of(std::string_view v) {
|
|
std::vector<std::size_t> out;
|
|
while (!v.empty()) {
|
|
const auto comma = v.find(',');
|
|
const auto n = number(v.substr(0, comma));
|
|
if (!n || *n == 0 || *n > 256 || (!out.empty() && *n < out.back()))
|
|
return std::nullopt;
|
|
out.push_back(*n);
|
|
v.remove_prefix(comma == std::string_view::npos ? v.size() : comma + 1);
|
|
}
|
|
if (out.empty())
|
|
return std::nullopt;
|
|
return out;
|
|
}
|
|
} // namespace
|
|
|
|
int main(int argc, char** argv) {
|
|
const std::vector<std::string_view> args(argv + 1, argv + argc);
|
|
Options o;
|
|
bool ok = true;
|
|
for (std::size_t i = 0; i < args.size() && ok; ++i) {
|
|
const auto flag = args[i];
|
|
if (flag == "--per-client") {
|
|
o.per_client = true;
|
|
continue;
|
|
}
|
|
if (flag == "--expect-established") {
|
|
o.expect_established = true;
|
|
continue;
|
|
}
|
|
if (i + 1 >= args.size()) {
|
|
ok = false;
|
|
break;
|
|
}
|
|
const auto value = args[++i];
|
|
const auto n = number(value);
|
|
if (flag == "--connect") {
|
|
const auto e = endpoint(value);
|
|
ok = e.has_value();
|
|
if (e)
|
|
std::tie(o.host, o.port) = *e;
|
|
} else if (flag == "--metrics") {
|
|
const auto e = endpoint(value);
|
|
ok = e.has_value();
|
|
if (e)
|
|
std::tie(o.metrics_host, o.metrics_port) = *e;
|
|
} else if (flag == "--server-name")
|
|
o.server_name = value;
|
|
else if (flag == "--ca")
|
|
o.ca = value;
|
|
else if (flag == "--issuer-key")
|
|
o.issuer_key = value;
|
|
else if (flag == "--grants")
|
|
o.grants_file = value;
|
|
else if (n && flag == "--grants-per-bot" && *n >= 1 && *n <= 64)
|
|
o.grants_per_bot = *n;
|
|
else if (flag == "--steps") {
|
|
const auto s = steps_of(value);
|
|
ok = s.has_value();
|
|
if (s)
|
|
o.steps = *s;
|
|
} else if (n && flag == "--world")
|
|
o.world = *n;
|
|
else if (n && flag == "--frame")
|
|
o.frame = *n;
|
|
else if (n && flag == "--region")
|
|
o.region = *n;
|
|
else if (n && flag == "--seed" && *n != 0)
|
|
o.seed = *n;
|
|
else if (n && flag == "--lifetime" && *n >= 60 && *n <= 14400)
|
|
o.lifetime = *n;
|
|
else if (n && flag == "--hold" && *n >= 1 && *n <= 3600)
|
|
o.hold_s = *n;
|
|
else if (n && flag == "--warmup" && *n <= 600)
|
|
o.warmup_s = *n;
|
|
else if (n && flag == "--ramp-ms" && *n <= 10'000)
|
|
o.ramp_ms = *n;
|
|
else if (n && flag == "--threads" && *n >= 1 && *n <= 64)
|
|
o.threads = *n;
|
|
else if (n && flag == "--response-budget-ms" && *n >= 1)
|
|
o.response_budget_ms = *n;
|
|
else
|
|
ok = false;
|
|
}
|
|
const bool signs = !o.issuer_key.empty();
|
|
if (!ok || o.host.empty() || o.server_name.empty() || o.ca.empty() ||
|
|
signs == !o.grants_file.empty() || (!signs && o.grants_per_bot == 0) || o.world == 0 ||
|
|
o.frame == 0 || o.region == 0) {
|
|
std::fputs("usage: evo-load --connect HOST:PORT --server-name NAME --ca FILE "
|
|
"(--issuer-key FILE | --grants FILE --grants-per-bot C) --world N "
|
|
"--frame N --region N [--steps N[,N]...] "
|
|
"[--hold S] [--warmup S] [--ramp-ms MS] [--threads T] [--seed X] "
|
|
"[--lifetime S] [--metrics HOST:PORT] [--response-budget-ms MS] "
|
|
"[--per-client] [--expect-established]\n",
|
|
stderr);
|
|
return 2;
|
|
}
|
|
std::optional<GrantSigningKey> key;
|
|
std::vector<SessionGrant> bundle;
|
|
if (signs) {
|
|
const auto scalar = read(o.issuer_key);
|
|
auto imported = GrantSigningKey::import(
|
|
DeploymentEnvironment::dev, 1,
|
|
std::span{reinterpret_cast<const std::uint8_t*>(scalar.data()), scalar.size()});
|
|
if (!imported) {
|
|
std::puts("error issuer-key");
|
|
return 2;
|
|
}
|
|
key.emplace(std::move(*imported));
|
|
} else {
|
|
const auto bytes = read(o.grants_file);
|
|
const auto needed = *std::ranges::max_element(o.steps) * o.grants_per_bot;
|
|
if (bytes.size() % session_grant_bytes != 0 ||
|
|
bytes.size() / session_grant_bytes < needed) {
|
|
std::printf("error grants (need %llu grants: bots x grants-per-bot)\n",
|
|
static_cast<unsigned long long>(needed));
|
|
return 2;
|
|
}
|
|
bundle.resize(bytes.size() / session_grant_bytes);
|
|
for (std::size_t i = 0; i < bundle.size(); ++i)
|
|
std::copy_n(bytes.begin() + static_cast<std::ptrdiff_t>(i * session_grant_bytes),
|
|
session_grant_bytes, reinterpret_cast<char*>(bundle[i].data()));
|
|
}
|
|
const auto address = quic::resolve_ipv4(o.host);
|
|
if (!address) {
|
|
std::puts("error resolve");
|
|
return 2;
|
|
}
|
|
std::printf("load start steps=%zu hold=%llu warmup=%llu threads=%llu budget_ms=%llu\n",
|
|
o.steps.size(), static_cast<unsigned long long>(o.hold_s),
|
|
static_cast<unsigned long long>(o.warmup_s),
|
|
static_cast<unsigned long long>(o.threads),
|
|
static_cast<unsigned long long>(o.response_budget_ms));
|
|
std::fflush(stdout);
|
|
bool smoke_ok = true;
|
|
std::optional<std::size_t> sustainable, saturated;
|
|
{
|
|
Load load{o, std::move(key), std::move(bundle), *address};
|
|
for (const auto n : o.steps) {
|
|
load.activate(n);
|
|
std::this_thread::sleep_for(std::chrono::seconds{o.warmup_s});
|
|
// The ramp's and warmup's logins (and closes) belong to this step; the rest of
|
|
// the warmup is discarded.
|
|
const auto warm = load.take(n);
|
|
const auto started = Clock::now();
|
|
const auto hold_start = now_seconds();
|
|
const auto cpu0 = cpu_seconds();
|
|
std::this_thread::sleep_for(std::chrono::seconds{o.hold_s});
|
|
auto samples = load.take(n);
|
|
for (std::size_t i = 0; i < n; ++i) {
|
|
append(samples[i].login_us, warm[i].login_us);
|
|
samples[i].logins += warm[i].logins;
|
|
samples[i].drops += warm[i].drops;
|
|
samples[i].login_failures += warm[i].login_failures;
|
|
for (const auto& [why, count] : warm[i].closes)
|
|
samples[i].closes[why] += count;
|
|
samples[i].refused += warm[i].refused;
|
|
}
|
|
const auto seconds = std::chrono::duration<double>(Clock::now() - started).count();
|
|
const auto established = load.established(n);
|
|
const bool within =
|
|
report(o, n, established, load.exhausted(n), {hold_start, now_seconds()}, samples,
|
|
seconds, cpu_seconds() - cpu0, scrape(o));
|
|
if (within && !saturated)
|
|
sustainable = n;
|
|
if (!within && !saturated)
|
|
saturated = n;
|
|
std::uint64_t snapshots = 0, outcomes = 0;
|
|
for (const auto& s : samples) {
|
|
snapshots += s.snapshots;
|
|
outcomes += s.outcome_us.size();
|
|
}
|
|
smoke_ok = smoke_ok && established == n && snapshots > 0 && outcomes > 0;
|
|
}
|
|
}
|
|
std::printf("load result sustainable=%zu saturated_at=%zu\n", sustainable.value_or(0),
|
|
saturated.value_or(0));
|
|
std::fflush(stdout);
|
|
return o.expect_established && !smoke_ok ? 1 : 0;
|
|
}
|