Add M10.9b evo-load multi-client load generator and process smoke
Foundation / checks (deploy) (push) Successful in 3m16s
Foundation / checks (ecs) (push) Successful in 1m54s
Foundation / checks (client) (push) Successful in 12m39s
Foundation / checks (network) (push) Successful in 12m56s
Foundation / checks (tsan) (push) Successful in 7m52s
Foundation / checks (checks) (push) Successful in 32m28s
Foundation / checks (postgres) (push) Failing after 26m52s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
Foundation / checks (deploy) (push) Successful in 3m16s
Foundation / checks (ecs) (push) Successful in 1m54s
Foundation / checks (client) (push) Successful in 12m39s
Foundation / checks (network) (push) Successful in 12m56s
Foundation / checks (tsan) (push) Successful in 7m52s
Foundation / checks (checks) (push) Successful in 32m28s
Foundation / checks (postgres) (push) Failing after 26m52s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
ADR-0097. evo-load runs bots on worker threads, each a RegionSessionClient with its own account and an in-process dev grant: one intention per tick at its input lead with marker prediction, actions every 2 s, interest moves every 10 s, and re-login after any close. Stepped ramps report pooled and per-client distributions (response, receipt, outcome, login, prediction error), snapshot rate, gaps and bandwidth, drops, failed logins and refusals, generator CPU, a starvation check among connected bots, a budget verdict and the region's evo_m10_* gauges, ending with the sustainable bot count. A Postgres process smoke runs 4 then 8 bots against a real evo-server. Fixes found while ramping: the process fixture's short pumps never read (a chatty child blocked on its stdout pipe), and the load governor refused on a single tick of debt (now 2 refuse, 4 shed). Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,764 @@
|
||||
// M10.9b multi-client load generator (ADR-0097). Development and qualification tool: only
|
||||
// runs against the evolution-dev reference worker count as evidence (10.9d); local runs are
|
||||
// smoke tests and development references.
|
||||
//
|
||||
// evo-load --connect HOST:PORT --server-name NAME --ca FILE --issuer-key FILE
|
||||
// --world N --frame N --region N [--steps N[,N]...] [--hold S] [--warmup S]
|
||||
// [--ramp-ms MS] [--threads T] [--seed X] [--lifetime S] [--metrics HOST:PORT]
|
||||
// [--response-budget-ms MS] [--per-client] [--expect-established]
|
||||
//
|
||||
// Each simulated client (bot) is a RegionSessionClient with its own account (from --seed and
|
||||
// its index) logging in with a dev grant signed here (the issuer key never leaves this
|
||||
// process). Bots run on T worker threads (each owns a slice). Every server tick a bot sends
|
||||
// one intention for tick + its input lead (a random walk turning every 2 s) and predicts its
|
||||
// probe marker; every 2 s it sends a probe action; every 10 s it moves its interest by a
|
||||
// cell (a fresh baseline). A closed or refused bot logs in again after 2 s with the next
|
||||
// serial.
|
||||
//
|
||||
// Steps: the bot count rises to each step's N (one login every --ramp-ms), waits --warmup
|
||||
// seconds, then measures for --hold seconds. Bots stay connected into the next step. Per
|
||||
// step it prints:
|
||||
// load step N=.. established=.. logins=.. drops=.. login_failures=.. refused=.. seconds=..
|
||||
// (drops: established sessions closed; login failures: closed before Welcome, including
|
||||
// transport refusals at the connection limit; refused: closed with capacity or
|
||||
// rate_limited, either way. Logins, drops and failures include the step's ramp and warmup;
|
||||
// everything else is the hold)
|
||||
// load response_ms p50/p95/p99/max=.. (input -> first own-marker snapshot applying it)
|
||||
// client_p95 min/med/max=..
|
||||
// load receipt_ms ... (intention -> receipt)
|
||||
// load outcome_ms ... (action -> final outcome)
|
||||
// load login_ms ... (connect -> Welcome)
|
||||
// load correction_mm ... (prediction error per authoritative marker)
|
||||
// load snapshots hz min/med/max=.. gap_ticks p99/max=.. kib_s med/max=..
|
||||
// intents=.. refused_intents=..
|
||||
// load client_cpu_pct=.. (this process, all threads; Linux)
|
||||
// load server ... (with --metrics: the region's evo_m10_* gauges at step end)
|
||||
// load starved N [indices] (among bots connected during the hold: snapshot rate < 80 % of
|
||||
// the median, or a response p95 over both twice the median and the budget)
|
||||
// load verdict N=.. within_budget=0|1 [reasons]
|
||||
// With --per-client each bot's own line follows its step. --expect-established exits 1
|
||||
// unless every step ended with all its bots established and every step saw snapshots and
|
||||
// final outcomes (the smoke test).
|
||||
#include "evo/client/motion_prediction.hpp"
|
||||
#include "evo/client/region_session.hpp"
|
||||
#include "evo/network/session_grant.hpp"
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <charconv>
|
||||
#include <chrono>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <ctime>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <iterator>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <thread>
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#ifndef _WIN32
|
||||
#include <arpa/inet.h>
|
||||
#include <netinet/in.h>
|
||||
#include <sys/resource.h>
|
||||
#include <sys/socket.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
using namespace evo;
|
||||
using namespace evo::network;
|
||||
using Clock = std::chrono::steady_clock;
|
||||
using std::chrono::milliseconds;
|
||||
|
||||
std::optional<std::uint64_t> number(std::string_view v) {
|
||||
std::uint64_t n = 0;
|
||||
const auto [end, ec] = std::from_chars(v.data(), v.data() + v.size(), n);
|
||||
if (ec != std::errc{} || end != v.data() + v.size())
|
||||
return std::nullopt;
|
||||
return n;
|
||||
}
|
||||
std::string read(const std::filesystem::path& path) {
|
||||
std::ifstream file{path, std::ios::binary};
|
||||
return {std::istreambuf_iterator<char>{file}, std::istreambuf_iterator<char>{}};
|
||||
}
|
||||
std::uint64_t now_seconds() {
|
||||
return static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(
|
||||
std::chrono::system_clock::now().time_since_epoch())
|
||||
.count());
|
||||
}
|
||||
std::uint64_t micros(Clock::duration d) {
|
||||
return static_cast<std::uint64_t>(std::max<std::int64_t>(
|
||||
0, std::chrono::duration_cast<std::chrono::microseconds>(d).count()));
|
||||
}
|
||||
// Nearest rank over a copy.
|
||||
std::uint64_t percentile(std::vector<std::uint64_t> v, double p) {
|
||||
if (v.empty())
|
||||
return 0;
|
||||
std::ranges::sort(v);
|
||||
const auto rank = static_cast<std::size_t>(std::ceil(p * static_cast<double>(v.size())));
|
||||
return v.at(std::clamp<std::size_t>(rank, 1, v.size()) - 1);
|
||||
}
|
||||
double ms(std::uint64_t us) {
|
||||
return static_cast<double>(us) / 1000.0;
|
||||
}
|
||||
|
||||
struct Options {
|
||||
std::string host, server_name, ca, issuer_key, metrics_host;
|
||||
std::uint16_t port{}, metrics_port{};
|
||||
std::uint64_t world{}, frame{}, region{}, seed{1}, lifetime{900};
|
||||
std::vector<std::size_t> steps{20};
|
||||
std::uint64_t hold_s{30}, warmup_s{5}, ramp_ms{100}, threads{4}, response_budget_ms{150};
|
||||
bool per_client{}, expect_established{};
|
||||
};
|
||||
|
||||
// What one bot measured since the last take.
|
||||
struct Samples {
|
||||
std::vector<std::uint64_t> response_us, receipt_us, outcome_us, login_us, correction_um,
|
||||
gap_ticks;
|
||||
std::uint64_t snapshots{}, bytes{}, intents{}, refused_intents{}, logins{}, drops{},
|
||||
login_failures{}, refused{};
|
||||
std::uint64_t connected_polls{}; // Polls while established (starvation is judged on these).
|
||||
};
|
||||
void append(std::vector<std::uint64_t>& to, const std::vector<std::uint64_t>& from) {
|
||||
to.insert(to.end(), from.begin(), from.end());
|
||||
}
|
||||
|
||||
struct Bot {
|
||||
std::size_t index{};
|
||||
AccountId account;
|
||||
std::uint64_t serial{};
|
||||
std::unique_ptr<client::RegionSessionClient> c;
|
||||
std::atomic<bool> active{false}, established{false};
|
||||
Clock::time_point connect_at, retry_at, next_action, next_interest, next_turn;
|
||||
client::MotionPredictor predictor;
|
||||
client::InputLead lead;
|
||||
std::uint64_t last_tick{}, last_view{}, last_bytes{}, next_action_id{1};
|
||||
std::uint64_t applied{}; // Newest applied intention sequence seen in an own marker.
|
||||
std::map<std::uint64_t, Clock::time_point> receipt_pending, response_pending, outcome_pending;
|
||||
std::map<std::uint64_t, std::uint64_t> intent_tick;
|
||||
std::int16_t vx{}, vy{};
|
||||
std::optional<SessionInterest> home;
|
||||
int interest_step{};
|
||||
std::mt19937_64 rng;
|
||||
Samples s;
|
||||
};
|
||||
|
||||
class Load {
|
||||
public:
|
||||
Load(Options o, GrantSigningKey key, std::string address)
|
||||
: o_{std::move(o)}, key_{std::move(key)}, address_{std::move(address)},
|
||||
authority_{read(o_.ca)} {
|
||||
const auto most = *std::ranges::max_element(o_.steps);
|
||||
bots_.reserve(most);
|
||||
for (std::size_t i = 0; i < most; ++i) {
|
||||
auto b = std::make_unique<Bot>();
|
||||
b->index = i;
|
||||
for (std::size_t k = 0; k < 8; ++k) {
|
||||
b->account.bytes.at(k) = static_cast<std::uint8_t>((o_.seed >> (8U * k)) & 0xFFU);
|
||||
b->account.bytes.at(8 + k) =
|
||||
static_cast<std::uint8_t>(((i + 1) >> (8U * k)) & 0xFFU);
|
||||
}
|
||||
b->serial = now_seconds() * 64; // Later runs supersede earlier ones.
|
||||
b->rng.seed(o_.seed * 1'000'003 + i);
|
||||
bots_.push_back(std::move(b));
|
||||
}
|
||||
const auto threads = std::clamp<std::size_t>(o_.threads, 1, 64);
|
||||
locks_ = std::vector<std::mutex>(threads);
|
||||
for (std::size_t t = 0; t < threads; ++t)
|
||||
workers_.emplace_back([this, t, threads] { work(t, threads); });
|
||||
}
|
||||
Load(const Load&) = delete;
|
||||
Load& operator=(const Load&) = delete;
|
||||
~Load() {
|
||||
stop_ = true;
|
||||
for (auto& w : workers_)
|
||||
w.join();
|
||||
}
|
||||
void activate(std::size_t n) {
|
||||
for (std::size_t i = 0; i < n && i < bots_.size(); ++i) {
|
||||
if (bots_[i]->active.exchange(true))
|
||||
continue;
|
||||
std::this_thread::sleep_for(milliseconds{o_.ramp_ms});
|
||||
}
|
||||
}
|
||||
// Every active bot's samples since the last take (cleared).
|
||||
std::vector<Samples> take(std::size_t n) {
|
||||
std::vector<Samples> out(n);
|
||||
for (std::size_t t = 0; t < locks_.size(); ++t) {
|
||||
const std::scoped_lock lock{locks_[t]};
|
||||
for (std::size_t i = t; i < n; i += locks_.size())
|
||||
out[i] = std::exchange(bots_[i]->s, Samples{});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
[[nodiscard]] std::size_t established(std::size_t n) const {
|
||||
return static_cast<std::size_t>(
|
||||
std::count_if(bots_.begin(), bots_.begin() + static_cast<std::ptrdiff_t>(n),
|
||||
[](const auto& b) { return b->established.load(); }));
|
||||
}
|
||||
|
||||
private:
|
||||
void work(std::size_t t, std::size_t threads) {
|
||||
while (!stop_) {
|
||||
{
|
||||
const std::scoped_lock lock{locks_[t]};
|
||||
const auto now = Clock::now();
|
||||
for (std::size_t i = t; i < bots_.size(); i += threads)
|
||||
if (bots_[i]->active)
|
||||
step(*bots_[i], now);
|
||||
}
|
||||
std::this_thread::sleep_for(milliseconds{2});
|
||||
}
|
||||
const std::scoped_lock lock{locks_[t]};
|
||||
for (std::size_t i = t; i < bots_.size(); i += threads)
|
||||
if (auto& c = bots_[i]->c; c && c->state() == client::RegionSessionState::established)
|
||||
c->close();
|
||||
for (int k = 0; k < 50; ++k) { // Let the closes flush.
|
||||
for (std::size_t i = t; i < bots_.size(); i += threads)
|
||||
if (bots_[i]->c)
|
||||
bots_[i]->c->poll(Clock::now());
|
||||
std::this_thread::sleep_for(milliseconds{2});
|
||||
}
|
||||
}
|
||||
void login(Bot& b, Clock::time_point now) {
|
||||
GrantClaims claims;
|
||||
claims.account = b.account;
|
||||
claims.serial = ++b.serial;
|
||||
claims.world = {o_.world, o_.frame};
|
||||
claims.region = o_.region;
|
||||
claims.issued_at = now_seconds();
|
||||
claims.expires_at = claims.issued_at + o_.lifetime;
|
||||
claims.grant_id = (b.serial << 16U) ^ (b.index + 1);
|
||||
const auto grant = key_.sign(claims);
|
||||
client::RegionSessionClientConfig config;
|
||||
config.transport.server_address = address_;
|
||||
config.transport.port = o_.port;
|
||||
config.transport.server_name = o_.server_name;
|
||||
config.transport.authority_pem = authority_;
|
||||
config.transport.authority_pem_file = o_.ca;
|
||||
if (!grant) {
|
||||
b.retry_at = now + std::chrono::seconds{2};
|
||||
return;
|
||||
}
|
||||
config.grant = *grant;
|
||||
auto made = client::RegionSessionClient::connect(std::move(config), now);
|
||||
if (!made) {
|
||||
b.retry_at = now + std::chrono::seconds{2};
|
||||
return;
|
||||
}
|
||||
b.c = std::move(*made);
|
||||
b.connect_at = now;
|
||||
b.last_tick = b.last_view = b.last_bytes = b.applied = 0;
|
||||
b.receipt_pending.clear();
|
||||
b.response_pending.clear();
|
||||
b.outcome_pending.clear();
|
||||
b.intent_tick.clear();
|
||||
b.predictor.reset();
|
||||
b.lead.reset();
|
||||
b.home.reset();
|
||||
}
|
||||
void step(Bot& b, Clock::time_point now) {
|
||||
if (!b.c) {
|
||||
if (now >= b.retry_at)
|
||||
login(b, now);
|
||||
return;
|
||||
}
|
||||
auto& c = *b.c;
|
||||
c.poll(now);
|
||||
if (c.state() == client::RegionSessionState::closed) {
|
||||
if (b.established.exchange(false))
|
||||
++b.s.drops;
|
||||
else
|
||||
++b.s.login_failures;
|
||||
if (c.close_reason() == SessionCloseReason::capacity ||
|
||||
c.close_reason() == SessionCloseReason::rate_limited)
|
||||
++b.s.refused;
|
||||
b.c.reset();
|
||||
b.retry_at = now + std::chrono::seconds{2} +
|
||||
milliseconds{std::uniform_int_distribution<int>{0, 500}(b.rng)};
|
||||
return;
|
||||
}
|
||||
if (c.state() != client::RegionSessionState::established)
|
||||
return;
|
||||
++b.s.connected_polls;
|
||||
if (!b.established.exchange(true)) {
|
||||
++b.s.logins;
|
||||
b.s.login_us.push_back(micros(now - b.connect_at));
|
||||
b.next_action = now + milliseconds{std::uniform_int_distribution<int>{0, 2000}(b.rng)};
|
||||
b.next_interest = now + std::chrono::seconds{10};
|
||||
}
|
||||
// Receipts: RTT, lead and refusals.
|
||||
for (const auto& r : c.take_intention_receipts()) {
|
||||
if (const auto it = b.receipt_pending.find(r.sequence); it != b.receipt_pending.end()) {
|
||||
b.s.receipt_us.push_back(micros(now - it->second));
|
||||
b.receipt_pending.erase(it);
|
||||
}
|
||||
const auto tick = b.intent_tick.find(r.sequence);
|
||||
if (r.status == AdmitStatus::admitted) {
|
||||
if (tick != b.intent_tick.end())
|
||||
b.lead.admitted(tick->second, r.server_tick);
|
||||
} else {
|
||||
++b.s.refused_intents;
|
||||
b.predictor.rejected(r.sequence);
|
||||
b.response_pending.erase(r.sequence);
|
||||
if (r.status == AdmitStatus::outside_window)
|
||||
b.lead.too_late();
|
||||
}
|
||||
if (tick != b.intent_tick.end())
|
||||
b.intent_tick.erase(tick);
|
||||
}
|
||||
// Snapshots: cadence, own marker (responsiveness, prediction errors), bytes.
|
||||
if (const auto* v = c.view(); v && v->tick != b.last_view) {
|
||||
if (b.last_view != 0 && v->tick > b.last_view)
|
||||
b.s.gap_ticks.push_back(v->tick - b.last_view);
|
||||
b.last_view = v->tick;
|
||||
++b.s.snapshots;
|
||||
if (!b.home && !v->environment.empty()) {
|
||||
std::int64_t x0 = v->environment.front().voxel.x, x1 = x0;
|
||||
std::int64_t y0 = v->environment.front().voxel.y, y1 = y0;
|
||||
for (const auto& cell : v->environment) {
|
||||
x0 = std::min(x0, cell.voxel.x);
|
||||
x1 = std::max(x1, cell.voxel.x);
|
||||
y0 = std::min(y0, cell.voxel.y);
|
||||
y1 = std::max(y1, cell.voxel.y);
|
||||
}
|
||||
b.home = SessionInterest{(x0 + x1) / 2, (y0 + y1) / 2, v->slice_z, 7};
|
||||
}
|
||||
for (const auto& m : v->markers) {
|
||||
if (!m.own)
|
||||
continue;
|
||||
b.predictor.authoritative({v->tick, m.applied_sequence, m.state, m.area});
|
||||
if (m.applied_sequence > b.applied) {
|
||||
b.applied = m.applied_sequence;
|
||||
while (!b.response_pending.empty() &&
|
||||
b.response_pending.begin()->first <= b.applied) {
|
||||
b.s.response_us.push_back(micros(now - b.response_pending.begin()->second));
|
||||
b.response_pending.erase(b.response_pending.begin());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const auto e : b.predictor.take_errors())
|
||||
b.s.correction_um.push_back(e);
|
||||
const auto bytes = c.snapshot_bytes();
|
||||
b.s.bytes += bytes - std::min(bytes, b.last_bytes);
|
||||
b.last_bytes = bytes;
|
||||
// Outcomes: final ones only.
|
||||
for (const auto& out : c.take_action_outcomes())
|
||||
if (out.result == ActionResult::committed_no_effect ||
|
||||
out.result == ActionResult::not_committed)
|
||||
if (const auto it = b.outcome_pending.find(out.action_id);
|
||||
it != b.outcome_pending.end()) {
|
||||
b.s.outcome_us.push_back(micros(now - it->second));
|
||||
b.outcome_pending.erase(it);
|
||||
}
|
||||
(void)c.take_action_receipts();
|
||||
// Once per server tick: an intention, a prediction, sometimes an action or interest.
|
||||
const auto tick = c.server_tick();
|
||||
if (tick == 0 || tick == b.last_tick)
|
||||
return;
|
||||
b.last_tick = tick;
|
||||
if (now >= b.next_turn) {
|
||||
const double angle = std::uniform_real_distribution<double>{0, 6.283185307}(b.rng);
|
||||
b.vx = static_cast<std::int16_t>(std::lround(700.0 * std::cos(angle)));
|
||||
b.vy = static_cast<std::int16_t>(std::lround(700.0 * std::sin(angle)));
|
||||
b.next_turn = now + std::chrono::seconds{2};
|
||||
}
|
||||
const auto target = tick + b.lead.lead();
|
||||
if (!b.predictor.full())
|
||||
if (const auto seq = c.send_intention(target, b.vx, b.vy)) {
|
||||
++b.s.intents;
|
||||
(void)b.predictor.input({*seq, target, {b.vx, b.vy}});
|
||||
b.receipt_pending[*seq] = now;
|
||||
b.response_pending[*seq] = now;
|
||||
b.intent_tick[*seq] = target;
|
||||
for (auto* pending : {&b.receipt_pending, &b.response_pending})
|
||||
while (pending->size() > 256)
|
||||
pending->erase(pending->begin());
|
||||
while (b.intent_tick.size() > 256)
|
||||
b.intent_tick.erase(b.intent_tick.begin());
|
||||
}
|
||||
(void)b.predictor.predict(target);
|
||||
(void)b.predictor.take_correction();
|
||||
if (now >= b.next_action) {
|
||||
const auto id = b.next_action_id++;
|
||||
if (c.send_action(id, ActionKind::probe, tick + 3)) {
|
||||
b.outcome_pending[id] = now;
|
||||
while (b.outcome_pending.size() > 64)
|
||||
b.outcome_pending.erase(b.outcome_pending.begin());
|
||||
}
|
||||
b.next_action = now + std::chrono::seconds{2};
|
||||
}
|
||||
if (now >= b.next_interest && b.home) {
|
||||
auto in = *b.home;
|
||||
in.x += (b.interest_step++ % 2 == 0) ? 1 : 0; // Alternate: a fresh baseline each.
|
||||
(void)c.set_interest(in);
|
||||
b.next_interest = now + std::chrono::seconds{10};
|
||||
}
|
||||
}
|
||||
|
||||
Options o_;
|
||||
GrantSigningKey key_;
|
||||
std::string address_, authority_;
|
||||
std::vector<std::unique_ptr<Bot>> bots_;
|
||||
std::vector<std::mutex> locks_;
|
||||
std::vector<std::thread> workers_;
|
||||
std::atomic<bool> stop_{false};
|
||||
};
|
||||
|
||||
// The region's /metrics (evo_m10_* and evo_region_tick_debt), unlabelled series only.
|
||||
std::map<std::string, std::uint64_t> scrape(const Options& o) {
|
||||
std::map<std::string, std::uint64_t> out;
|
||||
#ifndef _WIN32
|
||||
if (o.metrics_port == 0)
|
||||
return out;
|
||||
const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0);
|
||||
if (fd < 0)
|
||||
return out;
|
||||
sockaddr_in a{};
|
||||
a.sin_family = AF_INET;
|
||||
a.sin_port = htons(o.metrics_port);
|
||||
timeval timeout{3, 0};
|
||||
::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout);
|
||||
std::string raw;
|
||||
if (::inet_pton(AF_INET, o.metrics_host.c_str(), &a.sin_addr) == 1 &&
|
||||
::connect(fd, reinterpret_cast<const sockaddr*>(&a), sizeof a) == 0) {
|
||||
const std::string request = "GET /metrics HTTP/1.1\r\nHost: load\r\n\r\n";
|
||||
if (::send(fd, request.data(), request.size(), MSG_NOSIGNAL) ==
|
||||
static_cast<ssize_t>(request.size())) {
|
||||
std::array<char, 4096> chunk{};
|
||||
for (ssize_t n = 0; (n = ::recv(fd, chunk.data(), chunk.size(), 0)) > 0;)
|
||||
raw.append(chunk.data(), static_cast<std::size_t>(n));
|
||||
}
|
||||
}
|
||||
::close(fd);
|
||||
std::size_t at = 0;
|
||||
while (at < raw.size()) {
|
||||
const auto end = std::min(raw.find('\n', at), raw.size());
|
||||
const std::string_view line{raw.data() + at, end - at};
|
||||
at = end + 1;
|
||||
if (!line.starts_with("evo_m10_") && !line.starts_with("evo_region_tick_debt"))
|
||||
continue;
|
||||
const auto space = line.find(' ');
|
||||
if (space == std::string_view::npos || line.find('{') != std::string_view::npos)
|
||||
continue;
|
||||
if (const auto v = number(line.substr(space + 1)))
|
||||
out[std::string{line.substr(0, space)}] = *v;
|
||||
}
|
||||
#else
|
||||
(void)o;
|
||||
#endif
|
||||
return out;
|
||||
}
|
||||
double cpu_seconds() {
|
||||
#ifndef _WIN32
|
||||
rusage u{};
|
||||
::getrusage(RUSAGE_SELF, &u);
|
||||
return static_cast<double>(u.ru_utime.tv_sec + u.ru_stime.tv_sec) +
|
||||
(static_cast<double>(u.ru_utime.tv_usec + u.ru_stime.tv_usec) / 1e6);
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
void line(const char* name, const std::vector<std::uint64_t>& pooled,
|
||||
const std::vector<std::uint64_t>& client_p95, double scale, const char* unit) {
|
||||
const auto f = [&](std::uint64_t v) { return static_cast<double>(v) / scale; };
|
||||
std::printf("load %s_%s p50/p95/p99/max=%.1f/%.1f/%.1f/%.1f n=%zu client_p95 "
|
||||
"min/med/max=%.1f/%.1f/%.1f\n",
|
||||
name, unit, f(percentile(pooled, 0.50)), f(percentile(pooled, 0.95)),
|
||||
f(percentile(pooled, 0.99)), f(percentile(pooled, 1.0)), pooled.size(),
|
||||
f(percentile(client_p95, 0.0)), f(percentile(client_p95, 0.5)),
|
||||
f(percentile(client_p95, 1.0)));
|
||||
}
|
||||
|
||||
// One step's report; true when within budget.
|
||||
bool report(const Options& o, std::size_t n, std::size_t established,
|
||||
const std::vector<Samples>& all, double seconds, double cpu,
|
||||
const std::map<std::string, std::uint64_t>& server) {
|
||||
Samples pooled;
|
||||
std::vector<std::uint64_t> response95, receipt95, outcome95, login95, correction95, rates, kib;
|
||||
for (const auto& s : all) {
|
||||
append(pooled.response_us, s.response_us);
|
||||
append(pooled.receipt_us, s.receipt_us);
|
||||
append(pooled.outcome_us, s.outcome_us);
|
||||
append(pooled.login_us, s.login_us);
|
||||
append(pooled.correction_um, s.correction_um);
|
||||
append(pooled.gap_ticks, s.gap_ticks);
|
||||
pooled.snapshots += s.snapshots;
|
||||
pooled.intents += s.intents;
|
||||
pooled.refused_intents += s.refused_intents;
|
||||
pooled.logins += s.logins;
|
||||
pooled.drops += s.drops;
|
||||
pooled.login_failures += s.login_failures;
|
||||
pooled.refused += s.refused;
|
||||
response95.push_back(percentile(s.response_us, 0.95));
|
||||
receipt95.push_back(percentile(s.receipt_us, 0.95));
|
||||
outcome95.push_back(percentile(s.outcome_us, 0.95));
|
||||
if (!s.login_us.empty())
|
||||
login95.push_back(percentile(s.login_us, 0.95));
|
||||
correction95.push_back(percentile(s.correction_um, 0.95));
|
||||
rates.push_back(static_cast<std::uint64_t>(
|
||||
std::lround(1000.0 * static_cast<double>(s.snapshots) / std::max(seconds, 0.001))));
|
||||
kib.push_back(static_cast<std::uint64_t>(
|
||||
std::lround(static_cast<double>(s.bytes) / 1024.0 / std::max(seconds, 0.001))));
|
||||
}
|
||||
std::printf("load step N=%zu established=%zu logins=%llu drops=%llu login_failures=%llu "
|
||||
"refused=%llu seconds=%.1f\n",
|
||||
n, established, static_cast<unsigned long long>(pooled.logins),
|
||||
static_cast<unsigned long long>(pooled.drops),
|
||||
static_cast<unsigned long long>(pooled.login_failures),
|
||||
static_cast<unsigned long long>(pooled.refused), seconds);
|
||||
line("response", pooled.response_us, response95, 1000.0, "ms");
|
||||
line("receipt", pooled.receipt_us, receipt95, 1000.0, "ms");
|
||||
line("outcome", pooled.outcome_us, outcome95, 1000.0, "ms");
|
||||
line("login", pooled.login_us, login95, 1000.0, "ms");
|
||||
line("correction", pooled.correction_um, correction95, 1000.0, "mm");
|
||||
std::printf("load snapshots hz min/med/max=%.2f/%.2f/%.2f gap_ticks p99/max=%llu/%llu "
|
||||
"kib_s med/max=%llu/%llu intents=%llu refused_intents=%llu\n",
|
||||
static_cast<double>(percentile(rates, 0.0)) / 1000.0,
|
||||
static_cast<double>(percentile(rates, 0.5)) / 1000.0,
|
||||
static_cast<double>(percentile(rates, 1.0)) / 1000.0,
|
||||
static_cast<unsigned long long>(percentile(pooled.gap_ticks, 0.99)),
|
||||
static_cast<unsigned long long>(percentile(pooled.gap_ticks, 1.0)),
|
||||
static_cast<unsigned long long>(percentile(kib, 0.5)),
|
||||
static_cast<unsigned long long>(percentile(kib, 1.0)),
|
||||
static_cast<unsigned long long>(pooled.intents),
|
||||
static_cast<unsigned long long>(pooled.refused_intents));
|
||||
std::printf("load client_cpu_pct=%.0f\n", 100.0 * cpu / std::max(seconds, 0.001));
|
||||
if (!server.empty()) {
|
||||
std::printf("load server");
|
||||
for (const auto& [k, v] : server)
|
||||
std::printf(" %s=%llu", k.c_str(), static_cast<unsigned long long>(v));
|
||||
std::printf("\n");
|
||||
}
|
||||
// Starvation: throughput must not hide a client left behind. Judged among the bots that
|
||||
// were connected during the hold (the others are reported as not established).
|
||||
std::vector<std::uint64_t> connected_rates, connected_response;
|
||||
for (std::size_t i = 0; i < all.size(); ++i)
|
||||
if (all[i].connected_polls > 0) {
|
||||
connected_rates.push_back(rates[i]);
|
||||
connected_response.push_back(response95[i]);
|
||||
}
|
||||
const auto median_rate = percentile(connected_rates, 0.5);
|
||||
const auto median_response = percentile(connected_response, 0.5);
|
||||
const auto budget_us = o.response_budget_ms * 1000;
|
||||
std::vector<std::size_t> starved;
|
||||
for (std::size_t i = 0; i < all.size(); ++i)
|
||||
if (all[i].connected_polls > 0 &&
|
||||
(rates[i] * 5 < median_rate * 4 ||
|
||||
(response95[i] > 2 * median_response && response95[i] > budget_us)))
|
||||
starved.push_back(i);
|
||||
std::printf("load starved %zu", starved.size());
|
||||
for (const auto i : starved)
|
||||
std::printf(" %zu", i);
|
||||
std::printf("\n");
|
||||
std::vector<std::string> why;
|
||||
if (established < n)
|
||||
why.emplace_back("not-established");
|
||||
if (pooled.refused > 0 || pooled.login_failures > 0)
|
||||
why.emplace_back("refused");
|
||||
if (pooled.drops > 0)
|
||||
why.emplace_back("drops");
|
||||
if (!starved.empty())
|
||||
why.emplace_back("starved");
|
||||
if (pooled.response_us.empty() || percentile(pooled.response_us, 0.95) > budget_us)
|
||||
why.emplace_back("response");
|
||||
if (pooled.outcome_us.empty() || percentile(pooled.outcome_us, 0.95) > 1'500'000)
|
||||
why.emplace_back("outcome");
|
||||
const auto get = [&](const char* k) {
|
||||
const auto it = server.find(k);
|
||||
return it == server.end() ? 0 : it->second;
|
||||
};
|
||||
if (get("evo_m10_tick_work_p99_microseconds") > 25'000)
|
||||
why.emplace_back("tick-work");
|
||||
if (get("evo_m10_refusing") != 0 || get("evo_m10_shed_level") != 0)
|
||||
why.emplace_back("governor");
|
||||
if (get("evo_region_tick_debt") != 0)
|
||||
why.emplace_back("tick-debt");
|
||||
std::printf("load verdict N=%zu within_budget=%d", n, why.empty() ? 1 : 0);
|
||||
for (const auto& w : why)
|
||||
std::printf(" %s", w.c_str());
|
||||
std::printf("\n");
|
||||
if (o.per_client)
|
||||
for (std::size_t i = 0; i < all.size(); ++i)
|
||||
std::printf("load client %zu snapshots=%llu response_p95_ms=%.1f receipt_p95_ms=%.1f "
|
||||
"outcome_p95_ms=%.1f correction_p95_mm=%.1f drops=%llu "
|
||||
"login_failures=%llu\n",
|
||||
i, static_cast<unsigned long long>(all[i].snapshots), ms(response95[i]),
|
||||
ms(receipt95[i]), ms(outcome95[i]),
|
||||
static_cast<double>(correction95[i]) / 1000.0,
|
||||
static_cast<unsigned long long>(all[i].drops),
|
||||
static_cast<unsigned long long>(all[i].login_failures));
|
||||
std::fflush(stdout);
|
||||
return why.empty();
|
||||
}
|
||||
|
||||
std::optional<std::pair<std::string, std::uint16_t>> endpoint(std::string_view v) {
|
||||
const auto colon = v.rfind(':');
|
||||
if (colon == std::string_view::npos || colon == 0)
|
||||
return std::nullopt;
|
||||
const auto port = number(v.substr(colon + 1));
|
||||
if (!port || *port == 0 || *port > 65535)
|
||||
return std::nullopt;
|
||||
return std::pair{std::string{v.substr(0, colon)}, static_cast<std::uint16_t>(*port)};
|
||||
}
|
||||
std::optional<std::vector<std::size_t>> steps_of(std::string_view v) {
|
||||
std::vector<std::size_t> out;
|
||||
while (!v.empty()) {
|
||||
const auto comma = v.find(',');
|
||||
const auto n = number(v.substr(0, comma));
|
||||
if (!n || *n == 0 || *n > 256 || (!out.empty() && *n < out.back()))
|
||||
return std::nullopt;
|
||||
out.push_back(*n);
|
||||
v.remove_prefix(comma == std::string_view::npos ? v.size() : comma + 1);
|
||||
}
|
||||
if (out.empty())
|
||||
return std::nullopt;
|
||||
return out;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
const std::vector<std::string_view> args(argv + 1, argv + argc);
|
||||
Options o;
|
||||
bool ok = true;
|
||||
for (std::size_t i = 0; i < args.size() && ok; ++i) {
|
||||
const auto flag = args[i];
|
||||
if (flag == "--per-client") {
|
||||
o.per_client = true;
|
||||
continue;
|
||||
}
|
||||
if (flag == "--expect-established") {
|
||||
o.expect_established = true;
|
||||
continue;
|
||||
}
|
||||
if (i + 1 >= args.size()) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
const auto value = args[++i];
|
||||
const auto n = number(value);
|
||||
if (flag == "--connect") {
|
||||
const auto e = endpoint(value);
|
||||
ok = e.has_value();
|
||||
if (e)
|
||||
std::tie(o.host, o.port) = *e;
|
||||
} else if (flag == "--metrics") {
|
||||
const auto e = endpoint(value);
|
||||
ok = e.has_value();
|
||||
if (e)
|
||||
std::tie(o.metrics_host, o.metrics_port) = *e;
|
||||
} else if (flag == "--server-name")
|
||||
o.server_name = value;
|
||||
else if (flag == "--ca")
|
||||
o.ca = value;
|
||||
else if (flag == "--issuer-key")
|
||||
o.issuer_key = value;
|
||||
else if (flag == "--steps") {
|
||||
const auto s = steps_of(value);
|
||||
ok = s.has_value();
|
||||
if (s)
|
||||
o.steps = *s;
|
||||
} else if (n && flag == "--world")
|
||||
o.world = *n;
|
||||
else if (n && flag == "--frame")
|
||||
o.frame = *n;
|
||||
else if (n && flag == "--region")
|
||||
o.region = *n;
|
||||
else if (n && flag == "--seed" && *n != 0)
|
||||
o.seed = *n;
|
||||
else if (n && flag == "--lifetime" && *n >= 60 && *n <= 3600)
|
||||
o.lifetime = *n;
|
||||
else if (n && flag == "--hold" && *n >= 1 && *n <= 3600)
|
||||
o.hold_s = *n;
|
||||
else if (n && flag == "--warmup" && *n <= 600)
|
||||
o.warmup_s = *n;
|
||||
else if (n && flag == "--ramp-ms" && *n <= 10'000)
|
||||
o.ramp_ms = *n;
|
||||
else if (n && flag == "--threads" && *n >= 1 && *n <= 64)
|
||||
o.threads = *n;
|
||||
else if (n && flag == "--response-budget-ms" && *n >= 1)
|
||||
o.response_budget_ms = *n;
|
||||
else
|
||||
ok = false;
|
||||
}
|
||||
if (!ok || o.host.empty() || o.server_name.empty() || o.ca.empty() || o.issuer_key.empty() ||
|
||||
o.world == 0 || o.frame == 0 || o.region == 0) {
|
||||
std::fputs("usage: evo-load --connect HOST:PORT --server-name NAME --ca FILE "
|
||||
"--issuer-key FILE --world N --frame N --region N [--steps N[,N]...] "
|
||||
"[--hold S] [--warmup S] [--ramp-ms MS] [--threads T] [--seed X] "
|
||||
"[--lifetime S] [--metrics HOST:PORT] [--response-budget-ms MS] "
|
||||
"[--per-client] [--expect-established]\n",
|
||||
stderr);
|
||||
return 2;
|
||||
}
|
||||
const auto scalar = read(o.issuer_key);
|
||||
auto key = GrantSigningKey::import(
|
||||
DeploymentEnvironment::dev, 1,
|
||||
std::span{reinterpret_cast<const std::uint8_t*>(scalar.data()), scalar.size()});
|
||||
if (!key) {
|
||||
std::puts("error issuer-key");
|
||||
return 2;
|
||||
}
|
||||
const auto address = quic::resolve_ipv4(o.host);
|
||||
if (!address) {
|
||||
std::puts("error resolve");
|
||||
return 2;
|
||||
}
|
||||
std::printf("load start steps=%zu hold=%llu warmup=%llu threads=%llu budget_ms=%llu\n",
|
||||
o.steps.size(), static_cast<unsigned long long>(o.hold_s),
|
||||
static_cast<unsigned long long>(o.warmup_s),
|
||||
static_cast<unsigned long long>(o.threads),
|
||||
static_cast<unsigned long long>(o.response_budget_ms));
|
||||
std::fflush(stdout);
|
||||
bool smoke_ok = true;
|
||||
std::optional<std::size_t> sustainable, saturated;
|
||||
{
|
||||
Load load{o, std::move(*key), *address};
|
||||
for (const auto n : o.steps) {
|
||||
load.activate(n);
|
||||
std::this_thread::sleep_for(std::chrono::seconds{o.warmup_s});
|
||||
// The ramp's and warmup's logins (and closes) belong to this step; the rest of
|
||||
// the warmup is discarded.
|
||||
const auto warm = load.take(n);
|
||||
const auto started = Clock::now();
|
||||
const auto cpu0 = cpu_seconds();
|
||||
std::this_thread::sleep_for(std::chrono::seconds{o.hold_s});
|
||||
auto samples = load.take(n);
|
||||
for (std::size_t i = 0; i < n; ++i) {
|
||||
append(samples[i].login_us, warm[i].login_us);
|
||||
samples[i].logins += warm[i].logins;
|
||||
samples[i].drops += warm[i].drops;
|
||||
samples[i].login_failures += warm[i].login_failures;
|
||||
samples[i].refused += warm[i].refused;
|
||||
}
|
||||
const auto seconds = std::chrono::duration<double>(Clock::now() - started).count();
|
||||
const auto established = load.established(n);
|
||||
const bool within =
|
||||
report(o, n, established, samples, seconds, cpu_seconds() - cpu0, scrape(o));
|
||||
if (within && !saturated)
|
||||
sustainable = n;
|
||||
if (!within && !saturated)
|
||||
saturated = n;
|
||||
std::uint64_t snapshots = 0, outcomes = 0;
|
||||
for (const auto& s : samples) {
|
||||
snapshots += s.snapshots;
|
||||
outcomes += s.outcome_us.size();
|
||||
}
|
||||
smoke_ok = smoke_ok && established == n && snapshots > 0 && outcomes > 0;
|
||||
}
|
||||
}
|
||||
std::printf("load result sustainable=%zu saturated_at=%zu\n", sustainable.value_or(0),
|
||||
saturated.value_or(0));
|
||||
std::fflush(stdout);
|
||||
return o.expect_established && !smoke_ok ? 1 : 0;
|
||||
}
|
||||
Reference in New Issue
Block a user