Add M10.9b evo-load multi-client load generator and process smoke
Foundation / checks (deploy) (push) Successful in 3m16s
Foundation / checks (ecs) (push) Successful in 1m54s
Foundation / checks (client) (push) Successful in 12m39s
Foundation / checks (network) (push) Successful in 12m56s
Foundation / checks (tsan) (push) Successful in 7m52s
Foundation / checks (checks) (push) Successful in 32m28s
Foundation / checks (postgres) (push) Failing after 26m52s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped

ADR-0097. evo-load runs bots on worker threads, each a RegionSessionClient
with its own account and an in-process dev grant: one intention per tick
at its input lead with marker prediction, actions every 2 s, interest
moves every 10 s, and re-login after any close. Stepped ramps report
pooled and per-client distributions (response, receipt, outcome, login,
prediction error), snapshot rate, gaps and bandwidth, drops, failed
logins and refusals, generator CPU, a starvation check among connected
bots, a budget verdict and the region's evo_m10_* gauges, ending with the
sustainable bot count. A Postgres process smoke runs 4 then 8 bots
against a real evo-server.

Fixes found while ramping: the process fixture's short pumps never read
(a chatty child blocked on its stdout pipe), and the load governor
refused on a single tick of debt (now 2 refuse, 4 shed). Refreshes the
dev session window.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-07 09:02:57 -04:00
co-authored by Claude Opus 5.5
parent b8cb9db297
commit 1ba6748288
14 changed files with 983 additions and 22 deletions
+8 -1
View File
@@ -160,6 +160,13 @@ if(EVO_BUILD_QUIC)
target_link_libraries(evo-session-probe PRIVATE evo_session_client evo_network_tls)
evo_owned_target(evo-session-probe)
evo_msquic_runtime(evo-session-probe)
# M10.9b multi-client load generator (development and qualification; ADR-0097).
find_package(Threads REQUIRED)
add_executable(evo-load tools/network/load.cpp)
target_link_libraries(evo-load PRIVATE evo_session_client evo_network_tls evo_client_prediction
Threads::Threads)
evo_owned_target(evo-load)
evo_msquic_runtime(evo-load)
if(BUILD_TESTING)
# Owned impairment relay (tests and measurements only).
add_library(evo_network_impair STATIC tools/network/impair/udp_relay.cpp)
@@ -269,7 +276,7 @@ if(EVO_BUILD_POSTGRES)
tests/region_handoff_process_tests.cpp)
target_link_libraries(evo-postgres-tests PRIVATE evo_session_client)
target_compile_definitions(evo-postgres-tests PRIVATE EVO_POSTGRES_M10=1)
add_dependencies(evo-postgres-tests evo-session-probe)
add_dependencies(evo-postgres-tests evo-session-probe evo-load)
endif()
add_dependencies(evo-postgres-tests evo-server evo-world-clone) # Spawned by process tests.
evo_owned_target(evo-postgres-tests)
+1 -1
View File
@@ -24,7 +24,7 @@
{"name": "client", "configurePreset": "client", "targets": ["evo-client-state-tests", "evo-client", "evo-client-lifecycle-tests", "evo-client-environment-tests", "evo-client-organism-tests", "evo-client-ui-model-tests", "evo-client-panel-tests", "evo-client-bindings-tests"]},
{"name": "client-probe", "configurePreset": "client-probe", "targets": ["evo-client-probe"]},
{"name": "network", "configurePreset": "network", "targets": ["evo-client-state-tests", "evo-network-tests", "evo-dev-session"]},
{"name": "quic", "configurePreset": "quic", "targets": ["evo-quic-smoke-tests", "evo-quic-transport-tests", "evo-quic-session-tests", "evo-quic-hol-measure", "evo-session-probe", "evo-network-tests"]},
{"name": "quic", "configurePreset": "quic", "targets": ["evo-quic-smoke-tests", "evo-quic-transport-tests", "evo-quic-session-tests", "evo-quic-hol-measure", "evo-session-probe", "evo-load", "evo-network-tests"]},
{"name": "client-network", "configurePreset": "client-network", "targets": ["evo-client-state-tests", "evo-client", "evo-client-lifecycle-tests", "evo-client-environment-tests", "evo-client-organism-tests", "evo-client-ui-model-tests", "evo-client-panel-tests", "evo-client-bindings-tests", "evo-network-tests", "evo-dev-session", "evo-quic-session-tests", "evo-client-remote-tests", "evo-motion-trace", "evo-session-probe"]},
{"name": "client-package", "configurePreset": "client-package", "targets": ["evo-client-state-tests", "evo-client", "evo-client-lifecycle-tests", "evo-client-environment-tests", "evo-client-organism-tests", "evo-client-ui-model-tests", "evo-client-panel-tests", "evo-client-bindings-tests", "evo-network-tests", "evo-dev-session", "evo-quic-session-tests", "evo-client-remote-tests", "evo-motion-trace"]}
],
@@ -12,7 +12,7 @@ spec:
env:
- {name: REGION_SESSION, value: "dev"}
- {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"}
- {name: REGION_SESSION_WINDOW, value: "dev-20261007T102834-1d97:1791370114"}
- {name: REGION_SESSION_WINDOW, value: "dev-20261007T125653-8fc4:1791379013"}
ports:
# QUIC (ADR-0090): UDP only; nothing listens on TCP 7400.
- {name: session, containerPort: 7400, protocol: UDP}
+4 -3
View File
@@ -543,9 +543,10 @@ configuration done; 10.8d two evo-server processes on PostgreSQL with SIGKILL ar
durable step, duplicate/stale retries and a failed destination done (schema v14 recovery
sweep, replay guard). M10.8 complete. 10.9 split (ADR-0097; only evolution-dev reference-worker
runs qualify): 10.9a per-tick work measurement, load governor (refuse logins, then shed
replication with hysteresis) and its metrics done; 10.9b evo-load multi-client tool, 10.9c
abuse suite (fuzz, floods, slow peers, storms, lease loss) and 10.9d dev-cluster harness and
reference results follow.
replication with hysteresis) and its metrics done; 10.9b evo-load multi-client tool
(per-client distributions, starvation check, step verdicts, server gauges) and its process
smoke done; 10.9c abuse suite (fuzz, floods, slow peers, storms, lease loss) and 10.9d
dev-cluster harness and reference results follow.
**Entry:** M9 client/session evidence and M7/M8 dev/stage durability. Read
NETWORK_ARCHITECTURE, PERSISTENCE_ARCHITECTURE and ADR-0017. The M9 transport is
+7 -1
View File
@@ -107,7 +107,13 @@ layer) and feeds a load governor:
- both relax after 5 s of calm.
`/metrics` exposes the work percentiles (total and session layer) and the governor's
state. Next: 10.9b, the `evo-load` multi-client tool.
state. M10.9b adds `evo-load`, a multi-client load generator. It runs bots on worker
threads that send intentions every tick, actions, interest moves and reconnects. It reports
per step pooled and per-client distributions, a starvation check, a budget verdict and the
region's gauges, ending with the sustainable bot count. In a local development reference
(not evidence), one region stayed within budget up to 24 bots and the governor began
refusing at 32 (work p95 about 14 ms), with responses about 103 ms and outcomes about
665 ms throughout. Next: 10.9c, the abuse suite.
## Previous gate - 2026-10-02
@@ -44,10 +44,14 @@ snapshots.
- **Load governor** (`include/evo/server/load_governor.hpp`, portable, no clock). It keeps
a window of the last 40 samples (2 s). It reports the window's p95, p99 and maximum, plus
two outputs:
- **refusing** turns on when p95 ≥ 15 ms or tick debt > 0;
- **shed level** (0-3) rises by one when p95 ≥ 20 ms or tick debt ≥ 2, at most once per
- **refusing** turns on when p95 ≥ 15 ms or tick debt ≥ 2;
- **shed level** (0-3) rises by one when p95 ≥ 20 ms or tick debt ≥ 4, at most once per
second.
One tick of debt is jitter (a commit flush or a scheduler hiccup) and only interrupts calm.
As first written, any debt refused logins for at least 5 s, which made a login just after
a restart's catch-up fail intermittently in a process test (amended in 10.9b).
Calm means p95 < 10 ms and no debt. After 5 s of calm the level drops by one. At level 0,
5 s more of calm turns refusing off. Any non-calm sample restarts the calm timer.
Thresholds are configurable and validated (`relax < refuse ≤ shed ≤ tick budget`).
@@ -105,3 +109,49 @@ snapshots.
7 ms and p99 about 9 ms per tick. Of that, the session layer's p99 was about 4.5 ms. That
is far above 0.5 ms per session, so 10.9b examines it with evo-load. Likely contributors
are the per-tick snapshot source query and synchronous control renewals.
## 10.9b load tool (2026-10-07)
- **`evo-load`** (`tools/network/load.cpp`, quic preset; usage in its header). Bots run on
worker threads, each a `RegionSessionClient` with its own account and a dev grant signed
in-process. A bot:
- sends one intention per server tick at its `InputLead` (a random walk turning every
2 s) and predicts its marker with `MotionPredictor`;
- sends a probe action every 2 s and moves its interest every 10 s (a fresh baseline);
- logs in again 2 s after any close.
Steps raise the bot count (`--ramp-ms` apart), warm up, then hold. Each step reports:
- pooled p50, p95, p99 and maximum, plus the per-client p95 spread, for responsiveness
(input to the first own-marker snapshot applying it), receipt RTT, final-outcome
latency, login time and prediction error;
- snapshot rate, gaps and bandwidth per client;
- drops, failed logins and refusals;
- the generator's own CPU;
- with `--metrics`, the region's `evo_m10_*` gauges.
Starvation is judged among the bots connected during the hold: a snapshot rate under
80 % of the median, or a response p95 over both twice the median and the budget. A
step's verdict is within budget only with every bot established, no refusals or drops,
no starvation, response p95 within budget (150 ms by default), outcome p95 ≤ 1.5 s,
server work p99 ≤ 25 ms, no shedding or refusing, and no tick debt. The run ends with
`load result sustainable=N saturated_at=M`.
- **Smoke** (Postgres `region_m10_session_tests`): 4 then 8 bots against a real evo-server
with `--metrics`. Every bot logs in once with no drops or refusals, both steps measure
responses, outcomes and snapshots, and the report is complete.
- **Found while ramping:**
- The process-test fixture's `Child::pump(1ms)` rounded its wait down to 0 ms and never
read. A chatty evo-server then blocked on its full stdout pipe, which looked like a
server stall at 32 bots. The wait is now rounded up.
- The governor's any-debt refusal was too twitchy (see the amended decision above).
- **Local development reference** (WSL, Release, one region; not evidence):
| Bots | Verdict | Work p95 / max | Session p95 (per session) | Response p95 | Outcome p95 | Snapshots |
|---|---|---|---|---|---|---|
| 8 | within | 9.0 / 9.6 ms | 1.7 ms (0.22) | 106 ms | 659 ms | 20 Hz, no gaps |
| 16 | within | 11.0 / 11.2 ms | 3.5 ms (0.22) | 102 ms | 661 ms | 20 Hz |
| 24 | within | 13.1 / 13.3 ms | 5.8 ms (0.24) | 103 ms | 664 ms | 20 Hz |
| 32 | refusing | 14.3 / 17.0 ms | 8.4 ms (0.26) | 103 ms | 665 ms | 20 Hz |
At 48 and 64 bots, the bots beyond the 32-session cap fail at the transport connection
limit, and the 32 established sessions are unaffected. Prediction errors were 0 (no
loss on loopback), and no client starved at any step. The host step dominates the work;
the session layer stays near 0.25 ms per session (budget 0.5).
+40
View File
@@ -4948,3 +4948,43 @@ Not done:
- qualification on the evolution-dev reference worker (10.9d; needs v4 and v12-v14
deployed);
- load generation (10.9b) and the abuse suite (10.9c).
## 2026-10-07 — M10.9b evo-load (ADR-0097)
Implemented:
- `evo-load` (`tools/network/load.cpp`, quic preset): bots on worker threads (intentions
every tick at the input lead with prediction, actions every 2 s, interest moves every
10 s, re-login after any close), stepped ramps.
- Per step, it reports pooled and per-client distributions (response, receipt, outcome,
login, prediction error), snapshot rate, gaps and bandwidth, drops, failed logins and
refusals, the generator's CPU, a starvation check and a budget verdict. With
`--metrics`, it adds the region's gauges. The run ends with `sustainable` and
`saturated_at`.
- A Postgres process smoke (`region_m10_session_tests` `load_smoke`): 4 then 8 bots against
a real evo-server.
- Fixes found while ramping:
- The process fixture's `Child::pump` rounded short waits down to 0 ms and never read,
so a chatty child blocked on its stdout pipe.
- The load governor refused on a single tick of debt. It now refuses at 2 and sheds at
4; one tick only interrupts calm (ADR-0097 amended).
- The M10 end-to-end test waits until the server is not refusing before the headless
probe logs in, and dumps the probe and server lines on failure. One run had failed
there intermittently before the debt change.
Executed:
- **Linux GCC 14:**
- `postgres-check.sh` with QUIC, all suites, 3 runs in a row (2 after the final change);
- the smoke passes with every bot logged in once and both steps within budget;
- presets `quic` (ops, network, QUIC sessions) 3/3.
- **Local development reference ramp** (temporary steps 8-64, not committed, not
evidence): within budget up to 24 bots; the governor refused from 32 (work p95 about
14 ms, maximum 17 ms); session work stayed near 0.25 ms per session; responses p95 about
103 ms, outcomes about 665 ms, 20 Hz snapshots, no starvation. Beyond the 32-session cap,
extra logins failed at the transport limit and established sessions were unaffected.
- **Linux Clang 19:** clang-tidy 19 on `load.cpp`, the governor, ops and the M10 process
tests (clean); clang-format 19.
- **Windows MSVC 19.44:** presets `quic` 4/4 (`evo-load` builds) and `debug` 60/60.
Not done:
- the reference-worker run (10.9d);
- the abuse suite (10.9c).
+3
View File
@@ -19,6 +19,9 @@ struct LoadPolicy {
std::uint8_t max_shed{3};
};
inline constexpr std::size_t load_window_ticks = 40; // 2 s of 50 ms ticks.
// Tick debt (due minus simulated) that refuses logins and that sheds. One tick is jitter (a
// commit flush, a scheduler hiccup): it only interrupts calm.
inline constexpr std::uint64_t refuse_debt_ticks = 2, shed_debt_ticks = 4;
struct LoadState {
bool refusing{};
std::uint8_t shed{};
+1 -1
View File
@@ -35,7 +35,7 @@ in real time; changing it requires a versioned rules migration (ADR-0089).
| Per-tick region work | p99 <= 25 ms on the reference worker | budget | 10.9 (measured: `evo_m10_tick_work_*`, ADR-0097) |
| Session handling | <= 0.5 ms per session per tick | budget | 10.9 (measured: `evo_m10_session_work_*`, ADR-0097) |
| Host input queue | 64 inputs, 64 KiB | budget | region host |
| Overload response | shed interest, refuse new sessions before tick debt | budget | 10.9a load governor: refuse at p95 >= 15 ms or debt, shed at p95 >= 20 ms or debt >= 2, relax below 10 ms after 5 s (ADR-0097; provisional until 10.9d) |
| Overload response | shed interest, refuse new sessions before tick debt | budget | 10.9a load governor: refuse at p95 >= 15 ms or debt >= 2, shed at p95 >= 20 ms or debt >= 4, relax below 10 ms after 5 s (ADR-0097; provisional until 10.9d) |
Determinism: authoritative state is exact given the checkpoint and committed per-tick
inputs; timing, delivery, prediction, interpolation and presentation are tolerant
+2 -2
View File
@@ -42,8 +42,8 @@ bool LoadGovernor::observe(std::uint64_t work_us, std::uint64_t session_us, std:
state_.session_p99_us = rank(sessions, n, 99);
const auto before = std::pair{state_.refusing, state_.shed};
const bool hot = state_.p95_us >= policy_.refuse_us || tick_debt > 0;
const bool overloaded = state_.p95_us >= policy_.shed_us || tick_debt >= 2;
const bool hot = state_.p95_us >= policy_.refuse_us || tick_debt >= refuse_debt_ticks;
const bool overloaded = state_.p95_us >= policy_.shed_us || tick_debt >= shed_debt_ticks;
if (hot)
state_.refusing = true;
if (overloaded && state_.shed < policy_.max_shed &&
+3 -2
View File
@@ -119,8 +119,9 @@ class Child {
for (; seen < lines_.size(); ++seen)
if (want(lines_[seen]))
return true;
const auto left =
std::chrono::duration_cast<std::chrono::milliseconds>(deadline - Clock::now());
// Rounded up: a short pump (1 ms) must still read what is waiting, or a chatty
// child blocks on its full stdout pipe.
const auto left = std::chrono::ceil<std::chrono::milliseconds>(deadline - Clock::now());
if (left.count() <= 0 || !read_some(static_cast<int>(left.count())))
return false;
}
+5 -4
View File
@@ -189,10 +189,11 @@ void load_governor() {
// Down to 0 after another 5 s, then logins after 5 s more.
CHECK(run(100, 4'000).shed == 0 && g.state().refusing);
CHECK(run(99, 4'000).refusing && !run(1, 4'000).refusing);
// Tick debt alone: refuse at 1, shed at 2, even with light work.
CHECK(run(1, 1'000, 1).refusing && g.state().shed == 0);
CHECK(run(1, 1'000, 2).shed == 1);
CHECK(!g.observe(1'000, 250, 2, now + 10)); // Within the step interval: no change.
// Tick debt alone, even with light work: one tick is jitter, 2 refuse, 4 shed.
CHECK(!run(1, 1'000, 1).refusing);
CHECK(run(1, 1'000, 2).refusing && g.state().shed == 0);
CHECK(run(1, 1'000, 4).shed == 1);
CHECK(!g.observe(1'000, 250, 4, now + 10)); // Within the step interval: no change.
CHECK(g.state().session_p95_us == 1'000); // The window is mostly 4 ms ticks.
// Debt gone and calm (from the next sample): recovers.
CHECK(run(100, 1'000).shed == 1 && run(1, 1'000).shed == 0);
+92 -4
View File
@@ -356,7 +356,10 @@ void end_to_end() {
const auto o3 = outcome_of(*server, *c, 3, true);
CHECK(o3 && must(o3).result == ActionResult::committed_no_effect && !record(alice, 3).empty());
// The headless probe: login, intentions, actions and a query, then a clean exit.
// The headless probe: login, intentions, actions and a query, then a clean exit. After the
// restarts the load governor (ADR-0097) may still refuse logins while the host catches up;
// the probe does not retry, so wait until it accepts.
CHECK(until(*server, *c, [&] { return metric(server->ops, "evo_m10_refusing") == 0u; }));
const auto probe = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-session-probe";
Child p{{probe.string(), "run",
"--connect", "127.0.0.1:" + std::to_string(server->port),
@@ -381,9 +384,18 @@ void end_to_end() {
if (p.printed("done") || p.printed("timeout") || p.printed("closed"))
code = p.wait_exit(10s);
}
CHECK(code == 0 && p.printed("welcome ") && p.printed("intention 1 0") &&
p.printed("outcome 10 1 ") && p.printed("outcome 11 1 ") && p.printed("snapshot ") &&
p.printed("snapshot-bytes "));
const bool probed = code == 0 && p.printed("welcome ") && p.printed("intention 1 0") &&
p.printed("outcome 10 1 ") && p.printed("outcome 11 1 ") &&
p.printed("snapshot ") && p.printed("snapshot-bytes ");
if (!probed) {
for (const auto& line : p.lines())
if (!starts(line, "snapshot ") && !starts(line, "marker "))
std::printf("probe: %s\n", line.c_str());
for (const auto& line : server->child->lines())
if (starts(line, "m10 load") || starts(line, "m10 refused") || starts(line, "phase"))
std::printf("server: %s\n", line.c_str());
}
CHECK(probed);
CHECK(!record(bob, 10).empty() && !record(bob, 11).empty());
// The probe's marker (ADR-0094): its intentions (100, 0) were applied and moved it +x.
std::vector<std::array<long long, 4>> marks; // tick, x, y, applied
@@ -402,9 +414,85 @@ void end_to_end() {
CHECK(server->child->printed("m10 opened ") && server->child->printed("m10 committed ") &&
server->child->printed("m10 motion area "));
}
// M10.9b (ADR-0097): evo-load against a real evo-server, two steps with the region's
// metrics. A smoke test of the tool and the server under concurrent sessions, not
// qualification (only evolution-dev runs count): it checks that every bot logs in, that
// every step measures responses, outcomes and snapshots for its bots, and that the report
// is complete. The lines are kept in the log as a development reference.
void load_smoke() {
Setup s{152, 0xC6};
Server server{s};
const auto tool = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-load";
Child load{{tool.string(),
"--connect",
"127.0.0.1:" + std::to_string(server.port),
"--server-name",
name,
"--ca",
(s.region.root / "ca.pem").string(),
"--issuer-key",
(s.region.root / "issuer.key").string(),
"--world",
"7",
"--frame",
std::to_string(region_identity.frame),
"--region",
"152",
"--steps",
"4,8",
"--hold",
"8",
"--warmup",
"3",
"--ramp-ms",
"50",
"--threads",
"2",
"--lifetime",
"900",
"--metrics",
"127.0.0.1:" + std::to_string(server.ops),
"--per-client"}};
int code = -1;
for (const auto end = Clock::now() + 120s; Clock::now() < end && code < 0;) {
server.pump();
load.pump(5ms);
if (load.printed("load result "))
code = load.wait_exit(30s);
}
for (const auto& line : load.lines())
if (starts(line, "load "))
std::printf("%s\n", line.c_str());
CHECK(code == 0);
std::size_t steps = 0;
for (const auto& line : load.lines()) {
if (!starts(line, "load step "))
continue;
++steps;
// Every bot logged in once: 4 in the first step, 4 more in the second.
CHECK((starts(line, "load step N=4 established=4 logins=4 ") ||
starts(line, "load step N=8 established=8 logins=4 ")) &&
line.find(" drops=0 login_failures=0 refused=0 ") != std::string::npos);
}
const auto has = [&](std::string_view prefix) {
return std::ranges::count_if(load.lines(), [&](const std::string& l) {
return starts(l, std::string{prefix});
}) == 2;
};
CHECK(steps == 2 && has("load response_ms ") && has("load outcome_ms ") &&
has("load snapshots ") && has("load server ") && has("load verdict "));
CHECK(std::ranges::none_of(load.lines(), [](const std::string& l) {
return starts(l, "load response_ms") && l.find(" n=0 ") != std::string::npos;
}));
CHECK(std::ranges::count_if(
load.lines(), [](const std::string& l) { return starts(l, "load client "); }) == 12);
server.child->signal(SIGTERM);
CHECK(server.child->wait_exit(30s) == 0);
}
} // namespace
void run_region_m10_session_tests() {
end_to_end();
load_smoke();
std::puts("region m10 session integration passed");
}
+764
View File
@@ -0,0 +1,764 @@
// M10.9b multi-client load generator (ADR-0097). Development and qualification tool: only
// runs against the evolution-dev reference worker count as evidence (10.9d); local runs are
// smoke tests and development references.
//
// evo-load --connect HOST:PORT --server-name NAME --ca FILE --issuer-key FILE
// --world N --frame N --region N [--steps N[,N]...] [--hold S] [--warmup S]
// [--ramp-ms MS] [--threads T] [--seed X] [--lifetime S] [--metrics HOST:PORT]
// [--response-budget-ms MS] [--per-client] [--expect-established]
//
// Each simulated client (bot) is a RegionSessionClient with its own account (from --seed and
// its index) logging in with a dev grant signed here (the issuer key never leaves this
// process). Bots run on T worker threads (each owns a slice). Every server tick a bot sends
// one intention for tick + its input lead (a random walk turning every 2 s) and predicts its
// probe marker; every 2 s it sends a probe action; every 10 s it moves its interest by a
// cell (a fresh baseline). A closed or refused bot logs in again after 2 s with the next
// serial.
//
// Steps: the bot count rises to each step's N (one login every --ramp-ms), waits --warmup
// seconds, then measures for --hold seconds. Bots stay connected into the next step. Per
// step it prints:
// load step N=.. established=.. logins=.. drops=.. login_failures=.. refused=.. seconds=..
// (drops: established sessions closed; login failures: closed before Welcome, including
// transport refusals at the connection limit; refused: closed with capacity or
// rate_limited, either way. Logins, drops and failures include the step's ramp and warmup;
// everything else is the hold)
// load response_ms p50/p95/p99/max=.. (input -> first own-marker snapshot applying it)
// client_p95 min/med/max=..
// load receipt_ms ... (intention -> receipt)
// load outcome_ms ... (action -> final outcome)
// load login_ms ... (connect -> Welcome)
// load correction_mm ... (prediction error per authoritative marker)
// load snapshots hz min/med/max=.. gap_ticks p99/max=.. kib_s med/max=..
// intents=.. refused_intents=..
// load client_cpu_pct=.. (this process, all threads; Linux)
// load server ... (with --metrics: the region's evo_m10_* gauges at step end)
// load starved N [indices] (among bots connected during the hold: snapshot rate < 80 % of
// the median, or a response p95 over both twice the median and the budget)
// load verdict N=.. within_budget=0|1 [reasons]
// With --per-client each bot's own line follows its step. --expect-established exits 1
// unless every step ended with all its bots established and every step saw snapshots and
// final outcomes (the smoke test).
#include "evo/client/motion_prediction.hpp"
#include "evo/client/region_session.hpp"
#include "evo/network/session_grant.hpp"
#include <algorithm>
#include <array>
#include <atomic>
#include <charconv>
#include <chrono>
#include <cmath>
#include <cstdio>
#include <ctime>
#include <filesystem>
#include <fstream>
#include <iterator>
#include <map>
#include <memory>
#include <mutex>
#include <optional>
#include <random>
#include <string>
#include <string_view>
#include <thread>
#include <tuple>
#include <utility>
#include <vector>
#ifndef _WIN32
#include <arpa/inet.h>
#include <netinet/in.h>
#include <sys/resource.h>
#include <sys/socket.h>
#include <unistd.h>
#endif
namespace {
using namespace evo;
using namespace evo::network;
using Clock = std::chrono::steady_clock;
using std::chrono::milliseconds;
std::optional<std::uint64_t> number(std::string_view v) {
std::uint64_t n = 0;
const auto [end, ec] = std::from_chars(v.data(), v.data() + v.size(), n);
if (ec != std::errc{} || end != v.data() + v.size())
return std::nullopt;
return n;
}
std::string read(const std::filesystem::path& path) {
std::ifstream file{path, std::ios::binary};
return {std::istreambuf_iterator<char>{file}, std::istreambuf_iterator<char>{}};
}
std::uint64_t now_seconds() {
return static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count());
}
std::uint64_t micros(Clock::duration d) {
return static_cast<std::uint64_t>(std::max<std::int64_t>(
0, std::chrono::duration_cast<std::chrono::microseconds>(d).count()));
}
// Nearest rank over a copy.
std::uint64_t percentile(std::vector<std::uint64_t> v, double p) {
if (v.empty())
return 0;
std::ranges::sort(v);
const auto rank = static_cast<std::size_t>(std::ceil(p * static_cast<double>(v.size())));
return v.at(std::clamp<std::size_t>(rank, 1, v.size()) - 1);
}
double ms(std::uint64_t us) {
return static_cast<double>(us) / 1000.0;
}
struct Options {
std::string host, server_name, ca, issuer_key, metrics_host;
std::uint16_t port{}, metrics_port{};
std::uint64_t world{}, frame{}, region{}, seed{1}, lifetime{900};
std::vector<std::size_t> steps{20};
std::uint64_t hold_s{30}, warmup_s{5}, ramp_ms{100}, threads{4}, response_budget_ms{150};
bool per_client{}, expect_established{};
};
// What one bot measured since the last take.
struct Samples {
std::vector<std::uint64_t> response_us, receipt_us, outcome_us, login_us, correction_um,
gap_ticks;
std::uint64_t snapshots{}, bytes{}, intents{}, refused_intents{}, logins{}, drops{},
login_failures{}, refused{};
std::uint64_t connected_polls{}; // Polls while established (starvation is judged on these).
};
void append(std::vector<std::uint64_t>& to, const std::vector<std::uint64_t>& from) {
to.insert(to.end(), from.begin(), from.end());
}
struct Bot {
std::size_t index{};
AccountId account;
std::uint64_t serial{};
std::unique_ptr<client::RegionSessionClient> c;
std::atomic<bool> active{false}, established{false};
Clock::time_point connect_at, retry_at, next_action, next_interest, next_turn;
client::MotionPredictor predictor;
client::InputLead lead;
std::uint64_t last_tick{}, last_view{}, last_bytes{}, next_action_id{1};
std::uint64_t applied{}; // Newest applied intention sequence seen in an own marker.
std::map<std::uint64_t, Clock::time_point> receipt_pending, response_pending, outcome_pending;
std::map<std::uint64_t, std::uint64_t> intent_tick;
std::int16_t vx{}, vy{};
std::optional<SessionInterest> home;
int interest_step{};
std::mt19937_64 rng;
Samples s;
};
class Load {
public:
Load(Options o, GrantSigningKey key, std::string address)
: o_{std::move(o)}, key_{std::move(key)}, address_{std::move(address)},
authority_{read(o_.ca)} {
const auto most = *std::ranges::max_element(o_.steps);
bots_.reserve(most);
for (std::size_t i = 0; i < most; ++i) {
auto b = std::make_unique<Bot>();
b->index = i;
for (std::size_t k = 0; k < 8; ++k) {
b->account.bytes.at(k) = static_cast<std::uint8_t>((o_.seed >> (8U * k)) & 0xFFU);
b->account.bytes.at(8 + k) =
static_cast<std::uint8_t>(((i + 1) >> (8U * k)) & 0xFFU);
}
b->serial = now_seconds() * 64; // Later runs supersede earlier ones.
b->rng.seed(o_.seed * 1'000'003 + i);
bots_.push_back(std::move(b));
}
const auto threads = std::clamp<std::size_t>(o_.threads, 1, 64);
locks_ = std::vector<std::mutex>(threads);
for (std::size_t t = 0; t < threads; ++t)
workers_.emplace_back([this, t, threads] { work(t, threads); });
}
Load(const Load&) = delete;
Load& operator=(const Load&) = delete;
~Load() {
stop_ = true;
for (auto& w : workers_)
w.join();
}
void activate(std::size_t n) {
for (std::size_t i = 0; i < n && i < bots_.size(); ++i) {
if (bots_[i]->active.exchange(true))
continue;
std::this_thread::sleep_for(milliseconds{o_.ramp_ms});
}
}
// Every active bot's samples since the last take (cleared).
std::vector<Samples> take(std::size_t n) {
std::vector<Samples> out(n);
for (std::size_t t = 0; t < locks_.size(); ++t) {
const std::scoped_lock lock{locks_[t]};
for (std::size_t i = t; i < n; i += locks_.size())
out[i] = std::exchange(bots_[i]->s, Samples{});
}
return out;
}
[[nodiscard]] std::size_t established(std::size_t n) const {
return static_cast<std::size_t>(
std::count_if(bots_.begin(), bots_.begin() + static_cast<std::ptrdiff_t>(n),
[](const auto& b) { return b->established.load(); }));
}
private:
void work(std::size_t t, std::size_t threads) {
while (!stop_) {
{
const std::scoped_lock lock{locks_[t]};
const auto now = Clock::now();
for (std::size_t i = t; i < bots_.size(); i += threads)
if (bots_[i]->active)
step(*bots_[i], now);
}
std::this_thread::sleep_for(milliseconds{2});
}
const std::scoped_lock lock{locks_[t]};
for (std::size_t i = t; i < bots_.size(); i += threads)
if (auto& c = bots_[i]->c; c && c->state() == client::RegionSessionState::established)
c->close();
for (int k = 0; k < 50; ++k) { // Let the closes flush.
for (std::size_t i = t; i < bots_.size(); i += threads)
if (bots_[i]->c)
bots_[i]->c->poll(Clock::now());
std::this_thread::sleep_for(milliseconds{2});
}
}
void login(Bot& b, Clock::time_point now) {
GrantClaims claims;
claims.account = b.account;
claims.serial = ++b.serial;
claims.world = {o_.world, o_.frame};
claims.region = o_.region;
claims.issued_at = now_seconds();
claims.expires_at = claims.issued_at + o_.lifetime;
claims.grant_id = (b.serial << 16U) ^ (b.index + 1);
const auto grant = key_.sign(claims);
client::RegionSessionClientConfig config;
config.transport.server_address = address_;
config.transport.port = o_.port;
config.transport.server_name = o_.server_name;
config.transport.authority_pem = authority_;
config.transport.authority_pem_file = o_.ca;
if (!grant) {
b.retry_at = now + std::chrono::seconds{2};
return;
}
config.grant = *grant;
auto made = client::RegionSessionClient::connect(std::move(config), now);
if (!made) {
b.retry_at = now + std::chrono::seconds{2};
return;
}
b.c = std::move(*made);
b.connect_at = now;
b.last_tick = b.last_view = b.last_bytes = b.applied = 0;
b.receipt_pending.clear();
b.response_pending.clear();
b.outcome_pending.clear();
b.intent_tick.clear();
b.predictor.reset();
b.lead.reset();
b.home.reset();
}
void step(Bot& b, Clock::time_point now) {
if (!b.c) {
if (now >= b.retry_at)
login(b, now);
return;
}
auto& c = *b.c;
c.poll(now);
if (c.state() == client::RegionSessionState::closed) {
if (b.established.exchange(false))
++b.s.drops;
else
++b.s.login_failures;
if (c.close_reason() == SessionCloseReason::capacity ||
c.close_reason() == SessionCloseReason::rate_limited)
++b.s.refused;
b.c.reset();
b.retry_at = now + std::chrono::seconds{2} +
milliseconds{std::uniform_int_distribution<int>{0, 500}(b.rng)};
return;
}
if (c.state() != client::RegionSessionState::established)
return;
++b.s.connected_polls;
if (!b.established.exchange(true)) {
++b.s.logins;
b.s.login_us.push_back(micros(now - b.connect_at));
b.next_action = now + milliseconds{std::uniform_int_distribution<int>{0, 2000}(b.rng)};
b.next_interest = now + std::chrono::seconds{10};
}
// Receipts: RTT, lead and refusals.
for (const auto& r : c.take_intention_receipts()) {
if (const auto it = b.receipt_pending.find(r.sequence); it != b.receipt_pending.end()) {
b.s.receipt_us.push_back(micros(now - it->second));
b.receipt_pending.erase(it);
}
const auto tick = b.intent_tick.find(r.sequence);
if (r.status == AdmitStatus::admitted) {
if (tick != b.intent_tick.end())
b.lead.admitted(tick->second, r.server_tick);
} else {
++b.s.refused_intents;
b.predictor.rejected(r.sequence);
b.response_pending.erase(r.sequence);
if (r.status == AdmitStatus::outside_window)
b.lead.too_late();
}
if (tick != b.intent_tick.end())
b.intent_tick.erase(tick);
}
// Snapshots: cadence, own marker (responsiveness, prediction errors), bytes.
if (const auto* v = c.view(); v && v->tick != b.last_view) {
if (b.last_view != 0 && v->tick > b.last_view)
b.s.gap_ticks.push_back(v->tick - b.last_view);
b.last_view = v->tick;
++b.s.snapshots;
if (!b.home && !v->environment.empty()) {
std::int64_t x0 = v->environment.front().voxel.x, x1 = x0;
std::int64_t y0 = v->environment.front().voxel.y, y1 = y0;
for (const auto& cell : v->environment) {
x0 = std::min(x0, cell.voxel.x);
x1 = std::max(x1, cell.voxel.x);
y0 = std::min(y0, cell.voxel.y);
y1 = std::max(y1, cell.voxel.y);
}
b.home = SessionInterest{(x0 + x1) / 2, (y0 + y1) / 2, v->slice_z, 7};
}
for (const auto& m : v->markers) {
if (!m.own)
continue;
b.predictor.authoritative({v->tick, m.applied_sequence, m.state, m.area});
if (m.applied_sequence > b.applied) {
b.applied = m.applied_sequence;
while (!b.response_pending.empty() &&
b.response_pending.begin()->first <= b.applied) {
b.s.response_us.push_back(micros(now - b.response_pending.begin()->second));
b.response_pending.erase(b.response_pending.begin());
}
}
}
}
for (const auto e : b.predictor.take_errors())
b.s.correction_um.push_back(e);
const auto bytes = c.snapshot_bytes();
b.s.bytes += bytes - std::min(bytes, b.last_bytes);
b.last_bytes = bytes;
// Outcomes: final ones only.
for (const auto& out : c.take_action_outcomes())
if (out.result == ActionResult::committed_no_effect ||
out.result == ActionResult::not_committed)
if (const auto it = b.outcome_pending.find(out.action_id);
it != b.outcome_pending.end()) {
b.s.outcome_us.push_back(micros(now - it->second));
b.outcome_pending.erase(it);
}
(void)c.take_action_receipts();
// Once per server tick: an intention, a prediction, sometimes an action or interest.
const auto tick = c.server_tick();
if (tick == 0 || tick == b.last_tick)
return;
b.last_tick = tick;
if (now >= b.next_turn) {
const double angle = std::uniform_real_distribution<double>{0, 6.283185307}(b.rng);
b.vx = static_cast<std::int16_t>(std::lround(700.0 * std::cos(angle)));
b.vy = static_cast<std::int16_t>(std::lround(700.0 * std::sin(angle)));
b.next_turn = now + std::chrono::seconds{2};
}
const auto target = tick + b.lead.lead();
if (!b.predictor.full())
if (const auto seq = c.send_intention(target, b.vx, b.vy)) {
++b.s.intents;
(void)b.predictor.input({*seq, target, {b.vx, b.vy}});
b.receipt_pending[*seq] = now;
b.response_pending[*seq] = now;
b.intent_tick[*seq] = target;
for (auto* pending : {&b.receipt_pending, &b.response_pending})
while (pending->size() > 256)
pending->erase(pending->begin());
while (b.intent_tick.size() > 256)
b.intent_tick.erase(b.intent_tick.begin());
}
(void)b.predictor.predict(target);
(void)b.predictor.take_correction();
if (now >= b.next_action) {
const auto id = b.next_action_id++;
if (c.send_action(id, ActionKind::probe, tick + 3)) {
b.outcome_pending[id] = now;
while (b.outcome_pending.size() > 64)
b.outcome_pending.erase(b.outcome_pending.begin());
}
b.next_action = now + std::chrono::seconds{2};
}
if (now >= b.next_interest && b.home) {
auto in = *b.home;
in.x += (b.interest_step++ % 2 == 0) ? 1 : 0; // Alternate: a fresh baseline each.
(void)c.set_interest(in);
b.next_interest = now + std::chrono::seconds{10};
}
}
Options o_;
GrantSigningKey key_;
std::string address_, authority_;
std::vector<std::unique_ptr<Bot>> bots_;
std::vector<std::mutex> locks_;
std::vector<std::thread> workers_;
std::atomic<bool> stop_{false};
};
// The region's /metrics (evo_m10_* and evo_region_tick_debt), unlabelled series only.
std::map<std::string, std::uint64_t> scrape(const Options& o) {
std::map<std::string, std::uint64_t> out;
#ifndef _WIN32
if (o.metrics_port == 0)
return out;
const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0);
if (fd < 0)
return out;
sockaddr_in a{};
a.sin_family = AF_INET;
a.sin_port = htons(o.metrics_port);
timeval timeout{3, 0};
::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout);
std::string raw;
if (::inet_pton(AF_INET, o.metrics_host.c_str(), &a.sin_addr) == 1 &&
::connect(fd, reinterpret_cast<const sockaddr*>(&a), sizeof a) == 0) {
const std::string request = "GET /metrics HTTP/1.1\r\nHost: load\r\n\r\n";
if (::send(fd, request.data(), request.size(), MSG_NOSIGNAL) ==
static_cast<ssize_t>(request.size())) {
std::array<char, 4096> chunk{};
for (ssize_t n = 0; (n = ::recv(fd, chunk.data(), chunk.size(), 0)) > 0;)
raw.append(chunk.data(), static_cast<std::size_t>(n));
}
}
::close(fd);
std::size_t at = 0;
while (at < raw.size()) {
const auto end = std::min(raw.find('\n', at), raw.size());
const std::string_view line{raw.data() + at, end - at};
at = end + 1;
if (!line.starts_with("evo_m10_") && !line.starts_with("evo_region_tick_debt"))
continue;
const auto space = line.find(' ');
if (space == std::string_view::npos || line.find('{') != std::string_view::npos)
continue;
if (const auto v = number(line.substr(space + 1)))
out[std::string{line.substr(0, space)}] = *v;
}
#else
(void)o;
#endif
return out;
}
double cpu_seconds() {
#ifndef _WIN32
rusage u{};
::getrusage(RUSAGE_SELF, &u);
return static_cast<double>(u.ru_utime.tv_sec + u.ru_stime.tv_sec) +
(static_cast<double>(u.ru_utime.tv_usec + u.ru_stime.tv_usec) / 1e6);
#else
return 0;
#endif
}
void line(const char* name, const std::vector<std::uint64_t>& pooled,
const std::vector<std::uint64_t>& client_p95, double scale, const char* unit) {
const auto f = [&](std::uint64_t v) { return static_cast<double>(v) / scale; };
std::printf("load %s_%s p50/p95/p99/max=%.1f/%.1f/%.1f/%.1f n=%zu client_p95 "
"min/med/max=%.1f/%.1f/%.1f\n",
name, unit, f(percentile(pooled, 0.50)), f(percentile(pooled, 0.95)),
f(percentile(pooled, 0.99)), f(percentile(pooled, 1.0)), pooled.size(),
f(percentile(client_p95, 0.0)), f(percentile(client_p95, 0.5)),
f(percentile(client_p95, 1.0)));
}
// One step's report; true when within budget.
bool report(const Options& o, std::size_t n, std::size_t established,
const std::vector<Samples>& all, double seconds, double cpu,
const std::map<std::string, std::uint64_t>& server) {
Samples pooled;
std::vector<std::uint64_t> response95, receipt95, outcome95, login95, correction95, rates, kib;
for (const auto& s : all) {
append(pooled.response_us, s.response_us);
append(pooled.receipt_us, s.receipt_us);
append(pooled.outcome_us, s.outcome_us);
append(pooled.login_us, s.login_us);
append(pooled.correction_um, s.correction_um);
append(pooled.gap_ticks, s.gap_ticks);
pooled.snapshots += s.snapshots;
pooled.intents += s.intents;
pooled.refused_intents += s.refused_intents;
pooled.logins += s.logins;
pooled.drops += s.drops;
pooled.login_failures += s.login_failures;
pooled.refused += s.refused;
response95.push_back(percentile(s.response_us, 0.95));
receipt95.push_back(percentile(s.receipt_us, 0.95));
outcome95.push_back(percentile(s.outcome_us, 0.95));
if (!s.login_us.empty())
login95.push_back(percentile(s.login_us, 0.95));
correction95.push_back(percentile(s.correction_um, 0.95));
rates.push_back(static_cast<std::uint64_t>(
std::lround(1000.0 * static_cast<double>(s.snapshots) / std::max(seconds, 0.001))));
kib.push_back(static_cast<std::uint64_t>(
std::lround(static_cast<double>(s.bytes) / 1024.0 / std::max(seconds, 0.001))));
}
std::printf("load step N=%zu established=%zu logins=%llu drops=%llu login_failures=%llu "
"refused=%llu seconds=%.1f\n",
n, established, static_cast<unsigned long long>(pooled.logins),
static_cast<unsigned long long>(pooled.drops),
static_cast<unsigned long long>(pooled.login_failures),
static_cast<unsigned long long>(pooled.refused), seconds);
line("response", pooled.response_us, response95, 1000.0, "ms");
line("receipt", pooled.receipt_us, receipt95, 1000.0, "ms");
line("outcome", pooled.outcome_us, outcome95, 1000.0, "ms");
line("login", pooled.login_us, login95, 1000.0, "ms");
line("correction", pooled.correction_um, correction95, 1000.0, "mm");
std::printf("load snapshots hz min/med/max=%.2f/%.2f/%.2f gap_ticks p99/max=%llu/%llu "
"kib_s med/max=%llu/%llu intents=%llu refused_intents=%llu\n",
static_cast<double>(percentile(rates, 0.0)) / 1000.0,
static_cast<double>(percentile(rates, 0.5)) / 1000.0,
static_cast<double>(percentile(rates, 1.0)) / 1000.0,
static_cast<unsigned long long>(percentile(pooled.gap_ticks, 0.99)),
static_cast<unsigned long long>(percentile(pooled.gap_ticks, 1.0)),
static_cast<unsigned long long>(percentile(kib, 0.5)),
static_cast<unsigned long long>(percentile(kib, 1.0)),
static_cast<unsigned long long>(pooled.intents),
static_cast<unsigned long long>(pooled.refused_intents));
std::printf("load client_cpu_pct=%.0f\n", 100.0 * cpu / std::max(seconds, 0.001));
if (!server.empty()) {
std::printf("load server");
for (const auto& [k, v] : server)
std::printf(" %s=%llu", k.c_str(), static_cast<unsigned long long>(v));
std::printf("\n");
}
// Starvation: throughput must not hide a client left behind. Judged among the bots that
// were connected during the hold (the others are reported as not established).
std::vector<std::uint64_t> connected_rates, connected_response;
for (std::size_t i = 0; i < all.size(); ++i)
if (all[i].connected_polls > 0) {
connected_rates.push_back(rates[i]);
connected_response.push_back(response95[i]);
}
const auto median_rate = percentile(connected_rates, 0.5);
const auto median_response = percentile(connected_response, 0.5);
const auto budget_us = o.response_budget_ms * 1000;
std::vector<std::size_t> starved;
for (std::size_t i = 0; i < all.size(); ++i)
if (all[i].connected_polls > 0 &&
(rates[i] * 5 < median_rate * 4 ||
(response95[i] > 2 * median_response && response95[i] > budget_us)))
starved.push_back(i);
std::printf("load starved %zu", starved.size());
for (const auto i : starved)
std::printf(" %zu", i);
std::printf("\n");
std::vector<std::string> why;
if (established < n)
why.emplace_back("not-established");
if (pooled.refused > 0 || pooled.login_failures > 0)
why.emplace_back("refused");
if (pooled.drops > 0)
why.emplace_back("drops");
if (!starved.empty())
why.emplace_back("starved");
if (pooled.response_us.empty() || percentile(pooled.response_us, 0.95) > budget_us)
why.emplace_back("response");
if (pooled.outcome_us.empty() || percentile(pooled.outcome_us, 0.95) > 1'500'000)
why.emplace_back("outcome");
const auto get = [&](const char* k) {
const auto it = server.find(k);
return it == server.end() ? 0 : it->second;
};
if (get("evo_m10_tick_work_p99_microseconds") > 25'000)
why.emplace_back("tick-work");
if (get("evo_m10_refusing") != 0 || get("evo_m10_shed_level") != 0)
why.emplace_back("governor");
if (get("evo_region_tick_debt") != 0)
why.emplace_back("tick-debt");
std::printf("load verdict N=%zu within_budget=%d", n, why.empty() ? 1 : 0);
for (const auto& w : why)
std::printf(" %s", w.c_str());
std::printf("\n");
if (o.per_client)
for (std::size_t i = 0; i < all.size(); ++i)
std::printf("load client %zu snapshots=%llu response_p95_ms=%.1f receipt_p95_ms=%.1f "
"outcome_p95_ms=%.1f correction_p95_mm=%.1f drops=%llu "
"login_failures=%llu\n",
i, static_cast<unsigned long long>(all[i].snapshots), ms(response95[i]),
ms(receipt95[i]), ms(outcome95[i]),
static_cast<double>(correction95[i]) / 1000.0,
static_cast<unsigned long long>(all[i].drops),
static_cast<unsigned long long>(all[i].login_failures));
std::fflush(stdout);
return why.empty();
}
std::optional<std::pair<std::string, std::uint16_t>> endpoint(std::string_view v) {
const auto colon = v.rfind(':');
if (colon == std::string_view::npos || colon == 0)
return std::nullopt;
const auto port = number(v.substr(colon + 1));
if (!port || *port == 0 || *port > 65535)
return std::nullopt;
return std::pair{std::string{v.substr(0, colon)}, static_cast<std::uint16_t>(*port)};
}
std::optional<std::vector<std::size_t>> steps_of(std::string_view v) {
std::vector<std::size_t> out;
while (!v.empty()) {
const auto comma = v.find(',');
const auto n = number(v.substr(0, comma));
if (!n || *n == 0 || *n > 256 || (!out.empty() && *n < out.back()))
return std::nullopt;
out.push_back(*n);
v.remove_prefix(comma == std::string_view::npos ? v.size() : comma + 1);
}
if (out.empty())
return std::nullopt;
return out;
}
} // namespace
int main(int argc, char** argv) {
const std::vector<std::string_view> args(argv + 1, argv + argc);
Options o;
bool ok = true;
for (std::size_t i = 0; i < args.size() && ok; ++i) {
const auto flag = args[i];
if (flag == "--per-client") {
o.per_client = true;
continue;
}
if (flag == "--expect-established") {
o.expect_established = true;
continue;
}
if (i + 1 >= args.size()) {
ok = false;
break;
}
const auto value = args[++i];
const auto n = number(value);
if (flag == "--connect") {
const auto e = endpoint(value);
ok = e.has_value();
if (e)
std::tie(o.host, o.port) = *e;
} else if (flag == "--metrics") {
const auto e = endpoint(value);
ok = e.has_value();
if (e)
std::tie(o.metrics_host, o.metrics_port) = *e;
} else if (flag == "--server-name")
o.server_name = value;
else if (flag == "--ca")
o.ca = value;
else if (flag == "--issuer-key")
o.issuer_key = value;
else if (flag == "--steps") {
const auto s = steps_of(value);
ok = s.has_value();
if (s)
o.steps = *s;
} else if (n && flag == "--world")
o.world = *n;
else if (n && flag == "--frame")
o.frame = *n;
else if (n && flag == "--region")
o.region = *n;
else if (n && flag == "--seed" && *n != 0)
o.seed = *n;
else if (n && flag == "--lifetime" && *n >= 60 && *n <= 3600)
o.lifetime = *n;
else if (n && flag == "--hold" && *n >= 1 && *n <= 3600)
o.hold_s = *n;
else if (n && flag == "--warmup" && *n <= 600)
o.warmup_s = *n;
else if (n && flag == "--ramp-ms" && *n <= 10'000)
o.ramp_ms = *n;
else if (n && flag == "--threads" && *n >= 1 && *n <= 64)
o.threads = *n;
else if (n && flag == "--response-budget-ms" && *n >= 1)
o.response_budget_ms = *n;
else
ok = false;
}
if (!ok || o.host.empty() || o.server_name.empty() || o.ca.empty() || o.issuer_key.empty() ||
o.world == 0 || o.frame == 0 || o.region == 0) {
std::fputs("usage: evo-load --connect HOST:PORT --server-name NAME --ca FILE "
"--issuer-key FILE --world N --frame N --region N [--steps N[,N]...] "
"[--hold S] [--warmup S] [--ramp-ms MS] [--threads T] [--seed X] "
"[--lifetime S] [--metrics HOST:PORT] [--response-budget-ms MS] "
"[--per-client] [--expect-established]\n",
stderr);
return 2;
}
const auto scalar = read(o.issuer_key);
auto key = GrantSigningKey::import(
DeploymentEnvironment::dev, 1,
std::span{reinterpret_cast<const std::uint8_t*>(scalar.data()), scalar.size()});
if (!key) {
std::puts("error issuer-key");
return 2;
}
const auto address = quic::resolve_ipv4(o.host);
if (!address) {
std::puts("error resolve");
return 2;
}
std::printf("load start steps=%zu hold=%llu warmup=%llu threads=%llu budget_ms=%llu\n",
o.steps.size(), static_cast<unsigned long long>(o.hold_s),
static_cast<unsigned long long>(o.warmup_s),
static_cast<unsigned long long>(o.threads),
static_cast<unsigned long long>(o.response_budget_ms));
std::fflush(stdout);
bool smoke_ok = true;
std::optional<std::size_t> sustainable, saturated;
{
Load load{o, std::move(*key), *address};
for (const auto n : o.steps) {
load.activate(n);
std::this_thread::sleep_for(std::chrono::seconds{o.warmup_s});
// The ramp's and warmup's logins (and closes) belong to this step; the rest of
// the warmup is discarded.
const auto warm = load.take(n);
const auto started = Clock::now();
const auto cpu0 = cpu_seconds();
std::this_thread::sleep_for(std::chrono::seconds{o.hold_s});
auto samples = load.take(n);
for (std::size_t i = 0; i < n; ++i) {
append(samples[i].login_us, warm[i].login_us);
samples[i].logins += warm[i].logins;
samples[i].drops += warm[i].drops;
samples[i].login_failures += warm[i].login_failures;
samples[i].refused += warm[i].refused;
}
const auto seconds = std::chrono::duration<double>(Clock::now() - started).count();
const auto established = load.established(n);
const bool within =
report(o, n, established, samples, seconds, cpu_seconds() - cpu0, scrape(o));
if (within && !saturated)
sustainable = n;
if (!within && !saturated)
saturated = n;
std::uint64_t snapshots = 0, outcomes = 0;
for (const auto& s : samples) {
snapshots += s.snapshots;
outcomes += s.outcome_us.size();
}
smoke_ok = smoke_ok && established == n && snapshots > 0 && outcomes > 0;
}
}
std::printf("load result sustainable=%zu saturated_at=%zu\n", sustainable.value_or(0),
saturated.value_or(0));
std::fflush(stdout);
return o.expect_established && !smoke_ok ? 1 : 0;
}