Add M10.9b evo-load multi-client load generator and process smoke
Foundation / checks (deploy) (push) Successful in 3m16s
Foundation / checks (ecs) (push) Successful in 1m54s
Foundation / checks (client) (push) Successful in 12m39s
Foundation / checks (network) (push) Successful in 12m56s
Foundation / checks (tsan) (push) Successful in 7m52s
Foundation / checks (checks) (push) Successful in 32m28s
Foundation / checks (postgres) (push) Failing after 26m52s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
Foundation / checks (deploy) (push) Successful in 3m16s
Foundation / checks (ecs) (push) Successful in 1m54s
Foundation / checks (client) (push) Successful in 12m39s
Foundation / checks (network) (push) Successful in 12m56s
Foundation / checks (tsan) (push) Successful in 7m52s
Foundation / checks (checks) (push) Successful in 32m28s
Foundation / checks (postgres) (push) Failing after 26m52s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
ADR-0097. evo-load runs bots on worker threads, each a RegionSessionClient with its own account and an in-process dev grant: one intention per tick at its input lead with marker prediction, actions every 2 s, interest moves every 10 s, and re-login after any close. Stepped ramps report pooled and per-client distributions (response, receipt, outcome, login, prediction error), snapshot rate, gaps and bandwidth, drops, failed logins and refusals, generator CPU, a starvation check among connected bots, a budget verdict and the region's evo_m10_* gauges, ending with the sustainable bot count. A Postgres process smoke runs 4 then 8 bots against a real evo-server. Fixes found while ramping: the process fixture's short pumps never read (a chatty child blocked on its stdout pipe), and the load governor refused on a single tick of debt (now 2 refuse, 4 shed). Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+8
-1
@@ -160,6 +160,13 @@ if(EVO_BUILD_QUIC)
|
||||
target_link_libraries(evo-session-probe PRIVATE evo_session_client evo_network_tls)
|
||||
evo_owned_target(evo-session-probe)
|
||||
evo_msquic_runtime(evo-session-probe)
|
||||
# M10.9b multi-client load generator (development and qualification; ADR-0097).
|
||||
find_package(Threads REQUIRED)
|
||||
add_executable(evo-load tools/network/load.cpp)
|
||||
target_link_libraries(evo-load PRIVATE evo_session_client evo_network_tls evo_client_prediction
|
||||
Threads::Threads)
|
||||
evo_owned_target(evo-load)
|
||||
evo_msquic_runtime(evo-load)
|
||||
if(BUILD_TESTING)
|
||||
# Owned impairment relay (tests and measurements only).
|
||||
add_library(evo_network_impair STATIC tools/network/impair/udp_relay.cpp)
|
||||
@@ -269,7 +276,7 @@ if(EVO_BUILD_POSTGRES)
|
||||
tests/region_handoff_process_tests.cpp)
|
||||
target_link_libraries(evo-postgres-tests PRIVATE evo_session_client)
|
||||
target_compile_definitions(evo-postgres-tests PRIVATE EVO_POSTGRES_M10=1)
|
||||
add_dependencies(evo-postgres-tests evo-session-probe)
|
||||
add_dependencies(evo-postgres-tests evo-session-probe evo-load)
|
||||
endif()
|
||||
add_dependencies(evo-postgres-tests evo-server evo-world-clone) # Spawned by process tests.
|
||||
evo_owned_target(evo-postgres-tests)
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@
|
||||
{"name": "client", "configurePreset": "client", "targets": ["evo-client-state-tests", "evo-client", "evo-client-lifecycle-tests", "evo-client-environment-tests", "evo-client-organism-tests", "evo-client-ui-model-tests", "evo-client-panel-tests", "evo-client-bindings-tests"]},
|
||||
{"name": "client-probe", "configurePreset": "client-probe", "targets": ["evo-client-probe"]},
|
||||
{"name": "network", "configurePreset": "network", "targets": ["evo-client-state-tests", "evo-network-tests", "evo-dev-session"]},
|
||||
{"name": "quic", "configurePreset": "quic", "targets": ["evo-quic-smoke-tests", "evo-quic-transport-tests", "evo-quic-session-tests", "evo-quic-hol-measure", "evo-session-probe", "evo-network-tests"]},
|
||||
{"name": "quic", "configurePreset": "quic", "targets": ["evo-quic-smoke-tests", "evo-quic-transport-tests", "evo-quic-session-tests", "evo-quic-hol-measure", "evo-session-probe", "evo-load", "evo-network-tests"]},
|
||||
{"name": "client-network", "configurePreset": "client-network", "targets": ["evo-client-state-tests", "evo-client", "evo-client-lifecycle-tests", "evo-client-environment-tests", "evo-client-organism-tests", "evo-client-ui-model-tests", "evo-client-panel-tests", "evo-client-bindings-tests", "evo-network-tests", "evo-dev-session", "evo-quic-session-tests", "evo-client-remote-tests", "evo-motion-trace", "evo-session-probe"]},
|
||||
{"name": "client-package", "configurePreset": "client-package", "targets": ["evo-client-state-tests", "evo-client", "evo-client-lifecycle-tests", "evo-client-environment-tests", "evo-client-organism-tests", "evo-client-ui-model-tests", "evo-client-panel-tests", "evo-client-bindings-tests", "evo-network-tests", "evo-dev-session", "evo-quic-session-tests", "evo-client-remote-tests", "evo-motion-trace"]}
|
||||
],
|
||||
|
||||
@@ -12,7 +12,7 @@ spec:
|
||||
env:
|
||||
- {name: REGION_SESSION, value: "dev"}
|
||||
- {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"}
|
||||
- {name: REGION_SESSION_WINDOW, value: "dev-20261007T102834-1d97:1791370114"}
|
||||
- {name: REGION_SESSION_WINDOW, value: "dev-20261007T125653-8fc4:1791379013"}
|
||||
ports:
|
||||
# QUIC (ADR-0090): UDP only; nothing listens on TCP 7400.
|
||||
- {name: session, containerPort: 7400, protocol: UDP}
|
||||
|
||||
+4
-3
@@ -543,9 +543,10 @@ configuration done; 10.8d two evo-server processes on PostgreSQL with SIGKILL ar
|
||||
durable step, duplicate/stale retries and a failed destination done (schema v14 recovery
|
||||
sweep, replay guard). M10.8 complete. 10.9 split (ADR-0097; only evolution-dev reference-worker
|
||||
runs qualify): 10.9a per-tick work measurement, load governor (refuse logins, then shed
|
||||
replication with hysteresis) and its metrics done; 10.9b evo-load multi-client tool, 10.9c
|
||||
abuse suite (fuzz, floods, slow peers, storms, lease loss) and 10.9d dev-cluster harness and
|
||||
reference results follow.
|
||||
replication with hysteresis) and its metrics done; 10.9b evo-load multi-client tool
|
||||
(per-client distributions, starvation check, step verdicts, server gauges) and its process
|
||||
smoke done; 10.9c abuse suite (fuzz, floods, slow peers, storms, lease loss) and 10.9d
|
||||
dev-cluster harness and reference results follow.
|
||||
|
||||
**Entry:** M9 client/session evidence and M7/M8 dev/stage durability. Read
|
||||
NETWORK_ARCHITECTURE, PERSISTENCE_ARCHITECTURE and ADR-0017. The M9 transport is
|
||||
|
||||
+7
-1
@@ -107,7 +107,13 @@ layer) and feeds a load governor:
|
||||
- both relax after 5 s of calm.
|
||||
|
||||
`/metrics` exposes the work percentiles (total and session layer) and the governor's
|
||||
state. Next: 10.9b, the `evo-load` multi-client tool.
|
||||
state. M10.9b adds `evo-load`, a multi-client load generator. It runs bots on worker
|
||||
threads that send intentions every tick, actions, interest moves and reconnects. It reports
|
||||
per step pooled and per-client distributions, a starvation check, a budget verdict and the
|
||||
region's gauges, ending with the sustainable bot count. In a local development reference
|
||||
(not evidence), one region stayed within budget up to 24 bots and the governor began
|
||||
refusing at 32 (work p95 about 14 ms), with responses about 103 ms and outcomes about
|
||||
665 ms throughout. Next: 10.9c, the abuse suite.
|
||||
|
||||
## Previous gate - 2026-10-02
|
||||
|
||||
|
||||
@@ -44,10 +44,14 @@ snapshots.
|
||||
- **Load governor** (`include/evo/server/load_governor.hpp`, portable, no clock). It keeps
|
||||
a window of the last 40 samples (2 s). It reports the window's p95, p99 and maximum, plus
|
||||
two outputs:
|
||||
- **refusing** turns on when p95 ≥ 15 ms or tick debt > 0;
|
||||
- **shed level** (0-3) rises by one when p95 ≥ 20 ms or tick debt ≥ 2, at most once per
|
||||
- **refusing** turns on when p95 ≥ 15 ms or tick debt ≥ 2;
|
||||
- **shed level** (0-3) rises by one when p95 ≥ 20 ms or tick debt ≥ 4, at most once per
|
||||
second.
|
||||
|
||||
One tick of debt is jitter (a commit flush or a scheduler hiccup) and only interrupts calm.
|
||||
As first written, any debt refused logins for at least 5 s, which made a login just after
|
||||
a restart's catch-up fail intermittently in a process test (amended in 10.9b).
|
||||
|
||||
Calm means p95 < 10 ms and no debt. After 5 s of calm the level drops by one. At level 0,
|
||||
5 s more of calm turns refusing off. Any non-calm sample restarts the calm timer.
|
||||
Thresholds are configurable and validated (`relax < refuse ≤ shed ≤ tick budget`).
|
||||
@@ -105,3 +109,49 @@ snapshots.
|
||||
7 ms and p99 about 9 ms per tick. Of that, the session layer's p99 was about 4.5 ms. That
|
||||
is far above 0.5 ms per session, so 10.9b examines it with evo-load. Likely contributors
|
||||
are the per-tick snapshot source query and synchronous control renewals.
|
||||
|
||||
## 10.9b load tool (2026-10-07)
|
||||
- **`evo-load`** (`tools/network/load.cpp`, quic preset; usage in its header). Bots run on
|
||||
worker threads, each a `RegionSessionClient` with its own account and a dev grant signed
|
||||
in-process. A bot:
|
||||
- sends one intention per server tick at its `InputLead` (a random walk turning every
|
||||
2 s) and predicts its marker with `MotionPredictor`;
|
||||
- sends a probe action every 2 s and moves its interest every 10 s (a fresh baseline);
|
||||
- logs in again 2 s after any close.
|
||||
|
||||
Steps raise the bot count (`--ramp-ms` apart), warm up, then hold. Each step reports:
|
||||
- pooled p50, p95, p99 and maximum, plus the per-client p95 spread, for responsiveness
|
||||
(input to the first own-marker snapshot applying it), receipt RTT, final-outcome
|
||||
latency, login time and prediction error;
|
||||
- snapshot rate, gaps and bandwidth per client;
|
||||
- drops, failed logins and refusals;
|
||||
- the generator's own CPU;
|
||||
- with `--metrics`, the region's `evo_m10_*` gauges.
|
||||
|
||||
Starvation is judged among the bots connected during the hold: a snapshot rate under
|
||||
80 % of the median, or a response p95 over both twice the median and the budget. A
|
||||
step's verdict is within budget only with every bot established, no refusals or drops,
|
||||
no starvation, response p95 within budget (150 ms by default), outcome p95 ≤ 1.5 s,
|
||||
server work p99 ≤ 25 ms, no shedding or refusing, and no tick debt. The run ends with
|
||||
`load result sustainable=N saturated_at=M`.
|
||||
- **Smoke** (Postgres `region_m10_session_tests`): 4 then 8 bots against a real evo-server
|
||||
with `--metrics`. Every bot logs in once with no drops or refusals, both steps measure
|
||||
responses, outcomes and snapshots, and the report is complete.
|
||||
- **Found while ramping:**
|
||||
- The process-test fixture's `Child::pump(1ms)` rounded its wait down to 0 ms and never
|
||||
read. A chatty evo-server then blocked on its full stdout pipe, which looked like a
|
||||
server stall at 32 bots. The wait is now rounded up.
|
||||
- The governor's any-debt refusal was too twitchy (see the amended decision above).
|
||||
- **Local development reference** (WSL, Release, one region; not evidence):
|
||||
|
||||
| Bots | Verdict | Work p95 / max | Session p95 (per session) | Response p95 | Outcome p95 | Snapshots |
|
||||
|---|---|---|---|---|---|---|
|
||||
| 8 | within | 9.0 / 9.6 ms | 1.7 ms (0.22) | 106 ms | 659 ms | 20 Hz, no gaps |
|
||||
| 16 | within | 11.0 / 11.2 ms | 3.5 ms (0.22) | 102 ms | 661 ms | 20 Hz |
|
||||
| 24 | within | 13.1 / 13.3 ms | 5.8 ms (0.24) | 103 ms | 664 ms | 20 Hz |
|
||||
| 32 | refusing | 14.3 / 17.0 ms | 8.4 ms (0.26) | 103 ms | 665 ms | 20 Hz |
|
||||
|
||||
At 48 and 64 bots, the bots beyond the 32-session cap fail at the transport connection
|
||||
limit, and the 32 established sessions are unaffected. Prediction errors were 0 (no
|
||||
loss on loopback), and no client starved at any step. The host step dominates the work;
|
||||
the session layer stays near 0.25 ms per session (budget 0.5).
|
||||
|
||||
@@ -4948,3 +4948,43 @@ Not done:
|
||||
- qualification on the evolution-dev reference worker (10.9d; needs v4 and v12-v14
|
||||
deployed);
|
||||
- load generation (10.9b) and the abuse suite (10.9c).
|
||||
|
||||
## 2026-10-07 — M10.9b evo-load (ADR-0097)
|
||||
|
||||
Implemented:
|
||||
- `evo-load` (`tools/network/load.cpp`, quic preset): bots on worker threads (intentions
|
||||
every tick at the input lead with prediction, actions every 2 s, interest moves every
|
||||
10 s, re-login after any close), stepped ramps.
|
||||
- Per step, it reports pooled and per-client distributions (response, receipt, outcome,
|
||||
login, prediction error), snapshot rate, gaps and bandwidth, drops, failed logins and
|
||||
refusals, the generator's CPU, a starvation check and a budget verdict. With
|
||||
`--metrics`, it adds the region's gauges. The run ends with `sustainable` and
|
||||
`saturated_at`.
|
||||
- A Postgres process smoke (`region_m10_session_tests` `load_smoke`): 4 then 8 bots against
|
||||
a real evo-server.
|
||||
- Fixes found while ramping:
|
||||
- The process fixture's `Child::pump` rounded short waits down to 0 ms and never read,
|
||||
so a chatty child blocked on its stdout pipe.
|
||||
- The load governor refused on a single tick of debt. It now refuses at 2 and sheds at
|
||||
4; one tick only interrupts calm (ADR-0097 amended).
|
||||
- The M10 end-to-end test waits until the server is not refusing before the headless
|
||||
probe logs in, and dumps the probe and server lines on failure. One run had failed
|
||||
there intermittently before the debt change.
|
||||
|
||||
Executed:
|
||||
- **Linux GCC 14:**
|
||||
- `postgres-check.sh` with QUIC, all suites, 3 runs in a row (2 after the final change);
|
||||
- the smoke passes with every bot logged in once and both steps within budget;
|
||||
- presets `quic` (ops, network, QUIC sessions) 3/3.
|
||||
- **Local development reference ramp** (temporary steps 8-64, not committed, not
|
||||
evidence): within budget up to 24 bots; the governor refused from 32 (work p95 about
|
||||
14 ms, maximum 17 ms); session work stayed near 0.25 ms per session; responses p95 about
|
||||
103 ms, outcomes about 665 ms, 20 Hz snapshots, no starvation. Beyond the 32-session cap,
|
||||
extra logins failed at the transport limit and established sessions were unaffected.
|
||||
- **Linux Clang 19:** clang-tidy 19 on `load.cpp`, the governor, ops and the M10 process
|
||||
tests (clean); clang-format 19.
|
||||
- **Windows MSVC 19.44:** presets `quic` 4/4 (`evo-load` builds) and `debug` 60/60.
|
||||
|
||||
Not done:
|
||||
- the reference-worker run (10.9d);
|
||||
- the abuse suite (10.9c).
|
||||
|
||||
@@ -19,6 +19,9 @@ struct LoadPolicy {
|
||||
std::uint8_t max_shed{3};
|
||||
};
|
||||
inline constexpr std::size_t load_window_ticks = 40; // 2 s of 50 ms ticks.
|
||||
// Tick debt (due minus simulated) that refuses logins and that sheds. One tick is jitter (a
|
||||
// commit flush, a scheduler hiccup): it only interrupts calm.
|
||||
inline constexpr std::uint64_t refuse_debt_ticks = 2, shed_debt_ticks = 4;
|
||||
struct LoadState {
|
||||
bool refusing{};
|
||||
std::uint8_t shed{};
|
||||
|
||||
@@ -35,7 +35,7 @@ in real time; changing it requires a versioned rules migration (ADR-0089).
|
||||
| Per-tick region work | p99 <= 25 ms on the reference worker | budget | 10.9 (measured: `evo_m10_tick_work_*`, ADR-0097) |
|
||||
| Session handling | <= 0.5 ms per session per tick | budget | 10.9 (measured: `evo_m10_session_work_*`, ADR-0097) |
|
||||
| Host input queue | 64 inputs, 64 KiB | budget | region host |
|
||||
| Overload response | shed interest, refuse new sessions before tick debt | budget | 10.9a load governor: refuse at p95 >= 15 ms or debt, shed at p95 >= 20 ms or debt >= 2, relax below 10 ms after 5 s (ADR-0097; provisional until 10.9d) |
|
||||
| Overload response | shed interest, refuse new sessions before tick debt | budget | 10.9a load governor: refuse at p95 >= 15 ms or debt >= 2, shed at p95 >= 20 ms or debt >= 4, relax below 10 ms after 5 s (ADR-0097; provisional until 10.9d) |
|
||||
|
||||
Determinism: authoritative state is exact given the checkpoint and committed per-tick
|
||||
inputs; timing, delivery, prediction, interpolation and presentation are tolerant
|
||||
|
||||
@@ -42,8 +42,8 @@ bool LoadGovernor::observe(std::uint64_t work_us, std::uint64_t session_us, std:
|
||||
state_.session_p99_us = rank(sessions, n, 99);
|
||||
|
||||
const auto before = std::pair{state_.refusing, state_.shed};
|
||||
const bool hot = state_.p95_us >= policy_.refuse_us || tick_debt > 0;
|
||||
const bool overloaded = state_.p95_us >= policy_.shed_us || tick_debt >= 2;
|
||||
const bool hot = state_.p95_us >= policy_.refuse_us || tick_debt >= refuse_debt_ticks;
|
||||
const bool overloaded = state_.p95_us >= policy_.shed_us || tick_debt >= shed_debt_ticks;
|
||||
if (hot)
|
||||
state_.refusing = true;
|
||||
if (overloaded && state_.shed < policy_.max_shed &&
|
||||
|
||||
Vendored
+3
-2
@@ -119,8 +119,9 @@ class Child {
|
||||
for (; seen < lines_.size(); ++seen)
|
||||
if (want(lines_[seen]))
|
||||
return true;
|
||||
const auto left =
|
||||
std::chrono::duration_cast<std::chrono::milliseconds>(deadline - Clock::now());
|
||||
// Rounded up: a short pump (1 ms) must still read what is waiting, or a chatty
|
||||
// child blocks on its full stdout pipe.
|
||||
const auto left = std::chrono::ceil<std::chrono::milliseconds>(deadline - Clock::now());
|
||||
if (left.count() <= 0 || !read_some(static_cast<int>(left.count())))
|
||||
return false;
|
||||
}
|
||||
|
||||
+5
-4
@@ -189,10 +189,11 @@ void load_governor() {
|
||||
// Down to 0 after another 5 s, then logins after 5 s more.
|
||||
CHECK(run(100, 4'000).shed == 0 && g.state().refusing);
|
||||
CHECK(run(99, 4'000).refusing && !run(1, 4'000).refusing);
|
||||
// Tick debt alone: refuse at 1, shed at 2, even with light work.
|
||||
CHECK(run(1, 1'000, 1).refusing && g.state().shed == 0);
|
||||
CHECK(run(1, 1'000, 2).shed == 1);
|
||||
CHECK(!g.observe(1'000, 250, 2, now + 10)); // Within the step interval: no change.
|
||||
// Tick debt alone, even with light work: one tick is jitter, 2 refuse, 4 shed.
|
||||
CHECK(!run(1, 1'000, 1).refusing);
|
||||
CHECK(run(1, 1'000, 2).refusing && g.state().shed == 0);
|
||||
CHECK(run(1, 1'000, 4).shed == 1);
|
||||
CHECK(!g.observe(1'000, 250, 4, now + 10)); // Within the step interval: no change.
|
||||
CHECK(g.state().session_p95_us == 1'000); // The window is mostly 4 ms ticks.
|
||||
// Debt gone and calm (from the next sample): recovers.
|
||||
CHECK(run(100, 1'000).shed == 1 && run(1, 1'000).shed == 0);
|
||||
|
||||
@@ -356,7 +356,10 @@ void end_to_end() {
|
||||
const auto o3 = outcome_of(*server, *c, 3, true);
|
||||
CHECK(o3 && must(o3).result == ActionResult::committed_no_effect && !record(alice, 3).empty());
|
||||
|
||||
// The headless probe: login, intentions, actions and a query, then a clean exit.
|
||||
// The headless probe: login, intentions, actions and a query, then a clean exit. After the
|
||||
// restarts the load governor (ADR-0097) may still refuse logins while the host catches up;
|
||||
// the probe does not retry, so wait until it accepts.
|
||||
CHECK(until(*server, *c, [&] { return metric(server->ops, "evo_m10_refusing") == 0u; }));
|
||||
const auto probe = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-session-probe";
|
||||
Child p{{probe.string(), "run",
|
||||
"--connect", "127.0.0.1:" + std::to_string(server->port),
|
||||
@@ -381,9 +384,18 @@ void end_to_end() {
|
||||
if (p.printed("done") || p.printed("timeout") || p.printed("closed"))
|
||||
code = p.wait_exit(10s);
|
||||
}
|
||||
CHECK(code == 0 && p.printed("welcome ") && p.printed("intention 1 0") &&
|
||||
p.printed("outcome 10 1 ") && p.printed("outcome 11 1 ") && p.printed("snapshot ") &&
|
||||
p.printed("snapshot-bytes "));
|
||||
const bool probed = code == 0 && p.printed("welcome ") && p.printed("intention 1 0") &&
|
||||
p.printed("outcome 10 1 ") && p.printed("outcome 11 1 ") &&
|
||||
p.printed("snapshot ") && p.printed("snapshot-bytes ");
|
||||
if (!probed) {
|
||||
for (const auto& line : p.lines())
|
||||
if (!starts(line, "snapshot ") && !starts(line, "marker "))
|
||||
std::printf("probe: %s\n", line.c_str());
|
||||
for (const auto& line : server->child->lines())
|
||||
if (starts(line, "m10 load") || starts(line, "m10 refused") || starts(line, "phase"))
|
||||
std::printf("server: %s\n", line.c_str());
|
||||
}
|
||||
CHECK(probed);
|
||||
CHECK(!record(bob, 10).empty() && !record(bob, 11).empty());
|
||||
// The probe's marker (ADR-0094): its intentions (100, 0) were applied and moved it +x.
|
||||
std::vector<std::array<long long, 4>> marks; // tick, x, y, applied
|
||||
@@ -402,9 +414,85 @@ void end_to_end() {
|
||||
CHECK(server->child->printed("m10 opened ") && server->child->printed("m10 committed ") &&
|
||||
server->child->printed("m10 motion area "));
|
||||
}
|
||||
// M10.9b (ADR-0097): evo-load against a real evo-server, two steps with the region's
|
||||
// metrics. A smoke test of the tool and the server under concurrent sessions, not
|
||||
// qualification (only evolution-dev runs count): it checks that every bot logs in, that
|
||||
// every step measures responses, outcomes and snapshots for its bots, and that the report
|
||||
// is complete. The lines are kept in the log as a development reference.
|
||||
void load_smoke() {
|
||||
Setup s{152, 0xC6};
|
||||
Server server{s};
|
||||
const auto tool = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-load";
|
||||
Child load{{tool.string(),
|
||||
"--connect",
|
||||
"127.0.0.1:" + std::to_string(server.port),
|
||||
"--server-name",
|
||||
name,
|
||||
"--ca",
|
||||
(s.region.root / "ca.pem").string(),
|
||||
"--issuer-key",
|
||||
(s.region.root / "issuer.key").string(),
|
||||
"--world",
|
||||
"7",
|
||||
"--frame",
|
||||
std::to_string(region_identity.frame),
|
||||
"--region",
|
||||
"152",
|
||||
"--steps",
|
||||
"4,8",
|
||||
"--hold",
|
||||
"8",
|
||||
"--warmup",
|
||||
"3",
|
||||
"--ramp-ms",
|
||||
"50",
|
||||
"--threads",
|
||||
"2",
|
||||
"--lifetime",
|
||||
"900",
|
||||
"--metrics",
|
||||
"127.0.0.1:" + std::to_string(server.ops),
|
||||
"--per-client"}};
|
||||
int code = -1;
|
||||
for (const auto end = Clock::now() + 120s; Clock::now() < end && code < 0;) {
|
||||
server.pump();
|
||||
load.pump(5ms);
|
||||
if (load.printed("load result "))
|
||||
code = load.wait_exit(30s);
|
||||
}
|
||||
for (const auto& line : load.lines())
|
||||
if (starts(line, "load "))
|
||||
std::printf("%s\n", line.c_str());
|
||||
CHECK(code == 0);
|
||||
std::size_t steps = 0;
|
||||
for (const auto& line : load.lines()) {
|
||||
if (!starts(line, "load step "))
|
||||
continue;
|
||||
++steps;
|
||||
// Every bot logged in once: 4 in the first step, 4 more in the second.
|
||||
CHECK((starts(line, "load step N=4 established=4 logins=4 ") ||
|
||||
starts(line, "load step N=8 established=8 logins=4 ")) &&
|
||||
line.find(" drops=0 login_failures=0 refused=0 ") != std::string::npos);
|
||||
}
|
||||
const auto has = [&](std::string_view prefix) {
|
||||
return std::ranges::count_if(load.lines(), [&](const std::string& l) {
|
||||
return starts(l, std::string{prefix});
|
||||
}) == 2;
|
||||
};
|
||||
CHECK(steps == 2 && has("load response_ms ") && has("load outcome_ms ") &&
|
||||
has("load snapshots ") && has("load server ") && has("load verdict "));
|
||||
CHECK(std::ranges::none_of(load.lines(), [](const std::string& l) {
|
||||
return starts(l, "load response_ms") && l.find(" n=0 ") != std::string::npos;
|
||||
}));
|
||||
CHECK(std::ranges::count_if(
|
||||
load.lines(), [](const std::string& l) { return starts(l, "load client "); }) == 12);
|
||||
server.child->signal(SIGTERM);
|
||||
CHECK(server.child->wait_exit(30s) == 0);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void run_region_m10_session_tests() {
|
||||
end_to_end();
|
||||
load_smoke();
|
||||
std::puts("region m10 session integration passed");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,764 @@
|
||||
// M10.9b multi-client load generator (ADR-0097). Development and qualification tool: only
|
||||
// runs against the evolution-dev reference worker count as evidence (10.9d); local runs are
|
||||
// smoke tests and development references.
|
||||
//
|
||||
// evo-load --connect HOST:PORT --server-name NAME --ca FILE --issuer-key FILE
|
||||
// --world N --frame N --region N [--steps N[,N]...] [--hold S] [--warmup S]
|
||||
// [--ramp-ms MS] [--threads T] [--seed X] [--lifetime S] [--metrics HOST:PORT]
|
||||
// [--response-budget-ms MS] [--per-client] [--expect-established]
|
||||
//
|
||||
// Each simulated client (bot) is a RegionSessionClient with its own account (from --seed and
|
||||
// its index) logging in with a dev grant signed here (the issuer key never leaves this
|
||||
// process). Bots run on T worker threads (each owns a slice). Every server tick a bot sends
|
||||
// one intention for tick + its input lead (a random walk turning every 2 s) and predicts its
|
||||
// probe marker; every 2 s it sends a probe action; every 10 s it moves its interest by a
|
||||
// cell (a fresh baseline). A closed or refused bot logs in again after 2 s with the next
|
||||
// serial.
|
||||
//
|
||||
// Steps: the bot count rises to each step's N (one login every --ramp-ms), waits --warmup
|
||||
// seconds, then measures for --hold seconds. Bots stay connected into the next step. Per
|
||||
// step it prints:
|
||||
// load step N=.. established=.. logins=.. drops=.. login_failures=.. refused=.. seconds=..
|
||||
// (drops: established sessions closed; login failures: closed before Welcome, including
|
||||
// transport refusals at the connection limit; refused: closed with capacity or
|
||||
// rate_limited, either way. Logins, drops and failures include the step's ramp and warmup;
|
||||
// everything else is the hold)
|
||||
// load response_ms p50/p95/p99/max=.. (input -> first own-marker snapshot applying it)
|
||||
// client_p95 min/med/max=..
|
||||
// load receipt_ms ... (intention -> receipt)
|
||||
// load outcome_ms ... (action -> final outcome)
|
||||
// load login_ms ... (connect -> Welcome)
|
||||
// load correction_mm ... (prediction error per authoritative marker)
|
||||
// load snapshots hz min/med/max=.. gap_ticks p99/max=.. kib_s med/max=..
|
||||
// intents=.. refused_intents=..
|
||||
// load client_cpu_pct=.. (this process, all threads; Linux)
|
||||
// load server ... (with --metrics: the region's evo_m10_* gauges at step end)
|
||||
// load starved N [indices] (among bots connected during the hold: snapshot rate < 80 % of
|
||||
// the median, or a response p95 over both twice the median and the budget)
|
||||
// load verdict N=.. within_budget=0|1 [reasons]
|
||||
// With --per-client each bot's own line follows its step. --expect-established exits 1
|
||||
// unless every step ended with all its bots established and every step saw snapshots and
|
||||
// final outcomes (the smoke test).
|
||||
#include "evo/client/motion_prediction.hpp"
|
||||
#include "evo/client/region_session.hpp"
|
||||
#include "evo/network/session_grant.hpp"
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <charconv>
|
||||
#include <chrono>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <ctime>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <iterator>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <thread>
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#ifndef _WIN32
|
||||
#include <arpa/inet.h>
|
||||
#include <netinet/in.h>
|
||||
#include <sys/resource.h>
|
||||
#include <sys/socket.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
using namespace evo;
|
||||
using namespace evo::network;
|
||||
using Clock = std::chrono::steady_clock;
|
||||
using std::chrono::milliseconds;
|
||||
|
||||
std::optional<std::uint64_t> number(std::string_view v) {
|
||||
std::uint64_t n = 0;
|
||||
const auto [end, ec] = std::from_chars(v.data(), v.data() + v.size(), n);
|
||||
if (ec != std::errc{} || end != v.data() + v.size())
|
||||
return std::nullopt;
|
||||
return n;
|
||||
}
|
||||
std::string read(const std::filesystem::path& path) {
|
||||
std::ifstream file{path, std::ios::binary};
|
||||
return {std::istreambuf_iterator<char>{file}, std::istreambuf_iterator<char>{}};
|
||||
}
|
||||
std::uint64_t now_seconds() {
|
||||
return static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(
|
||||
std::chrono::system_clock::now().time_since_epoch())
|
||||
.count());
|
||||
}
|
||||
std::uint64_t micros(Clock::duration d) {
|
||||
return static_cast<std::uint64_t>(std::max<std::int64_t>(
|
||||
0, std::chrono::duration_cast<std::chrono::microseconds>(d).count()));
|
||||
}
|
||||
// Nearest rank over a copy.
|
||||
std::uint64_t percentile(std::vector<std::uint64_t> v, double p) {
|
||||
if (v.empty())
|
||||
return 0;
|
||||
std::ranges::sort(v);
|
||||
const auto rank = static_cast<std::size_t>(std::ceil(p * static_cast<double>(v.size())));
|
||||
return v.at(std::clamp<std::size_t>(rank, 1, v.size()) - 1);
|
||||
}
|
||||
double ms(std::uint64_t us) {
|
||||
return static_cast<double>(us) / 1000.0;
|
||||
}
|
||||
|
||||
struct Options {
|
||||
std::string host, server_name, ca, issuer_key, metrics_host;
|
||||
std::uint16_t port{}, metrics_port{};
|
||||
std::uint64_t world{}, frame{}, region{}, seed{1}, lifetime{900};
|
||||
std::vector<std::size_t> steps{20};
|
||||
std::uint64_t hold_s{30}, warmup_s{5}, ramp_ms{100}, threads{4}, response_budget_ms{150};
|
||||
bool per_client{}, expect_established{};
|
||||
};
|
||||
|
||||
// What one bot measured since the last take.
|
||||
struct Samples {
|
||||
std::vector<std::uint64_t> response_us, receipt_us, outcome_us, login_us, correction_um,
|
||||
gap_ticks;
|
||||
std::uint64_t snapshots{}, bytes{}, intents{}, refused_intents{}, logins{}, drops{},
|
||||
login_failures{}, refused{};
|
||||
std::uint64_t connected_polls{}; // Polls while established (starvation is judged on these).
|
||||
};
|
||||
void append(std::vector<std::uint64_t>& to, const std::vector<std::uint64_t>& from) {
|
||||
to.insert(to.end(), from.begin(), from.end());
|
||||
}
|
||||
|
||||
struct Bot {
|
||||
std::size_t index{};
|
||||
AccountId account;
|
||||
std::uint64_t serial{};
|
||||
std::unique_ptr<client::RegionSessionClient> c;
|
||||
std::atomic<bool> active{false}, established{false};
|
||||
Clock::time_point connect_at, retry_at, next_action, next_interest, next_turn;
|
||||
client::MotionPredictor predictor;
|
||||
client::InputLead lead;
|
||||
std::uint64_t last_tick{}, last_view{}, last_bytes{}, next_action_id{1};
|
||||
std::uint64_t applied{}; // Newest applied intention sequence seen in an own marker.
|
||||
std::map<std::uint64_t, Clock::time_point> receipt_pending, response_pending, outcome_pending;
|
||||
std::map<std::uint64_t, std::uint64_t> intent_tick;
|
||||
std::int16_t vx{}, vy{};
|
||||
std::optional<SessionInterest> home;
|
||||
int interest_step{};
|
||||
std::mt19937_64 rng;
|
||||
Samples s;
|
||||
};
|
||||
|
||||
class Load {
|
||||
public:
|
||||
Load(Options o, GrantSigningKey key, std::string address)
|
||||
: o_{std::move(o)}, key_{std::move(key)}, address_{std::move(address)},
|
||||
authority_{read(o_.ca)} {
|
||||
const auto most = *std::ranges::max_element(o_.steps);
|
||||
bots_.reserve(most);
|
||||
for (std::size_t i = 0; i < most; ++i) {
|
||||
auto b = std::make_unique<Bot>();
|
||||
b->index = i;
|
||||
for (std::size_t k = 0; k < 8; ++k) {
|
||||
b->account.bytes.at(k) = static_cast<std::uint8_t>((o_.seed >> (8U * k)) & 0xFFU);
|
||||
b->account.bytes.at(8 + k) =
|
||||
static_cast<std::uint8_t>(((i + 1) >> (8U * k)) & 0xFFU);
|
||||
}
|
||||
b->serial = now_seconds() * 64; // Later runs supersede earlier ones.
|
||||
b->rng.seed(o_.seed * 1'000'003 + i);
|
||||
bots_.push_back(std::move(b));
|
||||
}
|
||||
const auto threads = std::clamp<std::size_t>(o_.threads, 1, 64);
|
||||
locks_ = std::vector<std::mutex>(threads);
|
||||
for (std::size_t t = 0; t < threads; ++t)
|
||||
workers_.emplace_back([this, t, threads] { work(t, threads); });
|
||||
}
|
||||
Load(const Load&) = delete;
|
||||
Load& operator=(const Load&) = delete;
|
||||
~Load() {
|
||||
stop_ = true;
|
||||
for (auto& w : workers_)
|
||||
w.join();
|
||||
}
|
||||
void activate(std::size_t n) {
|
||||
for (std::size_t i = 0; i < n && i < bots_.size(); ++i) {
|
||||
if (bots_[i]->active.exchange(true))
|
||||
continue;
|
||||
std::this_thread::sleep_for(milliseconds{o_.ramp_ms});
|
||||
}
|
||||
}
|
||||
// Every active bot's samples since the last take (cleared).
|
||||
std::vector<Samples> take(std::size_t n) {
|
||||
std::vector<Samples> out(n);
|
||||
for (std::size_t t = 0; t < locks_.size(); ++t) {
|
||||
const std::scoped_lock lock{locks_[t]};
|
||||
for (std::size_t i = t; i < n; i += locks_.size())
|
||||
out[i] = std::exchange(bots_[i]->s, Samples{});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
[[nodiscard]] std::size_t established(std::size_t n) const {
|
||||
return static_cast<std::size_t>(
|
||||
std::count_if(bots_.begin(), bots_.begin() + static_cast<std::ptrdiff_t>(n),
|
||||
[](const auto& b) { return b->established.load(); }));
|
||||
}
|
||||
|
||||
private:
|
||||
void work(std::size_t t, std::size_t threads) {
|
||||
while (!stop_) {
|
||||
{
|
||||
const std::scoped_lock lock{locks_[t]};
|
||||
const auto now = Clock::now();
|
||||
for (std::size_t i = t; i < bots_.size(); i += threads)
|
||||
if (bots_[i]->active)
|
||||
step(*bots_[i], now);
|
||||
}
|
||||
std::this_thread::sleep_for(milliseconds{2});
|
||||
}
|
||||
const std::scoped_lock lock{locks_[t]};
|
||||
for (std::size_t i = t; i < bots_.size(); i += threads)
|
||||
if (auto& c = bots_[i]->c; c && c->state() == client::RegionSessionState::established)
|
||||
c->close();
|
||||
for (int k = 0; k < 50; ++k) { // Let the closes flush.
|
||||
for (std::size_t i = t; i < bots_.size(); i += threads)
|
||||
if (bots_[i]->c)
|
||||
bots_[i]->c->poll(Clock::now());
|
||||
std::this_thread::sleep_for(milliseconds{2});
|
||||
}
|
||||
}
|
||||
void login(Bot& b, Clock::time_point now) {
|
||||
GrantClaims claims;
|
||||
claims.account = b.account;
|
||||
claims.serial = ++b.serial;
|
||||
claims.world = {o_.world, o_.frame};
|
||||
claims.region = o_.region;
|
||||
claims.issued_at = now_seconds();
|
||||
claims.expires_at = claims.issued_at + o_.lifetime;
|
||||
claims.grant_id = (b.serial << 16U) ^ (b.index + 1);
|
||||
const auto grant = key_.sign(claims);
|
||||
client::RegionSessionClientConfig config;
|
||||
config.transport.server_address = address_;
|
||||
config.transport.port = o_.port;
|
||||
config.transport.server_name = o_.server_name;
|
||||
config.transport.authority_pem = authority_;
|
||||
config.transport.authority_pem_file = o_.ca;
|
||||
if (!grant) {
|
||||
b.retry_at = now + std::chrono::seconds{2};
|
||||
return;
|
||||
}
|
||||
config.grant = *grant;
|
||||
auto made = client::RegionSessionClient::connect(std::move(config), now);
|
||||
if (!made) {
|
||||
b.retry_at = now + std::chrono::seconds{2};
|
||||
return;
|
||||
}
|
||||
b.c = std::move(*made);
|
||||
b.connect_at = now;
|
||||
b.last_tick = b.last_view = b.last_bytes = b.applied = 0;
|
||||
b.receipt_pending.clear();
|
||||
b.response_pending.clear();
|
||||
b.outcome_pending.clear();
|
||||
b.intent_tick.clear();
|
||||
b.predictor.reset();
|
||||
b.lead.reset();
|
||||
b.home.reset();
|
||||
}
|
||||
void step(Bot& b, Clock::time_point now) {
|
||||
if (!b.c) {
|
||||
if (now >= b.retry_at)
|
||||
login(b, now);
|
||||
return;
|
||||
}
|
||||
auto& c = *b.c;
|
||||
c.poll(now);
|
||||
if (c.state() == client::RegionSessionState::closed) {
|
||||
if (b.established.exchange(false))
|
||||
++b.s.drops;
|
||||
else
|
||||
++b.s.login_failures;
|
||||
if (c.close_reason() == SessionCloseReason::capacity ||
|
||||
c.close_reason() == SessionCloseReason::rate_limited)
|
||||
++b.s.refused;
|
||||
b.c.reset();
|
||||
b.retry_at = now + std::chrono::seconds{2} +
|
||||
milliseconds{std::uniform_int_distribution<int>{0, 500}(b.rng)};
|
||||
return;
|
||||
}
|
||||
if (c.state() != client::RegionSessionState::established)
|
||||
return;
|
||||
++b.s.connected_polls;
|
||||
if (!b.established.exchange(true)) {
|
||||
++b.s.logins;
|
||||
b.s.login_us.push_back(micros(now - b.connect_at));
|
||||
b.next_action = now + milliseconds{std::uniform_int_distribution<int>{0, 2000}(b.rng)};
|
||||
b.next_interest = now + std::chrono::seconds{10};
|
||||
}
|
||||
// Receipts: RTT, lead and refusals.
|
||||
for (const auto& r : c.take_intention_receipts()) {
|
||||
if (const auto it = b.receipt_pending.find(r.sequence); it != b.receipt_pending.end()) {
|
||||
b.s.receipt_us.push_back(micros(now - it->second));
|
||||
b.receipt_pending.erase(it);
|
||||
}
|
||||
const auto tick = b.intent_tick.find(r.sequence);
|
||||
if (r.status == AdmitStatus::admitted) {
|
||||
if (tick != b.intent_tick.end())
|
||||
b.lead.admitted(tick->second, r.server_tick);
|
||||
} else {
|
||||
++b.s.refused_intents;
|
||||
b.predictor.rejected(r.sequence);
|
||||
b.response_pending.erase(r.sequence);
|
||||
if (r.status == AdmitStatus::outside_window)
|
||||
b.lead.too_late();
|
||||
}
|
||||
if (tick != b.intent_tick.end())
|
||||
b.intent_tick.erase(tick);
|
||||
}
|
||||
// Snapshots: cadence, own marker (responsiveness, prediction errors), bytes.
|
||||
if (const auto* v = c.view(); v && v->tick != b.last_view) {
|
||||
if (b.last_view != 0 && v->tick > b.last_view)
|
||||
b.s.gap_ticks.push_back(v->tick - b.last_view);
|
||||
b.last_view = v->tick;
|
||||
++b.s.snapshots;
|
||||
if (!b.home && !v->environment.empty()) {
|
||||
std::int64_t x0 = v->environment.front().voxel.x, x1 = x0;
|
||||
std::int64_t y0 = v->environment.front().voxel.y, y1 = y0;
|
||||
for (const auto& cell : v->environment) {
|
||||
x0 = std::min(x0, cell.voxel.x);
|
||||
x1 = std::max(x1, cell.voxel.x);
|
||||
y0 = std::min(y0, cell.voxel.y);
|
||||
y1 = std::max(y1, cell.voxel.y);
|
||||
}
|
||||
b.home = SessionInterest{(x0 + x1) / 2, (y0 + y1) / 2, v->slice_z, 7};
|
||||
}
|
||||
for (const auto& m : v->markers) {
|
||||
if (!m.own)
|
||||
continue;
|
||||
b.predictor.authoritative({v->tick, m.applied_sequence, m.state, m.area});
|
||||
if (m.applied_sequence > b.applied) {
|
||||
b.applied = m.applied_sequence;
|
||||
while (!b.response_pending.empty() &&
|
||||
b.response_pending.begin()->first <= b.applied) {
|
||||
b.s.response_us.push_back(micros(now - b.response_pending.begin()->second));
|
||||
b.response_pending.erase(b.response_pending.begin());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const auto e : b.predictor.take_errors())
|
||||
b.s.correction_um.push_back(e);
|
||||
const auto bytes = c.snapshot_bytes();
|
||||
b.s.bytes += bytes - std::min(bytes, b.last_bytes);
|
||||
b.last_bytes = bytes;
|
||||
// Outcomes: final ones only.
|
||||
for (const auto& out : c.take_action_outcomes())
|
||||
if (out.result == ActionResult::committed_no_effect ||
|
||||
out.result == ActionResult::not_committed)
|
||||
if (const auto it = b.outcome_pending.find(out.action_id);
|
||||
it != b.outcome_pending.end()) {
|
||||
b.s.outcome_us.push_back(micros(now - it->second));
|
||||
b.outcome_pending.erase(it);
|
||||
}
|
||||
(void)c.take_action_receipts();
|
||||
// Once per server tick: an intention, a prediction, sometimes an action or interest.
|
||||
const auto tick = c.server_tick();
|
||||
if (tick == 0 || tick == b.last_tick)
|
||||
return;
|
||||
b.last_tick = tick;
|
||||
if (now >= b.next_turn) {
|
||||
const double angle = std::uniform_real_distribution<double>{0, 6.283185307}(b.rng);
|
||||
b.vx = static_cast<std::int16_t>(std::lround(700.0 * std::cos(angle)));
|
||||
b.vy = static_cast<std::int16_t>(std::lround(700.0 * std::sin(angle)));
|
||||
b.next_turn = now + std::chrono::seconds{2};
|
||||
}
|
||||
const auto target = tick + b.lead.lead();
|
||||
if (!b.predictor.full())
|
||||
if (const auto seq = c.send_intention(target, b.vx, b.vy)) {
|
||||
++b.s.intents;
|
||||
(void)b.predictor.input({*seq, target, {b.vx, b.vy}});
|
||||
b.receipt_pending[*seq] = now;
|
||||
b.response_pending[*seq] = now;
|
||||
b.intent_tick[*seq] = target;
|
||||
for (auto* pending : {&b.receipt_pending, &b.response_pending})
|
||||
while (pending->size() > 256)
|
||||
pending->erase(pending->begin());
|
||||
while (b.intent_tick.size() > 256)
|
||||
b.intent_tick.erase(b.intent_tick.begin());
|
||||
}
|
||||
(void)b.predictor.predict(target);
|
||||
(void)b.predictor.take_correction();
|
||||
if (now >= b.next_action) {
|
||||
const auto id = b.next_action_id++;
|
||||
if (c.send_action(id, ActionKind::probe, tick + 3)) {
|
||||
b.outcome_pending[id] = now;
|
||||
while (b.outcome_pending.size() > 64)
|
||||
b.outcome_pending.erase(b.outcome_pending.begin());
|
||||
}
|
||||
b.next_action = now + std::chrono::seconds{2};
|
||||
}
|
||||
if (now >= b.next_interest && b.home) {
|
||||
auto in = *b.home;
|
||||
in.x += (b.interest_step++ % 2 == 0) ? 1 : 0; // Alternate: a fresh baseline each.
|
||||
(void)c.set_interest(in);
|
||||
b.next_interest = now + std::chrono::seconds{10};
|
||||
}
|
||||
}
|
||||
|
||||
Options o_;
|
||||
GrantSigningKey key_;
|
||||
std::string address_, authority_;
|
||||
std::vector<std::unique_ptr<Bot>> bots_;
|
||||
std::vector<std::mutex> locks_;
|
||||
std::vector<std::thread> workers_;
|
||||
std::atomic<bool> stop_{false};
|
||||
};
|
||||
|
||||
// The region's /metrics (evo_m10_* and evo_region_tick_debt), unlabelled series only.
|
||||
std::map<std::string, std::uint64_t> scrape(const Options& o) {
|
||||
std::map<std::string, std::uint64_t> out;
|
||||
#ifndef _WIN32
|
||||
if (o.metrics_port == 0)
|
||||
return out;
|
||||
const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0);
|
||||
if (fd < 0)
|
||||
return out;
|
||||
sockaddr_in a{};
|
||||
a.sin_family = AF_INET;
|
||||
a.sin_port = htons(o.metrics_port);
|
||||
timeval timeout{3, 0};
|
||||
::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout);
|
||||
std::string raw;
|
||||
if (::inet_pton(AF_INET, o.metrics_host.c_str(), &a.sin_addr) == 1 &&
|
||||
::connect(fd, reinterpret_cast<const sockaddr*>(&a), sizeof a) == 0) {
|
||||
const std::string request = "GET /metrics HTTP/1.1\r\nHost: load\r\n\r\n";
|
||||
if (::send(fd, request.data(), request.size(), MSG_NOSIGNAL) ==
|
||||
static_cast<ssize_t>(request.size())) {
|
||||
std::array<char, 4096> chunk{};
|
||||
for (ssize_t n = 0; (n = ::recv(fd, chunk.data(), chunk.size(), 0)) > 0;)
|
||||
raw.append(chunk.data(), static_cast<std::size_t>(n));
|
||||
}
|
||||
}
|
||||
::close(fd);
|
||||
std::size_t at = 0;
|
||||
while (at < raw.size()) {
|
||||
const auto end = std::min(raw.find('\n', at), raw.size());
|
||||
const std::string_view line{raw.data() + at, end - at};
|
||||
at = end + 1;
|
||||
if (!line.starts_with("evo_m10_") && !line.starts_with("evo_region_tick_debt"))
|
||||
continue;
|
||||
const auto space = line.find(' ');
|
||||
if (space == std::string_view::npos || line.find('{') != std::string_view::npos)
|
||||
continue;
|
||||
if (const auto v = number(line.substr(space + 1)))
|
||||
out[std::string{line.substr(0, space)}] = *v;
|
||||
}
|
||||
#else
|
||||
(void)o;
|
||||
#endif
|
||||
return out;
|
||||
}
|
||||
double cpu_seconds() {
|
||||
#ifndef _WIN32
|
||||
rusage u{};
|
||||
::getrusage(RUSAGE_SELF, &u);
|
||||
return static_cast<double>(u.ru_utime.tv_sec + u.ru_stime.tv_sec) +
|
||||
(static_cast<double>(u.ru_utime.tv_usec + u.ru_stime.tv_usec) / 1e6);
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
void line(const char* name, const std::vector<std::uint64_t>& pooled,
|
||||
const std::vector<std::uint64_t>& client_p95, double scale, const char* unit) {
|
||||
const auto f = [&](std::uint64_t v) { return static_cast<double>(v) / scale; };
|
||||
std::printf("load %s_%s p50/p95/p99/max=%.1f/%.1f/%.1f/%.1f n=%zu client_p95 "
|
||||
"min/med/max=%.1f/%.1f/%.1f\n",
|
||||
name, unit, f(percentile(pooled, 0.50)), f(percentile(pooled, 0.95)),
|
||||
f(percentile(pooled, 0.99)), f(percentile(pooled, 1.0)), pooled.size(),
|
||||
f(percentile(client_p95, 0.0)), f(percentile(client_p95, 0.5)),
|
||||
f(percentile(client_p95, 1.0)));
|
||||
}
|
||||
|
||||
// One step's report; true when within budget.
|
||||
bool report(const Options& o, std::size_t n, std::size_t established,
|
||||
const std::vector<Samples>& all, double seconds, double cpu,
|
||||
const std::map<std::string, std::uint64_t>& server) {
|
||||
Samples pooled;
|
||||
std::vector<std::uint64_t> response95, receipt95, outcome95, login95, correction95, rates, kib;
|
||||
for (const auto& s : all) {
|
||||
append(pooled.response_us, s.response_us);
|
||||
append(pooled.receipt_us, s.receipt_us);
|
||||
append(pooled.outcome_us, s.outcome_us);
|
||||
append(pooled.login_us, s.login_us);
|
||||
append(pooled.correction_um, s.correction_um);
|
||||
append(pooled.gap_ticks, s.gap_ticks);
|
||||
pooled.snapshots += s.snapshots;
|
||||
pooled.intents += s.intents;
|
||||
pooled.refused_intents += s.refused_intents;
|
||||
pooled.logins += s.logins;
|
||||
pooled.drops += s.drops;
|
||||
pooled.login_failures += s.login_failures;
|
||||
pooled.refused += s.refused;
|
||||
response95.push_back(percentile(s.response_us, 0.95));
|
||||
receipt95.push_back(percentile(s.receipt_us, 0.95));
|
||||
outcome95.push_back(percentile(s.outcome_us, 0.95));
|
||||
if (!s.login_us.empty())
|
||||
login95.push_back(percentile(s.login_us, 0.95));
|
||||
correction95.push_back(percentile(s.correction_um, 0.95));
|
||||
rates.push_back(static_cast<std::uint64_t>(
|
||||
std::lround(1000.0 * static_cast<double>(s.snapshots) / std::max(seconds, 0.001))));
|
||||
kib.push_back(static_cast<std::uint64_t>(
|
||||
std::lround(static_cast<double>(s.bytes) / 1024.0 / std::max(seconds, 0.001))));
|
||||
}
|
||||
std::printf("load step N=%zu established=%zu logins=%llu drops=%llu login_failures=%llu "
|
||||
"refused=%llu seconds=%.1f\n",
|
||||
n, established, static_cast<unsigned long long>(pooled.logins),
|
||||
static_cast<unsigned long long>(pooled.drops),
|
||||
static_cast<unsigned long long>(pooled.login_failures),
|
||||
static_cast<unsigned long long>(pooled.refused), seconds);
|
||||
line("response", pooled.response_us, response95, 1000.0, "ms");
|
||||
line("receipt", pooled.receipt_us, receipt95, 1000.0, "ms");
|
||||
line("outcome", pooled.outcome_us, outcome95, 1000.0, "ms");
|
||||
line("login", pooled.login_us, login95, 1000.0, "ms");
|
||||
line("correction", pooled.correction_um, correction95, 1000.0, "mm");
|
||||
std::printf("load snapshots hz min/med/max=%.2f/%.2f/%.2f gap_ticks p99/max=%llu/%llu "
|
||||
"kib_s med/max=%llu/%llu intents=%llu refused_intents=%llu\n",
|
||||
static_cast<double>(percentile(rates, 0.0)) / 1000.0,
|
||||
static_cast<double>(percentile(rates, 0.5)) / 1000.0,
|
||||
static_cast<double>(percentile(rates, 1.0)) / 1000.0,
|
||||
static_cast<unsigned long long>(percentile(pooled.gap_ticks, 0.99)),
|
||||
static_cast<unsigned long long>(percentile(pooled.gap_ticks, 1.0)),
|
||||
static_cast<unsigned long long>(percentile(kib, 0.5)),
|
||||
static_cast<unsigned long long>(percentile(kib, 1.0)),
|
||||
static_cast<unsigned long long>(pooled.intents),
|
||||
static_cast<unsigned long long>(pooled.refused_intents));
|
||||
std::printf("load client_cpu_pct=%.0f\n", 100.0 * cpu / std::max(seconds, 0.001));
|
||||
if (!server.empty()) {
|
||||
std::printf("load server");
|
||||
for (const auto& [k, v] : server)
|
||||
std::printf(" %s=%llu", k.c_str(), static_cast<unsigned long long>(v));
|
||||
std::printf("\n");
|
||||
}
|
||||
// Starvation: throughput must not hide a client left behind. Judged among the bots that
|
||||
// were connected during the hold (the others are reported as not established).
|
||||
std::vector<std::uint64_t> connected_rates, connected_response;
|
||||
for (std::size_t i = 0; i < all.size(); ++i)
|
||||
if (all[i].connected_polls > 0) {
|
||||
connected_rates.push_back(rates[i]);
|
||||
connected_response.push_back(response95[i]);
|
||||
}
|
||||
const auto median_rate = percentile(connected_rates, 0.5);
|
||||
const auto median_response = percentile(connected_response, 0.5);
|
||||
const auto budget_us = o.response_budget_ms * 1000;
|
||||
std::vector<std::size_t> starved;
|
||||
for (std::size_t i = 0; i < all.size(); ++i)
|
||||
if (all[i].connected_polls > 0 &&
|
||||
(rates[i] * 5 < median_rate * 4 ||
|
||||
(response95[i] > 2 * median_response && response95[i] > budget_us)))
|
||||
starved.push_back(i);
|
||||
std::printf("load starved %zu", starved.size());
|
||||
for (const auto i : starved)
|
||||
std::printf(" %zu", i);
|
||||
std::printf("\n");
|
||||
std::vector<std::string> why;
|
||||
if (established < n)
|
||||
why.emplace_back("not-established");
|
||||
if (pooled.refused > 0 || pooled.login_failures > 0)
|
||||
why.emplace_back("refused");
|
||||
if (pooled.drops > 0)
|
||||
why.emplace_back("drops");
|
||||
if (!starved.empty())
|
||||
why.emplace_back("starved");
|
||||
if (pooled.response_us.empty() || percentile(pooled.response_us, 0.95) > budget_us)
|
||||
why.emplace_back("response");
|
||||
if (pooled.outcome_us.empty() || percentile(pooled.outcome_us, 0.95) > 1'500'000)
|
||||
why.emplace_back("outcome");
|
||||
const auto get = [&](const char* k) {
|
||||
const auto it = server.find(k);
|
||||
return it == server.end() ? 0 : it->second;
|
||||
};
|
||||
if (get("evo_m10_tick_work_p99_microseconds") > 25'000)
|
||||
why.emplace_back("tick-work");
|
||||
if (get("evo_m10_refusing") != 0 || get("evo_m10_shed_level") != 0)
|
||||
why.emplace_back("governor");
|
||||
if (get("evo_region_tick_debt") != 0)
|
||||
why.emplace_back("tick-debt");
|
||||
std::printf("load verdict N=%zu within_budget=%d", n, why.empty() ? 1 : 0);
|
||||
for (const auto& w : why)
|
||||
std::printf(" %s", w.c_str());
|
||||
std::printf("\n");
|
||||
if (o.per_client)
|
||||
for (std::size_t i = 0; i < all.size(); ++i)
|
||||
std::printf("load client %zu snapshots=%llu response_p95_ms=%.1f receipt_p95_ms=%.1f "
|
||||
"outcome_p95_ms=%.1f correction_p95_mm=%.1f drops=%llu "
|
||||
"login_failures=%llu\n",
|
||||
i, static_cast<unsigned long long>(all[i].snapshots), ms(response95[i]),
|
||||
ms(receipt95[i]), ms(outcome95[i]),
|
||||
static_cast<double>(correction95[i]) / 1000.0,
|
||||
static_cast<unsigned long long>(all[i].drops),
|
||||
static_cast<unsigned long long>(all[i].login_failures));
|
||||
std::fflush(stdout);
|
||||
return why.empty();
|
||||
}
|
||||
|
||||
std::optional<std::pair<std::string, std::uint16_t>> endpoint(std::string_view v) {
|
||||
const auto colon = v.rfind(':');
|
||||
if (colon == std::string_view::npos || colon == 0)
|
||||
return std::nullopt;
|
||||
const auto port = number(v.substr(colon + 1));
|
||||
if (!port || *port == 0 || *port > 65535)
|
||||
return std::nullopt;
|
||||
return std::pair{std::string{v.substr(0, colon)}, static_cast<std::uint16_t>(*port)};
|
||||
}
|
||||
std::optional<std::vector<std::size_t>> steps_of(std::string_view v) {
|
||||
std::vector<std::size_t> out;
|
||||
while (!v.empty()) {
|
||||
const auto comma = v.find(',');
|
||||
const auto n = number(v.substr(0, comma));
|
||||
if (!n || *n == 0 || *n > 256 || (!out.empty() && *n < out.back()))
|
||||
return std::nullopt;
|
||||
out.push_back(*n);
|
||||
v.remove_prefix(comma == std::string_view::npos ? v.size() : comma + 1);
|
||||
}
|
||||
if (out.empty())
|
||||
return std::nullopt;
|
||||
return out;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
const std::vector<std::string_view> args(argv + 1, argv + argc);
|
||||
Options o;
|
||||
bool ok = true;
|
||||
for (std::size_t i = 0; i < args.size() && ok; ++i) {
|
||||
const auto flag = args[i];
|
||||
if (flag == "--per-client") {
|
||||
o.per_client = true;
|
||||
continue;
|
||||
}
|
||||
if (flag == "--expect-established") {
|
||||
o.expect_established = true;
|
||||
continue;
|
||||
}
|
||||
if (i + 1 >= args.size()) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
const auto value = args[++i];
|
||||
const auto n = number(value);
|
||||
if (flag == "--connect") {
|
||||
const auto e = endpoint(value);
|
||||
ok = e.has_value();
|
||||
if (e)
|
||||
std::tie(o.host, o.port) = *e;
|
||||
} else if (flag == "--metrics") {
|
||||
const auto e = endpoint(value);
|
||||
ok = e.has_value();
|
||||
if (e)
|
||||
std::tie(o.metrics_host, o.metrics_port) = *e;
|
||||
} else if (flag == "--server-name")
|
||||
o.server_name = value;
|
||||
else if (flag == "--ca")
|
||||
o.ca = value;
|
||||
else if (flag == "--issuer-key")
|
||||
o.issuer_key = value;
|
||||
else if (flag == "--steps") {
|
||||
const auto s = steps_of(value);
|
||||
ok = s.has_value();
|
||||
if (s)
|
||||
o.steps = *s;
|
||||
} else if (n && flag == "--world")
|
||||
o.world = *n;
|
||||
else if (n && flag == "--frame")
|
||||
o.frame = *n;
|
||||
else if (n && flag == "--region")
|
||||
o.region = *n;
|
||||
else if (n && flag == "--seed" && *n != 0)
|
||||
o.seed = *n;
|
||||
else if (n && flag == "--lifetime" && *n >= 60 && *n <= 3600)
|
||||
o.lifetime = *n;
|
||||
else if (n && flag == "--hold" && *n >= 1 && *n <= 3600)
|
||||
o.hold_s = *n;
|
||||
else if (n && flag == "--warmup" && *n <= 600)
|
||||
o.warmup_s = *n;
|
||||
else if (n && flag == "--ramp-ms" && *n <= 10'000)
|
||||
o.ramp_ms = *n;
|
||||
else if (n && flag == "--threads" && *n >= 1 && *n <= 64)
|
||||
o.threads = *n;
|
||||
else if (n && flag == "--response-budget-ms" && *n >= 1)
|
||||
o.response_budget_ms = *n;
|
||||
else
|
||||
ok = false;
|
||||
}
|
||||
if (!ok || o.host.empty() || o.server_name.empty() || o.ca.empty() || o.issuer_key.empty() ||
|
||||
o.world == 0 || o.frame == 0 || o.region == 0) {
|
||||
std::fputs("usage: evo-load --connect HOST:PORT --server-name NAME --ca FILE "
|
||||
"--issuer-key FILE --world N --frame N --region N [--steps N[,N]...] "
|
||||
"[--hold S] [--warmup S] [--ramp-ms MS] [--threads T] [--seed X] "
|
||||
"[--lifetime S] [--metrics HOST:PORT] [--response-budget-ms MS] "
|
||||
"[--per-client] [--expect-established]\n",
|
||||
stderr);
|
||||
return 2;
|
||||
}
|
||||
const auto scalar = read(o.issuer_key);
|
||||
auto key = GrantSigningKey::import(
|
||||
DeploymentEnvironment::dev, 1,
|
||||
std::span{reinterpret_cast<const std::uint8_t*>(scalar.data()), scalar.size()});
|
||||
if (!key) {
|
||||
std::puts("error issuer-key");
|
||||
return 2;
|
||||
}
|
||||
const auto address = quic::resolve_ipv4(o.host);
|
||||
if (!address) {
|
||||
std::puts("error resolve");
|
||||
return 2;
|
||||
}
|
||||
std::printf("load start steps=%zu hold=%llu warmup=%llu threads=%llu budget_ms=%llu\n",
|
||||
o.steps.size(), static_cast<unsigned long long>(o.hold_s),
|
||||
static_cast<unsigned long long>(o.warmup_s),
|
||||
static_cast<unsigned long long>(o.threads),
|
||||
static_cast<unsigned long long>(o.response_budget_ms));
|
||||
std::fflush(stdout);
|
||||
bool smoke_ok = true;
|
||||
std::optional<std::size_t> sustainable, saturated;
|
||||
{
|
||||
Load load{o, std::move(*key), *address};
|
||||
for (const auto n : o.steps) {
|
||||
load.activate(n);
|
||||
std::this_thread::sleep_for(std::chrono::seconds{o.warmup_s});
|
||||
// The ramp's and warmup's logins (and closes) belong to this step; the rest of
|
||||
// the warmup is discarded.
|
||||
const auto warm = load.take(n);
|
||||
const auto started = Clock::now();
|
||||
const auto cpu0 = cpu_seconds();
|
||||
std::this_thread::sleep_for(std::chrono::seconds{o.hold_s});
|
||||
auto samples = load.take(n);
|
||||
for (std::size_t i = 0; i < n; ++i) {
|
||||
append(samples[i].login_us, warm[i].login_us);
|
||||
samples[i].logins += warm[i].logins;
|
||||
samples[i].drops += warm[i].drops;
|
||||
samples[i].login_failures += warm[i].login_failures;
|
||||
samples[i].refused += warm[i].refused;
|
||||
}
|
||||
const auto seconds = std::chrono::duration<double>(Clock::now() - started).count();
|
||||
const auto established = load.established(n);
|
||||
const bool within =
|
||||
report(o, n, established, samples, seconds, cpu_seconds() - cpu0, scrape(o));
|
||||
if (within && !saturated)
|
||||
sustainable = n;
|
||||
if (!within && !saturated)
|
||||
saturated = n;
|
||||
std::uint64_t snapshots = 0, outcomes = 0;
|
||||
for (const auto& s : samples) {
|
||||
snapshots += s.snapshots;
|
||||
outcomes += s.outcome_us.size();
|
||||
}
|
||||
smoke_ok = smoke_ok && established == n && snapshots > 0 && outcomes > 0;
|
||||
}
|
||||
}
|
||||
std::printf("load result sustainable=%zu saturated_at=%zu\n", sustainable.value_or(0),
|
||||
saturated.value_or(0));
|
||||
std::fflush(stdout);
|
||||
return o.expect_established && !smoke_ok ? 1 : 0;
|
||||
}
|
||||
Reference in New Issue
Block a user