diff --git a/CMakeLists.txt b/CMakeLists.txt index d762323..5aa3eac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -96,7 +96,7 @@ target_include_directories(evo_region_lease PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/ target_compile_features(evo_region_lease PUBLIC cxx_std_23) evo_owned_target(evo_region_lease) # Portable host status and bounded operational exposition (M7.8). -add_library(evo_ops STATIC src/server/ops.cpp) +add_library(evo_ops STATIC src/server/ops.cpp src/server/load_governor.cpp) add_library(evo::ops ALIAS evo_ops) target_include_directories(evo_ops PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") target_compile_features(evo_ops PUBLIC cxx_std_23) diff --git a/apps/evo-server/m10_bridge.hpp b/apps/evo-server/m10_bridge.hpp index 0c9cc11..807fcb1 100644 --- a/apps/evo-server/m10_bridge.hpp +++ b/apps/evo-server/m10_bridge.hpp @@ -12,8 +12,12 @@ // M10.6 (ADR-0094): sessions' probe markers move within the habitat sites' bounding cells, // or within --m10-area. M10.8c (ADR-0096): with --m10-handoff-key, markers are durable probe // records and a marker crossing into a --m10-neighbour's area is handed over with a ticket. +// M10.9a (ADR-0097): the host's step and this bridge's work per simulated tick feed a load +// governor; above its thresholds grant logins are refused and replication is shed. #include "evo/persistence/postgres.hpp" #include "evo/persistence/region_lease.hpp" +#include "evo/server/ops.hpp" +#include #include #include #include @@ -23,6 +27,7 @@ #include "evo/client/environment_view.hpp" #include "evo/network/handoff_ticket.hpp" #include "evo/processes/lineage_region.hpp" +#include "evo/server/load_governor.hpp" #include "evo/server/pg_slot.hpp" #include "evo/server/postgres_action_ledger.hpp" #include "evo/server/postgres_control_ledger.hpp" @@ -183,9 +188,11 @@ template class M10Bridge { std::printf("m10 window closed activation=%s\n", window_->activation_id().c_str()); return false; } - void after_step() { + // `host_work`: this iteration's host step, measured by the caller. + void after_step(std::chrono::nanoseconds host_work) { if (!sessions_ || !accepting(wall_seconds())) return; + const auto started = std::chrono::steady_clock::now(); auto& host = *host_; const auto& s = host.status(); const server::HostFacts facts{s.ready && s.phase == server::HostPhase::running, s.epoch, @@ -233,6 +240,16 @@ template class M10Bridge { : "refused", static_cast(e.session), e.account.c_str(), static_cast(e.reason)); + measure(host_work, std::chrono::steady_clock::now() - started, s); + } + // The listener's load for /metrics. + void metrics(server::OpsCounters& c) const { + c.m10_enabled = sessions_ != nullptr; + if (!sessions_) + return; + c.m10_sessions = sessions_->sessions(); + c.m10_refused_logins = sessions_->refused_for_load(); + c.m10_load = governor_.state(); } void shutdown() { if (sessions_) @@ -240,6 +257,40 @@ template class M10Bridge { } private: + // Busy time accumulates until the simulated tick advances by k; it then counts as k equal + // per-tick samples (ADR-0097). A recovery's jump starts a new count instead. + void measure(std::chrono::nanoseconds host_work, std::chrono::nanoseconds session_work, + const server::HostStatus& s) { + busy_ += host_work + session_work; + session_busy_ += session_work; + if (s.simulated_tick <= measured_tick_) + return; + const auto ticks = s.simulated_tick - measured_tick_; + const bool counted = measured_tick_ != 0 && ticks <= server::load_window_ticks; + measured_tick_ = s.simulated_tick; + const auto per = [&](std::chrono::nanoseconds total) { + return static_cast( + std::chrono::duration_cast(total).count() / + static_cast(ticks)); + }; + const auto per_tick = per(busy_), session_per_tick = per(session_busy_); + busy_ = session_busy_ = {}; + if (!counted) + return; + const auto now_us = + static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); + bool changed = false; + for (std::uint64_t i = 0; i < ticks; ++i) + changed = governor_.observe(per_tick, session_per_tick, s.tick_debt, now_us) || changed; + if (!changed) + return; + const auto& load = governor_.state(); + sessions_->set_load(load.refusing, load.shed); + std::printf("m10 load refusing=%d shed=%u p95_us=%llu\n", load.refusing ? 1 : 0, + static_cast(load.shed), static_cast(load.p95_us)); + } static const processes::EnvironmentalCommunity& community(const processes::EnvironmentalCommunity& c) { return c; @@ -383,6 +434,9 @@ template class M10Bridge { std::unique_ptr slot_; std::unique_ptr control_; std::unique_ptr actions_; + server::LoadGovernor governor_{*server::LoadGovernor::create({})}; + std::chrono::nanoseconds busy_{}, session_busy_{}; + std::uint64_t measured_tick_{}; std::unique_ptr probes_; // Handoff mode only. std::unique_ptr handoff_; // Handoff mode only. std::optional> area_cells_; @@ -407,7 +461,10 @@ template class M10Bridge { [[nodiscard]] bool accepting(std::uint64_t) const noexcept { return false; } - void after_step() {} + void after_step(std::chrono::nanoseconds) {} + void metrics(server::OpsCounters& c) const { + c.m10_enabled = false; + } void shutdown() {} }; #endif diff --git a/apps/evo-server/main.cpp b/apps/evo-server/main.cpp index 4b73f09..eae99b5 100644 --- a/apps/evo-server/main.cpp +++ b/apps/evo-server/main.cpp @@ -418,6 +418,7 @@ template int serve(Options& o) { std::chrono::steady_clock::now() - last_checkpoint) .count()); counters.objects_available_bytes = host.objects().available_bytes().value_or(0); + m10->metrics(counters); return {200, server::render_metrics(s, counters), "text/plain; version=0.0.4; charset=utf-8"}; } @@ -516,7 +517,9 @@ template int serve(Options& o) { ++*synthetic_next; } } + const auto step_started = std::chrono::steady_clock::now(); const auto& s = host.step(scheduling_now()); + const auto host_work = std::chrono::steady_clock::now() - step_started; // ADR-0097. max_queued = std::max(max_queued, s.queued); // Checkpoint age restarts at recovery and at every confirmed commit. if ((!recovered && s.phase == server::HostPhase::running) || @@ -563,7 +566,7 @@ template int serve(Options& o) { if (ops) ops->serve(handler); sessions->step(host.state(), host.status()); - m10->after_step(); + m10->after_step(host_work); std::fflush(stdout); if (o.crash_point == "after-ack" && !acknowledged.empty() && ++ack_steps == o.crash_ordinal) crash_now(); diff --git a/deploy/kubernetes/region/dev-session-route/listener.yaml b/deploy/kubernetes/region/dev-session-route/listener.yaml index 99fcde7..c5c8d37 100644 --- a/deploy/kubernetes/region/dev-session-route/listener.yaml +++ b/deploy/kubernetes/region/dev-session-route/listener.yaml @@ -12,7 +12,7 @@ spec: env: - {name: REGION_SESSION, value: "dev"} - {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"} - - {name: REGION_SESSION_WINDOW, value: "dev-20261007T031149-a452:1791343909"} + - {name: REGION_SESSION_WINDOW, value: "dev-20261007T102834-1d97:1791370114"} ports: # QUIC (ADR-0090): UDP only; nothing listens on TCP 7400. - {name: session, containerPort: 7400, protocol: UDP} diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 5f3587a..50af1b4 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -541,7 +541,11 @@ durable probes in sessions, edge crossing, ghosts (schema v13) and a two-region done; 10.8c-2 client switch (offers, redirects, multi-region grants) and evo-server/probe configuration done; 10.8d two evo-server processes on PostgreSQL with SIGKILL around each durable step, duplicate/stale retries and a failed destination done (schema v14 recovery -sweep, replay guard). M10.8 complete. 10.9 planned. +sweep, replay guard). M10.8 complete. 10.9 split (ADR-0097; only evolution-dev reference-worker +runs qualify): 10.9a per-tick work measurement, load governor (refuse logins, then shed +replication with hysteresis) and its metrics done; 10.9b evo-load multi-client tool, 10.9c +abuse suite (fuzz, floods, slow peers, storms, lease loss) and 10.9d dev-cluster harness and +reference results follow. **Entry:** M9 client/session evidence and M7/M8 dev/stage durability. Read NETWORK_ARCHITECTURE, PERSISTENCE_ARCHITECTURE and ADR-0017. The M9 transport is diff --git a/docs/STATUS.md b/docs/STATUS.md index 6dc4cea..1826547 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -98,7 +98,16 @@ recovery gaps: - a ticket replayed at a restarted destination after the probe moved on is refused. M10.8 is complete. Deploying needs protocol v4 and schemas v12-v14 (a rebuilt migrations -image). Next: M10.9, load and abuse qualification. +image). M10.9a (ADR-0097) starts load qualification. Only runs on the evolution-dev reference +worker count as evidence. evo-server now measures each tick's work (host step plus session +layer) and feeds a load governor: +- above a p95 of 15 ms, or with tick debt, new grant logins are refused (`capacity`), while + crossings are still admitted; +- above 20 ms, replication sheds radius and far-update cadence in up to 3 levels; +- both relax after 5 s of calm. + +`/metrics` exposes the work percentiles (total and session layer) and the governor's +state. Next: 10.9b, the `evo-load` multi-client tool. ## Previous gate - 2026-10-02 diff --git a/docs/adr/0097-m10-load-and-abuse-qualification.md b/docs/adr/0097-m10-load-and-abuse-qualification.md new file mode 100644 index 0000000..70799ff --- /dev/null +++ b/docs/adr/0097-m10-load-and-abuse-qualification.md @@ -0,0 +1,107 @@ +# ADR-0097: M10 load and abuse qualification and overload response + +## Status +Accepted for M10.9 (2026-10-07). Decisions confirmed with the maintainer on 2026-10-07: +- **Venue.** Only measurements on the reference hardware count as qualification: one RKE2 + worker per region host on evolution-dev (ADR-0089). Local runs are smoke tests and + development references, never evidence. The gate stays open until the dev run is done. +- **Load tool.** One multi-client tool, `evo-load`: N session clients in one process, each a + bot, reporting per-client distributions so aggregate throughput cannot hide a starved + client. +- **Overload response: refuse, then shed.** Measured per-tick work drives two thresholds. + Above the first, new logins are refused. Above the second, the interest and far-update + cadence of existing sessions shrink step by step. Both are restored with hysteresis, so + ticks do not fall behind for admitted sessions. +- **Abuse coverage:** packet fuzz, floods, slow peers and connect/disconnect storms, and + region lease loss with 20 or more sessions attached. + +Delivery: +- **10.9a:** the work measurement, the load governor and its effects (this text). +- **10.9b:** `evo-load` and a local smoke test. +- **10.9c:** the abuse suite. +- **10.9d:** the evolution-dev run harness, then the reference results. + +Each segment amends this ADR. Every threshold here is an engineering budget under +`schemas/network/m10-budgets-v1.md`, not a game rule. + +## Context +ADR-0089 budgets: +- per-tick region work: p99 ≤ 25 ms on the reference worker; +- session handling: ≤ 0.5 ms per session per tick; +- at least 20 sessions per region (32 by default); +- overload response: shed interest and refuse new sessions before tick debt. + +Until now nothing measured the work, and the only bound was the session count. A region +that falls behind accrues tick debt, which delays every admitted session's outcomes and +snapshots. + +## Decision (10.9a) +- **Work measurement** (evo-server loop). Busy time is the host's step plus the session + bridge's work (session poll, replication, action carry and acknowledgements). Waits for + input and the loop's sleeps are excluded. When the simulated tick advances by k, the busy + time since the previous advance counts as k equal per-tick samples. The host's tick debt + is read with every sample. +- **Load governor** (`include/evo/server/load_governor.hpp`, portable, no clock). It keeps + a window of the last 40 samples (2 s). It reports the window's p95, p99 and maximum, plus + two outputs: + - **refusing** turns on when p95 ≥ 15 ms or tick debt > 0; + - **shed level** (0-3) rises by one when p95 ≥ 20 ms or tick debt ≥ 2, at most once per + second. + + Calm means p95 < 10 ms and no debt. After 5 s of calm the level drops by one. At level 0, + 5 s more of calm turns refusing off. Any non-calm sample restarts the calm timer. + Thresholds are configurable and validated (`relax < refuse ≤ shed ≤ tick budget`). +- **Effects** (`RegionSessions::set_load`): + - While refusing, a Hello carrying a grant is closed with `capacity` before signature + work. Handoff tickets are still admitted: a crossing moves an existing player between + regions, and refusing it would strand the probe until the source aborts the transfer + (up to 40 s). + - The shed level limits replication (`ReplicationServer::set_shed`): + + | Level | Radius cap | Near radius | Far updates | + |---|---|---|---| + | 0 | 7 | 3 | every 4 ticks | + | 1 | 7 | 3 | every 8 ticks | + | 2 | 5 | 3 | every 8 ticks | + | 3 | 3 | 2 | every 16 ticks | + + The stored interest is kept, so shedding forces no fresh baseline. Entities beyond the + cap are removed from the client's view and return when the level drops. Each session's + own marker is still sent every tick it changes. +- **Exposition.** `/metrics` adds `evo_m10_*` gauges: + - sessions; + - refusing; + - shed level; + - work p95, p99 and maximum in microseconds; + - the session layer's share, p95 and p99 (the ADR-0089 per-session budget is checked + against it); + - refused logins (counter). + + evo-server prints `m10 load refusing=R shed=L p95_us=P` on every change. No labels are + added. + +## Consequences +- Under overload, honest new players are refused (`capacity`, retry later) and existing + sessions see a smaller, slower-updating view rather than delayed ticks. Crossings + continue. +- The thresholds are provisional until 10.9d measures the reference worker. A change is + made by amending this ADR and the budgets table. +- No change to the session protocol: the client already handles `capacity` and removals. + +## 10.9a evidence (2026-10-07) +- Unit tests: + - the governor's thresholds, one step per second, hysteresis, the calm timer restarting + on a non-calm window, tick debt alone, and policy validation; + - the metrics families, present only with the listener and bounded at worst-case values; + - replication shedding: radius caps 5 and 3 without a fresh baseline or a change of the + stored interest, a far update every 16 ticks at level 3, and the full square back at + level 0; + - sessions under load: grant logins refused with `capacity` before signature work (the + grant stays usable), tickets admitted. +- Process test (`region_m10_session_tests.cpp`): a real evo-server with `--ops-listen` + exposes the `evo_m10_*` gauges. + + Local development reference (WSL, Release, one session; not evidence): work p95 about + 7 ms and p99 about 9 ms per tick. Of that, the session layer's p99 was about 4.5 ms. That + is far above 0.5 ms per session, so 10.9b examines it with evo-load. Likely contributors + are the per-tick snapshot source query and synchronous control renewals. diff --git a/docs/adr/README.md b/docs/adr/README.md index 529159e..e2d877f 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -98,6 +98,10 @@ design. No unadopted candidate is a link dependency of the proof. - [ADR-0084: Engineering UI toolkit](0084-engineering-ui-toolkit.md) - Dear ImGui 1.92.9b pinned (M9.8a) and integrated as the engineering client overlay with toolkit-first input routing (M9.8c). - [ADR-0090: M10 QUIC transport](0090-m10-quic-transport.md) - msquic 2.6.2 with OpenSSL 3.5.9 LTS (TLS 1.3 only) from pinned sources on Windows and Linux; DATAGRAM frames for superseding state, streams for reliable actions; M9 TLS/TCP kept as dev fallback; owned impairment relay; exact CA pinning due in 10.2b. +- [ADR-0097: M10 load and abuse qualification](0097-m10-load-and-abuse-qualification.md) - only evolution-dev reference-worker runs qualify; one multi-client evo-load tool with per-client distributions; a load governor over per-tick work (p95 window of 40 ticks) and tick debt refuses grant logins, then sheds replication radius and cadence in 3 levels with hysteresis (tickets still admitted); abuse suite of fuzz, floods, slow peers, storms and lease loss. +- [ADR-0096: M10 probe handoff](0096-m10-probe-handoff.md) - EXPERIMENT durable probe records with one owning region and a conserved test inventory; fenced prepared/staged/committed/aborted transfers; signed single-use transfer tickets; protocol v4 offers, redirects and ghosts; recovery sweep (schema v14) and replay guard proven by a two-process fault matrix. +- [ADR-0095: M10 interpolation and session loss](0095-m10-interpolation-and-session-loss.md) - drift-corrected client tick clock, adaptive 2-6 tick interpolation of remote markers with at most 2 ticks of extrapolation (organisms never), frozen stale view on loss and a fresh baseline on reconnect. +- [ADR-0094: M10 probe motion experiment](0094-m10-probe-motion-experiment.md) - EXPERIMENT fixed-speed probe marker (probe-motion-v1) moved by admitted intentions, protocol v3 marker records, own-marker prediction with reconciliation; not an organism movement rule. - [ADR-0093: M10 snapshots and interest](0093-m10-snapshots-and-interest.md) - session protocol v2: client-declared, server-capped interest (radius <= 7) with a perception boundary; per-tick snapshot chunks as datagrams, entity deltas against the acknowledged baseline (<= 4, <= 40 ticks), nearest first within 16 KiB, distant tier every 4 ticks; client assembly with 2 baselines. - [ADR-0092: M10 intentions and durable outcomes](0092-m10-intentions-and-durable-outcomes.md) - realtime intentions (datagram receipts and a cumulative simulated watermark) and account-scoped actions with stable ids, receipts and final outcomes (committed_no_effect / not_committed); no biological effect until organisms exist; durable action records in the commit transaction (10.4b). - [ADR-0091: M10 session identity and control](0091-m10-session-identity-and-control.md) - provider-independent 128-bit AccountIds, ECDSA P-256 session grants (regions hold public keys only, signed environment byte), per-account serials where the newest grant wins, takeover of an existing session, in-session refresh, bounded attempt rates; identity adapter and issuer as a library; durable control lease follows (10.3c). diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 5073864..57ea2e7 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -4904,3 +4904,47 @@ Not done: - deployment of protocol v4 and schemas v12-v14 (needs a rebuilt migrations image; not done from routine development); - a GUI run across two regions on evolution-dev (only one dev region exists). + +## 2026-10-07 — M10.9a load measurement and governor (ADR-0097) + +Implemented: +- evo-server measures each simulated tick's work: the host step plus the M10 bridge + (session poll, replication, action carry). Waits are excluded, and the session layer's + share is kept separately. +- `LoadGovernor` (`include/evo/server/load_governor.hpp`, portable): a window of 40 ticks, + refuse at p95 ≥ 15 ms or any tick debt, shed one level per second at p95 ≥ 20 ms or + debt ≥ 2, and relax one step per 5 s of calm (p95 < 10 ms, no debt). +- `RegionSessions::set_load`: while refusing, grant logins are closed with `capacity` + before signature work; handoff tickets are still admitted. +- `ReplicationServer::set_shed`: levels 0-3 cap the radius (7/7/5/3) and the near radius + (3/3/3/2) and stretch far updates (x1/x2/x2/x4). Stored interests are kept, and own + markers are unaffected. +- `/metrics`: `evo_m10_*` gauges (sessions, refusing, shed level, work and session-work + percentiles, maximum) and a refused-logins counter, rendered only with the listener. + evo-server prints `m10 load ...` on every change. +- The ADR index gains the missing entries for ADR-0094 to ADR-0096. + +Tests: +- `ops-contracts`: the governor state machine (thresholds, step interval, hysteresis, calm + restart, debt, policy validation) and the M10 metric families, including worst-case size. +- `network`: replication shedding (caps, no fresh baseline, 16-tick far cadence at level + 3, recovery at level 0). +- `quic-sessions`: grant logins refused under load (the grant is not consumed), a ticket + login admitted, and logins accepted after relaxing. +- Postgres `region_m10_session_tests`: a real evo-server's `/metrics` shows one session, + measured work, not refusing, shed 0. + +Executed: +- **Linux GCC 14:** `postgres-check.sh` with QUIC, all suites; presets `quic` (ops, + network, QUIC sessions) 3/3. The local reference (Release, one session) measured work + p95 7.1 ms and p99 8.8 ms, with the session layer at p99 4.5 ms. This is a development + reference only, flagged for 10.9b. +- **Linux Clang 19:** clang-tidy 19 on the governor, ops, replication, sessions, evo-server + `main.cpp` with the bridge, and the tests (one finding fixed); clang-format 19. +- **Windows MSVC 19.44:** presets `debug` 60/60 (ops-contracts included), `quic` 4/4 and + `client-network` 14/14. + +Not done: +- qualification on the evolution-dev reference worker (10.9d; needs v4 and v12-v14 + deployed); +- load generation (10.9b) and the abuse suite (10.9c). diff --git a/include/evo/network/replication.hpp b/include/evo/network/replication.hpp index b321852..6989824 100644 --- a/include/evo/network/replication.hpp +++ b/include/evo/network/replication.hpp @@ -89,6 +89,14 @@ class ReplicationServer { tick(std::uint64_t tick, const std::function& source, const Perceives& perceives, const std::function& payload); [[nodiscard]] ReplicationStats stats() const; + // M10.9a overload shedding (ADR-0097), level 0-3 (higher is clamped to 3): caps the + // radius encoded (7, 7, 5, 3), the near radius (3, 3, 3, 2) and stretches far updates + // (x1, x2, x2, x4). Stored interests are kept (no fresh baseline); entities beyond the + // cap are removed from views and return when the level drops. Own markers are unaffected. + void set_shed(std::uint8_t level) noexcept; + [[nodiscard]] std::uint8_t shed() const noexcept { + return shed_; + } private: // (kind, x | id, y, z): environment by voxel, organisms by id. @@ -109,6 +117,7 @@ class ReplicationServer { }; explicit ReplicationServer(ReplicationLimits limits) : limits_{limits} {} ReplicationLimits limits_; + std::uint8_t shed_{}; std::vector sessions_; ReplicationStats stats_; }; diff --git a/include/evo/server/load_governor.hpp b/include/evo/server/load_governor.hpp new file mode 100644 index 0000000..30d40ef --- /dev/null +++ b/include/evo/server/load_governor.hpp @@ -0,0 +1,56 @@ +#pragma once +#include +#include +#include +#include + +// M10.9a overload response (ADR-0097): a region's per-tick work and tick debt decide whether +// new logins are refused and how far replication is shed. Portable: no clocks (callers pass a +// monotonic time in microseconds), no transport. Thresholds are engineering budgets +// (schemas/network/m10-budgets-v1.md), not game rules. +namespace evo::server { +struct LoadPolicy { + std::uint64_t tick_budget_us{25'000}; // p99 per-tick region work budget (ADR-0089). + std::uint64_t refuse_us{15'000}; // Window p95 at or above: refuse new logins. + std::uint64_t shed_us{20'000}; // Window p95 at or above: shed one more level. + std::uint64_t relax_us{10'000}; // Window p95 below (and no debt): calm. + std::uint64_t step_us{1'000'000}; // At most one shed step per interval. + std::uint64_t settle_us{5'000'000}; // Calm this long relaxes one step. + std::uint8_t max_shed{3}; +}; +inline constexpr std::size_t load_window_ticks = 40; // 2 s of 50 ms ticks. +struct LoadState { + bool refusing{}; + std::uint8_t shed{}; + std::uint64_t p95_us{}, p99_us{}, max_us{}; // Over the window (0 before any sample). + std::size_t samples{}; // In the window. + // The session layer's share of the same ticks (ADR-0089 budgets it per session). + std::uint64_t session_p95_us{}, session_p99_us{}; + friend bool operator==(const LoadState&, const LoadState&) = default; +}; +enum class LoadPolicyError : std::uint8_t { invalid }; + +class LoadGovernor { + public: + [[nodiscard]] static std::expected create(LoadPolicy policy); + // One simulated tick's region work (of which `session_us` was the session layer's) and the + // host's tick debt, at monotonic `now_us`. Returns true when `refusing` or `shed` changed. + bool observe(std::uint64_t work_us, std::uint64_t session_us, std::uint64_t tick_debt, + std::uint64_t now_us); + [[nodiscard]] const LoadState& state() const noexcept { + return state_; + } + [[nodiscard]] const LoadPolicy& policy() const noexcept { + return policy_; + } + + private: + explicit LoadGovernor(LoadPolicy policy) : policy_{policy} {} + LoadPolicy policy_; + LoadState state_; + std::array window_{}, session_window_{}; + std::size_t next_{}; + bool stepped_{}, calm_{}; + std::uint64_t last_step_us_{}, calm_since_us_{}; +}; +} // namespace evo::server diff --git a/include/evo/server/ops.hpp b/include/evo/server/ops.hpp index d2f7755..e25bf39 100644 --- a/include/evo/server/ops.hpp +++ b/include/evo/server/ops.hpp @@ -1,4 +1,5 @@ #pragma once +#include "evo/server/load_governor.hpp" #include #include #include @@ -47,6 +48,11 @@ struct OpsCounters { // M8.5: object volume free space and the store's reserve (ADR-0070), so capacity // is alerted on before new objects are refused. std::uint64_t objects_available_bytes{}, objects_reserve_bytes{}; + // M10.9a (ADR-0097): the M10 session listener's load governor, rendered only when the + // listener is enabled. + bool m10_enabled{}; + std::uint64_t m10_sessions{}, m10_refused_logins{}; + LoadState m10_load; }; [[nodiscard]] const char* phase_name(HostPhase phase) noexcept; diff --git a/include/evo/server/region_sessions.hpp b/include/evo/server/region_sessions.hpp index 8e48ef8..89766f3 100644 --- a/include/evo/server/region_sessions.hpp +++ b/include/evo/server/region_sessions.hpp @@ -136,6 +136,12 @@ class RegionSessions { const std::function& source, const network::Perceives& perceives = {}); [[nodiscard]] network::ReplicationStats replication_stats() const; + // M10.9a (ADR-0097): the host's load governor output. While `refuse_logins`, a Hello + // carrying a grant is closed with `capacity` before signature work (handoff tickets are + // still admitted); `shed` (0-3) limits replication (ReplicationServer::set_shed). + void set_load(bool refuse_logins, std::uint8_t shed); + // Grant logins refused for load since creation. + [[nodiscard]] std::uint64_t refused_for_load() const noexcept; // Where probe markers live and spawn (ADR-0094); sessions already established receive a // marker. False for an invalid area. With neighbours (ADR-0096), markers move within the // bounding box of this area and the neighbours' areas, and leaving this area hands off. diff --git a/schemas/network/m10-budgets-v1.md b/schemas/network/m10-budgets-v1.md index de84c17..73a82e9 100644 --- a/schemas/network/m10-budgets-v1.md +++ b/schemas/network/m10-budgets-v1.md @@ -32,10 +32,10 @@ in real time; changing it requires a versioned rules migration (ADR-0089). | Client event queue | 1,024 events | budget | client session | | Heartbeat / liveness / handshake | 1 s / 5 s / 5 s | budget | 10.2, 10.7 | | Sessions per region | >= 20 required, 32 default | budget | 10.9 | -| Per-tick region work | p99 <= 25 ms on the reference worker | budget | 10.9 | -| Session handling | <= 0.5 ms per session per tick | budget | 10.9 | +| Per-tick region work | p99 <= 25 ms on the reference worker | budget | 10.9 (measured: `evo_m10_tick_work_*`, ADR-0097) | +| Session handling | <= 0.5 ms per session per tick | budget | 10.9 (measured: `evo_m10_session_work_*`, ADR-0097) | | Host input queue | 64 inputs, 64 KiB | budget | region host | -| Overload response | shed interest, refuse new sessions before tick debt | budget | 10.9 | +| Overload response | shed interest, refuse new sessions before tick debt | budget | 10.9a load governor: refuse at p95 >= 15 ms or debt, shed at p95 >= 20 ms or debt >= 2, relax below 10 ms after 5 s (ADR-0097; provisional until 10.9d) | Determinism: authoritative state is exact given the checkpoint and committed per-tick inputs; timing, delivery, prediction, interpolation and presentation are tolerant diff --git a/src/network/protocol/replication.cpp b/src/network/protocol/replication.cpp index 412e3d3..852a3ef 100644 --- a/src/network/protocol/replication.cpp +++ b/src/network/protocol/replication.cpp @@ -1,5 +1,6 @@ #include "evo/network/replication.hpp" #include +#include #include namespace evo::network { @@ -224,10 +225,20 @@ ReplicationServer::tick(std::uint64_t tick, return this->tick(tick, source, perceives, [payload](std::uint64_t) { return payload; }); } +void ReplicationServer::set_shed(std::uint8_t level) noexcept { + shed_ = std::min(level, 3); +} + std::vector>> ReplicationServer::tick( std::uint64_t tick, const std::function& source, const Perceives& perceives, const std::function& payload_of) { std::vector>> out; + // Shedding (ADR-0097): radius cap, near radius and far-update multiplier per level. + constexpr std::array shed_radius{7, 7, 5, 3}; + constexpr std::array shed_near{3, 3, 3, 2}; + constexpr std::array shed_far{1, 2, 2, 4}; + const auto near_radius = std::min(limits_.near_radius, shed_near.at(shed_)); + const auto far_interval = limits_.far_interval * shed_far.at(shed_); const auto overhead = session_header_bytes + snapshot_chunk_header_bytes; if (tick == 0) return out; @@ -239,12 +250,15 @@ std::vector>> ReplicationSer if (payload <= overhead) continue; // No datagram channel yet (or a path too small): nothing to send. const auto capacity = payload - overhead; // Record bytes per chunk. - const auto& in = *s.interest; - auto cached = std::ranges::find_if(sources, [&](const auto& e) { return e.first == in; }); + const auto& wanted = *s.interest; + auto cached = + std::ranges::find_if(sources, [&](const auto& e) { return e.first == wanted; }); if (cached == sources.end()) { - sources.emplace_back(in, source(in)); + sources.emplace_back(wanted, source(wanted)); cached = std::prev(sources.end()); } + auto in = wanted; // Encoded within the shed cap; the stored interest is unchanged. + in.radius = std::min(in.radius, shed_radius.at(shed_)); const auto& src = cached->second; // Spatial filter, then perception: nothing outside either is ever encoded. std::map, std::uint64_t>> truth; @@ -299,8 +313,8 @@ std::vector>> ReplicationSer const auto known = base_state.find(key); if (known != base_state.end() && known->second.record == value.first) continue; // The client already holds this value. - const bool due = known == base_state.end() || value.second <= limits_.near_radius || - tick - known->second.updated >= limits_.far_interval; + const bool due = known == base_state.end() || value.second <= near_radius || + tick - known->second.updated >= far_interval; if (!due) { ++deferred; continue; diff --git a/src/server/load_governor.cpp b/src/server/load_governor.cpp new file mode 100644 index 0000000..377ad32 --- /dev/null +++ b/src/server/load_governor.cpp @@ -0,0 +1,69 @@ +#include "evo/server/load_governor.hpp" + +#include +#include + +namespace evo::server { +std::expected LoadGovernor::create(LoadPolicy p) { + if (p.relax_us == 0 || p.relax_us >= p.refuse_us || p.refuse_us > p.shed_us || + p.shed_us > p.tick_budget_us || p.step_us == 0 || p.settle_us == 0 || p.max_shed > 3) + return std::unexpected(LoadPolicyError::invalid); + return LoadGovernor{p}; +} + +namespace { +using Window = std::array; +// The first `n` samples sorted; nearest rank: the smallest sample with at least `percent` of +// the window at or below it. +Window sorted_of(const Window& w, std::size_t n) { + Window sorted{}; + std::copy_n(w.begin(), n, sorted.begin()); + std::sort(sorted.begin(), sorted.begin() + static_cast(n)); + return sorted; +} +std::uint64_t rank(const Window& sorted, std::size_t n, std::size_t percent) { + return sorted.at((((n * percent) + 99) / 100) - 1); +} +} // namespace + +bool LoadGovernor::observe(std::uint64_t work_us, std::uint64_t session_us, std::uint64_t tick_debt, + std::uint64_t now_us) { + window_.at(next_) = work_us; + session_window_.at(next_) = std::min(session_us, work_us); + next_ = (next_ + 1) % window_.size(); + state_.samples = std::min(state_.samples + 1, window_.size()); + const auto n = state_.samples; + const auto sorted = sorted_of(window_, n); + state_.p95_us = rank(sorted, n, 95); + state_.p99_us = rank(sorted, n, 99); + state_.max_us = sorted.at(n - 1); + const auto sessions = sorted_of(session_window_, n); + state_.session_p95_us = rank(sessions, n, 95); + state_.session_p99_us = rank(sessions, n, 99); + + const auto before = std::pair{state_.refusing, state_.shed}; + const bool hot = state_.p95_us >= policy_.refuse_us || tick_debt > 0; + const bool overloaded = state_.p95_us >= policy_.shed_us || tick_debt >= 2; + if (hot) + state_.refusing = true; + if (overloaded && state_.shed < policy_.max_shed && + (!stepped_ || now_us - last_step_us_ >= policy_.step_us)) { + ++state_.shed; + stepped_ = true; + last_step_us_ = now_us; + } + if (state_.p95_us >= policy_.relax_us || tick_debt > 0) { + calm_ = false; + } else if (!calm_) { + calm_ = true; + calm_since_us_ = now_us; + } else if (now_us - calm_since_us_ >= policy_.settle_us) { + if (state_.shed > 0) + --state_.shed; // One level per settle interval, then logins. + else + state_.refusing = false; + calm_since_us_ = now_us; + } + return before != std::pair{state_.refusing, state_.shed}; +} +} // namespace evo::server diff --git a/src/server/ops.cpp b/src/server/ops.cpp index 2a6e063..45f8adb 100644 --- a/src/server/ops.cpp +++ b/src/server/ops.cpp @@ -103,6 +103,24 @@ std::string render_metrics(const HostStatus& s, const OpsCounters& c) { .append("\"} ") .append(std::to_string(c.dropped[i])) .append("\n"); + if (c.m10_enabled) { + metric(out, "evo_m10_sessions", "gauge", "Open M10 sessions", c.m10_sessions); + metric(out, "evo_m10_refusing", "gauge", "1 while new logins are refused for load", + c.m10_load.refusing); + metric(out, "evo_m10_shed_level", "gauge", "Replication shed level (0-3)", c.m10_load.shed); + metric(out, "evo_m10_tick_work_p95_microseconds", "gauge", + "p95 per-tick region work over the last 40 ticks", c.m10_load.p95_us); + metric(out, "evo_m10_tick_work_p99_microseconds", "gauge", + "p99 per-tick region work over the last 40 ticks", c.m10_load.p99_us); + metric(out, "evo_m10_tick_work_max_microseconds", "gauge", + "Largest per-tick region work over the last 40 ticks", c.m10_load.max_us); + metric(out, "evo_m10_session_work_p95_microseconds", "gauge", + "p95 per-tick session-layer work over the last 40 ticks", c.m10_load.session_p95_us); + metric(out, "evo_m10_session_work_p99_microseconds", "gauge", + "p99 per-tick session-layer work over the last 40 ticks", c.m10_load.session_p99_us); + metric(out, "evo_m10_refused_logins_total", "counter", "Logins refused for load", + c.m10_refused_logins); + } return out; } } // namespace evo::server diff --git a/src/server/session/region_sessions.cpp b/src/server/session/region_sessions.cpp index 7cc805e..82185f4 100644 --- a/src/server/session/region_sessions.cpp +++ b/src/server/session/region_sessions.cpp @@ -108,6 +108,8 @@ struct RegionSessions::Impl { std::vector outgoing; // Handed off, not yet known committed. Clock::time_point next_ghosts{}, next_overdue{}; std::optional default_interest; + bool refuse_logins{}; // Load governor (ADR-0097). + std::uint64_t refused_load{}; // Grant logins refused for load. HostFacts facts; std::vector connections; std::vector events; @@ -216,6 +218,10 @@ struct RegionSessions::Impl { return close(c, SessionCloseReason::rate_limited, now); if (network::looks_like_handoff_ticket(m.grant)) return ticket_hello(c, m, now_seconds, now); + if (refuse_logins) { // Overload (ADR-0097): before signature work. + ++refused_load; + return close(c, SessionCloseReason::capacity, now); + } const auto grant = verifier.verify(m.grant); if (!grant) return close(c, reason_for(grant.error()), now); @@ -774,6 +780,15 @@ std::optional RegionSessions::holder(const network::AccountId& ac return impl_->control.holder(account); } +void RegionSessions::set_load(bool refuse_logins, std::uint8_t shed) { + impl_->refuse_logins = refuse_logins; + impl_->replication.set_shed(shed); +} + +std::uint64_t RegionSessions::refused_for_load() const noexcept { + return impl_->refused_load; +} + std::size_t RegionSessions::sessions() const noexcept { return impl_->control.sessions(); } diff --git a/tests/network_replication.cpp b/tests/network_replication.cpp index e61a21f..3efa9aa 100644 --- a/tests/network_replication.cpp +++ b/tests/network_replication.cpp @@ -5,6 +5,7 @@ #include "network_test.hpp" #include #include +#include #include #include #include @@ -385,9 +386,71 @@ void dense_report() { } } // namespace +// M10.9a overload shedding (ADR-0097): the encoded radius shrinks and far updates slow without a +// fresh baseline or a change of the stored interest; the view grows back when the level drops. +void shedding() { + auto server = *ReplicationServer::create(limits()); + World world; + CHECK(server.open(1, 5) && server.set_interest(1, {0, 0, slice, 7})); + SnapshotAssembler client{identity, 5}; + const auto source = [&](const SessionInterest&) { return world.source(); }; + std::uint64_t tick = 0; + std::uint64_t baseline = 0; + const auto step = [&] { + const auto out = server.tick(++tick, source, {}); + const auto& chunks = for_session(out, 1); + baseline = chunks.front().baseline; + for (const auto& a : deliver(client, chunks)) + server.acknowledge(1, a); + return client.latest()->environment.size(); + }; + while (step() < 225 && tick < 10) { + } + CHECK(client.latest()->environment.size() == 225); + const auto square = [&](std::int64_t r) { + return std::ranges::all_of(client.latest()->environment, [&](const auto& c) { + return std::max(std::abs(c.voxel.x), std::abs(c.voxel.y)) <= r; + }); + }; + server.set_shed(2); + CHECK(server.shed() == 2 && step() == 121 && square(5) && baseline != 0); + CHECK(server.interest(1).value_or(SessionInterest{}).radius == 7); // The stored interest. + server.set_shed(9); + CHECK(server.shed() == 3 && step() == 49 && square(3) && baseline != 0); + // Level 3: near radius 2, far updates every 16 ticks. A cell at distance 3 changed twice + // arrives at once (last sent long ago), then not again for 16 ticks. + const auto at = [&](std::int64_t x, std::int64_t y) { + return std::ranges::find_if(client.latest()->environment, + [&](const auto& c) { return c.voxel.x == x && c.voxel.y == y; }) + ->quantity[0]; + }; + for (int i = 0; i < 16; ++i) + step(); + world.quantity[{3, 0}] = 41; + world.quantity[{2, 0}] = 42; + step(); + CHECK(at(3, 0) == 41 && at(2, 0) == 42); + world.quantity[{3, 0}] = 51; + world.quantity[{2, 0}] = 52; + step(); + CHECK(at(2, 0) == 52 && at(3, 0) == 41); // Near: every tick. Far: deferred. + int waited = 1; + while (at(3, 0) != 51 && waited < 40) { + step(); + ++waited; + } + CHECK(waited == 16); + // Back to level 0: the full square returns, still from acknowledged baselines. + server.set_shed(0); + for (int i = 0; i < 6 && client.latest()->environment.size() < 225; ++i) + step(); + CHECK(client.latest()->environment.size() == 225 && baseline != 0); +} + void replication_tests() { codec_v2(); convergence_and_deltas(); + shedding(); concealment(); client_robustness(); dense_report(); diff --git a/tests/ops_tests.cpp b/tests/ops_tests.cpp index accb4e0..3869f7d 100644 --- a/tests/ops_tests.cpp +++ b/tests/ops_tests.cpp @@ -52,6 +52,17 @@ std::vector parse(const std::string& text) { return out; } +// Every host counter at its largest value (the M10 section stays disabled). +OpsCounters worst_counters() { + constexpr auto most = std::numeric_limits::max(); + OpsCounters c; + c.queue_capacity_inputs = c.queue_capacity_bytes = most; + c.checkpoint_age_ms = c.acks = c.outcomes = most; + c.dropped = {most, most, most, most}; + c.objects_available_bytes = c.objects_reserve_bytes = most; + return c; +} + void exposition_is_bounded_and_well_formed() { HostStatus s; s.phase = HostPhase::running; @@ -93,7 +104,7 @@ void exposition_is_bounded_and_well_formed() { big.max_tick_debt = big.commits = big.store_failures = big.unknown_outcomes = most; big.fences = big.lease_remaining_ms = most; big.simulated_tick = 1; // Lag saturates at zero rather than wrapping. - OpsCounters bigc{most, most, most, most, most, {most, most, most, most}, most, most}; + const auto bigc = worst_counters(); const auto large = render_metrics(big, bigc); CHECK(large.size() <= max_metrics_bytes); CHECK(large.find("evo_region_commit_lag_ticks 0\n") != std::string::npos); @@ -105,11 +116,95 @@ void exposition_is_bounded_and_well_formed() { CHECK(std::string(unacknowledged_name(static_cast(i))) != "unknown"); std::printf("ops_metrics_bytes typical=%zu worst=%zu\n", text.size(), large.size()); } + +// M10.9a (ADR-0097): the listener's governor families appear only when it is enabled. +void m10_load_metrics() { + HostStatus s; + OpsCounters c; + const auto without = parse(render_metrics(s, c)).size(); + c.m10_enabled = true; + c.m10_sessions = 21; + c.m10_refused_logins = 3; + c.m10_load = {true, 2, 17'000, 19'500, 23'000, 40, 900, 1'100}; + const auto text = render_metrics(s, c); + CHECK(parse(text).size() == without + 9); + CHECK(text.find("evo_m10_session_work_p99_microseconds 1100\n") != std::string::npos); + CHECK(text.find("evo_m10_sessions 21\n") != std::string::npos); + CHECK(text.find("evo_m10_refusing 1\n") != std::string::npos); + CHECK(text.find("evo_m10_shed_level 2\n") != std::string::npos); + CHECK(text.find("evo_m10_tick_work_p99_microseconds 19500\n") != std::string::npos); + CHECK(text.find("evo_m10_refused_logins_total 3\n") != std::string::npos); + constexpr auto most = std::numeric_limits::max(); + HostStatus big; + big.epoch = big.committed_tick = big.committed_sequence = big.tick_debt = most; + big.max_tick_debt = big.commits = big.store_failures = big.unknown_outcomes = most; + big.fences = big.lease_remaining_ms = most; + auto bigc = worst_counters(); + bigc.m10_enabled = true; + bigc.m10_sessions = bigc.m10_refused_logins = most; + bigc.m10_load = {true, 3, most, most, most, 40, most, most}; + CHECK(render_metrics(big, bigc).size() <= max_metrics_bytes); +} + +// M10.9a (ADR-0097): refuse, then shed one level per second; relax one level per 5 s of +// calm, then accept again. Samples are 50 ms ticks. +void load_governor() { + CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 15'000, 1'000'000, 5'000'000, 3})); + CHECK(!LoadGovernor::create({25'000, 21'000, 20'000, 10'000, 1'000'000, 5'000'000, 3})); + CHECK(!LoadGovernor::create({19'000, 15'000, 20'000, 10'000, 1'000'000, 5'000'000, 3})); + CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 10'000, 0, 5'000'000, 3})); + CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 10'000, 1'000'000, 5'000'000, 4})); + auto made = LoadGovernor::create({}); + CHECK(made.has_value()); + auto& g = *made; + std::uint64_t now = 0; + const auto run = [&](std::size_t ticks, std::uint64_t work, std::uint64_t debt = 0) { + for (std::size_t i = 0; i < ticks; ++i) { + now += 50'000; + g.observe(work, work / 4, debt, now); + } + return g.state(); + }; + // Quiet: nothing changes; the window reports its percentiles. + CHECK(run(60, 5'000) == LoadState{false, 0, 5'000, 5'000, 5'000, 40, 1'250, 1'250}); + // A single slow tick is not the p95 of 40. + CHECK(run(1, 30'000).max_us == 30'000 && !g.state().refusing && g.state().p95_us == 5'000); + // Sustained 16 ms: refuse, no shedding. + auto st = run(40, 16'000); + CHECK(st.refusing && st.shed == 0 && st.p95_us == 16'000); + // 22 ms: one shed level per second, never past 3. + // The p95 (nearest rank, 38th of 40) reaches the threshold with the third slow tick. + CHECK(run(2, 22'000).shed == 0 && run(1, 22'000).shed == 1); + CHECK(run(18, 22'000).shed == 1 && run(2, 22'000).shed == 2); + CHECK(run(100, 22'000).shed == 3); + // Calm only once the window's p95 is below 10 ms; then one level per 5 s. + st = run(37, 4'000); + CHECK(st.shed == 3 && st.p95_us == 22'000); + CHECK(run(1, 4'000).p95_us == 4'000); // Calm starts now. + CHECK(run(99, 4'000).shed == 3 && run(1, 4'000).shed == 2); + // A non-calm window (p95 >= 10 ms, still under the refuse threshold) restarts the timer. + CHECK(run(50, 4'000).shed == 2 && run(3, 12'000).p95_us == 12'000); + CHECK(run(37, 4'000).p95_us == 12'000 && run(1, 4'000).p95_us == 4'000); + CHECK(run(99, 4'000).shed == 2 && run(1, 4'000).shed == 1); + // Down to 0 after another 5 s, then logins after 5 s more. + CHECK(run(100, 4'000).shed == 0 && g.state().refusing); + CHECK(run(99, 4'000).refusing && !run(1, 4'000).refusing); + // Tick debt alone: refuse at 1, shed at 2, even with light work. + CHECK(run(1, 1'000, 1).refusing && g.state().shed == 0); + CHECK(run(1, 1'000, 2).shed == 1); + CHECK(!g.observe(1'000, 250, 2, now + 10)); // Within the step interval: no change. + CHECK(g.state().session_p95_us == 1'000); // The window is mostly 4 ms ticks. + // Debt gone and calm (from the next sample): recovers. + CHECK(run(100, 1'000).shed == 1 && run(1, 1'000).shed == 0); + CHECK(run(99, 1'000).refusing && !run(1, 1'000).refusing); +} } // namespace int main() { try { exposition_is_bounded_and_well_formed(); + m10_load_metrics(); + load_governor(); std::puts("ops contracts passed"); return 0; } catch (const std::exception& e) { diff --git a/tests/quic_sessions.cpp b/tests/quic_sessions.cpp index 4b6906d..2bf8b2d 100644 --- a/tests/quic_sessions.cpp +++ b/tests/quic_sessions.cpp @@ -627,6 +627,19 @@ void ticket_login(const Files& f) { handoff.error = HandoffLedgerError::unavailable; CHECK(refused_with(h, ticket(account(3), 0x65), Reason::capacity)); CHECK(a->state() == RegionSessionState::established && handoff.admitted.size() == 1); + // M10.9a (ADR-0097): under load, grant logins are refused before signature work while + // tickets (crossings) are still admitted; the refused grant stays usable. + handoff.error.reset(); + h.server->set_load(true, 3); + CHECK(refused_with(h, h.issuer.grant(account(5), 1), Reason::capacity)); + CHECK(refused_with(h, h.issuer.grant(account(5), 1), Reason::capacity)); + CHECK(h.server->refused_for_load() == 2 && a->state() == RegionSessionState::established); + auto crossed = h.connect(ticket(account(6), 0x66)); + CHECK(h.settled(*crossed) && crossed->state() == RegionSessionState::established); + h.server->set_load(false, 0); + auto later = h.connect(h.issuer.grant(account(5), 1)); + CHECK(h.settled(*later) && later->state() == RegionSessionState::established && + h.server->refused_for_load() == 2); } // M10.8c (ADR-0096): two regions with adjacent areas in one process, over the fake world diff --git a/tests/region_m10_session_tests.cpp b/tests/region_m10_session_tests.cpp index d8c128d..3a38570 100644 --- a/tests/region_m10_session_tests.cpp +++ b/tests/region_m10_session_tests.cpp @@ -9,14 +9,17 @@ #include "evo/network/session_grant.hpp" #include "fixtures/server_process.hpp" #include +#include #include #include #include #include +#include #include #include #include #include +#include #include #include @@ -69,6 +72,34 @@ std::string hex(const AccountId& a) { return out; } +// GET /metrics from the ops listener; the value of an unlabelled series (ADR-0097 adds the +// evo_m10_* families when the listener is enabled). +std::optional metric(std::uint16_t port, const std::string& series) { + const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); + expect(fd >= 0, "socket"); + sockaddr_in a{}; + a.sin_family = AF_INET; + a.sin_port = htons(port); + a.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + timeval timeout{3, 0}; + ::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout); + std::string raw; + if (::connect(fd, reinterpret_cast(&a), sizeof a) == 0) { + const std::string request = "GET /metrics HTTP/1.1\r\nHost: test\r\n\r\n"; + if (::send(fd, request.data(), request.size(), MSG_NOSIGNAL) == + static_cast(request.size())) { + std::array chunk{}; + for (ssize_t n = 0; (n = ::recv(fd, chunk.data(), chunk.size(), 0)) > 0;) + raw.append(chunk.data(), static_cast(n)); + } + } + ::close(fd); + const auto at = raw.find("\n" + series + " "); + if (at == std::string::npos) + return std::nullopt; + return std::stoull(raw.substr(at + series.size() + 2)); +} + // One region's credentials, issuer key and server process arguments. struct Setup { ServerRegion region; @@ -95,22 +126,23 @@ struct Setup { } [[nodiscard]] std::vector args() const { return server_args( - region, {"--synthetic-inputs", "development-v1", "--m10-listen", "127.0.0.1:0", - "--m10-cert", (region.root / "server.pem").string(), "--m10-key", - (region.root / "server.key").string(), "--m10-issuer", - (region.root / "issuer.pub").string(), "--m10-environment", "dev", - "--m10-max-grant-lifetime", "900", "--session-window", - "test-" + - std::to_string(static_cast( - std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count())) + - ":" + - std::to_string(static_cast( - std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count()) + - 1200)}); + region, + {"--synthetic-inputs", "development-v1", "--ops-listen", "127.0.0.1:0", "--m10-listen", + "127.0.0.1:0", "--m10-cert", (region.root / "server.pem").string(), "--m10-key", + (region.root / "server.key").string(), "--m10-issuer", + (region.root / "issuer.pub").string(), "--m10-environment", "dev", + "--m10-max-grant-lifetime", "900", "--session-window", + "test-" + + std::to_string(static_cast( + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count())) + + ":" + + std::to_string(static_cast( + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count()) + + 1200)}); } [[nodiscard]] WireGrant grant(const AccountId& who, std::uint64_t serial) const { GrantClaims c; @@ -133,12 +165,16 @@ struct Setup { // A running evo-server and its QUIC port, its output drained while waiting. struct Server { std::unique_ptr child; - std::uint16_t port{}; + std::uint16_t port{}, ops{}; explicit Server(const Setup& s) : child{std::make_unique(s.args())} { CHECK(child->wait_line("m10 listening ", 30s)); - for (const auto& l : child->lines()) + for (const auto& l : child->lines()) { if (starts(l, "m10 listening ")) port = static_cast(std::stoul(l.substr(14))); + if (starts(l, "ops listening ")) + ops = static_cast(std::stoul(l.substr(14))); + } + CHECK(ops != 0); CHECK(port != 0 && child->wait_line("ready 1", 30s)); } void pump() { @@ -237,6 +273,20 @@ void end_to_end() { CHECK(std::ranges::all_of(shown->cells(), [&](const auto& c) { return c.voxel.z == shown->slice_z(); })); CHECK(until(*server, *a, [&] { return a->server_tick() >= first_tick + 40; })); + // M10.9a (ADR-0097): the listener's load is exposed; a lightly loaded region measures + // per-tick work, accepts logins and sheds nothing. + const auto sessions = metric(server->ops, "evo_m10_sessions"); + const auto p95 = metric(server->ops, "evo_m10_tick_work_p95_microseconds"); + std::printf("m10 load report sessions=%llu p95_us=%llu p99_us=%llu session_p99_us=%llu\n", + static_cast(sessions.value_or(0)), + static_cast(p95.value_or(0)), + static_cast( + metric(server->ops, "evo_m10_tick_work_p99_microseconds").value_or(0)), + static_cast( + metric(server->ops, "evo_m10_session_work_p99_microseconds").value_or(0))); + CHECK(sessions == 1u && p95.value_or(0) > 0 && metric(server->ops, "evo_m10_refusing") == 0u && + metric(server->ops, "evo_m10_shed_level") == 0u && + metric(server->ops, "evo_m10_refused_logins_total") == 0u); std::printf("m10 snapshot report cells=%zu ticks=%llu chunks=%llu bytes=%llu\n", shown->cells().size(), static_cast(a->server_tick() - first_tick),