Add M10.9a load measurement, governor and overload shedding
Foundation / checks (deploy) (push) Successful in 3m20s
Foundation / checks (ecs) (push) Successful in 1m59s
Foundation / checks (client) (push) Failing after 10m0s
Foundation / checks (network) (push) Successful in 13m9s
Foundation / checks (tsan) (push) Successful in 8m1s
Foundation / checks (checks) (push) Successful in 32m2s
Foundation / checks (postgres) (push) Failing after 26m39s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped

ADR-0097 (M10.9 load and abuse qualification; only evolution-dev
reference-worker runs qualify). evo-server measures each simulated tick's
work (host step plus the M10 bridge, with the session layer's share kept
separately) and feeds a portable LoadGovernor: grant logins are refused
at a 2 s p95 of 15 ms or any tick debt, replication is shed in up to
three levels at 20 ms or debt >= 2 (radius cap 7/7/5/3, far updates
x1/x2/x2/x4, no fresh baselines, own markers unaffected), and both relax
after 5 s of calm. Handoff tickets are still admitted. /metrics gains
evo_m10_* gauges when the listener is enabled.

Adds the missing ADR index entries for ADR-0094 to ADR-0096. Refreshes
the dev session window.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-07 06:34:40 -04:00
co-authored by Claude Opus 5.5
parent 265163a9a6
commit b8cb9db297
22 changed files with 676 additions and 34 deletions
+1 -1
View File
@@ -96,7 +96,7 @@ target_include_directories(evo_region_lease PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/
target_compile_features(evo_region_lease PUBLIC cxx_std_23)
evo_owned_target(evo_region_lease)
# Portable host status and bounded operational exposition (M7.8).
add_library(evo_ops STATIC src/server/ops.cpp)
add_library(evo_ops STATIC src/server/ops.cpp src/server/load_governor.cpp)
add_library(evo::ops ALIAS evo_ops)
target_include_directories(evo_ops PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
target_compile_features(evo_ops PUBLIC cxx_std_23)
+59 -2
View File
@@ -12,8 +12,12 @@
// M10.6 (ADR-0094): sessions' probe markers move within the habitat sites' bounding cells,
// or within --m10-area. M10.8c (ADR-0096): with --m10-handoff-key, markers are durable probe
// records and a marker crossing into a --m10-neighbour's area is handed over with a ticket.
// M10.9a (ADR-0097): the host's step and this bridge's work per simulated tick feed a load
// governor; above its thresholds grant logins are refused and replication is shed.
#include "evo/persistence/postgres.hpp"
#include "evo/persistence/region_lease.hpp"
#include "evo/server/ops.hpp"
#include <chrono>
#include <cstdint>
#include <optional>
#include <string>
@@ -23,6 +27,7 @@
#include "evo/client/environment_view.hpp"
#include "evo/network/handoff_ticket.hpp"
#include "evo/processes/lineage_region.hpp"
#include "evo/server/load_governor.hpp"
#include "evo/server/pg_slot.hpp"
#include "evo/server/postgres_action_ledger.hpp"
#include "evo/server/postgres_control_ledger.hpp"
@@ -183,9 +188,11 @@ template <typename Host> class M10Bridge {
std::printf("m10 window closed activation=%s\n", window_->activation_id().c_str());
return false;
}
void after_step() {
// `host_work`: this iteration's host step, measured by the caller.
void after_step(std::chrono::nanoseconds host_work) {
if (!sessions_ || !accepting(wall_seconds()))
return;
const auto started = std::chrono::steady_clock::now();
auto& host = *host_;
const auto& s = host.status();
const server::HostFacts facts{s.ready && s.phase == server::HostPhase::running, s.epoch,
@@ -233,6 +240,16 @@ template <typename Host> class M10Bridge {
: "refused",
static_cast<unsigned long long>(e.session), e.account.c_str(),
static_cast<unsigned>(e.reason));
measure(host_work, std::chrono::steady_clock::now() - started, s);
}
// The listener's load for /metrics.
void metrics(server::OpsCounters& c) const {
c.m10_enabled = sessions_ != nullptr;
if (!sessions_)
return;
c.m10_sessions = sessions_->sessions();
c.m10_refused_logins = sessions_->refused_for_load();
c.m10_load = governor_.state();
}
void shutdown() {
if (sessions_)
@@ -240,6 +257,40 @@ template <typename Host> class M10Bridge {
}
private:
// Busy time accumulates until the simulated tick advances by k; it then counts as k equal
// per-tick samples (ADR-0097). A recovery's jump starts a new count instead.
void measure(std::chrono::nanoseconds host_work, std::chrono::nanoseconds session_work,
const server::HostStatus& s) {
busy_ += host_work + session_work;
session_busy_ += session_work;
if (s.simulated_tick <= measured_tick_)
return;
const auto ticks = s.simulated_tick - measured_tick_;
const bool counted = measured_tick_ != 0 && ticks <= server::load_window_ticks;
measured_tick_ = s.simulated_tick;
const auto per = [&](std::chrono::nanoseconds total) {
return static_cast<std::uint64_t>(
std::chrono::duration_cast<std::chrono::microseconds>(total).count() /
static_cast<std::int64_t>(ticks));
};
const auto per_tick = per(busy_), session_per_tick = per(session_busy_);
busy_ = session_busy_ = {};
if (!counted)
return;
const auto now_us =
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now().time_since_epoch())
.count());
bool changed = false;
for (std::uint64_t i = 0; i < ticks; ++i)
changed = governor_.observe(per_tick, session_per_tick, s.tick_debt, now_us) || changed;
if (!changed)
return;
const auto& load = governor_.state();
sessions_->set_load(load.refusing, load.shed);
std::printf("m10 load refusing=%d shed=%u p95_us=%llu\n", load.refusing ? 1 : 0,
static_cast<unsigned>(load.shed), static_cast<unsigned long long>(load.p95_us));
}
static const processes::EnvironmentalCommunity&
community(const processes::EnvironmentalCommunity& c) {
return c;
@@ -383,6 +434,9 @@ template <typename Host> class M10Bridge {
std::unique_ptr<server::PgSlot> slot_;
std::unique_ptr<server::PostgresControlLedger> control_;
std::unique_ptr<server::PostgresActionLedger> actions_;
server::LoadGovernor governor_{*server::LoadGovernor::create({})};
std::chrono::nanoseconds busy_{}, session_busy_{};
std::uint64_t measured_tick_{};
std::unique_ptr<server::PostgresProbeLedger> probes_; // Handoff mode only.
std::unique_ptr<server::PostgresHandoffLedger> handoff_; // Handoff mode only.
std::optional<std::array<std::int64_t, 4>> area_cells_;
@@ -407,7 +461,10 @@ template <typename Host> class M10Bridge {
[[nodiscard]] bool accepting(std::uint64_t) const noexcept {
return false;
}
void after_step() {}
void after_step(std::chrono::nanoseconds) {}
void metrics(server::OpsCounters& c) const {
c.m10_enabled = false;
}
void shutdown() {}
};
#endif
+4 -1
View File
@@ -418,6 +418,7 @@ template <typename State> int serve(Options& o) {
std::chrono::steady_clock::now() - last_checkpoint)
.count());
counters.objects_available_bytes = host.objects().available_bytes().value_or(0);
m10->metrics(counters);
return {200, server::render_metrics(s, counters),
"text/plain; version=0.0.4; charset=utf-8"};
}
@@ -516,7 +517,9 @@ template <typename State> int serve(Options& o) {
++*synthetic_next;
}
}
const auto step_started = std::chrono::steady_clock::now();
const auto& s = host.step(scheduling_now());
const auto host_work = std::chrono::steady_clock::now() - step_started; // ADR-0097.
max_queued = std::max(max_queued, s.queued);
// Checkpoint age restarts at recovery and at every confirmed commit.
if ((!recovered && s.phase == server::HostPhase::running) ||
@@ -563,7 +566,7 @@ template <typename State> int serve(Options& o) {
if (ops)
ops->serve(handler);
sessions->step(host.state(), host.status());
m10->after_step();
m10->after_step(host_work);
std::fflush(stdout);
if (o.crash_point == "after-ack" && !acknowledged.empty() && ++ack_steps == o.crash_ordinal)
crash_now();
@@ -12,7 +12,7 @@ spec:
env:
- {name: REGION_SESSION, value: "dev"}
- {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"}
- {name: REGION_SESSION_WINDOW, value: "dev-20261007T031149-a452:1791343909"}
- {name: REGION_SESSION_WINDOW, value: "dev-20261007T102834-1d97:1791370114"}
ports:
# QUIC (ADR-0090): UDP only; nothing listens on TCP 7400.
- {name: session, containerPort: 7400, protocol: UDP}
+5 -1
View File
@@ -541,7 +541,11 @@ durable probes in sessions, edge crossing, ghosts (schema v13) and a two-region
done; 10.8c-2 client switch (offers, redirects, multi-region grants) and evo-server/probe
configuration done; 10.8d two evo-server processes on PostgreSQL with SIGKILL around each
durable step, duplicate/stale retries and a failed destination done (schema v14 recovery
sweep, replay guard). M10.8 complete. 10.9 planned.
sweep, replay guard). M10.8 complete. 10.9 split (ADR-0097; only evolution-dev reference-worker
runs qualify): 10.9a per-tick work measurement, load governor (refuse logins, then shed
replication with hysteresis) and its metrics done; 10.9b evo-load multi-client tool, 10.9c
abuse suite (fuzz, floods, slow peers, storms, lease loss) and 10.9d dev-cluster harness and
reference results follow.
**Entry:** M9 client/session evidence and M7/M8 dev/stage durability. Read
NETWORK_ARCHITECTURE, PERSISTENCE_ARCHITECTURE and ADR-0017. The M9 transport is
+10 -1
View File
@@ -98,7 +98,16 @@ recovery gaps:
- a ticket replayed at a restarted destination after the probe moved on is refused.
M10.8 is complete. Deploying needs protocol v4 and schemas v12-v14 (a rebuilt migrations
image). Next: M10.9, load and abuse qualification.
image). M10.9a (ADR-0097) starts load qualification. Only runs on the evolution-dev reference
worker count as evidence. evo-server now measures each tick's work (host step plus session
layer) and feeds a load governor:
- above a p95 of 15 ms, or with tick debt, new grant logins are refused (`capacity`), while
crossings are still admitted;
- above 20 ms, replication sheds radius and far-update cadence in up to 3 levels;
- both relax after 5 s of calm.
`/metrics` exposes the work percentiles (total and session layer) and the governor's
state. Next: 10.9b, the `evo-load` multi-client tool.
## Previous gate - 2026-10-02
@@ -0,0 +1,107 @@
# ADR-0097: M10 load and abuse qualification and overload response
## Status
Accepted for M10.9 (2026-10-07). Decisions confirmed with the maintainer on 2026-10-07:
- **Venue.** Only measurements on the reference hardware count as qualification: one RKE2
worker per region host on evolution-dev (ADR-0089). Local runs are smoke tests and
development references, never evidence. The gate stays open until the dev run is done.
- **Load tool.** One multi-client tool, `evo-load`: N session clients in one process, each a
bot, reporting per-client distributions so aggregate throughput cannot hide a starved
client.
- **Overload response: refuse, then shed.** Measured per-tick work drives two thresholds.
Above the first, new logins are refused. Above the second, the interest and far-update
cadence of existing sessions shrink step by step. Both are restored with hysteresis, so
ticks do not fall behind for admitted sessions.
- **Abuse coverage:** packet fuzz, floods, slow peers and connect/disconnect storms, and
region lease loss with 20 or more sessions attached.
Delivery:
- **10.9a:** the work measurement, the load governor and its effects (this text).
- **10.9b:** `evo-load` and a local smoke test.
- **10.9c:** the abuse suite.
- **10.9d:** the evolution-dev run harness, then the reference results.
Each segment amends this ADR. Every threshold here is an engineering budget under
`schemas/network/m10-budgets-v1.md`, not a game rule.
## Context
ADR-0089 budgets:
- per-tick region work: p99 ≤ 25 ms on the reference worker;
- session handling: ≤ 0.5 ms per session per tick;
- at least 20 sessions per region (32 by default);
- overload response: shed interest and refuse new sessions before tick debt.
Until now nothing measured the work, and the only bound was the session count. A region
that falls behind accrues tick debt, which delays every admitted session's outcomes and
snapshots.
## Decision (10.9a)
- **Work measurement** (evo-server loop). Busy time is the host's step plus the session
bridge's work (session poll, replication, action carry and acknowledgements). Waits for
input and the loop's sleeps are excluded. When the simulated tick advances by k, the busy
time since the previous advance counts as k equal per-tick samples. The host's tick debt
is read with every sample.
- **Load governor** (`include/evo/server/load_governor.hpp`, portable, no clock). It keeps
a window of the last 40 samples (2 s). It reports the window's p95, p99 and maximum, plus
two outputs:
- **refusing** turns on when p95 ≥ 15 ms or tick debt > 0;
- **shed level** (0-3) rises by one when p95 ≥ 20 ms or tick debt ≥ 2, at most once per
second.
Calm means p95 < 10 ms and no debt. After 5 s of calm the level drops by one. At level 0,
5 s more of calm turns refusing off. Any non-calm sample restarts the calm timer.
Thresholds are configurable and validated (`relax < refuse ≤ shed ≤ tick budget`).
- **Effects** (`RegionSessions::set_load`):
- While refusing, a Hello carrying a grant is closed with `capacity` before signature
work. Handoff tickets are still admitted: a crossing moves an existing player between
regions, and refusing it would strand the probe until the source aborts the transfer
(up to 40 s).
- The shed level limits replication (`ReplicationServer::set_shed`):
| Level | Radius cap | Near radius | Far updates |
|---|---|---|---|
| 0 | 7 | 3 | every 4 ticks |
| 1 | 7 | 3 | every 8 ticks |
| 2 | 5 | 3 | every 8 ticks |
| 3 | 3 | 2 | every 16 ticks |
The stored interest is kept, so shedding forces no fresh baseline. Entities beyond the
cap are removed from the client's view and return when the level drops. Each session's
own marker is still sent every tick it changes.
- **Exposition.** `/metrics` adds `evo_m10_*` gauges:
- sessions;
- refusing;
- shed level;
- work p95, p99 and maximum in microseconds;
- the session layer's share, p95 and p99 (the ADR-0089 per-session budget is checked
against it);
- refused logins (counter).
evo-server prints `m10 load refusing=R shed=L p95_us=P` on every change. No labels are
added.
## Consequences
- Under overload, honest new players are refused (`capacity`, retry later) and existing
sessions see a smaller, slower-updating view rather than delayed ticks. Crossings
continue.
- The thresholds are provisional until 10.9d measures the reference worker. A change is
made by amending this ADR and the budgets table.
- No change to the session protocol: the client already handles `capacity` and removals.
## 10.9a evidence (2026-10-07)
- Unit tests:
- the governor's thresholds, one step per second, hysteresis, the calm timer restarting
on a non-calm window, tick debt alone, and policy validation;
- the metrics families, present only with the listener and bounded at worst-case values;
- replication shedding: radius caps 5 and 3 without a fresh baseline or a change of the
stored interest, a far update every 16 ticks at level 3, and the full square back at
level 0;
- sessions under load: grant logins refused with `capacity` before signature work (the
grant stays usable), tickets admitted.
- Process test (`region_m10_session_tests.cpp`): a real evo-server with `--ops-listen`
exposes the `evo_m10_*` gauges.
Local development reference (WSL, Release, one session; not evidence): work p95 about
7 ms and p99 about 9 ms per tick. Of that, the session layer's p99 was about 4.5 ms. That
is far above 0.5 ms per session, so 10.9b examines it with evo-load. Likely contributors
are the per-tick snapshot source query and synchronous control renewals.
+4
View File
@@ -98,6 +98,10 @@ design. No unadopted candidate is a link dependency of the proof.
- [ADR-0084: Engineering UI toolkit](0084-engineering-ui-toolkit.md) - Dear ImGui 1.92.9b pinned (M9.8a) and integrated as the engineering client overlay with toolkit-first input routing (M9.8c).
- [ADR-0090: M10 QUIC transport](0090-m10-quic-transport.md) - msquic 2.6.2 with OpenSSL 3.5.9 LTS (TLS 1.3 only) from pinned sources on Windows and Linux; DATAGRAM frames for superseding state, streams for reliable actions; M9 TLS/TCP kept as dev fallback; owned impairment relay; exact CA pinning due in 10.2b.
- [ADR-0097: M10 load and abuse qualification](0097-m10-load-and-abuse-qualification.md) - only evolution-dev reference-worker runs qualify; one multi-client evo-load tool with per-client distributions; a load governor over per-tick work (p95 window of 40 ticks) and tick debt refuses grant logins, then sheds replication radius and cadence in 3 levels with hysteresis (tickets still admitted); abuse suite of fuzz, floods, slow peers, storms and lease loss.
- [ADR-0096: M10 probe handoff](0096-m10-probe-handoff.md) - EXPERIMENT durable probe records with one owning region and a conserved test inventory; fenced prepared/staged/committed/aborted transfers; signed single-use transfer tickets; protocol v4 offers, redirects and ghosts; recovery sweep (schema v14) and replay guard proven by a two-process fault matrix.
- [ADR-0095: M10 interpolation and session loss](0095-m10-interpolation-and-session-loss.md) - drift-corrected client tick clock, adaptive 2-6 tick interpolation of remote markers with at most 2 ticks of extrapolation (organisms never), frozen stale view on loss and a fresh baseline on reconnect.
- [ADR-0094: M10 probe motion experiment](0094-m10-probe-motion-experiment.md) - EXPERIMENT fixed-speed probe marker (probe-motion-v1) moved by admitted intentions, protocol v3 marker records, own-marker prediction with reconciliation; not an organism movement rule.
- [ADR-0093: M10 snapshots and interest](0093-m10-snapshots-and-interest.md) - session protocol v2: client-declared, server-capped interest (radius <= 7) with a perception boundary; per-tick snapshot chunks as datagrams, entity deltas against the acknowledged baseline (<= 4, <= 40 ticks), nearest first within 16 KiB, distant tier every 4 ticks; client assembly with 2 baselines.
- [ADR-0092: M10 intentions and durable outcomes](0092-m10-intentions-and-durable-outcomes.md) - realtime intentions (datagram receipts and a cumulative simulated watermark) and account-scoped actions with stable ids, receipts and final outcomes (committed_no_effect / not_committed); no biological effect until organisms exist; durable action records in the commit transaction (10.4b).
- [ADR-0091: M10 session identity and control](0091-m10-session-identity-and-control.md) - provider-independent 128-bit AccountIds, ECDSA P-256 session grants (regions hold public keys only, signed environment byte), per-account serials where the newest grant wins, takeover of an existing session, in-session refresh, bounded attempt rates; identity adapter and issuer as a library; durable control lease follows (10.3c).
+44
View File
@@ -4904,3 +4904,47 @@ Not done:
- deployment of protocol v4 and schemas v12-v14 (needs a rebuilt migrations image; not done
from routine development);
- a GUI run across two regions on evolution-dev (only one dev region exists).
## 2026-10-07 — M10.9a load measurement and governor (ADR-0097)
Implemented:
- evo-server measures each simulated tick's work: the host step plus the M10 bridge
(session poll, replication, action carry). Waits are excluded, and the session layer's
share is kept separately.
- `LoadGovernor` (`include/evo/server/load_governor.hpp`, portable): a window of 40 ticks,
refuse at p95 ≥ 15 ms or any tick debt, shed one level per second at p95 ≥ 20 ms or
debt ≥ 2, and relax one step per 5 s of calm (p95 < 10 ms, no debt).
- `RegionSessions::set_load`: while refusing, grant logins are closed with `capacity`
before signature work; handoff tickets are still admitted.
- `ReplicationServer::set_shed`: levels 0-3 cap the radius (7/7/5/3) and the near radius
(3/3/3/2) and stretch far updates (x1/x2/x2/x4). Stored interests are kept, and own
markers are unaffected.
- `/metrics`: `evo_m10_*` gauges (sessions, refusing, shed level, work and session-work
percentiles, maximum) and a refused-logins counter, rendered only with the listener.
evo-server prints `m10 load ...` on every change.
- The ADR index gains the missing entries for ADR-0094 to ADR-0096.
Tests:
- `ops-contracts`: the governor state machine (thresholds, step interval, hysteresis, calm
restart, debt, policy validation) and the M10 metric families, including worst-case size.
- `network`: replication shedding (caps, no fresh baseline, 16-tick far cadence at level
3, recovery at level 0).
- `quic-sessions`: grant logins refused under load (the grant is not consumed), a ticket
login admitted, and logins accepted after relaxing.
- Postgres `region_m10_session_tests`: a real evo-server's `/metrics` shows one session,
measured work, not refusing, shed 0.
Executed:
- **Linux GCC 14:** `postgres-check.sh` with QUIC, all suites; presets `quic` (ops,
network, QUIC sessions) 3/3. The local reference (Release, one session) measured work
p95 7.1 ms and p99 8.8 ms, with the session layer at p99 4.5 ms. This is a development
reference only, flagged for 10.9b.
- **Linux Clang 19:** clang-tidy 19 on the governor, ops, replication, sessions, evo-server
`main.cpp` with the bridge, and the tests (one finding fixed); clang-format 19.
- **Windows MSVC 19.44:** presets `debug` 60/60 (ops-contracts included), `quic` 4/4 and
`client-network` 14/14.
Not done:
- qualification on the evolution-dev reference worker (10.9d; needs v4 and v12-v14
deployed);
- load generation (10.9b) and the abuse suite (10.9c).
+9
View File
@@ -89,6 +89,14 @@ class ReplicationServer {
tick(std::uint64_t tick, const std::function<ReplicationSource(const SessionInterest&)>& source,
const Perceives& perceives, const std::function<std::size_t(std::uint64_t)>& payload);
[[nodiscard]] ReplicationStats stats() const;
// M10.9a overload shedding (ADR-0097), level 0-3 (higher is clamped to 3): caps the
// radius encoded (7, 7, 5, 3), the near radius (3, 3, 3, 2) and stretches far updates
// (x1, x2, x2, x4). Stored interests are kept (no fresh baseline); entities beyond the
// cap are removed from views and return when the level drops. Own markers are unaffected.
void set_shed(std::uint8_t level) noexcept;
[[nodiscard]] std::uint8_t shed() const noexcept {
return shed_;
}
private:
// (kind, x | id, y, z): environment by voxel, organisms by id.
@@ -109,6 +117,7 @@ class ReplicationServer {
};
explicit ReplicationServer(ReplicationLimits limits) : limits_{limits} {}
ReplicationLimits limits_;
std::uint8_t shed_{};
std::vector<Session> sessions_;
ReplicationStats stats_;
};
+56
View File
@@ -0,0 +1,56 @@
#pragma once
#include <array>
#include <cstddef>
#include <cstdint>
#include <expected>
// M10.9a overload response (ADR-0097): a region's per-tick work and tick debt decide whether
// new logins are refused and how far replication is shed. Portable: no clocks (callers pass a
// monotonic time in microseconds), no transport. Thresholds are engineering budgets
// (schemas/network/m10-budgets-v1.md), not game rules.
namespace evo::server {
struct LoadPolicy {
std::uint64_t tick_budget_us{25'000}; // p99 per-tick region work budget (ADR-0089).
std::uint64_t refuse_us{15'000}; // Window p95 at or above: refuse new logins.
std::uint64_t shed_us{20'000}; // Window p95 at or above: shed one more level.
std::uint64_t relax_us{10'000}; // Window p95 below (and no debt): calm.
std::uint64_t step_us{1'000'000}; // At most one shed step per interval.
std::uint64_t settle_us{5'000'000}; // Calm this long relaxes one step.
std::uint8_t max_shed{3};
};
inline constexpr std::size_t load_window_ticks = 40; // 2 s of 50 ms ticks.
struct LoadState {
bool refusing{};
std::uint8_t shed{};
std::uint64_t p95_us{}, p99_us{}, max_us{}; // Over the window (0 before any sample).
std::size_t samples{}; // In the window.
// The session layer's share of the same ticks (ADR-0089 budgets it per session).
std::uint64_t session_p95_us{}, session_p99_us{};
friend bool operator==(const LoadState&, const LoadState&) = default;
};
enum class LoadPolicyError : std::uint8_t { invalid };
class LoadGovernor {
public:
[[nodiscard]] static std::expected<LoadGovernor, LoadPolicyError> create(LoadPolicy policy);
// One simulated tick's region work (of which `session_us` was the session layer's) and the
// host's tick debt, at monotonic `now_us`. Returns true when `refusing` or `shed` changed.
bool observe(std::uint64_t work_us, std::uint64_t session_us, std::uint64_t tick_debt,
std::uint64_t now_us);
[[nodiscard]] const LoadState& state() const noexcept {
return state_;
}
[[nodiscard]] const LoadPolicy& policy() const noexcept {
return policy_;
}
private:
explicit LoadGovernor(LoadPolicy policy) : policy_{policy} {}
LoadPolicy policy_;
LoadState state_;
std::array<std::uint64_t, load_window_ticks> window_{}, session_window_{};
std::size_t next_{};
bool stepped_{}, calm_{};
std::uint64_t last_step_us_{}, calm_since_us_{};
};
} // namespace evo::server
+6
View File
@@ -1,4 +1,5 @@
#pragma once
#include "evo/server/load_governor.hpp"
#include <array>
#include <cstddef>
#include <cstdint>
@@ -47,6 +48,11 @@ struct OpsCounters {
// M8.5: object volume free space and the store's reserve (ADR-0070), so capacity
// is alerted on before new objects are refused.
std::uint64_t objects_available_bytes{}, objects_reserve_bytes{};
// M10.9a (ADR-0097): the M10 session listener's load governor, rendered only when the
// listener is enabled.
bool m10_enabled{};
std::uint64_t m10_sessions{}, m10_refused_logins{};
LoadState m10_load;
};
[[nodiscard]] const char* phase_name(HostPhase phase) noexcept;
+6
View File
@@ -136,6 +136,12 @@ class RegionSessions {
const std::function<network::ReplicationSource(const network::SessionInterest&)>& source,
const network::Perceives& perceives = {});
[[nodiscard]] network::ReplicationStats replication_stats() const;
// M10.9a (ADR-0097): the host's load governor output. While `refuse_logins`, a Hello
// carrying a grant is closed with `capacity` before signature work (handoff tickets are
// still admitted); `shed` (0-3) limits replication (ReplicationServer::set_shed).
void set_load(bool refuse_logins, std::uint8_t shed);
// Grant logins refused for load since creation.
[[nodiscard]] std::uint64_t refused_for_load() const noexcept;
// Where probe markers live and spawn (ADR-0094); sessions already established receive a
// marker. False for an invalid area. With neighbours (ADR-0096), markers move within the
// bounding box of this area and the neighbours' areas, and leaving this area hands off.
+3 -3
View File
@@ -32,10 +32,10 @@ in real time; changing it requires a versioned rules migration (ADR-0089).
| Client event queue | 1,024 events | budget | client session |
| Heartbeat / liveness / handshake | 1 s / 5 s / 5 s | budget | 10.2, 10.7 |
| Sessions per region | >= 20 required, 32 default | budget | 10.9 |
| Per-tick region work | p99 <= 25 ms on the reference worker | budget | 10.9 |
| Session handling | <= 0.5 ms per session per tick | budget | 10.9 |
| Per-tick region work | p99 <= 25 ms on the reference worker | budget | 10.9 (measured: `evo_m10_tick_work_*`, ADR-0097) |
| Session handling | <= 0.5 ms per session per tick | budget | 10.9 (measured: `evo_m10_session_work_*`, ADR-0097) |
| Host input queue | 64 inputs, 64 KiB | budget | region host |
| Overload response | shed interest, refuse new sessions before tick debt | budget | 10.9 |
| Overload response | shed interest, refuse new sessions before tick debt | budget | 10.9a load governor: refuse at p95 >= 15 ms or debt, shed at p95 >= 20 ms or debt >= 2, relax below 10 ms after 5 s (ADR-0097; provisional until 10.9d) |
Determinism: authoritative state is exact given the checkpoint and committed per-tick
inputs; timing, delivery, prediction, interpolation and presentation are tolerant
+19 -5
View File
@@ -1,5 +1,6 @@
#include "evo/network/replication.hpp"
#include <algorithm>
#include <array>
#include <limits>
namespace evo::network {
@@ -224,10 +225,20 @@ ReplicationServer::tick(std::uint64_t tick,
return this->tick(tick, source, perceives, [payload](std::uint64_t) { return payload; });
}
void ReplicationServer::set_shed(std::uint8_t level) noexcept {
shed_ = std::min<std::uint8_t>(level, 3);
}
std::vector<std::pair<std::uint64_t, std::vector<SnapshotChunk>>> ReplicationServer::tick(
std::uint64_t tick, const std::function<ReplicationSource(const SessionInterest&)>& source,
const Perceives& perceives, const std::function<std::size_t(std::uint64_t)>& payload_of) {
std::vector<std::pair<std::uint64_t, std::vector<SnapshotChunk>>> out;
// Shedding (ADR-0097): radius cap, near radius and far-update multiplier per level.
constexpr std::array<std::uint16_t, 4> shed_radius{7, 7, 5, 3};
constexpr std::array<std::uint16_t, 4> shed_near{3, 3, 3, 2};
constexpr std::array<std::uint64_t, 4> shed_far{1, 2, 2, 4};
const auto near_radius = std::min(limits_.near_radius, shed_near.at(shed_));
const auto far_interval = limits_.far_interval * shed_far.at(shed_);
const auto overhead = session_header_bytes + snapshot_chunk_header_bytes;
if (tick == 0)
return out;
@@ -239,12 +250,15 @@ std::vector<std::pair<std::uint64_t, std::vector<SnapshotChunk>>> ReplicationSer
if (payload <= overhead)
continue; // No datagram channel yet (or a path too small): nothing to send.
const auto capacity = payload - overhead; // Record bytes per chunk.
const auto& in = *s.interest;
auto cached = std::ranges::find_if(sources, [&](const auto& e) { return e.first == in; });
const auto& wanted = *s.interest;
auto cached =
std::ranges::find_if(sources, [&](const auto& e) { return e.first == wanted; });
if (cached == sources.end()) {
sources.emplace_back(in, source(in));
sources.emplace_back(wanted, source(wanted));
cached = std::prev(sources.end());
}
auto in = wanted; // Encoded within the shed cap; the stored interest is unchanged.
in.radius = std::min(in.radius, shed_radius.at(shed_));
const auto& src = cached->second;
// Spatial filter, then perception: nothing outside either is ever encoded.
std::map<Key, std::pair<std::vector<std::uint8_t>, std::uint64_t>> truth;
@@ -299,8 +313,8 @@ std::vector<std::pair<std::uint64_t, std::vector<SnapshotChunk>>> ReplicationSer
const auto known = base_state.find(key);
if (known != base_state.end() && known->second.record == value.first)
continue; // The client already holds this value.
const bool due = known == base_state.end() || value.second <= limits_.near_radius ||
tick - known->second.updated >= limits_.far_interval;
const bool due = known == base_state.end() || value.second <= near_radius ||
tick - known->second.updated >= far_interval;
if (!due) {
++deferred;
continue;
+69
View File
@@ -0,0 +1,69 @@
#include "evo/server/load_governor.hpp"
#include <algorithm>
#include <utility>
namespace evo::server {
std::expected<LoadGovernor, LoadPolicyError> LoadGovernor::create(LoadPolicy p) {
if (p.relax_us == 0 || p.relax_us >= p.refuse_us || p.refuse_us > p.shed_us ||
p.shed_us > p.tick_budget_us || p.step_us == 0 || p.settle_us == 0 || p.max_shed > 3)
return std::unexpected(LoadPolicyError::invalid);
return LoadGovernor{p};
}
namespace {
using Window = std::array<std::uint64_t, load_window_ticks>;
// The first `n` samples sorted; nearest rank: the smallest sample with at least `percent` of
// the window at or below it.
Window sorted_of(const Window& w, std::size_t n) {
Window sorted{};
std::copy_n(w.begin(), n, sorted.begin());
std::sort(sorted.begin(), sorted.begin() + static_cast<std::ptrdiff_t>(n));
return sorted;
}
std::uint64_t rank(const Window& sorted, std::size_t n, std::size_t percent) {
return sorted.at((((n * percent) + 99) / 100) - 1);
}
} // namespace
bool LoadGovernor::observe(std::uint64_t work_us, std::uint64_t session_us, std::uint64_t tick_debt,
std::uint64_t now_us) {
window_.at(next_) = work_us;
session_window_.at(next_) = std::min(session_us, work_us);
next_ = (next_ + 1) % window_.size();
state_.samples = std::min(state_.samples + 1, window_.size());
const auto n = state_.samples;
const auto sorted = sorted_of(window_, n);
state_.p95_us = rank(sorted, n, 95);
state_.p99_us = rank(sorted, n, 99);
state_.max_us = sorted.at(n - 1);
const auto sessions = sorted_of(session_window_, n);
state_.session_p95_us = rank(sessions, n, 95);
state_.session_p99_us = rank(sessions, n, 99);
const auto before = std::pair{state_.refusing, state_.shed};
const bool hot = state_.p95_us >= policy_.refuse_us || tick_debt > 0;
const bool overloaded = state_.p95_us >= policy_.shed_us || tick_debt >= 2;
if (hot)
state_.refusing = true;
if (overloaded && state_.shed < policy_.max_shed &&
(!stepped_ || now_us - last_step_us_ >= policy_.step_us)) {
++state_.shed;
stepped_ = true;
last_step_us_ = now_us;
}
if (state_.p95_us >= policy_.relax_us || tick_debt > 0) {
calm_ = false;
} else if (!calm_) {
calm_ = true;
calm_since_us_ = now_us;
} else if (now_us - calm_since_us_ >= policy_.settle_us) {
if (state_.shed > 0)
--state_.shed; // One level per settle interval, then logins.
else
state_.refusing = false;
calm_since_us_ = now_us;
}
return before != std::pair{state_.refusing, state_.shed};
}
} // namespace evo::server
+18
View File
@@ -103,6 +103,24 @@ std::string render_metrics(const HostStatus& s, const OpsCounters& c) {
.append("\"} ")
.append(std::to_string(c.dropped[i]))
.append("\n");
if (c.m10_enabled) {
metric(out, "evo_m10_sessions", "gauge", "Open M10 sessions", c.m10_sessions);
metric(out, "evo_m10_refusing", "gauge", "1 while new logins are refused for load",
c.m10_load.refusing);
metric(out, "evo_m10_shed_level", "gauge", "Replication shed level (0-3)", c.m10_load.shed);
metric(out, "evo_m10_tick_work_p95_microseconds", "gauge",
"p95 per-tick region work over the last 40 ticks", c.m10_load.p95_us);
metric(out, "evo_m10_tick_work_p99_microseconds", "gauge",
"p99 per-tick region work over the last 40 ticks", c.m10_load.p99_us);
metric(out, "evo_m10_tick_work_max_microseconds", "gauge",
"Largest per-tick region work over the last 40 ticks", c.m10_load.max_us);
metric(out, "evo_m10_session_work_p95_microseconds", "gauge",
"p95 per-tick session-layer work over the last 40 ticks", c.m10_load.session_p95_us);
metric(out, "evo_m10_session_work_p99_microseconds", "gauge",
"p99 per-tick session-layer work over the last 40 ticks", c.m10_load.session_p99_us);
metric(out, "evo_m10_refused_logins_total", "counter", "Logins refused for load",
c.m10_refused_logins);
}
return out;
}
} // namespace evo::server
+15
View File
@@ -108,6 +108,8 @@ struct RegionSessions::Impl {
std::vector<Outgoing> outgoing; // Handed off, not yet known committed.
Clock::time_point next_ghosts{}, next_overdue{};
std::optional<network::SessionInterest> default_interest;
bool refuse_logins{}; // Load governor (ADR-0097).
std::uint64_t refused_load{}; // Grant logins refused for load.
HostFacts facts;
std::vector<Connection> connections;
std::vector<RegionSessionEvent> events;
@@ -216,6 +218,10 @@ struct RegionSessions::Impl {
return close(c, SessionCloseReason::rate_limited, now);
if (network::looks_like_handoff_ticket(m.grant))
return ticket_hello(c, m, now_seconds, now);
if (refuse_logins) { // Overload (ADR-0097): before signature work.
++refused_load;
return close(c, SessionCloseReason::capacity, now);
}
const auto grant = verifier.verify(m.grant);
if (!grant)
return close(c, reason_for(grant.error()), now);
@@ -774,6 +780,15 @@ std::optional<std::uint64_t> RegionSessions::holder(const network::AccountId& ac
return impl_->control.holder(account);
}
void RegionSessions::set_load(bool refuse_logins, std::uint8_t shed) {
impl_->refuse_logins = refuse_logins;
impl_->replication.set_shed(shed);
}
std::uint64_t RegionSessions::refused_for_load() const noexcept {
return impl_->refused_load;
}
std::size_t RegionSessions::sessions() const noexcept {
return impl_->control.sessions();
}
+63
View File
@@ -5,6 +5,7 @@
#include "network_test.hpp"
#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <map>
#include <set>
#include <vector>
@@ -385,9 +386,71 @@ void dense_report() {
}
} // namespace
// M10.9a overload shedding (ADR-0097): the encoded radius shrinks and far updates slow without a
// fresh baseline or a change of the stored interest; the view grows back when the level drops.
void shedding() {
auto server = *ReplicationServer::create(limits());
World world;
CHECK(server.open(1, 5) && server.set_interest(1, {0, 0, slice, 7}));
SnapshotAssembler client{identity, 5};
const auto source = [&](const SessionInterest&) { return world.source(); };
std::uint64_t tick = 0;
std::uint64_t baseline = 0;
const auto step = [&] {
const auto out = server.tick(++tick, source, {});
const auto& chunks = for_session(out, 1);
baseline = chunks.front().baseline;
for (const auto& a : deliver(client, chunks))
server.acknowledge(1, a);
return client.latest()->environment.size();
};
while (step() < 225 && tick < 10) {
}
CHECK(client.latest()->environment.size() == 225);
const auto square = [&](std::int64_t r) {
return std::ranges::all_of(client.latest()->environment, [&](const auto& c) {
return std::max(std::abs(c.voxel.x), std::abs(c.voxel.y)) <= r;
});
};
server.set_shed(2);
CHECK(server.shed() == 2 && step() == 121 && square(5) && baseline != 0);
CHECK(server.interest(1).value_or(SessionInterest{}).radius == 7); // The stored interest.
server.set_shed(9);
CHECK(server.shed() == 3 && step() == 49 && square(3) && baseline != 0);
// Level 3: near radius 2, far updates every 16 ticks. A cell at distance 3 changed twice
// arrives at once (last sent long ago), then not again for 16 ticks.
const auto at = [&](std::int64_t x, std::int64_t y) {
return std::ranges::find_if(client.latest()->environment,
[&](const auto& c) { return c.voxel.x == x && c.voxel.y == y; })
->quantity[0];
};
for (int i = 0; i < 16; ++i)
step();
world.quantity[{3, 0}] = 41;
world.quantity[{2, 0}] = 42;
step();
CHECK(at(3, 0) == 41 && at(2, 0) == 42);
world.quantity[{3, 0}] = 51;
world.quantity[{2, 0}] = 52;
step();
CHECK(at(2, 0) == 52 && at(3, 0) == 41); // Near: every tick. Far: deferred.
int waited = 1;
while (at(3, 0) != 51 && waited < 40) {
step();
++waited;
}
CHECK(waited == 16);
// Back to level 0: the full square returns, still from acknowledged baselines.
server.set_shed(0);
for (int i = 0; i < 6 && client.latest()->environment.size() < 225; ++i)
step();
CHECK(client.latest()->environment.size() == 225 && baseline != 0);
}
void replication_tests() {
codec_v2();
convergence_and_deltas();
shedding();
concealment();
client_robustness();
dense_report();
+96 -1
View File
@@ -52,6 +52,17 @@ std::vector<Sample> parse(const std::string& text) {
return out;
}
// Every host counter at its largest value (the M10 section stays disabled).
OpsCounters worst_counters() {
constexpr auto most = std::numeric_limits<std::uint64_t>::max();
OpsCounters c;
c.queue_capacity_inputs = c.queue_capacity_bytes = most;
c.checkpoint_age_ms = c.acks = c.outcomes = most;
c.dropped = {most, most, most, most};
c.objects_available_bytes = c.objects_reserve_bytes = most;
return c;
}
void exposition_is_bounded_and_well_formed() {
HostStatus s;
s.phase = HostPhase::running;
@@ -93,7 +104,7 @@ void exposition_is_bounded_and_well_formed() {
big.max_tick_debt = big.commits = big.store_failures = big.unknown_outcomes = most;
big.fences = big.lease_remaining_ms = most;
big.simulated_tick = 1; // Lag saturates at zero rather than wrapping.
OpsCounters bigc{most, most, most, most, most, {most, most, most, most}, most, most};
const auto bigc = worst_counters();
const auto large = render_metrics(big, bigc);
CHECK(large.size() <= max_metrics_bytes);
CHECK(large.find("evo_region_commit_lag_ticks 0\n") != std::string::npos);
@@ -105,11 +116,95 @@ void exposition_is_bounded_and_well_formed() {
CHECK(std::string(unacknowledged_name(static_cast<Unacknowledged>(i))) != "unknown");
std::printf("ops_metrics_bytes typical=%zu worst=%zu\n", text.size(), large.size());
}
// M10.9a (ADR-0097): the listener's governor families appear only when it is enabled.
void m10_load_metrics() {
HostStatus s;
OpsCounters c;
const auto without = parse(render_metrics(s, c)).size();
c.m10_enabled = true;
c.m10_sessions = 21;
c.m10_refused_logins = 3;
c.m10_load = {true, 2, 17'000, 19'500, 23'000, 40, 900, 1'100};
const auto text = render_metrics(s, c);
CHECK(parse(text).size() == without + 9);
CHECK(text.find("evo_m10_session_work_p99_microseconds 1100\n") != std::string::npos);
CHECK(text.find("evo_m10_sessions 21\n") != std::string::npos);
CHECK(text.find("evo_m10_refusing 1\n") != std::string::npos);
CHECK(text.find("evo_m10_shed_level 2\n") != std::string::npos);
CHECK(text.find("evo_m10_tick_work_p99_microseconds 19500\n") != std::string::npos);
CHECK(text.find("evo_m10_refused_logins_total 3\n") != std::string::npos);
constexpr auto most = std::numeric_limits<std::uint64_t>::max();
HostStatus big;
big.epoch = big.committed_tick = big.committed_sequence = big.tick_debt = most;
big.max_tick_debt = big.commits = big.store_failures = big.unknown_outcomes = most;
big.fences = big.lease_remaining_ms = most;
auto bigc = worst_counters();
bigc.m10_enabled = true;
bigc.m10_sessions = bigc.m10_refused_logins = most;
bigc.m10_load = {true, 3, most, most, most, 40, most, most};
CHECK(render_metrics(big, bigc).size() <= max_metrics_bytes);
}
// M10.9a (ADR-0097): refuse, then shed one level per second; relax one level per 5 s of
// calm, then accept again. Samples are 50 ms ticks.
void load_governor() {
CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 15'000, 1'000'000, 5'000'000, 3}));
CHECK(!LoadGovernor::create({25'000, 21'000, 20'000, 10'000, 1'000'000, 5'000'000, 3}));
CHECK(!LoadGovernor::create({19'000, 15'000, 20'000, 10'000, 1'000'000, 5'000'000, 3}));
CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 10'000, 0, 5'000'000, 3}));
CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 10'000, 1'000'000, 5'000'000, 4}));
auto made = LoadGovernor::create({});
CHECK(made.has_value());
auto& g = *made;
std::uint64_t now = 0;
const auto run = [&](std::size_t ticks, std::uint64_t work, std::uint64_t debt = 0) {
for (std::size_t i = 0; i < ticks; ++i) {
now += 50'000;
g.observe(work, work / 4, debt, now);
}
return g.state();
};
// Quiet: nothing changes; the window reports its percentiles.
CHECK(run(60, 5'000) == LoadState{false, 0, 5'000, 5'000, 5'000, 40, 1'250, 1'250});
// A single slow tick is not the p95 of 40.
CHECK(run(1, 30'000).max_us == 30'000 && !g.state().refusing && g.state().p95_us == 5'000);
// Sustained 16 ms: refuse, no shedding.
auto st = run(40, 16'000);
CHECK(st.refusing && st.shed == 0 && st.p95_us == 16'000);
// 22 ms: one shed level per second, never past 3.
// The p95 (nearest rank, 38th of 40) reaches the threshold with the third slow tick.
CHECK(run(2, 22'000).shed == 0 && run(1, 22'000).shed == 1);
CHECK(run(18, 22'000).shed == 1 && run(2, 22'000).shed == 2);
CHECK(run(100, 22'000).shed == 3);
// Calm only once the window's p95 is below 10 ms; then one level per 5 s.
st = run(37, 4'000);
CHECK(st.shed == 3 && st.p95_us == 22'000);
CHECK(run(1, 4'000).p95_us == 4'000); // Calm starts now.
CHECK(run(99, 4'000).shed == 3 && run(1, 4'000).shed == 2);
// A non-calm window (p95 >= 10 ms, still under the refuse threshold) restarts the timer.
CHECK(run(50, 4'000).shed == 2 && run(3, 12'000).p95_us == 12'000);
CHECK(run(37, 4'000).p95_us == 12'000 && run(1, 4'000).p95_us == 4'000);
CHECK(run(99, 4'000).shed == 2 && run(1, 4'000).shed == 1);
// Down to 0 after another 5 s, then logins after 5 s more.
CHECK(run(100, 4'000).shed == 0 && g.state().refusing);
CHECK(run(99, 4'000).refusing && !run(1, 4'000).refusing);
// Tick debt alone: refuse at 1, shed at 2, even with light work.
CHECK(run(1, 1'000, 1).refusing && g.state().shed == 0);
CHECK(run(1, 1'000, 2).shed == 1);
CHECK(!g.observe(1'000, 250, 2, now + 10)); // Within the step interval: no change.
CHECK(g.state().session_p95_us == 1'000); // The window is mostly 4 ms ticks.
// Debt gone and calm (from the next sample): recovers.
CHECK(run(100, 1'000).shed == 1 && run(1, 1'000).shed == 0);
CHECK(run(99, 1'000).refusing && !run(1, 1'000).refusing);
}
} // namespace
int main() {
try {
exposition_is_bounded_and_well_formed();
m10_load_metrics();
load_governor();
std::puts("ops contracts passed");
return 0;
} catch (const std::exception& e) {
+13
View File
@@ -627,6 +627,19 @@ void ticket_login(const Files& f) {
handoff.error = HandoffLedgerError::unavailable;
CHECK(refused_with(h, ticket(account(3), 0x65), Reason::capacity));
CHECK(a->state() == RegionSessionState::established && handoff.admitted.size() == 1);
// M10.9a (ADR-0097): under load, grant logins are refused before signature work while
// tickets (crossings) are still admitted; the refused grant stays usable.
handoff.error.reset();
h.server->set_load(true, 3);
CHECK(refused_with(h, h.issuer.grant(account(5), 1), Reason::capacity));
CHECK(refused_with(h, h.issuer.grant(account(5), 1), Reason::capacity));
CHECK(h.server->refused_for_load() == 2 && a->state() == RegionSessionState::established);
auto crossed = h.connect(ticket(account(6), 0x66));
CHECK(h.settled(*crossed) && crossed->state() == RegionSessionState::established);
h.server->set_load(false, 0);
auto later = h.connect(h.issuer.grant(account(5), 1));
CHECK(h.settled(*later) && later->state() == RegionSessionState::established &&
h.server->refused_for_load() == 2);
}
// M10.8c (ADR-0096): two regions with adjacent areas in one process, over the fake world
+68 -18
View File
@@ -9,14 +9,17 @@
#include "evo/network/session_grant.hpp"
#include "fixtures/server_process.hpp"
#include <algorithm>
#include <arpa/inet.h>
#include <array>
#include <chrono>
#include <fcntl.h>
#include <functional>
#include <netinet/in.h>
#include <optional>
#include <source_location>
#include <sstream>
#include <string>
#include <sys/socket.h>
#include <thread>
#include <unistd.h>
@@ -69,6 +72,34 @@ std::string hex(const AccountId& a) {
return out;
}
// GET /metrics from the ops listener; the value of an unlabelled series (ADR-0097 adds the
// evo_m10_* families when the listener is enabled).
std::optional<std::uint64_t> metric(std::uint16_t port, const std::string& series) {
const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0);
expect(fd >= 0, "socket");
sockaddr_in a{};
a.sin_family = AF_INET;
a.sin_port = htons(port);
a.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
timeval timeout{3, 0};
::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout);
std::string raw;
if (::connect(fd, reinterpret_cast<const sockaddr*>(&a), sizeof a) == 0) {
const std::string request = "GET /metrics HTTP/1.1\r\nHost: test\r\n\r\n";
if (::send(fd, request.data(), request.size(), MSG_NOSIGNAL) ==
static_cast<ssize_t>(request.size())) {
std::array<char, 4096> chunk{};
for (ssize_t n = 0; (n = ::recv(fd, chunk.data(), chunk.size(), 0)) > 0;)
raw.append(chunk.data(), static_cast<std::size_t>(n));
}
}
::close(fd);
const auto at = raw.find("\n" + series + " ");
if (at == std::string::npos)
return std::nullopt;
return std::stoull(raw.substr(at + series.size() + 2));
}
// One region's credentials, issuer key and server process arguments.
struct Setup {
ServerRegion region;
@@ -95,22 +126,23 @@ struct Setup {
}
[[nodiscard]] std::vector<std::string> args() const {
return server_args(
region, {"--synthetic-inputs", "development-v1", "--m10-listen", "127.0.0.1:0",
"--m10-cert", (region.root / "server.pem").string(), "--m10-key",
(region.root / "server.key").string(), "--m10-issuer",
(region.root / "issuer.pub").string(), "--m10-environment", "dev",
"--m10-max-grant-lifetime", "900", "--session-window",
"test-" +
std::to_string(static_cast<std::uint64_t>(
std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count())) +
":" +
std::to_string(static_cast<std::uint64_t>(
std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count()) +
1200)});
region,
{"--synthetic-inputs", "development-v1", "--ops-listen", "127.0.0.1:0", "--m10-listen",
"127.0.0.1:0", "--m10-cert", (region.root / "server.pem").string(), "--m10-key",
(region.root / "server.key").string(), "--m10-issuer",
(region.root / "issuer.pub").string(), "--m10-environment", "dev",
"--m10-max-grant-lifetime", "900", "--session-window",
"test-" +
std::to_string(static_cast<std::uint64_t>(
std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count())) +
":" +
std::to_string(static_cast<std::uint64_t>(
std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count()) +
1200)});
}
[[nodiscard]] WireGrant grant(const AccountId& who, std::uint64_t serial) const {
GrantClaims c;
@@ -133,12 +165,16 @@ struct Setup {
// A running evo-server and its QUIC port, its output drained while waiting.
struct Server {
std::unique_ptr<Child> child;
std::uint16_t port{};
std::uint16_t port{}, ops{};
explicit Server(const Setup& s) : child{std::make_unique<Child>(s.args())} {
CHECK(child->wait_line("m10 listening ", 30s));
for (const auto& l : child->lines())
for (const auto& l : child->lines()) {
if (starts(l, "m10 listening "))
port = static_cast<std::uint16_t>(std::stoul(l.substr(14)));
if (starts(l, "ops listening "))
ops = static_cast<std::uint16_t>(std::stoul(l.substr(14)));
}
CHECK(ops != 0);
CHECK(port != 0 && child->wait_line("ready 1", 30s));
}
void pump() {
@@ -237,6 +273,20 @@ void end_to_end() {
CHECK(std::ranges::all_of(shown->cells(),
[&](const auto& c) { return c.voxel.z == shown->slice_z(); }));
CHECK(until(*server, *a, [&] { return a->server_tick() >= first_tick + 40; }));
// M10.9a (ADR-0097): the listener's load is exposed; a lightly loaded region measures
// per-tick work, accepts logins and sheds nothing.
const auto sessions = metric(server->ops, "evo_m10_sessions");
const auto p95 = metric(server->ops, "evo_m10_tick_work_p95_microseconds");
std::printf("m10 load report sessions=%llu p95_us=%llu p99_us=%llu session_p99_us=%llu\n",
static_cast<unsigned long long>(sessions.value_or(0)),
static_cast<unsigned long long>(p95.value_or(0)),
static_cast<unsigned long long>(
metric(server->ops, "evo_m10_tick_work_p99_microseconds").value_or(0)),
static_cast<unsigned long long>(
metric(server->ops, "evo_m10_session_work_p99_microseconds").value_or(0)));
CHECK(sessions == 1u && p95.value_or(0) > 0 && metric(server->ops, "evo_m10_refusing") == 0u &&
metric(server->ops, "evo_m10_shed_level") == 0u &&
metric(server->ops, "evo_m10_refused_logins_total") == 0u);
std::printf("m10 snapshot report cells=%zu ticks=%llu chunks=%llu bytes=%llu\n",
shown->cells().size(),
static_cast<unsigned long long>(a->server_tick() - first_tick),