Add M10.9a load measurement, governor and overload shedding
Foundation / checks (deploy) (push) Successful in 3m20s
Foundation / checks (ecs) (push) Successful in 1m59s
Foundation / checks (client) (push) Failing after 10m0s
Foundation / checks (network) (push) Successful in 13m9s
Foundation / checks (tsan) (push) Successful in 8m1s
Foundation / checks (checks) (push) Successful in 32m2s
Foundation / checks (postgres) (push) Failing after 26m39s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
Foundation / checks (deploy) (push) Successful in 3m20s
Foundation / checks (ecs) (push) Successful in 1m59s
Foundation / checks (client) (push) Failing after 10m0s
Foundation / checks (network) (push) Successful in 13m9s
Foundation / checks (tsan) (push) Successful in 8m1s
Foundation / checks (checks) (push) Successful in 32m2s
Foundation / checks (postgres) (push) Failing after 26m39s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
ADR-0097 (M10.9 load and abuse qualification; only evolution-dev reference-worker runs qualify). evo-server measures each simulated tick's work (host step plus the M10 bridge, with the session layer's share kept separately) and feeds a portable LoadGovernor: grant logins are refused at a 2 s p95 of 15 ms or any tick debt, replication is shed in up to three levels at 20 ms or debt >= 2 (radius cap 7/7/5/3, far updates x1/x2/x2/x4, no fresh baselines, own markers unaffected), and both relax after 5 s of calm. Handoff tickets are still admitted. /metrics gains evo_m10_* gauges when the listener is enabled. Adds the missing ADR index entries for ADR-0094 to ADR-0096. Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+1
-1
@@ -96,7 +96,7 @@ target_include_directories(evo_region_lease PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/
|
||||
target_compile_features(evo_region_lease PUBLIC cxx_std_23)
|
||||
evo_owned_target(evo_region_lease)
|
||||
# Portable host status and bounded operational exposition (M7.8).
|
||||
add_library(evo_ops STATIC src/server/ops.cpp)
|
||||
add_library(evo_ops STATIC src/server/ops.cpp src/server/load_governor.cpp)
|
||||
add_library(evo::ops ALIAS evo_ops)
|
||||
target_include_directories(evo_ops PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
|
||||
target_compile_features(evo_ops PUBLIC cxx_std_23)
|
||||
|
||||
@@ -12,8 +12,12 @@
|
||||
// M10.6 (ADR-0094): sessions' probe markers move within the habitat sites' bounding cells,
|
||||
// or within --m10-area. M10.8c (ADR-0096): with --m10-handoff-key, markers are durable probe
|
||||
// records and a marker crossing into a --m10-neighbour's area is handed over with a ticket.
|
||||
// M10.9a (ADR-0097): the host's step and this bridge's work per simulated tick feed a load
|
||||
// governor; above its thresholds grant logins are refused and replication is shed.
|
||||
#include "evo/persistence/postgres.hpp"
|
||||
#include "evo/persistence/region_lease.hpp"
|
||||
#include "evo/server/ops.hpp"
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
@@ -23,6 +27,7 @@
|
||||
#include "evo/client/environment_view.hpp"
|
||||
#include "evo/network/handoff_ticket.hpp"
|
||||
#include "evo/processes/lineage_region.hpp"
|
||||
#include "evo/server/load_governor.hpp"
|
||||
#include "evo/server/pg_slot.hpp"
|
||||
#include "evo/server/postgres_action_ledger.hpp"
|
||||
#include "evo/server/postgres_control_ledger.hpp"
|
||||
@@ -183,9 +188,11 @@ template <typename Host> class M10Bridge {
|
||||
std::printf("m10 window closed activation=%s\n", window_->activation_id().c_str());
|
||||
return false;
|
||||
}
|
||||
void after_step() {
|
||||
// `host_work`: this iteration's host step, measured by the caller.
|
||||
void after_step(std::chrono::nanoseconds host_work) {
|
||||
if (!sessions_ || !accepting(wall_seconds()))
|
||||
return;
|
||||
const auto started = std::chrono::steady_clock::now();
|
||||
auto& host = *host_;
|
||||
const auto& s = host.status();
|
||||
const server::HostFacts facts{s.ready && s.phase == server::HostPhase::running, s.epoch,
|
||||
@@ -233,6 +240,16 @@ template <typename Host> class M10Bridge {
|
||||
: "refused",
|
||||
static_cast<unsigned long long>(e.session), e.account.c_str(),
|
||||
static_cast<unsigned>(e.reason));
|
||||
measure(host_work, std::chrono::steady_clock::now() - started, s);
|
||||
}
|
||||
// The listener's load for /metrics.
|
||||
void metrics(server::OpsCounters& c) const {
|
||||
c.m10_enabled = sessions_ != nullptr;
|
||||
if (!sessions_)
|
||||
return;
|
||||
c.m10_sessions = sessions_->sessions();
|
||||
c.m10_refused_logins = sessions_->refused_for_load();
|
||||
c.m10_load = governor_.state();
|
||||
}
|
||||
void shutdown() {
|
||||
if (sessions_)
|
||||
@@ -240,6 +257,40 @@ template <typename Host> class M10Bridge {
|
||||
}
|
||||
|
||||
private:
|
||||
// Busy time accumulates until the simulated tick advances by k; it then counts as k equal
|
||||
// per-tick samples (ADR-0097). A recovery's jump starts a new count instead.
|
||||
void measure(std::chrono::nanoseconds host_work, std::chrono::nanoseconds session_work,
|
||||
const server::HostStatus& s) {
|
||||
busy_ += host_work + session_work;
|
||||
session_busy_ += session_work;
|
||||
if (s.simulated_tick <= measured_tick_)
|
||||
return;
|
||||
const auto ticks = s.simulated_tick - measured_tick_;
|
||||
const bool counted = measured_tick_ != 0 && ticks <= server::load_window_ticks;
|
||||
measured_tick_ = s.simulated_tick;
|
||||
const auto per = [&](std::chrono::nanoseconds total) {
|
||||
return static_cast<std::uint64_t>(
|
||||
std::chrono::duration_cast<std::chrono::microseconds>(total).count() /
|
||||
static_cast<std::int64_t>(ticks));
|
||||
};
|
||||
const auto per_tick = per(busy_), session_per_tick = per(session_busy_);
|
||||
busy_ = session_busy_ = {};
|
||||
if (!counted)
|
||||
return;
|
||||
const auto now_us =
|
||||
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::microseconds>(
|
||||
std::chrono::steady_clock::now().time_since_epoch())
|
||||
.count());
|
||||
bool changed = false;
|
||||
for (std::uint64_t i = 0; i < ticks; ++i)
|
||||
changed = governor_.observe(per_tick, session_per_tick, s.tick_debt, now_us) || changed;
|
||||
if (!changed)
|
||||
return;
|
||||
const auto& load = governor_.state();
|
||||
sessions_->set_load(load.refusing, load.shed);
|
||||
std::printf("m10 load refusing=%d shed=%u p95_us=%llu\n", load.refusing ? 1 : 0,
|
||||
static_cast<unsigned>(load.shed), static_cast<unsigned long long>(load.p95_us));
|
||||
}
|
||||
static const processes::EnvironmentalCommunity&
|
||||
community(const processes::EnvironmentalCommunity& c) {
|
||||
return c;
|
||||
@@ -383,6 +434,9 @@ template <typename Host> class M10Bridge {
|
||||
std::unique_ptr<server::PgSlot> slot_;
|
||||
std::unique_ptr<server::PostgresControlLedger> control_;
|
||||
std::unique_ptr<server::PostgresActionLedger> actions_;
|
||||
server::LoadGovernor governor_{*server::LoadGovernor::create({})};
|
||||
std::chrono::nanoseconds busy_{}, session_busy_{};
|
||||
std::uint64_t measured_tick_{};
|
||||
std::unique_ptr<server::PostgresProbeLedger> probes_; // Handoff mode only.
|
||||
std::unique_ptr<server::PostgresHandoffLedger> handoff_; // Handoff mode only.
|
||||
std::optional<std::array<std::int64_t, 4>> area_cells_;
|
||||
@@ -407,7 +461,10 @@ template <typename Host> class M10Bridge {
|
||||
[[nodiscard]] bool accepting(std::uint64_t) const noexcept {
|
||||
return false;
|
||||
}
|
||||
void after_step() {}
|
||||
void after_step(std::chrono::nanoseconds) {}
|
||||
void metrics(server::OpsCounters& c) const {
|
||||
c.m10_enabled = false;
|
||||
}
|
||||
void shutdown() {}
|
||||
};
|
||||
#endif
|
||||
|
||||
@@ -418,6 +418,7 @@ template <typename State> int serve(Options& o) {
|
||||
std::chrono::steady_clock::now() - last_checkpoint)
|
||||
.count());
|
||||
counters.objects_available_bytes = host.objects().available_bytes().value_or(0);
|
||||
m10->metrics(counters);
|
||||
return {200, server::render_metrics(s, counters),
|
||||
"text/plain; version=0.0.4; charset=utf-8"};
|
||||
}
|
||||
@@ -516,7 +517,9 @@ template <typename State> int serve(Options& o) {
|
||||
++*synthetic_next;
|
||||
}
|
||||
}
|
||||
const auto step_started = std::chrono::steady_clock::now();
|
||||
const auto& s = host.step(scheduling_now());
|
||||
const auto host_work = std::chrono::steady_clock::now() - step_started; // ADR-0097.
|
||||
max_queued = std::max(max_queued, s.queued);
|
||||
// Checkpoint age restarts at recovery and at every confirmed commit.
|
||||
if ((!recovered && s.phase == server::HostPhase::running) ||
|
||||
@@ -563,7 +566,7 @@ template <typename State> int serve(Options& o) {
|
||||
if (ops)
|
||||
ops->serve(handler);
|
||||
sessions->step(host.state(), host.status());
|
||||
m10->after_step();
|
||||
m10->after_step(host_work);
|
||||
std::fflush(stdout);
|
||||
if (o.crash_point == "after-ack" && !acknowledged.empty() && ++ack_steps == o.crash_ordinal)
|
||||
crash_now();
|
||||
|
||||
@@ -12,7 +12,7 @@ spec:
|
||||
env:
|
||||
- {name: REGION_SESSION, value: "dev"}
|
||||
- {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"}
|
||||
- {name: REGION_SESSION_WINDOW, value: "dev-20261007T031149-a452:1791343909"}
|
||||
- {name: REGION_SESSION_WINDOW, value: "dev-20261007T102834-1d97:1791370114"}
|
||||
ports:
|
||||
# QUIC (ADR-0090): UDP only; nothing listens on TCP 7400.
|
||||
- {name: session, containerPort: 7400, protocol: UDP}
|
||||
|
||||
+5
-1
@@ -541,7 +541,11 @@ durable probes in sessions, edge crossing, ghosts (schema v13) and a two-region
|
||||
done; 10.8c-2 client switch (offers, redirects, multi-region grants) and evo-server/probe
|
||||
configuration done; 10.8d two evo-server processes on PostgreSQL with SIGKILL around each
|
||||
durable step, duplicate/stale retries and a failed destination done (schema v14 recovery
|
||||
sweep, replay guard). M10.8 complete. 10.9 planned.
|
||||
sweep, replay guard). M10.8 complete. 10.9 split (ADR-0097; only evolution-dev reference-worker
|
||||
runs qualify): 10.9a per-tick work measurement, load governor (refuse logins, then shed
|
||||
replication with hysteresis) and its metrics done; 10.9b evo-load multi-client tool, 10.9c
|
||||
abuse suite (fuzz, floods, slow peers, storms, lease loss) and 10.9d dev-cluster harness and
|
||||
reference results follow.
|
||||
|
||||
**Entry:** M9 client/session evidence and M7/M8 dev/stage durability. Read
|
||||
NETWORK_ARCHITECTURE, PERSISTENCE_ARCHITECTURE and ADR-0017. The M9 transport is
|
||||
|
||||
+10
-1
@@ -98,7 +98,16 @@ recovery gaps:
|
||||
- a ticket replayed at a restarted destination after the probe moved on is refused.
|
||||
|
||||
M10.8 is complete. Deploying needs protocol v4 and schemas v12-v14 (a rebuilt migrations
|
||||
image). Next: M10.9, load and abuse qualification.
|
||||
image). M10.9a (ADR-0097) starts load qualification. Only runs on the evolution-dev reference
|
||||
worker count as evidence. evo-server now measures each tick's work (host step plus session
|
||||
layer) and feeds a load governor:
|
||||
- above a p95 of 15 ms, or with tick debt, new grant logins are refused (`capacity`), while
|
||||
crossings are still admitted;
|
||||
- above 20 ms, replication sheds radius and far-update cadence in up to 3 levels;
|
||||
- both relax after 5 s of calm.
|
||||
|
||||
`/metrics` exposes the work percentiles (total and session layer) and the governor's
|
||||
state. Next: 10.9b, the `evo-load` multi-client tool.
|
||||
|
||||
## Previous gate - 2026-10-02
|
||||
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
# ADR-0097: M10 load and abuse qualification and overload response
|
||||
|
||||
## Status
|
||||
Accepted for M10.9 (2026-10-07). Decisions confirmed with the maintainer on 2026-10-07:
|
||||
- **Venue.** Only measurements on the reference hardware count as qualification: one RKE2
|
||||
worker per region host on evolution-dev (ADR-0089). Local runs are smoke tests and
|
||||
development references, never evidence. The gate stays open until the dev run is done.
|
||||
- **Load tool.** One multi-client tool, `evo-load`: N session clients in one process, each a
|
||||
bot, reporting per-client distributions so aggregate throughput cannot hide a starved
|
||||
client.
|
||||
- **Overload response: refuse, then shed.** Measured per-tick work drives two thresholds.
|
||||
Above the first, new logins are refused. Above the second, the interest and far-update
|
||||
cadence of existing sessions shrink step by step. Both are restored with hysteresis, so
|
||||
ticks do not fall behind for admitted sessions.
|
||||
- **Abuse coverage:** packet fuzz, floods, slow peers and connect/disconnect storms, and
|
||||
region lease loss with 20 or more sessions attached.
|
||||
|
||||
Delivery:
|
||||
- **10.9a:** the work measurement, the load governor and its effects (this text).
|
||||
- **10.9b:** `evo-load` and a local smoke test.
|
||||
- **10.9c:** the abuse suite.
|
||||
- **10.9d:** the evolution-dev run harness, then the reference results.
|
||||
|
||||
Each segment amends this ADR. Every threshold here is an engineering budget under
|
||||
`schemas/network/m10-budgets-v1.md`, not a game rule.
|
||||
|
||||
## Context
|
||||
ADR-0089 budgets:
|
||||
- per-tick region work: p99 ≤ 25 ms on the reference worker;
|
||||
- session handling: ≤ 0.5 ms per session per tick;
|
||||
- at least 20 sessions per region (32 by default);
|
||||
- overload response: shed interest and refuse new sessions before tick debt.
|
||||
|
||||
Until now nothing measured the work, and the only bound was the session count. A region
|
||||
that falls behind accrues tick debt, which delays every admitted session's outcomes and
|
||||
snapshots.
|
||||
|
||||
## Decision (10.9a)
|
||||
- **Work measurement** (evo-server loop). Busy time is the host's step plus the session
|
||||
bridge's work (session poll, replication, action carry and acknowledgements). Waits for
|
||||
input and the loop's sleeps are excluded. When the simulated tick advances by k, the busy
|
||||
time since the previous advance counts as k equal per-tick samples. The host's tick debt
|
||||
is read with every sample.
|
||||
- **Load governor** (`include/evo/server/load_governor.hpp`, portable, no clock). It keeps
|
||||
a window of the last 40 samples (2 s). It reports the window's p95, p99 and maximum, plus
|
||||
two outputs:
|
||||
- **refusing** turns on when p95 ≥ 15 ms or tick debt > 0;
|
||||
- **shed level** (0-3) rises by one when p95 ≥ 20 ms or tick debt ≥ 2, at most once per
|
||||
second.
|
||||
|
||||
Calm means p95 < 10 ms and no debt. After 5 s of calm the level drops by one. At level 0,
|
||||
5 s more of calm turns refusing off. Any non-calm sample restarts the calm timer.
|
||||
Thresholds are configurable and validated (`relax < refuse ≤ shed ≤ tick budget`).
|
||||
- **Effects** (`RegionSessions::set_load`):
|
||||
- While refusing, a Hello carrying a grant is closed with `capacity` before signature
|
||||
work. Handoff tickets are still admitted: a crossing moves an existing player between
|
||||
regions, and refusing it would strand the probe until the source aborts the transfer
|
||||
(up to 40 s).
|
||||
- The shed level limits replication (`ReplicationServer::set_shed`):
|
||||
|
||||
| Level | Radius cap | Near radius | Far updates |
|
||||
|---|---|---|---|
|
||||
| 0 | 7 | 3 | every 4 ticks |
|
||||
| 1 | 7 | 3 | every 8 ticks |
|
||||
| 2 | 5 | 3 | every 8 ticks |
|
||||
| 3 | 3 | 2 | every 16 ticks |
|
||||
|
||||
The stored interest is kept, so shedding forces no fresh baseline. Entities beyond the
|
||||
cap are removed from the client's view and return when the level drops. Each session's
|
||||
own marker is still sent every tick it changes.
|
||||
- **Exposition.** `/metrics` adds `evo_m10_*` gauges:
|
||||
- sessions;
|
||||
- refusing;
|
||||
- shed level;
|
||||
- work p95, p99 and maximum in microseconds;
|
||||
- the session layer's share, p95 and p99 (the ADR-0089 per-session budget is checked
|
||||
against it);
|
||||
- refused logins (counter).
|
||||
|
||||
evo-server prints `m10 load refusing=R shed=L p95_us=P` on every change. No labels are
|
||||
added.
|
||||
|
||||
## Consequences
|
||||
- Under overload, honest new players are refused (`capacity`, retry later) and existing
|
||||
sessions see a smaller, slower-updating view rather than delayed ticks. Crossings
|
||||
continue.
|
||||
- The thresholds are provisional until 10.9d measures the reference worker. A change is
|
||||
made by amending this ADR and the budgets table.
|
||||
- No change to the session protocol: the client already handles `capacity` and removals.
|
||||
|
||||
## 10.9a evidence (2026-10-07)
|
||||
- Unit tests:
|
||||
- the governor's thresholds, one step per second, hysteresis, the calm timer restarting
|
||||
on a non-calm window, tick debt alone, and policy validation;
|
||||
- the metrics families, present only with the listener and bounded at worst-case values;
|
||||
- replication shedding: radius caps 5 and 3 without a fresh baseline or a change of the
|
||||
stored interest, a far update every 16 ticks at level 3, and the full square back at
|
||||
level 0;
|
||||
- sessions under load: grant logins refused with `capacity` before signature work (the
|
||||
grant stays usable), tickets admitted.
|
||||
- Process test (`region_m10_session_tests.cpp`): a real evo-server with `--ops-listen`
|
||||
exposes the `evo_m10_*` gauges.
|
||||
|
||||
Local development reference (WSL, Release, one session; not evidence): work p95 about
|
||||
7 ms and p99 about 9 ms per tick. Of that, the session layer's p99 was about 4.5 ms. That
|
||||
is far above 0.5 ms per session, so 10.9b examines it with evo-load. Likely contributors
|
||||
are the per-tick snapshot source query and synchronous control renewals.
|
||||
@@ -98,6 +98,10 @@ design. No unadopted candidate is a link dependency of the proof.
|
||||
|
||||
- [ADR-0084: Engineering UI toolkit](0084-engineering-ui-toolkit.md) - Dear ImGui 1.92.9b pinned (M9.8a) and integrated as the engineering client overlay with toolkit-first input routing (M9.8c).
|
||||
- [ADR-0090: M10 QUIC transport](0090-m10-quic-transport.md) - msquic 2.6.2 with OpenSSL 3.5.9 LTS (TLS 1.3 only) from pinned sources on Windows and Linux; DATAGRAM frames for superseding state, streams for reliable actions; M9 TLS/TCP kept as dev fallback; owned impairment relay; exact CA pinning due in 10.2b.
|
||||
- [ADR-0097: M10 load and abuse qualification](0097-m10-load-and-abuse-qualification.md) - only evolution-dev reference-worker runs qualify; one multi-client evo-load tool with per-client distributions; a load governor over per-tick work (p95 window of 40 ticks) and tick debt refuses grant logins, then sheds replication radius and cadence in 3 levels with hysteresis (tickets still admitted); abuse suite of fuzz, floods, slow peers, storms and lease loss.
|
||||
- [ADR-0096: M10 probe handoff](0096-m10-probe-handoff.md) - EXPERIMENT durable probe records with one owning region and a conserved test inventory; fenced prepared/staged/committed/aborted transfers; signed single-use transfer tickets; protocol v4 offers, redirects and ghosts; recovery sweep (schema v14) and replay guard proven by a two-process fault matrix.
|
||||
- [ADR-0095: M10 interpolation and session loss](0095-m10-interpolation-and-session-loss.md) - drift-corrected client tick clock, adaptive 2-6 tick interpolation of remote markers with at most 2 ticks of extrapolation (organisms never), frozen stale view on loss and a fresh baseline on reconnect.
|
||||
- [ADR-0094: M10 probe motion experiment](0094-m10-probe-motion-experiment.md) - EXPERIMENT fixed-speed probe marker (probe-motion-v1) moved by admitted intentions, protocol v3 marker records, own-marker prediction with reconciliation; not an organism movement rule.
|
||||
- [ADR-0093: M10 snapshots and interest](0093-m10-snapshots-and-interest.md) - session protocol v2: client-declared, server-capped interest (radius <= 7) with a perception boundary; per-tick snapshot chunks as datagrams, entity deltas against the acknowledged baseline (<= 4, <= 40 ticks), nearest first within 16 KiB, distant tier every 4 ticks; client assembly with 2 baselines.
|
||||
- [ADR-0092: M10 intentions and durable outcomes](0092-m10-intentions-and-durable-outcomes.md) - realtime intentions (datagram receipts and a cumulative simulated watermark) and account-scoped actions with stable ids, receipts and final outcomes (committed_no_effect / not_committed); no biological effect until organisms exist; durable action records in the commit transaction (10.4b).
|
||||
- [ADR-0091: M10 session identity and control](0091-m10-session-identity-and-control.md) - provider-independent 128-bit AccountIds, ECDSA P-256 session grants (regions hold public keys only, signed environment byte), per-account serials where the newest grant wins, takeover of an existing session, in-session refresh, bounded attempt rates; identity adapter and issuer as a library; durable control lease follows (10.3c).
|
||||
|
||||
@@ -4904,3 +4904,47 @@ Not done:
|
||||
- deployment of protocol v4 and schemas v12-v14 (needs a rebuilt migrations image; not done
|
||||
from routine development);
|
||||
- a GUI run across two regions on evolution-dev (only one dev region exists).
|
||||
|
||||
## 2026-10-07 — M10.9a load measurement and governor (ADR-0097)
|
||||
|
||||
Implemented:
|
||||
- evo-server measures each simulated tick's work: the host step plus the M10 bridge
|
||||
(session poll, replication, action carry). Waits are excluded, and the session layer's
|
||||
share is kept separately.
|
||||
- `LoadGovernor` (`include/evo/server/load_governor.hpp`, portable): a window of 40 ticks,
|
||||
refuse at p95 ≥ 15 ms or any tick debt, shed one level per second at p95 ≥ 20 ms or
|
||||
debt ≥ 2, and relax one step per 5 s of calm (p95 < 10 ms, no debt).
|
||||
- `RegionSessions::set_load`: while refusing, grant logins are closed with `capacity`
|
||||
before signature work; handoff tickets are still admitted.
|
||||
- `ReplicationServer::set_shed`: levels 0-3 cap the radius (7/7/5/3) and the near radius
|
||||
(3/3/3/2) and stretch far updates (x1/x2/x2/x4). Stored interests are kept, and own
|
||||
markers are unaffected.
|
||||
- `/metrics`: `evo_m10_*` gauges (sessions, refusing, shed level, work and session-work
|
||||
percentiles, maximum) and a refused-logins counter, rendered only with the listener.
|
||||
evo-server prints `m10 load ...` on every change.
|
||||
- The ADR index gains the missing entries for ADR-0094 to ADR-0096.
|
||||
|
||||
Tests:
|
||||
- `ops-contracts`: the governor state machine (thresholds, step interval, hysteresis, calm
|
||||
restart, debt, policy validation) and the M10 metric families, including worst-case size.
|
||||
- `network`: replication shedding (caps, no fresh baseline, 16-tick far cadence at level
|
||||
3, recovery at level 0).
|
||||
- `quic-sessions`: grant logins refused under load (the grant is not consumed), a ticket
|
||||
login admitted, and logins accepted after relaxing.
|
||||
- Postgres `region_m10_session_tests`: a real evo-server's `/metrics` shows one session,
|
||||
measured work, not refusing, shed 0.
|
||||
|
||||
Executed:
|
||||
- **Linux GCC 14:** `postgres-check.sh` with QUIC, all suites; presets `quic` (ops,
|
||||
network, QUIC sessions) 3/3. The local reference (Release, one session) measured work
|
||||
p95 7.1 ms and p99 8.8 ms, with the session layer at p99 4.5 ms. This is a development
|
||||
reference only, flagged for 10.9b.
|
||||
- **Linux Clang 19:** clang-tidy 19 on the governor, ops, replication, sessions, evo-server
|
||||
`main.cpp` with the bridge, and the tests (one finding fixed); clang-format 19.
|
||||
- **Windows MSVC 19.44:** presets `debug` 60/60 (ops-contracts included), `quic` 4/4 and
|
||||
`client-network` 14/14.
|
||||
|
||||
Not done:
|
||||
- qualification on the evolution-dev reference worker (10.9d; needs v4 and v12-v14
|
||||
deployed);
|
||||
- load generation (10.9b) and the abuse suite (10.9c).
|
||||
|
||||
@@ -89,6 +89,14 @@ class ReplicationServer {
|
||||
tick(std::uint64_t tick, const std::function<ReplicationSource(const SessionInterest&)>& source,
|
||||
const Perceives& perceives, const std::function<std::size_t(std::uint64_t)>& payload);
|
||||
[[nodiscard]] ReplicationStats stats() const;
|
||||
// M10.9a overload shedding (ADR-0097), level 0-3 (higher is clamped to 3): caps the
|
||||
// radius encoded (7, 7, 5, 3), the near radius (3, 3, 3, 2) and stretches far updates
|
||||
// (x1, x2, x2, x4). Stored interests are kept (no fresh baseline); entities beyond the
|
||||
// cap are removed from views and return when the level drops. Own markers are unaffected.
|
||||
void set_shed(std::uint8_t level) noexcept;
|
||||
[[nodiscard]] std::uint8_t shed() const noexcept {
|
||||
return shed_;
|
||||
}
|
||||
|
||||
private:
|
||||
// (kind, x | id, y, z): environment by voxel, organisms by id.
|
||||
@@ -109,6 +117,7 @@ class ReplicationServer {
|
||||
};
|
||||
explicit ReplicationServer(ReplicationLimits limits) : limits_{limits} {}
|
||||
ReplicationLimits limits_;
|
||||
std::uint8_t shed_{};
|
||||
std::vector<Session> sessions_;
|
||||
ReplicationStats stats_;
|
||||
};
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
#pragma once
|
||||
#include <array>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <expected>
|
||||
|
||||
// M10.9a overload response (ADR-0097): a region's per-tick work and tick debt decide whether
|
||||
// new logins are refused and how far replication is shed. Portable: no clocks (callers pass a
|
||||
// monotonic time in microseconds), no transport. Thresholds are engineering budgets
|
||||
// (schemas/network/m10-budgets-v1.md), not game rules.
|
||||
namespace evo::server {
|
||||
struct LoadPolicy {
|
||||
std::uint64_t tick_budget_us{25'000}; // p99 per-tick region work budget (ADR-0089).
|
||||
std::uint64_t refuse_us{15'000}; // Window p95 at or above: refuse new logins.
|
||||
std::uint64_t shed_us{20'000}; // Window p95 at or above: shed one more level.
|
||||
std::uint64_t relax_us{10'000}; // Window p95 below (and no debt): calm.
|
||||
std::uint64_t step_us{1'000'000}; // At most one shed step per interval.
|
||||
std::uint64_t settle_us{5'000'000}; // Calm this long relaxes one step.
|
||||
std::uint8_t max_shed{3};
|
||||
};
|
||||
inline constexpr std::size_t load_window_ticks = 40; // 2 s of 50 ms ticks.
|
||||
struct LoadState {
|
||||
bool refusing{};
|
||||
std::uint8_t shed{};
|
||||
std::uint64_t p95_us{}, p99_us{}, max_us{}; // Over the window (0 before any sample).
|
||||
std::size_t samples{}; // In the window.
|
||||
// The session layer's share of the same ticks (ADR-0089 budgets it per session).
|
||||
std::uint64_t session_p95_us{}, session_p99_us{};
|
||||
friend bool operator==(const LoadState&, const LoadState&) = default;
|
||||
};
|
||||
enum class LoadPolicyError : std::uint8_t { invalid };
|
||||
|
||||
class LoadGovernor {
|
||||
public:
|
||||
[[nodiscard]] static std::expected<LoadGovernor, LoadPolicyError> create(LoadPolicy policy);
|
||||
// One simulated tick's region work (of which `session_us` was the session layer's) and the
|
||||
// host's tick debt, at monotonic `now_us`. Returns true when `refusing` or `shed` changed.
|
||||
bool observe(std::uint64_t work_us, std::uint64_t session_us, std::uint64_t tick_debt,
|
||||
std::uint64_t now_us);
|
||||
[[nodiscard]] const LoadState& state() const noexcept {
|
||||
return state_;
|
||||
}
|
||||
[[nodiscard]] const LoadPolicy& policy() const noexcept {
|
||||
return policy_;
|
||||
}
|
||||
|
||||
private:
|
||||
explicit LoadGovernor(LoadPolicy policy) : policy_{policy} {}
|
||||
LoadPolicy policy_;
|
||||
LoadState state_;
|
||||
std::array<std::uint64_t, load_window_ticks> window_{}, session_window_{};
|
||||
std::size_t next_{};
|
||||
bool stepped_{}, calm_{};
|
||||
std::uint64_t last_step_us_{}, calm_since_us_{};
|
||||
};
|
||||
} // namespace evo::server
|
||||
@@ -1,4 +1,5 @@
|
||||
#pragma once
|
||||
#include "evo/server/load_governor.hpp"
|
||||
#include <array>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
@@ -47,6 +48,11 @@ struct OpsCounters {
|
||||
// M8.5: object volume free space and the store's reserve (ADR-0070), so capacity
|
||||
// is alerted on before new objects are refused.
|
||||
std::uint64_t objects_available_bytes{}, objects_reserve_bytes{};
|
||||
// M10.9a (ADR-0097): the M10 session listener's load governor, rendered only when the
|
||||
// listener is enabled.
|
||||
bool m10_enabled{};
|
||||
std::uint64_t m10_sessions{}, m10_refused_logins{};
|
||||
LoadState m10_load;
|
||||
};
|
||||
|
||||
[[nodiscard]] const char* phase_name(HostPhase phase) noexcept;
|
||||
|
||||
@@ -136,6 +136,12 @@ class RegionSessions {
|
||||
const std::function<network::ReplicationSource(const network::SessionInterest&)>& source,
|
||||
const network::Perceives& perceives = {});
|
||||
[[nodiscard]] network::ReplicationStats replication_stats() const;
|
||||
// M10.9a (ADR-0097): the host's load governor output. While `refuse_logins`, a Hello
|
||||
// carrying a grant is closed with `capacity` before signature work (handoff tickets are
|
||||
// still admitted); `shed` (0-3) limits replication (ReplicationServer::set_shed).
|
||||
void set_load(bool refuse_logins, std::uint8_t shed);
|
||||
// Grant logins refused for load since creation.
|
||||
[[nodiscard]] std::uint64_t refused_for_load() const noexcept;
|
||||
// Where probe markers live and spawn (ADR-0094); sessions already established receive a
|
||||
// marker. False for an invalid area. With neighbours (ADR-0096), markers move within the
|
||||
// bounding box of this area and the neighbours' areas, and leaving this area hands off.
|
||||
|
||||
@@ -32,10 +32,10 @@ in real time; changing it requires a versioned rules migration (ADR-0089).
|
||||
| Client event queue | 1,024 events | budget | client session |
|
||||
| Heartbeat / liveness / handshake | 1 s / 5 s / 5 s | budget | 10.2, 10.7 |
|
||||
| Sessions per region | >= 20 required, 32 default | budget | 10.9 |
|
||||
| Per-tick region work | p99 <= 25 ms on the reference worker | budget | 10.9 |
|
||||
| Session handling | <= 0.5 ms per session per tick | budget | 10.9 |
|
||||
| Per-tick region work | p99 <= 25 ms on the reference worker | budget | 10.9 (measured: `evo_m10_tick_work_*`, ADR-0097) |
|
||||
| Session handling | <= 0.5 ms per session per tick | budget | 10.9 (measured: `evo_m10_session_work_*`, ADR-0097) |
|
||||
| Host input queue | 64 inputs, 64 KiB | budget | region host |
|
||||
| Overload response | shed interest, refuse new sessions before tick debt | budget | 10.9 |
|
||||
| Overload response | shed interest, refuse new sessions before tick debt | budget | 10.9a load governor: refuse at p95 >= 15 ms or debt, shed at p95 >= 20 ms or debt >= 2, relax below 10 ms after 5 s (ADR-0097; provisional until 10.9d) |
|
||||
|
||||
Determinism: authoritative state is exact given the checkpoint and committed per-tick
|
||||
inputs; timing, delivery, prediction, interpolation and presentation are tolerant
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#include "evo/network/replication.hpp"
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
namespace evo::network {
|
||||
@@ -224,10 +225,20 @@ ReplicationServer::tick(std::uint64_t tick,
|
||||
return this->tick(tick, source, perceives, [payload](std::uint64_t) { return payload; });
|
||||
}
|
||||
|
||||
void ReplicationServer::set_shed(std::uint8_t level) noexcept {
|
||||
shed_ = std::min<std::uint8_t>(level, 3);
|
||||
}
|
||||
|
||||
std::vector<std::pair<std::uint64_t, std::vector<SnapshotChunk>>> ReplicationServer::tick(
|
||||
std::uint64_t tick, const std::function<ReplicationSource(const SessionInterest&)>& source,
|
||||
const Perceives& perceives, const std::function<std::size_t(std::uint64_t)>& payload_of) {
|
||||
std::vector<std::pair<std::uint64_t, std::vector<SnapshotChunk>>> out;
|
||||
// Shedding (ADR-0097): radius cap, near radius and far-update multiplier per level.
|
||||
constexpr std::array<std::uint16_t, 4> shed_radius{7, 7, 5, 3};
|
||||
constexpr std::array<std::uint16_t, 4> shed_near{3, 3, 3, 2};
|
||||
constexpr std::array<std::uint64_t, 4> shed_far{1, 2, 2, 4};
|
||||
const auto near_radius = std::min(limits_.near_radius, shed_near.at(shed_));
|
||||
const auto far_interval = limits_.far_interval * shed_far.at(shed_);
|
||||
const auto overhead = session_header_bytes + snapshot_chunk_header_bytes;
|
||||
if (tick == 0)
|
||||
return out;
|
||||
@@ -239,12 +250,15 @@ std::vector<std::pair<std::uint64_t, std::vector<SnapshotChunk>>> ReplicationSer
|
||||
if (payload <= overhead)
|
||||
continue; // No datagram channel yet (or a path too small): nothing to send.
|
||||
const auto capacity = payload - overhead; // Record bytes per chunk.
|
||||
const auto& in = *s.interest;
|
||||
auto cached = std::ranges::find_if(sources, [&](const auto& e) { return e.first == in; });
|
||||
const auto& wanted = *s.interest;
|
||||
auto cached =
|
||||
std::ranges::find_if(sources, [&](const auto& e) { return e.first == wanted; });
|
||||
if (cached == sources.end()) {
|
||||
sources.emplace_back(in, source(in));
|
||||
sources.emplace_back(wanted, source(wanted));
|
||||
cached = std::prev(sources.end());
|
||||
}
|
||||
auto in = wanted; // Encoded within the shed cap; the stored interest is unchanged.
|
||||
in.radius = std::min(in.radius, shed_radius.at(shed_));
|
||||
const auto& src = cached->second;
|
||||
// Spatial filter, then perception: nothing outside either is ever encoded.
|
||||
std::map<Key, std::pair<std::vector<std::uint8_t>, std::uint64_t>> truth;
|
||||
@@ -299,8 +313,8 @@ std::vector<std::pair<std::uint64_t, std::vector<SnapshotChunk>>> ReplicationSer
|
||||
const auto known = base_state.find(key);
|
||||
if (known != base_state.end() && known->second.record == value.first)
|
||||
continue; // The client already holds this value.
|
||||
const bool due = known == base_state.end() || value.second <= limits_.near_radius ||
|
||||
tick - known->second.updated >= limits_.far_interval;
|
||||
const bool due = known == base_state.end() || value.second <= near_radius ||
|
||||
tick - known->second.updated >= far_interval;
|
||||
if (!due) {
|
||||
++deferred;
|
||||
continue;
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
#include "evo/server/load_governor.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <utility>
|
||||
|
||||
namespace evo::server {
|
||||
std::expected<LoadGovernor, LoadPolicyError> LoadGovernor::create(LoadPolicy p) {
|
||||
if (p.relax_us == 0 || p.relax_us >= p.refuse_us || p.refuse_us > p.shed_us ||
|
||||
p.shed_us > p.tick_budget_us || p.step_us == 0 || p.settle_us == 0 || p.max_shed > 3)
|
||||
return std::unexpected(LoadPolicyError::invalid);
|
||||
return LoadGovernor{p};
|
||||
}
|
||||
|
||||
namespace {
|
||||
using Window = std::array<std::uint64_t, load_window_ticks>;
|
||||
// The first `n` samples sorted; nearest rank: the smallest sample with at least `percent` of
|
||||
// the window at or below it.
|
||||
Window sorted_of(const Window& w, std::size_t n) {
|
||||
Window sorted{};
|
||||
std::copy_n(w.begin(), n, sorted.begin());
|
||||
std::sort(sorted.begin(), sorted.begin() + static_cast<std::ptrdiff_t>(n));
|
||||
return sorted;
|
||||
}
|
||||
std::uint64_t rank(const Window& sorted, std::size_t n, std::size_t percent) {
|
||||
return sorted.at((((n * percent) + 99) / 100) - 1);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
bool LoadGovernor::observe(std::uint64_t work_us, std::uint64_t session_us, std::uint64_t tick_debt,
|
||||
std::uint64_t now_us) {
|
||||
window_.at(next_) = work_us;
|
||||
session_window_.at(next_) = std::min(session_us, work_us);
|
||||
next_ = (next_ + 1) % window_.size();
|
||||
state_.samples = std::min(state_.samples + 1, window_.size());
|
||||
const auto n = state_.samples;
|
||||
const auto sorted = sorted_of(window_, n);
|
||||
state_.p95_us = rank(sorted, n, 95);
|
||||
state_.p99_us = rank(sorted, n, 99);
|
||||
state_.max_us = sorted.at(n - 1);
|
||||
const auto sessions = sorted_of(session_window_, n);
|
||||
state_.session_p95_us = rank(sessions, n, 95);
|
||||
state_.session_p99_us = rank(sessions, n, 99);
|
||||
|
||||
const auto before = std::pair{state_.refusing, state_.shed};
|
||||
const bool hot = state_.p95_us >= policy_.refuse_us || tick_debt > 0;
|
||||
const bool overloaded = state_.p95_us >= policy_.shed_us || tick_debt >= 2;
|
||||
if (hot)
|
||||
state_.refusing = true;
|
||||
if (overloaded && state_.shed < policy_.max_shed &&
|
||||
(!stepped_ || now_us - last_step_us_ >= policy_.step_us)) {
|
||||
++state_.shed;
|
||||
stepped_ = true;
|
||||
last_step_us_ = now_us;
|
||||
}
|
||||
if (state_.p95_us >= policy_.relax_us || tick_debt > 0) {
|
||||
calm_ = false;
|
||||
} else if (!calm_) {
|
||||
calm_ = true;
|
||||
calm_since_us_ = now_us;
|
||||
} else if (now_us - calm_since_us_ >= policy_.settle_us) {
|
||||
if (state_.shed > 0)
|
||||
--state_.shed; // One level per settle interval, then logins.
|
||||
else
|
||||
state_.refusing = false;
|
||||
calm_since_us_ = now_us;
|
||||
}
|
||||
return before != std::pair{state_.refusing, state_.shed};
|
||||
}
|
||||
} // namespace evo::server
|
||||
@@ -103,6 +103,24 @@ std::string render_metrics(const HostStatus& s, const OpsCounters& c) {
|
||||
.append("\"} ")
|
||||
.append(std::to_string(c.dropped[i]))
|
||||
.append("\n");
|
||||
if (c.m10_enabled) {
|
||||
metric(out, "evo_m10_sessions", "gauge", "Open M10 sessions", c.m10_sessions);
|
||||
metric(out, "evo_m10_refusing", "gauge", "1 while new logins are refused for load",
|
||||
c.m10_load.refusing);
|
||||
metric(out, "evo_m10_shed_level", "gauge", "Replication shed level (0-3)", c.m10_load.shed);
|
||||
metric(out, "evo_m10_tick_work_p95_microseconds", "gauge",
|
||||
"p95 per-tick region work over the last 40 ticks", c.m10_load.p95_us);
|
||||
metric(out, "evo_m10_tick_work_p99_microseconds", "gauge",
|
||||
"p99 per-tick region work over the last 40 ticks", c.m10_load.p99_us);
|
||||
metric(out, "evo_m10_tick_work_max_microseconds", "gauge",
|
||||
"Largest per-tick region work over the last 40 ticks", c.m10_load.max_us);
|
||||
metric(out, "evo_m10_session_work_p95_microseconds", "gauge",
|
||||
"p95 per-tick session-layer work over the last 40 ticks", c.m10_load.session_p95_us);
|
||||
metric(out, "evo_m10_session_work_p99_microseconds", "gauge",
|
||||
"p99 per-tick session-layer work over the last 40 ticks", c.m10_load.session_p99_us);
|
||||
metric(out, "evo_m10_refused_logins_total", "counter", "Logins refused for load",
|
||||
c.m10_refused_logins);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
} // namespace evo::server
|
||||
|
||||
@@ -108,6 +108,8 @@ struct RegionSessions::Impl {
|
||||
std::vector<Outgoing> outgoing; // Handed off, not yet known committed.
|
||||
Clock::time_point next_ghosts{}, next_overdue{};
|
||||
std::optional<network::SessionInterest> default_interest;
|
||||
bool refuse_logins{}; // Load governor (ADR-0097).
|
||||
std::uint64_t refused_load{}; // Grant logins refused for load.
|
||||
HostFacts facts;
|
||||
std::vector<Connection> connections;
|
||||
std::vector<RegionSessionEvent> events;
|
||||
@@ -216,6 +218,10 @@ struct RegionSessions::Impl {
|
||||
return close(c, SessionCloseReason::rate_limited, now);
|
||||
if (network::looks_like_handoff_ticket(m.grant))
|
||||
return ticket_hello(c, m, now_seconds, now);
|
||||
if (refuse_logins) { // Overload (ADR-0097): before signature work.
|
||||
++refused_load;
|
||||
return close(c, SessionCloseReason::capacity, now);
|
||||
}
|
||||
const auto grant = verifier.verify(m.grant);
|
||||
if (!grant)
|
||||
return close(c, reason_for(grant.error()), now);
|
||||
@@ -774,6 +780,15 @@ std::optional<std::uint64_t> RegionSessions::holder(const network::AccountId& ac
|
||||
return impl_->control.holder(account);
|
||||
}
|
||||
|
||||
void RegionSessions::set_load(bool refuse_logins, std::uint8_t shed) {
|
||||
impl_->refuse_logins = refuse_logins;
|
||||
impl_->replication.set_shed(shed);
|
||||
}
|
||||
|
||||
std::uint64_t RegionSessions::refused_for_load() const noexcept {
|
||||
return impl_->refused_load;
|
||||
}
|
||||
|
||||
std::size_t RegionSessions::sessions() const noexcept {
|
||||
return impl_->control.sessions();
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#include "network_test.hpp"
|
||||
#include <algorithm>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
@@ -385,9 +386,71 @@ void dense_report() {
|
||||
}
|
||||
} // namespace
|
||||
|
||||
// M10.9a overload shedding (ADR-0097): the encoded radius shrinks and far updates slow without a
|
||||
// fresh baseline or a change of the stored interest; the view grows back when the level drops.
|
||||
void shedding() {
|
||||
auto server = *ReplicationServer::create(limits());
|
||||
World world;
|
||||
CHECK(server.open(1, 5) && server.set_interest(1, {0, 0, slice, 7}));
|
||||
SnapshotAssembler client{identity, 5};
|
||||
const auto source = [&](const SessionInterest&) { return world.source(); };
|
||||
std::uint64_t tick = 0;
|
||||
std::uint64_t baseline = 0;
|
||||
const auto step = [&] {
|
||||
const auto out = server.tick(++tick, source, {});
|
||||
const auto& chunks = for_session(out, 1);
|
||||
baseline = chunks.front().baseline;
|
||||
for (const auto& a : deliver(client, chunks))
|
||||
server.acknowledge(1, a);
|
||||
return client.latest()->environment.size();
|
||||
};
|
||||
while (step() < 225 && tick < 10) {
|
||||
}
|
||||
CHECK(client.latest()->environment.size() == 225);
|
||||
const auto square = [&](std::int64_t r) {
|
||||
return std::ranges::all_of(client.latest()->environment, [&](const auto& c) {
|
||||
return std::max(std::abs(c.voxel.x), std::abs(c.voxel.y)) <= r;
|
||||
});
|
||||
};
|
||||
server.set_shed(2);
|
||||
CHECK(server.shed() == 2 && step() == 121 && square(5) && baseline != 0);
|
||||
CHECK(server.interest(1).value_or(SessionInterest{}).radius == 7); // The stored interest.
|
||||
server.set_shed(9);
|
||||
CHECK(server.shed() == 3 && step() == 49 && square(3) && baseline != 0);
|
||||
// Level 3: near radius 2, far updates every 16 ticks. A cell at distance 3 changed twice
|
||||
// arrives at once (last sent long ago), then not again for 16 ticks.
|
||||
const auto at = [&](std::int64_t x, std::int64_t y) {
|
||||
return std::ranges::find_if(client.latest()->environment,
|
||||
[&](const auto& c) { return c.voxel.x == x && c.voxel.y == y; })
|
||||
->quantity[0];
|
||||
};
|
||||
for (int i = 0; i < 16; ++i)
|
||||
step();
|
||||
world.quantity[{3, 0}] = 41;
|
||||
world.quantity[{2, 0}] = 42;
|
||||
step();
|
||||
CHECK(at(3, 0) == 41 && at(2, 0) == 42);
|
||||
world.quantity[{3, 0}] = 51;
|
||||
world.quantity[{2, 0}] = 52;
|
||||
step();
|
||||
CHECK(at(2, 0) == 52 && at(3, 0) == 41); // Near: every tick. Far: deferred.
|
||||
int waited = 1;
|
||||
while (at(3, 0) != 51 && waited < 40) {
|
||||
step();
|
||||
++waited;
|
||||
}
|
||||
CHECK(waited == 16);
|
||||
// Back to level 0: the full square returns, still from acknowledged baselines.
|
||||
server.set_shed(0);
|
||||
for (int i = 0; i < 6 && client.latest()->environment.size() < 225; ++i)
|
||||
step();
|
||||
CHECK(client.latest()->environment.size() == 225 && baseline != 0);
|
||||
}
|
||||
|
||||
void replication_tests() {
|
||||
codec_v2();
|
||||
convergence_and_deltas();
|
||||
shedding();
|
||||
concealment();
|
||||
client_robustness();
|
||||
dense_report();
|
||||
|
||||
+96
-1
@@ -52,6 +52,17 @@ std::vector<Sample> parse(const std::string& text) {
|
||||
return out;
|
||||
}
|
||||
|
||||
// Every host counter at its largest value (the M10 section stays disabled).
|
||||
OpsCounters worst_counters() {
|
||||
constexpr auto most = std::numeric_limits<std::uint64_t>::max();
|
||||
OpsCounters c;
|
||||
c.queue_capacity_inputs = c.queue_capacity_bytes = most;
|
||||
c.checkpoint_age_ms = c.acks = c.outcomes = most;
|
||||
c.dropped = {most, most, most, most};
|
||||
c.objects_available_bytes = c.objects_reserve_bytes = most;
|
||||
return c;
|
||||
}
|
||||
|
||||
void exposition_is_bounded_and_well_formed() {
|
||||
HostStatus s;
|
||||
s.phase = HostPhase::running;
|
||||
@@ -93,7 +104,7 @@ void exposition_is_bounded_and_well_formed() {
|
||||
big.max_tick_debt = big.commits = big.store_failures = big.unknown_outcomes = most;
|
||||
big.fences = big.lease_remaining_ms = most;
|
||||
big.simulated_tick = 1; // Lag saturates at zero rather than wrapping.
|
||||
OpsCounters bigc{most, most, most, most, most, {most, most, most, most}, most, most};
|
||||
const auto bigc = worst_counters();
|
||||
const auto large = render_metrics(big, bigc);
|
||||
CHECK(large.size() <= max_metrics_bytes);
|
||||
CHECK(large.find("evo_region_commit_lag_ticks 0\n") != std::string::npos);
|
||||
@@ -105,11 +116,95 @@ void exposition_is_bounded_and_well_formed() {
|
||||
CHECK(std::string(unacknowledged_name(static_cast<Unacknowledged>(i))) != "unknown");
|
||||
std::printf("ops_metrics_bytes typical=%zu worst=%zu\n", text.size(), large.size());
|
||||
}
|
||||
|
||||
// M10.9a (ADR-0097): the listener's governor families appear only when it is enabled.
|
||||
void m10_load_metrics() {
|
||||
HostStatus s;
|
||||
OpsCounters c;
|
||||
const auto without = parse(render_metrics(s, c)).size();
|
||||
c.m10_enabled = true;
|
||||
c.m10_sessions = 21;
|
||||
c.m10_refused_logins = 3;
|
||||
c.m10_load = {true, 2, 17'000, 19'500, 23'000, 40, 900, 1'100};
|
||||
const auto text = render_metrics(s, c);
|
||||
CHECK(parse(text).size() == without + 9);
|
||||
CHECK(text.find("evo_m10_session_work_p99_microseconds 1100\n") != std::string::npos);
|
||||
CHECK(text.find("evo_m10_sessions 21\n") != std::string::npos);
|
||||
CHECK(text.find("evo_m10_refusing 1\n") != std::string::npos);
|
||||
CHECK(text.find("evo_m10_shed_level 2\n") != std::string::npos);
|
||||
CHECK(text.find("evo_m10_tick_work_p99_microseconds 19500\n") != std::string::npos);
|
||||
CHECK(text.find("evo_m10_refused_logins_total 3\n") != std::string::npos);
|
||||
constexpr auto most = std::numeric_limits<std::uint64_t>::max();
|
||||
HostStatus big;
|
||||
big.epoch = big.committed_tick = big.committed_sequence = big.tick_debt = most;
|
||||
big.max_tick_debt = big.commits = big.store_failures = big.unknown_outcomes = most;
|
||||
big.fences = big.lease_remaining_ms = most;
|
||||
auto bigc = worst_counters();
|
||||
bigc.m10_enabled = true;
|
||||
bigc.m10_sessions = bigc.m10_refused_logins = most;
|
||||
bigc.m10_load = {true, 3, most, most, most, 40, most, most};
|
||||
CHECK(render_metrics(big, bigc).size() <= max_metrics_bytes);
|
||||
}
|
||||
|
||||
// M10.9a (ADR-0097): refuse, then shed one level per second; relax one level per 5 s of
|
||||
// calm, then accept again. Samples are 50 ms ticks.
|
||||
void load_governor() {
|
||||
CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 15'000, 1'000'000, 5'000'000, 3}));
|
||||
CHECK(!LoadGovernor::create({25'000, 21'000, 20'000, 10'000, 1'000'000, 5'000'000, 3}));
|
||||
CHECK(!LoadGovernor::create({19'000, 15'000, 20'000, 10'000, 1'000'000, 5'000'000, 3}));
|
||||
CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 10'000, 0, 5'000'000, 3}));
|
||||
CHECK(!LoadGovernor::create({25'000, 15'000, 20'000, 10'000, 1'000'000, 5'000'000, 4}));
|
||||
auto made = LoadGovernor::create({});
|
||||
CHECK(made.has_value());
|
||||
auto& g = *made;
|
||||
std::uint64_t now = 0;
|
||||
const auto run = [&](std::size_t ticks, std::uint64_t work, std::uint64_t debt = 0) {
|
||||
for (std::size_t i = 0; i < ticks; ++i) {
|
||||
now += 50'000;
|
||||
g.observe(work, work / 4, debt, now);
|
||||
}
|
||||
return g.state();
|
||||
};
|
||||
// Quiet: nothing changes; the window reports its percentiles.
|
||||
CHECK(run(60, 5'000) == LoadState{false, 0, 5'000, 5'000, 5'000, 40, 1'250, 1'250});
|
||||
// A single slow tick is not the p95 of 40.
|
||||
CHECK(run(1, 30'000).max_us == 30'000 && !g.state().refusing && g.state().p95_us == 5'000);
|
||||
// Sustained 16 ms: refuse, no shedding.
|
||||
auto st = run(40, 16'000);
|
||||
CHECK(st.refusing && st.shed == 0 && st.p95_us == 16'000);
|
||||
// 22 ms: one shed level per second, never past 3.
|
||||
// The p95 (nearest rank, 38th of 40) reaches the threshold with the third slow tick.
|
||||
CHECK(run(2, 22'000).shed == 0 && run(1, 22'000).shed == 1);
|
||||
CHECK(run(18, 22'000).shed == 1 && run(2, 22'000).shed == 2);
|
||||
CHECK(run(100, 22'000).shed == 3);
|
||||
// Calm only once the window's p95 is below 10 ms; then one level per 5 s.
|
||||
st = run(37, 4'000);
|
||||
CHECK(st.shed == 3 && st.p95_us == 22'000);
|
||||
CHECK(run(1, 4'000).p95_us == 4'000); // Calm starts now.
|
||||
CHECK(run(99, 4'000).shed == 3 && run(1, 4'000).shed == 2);
|
||||
// A non-calm window (p95 >= 10 ms, still under the refuse threshold) restarts the timer.
|
||||
CHECK(run(50, 4'000).shed == 2 && run(3, 12'000).p95_us == 12'000);
|
||||
CHECK(run(37, 4'000).p95_us == 12'000 && run(1, 4'000).p95_us == 4'000);
|
||||
CHECK(run(99, 4'000).shed == 2 && run(1, 4'000).shed == 1);
|
||||
// Down to 0 after another 5 s, then logins after 5 s more.
|
||||
CHECK(run(100, 4'000).shed == 0 && g.state().refusing);
|
||||
CHECK(run(99, 4'000).refusing && !run(1, 4'000).refusing);
|
||||
// Tick debt alone: refuse at 1, shed at 2, even with light work.
|
||||
CHECK(run(1, 1'000, 1).refusing && g.state().shed == 0);
|
||||
CHECK(run(1, 1'000, 2).shed == 1);
|
||||
CHECK(!g.observe(1'000, 250, 2, now + 10)); // Within the step interval: no change.
|
||||
CHECK(g.state().session_p95_us == 1'000); // The window is mostly 4 ms ticks.
|
||||
// Debt gone and calm (from the next sample): recovers.
|
||||
CHECK(run(100, 1'000).shed == 1 && run(1, 1'000).shed == 0);
|
||||
CHECK(run(99, 1'000).refusing && !run(1, 1'000).refusing);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
try {
|
||||
exposition_is_bounded_and_well_formed();
|
||||
m10_load_metrics();
|
||||
load_governor();
|
||||
std::puts("ops contracts passed");
|
||||
return 0;
|
||||
} catch (const std::exception& e) {
|
||||
|
||||
@@ -627,6 +627,19 @@ void ticket_login(const Files& f) {
|
||||
handoff.error = HandoffLedgerError::unavailable;
|
||||
CHECK(refused_with(h, ticket(account(3), 0x65), Reason::capacity));
|
||||
CHECK(a->state() == RegionSessionState::established && handoff.admitted.size() == 1);
|
||||
// M10.9a (ADR-0097): under load, grant logins are refused before signature work while
|
||||
// tickets (crossings) are still admitted; the refused grant stays usable.
|
||||
handoff.error.reset();
|
||||
h.server->set_load(true, 3);
|
||||
CHECK(refused_with(h, h.issuer.grant(account(5), 1), Reason::capacity));
|
||||
CHECK(refused_with(h, h.issuer.grant(account(5), 1), Reason::capacity));
|
||||
CHECK(h.server->refused_for_load() == 2 && a->state() == RegionSessionState::established);
|
||||
auto crossed = h.connect(ticket(account(6), 0x66));
|
||||
CHECK(h.settled(*crossed) && crossed->state() == RegionSessionState::established);
|
||||
h.server->set_load(false, 0);
|
||||
auto later = h.connect(h.issuer.grant(account(5), 1));
|
||||
CHECK(h.settled(*later) && later->state() == RegionSessionState::established &&
|
||||
h.server->refused_for_load() == 2);
|
||||
}
|
||||
|
||||
// M10.8c (ADR-0096): two regions with adjacent areas in one process, over the fake world
|
||||
|
||||
@@ -9,14 +9,17 @@
|
||||
#include "evo/network/session_grant.hpp"
|
||||
#include "fixtures/server_process.hpp"
|
||||
#include <algorithm>
|
||||
#include <arpa/inet.h>
|
||||
#include <array>
|
||||
#include <chrono>
|
||||
#include <fcntl.h>
|
||||
#include <functional>
|
||||
#include <netinet/in.h>
|
||||
#include <optional>
|
||||
#include <source_location>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <sys/socket.h>
|
||||
#include <thread>
|
||||
#include <unistd.h>
|
||||
|
||||
@@ -69,6 +72,34 @@ std::string hex(const AccountId& a) {
|
||||
return out;
|
||||
}
|
||||
|
||||
// GET /metrics from the ops listener; the value of an unlabelled series (ADR-0097 adds the
|
||||
// evo_m10_* families when the listener is enabled).
|
||||
std::optional<std::uint64_t> metric(std::uint16_t port, const std::string& series) {
|
||||
const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0);
|
||||
expect(fd >= 0, "socket");
|
||||
sockaddr_in a{};
|
||||
a.sin_family = AF_INET;
|
||||
a.sin_port = htons(port);
|
||||
a.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
|
||||
timeval timeout{3, 0};
|
||||
::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout);
|
||||
std::string raw;
|
||||
if (::connect(fd, reinterpret_cast<const sockaddr*>(&a), sizeof a) == 0) {
|
||||
const std::string request = "GET /metrics HTTP/1.1\r\nHost: test\r\n\r\n";
|
||||
if (::send(fd, request.data(), request.size(), MSG_NOSIGNAL) ==
|
||||
static_cast<ssize_t>(request.size())) {
|
||||
std::array<char, 4096> chunk{};
|
||||
for (ssize_t n = 0; (n = ::recv(fd, chunk.data(), chunk.size(), 0)) > 0;)
|
||||
raw.append(chunk.data(), static_cast<std::size_t>(n));
|
||||
}
|
||||
}
|
||||
::close(fd);
|
||||
const auto at = raw.find("\n" + series + " ");
|
||||
if (at == std::string::npos)
|
||||
return std::nullopt;
|
||||
return std::stoull(raw.substr(at + series.size() + 2));
|
||||
}
|
||||
|
||||
// One region's credentials, issuer key and server process arguments.
|
||||
struct Setup {
|
||||
ServerRegion region;
|
||||
@@ -95,22 +126,23 @@ struct Setup {
|
||||
}
|
||||
[[nodiscard]] std::vector<std::string> args() const {
|
||||
return server_args(
|
||||
region, {"--synthetic-inputs", "development-v1", "--m10-listen", "127.0.0.1:0",
|
||||
"--m10-cert", (region.root / "server.pem").string(), "--m10-key",
|
||||
(region.root / "server.key").string(), "--m10-issuer",
|
||||
(region.root / "issuer.pub").string(), "--m10-environment", "dev",
|
||||
"--m10-max-grant-lifetime", "900", "--session-window",
|
||||
"test-" +
|
||||
std::to_string(static_cast<std::uint64_t>(
|
||||
std::chrono::duration_cast<std::chrono::seconds>(
|
||||
std::chrono::system_clock::now().time_since_epoch())
|
||||
.count())) +
|
||||
":" +
|
||||
std::to_string(static_cast<std::uint64_t>(
|
||||
std::chrono::duration_cast<std::chrono::seconds>(
|
||||
std::chrono::system_clock::now().time_since_epoch())
|
||||
.count()) +
|
||||
1200)});
|
||||
region,
|
||||
{"--synthetic-inputs", "development-v1", "--ops-listen", "127.0.0.1:0", "--m10-listen",
|
||||
"127.0.0.1:0", "--m10-cert", (region.root / "server.pem").string(), "--m10-key",
|
||||
(region.root / "server.key").string(), "--m10-issuer",
|
||||
(region.root / "issuer.pub").string(), "--m10-environment", "dev",
|
||||
"--m10-max-grant-lifetime", "900", "--session-window",
|
||||
"test-" +
|
||||
std::to_string(static_cast<std::uint64_t>(
|
||||
std::chrono::duration_cast<std::chrono::seconds>(
|
||||
std::chrono::system_clock::now().time_since_epoch())
|
||||
.count())) +
|
||||
":" +
|
||||
std::to_string(static_cast<std::uint64_t>(
|
||||
std::chrono::duration_cast<std::chrono::seconds>(
|
||||
std::chrono::system_clock::now().time_since_epoch())
|
||||
.count()) +
|
||||
1200)});
|
||||
}
|
||||
[[nodiscard]] WireGrant grant(const AccountId& who, std::uint64_t serial) const {
|
||||
GrantClaims c;
|
||||
@@ -133,12 +165,16 @@ struct Setup {
|
||||
// A running evo-server and its QUIC port, its output drained while waiting.
|
||||
struct Server {
|
||||
std::unique_ptr<Child> child;
|
||||
std::uint16_t port{};
|
||||
std::uint16_t port{}, ops{};
|
||||
explicit Server(const Setup& s) : child{std::make_unique<Child>(s.args())} {
|
||||
CHECK(child->wait_line("m10 listening ", 30s));
|
||||
for (const auto& l : child->lines())
|
||||
for (const auto& l : child->lines()) {
|
||||
if (starts(l, "m10 listening "))
|
||||
port = static_cast<std::uint16_t>(std::stoul(l.substr(14)));
|
||||
if (starts(l, "ops listening "))
|
||||
ops = static_cast<std::uint16_t>(std::stoul(l.substr(14)));
|
||||
}
|
||||
CHECK(ops != 0);
|
||||
CHECK(port != 0 && child->wait_line("ready 1", 30s));
|
||||
}
|
||||
void pump() {
|
||||
@@ -237,6 +273,20 @@ void end_to_end() {
|
||||
CHECK(std::ranges::all_of(shown->cells(),
|
||||
[&](const auto& c) { return c.voxel.z == shown->slice_z(); }));
|
||||
CHECK(until(*server, *a, [&] { return a->server_tick() >= first_tick + 40; }));
|
||||
// M10.9a (ADR-0097): the listener's load is exposed; a lightly loaded region measures
|
||||
// per-tick work, accepts logins and sheds nothing.
|
||||
const auto sessions = metric(server->ops, "evo_m10_sessions");
|
||||
const auto p95 = metric(server->ops, "evo_m10_tick_work_p95_microseconds");
|
||||
std::printf("m10 load report sessions=%llu p95_us=%llu p99_us=%llu session_p99_us=%llu\n",
|
||||
static_cast<unsigned long long>(sessions.value_or(0)),
|
||||
static_cast<unsigned long long>(p95.value_or(0)),
|
||||
static_cast<unsigned long long>(
|
||||
metric(server->ops, "evo_m10_tick_work_p99_microseconds").value_or(0)),
|
||||
static_cast<unsigned long long>(
|
||||
metric(server->ops, "evo_m10_session_work_p99_microseconds").value_or(0)));
|
||||
CHECK(sessions == 1u && p95.value_or(0) > 0 && metric(server->ops, "evo_m10_refusing") == 0u &&
|
||||
metric(server->ops, "evo_m10_shed_level") == 0u &&
|
||||
metric(server->ops, "evo_m10_refused_logins_total") == 0u);
|
||||
std::printf("m10 snapshot report cells=%zu ticks=%llu chunks=%llu bytes=%llu\n",
|
||||
shown->cells().size(),
|
||||
static_cast<unsigned long long>(a->server_tick() - first_tick),
|
||||
|
||||
Reference in New Issue
Block a user