From 9f9a93d4c877059fb35e9990d20cc9abb25a0e68 Mon Sep 17 00:00:00 2001 From: dingo Date: Thu, 8 Oct 2026 18:00:26 -0400 Subject: [PATCH] M10.9d: watermark coalescing, budgets v2, evo-load action ids - IntentLimits::watermark_coalesce (evo-server: every 4, coalesce 4); ADR-0092 amendment. - m10 load-report splits poll/replicate and commit-tick/other-tick maxima. - Budgets v2: work p95 <= 25 ms over all ticks, commit ticks <= 50 ms (ADR-0097); moving the commit off the tick is a recorded follow-up. - evo-load: time-based action ids, duplicate_actions, receipts before outcomes; drill ramps at 500 ms and judges by v2. - Refresh the dev session window. Co-Authored-By: Claude Opus 5.5 --- apps/evo-server/m10_bridge.hpp | 48 +++++++++++++++-- .../region/dev-session-route/listener.yaml | 2 +- ...092-m10-intentions-and-durable-outcomes.md | 10 ++++ .../0097-m10-load-and-abuse-qualification.md | 39 ++++++++++++++ docs/engineering/VALIDATION.md | 44 ++++++++++++++++ include/evo/network/intent_admission.hpp | 7 ++- schemas/network/m10-budgets-v1.md | 3 ++ schemas/network/m10-budgets-v2.md | 51 +++++++++++++++++++ src/network/protocol/intent_admission.cpp | 10 ++-- tests/network_control.cpp | 25 +++++++++ tools/network/load.cpp | 37 +++++++++----- tools/ops/load-drill.sh | 19 ++++--- 12 files changed, 267 insertions(+), 28 deletions(-) create mode 100644 schemas/network/m10-budgets-v2.md diff --git a/apps/evo-server/m10_bridge.hpp b/apps/evo-server/m10_bridge.hpp index 5ab4cde..665f5fc 100644 --- a/apps/evo-server/m10_bridge.hpp +++ b/apps/evo-server/m10_bridge.hpp @@ -104,6 +104,10 @@ template class M10Bridge { if (issuer.size() != network::grant_public_key_bytes) return fail("error m10-issuer"); server::RegionSessionsConfig config; + // ADR-0097: snapshots carry the server tick every tick (10.5), so watermarks need not; + // an advancing one goes out at most every 4 ticks (cumulative, nothing is lost). + config.intents.watermark_every = 4; + config.intents.watermark_coalesce = 4; config.transport.listen_address = o.listen.substr(0, colon); config.transport.port = port; config.transport.certificate_pem_file = o.certificate; @@ -206,6 +210,8 @@ template class M10Bridge { const server::HostFacts facts{s.ready && s.phase == server::HostPhase::running, s.epoch, s.simulated_tick}; sessions_->poll(facts, wall_seconds(), std::chrono::steady_clock::now()); + const auto polled = std::chrono::steady_clock::now(); + poll_busy_ += polled - started; if (facts.ready && host.state() != nullptr && s.simulated_tick > replicated_) { replicated_ = s.simulated_tick; const auto& c = community(*host.state()); @@ -219,6 +225,7 @@ template class M10Bridge { sessions_->replicate(s.simulated_tick, [&](const network::SessionInterest& in) { return source(c, in, s.simulated_tick); }); + replicate_busy_ += std::chrono::steady_clock::now() - polled; } if (facts.ready) for (const auto& a : sessions_->carry(s.simulated_tick + m10_carry_ahead_ticks)) { @@ -286,9 +293,24 @@ template class M10Bridge { static_cast(ticks)); }; const auto per_tick = per(busy_), session_per_tick = per(session_busy_); - busy_ = session_busy_ = {}; + const auto poll_per_tick = per(poll_busy_), replicate_per_tick = per(replicate_busy_); + busy_ = session_busy_ = poll_busy_ = replicate_busy_ = {}; + // A tick whose host step committed a batch (ADR-0097 measures those separately). + const bool commit = s.commits != commits_seen_; + commits_seen_ = s.commits; if (!counted) return; + interval_.ticks += ticks; + interval_.poll_us += poll_per_tick * ticks; + interval_.replicate_us += replicate_per_tick * ticks; + interval_.poll_max_us = std::max(interval_.poll_max_us, poll_per_tick); + interval_.replicate_max_us = std::max(interval_.replicate_max_us, replicate_per_tick); + if (commit) { + ++interval_.commit_ticks; + interval_.commit_max_us = std::max(interval_.commit_max_us, per_tick); + } else { + interval_.other_max_us = std::max(interval_.other_max_us, per_tick); + } const auto now_us = static_cast(std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) @@ -311,15 +333,28 @@ template class M10Bridge { return; reported_us_ = now_us; const auto& load = governor_.state(); + const auto ticks = std::max(interval_.ticks, 1); + // Since the last report: commit ticks (host committed a batch) apart from the others, + // and the session layer split into poll (transport, control, intents) and replicate. std::printf("m10 load-report sessions=%zu p95_us=%llu p99_us=%llu max_us=%llu " - "session_p95_us=%llu session_p99_us=%llu refusing=%d shed=%u tick_debt=%llu\n", + "session_p95_us=%llu session_p99_us=%llu refusing=%d shed=%u tick_debt=%llu " + "commit_ticks=%llu commit_max_us=%llu other_max_us=%llu poll_avg_us=%llu " + "poll_max_us=%llu replicate_avg_us=%llu replicate_max_us=%llu\n", sessions_->sessions(), static_cast(load.p95_us), static_cast(load.p99_us), static_cast(load.max_us), static_cast(load.session_p95_us), static_cast(load.session_p99_us), governed_ && load.refusing ? 1 : 0, governed_ ? load.shed : 0U, - static_cast(s.tick_debt)); + static_cast(s.tick_debt), + static_cast(interval_.commit_ticks), + static_cast(interval_.commit_max_us), + static_cast(interval_.other_max_us), + static_cast(interval_.poll_us / ticks), + static_cast(interval_.poll_max_us), + static_cast(interval_.replicate_us / ticks), + static_cast(interval_.replicate_max_us)); + interval_ = {}; } static const processes::EnvironmentalCommunity& community(const processes::EnvironmentalCommunity& c) { @@ -467,7 +502,12 @@ template class M10Bridge { server::LoadGovernor governor_{*server::LoadGovernor::create({})}; bool governed_{true}; std::uint64_t reported_us_{}; - std::chrono::nanoseconds busy_{}, session_busy_{}; + std::chrono::nanoseconds busy_{}, session_busy_{}, poll_busy_{}, replicate_busy_{}; + std::uint64_t commits_seen_{}; + struct Interval { // Since the last load-report. + std::uint64_t ticks{}, poll_us{}, replicate_us{}, poll_max_us{}, replicate_max_us{}, + commit_ticks{}, commit_max_us{}, other_max_us{}; + } interval_; std::uint64_t measured_tick_{}; std::unique_ptr probes_; // Handoff mode only. std::unique_ptr handoff_; // Handoff mode only. diff --git a/deploy/kubernetes/region/dev-session-route/listener.yaml b/deploy/kubernetes/region/dev-session-route/listener.yaml index 6323b61..cf462f7 100644 --- a/deploy/kubernetes/region/dev-session-route/listener.yaml +++ b/deploy/kubernetes/region/dev-session-route/listener.yaml @@ -12,7 +12,7 @@ spec: env: - {name: REGION_SESSION, value: "dev"} - {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"} - - {name: REGION_SESSION_WINDOW, value: "dev-20261008T184450-d705:1791498890"} + - {name: REGION_SESSION_WINDOW, value: "dev-20261008T215407-71fa:1791510247"} - {name: REGION_M10_MAX_GRANT_LIFETIME, value: "14400"} ports: # QUIC (ADR-0090): UDP only; nothing listens on TCP 7400. diff --git a/docs/adr/0092-m10-intentions-and-durable-outcomes.md b/docs/adr/0092-m10-intentions-and-durable-outcomes.md index eaeb234..4b065ef 100644 --- a/docs/adr/0092-m10-intentions-and-durable-outcomes.md +++ b/docs/adr/0092-m10-intentions-and-durable-outcomes.md @@ -119,3 +119,13 @@ Schema v11 (`migrations/0011_region_actions.sql`, [region-actions-v1](../../sche the record by the next process, and a retry is a duplicate with the same outcome); an action whose tick had not run at SIGKILL has no record, answers `unknown`, and commits exactly once when retried. + +## Amendment (2026-10-08, M10.9d, ADR-0097): watermark cadence +Snapshots carry the server tick every tick (10.5), so the watermark no longer has to be +every client's clock. In evo-server, an advancing `SimulatedThrough` goes to a session at +most every 4 ticks, and idle sessions get one every 4 ticks +(`IntentLimits::watermark_coalesce` and `watermark_every`; the library defaults stay 1). +It is cumulative, so no admitted intention is ever missing from a later watermark; a client +learns "simulated" up to 3 ticks (150 ms) later. Receipts are unchanged: one per intention. +Reason: on the reference worker, datagram sends dominated the session layer, and the +per-tick watermark was a quarter of them. diff --git a/docs/adr/0097-m10-load-and-abuse-qualification.md b/docs/adr/0097-m10-load-and-abuse-qualification.md index adfe925..d303514 100644 --- a/docs/adr/0097-m10-load-and-abuse-qualification.md +++ b/docs/adr/0097-m10-load-and-abuse-qualification.md @@ -348,3 +348,42 @@ Same drill (steps 20/24/28/32, 60 s holds), 18:21–18:26 UTC, restored and veri from about 970 µs to about 360 µs per tick (2.7×). - **Next:** a third run with the fix. The remaining per-session cost is the comparison with each session's acknowledged baseline. + +## 10.9d third reference run and decisions (2026-10-08, image `f378a43d…`) +The renewal had rotated the dev CA, but the running pod still held the old server +certificate. A `rollout restart` loaded the new one first (a throwaway-key login checked +the trust), then the drill ran at 21:00–21:05 UTC and was restored. +- **24 sessions held with every client budget met:** response p95 111 ms; 20 Hz; no + drops or starvation. The governor (refusing at p95 ≥ 15 ms) refused logins beyond 24. +- **Server at 24 sessions:** + - tick work p95 9–16 ms, p99 25–34 ms; + - with 0 sessions the p99 is already 23–25 ms: the synchronous commit flush; + - session layer 0.62–0.69 ms per session at p99, 0.35–0.5 at p95; + - no tick debt. +- **Outcome latency in this run is invalid.** The bundle reused the earlier runs' + accounts and the bots numbered actions from 1, so every action was a duplicate + answered at once with its recorded outcome (16 ms). `evo-load` now numbers actions from + the current time and counts `duplicate_actions` apart from outcome latency. + +**Maintainer decisions (2026-10-08):** +1. **Budget amended.** Per-tick region work is p95 ≤ 25 ms over all ticks, with ticks + that commit a batch ≤ 50 ms, measured separately (`schemas/network/m10-budgets-v2.md`). + Moving the commit off the simulation thread is a recorded follow-up. +2. **Optimize first, then declare** the measured sustainable session count. + +**Profile and second optimisation** (local, 24 bots): +- The `load-report` now splits the session layer into `poll` and `replicate` and gives + commit-tick and other-tick maxima. +- `poll` cost about 1.3 ms per tick against 0.45 ms for `replicate`. Probes inside `poll` + (temporary, not committed) attributed it to datagram sends of about 15 µs each, almost + all in msquic's `DatagramSend`: a receipt and a watermark per session per tick, plus + snapshot chunks. +- Snapshots carry the server tick every tick (10.5), so the per-tick watermark is no + longer needed for the client's clock. An advancing watermark is now sent at most every + 4 ticks (`IntentLimits::watermark_coalesce`; evo-server uses 4, the library default + stays 1). Watermarks dropped from 4,800 to 1,200 per 10 s and `poll` from about 1.36 to + 1.05 ms per tick. Receipts stay one per intention (ADR-0092). +- Further gains need fewer datagrams per tick, which is a protocol change and out of + scope here. +- The drill summary now judges by budgets v2 and prints commit and other-tick maxima and + the poll and replicate averages. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index d46dad1..cc496d5 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -5210,3 +5210,47 @@ Fix: Executed: see the commit's check list (Linux suites, clang-tidy, Windows). Not done: the third reference run (needs the image, a new bundle and the go-ahead). + +## 2026-10-08 — M10.9d third evolution-dev drill, budget v2, watermark coalescing + +Third drill (image `f378a43d…`, 21:00–21:05 UTC). The pod was restarted first to load the +renewed certificate (checked with a throwaway-key login). +- **24 sessions, client:** all within budget (response p95 111 ms, 20 Hz, no starvation). +- **24 sessions, server:** + - tick work p95 9–16 ms, p99 25–34 ms; + - with no session the p99 is 23–25 ms (the commit flush); + - session layer 0.62–0.69 ms per session at p99; + - governor cap at 24. +- **Outcome latency invalid:** duplicate action ids, the reused accounts again. + +Maintainer decisions: +- budgets v2 (p95 ≤ 25 ms over all ticks, commit ticks ≤ 50 ms; follow-up to move the + commit off the tick); +- optimise before declaring. + +Changes: +- `evo-load`: action ids from the current time, `duplicate_actions`, receipts read before + outcomes; the drill ramps at 500 ms. +- The `load-report` splits poll and replicate and gives commit-tick and other-tick maxima. +- `IntentLimits::watermark_coalesce`, 4 in evo-server, plus an ADR-0092 amendment. +- `schemas/network/m10-budgets-v2.md`. +- The drill summary judges by v2. + +Local profile, 24 bots (WSL, Release): +- `poll` about 1.3 ms against `replicate` about 0.45 ms per tick; +- temporary probes inside `poll` (reverted): event handling about 0.76 ms per tick + (9,600 datagrams per 10 s), watermark sends about 0.36 ms, control renewals about + 0.8 ms each; +- with coalescing: watermark sends 4,800 → 1,200 per 10 s, `poll` about 1.05 ms per tick. + +Executed: +- WSL Release `quic` build plus the quic and network CTest labels (including ops-contracts and + `session-fuzz-smoke`), passed. +- The 18 Postgres suites (including the `evo-load` smoke), passed. +- clang-tidy 19 on the changed sources, clean. +- `bash -n tools/ops/load-drill.sh`, passed. +- Windows `debug` 60/60, `quic` 5/5, `client-network` 15/15. + +Not done: +- shellcheck: not installed in this WSL. +- The fourth drill (needs the new image, a fresh bundle and the go-ahead). diff --git a/include/evo/network/intent_admission.hpp b/include/evo/network/intent_admission.hpp index 02e6daf..2876572 100644 --- a/include/evo/network/intent_admission.hpp +++ b/include/evo/network/intent_admission.hpp @@ -30,6 +30,10 @@ struct IntentLimits { // Ticks between watermarks to each session (1: every simulated tick). The watermark is // also every client's clock for the server tick until snapshots carry it (10.5). std::uint64_t watermark_every{1}; + // M10.9d (ADR-0097): while a session's watermark advances, at most one per this many + // ticks (it is cumulative: nothing is lost, "simulated" is learned up to N-1 ticks later). + // 1 keeps one per advance. Datagram sends dominate the session layer under load. + std::uint64_t watermark_coalesce{1}; }; // The host's authority as admission must see it. struct AdmissionFacts { @@ -90,7 +94,8 @@ class IntentAdmission { const AdmissionFacts& facts); // Watermarks to send after the host simulated through facts.server_tick: to every // session (sequence 0 before its first simulated intention) at least every - // `watermark_every` ticks, and whenever its sequence advances. + // `watermark_every` ticks, and when its sequence advances (at most every + // `watermark_coalesce` ticks). [[nodiscard]] std::vector> simulated(const AdmissionFacts& facts); // Scheduled actions whose tick is <= `tick`: they ride that tick's batch from now on. diff --git a/schemas/network/m10-budgets-v1.md b/schemas/network/m10-budgets-v1.md index 98a79e3..86c7253 100644 --- a/schemas/network/m10-budgets-v1.md +++ b/schemas/network/m10-budgets-v1.md @@ -1,5 +1,8 @@ # M10 timing and workload budgets, version 1 +Superseded by `m10-budgets-v2.md` (2026-10-08): per-tick region work is bounded at p95, +with commit ticks bounded separately. Kept for history. + Normative table for ADR-0089. "Ratified" values were decided by the maintainer (2026-10-03); "budget" values are engineering limits derived from them and from the M10.1 traces and change only with measurements and an ADR-0089 amendment. Version 1 diff --git a/schemas/network/m10-budgets-v2.md b/schemas/network/m10-budgets-v2.md new file mode 100644 index 0000000..4828a48 --- /dev/null +++ b/schemas/network/m10-budgets-v2.md @@ -0,0 +1,51 @@ +# M10 timing and workload budgets, version 2 + +Version 2 (2026-10-08, maintainer decision in M10.9d, ADR-0097) changes one row: +per-tick region work. The first evolution-dev reference runs showed the host's synchronous +commit flush alone reaching 23-25 ms at p99 with no session, about once per second, while +other ticks stayed near 3.5 ms. Work is now bounded at p95 over all ticks, plus a separate +bound for ticks that commit a batch. Moving the commit I/O off the simulation thread is a +recorded follow-up (a persistence/ADR change), not part of M10. Affected: the load drill's +summary (`tools/ops/load-drill.sh`) and the governor documentation; the governor itself +already acts on p95. Version 1 is kept for history. + +Normative table for ADR-0089. "Ratified" values were decided by the maintainer +(2026-10-03); "budget" values are engineering limits derived from them and from the +M10.1 traces and change only with measurements and an ADR-0089 amendment. Version 1 +applies to all M10 segments; a change bumps the version and lists affected tests. +None of these values is a biological or game rule. Exception to note: genome v1 and +metabolism express rates per tick, so the tick quantum currently fixes biological rates +in real time; changing it requires a versioned rules migration (ADR-0089). + +| Item | Value | Kind | Enforced/tested in | +|---|---|---|---| +| Authoritative tick quantum | 50,000 us (20 Hz) | ratified | checkpoints (`tick_microseconds`), region host | +| Commit flush | 1,000 ms | ratified | region host `flush_interval` | +| Final outcome delivery | p95 <= 1.5 s after admission (dev/stage, nominal network) | ratified | 10.4, 10.9 | +| Responsiveness (input -> first reflecting snapshot) | LAN p95 <= 150 ms; 100 ms RTT p95 <= 250 ms | ratified | 10.6, 10.9 | +| Reference hardware | server: one RKE2 worker (8 vCPU, 31.3 GiB) per region host; client: low-spec VM (M9.8g) | ratified | 10.9 | +| Input scheduling window | server_tick + 1 ..= server_tick + 8 ticks | budget | 10.4 admission | +| Client input lead | ceil((rtt/2 + jitter)/50 ms) + 1, clamped 2..8 ticks | budget | 10.4, 10.6 | +| Late input | rejected, never rewound | budget | 10.4 | +| Snapshot cadence (in interest) | every tick | budget | 10.5 | +| Baselines retained | server <= 4 per session, each <= 40 ticks old; client <= 2 | budget | 10.5 | +| Intention sequence | u64 per session, from 1, strictly increasing, no wrap | budget | 10.4 | +| Tick, baseline id | u64, no wrap | budget | 10.4, 10.5 | +| Realtime packet sequence | serial-number arithmetic >= 16 bits, window < half space | budget | 10.2 | +| Session id / epoch | u64 server-assigned, bound to connection; epoch on every message; stale closes | budget | 10.3, 10.4 | +| Intentions per session | 1 per tick (8 per 8-tick window, every attempt counted) | budget | 10.4 | +| Unresolved intentions per session | 256 | budget | 10.4 | +| Realtime datagram payload | <= 1,200 bytes | budget | 10.2 | +| Reliable frame | <= 64 KiB | budget | 10.2 | +| Replication per session | <= 16 KiB per tick | budget | 10.5 | +| Client event queue | 1,024 events | budget | client session | +| Heartbeat / liveness / handshake | 1 s / 5 s / 5 s | budget | 10.2, 10.7 | +| Sessions per region | >= 20 required, 32 default | budget | 10.9 | +| Per-tick region work | p95 <= 25 ms over all ticks on the reference worker; ticks that commit a batch <= 50 ms (measured separately) | budget (v2) | 10.9 (`m10 load-report` commit_max_us / other_max_us, `evo_m10_tick_work_*`, ADR-0097) | +| Session handling | <= 0.5 ms per session per tick | budget | 10.9 (measured: `evo_m10_session_work_*`, ADR-0097) | +| Host input queue | 64 inputs, 64 KiB | budget | region host | +| Overload response | shed interest, refuse new sessions before tick debt | budget | 10.9a load governor: refuse at p95 >= 15 ms or debt >= 2, shed at p95 >= 20 ms or debt >= 4, relax below 10 ms after 5 s (ADR-0097; provisional until 10.9d) | + +Determinism: authoritative state is exact given the checkpoint and committed per-tick +inputs; timing, delivery, prediction, interpolation and presentation are tolerant +(ADR-0089). diff --git a/src/network/protocol/intent_admission.cpp b/src/network/protocol/intent_admission.cpp index 90d42a7..cdfd512 100644 --- a/src/network/protocol/intent_admission.cpp +++ b/src/network/protocol/intent_admission.cpp @@ -17,7 +17,8 @@ std::expected IntentAdmission::create(Intent if (limits.intent_window == 0 || limits.intent_window > 64 || limits.window_ticks == 0 || limits.max_intents_per_window == 0 || limits.max_actions_per_window == 0 || limits.max_unresolved == 0 || limits.max_unresolved > 4096 || limits.max_sessions == 0 || - limits.max_sessions > 1024 || limits.max_finals == 0 || limits.watermark_every == 0) + limits.max_sessions > 1024 || limits.max_finals == 0 || limits.watermark_every == 0 || + limits.watermark_coalesce == 0) return std::unexpected(IntentConfigError::invalid); IntentAdmission admission{limits}; admission.sessions_.reserve(limits.max_sessions); @@ -209,9 +210,10 @@ IntentAdmission::simulated(const AdmissionFacts& facts) { s.watermark = s.pending.front().sequence; s.pending.pop_front(); } - const bool changed = s.watermark != s.sent_watermark; - const bool repeat = facts.server_tick < s.sent_at_tick || - facts.server_tick - s.sent_at_tick >= limits_.watermark_every; + const auto since = facts.server_tick < s.sent_at_tick ? limits_.watermark_every + : facts.server_tick - s.sent_at_tick; + const bool changed = s.watermark != s.sent_watermark && since >= limits_.watermark_coalesce; + const bool repeat = since >= limits_.watermark_every; if (changed || repeat) { s.sent_watermark = s.watermark; s.sent_at_tick = facts.server_tick; diff --git a/tests/network_control.cpp b/tests/network_control.cpp index e900a8f..5e000aa 100644 --- a/tests/network_control.cpp +++ b/tests/network_control.cpp @@ -551,6 +551,31 @@ void intent_admission() { marks = a.simulated(f); // Repeated after 20 ticks so a lost datagram is covered. CHECK(mark_of(marks, 1) == SimulatedThrough{5, 3, 130} && mark_of(marks, 2) == SimulatedThrough{5, 0, 130}); + // M10.9d: coalesced, an advancing watermark goes out at most every 4 ticks, still + // cumulative (the latest sequence), and idle repeats keep their interval. + { + IntentLimits l; + l.watermark_every = 20; + l.watermark_coalesce = 4; + IntentLimits zero; + zero.watermark_coalesce = 0; + CHECK(!IntentAdmission::create(zero)); + auto c = *IntentAdmission::create(l); + CHECK(c.open(3, alice)); + AdmissionFacts g{true, 5, 200}; + std::uint64_t sent = 0; + for (std::uint64_t tick = 201; tick <= 216; ++tick) { + CHECK(c.intention(3, SessionIntention{3, 5, tick - 200, tick, 0, 0}, g).status == + AdmitStatus::admitted); + g.server_tick = tick; + const auto m = mark_of(c.simulated(g), 3); + if (m) { + ++sent; + CHECK(m->sequence == tick - 200); // The newest simulated intention. + } + } + CHECK(sent == 4); // Ticks 201 (first), 205, 209, 213. + } // By default every session hears every simulated tick. auto every = *IntentAdmission::create(IntentLimits{}); CHECK(every.open(7, alice)); diff --git a/tools/network/load.cpp b/tools/network/load.cpp index 776d3c5..3f1250c 100644 --- a/tools/network/load.cpp +++ b/tools/network/load.cpp @@ -132,6 +132,7 @@ struct Options { struct Samples { std::vector response_us, receipt_us, outcome_us, login_us, correction_um, gap_ticks; + std::uint64_t duplicate_actions{}; // Receipts saying duplicate: no outcome latency. std::uint64_t snapshots{}, bytes{}, intents{}, refused_intents{}, logins{}, drops{}, login_failures{}, refused{}; std::map closes; // By session close reason (255: no Close). @@ -181,6 +182,10 @@ class Load { static_cast(((i + 1) >> (8U * k)) & 0xFFU); } b->serial = now_seconds() * 64; // Later runs supersede earlier ones. + // Action ids are unique per account across runs too: a reused id is a duplicate, + // answered at once with the recorded outcome (10.9d: the third dev drill's 16 ms + // "outcomes" were duplicates of an earlier run's actions). + b->next_action_id = now_seconds() * 1000; b->rng.seed(o_.seed * 1'000'003 + i); bots_.push_back(std::move(b)); } @@ -396,6 +401,12 @@ class Load { const auto bytes = c.snapshot_bytes(); b.s.bytes += bytes - std::min(bytes, b.last_bytes); b.last_bytes = bytes; + // Receipts first: a duplicate's recorded outcome is not a latency sample. + for (const auto& r : c.take_action_receipts()) + if (r.status == AdmitStatus::duplicate) { + ++b.s.duplicate_actions; + b.outcome_pending.erase(r.action_id); + } // Outcomes: final ones only. for (const auto& out : c.take_action_outcomes()) if (out.result == ActionResult::committed_no_effect || @@ -405,7 +416,6 @@ class Load { b.s.outcome_us.push_back(micros(now - it->second)); b.outcome_pending.erase(it); } - (void)c.take_action_receipts(); // Once per server tick: an intention, a prediction, sometimes an action or interest. const auto tick = c.server_tick(); if (tick == 0 || tick == b.last_tick) @@ -542,6 +552,7 @@ bool report(const Options& o, std::size_t n, std::size_t established, std::size_ pooled.snapshots += s.snapshots; pooled.intents += s.intents; pooled.refused_intents += s.refused_intents; + pooled.duplicate_actions += s.duplicate_actions; pooled.logins += s.logins; pooled.drops += s.drops; pooled.login_failures += s.login_failures; @@ -576,17 +587,19 @@ bool report(const Options& o, std::size_t n, std::size_t established, std::size_ line("outcome", pooled.outcome_us, outcome95, 1000.0, "ms"); line("login", pooled.login_us, login95, 1000.0, "ms"); line("correction", pooled.correction_um, correction95, 1000.0, "mm"); - std::printf("load snapshots hz min/med/max=%.2f/%.2f/%.2f gap_ticks p99/max=%llu/%llu " - "kib_s med/max=%llu/%llu intents=%llu refused_intents=%llu\n", - static_cast(percentile(rates, 0.0)) / 1000.0, - static_cast(percentile(rates, 0.5)) / 1000.0, - static_cast(percentile(rates, 1.0)) / 1000.0, - static_cast(percentile(pooled.gap_ticks, 0.99)), - static_cast(percentile(pooled.gap_ticks, 1.0)), - static_cast(percentile(kib, 0.5)), - static_cast(percentile(kib, 1.0)), - static_cast(pooled.intents), - static_cast(pooled.refused_intents)); + std::printf( + "load snapshots hz min/med/max=%.2f/%.2f/%.2f gap_ticks p99/max=%llu/%llu " + "kib_s med/max=%llu/%llu intents=%llu refused_intents=%llu duplicate_actions=%llu\n", + static_cast(percentile(rates, 0.0)) / 1000.0, + static_cast(percentile(rates, 0.5)) / 1000.0, + static_cast(percentile(rates, 1.0)) / 1000.0, + static_cast(percentile(pooled.gap_ticks, 0.99)), + static_cast(percentile(pooled.gap_ticks, 1.0)), + static_cast(percentile(kib, 0.5)), + static_cast(percentile(kib, 1.0)), + static_cast(pooled.intents), + static_cast(pooled.refused_intents), + static_cast(pooled.duplicate_actions)); std::printf("load client_cpu_pct=%.0f\n", 100.0 * cpu / std::max(seconds, 0.001)); if (!server.empty()) { std::printf("load server"); diff --git a/tools/ops/load-drill.sh b/tools/ops/load-drill.sh index dfc6c4b..dfd2e7b 100644 --- a/tools/ops/load-drill.sh +++ b/tools/ops/load-drill.sh @@ -129,7 +129,7 @@ set +e "$tool" --connect "$connect" --server-name "$server_name" --ca "$tokens/dev-ca.pem" \ --grants "$tokens/load-grants.bin" --grants-per-bot "$per_bot" --world "$world" \ --frame "$frame" --region "$region" --steps "$steps" --hold "$hold" --warmup "$warmup" \ - --threads 8 --per-client > "$work/load.log" 2>&1 + --threads 8 --ramp-ms 500 --per-client > "$work/load.log" 2>&1 load_exit=$? set -e sleep 12 # One more server report after the last hold. @@ -162,14 +162,21 @@ for line in open(load, encoding='utf-8', errors='replace'): if not inside: print(f"drill step N={n} server_reports=0") continue - worst = lambda key: max(r[key] for r in inside) + worst = lambda key: max(r.get(key, 0) for r in inside) + mean = lambda key: sum(r.get(key, 0) for r in inside) / len(inside) sessions = max(1, worst('sessions')) + # Budgets v2 (schemas/network/m10-budgets-v2.md): tick work p95 <= 25 ms over all ticks, + # ticks that commit a batch <= 50 ms; session layer p99 <= 0.5 ms per session; no debt. + within = (worst('p95_us') <= 25000 and worst('commit_max_us') <= 50000 and + worst('session_p99_us') <= 500 * sessions and worst('tick_debt') == 0) print(f"drill step N={n} server_reports={len(inside)} sessions_max={worst('sessions')} " - f"work_p99_max_ms={worst('p99_us') / 1000:.1f} work_max_ms={worst('max_us') / 1000:.1f} " + f"work_p95_max_ms={worst('p95_us') / 1000:.1f} work_p99_max_ms={worst('p99_us') / 1000:.1f} " + f"commit_max_ms={worst('commit_max_us') / 1000:.1f} other_max_ms={worst('other_max_us') / 1000:.1f} " + f"session_p95_max_ms={worst('session_p95_us') / 1000:.1f} " f"session_p99_max_ms={worst('session_p99_us') / 1000:.1f} " - f"per_session_ms={worst('session_p99_us') / 1000 / sessions:.3f} " + f"per_session_p99_ms={worst('session_p99_us') / 1000 / sessions:.3f} " + f"poll_avg_ms={mean('poll_avg_us') / 1000:.2f} replicate_avg_ms={mean('replicate_avg_us') / 1000:.2f} " f"refusing_max={worst('refusing')} shed_max={worst('shed')} " - f"tick_debt_max={worst('tick_debt')} " - f"server_within_budget={int(worst('p99_us') <= 25000 and worst('session_p99_us') <= 500 * sessions and worst('tick_debt') == 0)}") + f"tick_debt_max={worst('tick_debt')} server_within_budget={int(within)}") PY say "drill complete (evo-load exit $load_exit); restoring"