M10.9d: watermark coalescing, budgets v2, evo-load action ids
Foundation / checks (client) (push) Successful in 14s
Foundation / checks (checks) (push) Successful in 16s
Foundation / checks (deploy) (push) Successful in 15s
Foundation / checks (network) (push) Successful in 18s
Foundation / checks (postgres) (push) Successful in 1m34s
Foundation / checks (ecs) (push) Successful in 1m53s
Foundation / checks (tsan) (push) Successful in 5m44s
Foundation / server-image (push) Successful in 1m0s
Foundation / postgres-tools (push) Successful in 1m38s
Foundation / image (push) Successful in 2m34s

- IntentLimits::watermark_coalesce (evo-server: every 4, coalesce 4); ADR-0092 amendment.
- m10 load-report splits poll/replicate and commit-tick/other-tick maxima.
- Budgets v2: work p95 <= 25 ms over all ticks, commit ticks <= 50 ms (ADR-0097);
  moving the commit off the tick is a recorded follow-up.
- evo-load: time-based action ids, duplicate_actions, receipts before outcomes;
  drill ramps at 500 ms and judges by v2.
- Refresh the dev session window.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-08 18:00:41 -04:00
co-authored by Claude Opus 5.5
parent 1b266b12be
commit 9f9a93d4c8
12 changed files with 267 additions and 28 deletions
+44 -4
View File
@@ -104,6 +104,10 @@ template <typename Host> class M10Bridge {
if (issuer.size() != network::grant_public_key_bytes)
return fail("error m10-issuer");
server::RegionSessionsConfig config;
// ADR-0097: snapshots carry the server tick every tick (10.5), so watermarks need not;
// an advancing one goes out at most every 4 ticks (cumulative, nothing is lost).
config.intents.watermark_every = 4;
config.intents.watermark_coalesce = 4;
config.transport.listen_address = o.listen.substr(0, colon);
config.transport.port = port;
config.transport.certificate_pem_file = o.certificate;
@@ -206,6 +210,8 @@ template <typename Host> class M10Bridge {
const server::HostFacts facts{s.ready && s.phase == server::HostPhase::running, s.epoch,
s.simulated_tick};
sessions_->poll(facts, wall_seconds(), std::chrono::steady_clock::now());
const auto polled = std::chrono::steady_clock::now();
poll_busy_ += polled - started;
if (facts.ready && host.state() != nullptr && s.simulated_tick > replicated_) {
replicated_ = s.simulated_tick;
const auto& c = community(*host.state());
@@ -219,6 +225,7 @@ template <typename Host> class M10Bridge {
sessions_->replicate(s.simulated_tick, [&](const network::SessionInterest& in) {
return source(c, in, s.simulated_tick);
});
replicate_busy_ += std::chrono::steady_clock::now() - polled;
}
if (facts.ready)
for (const auto& a : sessions_->carry(s.simulated_tick + m10_carry_ahead_ticks)) {
@@ -286,9 +293,24 @@ template <typename Host> class M10Bridge {
static_cast<std::int64_t>(ticks));
};
const auto per_tick = per(busy_), session_per_tick = per(session_busy_);
busy_ = session_busy_ = {};
const auto poll_per_tick = per(poll_busy_), replicate_per_tick = per(replicate_busy_);
busy_ = session_busy_ = poll_busy_ = replicate_busy_ = {};
// A tick whose host step committed a batch (ADR-0097 measures those separately).
const bool commit = s.commits != commits_seen_;
commits_seen_ = s.commits;
if (!counted)
return;
interval_.ticks += ticks;
interval_.poll_us += poll_per_tick * ticks;
interval_.replicate_us += replicate_per_tick * ticks;
interval_.poll_max_us = std::max(interval_.poll_max_us, poll_per_tick);
interval_.replicate_max_us = std::max(interval_.replicate_max_us, replicate_per_tick);
if (commit) {
++interval_.commit_ticks;
interval_.commit_max_us = std::max(interval_.commit_max_us, per_tick);
} else {
interval_.other_max_us = std::max(interval_.other_max_us, per_tick);
}
const auto now_us =
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now().time_since_epoch())
@@ -311,15 +333,28 @@ template <typename Host> class M10Bridge {
return;
reported_us_ = now_us;
const auto& load = governor_.state();
const auto ticks = std::max<std::uint64_t>(interval_.ticks, 1);
// Since the last report: commit ticks (host committed a batch) apart from the others,
// and the session layer split into poll (transport, control, intents) and replicate.
std::printf("m10 load-report sessions=%zu p95_us=%llu p99_us=%llu max_us=%llu "
"session_p95_us=%llu session_p99_us=%llu refusing=%d shed=%u tick_debt=%llu\n",
"session_p95_us=%llu session_p99_us=%llu refusing=%d shed=%u tick_debt=%llu "
"commit_ticks=%llu commit_max_us=%llu other_max_us=%llu poll_avg_us=%llu "
"poll_max_us=%llu replicate_avg_us=%llu replicate_max_us=%llu\n",
sessions_->sessions(), static_cast<unsigned long long>(load.p95_us),
static_cast<unsigned long long>(load.p99_us),
static_cast<unsigned long long>(load.max_us),
static_cast<unsigned long long>(load.session_p95_us),
static_cast<unsigned long long>(load.session_p99_us),
governed_ && load.refusing ? 1 : 0, governed_ ? load.shed : 0U,
static_cast<unsigned long long>(s.tick_debt));
static_cast<unsigned long long>(s.tick_debt),
static_cast<unsigned long long>(interval_.commit_ticks),
static_cast<unsigned long long>(interval_.commit_max_us),
static_cast<unsigned long long>(interval_.other_max_us),
static_cast<unsigned long long>(interval_.poll_us / ticks),
static_cast<unsigned long long>(interval_.poll_max_us),
static_cast<unsigned long long>(interval_.replicate_us / ticks),
static_cast<unsigned long long>(interval_.replicate_max_us));
interval_ = {};
}
static const processes::EnvironmentalCommunity&
community(const processes::EnvironmentalCommunity& c) {
@@ -467,7 +502,12 @@ template <typename Host> class M10Bridge {
server::LoadGovernor governor_{*server::LoadGovernor::create({})};
bool governed_{true};
std::uint64_t reported_us_{};
std::chrono::nanoseconds busy_{}, session_busy_{};
std::chrono::nanoseconds busy_{}, session_busy_{}, poll_busy_{}, replicate_busy_{};
std::uint64_t commits_seen_{};
struct Interval { // Since the last load-report.
std::uint64_t ticks{}, poll_us{}, replicate_us{}, poll_max_us{}, replicate_max_us{},
commit_ticks{}, commit_max_us{}, other_max_us{};
} interval_;
std::uint64_t measured_tick_{};
std::unique_ptr<server::PostgresProbeLedger> probes_; // Handoff mode only.
std::unique_ptr<server::PostgresHandoffLedger> handoff_; // Handoff mode only.
@@ -12,7 +12,7 @@ spec:
env:
- {name: REGION_SESSION, value: "dev"}
- {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"}
- {name: REGION_SESSION_WINDOW, value: "dev-20261008T184450-d705:1791498890"}
- {name: REGION_SESSION_WINDOW, value: "dev-20261008T215407-71fa:1791510247"}
- {name: REGION_M10_MAX_GRANT_LIFETIME, value: "14400"}
ports:
# QUIC (ADR-0090): UDP only; nothing listens on TCP 7400.
@@ -119,3 +119,13 @@ Schema v11 (`migrations/0011_region_actions.sql`, [region-actions-v1](../../sche
the record by the next process, and a retry is a duplicate with the same outcome); an
action whose tick had not run at SIGKILL has no record, answers `unknown`, and commits
exactly once when retried.
## Amendment (2026-10-08, M10.9d, ADR-0097): watermark cadence
Snapshots carry the server tick every tick (10.5), so the watermark no longer has to be
every client's clock. In evo-server, an advancing `SimulatedThrough` goes to a session at
most every 4 ticks, and idle sessions get one every 4 ticks
(`IntentLimits::watermark_coalesce` and `watermark_every`; the library defaults stay 1).
It is cumulative, so no admitted intention is ever missing from a later watermark; a client
learns "simulated" up to 3 ticks (150 ms) later. Receipts are unchanged: one per intention.
Reason: on the reference worker, datagram sends dominated the session layer, and the
per-tick watermark was a quarter of them.
@@ -348,3 +348,42 @@ Same drill (steps 20/24/28/32, 60 s holds), 18:21–18:26 UTC, restored and veri
from about 970 µs to about 360 µs per tick (2.7×).
- **Next:** a third run with the fix. The remaining per-session cost is the comparison
with each session's acknowledged baseline.
## 10.9d third reference run and decisions (2026-10-08, image `f378a43d…`)
The renewal had rotated the dev CA, but the running pod still held the old server
certificate. A `rollout restart` loaded the new one first (a throwaway-key login checked
the trust), then the drill ran at 21:00–21:05 UTC and was restored.
- **24 sessions held with every client budget met:** response p95 111 ms; 20 Hz; no
drops or starvation. The governor (refusing at p95 ≥ 15 ms) refused logins beyond 24.
- **Server at 24 sessions:**
- tick work p95 9–16 ms, p99 25–34 ms;
- with 0 sessions the p99 is already 23–25 ms: the synchronous commit flush;
- session layer 0.62–0.69 ms per session at p99, 0.35–0.5 at p95;
- no tick debt.
- **Outcome latency in this run is invalid.** The bundle reused the earlier runs'
accounts and the bots numbered actions from 1, so every action was a duplicate
answered at once with its recorded outcome (16 ms). `evo-load` now numbers actions from
the current time and counts `duplicate_actions` apart from outcome latency.
**Maintainer decisions (2026-10-08):**
1. **Budget amended.** Per-tick region work is p95 ≤ 25 ms over all ticks, with ticks
that commit a batch ≤ 50 ms, measured separately (`schemas/network/m10-budgets-v2.md`).
Moving the commit off the simulation thread is a recorded follow-up.
2. **Optimize first, then declare** the measured sustainable session count.
**Profile and second optimisation** (local, 24 bots):
- The `load-report` now splits the session layer into `poll` and `replicate` and gives
commit-tick and other-tick maxima.
- `poll` cost about 1.3 ms per tick against 0.45 ms for `replicate`. Probes inside `poll`
(temporary, not committed) attributed it to datagram sends of about 15 µs each, almost
all in msquic's `DatagramSend`: a receipt and a watermark per session per tick, plus
snapshot chunks.
- Snapshots carry the server tick every tick (10.5), so the per-tick watermark is no
longer needed for the client's clock. An advancing watermark is now sent at most every
4 ticks (`IntentLimits::watermark_coalesce`; evo-server uses 4, the library default
stays 1). Watermarks dropped from 4,800 to 1,200 per 10 s and `poll` from about 1.36 to
1.05 ms per tick. Receipts stay one per intention (ADR-0092).
- Further gains need fewer datagrams per tick, which is a protocol change and out of
scope here.
- The drill summary now judges by budgets v2 and prints commit and other-tick maxima and
the poll and replicate averages.
+44
View File
@@ -5210,3 +5210,47 @@ Fix:
Executed: see the commit's check list (Linux suites, clang-tidy, Windows). Not done: the
third reference run (needs the image, a new bundle and the go-ahead).
## 2026-10-08 — M10.9d third evolution-dev drill, budget v2, watermark coalescing
Third drill (image `f378a43d…`, 21:00–21:05 UTC). The pod was restarted first to load the
renewed certificate (checked with a throwaway-key login).
- **24 sessions, client:** all within budget (response p95 111 ms, 20 Hz, no starvation).
- **24 sessions, server:**
- tick work p95 9–16 ms, p99 25–34 ms;
- with no session the p99 is 23–25 ms (the commit flush);
- session layer 0.62–0.69 ms per session at p99;
- governor cap at 24.
- **Outcome latency invalid:** duplicate action ids, the reused accounts again.
Maintainer decisions:
- budgets v2 (p95 ≤ 25 ms over all ticks, commit ticks ≤ 50 ms; follow-up to move the
commit off the tick);
- optimise before declaring.
Changes:
- `evo-load`: action ids from the current time, `duplicate_actions`, receipts read before
outcomes; the drill ramps at 500 ms.
- The `load-report` splits poll and replicate and gives commit-tick and other-tick maxima.
- `IntentLimits::watermark_coalesce`, 4 in evo-server, plus an ADR-0092 amendment.
- `schemas/network/m10-budgets-v2.md`.
- The drill summary judges by v2.
Local profile, 24 bots (WSL, Release):
- `poll` about 1.3 ms against `replicate` about 0.45 ms per tick;
- temporary probes inside `poll` (reverted): event handling about 0.76 ms per tick
(9,600 datagrams per 10 s), watermark sends about 0.36 ms, control renewals about
0.8 ms each;
- with coalescing: watermark sends 4,800 → 1,200 per 10 s, `poll` about 1.05 ms per tick.
Executed:
- WSL Release `quic` build plus the quic and network CTest labels (including ops-contracts and
`session-fuzz-smoke`), passed.
- The 18 Postgres suites (including the `evo-load` smoke), passed.
- clang-tidy 19 on the changed sources, clean.
- `bash -n tools/ops/load-drill.sh`, passed.
- Windows `debug` 60/60, `quic` 5/5, `client-network` 15/15.
Not done:
- shellcheck: not installed in this WSL.
- The fourth drill (needs the new image, a fresh bundle and the go-ahead).
+6 -1
View File
@@ -30,6 +30,10 @@ struct IntentLimits {
// Ticks between watermarks to each session (1: every simulated tick). The watermark is
// also every client's clock for the server tick until snapshots carry it (10.5).
std::uint64_t watermark_every{1};
// M10.9d (ADR-0097): while a session's watermark advances, at most one per this many
// ticks (it is cumulative: nothing is lost, "simulated" is learned up to N-1 ticks later).
// 1 keeps one per advance. Datagram sends dominate the session layer under load.
std::uint64_t watermark_coalesce{1};
};
// The host's authority as admission must see it.
struct AdmissionFacts {
@@ -90,7 +94,8 @@ class IntentAdmission {
const AdmissionFacts& facts);
// Watermarks to send after the host simulated through facts.server_tick: to every
// session (sequence 0 before its first simulated intention) at least every
// `watermark_every` ticks, and whenever its sequence advances.
// `watermark_every` ticks, and when its sequence advances (at most every
// `watermark_coalesce` ticks).
[[nodiscard]] std::vector<std::pair<std::uint64_t, SimulatedThrough>>
simulated(const AdmissionFacts& facts);
// Scheduled actions whose tick is <= `tick`: they ride that tick's batch from now on.
+3
View File
@@ -1,5 +1,8 @@
# M10 timing and workload budgets, version 1
Superseded by `m10-budgets-v2.md` (2026-10-08): per-tick region work is bounded at p95,
with commit ticks bounded separately. Kept for history.
Normative table for ADR-0089. "Ratified" values were decided by the maintainer
(2026-10-03); "budget" values are engineering limits derived from them and from the
M10.1 traces and change only with measurements and an ADR-0089 amendment. Version 1
+51
View File
@@ -0,0 +1,51 @@
# M10 timing and workload budgets, version 2
Version 2 (2026-10-08, maintainer decision in M10.9d, ADR-0097) changes one row:
per-tick region work. The first evolution-dev reference runs showed the host's synchronous
commit flush alone reaching 23-25 ms at p99 with no session, about once per second, while
other ticks stayed near 3.5 ms. Work is now bounded at p95 over all ticks, plus a separate
bound for ticks that commit a batch. Moving the commit I/O off the simulation thread is a
recorded follow-up (a persistence/ADR change), not part of M10. Affected: the load drill's
summary (`tools/ops/load-drill.sh`) and the governor documentation; the governor itself
already acts on p95. Version 1 is kept for history.
Normative table for ADR-0089. "Ratified" values were decided by the maintainer
(2026-10-03); "budget" values are engineering limits derived from them and from the
M10.1 traces and change only with measurements and an ADR-0089 amendment. Version 1
applies to all M10 segments; a change bumps the version and lists affected tests.
None of these values is a biological or game rule. Exception to note: genome v1 and
metabolism express rates per tick, so the tick quantum currently fixes biological rates
in real time; changing it requires a versioned rules migration (ADR-0089).
| Item | Value | Kind | Enforced/tested in |
|---|---|---|---|
| Authoritative tick quantum | 50,000 us (20 Hz) | ratified | checkpoints (`tick_microseconds`), region host |
| Commit flush | 1,000 ms | ratified | region host `flush_interval` |
| Final outcome delivery | p95 <= 1.5 s after admission (dev/stage, nominal network) | ratified | 10.4, 10.9 |
| Responsiveness (input -> first reflecting snapshot) | LAN p95 <= 150 ms; 100 ms RTT p95 <= 250 ms | ratified | 10.6, 10.9 |
| Reference hardware | server: one RKE2 worker (8 vCPU, 31.3 GiB) per region host; client: low-spec VM (M9.8g) | ratified | 10.9 |
| Input scheduling window | server_tick + 1 ..= server_tick + 8 ticks | budget | 10.4 admission |
| Client input lead | ceil((rtt/2 + jitter)/50 ms) + 1, clamped 2..8 ticks | budget | 10.4, 10.6 |
| Late input | rejected, never rewound | budget | 10.4 |
| Snapshot cadence (in interest) | every tick | budget | 10.5 |
| Baselines retained | server <= 4 per session, each <= 40 ticks old; client <= 2 | budget | 10.5 |
| Intention sequence | u64 per session, from 1, strictly increasing, no wrap | budget | 10.4 |
| Tick, baseline id | u64, no wrap | budget | 10.4, 10.5 |
| Realtime packet sequence | serial-number arithmetic >= 16 bits, window < half space | budget | 10.2 |
| Session id / epoch | u64 server-assigned, bound to connection; epoch on every message; stale closes | budget | 10.3, 10.4 |
| Intentions per session | 1 per tick (8 per 8-tick window, every attempt counted) | budget | 10.4 |
| Unresolved intentions per session | 256 | budget | 10.4 |
| Realtime datagram payload | <= 1,200 bytes | budget | 10.2 |
| Reliable frame | <= 64 KiB | budget | 10.2 |
| Replication per session | <= 16 KiB per tick | budget | 10.5 |
| Client event queue | 1,024 events | budget | client session |
| Heartbeat / liveness / handshake | 1 s / 5 s / 5 s | budget | 10.2, 10.7 |
| Sessions per region | >= 20 required, 32 default | budget | 10.9 |
| Per-tick region work | p95 <= 25 ms over all ticks on the reference worker; ticks that commit a batch <= 50 ms (measured separately) | budget (v2) | 10.9 (`m10 load-report` commit_max_us / other_max_us, `evo_m10_tick_work_*`, ADR-0097) |
| Session handling | <= 0.5 ms per session per tick | budget | 10.9 (measured: `evo_m10_session_work_*`, ADR-0097) |
| Host input queue | 64 inputs, 64 KiB | budget | region host |
| Overload response | shed interest, refuse new sessions before tick debt | budget | 10.9a load governor: refuse at p95 >= 15 ms or debt >= 2, shed at p95 >= 20 ms or debt >= 4, relax below 10 ms after 5 s (ADR-0097; provisional until 10.9d) |
Determinism: authoritative state is exact given the checkpoint and committed per-tick
inputs; timing, delivery, prediction, interpolation and presentation are tolerant
(ADR-0089).
+6 -4
View File
@@ -17,7 +17,8 @@ std::expected<IntentAdmission, IntentConfigError> IntentAdmission::create(Intent
if (limits.intent_window == 0 || limits.intent_window > 64 || limits.window_ticks == 0 ||
limits.max_intents_per_window == 0 || limits.max_actions_per_window == 0 ||
limits.max_unresolved == 0 || limits.max_unresolved > 4096 || limits.max_sessions == 0 ||
limits.max_sessions > 1024 || limits.max_finals == 0 || limits.watermark_every == 0)
limits.max_sessions > 1024 || limits.max_finals == 0 || limits.watermark_every == 0 ||
limits.watermark_coalesce == 0)
return std::unexpected(IntentConfigError::invalid);
IntentAdmission admission{limits};
admission.sessions_.reserve(limits.max_sessions);
@@ -209,9 +210,10 @@ IntentAdmission::simulated(const AdmissionFacts& facts) {
s.watermark = s.pending.front().sequence;
s.pending.pop_front();
}
const bool changed = s.watermark != s.sent_watermark;
const bool repeat = facts.server_tick < s.sent_at_tick ||
facts.server_tick - s.sent_at_tick >= limits_.watermark_every;
const auto since = facts.server_tick < s.sent_at_tick ? limits_.watermark_every
: facts.server_tick - s.sent_at_tick;
const bool changed = s.watermark != s.sent_watermark && since >= limits_.watermark_coalesce;
const bool repeat = since >= limits_.watermark_every;
if (changed || repeat) {
s.sent_watermark = s.watermark;
s.sent_at_tick = facts.server_tick;
+25
View File
@@ -551,6 +551,31 @@ void intent_admission() {
marks = a.simulated(f); // Repeated after 20 ticks so a lost datagram is covered.
CHECK(mark_of(marks, 1) == SimulatedThrough{5, 3, 130} &&
mark_of(marks, 2) == SimulatedThrough{5, 0, 130});
// M10.9d: coalesced, an advancing watermark goes out at most every 4 ticks, still
// cumulative (the latest sequence), and idle repeats keep their interval.
{
IntentLimits l;
l.watermark_every = 20;
l.watermark_coalesce = 4;
IntentLimits zero;
zero.watermark_coalesce = 0;
CHECK(!IntentAdmission::create(zero));
auto c = *IntentAdmission::create(l);
CHECK(c.open(3, alice));
AdmissionFacts g{true, 5, 200};
std::uint64_t sent = 0;
for (std::uint64_t tick = 201; tick <= 216; ++tick) {
CHECK(c.intention(3, SessionIntention{3, 5, tick - 200, tick, 0, 0}, g).status ==
AdmitStatus::admitted);
g.server_tick = tick;
const auto m = mark_of(c.simulated(g), 3);
if (m) {
++sent;
CHECK(m->sequence == tick - 200); // The newest simulated intention.
}
}
CHECK(sent == 4); // Ticks 201 (first), 205, 209, 213.
}
// By default every session hears every simulated tick.
auto every = *IntentAdmission::create(IntentLimits{});
CHECK(every.open(7, alice));
+25 -12
View File
@@ -132,6 +132,7 @@ struct Options {
struct Samples {
std::vector<std::uint64_t> response_us, receipt_us, outcome_us, login_us, correction_um,
gap_ticks;
std::uint64_t duplicate_actions{}; // Receipts saying duplicate: no outcome latency.
std::uint64_t snapshots{}, bytes{}, intents{}, refused_intents{}, logins{}, drops{},
login_failures{}, refused{};
std::map<unsigned, std::uint64_t> closes; // By session close reason (255: no Close).
@@ -181,6 +182,10 @@ class Load {
static_cast<std::uint8_t>(((i + 1) >> (8U * k)) & 0xFFU);
}
b->serial = now_seconds() * 64; // Later runs supersede earlier ones.
// Action ids are unique per account across runs too: a reused id is a duplicate,
// answered at once with the recorded outcome (10.9d: the third dev drill's 16 ms
// "outcomes" were duplicates of an earlier run's actions).
b->next_action_id = now_seconds() * 1000;
b->rng.seed(o_.seed * 1'000'003 + i);
bots_.push_back(std::move(b));
}
@@ -396,6 +401,12 @@ class Load {
const auto bytes = c.snapshot_bytes();
b.s.bytes += bytes - std::min(bytes, b.last_bytes);
b.last_bytes = bytes;
// Receipts first: a duplicate's recorded outcome is not a latency sample.
for (const auto& r : c.take_action_receipts())
if (r.status == AdmitStatus::duplicate) {
++b.s.duplicate_actions;
b.outcome_pending.erase(r.action_id);
}
// Outcomes: final ones only.
for (const auto& out : c.take_action_outcomes())
if (out.result == ActionResult::committed_no_effect ||
@@ -405,7 +416,6 @@ class Load {
b.s.outcome_us.push_back(micros(now - it->second));
b.outcome_pending.erase(it);
}
(void)c.take_action_receipts();
// Once per server tick: an intention, a prediction, sometimes an action or interest.
const auto tick = c.server_tick();
if (tick == 0 || tick == b.last_tick)
@@ -542,6 +552,7 @@ bool report(const Options& o, std::size_t n, std::size_t established, std::size_
pooled.snapshots += s.snapshots;
pooled.intents += s.intents;
pooled.refused_intents += s.refused_intents;
pooled.duplicate_actions += s.duplicate_actions;
pooled.logins += s.logins;
pooled.drops += s.drops;
pooled.login_failures += s.login_failures;
@@ -576,17 +587,19 @@ bool report(const Options& o, std::size_t n, std::size_t established, std::size_
line("outcome", pooled.outcome_us, outcome95, 1000.0, "ms");
line("login", pooled.login_us, login95, 1000.0, "ms");
line("correction", pooled.correction_um, correction95, 1000.0, "mm");
std::printf("load snapshots hz min/med/max=%.2f/%.2f/%.2f gap_ticks p99/max=%llu/%llu "
"kib_s med/max=%llu/%llu intents=%llu refused_intents=%llu\n",
static_cast<double>(percentile(rates, 0.0)) / 1000.0,
static_cast<double>(percentile(rates, 0.5)) / 1000.0,
static_cast<double>(percentile(rates, 1.0)) / 1000.0,
static_cast<unsigned long long>(percentile(pooled.gap_ticks, 0.99)),
static_cast<unsigned long long>(percentile(pooled.gap_ticks, 1.0)),
static_cast<unsigned long long>(percentile(kib, 0.5)),
static_cast<unsigned long long>(percentile(kib, 1.0)),
static_cast<unsigned long long>(pooled.intents),
static_cast<unsigned long long>(pooled.refused_intents));
std::printf(
"load snapshots hz min/med/max=%.2f/%.2f/%.2f gap_ticks p99/max=%llu/%llu "
"kib_s med/max=%llu/%llu intents=%llu refused_intents=%llu duplicate_actions=%llu\n",
static_cast<double>(percentile(rates, 0.0)) / 1000.0,
static_cast<double>(percentile(rates, 0.5)) / 1000.0,
static_cast<double>(percentile(rates, 1.0)) / 1000.0,
static_cast<unsigned long long>(percentile(pooled.gap_ticks, 0.99)),
static_cast<unsigned long long>(percentile(pooled.gap_ticks, 1.0)),
static_cast<unsigned long long>(percentile(kib, 0.5)),
static_cast<unsigned long long>(percentile(kib, 1.0)),
static_cast<unsigned long long>(pooled.intents),
static_cast<unsigned long long>(pooled.refused_intents),
static_cast<unsigned long long>(pooled.duplicate_actions));
std::printf("load client_cpu_pct=%.0f\n", 100.0 * cpu / std::max(seconds, 0.001));
if (!server.empty()) {
std::printf("load server");
+13 -6
View File
@@ -129,7 +129,7 @@ set +e
"$tool" --connect "$connect" --server-name "$server_name" --ca "$tokens/dev-ca.pem" \
--grants "$tokens/load-grants.bin" --grants-per-bot "$per_bot" --world "$world" \
--frame "$frame" --region "$region" --steps "$steps" --hold "$hold" --warmup "$warmup" \
--threads 8 --per-client > "$work/load.log" 2>&1
--threads 8 --ramp-ms 500 --per-client > "$work/load.log" 2>&1
load_exit=$?
set -e
sleep 12 # One more server report after the last hold.
@@ -162,14 +162,21 @@ for line in open(load, encoding='utf-8', errors='replace'):
if not inside:
print(f"drill step N={n} server_reports=0")
continue
worst = lambda key: max(r[key] for r in inside)
worst = lambda key: max(r.get(key, 0) for r in inside)
mean = lambda key: sum(r.get(key, 0) for r in inside) / len(inside)
sessions = max(1, worst('sessions'))
# Budgets v2 (schemas/network/m10-budgets-v2.md): tick work p95 <= 25 ms over all ticks,
# ticks that commit a batch <= 50 ms; session layer p99 <= 0.5 ms per session; no debt.
within = (worst('p95_us') <= 25000 and worst('commit_max_us') <= 50000 and
worst('session_p99_us') <= 500 * sessions and worst('tick_debt') == 0)
print(f"drill step N={n} server_reports={len(inside)} sessions_max={worst('sessions')} "
f"work_p99_max_ms={worst('p99_us') / 1000:.1f} work_max_ms={worst('max_us') / 1000:.1f} "
f"work_p95_max_ms={worst('p95_us') / 1000:.1f} work_p99_max_ms={worst('p99_us') / 1000:.1f} "
f"commit_max_ms={worst('commit_max_us') / 1000:.1f} other_max_ms={worst('other_max_us') / 1000:.1f} "
f"session_p95_max_ms={worst('session_p95_us') / 1000:.1f} "
f"session_p99_max_ms={worst('session_p99_us') / 1000:.1f} "
f"per_session_ms={worst('session_p99_us') / 1000 / sessions:.3f} "
f"per_session_p99_ms={worst('session_p99_us') / 1000 / sessions:.3f} "
f"poll_avg_ms={mean('poll_avg_us') / 1000:.2f} replicate_avg_ms={mean('replicate_avg_us') / 1000:.2f} "
f"refusing_max={worst('refusing')} shed_max={worst('shed')} "
f"tick_debt_max={worst('tick_debt')} "
f"server_within_budget={int(worst('p99_us') <= 25000 and worst('session_p99_us') <= 500 * sessions and worst('tick_debt') == 0)}")
f"tick_debt_max={worst('tick_debt')} server_within_budget={int(within)}")
PY
say "drill complete (evo-load exit $load_exit); restoring"