M10.9d: watermark coalescing, budgets v2, evo-load action ids
Foundation / checks (client) (push) Successful in 14s
Foundation / checks (checks) (push) Successful in 16s
Foundation / checks (deploy) (push) Successful in 15s
Foundation / checks (network) (push) Successful in 18s
Foundation / checks (postgres) (push) Successful in 1m34s
Foundation / checks (ecs) (push) Successful in 1m53s
Foundation / checks (tsan) (push) Successful in 5m44s
Foundation / server-image (push) Successful in 1m0s
Foundation / postgres-tools (push) Successful in 1m38s
Foundation / image (push) Successful in 2m34s
Foundation / checks (client) (push) Successful in 14s
Foundation / checks (checks) (push) Successful in 16s
Foundation / checks (deploy) (push) Successful in 15s
Foundation / checks (network) (push) Successful in 18s
Foundation / checks (postgres) (push) Successful in 1m34s
Foundation / checks (ecs) (push) Successful in 1m53s
Foundation / checks (tsan) (push) Successful in 5m44s
Foundation / server-image (push) Successful in 1m0s
Foundation / postgres-tools (push) Successful in 1m38s
Foundation / image (push) Successful in 2m34s
- IntentLimits::watermark_coalesce (evo-server: every 4, coalesce 4); ADR-0092 amendment. - m10 load-report splits poll/replicate and commit-tick/other-tick maxima. - Budgets v2: work p95 <= 25 ms over all ticks, commit ticks <= 50 ms (ADR-0097); moving the commit off the tick is a recorded follow-up. - evo-load: time-based action ids, duplicate_actions, receipts before outcomes; drill ramps at 500 ms and judges by v2. - Refresh the dev session window. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -104,6 +104,10 @@ template <typename Host> class M10Bridge {
|
||||
if (issuer.size() != network::grant_public_key_bytes)
|
||||
return fail("error m10-issuer");
|
||||
server::RegionSessionsConfig config;
|
||||
// ADR-0097: snapshots carry the server tick every tick (10.5), so watermarks need not;
|
||||
// an advancing one goes out at most every 4 ticks (cumulative, nothing is lost).
|
||||
config.intents.watermark_every = 4;
|
||||
config.intents.watermark_coalesce = 4;
|
||||
config.transport.listen_address = o.listen.substr(0, colon);
|
||||
config.transport.port = port;
|
||||
config.transport.certificate_pem_file = o.certificate;
|
||||
@@ -206,6 +210,8 @@ template <typename Host> class M10Bridge {
|
||||
const server::HostFacts facts{s.ready && s.phase == server::HostPhase::running, s.epoch,
|
||||
s.simulated_tick};
|
||||
sessions_->poll(facts, wall_seconds(), std::chrono::steady_clock::now());
|
||||
const auto polled = std::chrono::steady_clock::now();
|
||||
poll_busy_ += polled - started;
|
||||
if (facts.ready && host.state() != nullptr && s.simulated_tick > replicated_) {
|
||||
replicated_ = s.simulated_tick;
|
||||
const auto& c = community(*host.state());
|
||||
@@ -219,6 +225,7 @@ template <typename Host> class M10Bridge {
|
||||
sessions_->replicate(s.simulated_tick, [&](const network::SessionInterest& in) {
|
||||
return source(c, in, s.simulated_tick);
|
||||
});
|
||||
replicate_busy_ += std::chrono::steady_clock::now() - polled;
|
||||
}
|
||||
if (facts.ready)
|
||||
for (const auto& a : sessions_->carry(s.simulated_tick + m10_carry_ahead_ticks)) {
|
||||
@@ -286,9 +293,24 @@ template <typename Host> class M10Bridge {
|
||||
static_cast<std::int64_t>(ticks));
|
||||
};
|
||||
const auto per_tick = per(busy_), session_per_tick = per(session_busy_);
|
||||
busy_ = session_busy_ = {};
|
||||
const auto poll_per_tick = per(poll_busy_), replicate_per_tick = per(replicate_busy_);
|
||||
busy_ = session_busy_ = poll_busy_ = replicate_busy_ = {};
|
||||
// A tick whose host step committed a batch (ADR-0097 measures those separately).
|
||||
const bool commit = s.commits != commits_seen_;
|
||||
commits_seen_ = s.commits;
|
||||
if (!counted)
|
||||
return;
|
||||
interval_.ticks += ticks;
|
||||
interval_.poll_us += poll_per_tick * ticks;
|
||||
interval_.replicate_us += replicate_per_tick * ticks;
|
||||
interval_.poll_max_us = std::max(interval_.poll_max_us, poll_per_tick);
|
||||
interval_.replicate_max_us = std::max(interval_.replicate_max_us, replicate_per_tick);
|
||||
if (commit) {
|
||||
++interval_.commit_ticks;
|
||||
interval_.commit_max_us = std::max(interval_.commit_max_us, per_tick);
|
||||
} else {
|
||||
interval_.other_max_us = std::max(interval_.other_max_us, per_tick);
|
||||
}
|
||||
const auto now_us =
|
||||
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::microseconds>(
|
||||
std::chrono::steady_clock::now().time_since_epoch())
|
||||
@@ -311,15 +333,28 @@ template <typename Host> class M10Bridge {
|
||||
return;
|
||||
reported_us_ = now_us;
|
||||
const auto& load = governor_.state();
|
||||
const auto ticks = std::max<std::uint64_t>(interval_.ticks, 1);
|
||||
// Since the last report: commit ticks (host committed a batch) apart from the others,
|
||||
// and the session layer split into poll (transport, control, intents) and replicate.
|
||||
std::printf("m10 load-report sessions=%zu p95_us=%llu p99_us=%llu max_us=%llu "
|
||||
"session_p95_us=%llu session_p99_us=%llu refusing=%d shed=%u tick_debt=%llu\n",
|
||||
"session_p95_us=%llu session_p99_us=%llu refusing=%d shed=%u tick_debt=%llu "
|
||||
"commit_ticks=%llu commit_max_us=%llu other_max_us=%llu poll_avg_us=%llu "
|
||||
"poll_max_us=%llu replicate_avg_us=%llu replicate_max_us=%llu\n",
|
||||
sessions_->sessions(), static_cast<unsigned long long>(load.p95_us),
|
||||
static_cast<unsigned long long>(load.p99_us),
|
||||
static_cast<unsigned long long>(load.max_us),
|
||||
static_cast<unsigned long long>(load.session_p95_us),
|
||||
static_cast<unsigned long long>(load.session_p99_us),
|
||||
governed_ && load.refusing ? 1 : 0, governed_ ? load.shed : 0U,
|
||||
static_cast<unsigned long long>(s.tick_debt));
|
||||
static_cast<unsigned long long>(s.tick_debt),
|
||||
static_cast<unsigned long long>(interval_.commit_ticks),
|
||||
static_cast<unsigned long long>(interval_.commit_max_us),
|
||||
static_cast<unsigned long long>(interval_.other_max_us),
|
||||
static_cast<unsigned long long>(interval_.poll_us / ticks),
|
||||
static_cast<unsigned long long>(interval_.poll_max_us),
|
||||
static_cast<unsigned long long>(interval_.replicate_us / ticks),
|
||||
static_cast<unsigned long long>(interval_.replicate_max_us));
|
||||
interval_ = {};
|
||||
}
|
||||
static const processes::EnvironmentalCommunity&
|
||||
community(const processes::EnvironmentalCommunity& c) {
|
||||
@@ -467,7 +502,12 @@ template <typename Host> class M10Bridge {
|
||||
server::LoadGovernor governor_{*server::LoadGovernor::create({})};
|
||||
bool governed_{true};
|
||||
std::uint64_t reported_us_{};
|
||||
std::chrono::nanoseconds busy_{}, session_busy_{};
|
||||
std::chrono::nanoseconds busy_{}, session_busy_{}, poll_busy_{}, replicate_busy_{};
|
||||
std::uint64_t commits_seen_{};
|
||||
struct Interval { // Since the last load-report.
|
||||
std::uint64_t ticks{}, poll_us{}, replicate_us{}, poll_max_us{}, replicate_max_us{},
|
||||
commit_ticks{}, commit_max_us{}, other_max_us{};
|
||||
} interval_;
|
||||
std::uint64_t measured_tick_{};
|
||||
std::unique_ptr<server::PostgresProbeLedger> probes_; // Handoff mode only.
|
||||
std::unique_ptr<server::PostgresHandoffLedger> handoff_; // Handoff mode only.
|
||||
|
||||
@@ -12,7 +12,7 @@ spec:
|
||||
env:
|
||||
- {name: REGION_SESSION, value: "dev"}
|
||||
- {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"}
|
||||
- {name: REGION_SESSION_WINDOW, value: "dev-20261008T184450-d705:1791498890"}
|
||||
- {name: REGION_SESSION_WINDOW, value: "dev-20261008T215407-71fa:1791510247"}
|
||||
- {name: REGION_M10_MAX_GRANT_LIFETIME, value: "14400"}
|
||||
ports:
|
||||
# QUIC (ADR-0090): UDP only; nothing listens on TCP 7400.
|
||||
|
||||
@@ -119,3 +119,13 @@ Schema v11 (`migrations/0011_region_actions.sql`, [region-actions-v1](../../sche
|
||||
the record by the next process, and a retry is a duplicate with the same outcome); an
|
||||
action whose tick had not run at SIGKILL has no record, answers `unknown`, and commits
|
||||
exactly once when retried.
|
||||
|
||||
## Amendment (2026-10-08, M10.9d, ADR-0097): watermark cadence
|
||||
Snapshots carry the server tick every tick (10.5), so the watermark no longer has to be
|
||||
every client's clock. In evo-server, an advancing `SimulatedThrough` goes to a session at
|
||||
most every 4 ticks, and idle sessions get one every 4 ticks
|
||||
(`IntentLimits::watermark_coalesce` and `watermark_every`; the library defaults stay 1).
|
||||
It is cumulative, so no admitted intention is ever missing from a later watermark; a client
|
||||
learns "simulated" up to 3 ticks (150 ms) later. Receipts are unchanged: one per intention.
|
||||
Reason: on the reference worker, datagram sends dominated the session layer, and the
|
||||
per-tick watermark was a quarter of them.
|
||||
|
||||
@@ -348,3 +348,42 @@ Same drill (steps 20/24/28/32, 60 s holds), 18:21–18:26 UTC, restored and veri
|
||||
from about 970 µs to about 360 µs per tick (2.7×).
|
||||
- **Next:** a third run with the fix. The remaining per-session cost is the comparison
|
||||
with each session's acknowledged baseline.
|
||||
|
||||
## 10.9d third reference run and decisions (2026-10-08, image `f378a43d…`)
|
||||
The renewal had rotated the dev CA, but the running pod still held the old server
|
||||
certificate. A `rollout restart` loaded the new one first (a throwaway-key login checked
|
||||
the trust), then the drill ran at 21:00–21:05 UTC and was restored.
|
||||
- **24 sessions held with every client budget met:** response p95 111 ms; 20 Hz; no
|
||||
drops or starvation. The governor (refusing at p95 ≥ 15 ms) refused logins beyond 24.
|
||||
- **Server at 24 sessions:**
|
||||
- tick work p95 9–16 ms, p99 25–34 ms;
|
||||
- with 0 sessions the p99 is already 23–25 ms: the synchronous commit flush;
|
||||
- session layer 0.62–0.69 ms per session at p99, 0.35–0.5 at p95;
|
||||
- no tick debt.
|
||||
- **Outcome latency in this run is invalid.** The bundle reused the earlier runs'
|
||||
accounts and the bots numbered actions from 1, so every action was a duplicate
|
||||
answered at once with its recorded outcome (16 ms). `evo-load` now numbers actions from
|
||||
the current time and counts `duplicate_actions` apart from outcome latency.
|
||||
|
||||
**Maintainer decisions (2026-10-08):**
|
||||
1. **Budget amended.** Per-tick region work is p95 ≤ 25 ms over all ticks, with ticks
|
||||
that commit a batch ≤ 50 ms, measured separately (`schemas/network/m10-budgets-v2.md`).
|
||||
Moving the commit off the simulation thread is a recorded follow-up.
|
||||
2. **Optimize first, then declare** the measured sustainable session count.
|
||||
|
||||
**Profile and second optimisation** (local, 24 bots):
|
||||
- The `load-report` now splits the session layer into `poll` and `replicate` and gives
|
||||
commit-tick and other-tick maxima.
|
||||
- `poll` cost about 1.3 ms per tick against 0.45 ms for `replicate`. Probes inside `poll`
|
||||
(temporary, not committed) attributed it to datagram sends of about 15 µs each, almost
|
||||
all in msquic's `DatagramSend`: a receipt and a watermark per session per tick, plus
|
||||
snapshot chunks.
|
||||
- Snapshots carry the server tick every tick (10.5), so the per-tick watermark is no
|
||||
longer needed for the client's clock. An advancing watermark is now sent at most every
|
||||
4 ticks (`IntentLimits::watermark_coalesce`; evo-server uses 4, the library default
|
||||
stays 1). Watermarks dropped from 4,800 to 1,200 per 10 s and `poll` from about 1.36 to
|
||||
1.05 ms per tick. Receipts stay one per intention (ADR-0092).
|
||||
- Further gains need fewer datagrams per tick, which is a protocol change and out of
|
||||
scope here.
|
||||
- The drill summary now judges by budgets v2 and prints commit and other-tick maxima and
|
||||
the poll and replicate averages.
|
||||
|
||||
@@ -5210,3 +5210,47 @@ Fix:
|
||||
|
||||
Executed: see the commit's check list (Linux suites, clang-tidy, Windows). Not done: the
|
||||
third reference run (needs the image, a new bundle and the go-ahead).
|
||||
|
||||
## 2026-10-08 — M10.9d third evolution-dev drill, budget v2, watermark coalescing
|
||||
|
||||
Third drill (image `f378a43d…`, 21:00–21:05 UTC). The pod was restarted first to load the
|
||||
renewed certificate (checked with a throwaway-key login).
|
||||
- **24 sessions, client:** all within budget (response p95 111 ms, 20 Hz, no starvation).
|
||||
- **24 sessions, server:**
|
||||
- tick work p95 9–16 ms, p99 25–34 ms;
|
||||
- with no session the p99 is 23–25 ms (the commit flush);
|
||||
- session layer 0.62–0.69 ms per session at p99;
|
||||
- governor cap at 24.
|
||||
- **Outcome latency invalid:** duplicate action ids, the reused accounts again.
|
||||
|
||||
Maintainer decisions:
|
||||
- budgets v2 (p95 ≤ 25 ms over all ticks, commit ticks ≤ 50 ms; follow-up to move the
|
||||
commit off the tick);
|
||||
- optimise before declaring.
|
||||
|
||||
Changes:
|
||||
- `evo-load`: action ids from the current time, `duplicate_actions`, receipts read before
|
||||
outcomes; the drill ramps at 500 ms.
|
||||
- The `load-report` splits poll and replicate and gives commit-tick and other-tick maxima.
|
||||
- `IntentLimits::watermark_coalesce`, 4 in evo-server, plus an ADR-0092 amendment.
|
||||
- `schemas/network/m10-budgets-v2.md`.
|
||||
- The drill summary judges by v2.
|
||||
|
||||
Local profile, 24 bots (WSL, Release):
|
||||
- `poll` about 1.3 ms against `replicate` about 0.45 ms per tick;
|
||||
- temporary probes inside `poll` (reverted): event handling about 0.76 ms per tick
|
||||
(9,600 datagrams per 10 s), watermark sends about 0.36 ms, control renewals about
|
||||
0.8 ms each;
|
||||
- with coalescing: watermark sends 4,800 → 1,200 per 10 s, `poll` about 1.05 ms per tick.
|
||||
|
||||
Executed:
|
||||
- WSL Release `quic` build plus the quic and network CTest labels (including ops-contracts and
|
||||
`session-fuzz-smoke`), passed.
|
||||
- The 18 Postgres suites (including the `evo-load` smoke), passed.
|
||||
- clang-tidy 19 on the changed sources, clean.
|
||||
- `bash -n tools/ops/load-drill.sh`, passed.
|
||||
- Windows `debug` 60/60, `quic` 5/5, `client-network` 15/15.
|
||||
|
||||
Not done:
|
||||
- shellcheck: not installed in this WSL.
|
||||
- The fourth drill (needs the new image, a fresh bundle and the go-ahead).
|
||||
|
||||
@@ -30,6 +30,10 @@ struct IntentLimits {
|
||||
// Ticks between watermarks to each session (1: every simulated tick). The watermark is
|
||||
// also every client's clock for the server tick until snapshots carry it (10.5).
|
||||
std::uint64_t watermark_every{1};
|
||||
// M10.9d (ADR-0097): while a session's watermark advances, at most one per this many
|
||||
// ticks (it is cumulative: nothing is lost, "simulated" is learned up to N-1 ticks later).
|
||||
// 1 keeps one per advance. Datagram sends dominate the session layer under load.
|
||||
std::uint64_t watermark_coalesce{1};
|
||||
};
|
||||
// The host's authority as admission must see it.
|
||||
struct AdmissionFacts {
|
||||
@@ -90,7 +94,8 @@ class IntentAdmission {
|
||||
const AdmissionFacts& facts);
|
||||
// Watermarks to send after the host simulated through facts.server_tick: to every
|
||||
// session (sequence 0 before its first simulated intention) at least every
|
||||
// `watermark_every` ticks, and whenever its sequence advances.
|
||||
// `watermark_every` ticks, and when its sequence advances (at most every
|
||||
// `watermark_coalesce` ticks).
|
||||
[[nodiscard]] std::vector<std::pair<std::uint64_t, SimulatedThrough>>
|
||||
simulated(const AdmissionFacts& facts);
|
||||
// Scheduled actions whose tick is <= `tick`: they ride that tick's batch from now on.
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
# M10 timing and workload budgets, version 1
|
||||
|
||||
Superseded by `m10-budgets-v2.md` (2026-10-08): per-tick region work is bounded at p95,
|
||||
with commit ticks bounded separately. Kept for history.
|
||||
|
||||
Normative table for ADR-0089. "Ratified" values were decided by the maintainer
|
||||
(2026-10-03); "budget" values are engineering limits derived from them and from the
|
||||
M10.1 traces and change only with measurements and an ADR-0089 amendment. Version 1
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
# M10 timing and workload budgets, version 2
|
||||
|
||||
Version 2 (2026-10-08, maintainer decision in M10.9d, ADR-0097) changes one row:
|
||||
per-tick region work. The first evolution-dev reference runs showed the host's synchronous
|
||||
commit flush alone reaching 23-25 ms at p99 with no session, about once per second, while
|
||||
other ticks stayed near 3.5 ms. Work is now bounded at p95 over all ticks, plus a separate
|
||||
bound for ticks that commit a batch. Moving the commit I/O off the simulation thread is a
|
||||
recorded follow-up (a persistence/ADR change), not part of M10. Affected: the load drill's
|
||||
summary (`tools/ops/load-drill.sh`) and the governor documentation; the governor itself
|
||||
already acts on p95. Version 1 is kept for history.
|
||||
|
||||
Normative table for ADR-0089. "Ratified" values were decided by the maintainer
|
||||
(2026-10-03); "budget" values are engineering limits derived from them and from the
|
||||
M10.1 traces and change only with measurements and an ADR-0089 amendment. Version 1
|
||||
applies to all M10 segments; a change bumps the version and lists affected tests.
|
||||
None of these values is a biological or game rule. Exception to note: genome v1 and
|
||||
metabolism express rates per tick, so the tick quantum currently fixes biological rates
|
||||
in real time; changing it requires a versioned rules migration (ADR-0089).
|
||||
|
||||
| Item | Value | Kind | Enforced/tested in |
|
||||
|---|---|---|---|
|
||||
| Authoritative tick quantum | 50,000 us (20 Hz) | ratified | checkpoints (`tick_microseconds`), region host |
|
||||
| Commit flush | 1,000 ms | ratified | region host `flush_interval` |
|
||||
| Final outcome delivery | p95 <= 1.5 s after admission (dev/stage, nominal network) | ratified | 10.4, 10.9 |
|
||||
| Responsiveness (input -> first reflecting snapshot) | LAN p95 <= 150 ms; 100 ms RTT p95 <= 250 ms | ratified | 10.6, 10.9 |
|
||||
| Reference hardware | server: one RKE2 worker (8 vCPU, 31.3 GiB) per region host; client: low-spec VM (M9.8g) | ratified | 10.9 |
|
||||
| Input scheduling window | server_tick + 1 ..= server_tick + 8 ticks | budget | 10.4 admission |
|
||||
| Client input lead | ceil((rtt/2 + jitter)/50 ms) + 1, clamped 2..8 ticks | budget | 10.4, 10.6 |
|
||||
| Late input | rejected, never rewound | budget | 10.4 |
|
||||
| Snapshot cadence (in interest) | every tick | budget | 10.5 |
|
||||
| Baselines retained | server <= 4 per session, each <= 40 ticks old; client <= 2 | budget | 10.5 |
|
||||
| Intention sequence | u64 per session, from 1, strictly increasing, no wrap | budget | 10.4 |
|
||||
| Tick, baseline id | u64, no wrap | budget | 10.4, 10.5 |
|
||||
| Realtime packet sequence | serial-number arithmetic >= 16 bits, window < half space | budget | 10.2 |
|
||||
| Session id / epoch | u64 server-assigned, bound to connection; epoch on every message; stale closes | budget | 10.3, 10.4 |
|
||||
| Intentions per session | 1 per tick (8 per 8-tick window, every attempt counted) | budget | 10.4 |
|
||||
| Unresolved intentions per session | 256 | budget | 10.4 |
|
||||
| Realtime datagram payload | <= 1,200 bytes | budget | 10.2 |
|
||||
| Reliable frame | <= 64 KiB | budget | 10.2 |
|
||||
| Replication per session | <= 16 KiB per tick | budget | 10.5 |
|
||||
| Client event queue | 1,024 events | budget | client session |
|
||||
| Heartbeat / liveness / handshake | 1 s / 5 s / 5 s | budget | 10.2, 10.7 |
|
||||
| Sessions per region | >= 20 required, 32 default | budget | 10.9 |
|
||||
| Per-tick region work | p95 <= 25 ms over all ticks on the reference worker; ticks that commit a batch <= 50 ms (measured separately) | budget (v2) | 10.9 (`m10 load-report` commit_max_us / other_max_us, `evo_m10_tick_work_*`, ADR-0097) |
|
||||
| Session handling | <= 0.5 ms per session per tick | budget | 10.9 (measured: `evo_m10_session_work_*`, ADR-0097) |
|
||||
| Host input queue | 64 inputs, 64 KiB | budget | region host |
|
||||
| Overload response | shed interest, refuse new sessions before tick debt | budget | 10.9a load governor: refuse at p95 >= 15 ms or debt >= 2, shed at p95 >= 20 ms or debt >= 4, relax below 10 ms after 5 s (ADR-0097; provisional until 10.9d) |
|
||||
|
||||
Determinism: authoritative state is exact given the checkpoint and committed per-tick
|
||||
inputs; timing, delivery, prediction, interpolation and presentation are tolerant
|
||||
(ADR-0089).
|
||||
@@ -17,7 +17,8 @@ std::expected<IntentAdmission, IntentConfigError> IntentAdmission::create(Intent
|
||||
if (limits.intent_window == 0 || limits.intent_window > 64 || limits.window_ticks == 0 ||
|
||||
limits.max_intents_per_window == 0 || limits.max_actions_per_window == 0 ||
|
||||
limits.max_unresolved == 0 || limits.max_unresolved > 4096 || limits.max_sessions == 0 ||
|
||||
limits.max_sessions > 1024 || limits.max_finals == 0 || limits.watermark_every == 0)
|
||||
limits.max_sessions > 1024 || limits.max_finals == 0 || limits.watermark_every == 0 ||
|
||||
limits.watermark_coalesce == 0)
|
||||
return std::unexpected(IntentConfigError::invalid);
|
||||
IntentAdmission admission{limits};
|
||||
admission.sessions_.reserve(limits.max_sessions);
|
||||
@@ -209,9 +210,10 @@ IntentAdmission::simulated(const AdmissionFacts& facts) {
|
||||
s.watermark = s.pending.front().sequence;
|
||||
s.pending.pop_front();
|
||||
}
|
||||
const bool changed = s.watermark != s.sent_watermark;
|
||||
const bool repeat = facts.server_tick < s.sent_at_tick ||
|
||||
facts.server_tick - s.sent_at_tick >= limits_.watermark_every;
|
||||
const auto since = facts.server_tick < s.sent_at_tick ? limits_.watermark_every
|
||||
: facts.server_tick - s.sent_at_tick;
|
||||
const bool changed = s.watermark != s.sent_watermark && since >= limits_.watermark_coalesce;
|
||||
const bool repeat = since >= limits_.watermark_every;
|
||||
if (changed || repeat) {
|
||||
s.sent_watermark = s.watermark;
|
||||
s.sent_at_tick = facts.server_tick;
|
||||
|
||||
@@ -551,6 +551,31 @@ void intent_admission() {
|
||||
marks = a.simulated(f); // Repeated after 20 ticks so a lost datagram is covered.
|
||||
CHECK(mark_of(marks, 1) == SimulatedThrough{5, 3, 130} &&
|
||||
mark_of(marks, 2) == SimulatedThrough{5, 0, 130});
|
||||
// M10.9d: coalesced, an advancing watermark goes out at most every 4 ticks, still
|
||||
// cumulative (the latest sequence), and idle repeats keep their interval.
|
||||
{
|
||||
IntentLimits l;
|
||||
l.watermark_every = 20;
|
||||
l.watermark_coalesce = 4;
|
||||
IntentLimits zero;
|
||||
zero.watermark_coalesce = 0;
|
||||
CHECK(!IntentAdmission::create(zero));
|
||||
auto c = *IntentAdmission::create(l);
|
||||
CHECK(c.open(3, alice));
|
||||
AdmissionFacts g{true, 5, 200};
|
||||
std::uint64_t sent = 0;
|
||||
for (std::uint64_t tick = 201; tick <= 216; ++tick) {
|
||||
CHECK(c.intention(3, SessionIntention{3, 5, tick - 200, tick, 0, 0}, g).status ==
|
||||
AdmitStatus::admitted);
|
||||
g.server_tick = tick;
|
||||
const auto m = mark_of(c.simulated(g), 3);
|
||||
if (m) {
|
||||
++sent;
|
||||
CHECK(m->sequence == tick - 200); // The newest simulated intention.
|
||||
}
|
||||
}
|
||||
CHECK(sent == 4); // Ticks 201 (first), 205, 209, 213.
|
||||
}
|
||||
// By default every session hears every simulated tick.
|
||||
auto every = *IntentAdmission::create(IntentLimits{});
|
||||
CHECK(every.open(7, alice));
|
||||
|
||||
+25
-12
@@ -132,6 +132,7 @@ struct Options {
|
||||
struct Samples {
|
||||
std::vector<std::uint64_t> response_us, receipt_us, outcome_us, login_us, correction_um,
|
||||
gap_ticks;
|
||||
std::uint64_t duplicate_actions{}; // Receipts saying duplicate: no outcome latency.
|
||||
std::uint64_t snapshots{}, bytes{}, intents{}, refused_intents{}, logins{}, drops{},
|
||||
login_failures{}, refused{};
|
||||
std::map<unsigned, std::uint64_t> closes; // By session close reason (255: no Close).
|
||||
@@ -181,6 +182,10 @@ class Load {
|
||||
static_cast<std::uint8_t>(((i + 1) >> (8U * k)) & 0xFFU);
|
||||
}
|
||||
b->serial = now_seconds() * 64; // Later runs supersede earlier ones.
|
||||
// Action ids are unique per account across runs too: a reused id is a duplicate,
|
||||
// answered at once with the recorded outcome (10.9d: the third dev drill's 16 ms
|
||||
// "outcomes" were duplicates of an earlier run's actions).
|
||||
b->next_action_id = now_seconds() * 1000;
|
||||
b->rng.seed(o_.seed * 1'000'003 + i);
|
||||
bots_.push_back(std::move(b));
|
||||
}
|
||||
@@ -396,6 +401,12 @@ class Load {
|
||||
const auto bytes = c.snapshot_bytes();
|
||||
b.s.bytes += bytes - std::min(bytes, b.last_bytes);
|
||||
b.last_bytes = bytes;
|
||||
// Receipts first: a duplicate's recorded outcome is not a latency sample.
|
||||
for (const auto& r : c.take_action_receipts())
|
||||
if (r.status == AdmitStatus::duplicate) {
|
||||
++b.s.duplicate_actions;
|
||||
b.outcome_pending.erase(r.action_id);
|
||||
}
|
||||
// Outcomes: final ones only.
|
||||
for (const auto& out : c.take_action_outcomes())
|
||||
if (out.result == ActionResult::committed_no_effect ||
|
||||
@@ -405,7 +416,6 @@ class Load {
|
||||
b.s.outcome_us.push_back(micros(now - it->second));
|
||||
b.outcome_pending.erase(it);
|
||||
}
|
||||
(void)c.take_action_receipts();
|
||||
// Once per server tick: an intention, a prediction, sometimes an action or interest.
|
||||
const auto tick = c.server_tick();
|
||||
if (tick == 0 || tick == b.last_tick)
|
||||
@@ -542,6 +552,7 @@ bool report(const Options& o, std::size_t n, std::size_t established, std::size_
|
||||
pooled.snapshots += s.snapshots;
|
||||
pooled.intents += s.intents;
|
||||
pooled.refused_intents += s.refused_intents;
|
||||
pooled.duplicate_actions += s.duplicate_actions;
|
||||
pooled.logins += s.logins;
|
||||
pooled.drops += s.drops;
|
||||
pooled.login_failures += s.login_failures;
|
||||
@@ -576,17 +587,19 @@ bool report(const Options& o, std::size_t n, std::size_t established, std::size_
|
||||
line("outcome", pooled.outcome_us, outcome95, 1000.0, "ms");
|
||||
line("login", pooled.login_us, login95, 1000.0, "ms");
|
||||
line("correction", pooled.correction_um, correction95, 1000.0, "mm");
|
||||
std::printf("load snapshots hz min/med/max=%.2f/%.2f/%.2f gap_ticks p99/max=%llu/%llu "
|
||||
"kib_s med/max=%llu/%llu intents=%llu refused_intents=%llu\n",
|
||||
static_cast<double>(percentile(rates, 0.0)) / 1000.0,
|
||||
static_cast<double>(percentile(rates, 0.5)) / 1000.0,
|
||||
static_cast<double>(percentile(rates, 1.0)) / 1000.0,
|
||||
static_cast<unsigned long long>(percentile(pooled.gap_ticks, 0.99)),
|
||||
static_cast<unsigned long long>(percentile(pooled.gap_ticks, 1.0)),
|
||||
static_cast<unsigned long long>(percentile(kib, 0.5)),
|
||||
static_cast<unsigned long long>(percentile(kib, 1.0)),
|
||||
static_cast<unsigned long long>(pooled.intents),
|
||||
static_cast<unsigned long long>(pooled.refused_intents));
|
||||
std::printf(
|
||||
"load snapshots hz min/med/max=%.2f/%.2f/%.2f gap_ticks p99/max=%llu/%llu "
|
||||
"kib_s med/max=%llu/%llu intents=%llu refused_intents=%llu duplicate_actions=%llu\n",
|
||||
static_cast<double>(percentile(rates, 0.0)) / 1000.0,
|
||||
static_cast<double>(percentile(rates, 0.5)) / 1000.0,
|
||||
static_cast<double>(percentile(rates, 1.0)) / 1000.0,
|
||||
static_cast<unsigned long long>(percentile(pooled.gap_ticks, 0.99)),
|
||||
static_cast<unsigned long long>(percentile(pooled.gap_ticks, 1.0)),
|
||||
static_cast<unsigned long long>(percentile(kib, 0.5)),
|
||||
static_cast<unsigned long long>(percentile(kib, 1.0)),
|
||||
static_cast<unsigned long long>(pooled.intents),
|
||||
static_cast<unsigned long long>(pooled.refused_intents),
|
||||
static_cast<unsigned long long>(pooled.duplicate_actions));
|
||||
std::printf("load client_cpu_pct=%.0f\n", 100.0 * cpu / std::max(seconds, 0.001));
|
||||
if (!server.empty()) {
|
||||
std::printf("load server");
|
||||
|
||||
+13
-6
@@ -129,7 +129,7 @@ set +e
|
||||
"$tool" --connect "$connect" --server-name "$server_name" --ca "$tokens/dev-ca.pem" \
|
||||
--grants "$tokens/load-grants.bin" --grants-per-bot "$per_bot" --world "$world" \
|
||||
--frame "$frame" --region "$region" --steps "$steps" --hold "$hold" --warmup "$warmup" \
|
||||
--threads 8 --per-client > "$work/load.log" 2>&1
|
||||
--threads 8 --ramp-ms 500 --per-client > "$work/load.log" 2>&1
|
||||
load_exit=$?
|
||||
set -e
|
||||
sleep 12 # One more server report after the last hold.
|
||||
@@ -162,14 +162,21 @@ for line in open(load, encoding='utf-8', errors='replace'):
|
||||
if not inside:
|
||||
print(f"drill step N={n} server_reports=0")
|
||||
continue
|
||||
worst = lambda key: max(r[key] for r in inside)
|
||||
worst = lambda key: max(r.get(key, 0) for r in inside)
|
||||
mean = lambda key: sum(r.get(key, 0) for r in inside) / len(inside)
|
||||
sessions = max(1, worst('sessions'))
|
||||
# Budgets v2 (schemas/network/m10-budgets-v2.md): tick work p95 <= 25 ms over all ticks,
|
||||
# ticks that commit a batch <= 50 ms; session layer p99 <= 0.5 ms per session; no debt.
|
||||
within = (worst('p95_us') <= 25000 and worst('commit_max_us') <= 50000 and
|
||||
worst('session_p99_us') <= 500 * sessions and worst('tick_debt') == 0)
|
||||
print(f"drill step N={n} server_reports={len(inside)} sessions_max={worst('sessions')} "
|
||||
f"work_p99_max_ms={worst('p99_us') / 1000:.1f} work_max_ms={worst('max_us') / 1000:.1f} "
|
||||
f"work_p95_max_ms={worst('p95_us') / 1000:.1f} work_p99_max_ms={worst('p99_us') / 1000:.1f} "
|
||||
f"commit_max_ms={worst('commit_max_us') / 1000:.1f} other_max_ms={worst('other_max_us') / 1000:.1f} "
|
||||
f"session_p95_max_ms={worst('session_p95_us') / 1000:.1f} "
|
||||
f"session_p99_max_ms={worst('session_p99_us') / 1000:.1f} "
|
||||
f"per_session_ms={worst('session_p99_us') / 1000 / sessions:.3f} "
|
||||
f"per_session_p99_ms={worst('session_p99_us') / 1000 / sessions:.3f} "
|
||||
f"poll_avg_ms={mean('poll_avg_us') / 1000:.2f} replicate_avg_ms={mean('replicate_avg_us') / 1000:.2f} "
|
||||
f"refusing_max={worst('refusing')} shed_max={worst('shed')} "
|
||||
f"tick_debt_max={worst('tick_debt')} "
|
||||
f"server_within_budget={int(worst('p99_us') <= 25000 and worst('session_p99_us') <= 500 * sessions and worst('tick_debt') == 0)}")
|
||||
f"tick_debt_max={worst('tick_debt')} server_within_budget={int(within)}")
|
||||
PY
|
||||
say "drill complete (evo-load exit $load_exit); restoring"
|
||||
|
||||
Reference in New Issue
Block a user