From 46b6358606323f417e4e70333b7bf8268d3ffc69 Mon Sep 17 00:00:00 2001 From: dingo Date: Thu, 8 Oct 2026 15:03:45 -0400 Subject: [PATCH] Share replication encoding per interest; fix dev retained builds ADR-0097. The second evolution-dev load drill (image 7952185d) held 20 sessions with client budgets met (response p95 111 ms, outcome p95 about 1.2 s, 20 Hz, no starvation) but the server over its own: tick work p99 48-70 ms (25 ms budget), session layer 1.2-2.0 ms per session at p99 (0.5 ms budget), governor refusing and shedding to level 3 to little effect. ReplicationServer::tick re-encoded every environment cell and organism for each session. Sessions with the same requested interest (no perception predicate) now share one encoding per tick; markers stay per session. Locally 21 sessions: about 970 us -> 360 us per tick. The drill now waits for two consecutive calm reports before the bots. The dev overlay's REGION_RETAINED listed the deployed build itself (61df6a1, f88bee2), which the deploy policy refuses (CI deploy failed on those commits); it now lists only the other builds, keeping 6c96629e, the build of the region head. Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 --- .../region/dev-session-route/listener.yaml | 2 +- .../kubernetes/region/dev/region-values.yaml | 2 +- .../0097-m10-load-and-abuse-qualification.md | 29 +++++ docs/engineering/VALIDATION.md | 26 +++++ src/network/protocol/replication.cpp | 107 ++++++++++++------ tests/network_replication.cpp | 51 +++++++++ tools/ops/load-drill.sh | 11 +- 7 files changed, 187 insertions(+), 41 deletions(-) diff --git a/deploy/kubernetes/region/dev-session-route/listener.yaml b/deploy/kubernetes/region/dev-session-route/listener.yaml index a7d91cb..6323b61 100644 --- a/deploy/kubernetes/region/dev-session-route/listener.yaml +++ b/deploy/kubernetes/region/dev-session-route/listener.yaml @@ -12,7 +12,7 @@ spec: env: - {name: REGION_SESSION, value: "dev"} - {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"} - - {name: REGION_SESSION_WINDOW, value: "dev-renew-20261008T181816Z:1791488896"} + - {name: REGION_SESSION_WINDOW, value: "dev-20261008T184450-d705:1791498890"} - {name: REGION_M10_MAX_GRANT_LIFETIME, value: "14400"} ports: # QUIC (ADR-0090): UDP only; nothing listens on TCP 7400. diff --git a/deploy/kubernetes/region/dev/region-values.yaml b/deploy/kubernetes/region/dev/region-values.yaml index d87cd52..b87703f 100644 --- a/deploy/kubernetes/region/dev/region-values.yaml +++ b/deploy/kubernetes/region/dev/region-values.yaml @@ -13,7 +13,7 @@ spec: - {name: REGION_ID, value: "1"} - {name: REGION_BUILD, value: 7952185d5ef96504b5793b76fe72e42cbe9c9ac19149bf912adb2c3a735f8dbf} # Keep the currently promoted build plus prior committed builds recoverable. - - {name: REGION_RETAINED, value: "7952185d5ef96504b5793b76fe72e42cbe9c9ac19149bf912adb2c3a735f8dbf 6c96629eb3c26513d55207b109fe2a9873a58f43510bd067b7d5422572e3fb42 ecac8f9f2489c547ce58f12ed01baaa3939c8adbf8aed05c313abbf422cdc855 4af7209941643e0c6a100e2b4e2fef8a959b87fdbfac2173b7867cfc1808bd66 edb39f956df1f481d7ba724cac4ceb3a4fe494efc02f994d31d6c96dfae04a78 0ded41070dd3e0c0daa43e13dadc58c449865eec00530cfceff3c1579c66e916"} + - {name: REGION_RETAINED, value: "6c96629eb3c26513d55207b109fe2a9873a58f43510bd067b7d5422572e3fb42 ecac8f9f2489c547ce58f12ed01baaa3939c8adbf8aed05c313abbf422cdc855 4af7209941643e0c6a100e2b4e2fef8a959b87fdbfac2173b7867cfc1808bd66 edb39f956df1f481d7ba724cac4ceb3a4fe494efc02f994d31d6c96dfae04a78 0ded41070dd3e0c0daa43e13dadc58c449865eec00530cfceff3c1579c66e916"} --- apiVersion: batch/v1 kind: Job diff --git a/docs/adr/0097-m10-load-and-abuse-qualification.md b/docs/adr/0097-m10-load-and-abuse-qualification.md index f579d3f..adfe925 100644 --- a/docs/adr/0097-m10-load-and-abuse-qualification.md +++ b/docs/adr/0097-m10-load-and-abuse-qualification.md @@ -319,3 +319,32 @@ Fixes before the next run: - waits after restoring for a Ready pod other than the drill pod (the old check could accept a pod still running during the rollout); - checks that `REGION_INPUTS` is empty again. + +## 10.9d second reference run (2026-10-08, image `7952185d…`) +Same drill (steps 20/24/28/32, 60 s holds), 18:21–18:26 UTC, restored and verified. +- **Client side, 20 sessions:** all established, no drops, no starvation; response p95 + 111 ms (LAN budget 150); outcome p95 1.18–1.21 s (budget 1.5); snapshots at a median of + 19.8–20 Hz. No login beyond 20 succeeded. +- **Server side, 20 sessions:** + + | Measure | Value | Budget | + |---|---|---| + | Tick work p95 | 8–16 ms (one report 27 ms) | — | + | Tick work p99 | 48–70 ms | 25 ms | + | Session layer p95 | about the whole tick work | — | + | Session layer p99 | 23–41 ms, 1.2–2.0 ms per session | 0.5 ms per session | + + There was no tick debt. The governor refused from 18:22:15 and shed to level 3 from + 18:22:42, which barely lowered the cost. +- **The startup trip happened again, briefly.** The first calm report came 4 s after the + restart, and the governor (now over a full window of startup ticks) refused the first + logins until 18:21:36. The drill now waits for two consecutive calm reports. +- **Cause:** `ReplicationServer::tick` encoded every environment cell and organism again + for each session, about 225 records × 20 sessions each tick. +- **Fix:** sessions with the same requested interest and no perception predicate share one + encoding per tick. Markers, and everything when perception filters, stay per session. + Behaviour is unchanged: the existing replication tests pass, and a shared-interest test + checks 20 sessions on one square and one on another. Measured locally, 21 sessions went + from about 970 µs to about 360 µs per tick (2.7×). +- **Next:** a third run with the fix. The remaining per-session cost is the comparison + with each session's acknowledged baseline. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 497b396..d46dad1 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -5184,3 +5184,29 @@ Fixes (in the following commit): stricter restore check. The next run needs a new image and a new bundle: some of this bundle's grants are spent. + +## 2026-10-08 — M10.9d second evolution-dev load drill and the shared replication truth + +Second drill (image `7952185d…`), 18:21–18:26 UTC, restore verified: +- **20 sessions, client side:** all established; response p95 111 ms; outcome p95 about + 1.2 s; snapshots at about 20 Hz; no starvation. +- **20 sessions, server side:** + - tick work p95 8–16 ms and p99 48–70 ms (budget 25 ms); + - session layer p99 1.2–2.0 ms per session (budget 0.5 ms); + - no tick debt; + - the governor refused from 18:22:15 and shed to level 3. +- **Startup:** the governor refused the first logins until 18:21:36, after a single calm + report. The drill now waits for two. + +Fix: +- `ReplicationServer::tick` shares one encoding of the environment and organisms per + distinct interest; per session only markers, or everything with a perception + predicate. +- New `network` test: 21 sessions on two interests. +- Development reference, Linux Release: + - 21 sessions: about 356–366 µs per tick shared, against 947–995 µs per session + (old code, same test); + - existing replication tests unchanged. + +Executed: see the commit's check list (Linux suites, clang-tidy, Windows). Not done: the +third reference run (needs the image, a new bundle and the go-ahead). diff --git a/src/network/protocol/replication.cpp b/src/network/protocol/replication.cpp index 852a3ef..ab1db3a 100644 --- a/src/network/protocol/replication.cpp +++ b/src/network/protocol/replication.cpp @@ -1,7 +1,9 @@ #include "evo/network/replication.hpp" #include #include +#include #include +#include namespace evo::network { namespace { @@ -242,7 +244,17 @@ std::vector>> ReplicationSer const auto overhead = session_header_bytes + snapshot_chunk_header_bytes; if (tick == 0) return out; - std::vector> sources; + // Per distinct requested interest: the source, and (without a perception predicate) the + // encoded environment and organisms in its effective square, shared by every session + // with that interest (10.9d, ADR-0097: encoding them per session dominated the session + // layer on the reference worker). A deque keeps references stable as entries are added. + using Truth = std::map, std::uint64_t>>; + struct Shared { + SessionInterest interest; + ReplicationSource source; + std::optional truth; + }; + std::deque sources; for (auto& s : sessions_) { if (!s.interest || (!s.baselines.empty() && s.baselines.back().tick >= tick)) continue; @@ -252,36 +264,55 @@ std::vector>> ReplicationSer const auto capacity = payload - overhead; // Record bytes per chunk. const auto& wanted = *s.interest; auto cached = - std::ranges::find_if(sources, [&](const auto& e) { return e.first == wanted; }); + std::ranges::find_if(sources, [&](const Shared& e) { return e.interest == wanted; }); if (cached == sources.end()) { - sources.emplace_back(wanted, source(wanted)); + sources.push_back({wanted, source(wanted), std::nullopt}); cached = std::prev(sources.end()); } auto in = wanted; // Encoded within the shed cap; the stored interest is unchanged. in.radius = std::min(in.radius, shed_radius.at(shed_)); - const auto& src = cached->second; + const auto& src = cached->source; // Spatial filter, then perception: nothing outside either is ever encoded. - std::map, std::uint64_t>> truth; - for (const auto& c : src.environment) - if (c.voxel.z == in.z && distance(c.voxel.x, c.voxel.y, in) <= in.radius) - truth[Key{environment_kind, c.voxel.x, c.voxel.y, c.voxel.z}] = { - encode_record(c), distance(c.voxel.x, c.voxel.y, in)}; - for (const auto& o : src.organisms) - if (o.voxel.z == in.z && distance(o.voxel.x, o.voxel.y, in) <= in.radius && - (!perceives || perceives(s.id, o))) - truth[Key{organism_kind, static_cast(o.id), 0, 0}] = { - encode_record(o), distance(o.voxel.x, o.voxel.y, in)}; + const auto fill = [&](Truth& into, bool filtered) { + for (const auto& c : src.environment) + if (c.voxel.z == in.z && distance(c.voxel.x, c.voxel.y, in) <= in.radius) + into[Key{environment_kind, c.voxel.x, c.voxel.y, c.voxel.z}] = { + encode_record(c), distance(c.voxel.x, c.voxel.y, in)}; + for (const auto& o : src.organisms) + if (o.voxel.z == in.z && distance(o.voxel.x, o.voxel.y, in) <= in.radius && + (!filtered || perceives(s.id, o))) + into[Key{organism_kind, static_cast(o.id), 0, 0}] = { + encode_record(o), distance(o.voxel.x, o.voxel.y, in)}; + }; + // `own`: this session's entries (markers; everything when perception filters). + Truth own; + const Truth* shared = nullptr; + if (perceives) { + fill(own, true); + } else { + if (!cached->truth) { + cached->truth.emplace(); + fill(*cached->truth, false); + } + shared = &*cached->truth; + } // Markers: the own one always; others in the square of the slice. Distance 0 puts // every marker first and makes it due every tick it changed. if (src.marker_area) for (const auto& m : src.markers) { - const bool own = !m.ghost && m.session == s.id; - if (!own && (src.marker_area->z != in.z || - distance(cell_of(m.state.x), cell_of(m.state.y), in) > in.radius)) + const bool own_marker = !m.ghost && m.session == s.id; + if (!own_marker && + (src.marker_area->z != in.z || + distance(cell_of(m.state.x), cell_of(m.state.y), in) > in.radius)) continue; - truth[Key{marker_kind, static_cast(m.id), 0, 0}] = { - encode_record(m, src.marker_area->z, own ? &*src.marker_area : nullptr), 0}; + own[Key{marker_kind, static_cast(m.id), 0, 0}] = { + encode_record(m, src.marker_area->z, own_marker ? &*src.marker_area : nullptr), + 0}; } + // Marker keys never collide with environment or organism keys (the kind differs). + const auto contains = [&](const Key& key) { + return own.contains(key) || (shared != nullptr && shared->contains(key)); + }; // Baselines older than the limit are forgotten; an expired ack means a fresh one. while (!s.baselines.empty() && tick - s.baselines.front().tick > limits_.max_baseline_age) s.baselines.pop_front(); @@ -298,7 +329,7 @@ std::vector>> ReplicationSer std::vector> removals; std::vector removed; for (const auto& [key, entry] : base_state) - if (!truth.contains(key)) { + if (!contains(key)) { removals.push_back( std::get<0>(key) == environment_kind ? remove_environment({std::get<1>(key), std::get<2>(key), std::get<3>(key)}) @@ -309,21 +340,27 @@ std::vector>> ReplicationSer } std::vector candidates; std::uint64_t deferred = 0; - for (const auto& [key, value] : truth) { - const auto known = base_state.find(key); - if (known != base_state.end() && known->second.record == value.first) - continue; // The client already holds this value. - const bool due = known == base_state.end() || value.second <= near_radius || - tick - known->second.updated >= far_interval; - if (!due) { - ++deferred; - continue; - } - const auto staleness = known == base_state.end() - ? std::numeric_limits::max() - : tick - known->second.updated; - candidates.push_back({value.second, staleness, key, &value.first}); - } + const auto consider = + [&](const Key& key, const std::pair, std::uint64_t>& value) { + const auto known = base_state.find(key); + if (known != base_state.end() && known->second.record == value.first) + return; // The client already holds this value. + const bool due = known == base_state.end() || value.second <= near_radius || + tick - known->second.updated >= far_interval; + if (!due) { + ++deferred; + return; + } + const auto staleness = known == base_state.end() + ? std::numeric_limits::max() + : tick - known->second.updated; + candidates.push_back({value.second, staleness, key, &value.first}); + }; + if (shared != nullptr) + for (const auto& [key, value] : *shared) + consider(key, value); + for (const auto& [key, value] : own) + consider(key, value); std::ranges::sort(candidates, [](const Candidate& a, const Candidate& b) { return std::tie(a.distance, b.staleness, a.key) < std::tie(b.distance, a.staleness, b.key); diff --git a/tests/network_replication.cpp b/tests/network_replication.cpp index 3efa9aa..b24efcd 100644 --- a/tests/network_replication.cpp +++ b/tests/network_replication.cpp @@ -4,6 +4,7 @@ #include "evo/network/replication.hpp" #include "network_test.hpp" #include +#include #include #include #include @@ -447,8 +448,58 @@ void shedding() { CHECK(client.latest()->environment.size() == 225 && baseline != 0); } +// M10.9d (ADR-0097): sessions with the same interest share one encoding of the environment +// per tick; each still converges on exactly its square, and another interest gets its own. +// Prints the server's time per tick for 20 sessions (a development reference). +void shared_interest() { + auto server = *ReplicationServer::create(limits()); + World world; + constexpr std::uint64_t sessions = 21; + std::vector clients; + for (std::uint64_t id = 1; id <= sessions; ++id) { + CHECK(server.open(id, 5)); + // Session 21 looks elsewhere (centre 3, 0): its own square. + CHECK(server.set_interest(id, {id == sessions ? 3 : 0, 0, slice, 7})); + clients.emplace_back(identity, 5); + } + const auto source = [&](const SessionInterest&) { return world.source(); }; + std::uint64_t tick = 0; + std::chrono::nanoseconds spent{}; + const auto step = [&] { + const auto started = std::chrono::steady_clock::now(); + const auto out = server.tick(++tick, source, {}); + spent += std::chrono::steady_clock::now() - started; + for (std::uint64_t id = 1; id <= sessions; ++id) + for (const auto& a : deliver(clients[id - 1], for_session(out, id))) + server.acknowledge(id, a); + }; + for (int i = 0; i < 6; ++i) + step(); + for (std::uint64_t id = 1; id <= sessions; ++id) { + const auto& view = clients[id - 1].latest()->environment; + const std::int64_t centre = id == sessions ? 3 : 0; + CHECK(view.size() == 225 && std::ranges::all_of(view, [&](const auto& c) { + return std::abs(c.voxel.x - centre) <= 7 && std::abs(c.voxel.y) <= 7; + })); + } + // A change reaches every session that sees it. + world.quantity[{1, 1}] = 77; + spent = {}; + for (int i = 0; i < 40; ++i) + step(); + for (auto& c : clients) + CHECK(std::ranges::any_of(c.latest()->environment, [](const auto& e) { + return e.voxel.x == 1 && e.voxel.y == 1 && e.quantity[0] == 77; + })); + std::printf("replication shared_interest sessions=%llu us_per_tick=%lld\n", + static_cast(sessions), + static_cast( + std::chrono::duration_cast(spent).count() / 40)); +} + void replication_tests() { codec_v2(); + shared_interest(); convergence_and_deltas(); shedding(); concealment(); diff --git a/tools/ops/load-drill.sh b/tools/ops/load-drill.sh index 57d521c..dfc6c4b 100644 --- a/tools/ops/load-drill.sh +++ b/tools/ops/load-drill.sh @@ -112,14 +112,17 @@ say "enabling synthetic inputs in $ns (drill-time override, restored on exit)" "${k[@]}" set env -n "$ns" deployment/evo-region REGION_INPUTS=development-v1 > /dev/null pod="$(ready_pod "$before" 240)" || fail "no Ready evo-region pod with synthetic inputs" drill_pod="$pod" -# Warm up: wait for a full governor window and a calm report (refusing=0 shed=0). +# Warm up: two consecutive calm reports (refusing=0 shed=0, about 10 s apart), so the +# governor's window no longer holds the restart's first ticks when the bots arrive (the +# second reference run met one calm report, then a refusal at the first login). calm="" for _ in $(seq 1 60); do sleep 3 - last="$("${k[@]}" logs -n "$ns" --since=15s "$pod" 2> /dev/null | grep "m10 load-report" | tail -1)" - case "$last" in *"refusing=0 shed=0"*) calm=1; break ;; esac + last2="$("${k[@]}" logs -n "$ns" --since=25s "$pod" 2> /dev/null | grep "m10 load-report" | tail -2)" + last="$(echo "$last2" | tail -1)" + [ "$(echo "$last2" | grep -c "refusing=0 shed=0")" = 2 ] && { calm=1; break; } done -[ -n "$calm" ] || fail "the region did not report calm within 3 minutes of starting: $last" +[ -n "$calm" ] || fail "the region did not report calm twice within 3 minutes of starting: $last" say "region calm: $last" say "running evo-load against $connect" set +e