diff --git a/deploy/kubernetes/region/dev-session-route/listener.yaml b/deploy/kubernetes/region/dev-session-route/listener.yaml index a7d91cb..6323b61 100644 --- a/deploy/kubernetes/region/dev-session-route/listener.yaml +++ b/deploy/kubernetes/region/dev-session-route/listener.yaml @@ -12,7 +12,7 @@ spec: env: - {name: REGION_SESSION, value: "dev"} - {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"} - - {name: REGION_SESSION_WINDOW, value: "dev-renew-20261008T181816Z:1791488896"} + - {name: REGION_SESSION_WINDOW, value: "dev-20261008T184450-d705:1791498890"} - {name: REGION_M10_MAX_GRANT_LIFETIME, value: "14400"} ports: # QUIC (ADR-0090): UDP only; nothing listens on TCP 7400. diff --git a/deploy/kubernetes/region/dev/region-values.yaml b/deploy/kubernetes/region/dev/region-values.yaml index d87cd52..b87703f 100644 --- a/deploy/kubernetes/region/dev/region-values.yaml +++ b/deploy/kubernetes/region/dev/region-values.yaml @@ -13,7 +13,7 @@ spec: - {name: REGION_ID, value: "1"} - {name: REGION_BUILD, value: 7952185d5ef96504b5793b76fe72e42cbe9c9ac19149bf912adb2c3a735f8dbf} # Keep the currently promoted build plus prior committed builds recoverable. - - {name: REGION_RETAINED, value: "7952185d5ef96504b5793b76fe72e42cbe9c9ac19149bf912adb2c3a735f8dbf 6c96629eb3c26513d55207b109fe2a9873a58f43510bd067b7d5422572e3fb42 ecac8f9f2489c547ce58f12ed01baaa3939c8adbf8aed05c313abbf422cdc855 4af7209941643e0c6a100e2b4e2fef8a959b87fdbfac2173b7867cfc1808bd66 edb39f956df1f481d7ba724cac4ceb3a4fe494efc02f994d31d6c96dfae04a78 0ded41070dd3e0c0daa43e13dadc58c449865eec00530cfceff3c1579c66e916"} + - {name: REGION_RETAINED, value: "6c96629eb3c26513d55207b109fe2a9873a58f43510bd067b7d5422572e3fb42 ecac8f9f2489c547ce58f12ed01baaa3939c8adbf8aed05c313abbf422cdc855 4af7209941643e0c6a100e2b4e2fef8a959b87fdbfac2173b7867cfc1808bd66 edb39f956df1f481d7ba724cac4ceb3a4fe494efc02f994d31d6c96dfae04a78 0ded41070dd3e0c0daa43e13dadc58c449865eec00530cfceff3c1579c66e916"} --- apiVersion: batch/v1 kind: Job diff --git a/docs/adr/0097-m10-load-and-abuse-qualification.md b/docs/adr/0097-m10-load-and-abuse-qualification.md index f579d3f..adfe925 100644 --- a/docs/adr/0097-m10-load-and-abuse-qualification.md +++ b/docs/adr/0097-m10-load-and-abuse-qualification.md @@ -319,3 +319,32 @@ Fixes before the next run: - waits after restoring for a Ready pod other than the drill pod (the old check could accept a pod still running during the rollout); - checks that `REGION_INPUTS` is empty again. + +## 10.9d second reference run (2026-10-08, image `7952185d…`) +Same drill (steps 20/24/28/32, 60 s holds), 18:21–18:26 UTC, restored and verified. +- **Client side, 20 sessions:** all established, no drops, no starvation; response p95 + 111 ms (LAN budget 150); outcome p95 1.18–1.21 s (budget 1.5); snapshots at a median of + 19.8–20 Hz. No login beyond 20 succeeded. +- **Server side, 20 sessions:** + + | Measure | Value | Budget | + |---|---|---| + | Tick work p95 | 8–16 ms (one report 27 ms) | — | + | Tick work p99 | 48–70 ms | 25 ms | + | Session layer p95 | about the whole tick work | — | + | Session layer p99 | 23–41 ms, 1.2–2.0 ms per session | 0.5 ms per session | + + There was no tick debt. The governor refused from 18:22:15 and shed to level 3 from + 18:22:42, which barely lowered the cost. +- **The startup trip happened again, briefly.** The first calm report came 4 s after the + restart, and the governor (now over a full window of startup ticks) refused the first + logins until 18:21:36. The drill now waits for two consecutive calm reports. +- **Cause:** `ReplicationServer::tick` encoded every environment cell and organism again + for each session, about 225 records × 20 sessions each tick. +- **Fix:** sessions with the same requested interest and no perception predicate share one + encoding per tick. Markers, and everything when perception filters, stay per session. + Behaviour is unchanged: the existing replication tests pass, and a shared-interest test + checks 20 sessions on one square and one on another. Measured locally, 21 sessions went + from about 970 µs to about 360 µs per tick (2.7×). +- **Next:** a third run with the fix. The remaining per-session cost is the comparison + with each session's acknowledged baseline. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 497b396..d46dad1 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -5184,3 +5184,29 @@ Fixes (in the following commit): stricter restore check. The next run needs a new image and a new bundle: some of this bundle's grants are spent. + +## 2026-10-08 — M10.9d second evolution-dev load drill and the shared replication truth + +Second drill (image `7952185d…`), 18:21–18:26 UTC, restore verified: +- **20 sessions, client side:** all established; response p95 111 ms; outcome p95 about + 1.2 s; snapshots at about 20 Hz; no starvation. +- **20 sessions, server side:** + - tick work p95 8–16 ms and p99 48–70 ms (budget 25 ms); + - session layer p99 1.2–2.0 ms per session (budget 0.5 ms); + - no tick debt; + - the governor refused from 18:22:15 and shed to level 3. +- **Startup:** the governor refused the first logins until 18:21:36, after a single calm + report. The drill now waits for two. + +Fix: +- `ReplicationServer::tick` shares one encoding of the environment and organisms per + distinct interest; per session only markers, or everything with a perception + predicate. +- New `network` test: 21 sessions on two interests. +- Development reference, Linux Release: + - 21 sessions: about 356–366 µs per tick shared, against 947–995 µs per session + (old code, same test); + - existing replication tests unchanged. + +Executed: see the commit's check list (Linux suites, clang-tidy, Windows). Not done: the +third reference run (needs the image, a new bundle and the go-ahead). diff --git a/src/network/protocol/replication.cpp b/src/network/protocol/replication.cpp index 852a3ef..ab1db3a 100644 --- a/src/network/protocol/replication.cpp +++ b/src/network/protocol/replication.cpp @@ -1,7 +1,9 @@ #include "evo/network/replication.hpp" #include #include +#include #include +#include namespace evo::network { namespace { @@ -242,7 +244,17 @@ std::vector>> ReplicationSer const auto overhead = session_header_bytes + snapshot_chunk_header_bytes; if (tick == 0) return out; - std::vector> sources; + // Per distinct requested interest: the source, and (without a perception predicate) the + // encoded environment and organisms in its effective square, shared by every session + // with that interest (10.9d, ADR-0097: encoding them per session dominated the session + // layer on the reference worker). A deque keeps references stable as entries are added. + using Truth = std::map, std::uint64_t>>; + struct Shared { + SessionInterest interest; + ReplicationSource source; + std::optional truth; + }; + std::deque sources; for (auto& s : sessions_) { if (!s.interest || (!s.baselines.empty() && s.baselines.back().tick >= tick)) continue; @@ -252,36 +264,55 @@ std::vector>> ReplicationSer const auto capacity = payload - overhead; // Record bytes per chunk. const auto& wanted = *s.interest; auto cached = - std::ranges::find_if(sources, [&](const auto& e) { return e.first == wanted; }); + std::ranges::find_if(sources, [&](const Shared& e) { return e.interest == wanted; }); if (cached == sources.end()) { - sources.emplace_back(wanted, source(wanted)); + sources.push_back({wanted, source(wanted), std::nullopt}); cached = std::prev(sources.end()); } auto in = wanted; // Encoded within the shed cap; the stored interest is unchanged. in.radius = std::min(in.radius, shed_radius.at(shed_)); - const auto& src = cached->second; + const auto& src = cached->source; // Spatial filter, then perception: nothing outside either is ever encoded. - std::map, std::uint64_t>> truth; - for (const auto& c : src.environment) - if (c.voxel.z == in.z && distance(c.voxel.x, c.voxel.y, in) <= in.radius) - truth[Key{environment_kind, c.voxel.x, c.voxel.y, c.voxel.z}] = { - encode_record(c), distance(c.voxel.x, c.voxel.y, in)}; - for (const auto& o : src.organisms) - if (o.voxel.z == in.z && distance(o.voxel.x, o.voxel.y, in) <= in.radius && - (!perceives || perceives(s.id, o))) - truth[Key{organism_kind, static_cast(o.id), 0, 0}] = { - encode_record(o), distance(o.voxel.x, o.voxel.y, in)}; + const auto fill = [&](Truth& into, bool filtered) { + for (const auto& c : src.environment) + if (c.voxel.z == in.z && distance(c.voxel.x, c.voxel.y, in) <= in.radius) + into[Key{environment_kind, c.voxel.x, c.voxel.y, c.voxel.z}] = { + encode_record(c), distance(c.voxel.x, c.voxel.y, in)}; + for (const auto& o : src.organisms) + if (o.voxel.z == in.z && distance(o.voxel.x, o.voxel.y, in) <= in.radius && + (!filtered || perceives(s.id, o))) + into[Key{organism_kind, static_cast(o.id), 0, 0}] = { + encode_record(o), distance(o.voxel.x, o.voxel.y, in)}; + }; + // `own`: this session's entries (markers; everything when perception filters). + Truth own; + const Truth* shared = nullptr; + if (perceives) { + fill(own, true); + } else { + if (!cached->truth) { + cached->truth.emplace(); + fill(*cached->truth, false); + } + shared = &*cached->truth; + } // Markers: the own one always; others in the square of the slice. Distance 0 puts // every marker first and makes it due every tick it changed. if (src.marker_area) for (const auto& m : src.markers) { - const bool own = !m.ghost && m.session == s.id; - if (!own && (src.marker_area->z != in.z || - distance(cell_of(m.state.x), cell_of(m.state.y), in) > in.radius)) + const bool own_marker = !m.ghost && m.session == s.id; + if (!own_marker && + (src.marker_area->z != in.z || + distance(cell_of(m.state.x), cell_of(m.state.y), in) > in.radius)) continue; - truth[Key{marker_kind, static_cast(m.id), 0, 0}] = { - encode_record(m, src.marker_area->z, own ? &*src.marker_area : nullptr), 0}; + own[Key{marker_kind, static_cast(m.id), 0, 0}] = { + encode_record(m, src.marker_area->z, own_marker ? &*src.marker_area : nullptr), + 0}; } + // Marker keys never collide with environment or organism keys (the kind differs). + const auto contains = [&](const Key& key) { + return own.contains(key) || (shared != nullptr && shared->contains(key)); + }; // Baselines older than the limit are forgotten; an expired ack means a fresh one. while (!s.baselines.empty() && tick - s.baselines.front().tick > limits_.max_baseline_age) s.baselines.pop_front(); @@ -298,7 +329,7 @@ std::vector>> ReplicationSer std::vector> removals; std::vector removed; for (const auto& [key, entry] : base_state) - if (!truth.contains(key)) { + if (!contains(key)) { removals.push_back( std::get<0>(key) == environment_kind ? remove_environment({std::get<1>(key), std::get<2>(key), std::get<3>(key)}) @@ -309,21 +340,27 @@ std::vector>> ReplicationSer } std::vector candidates; std::uint64_t deferred = 0; - for (const auto& [key, value] : truth) { - const auto known = base_state.find(key); - if (known != base_state.end() && known->second.record == value.first) - continue; // The client already holds this value. - const bool due = known == base_state.end() || value.second <= near_radius || - tick - known->second.updated >= far_interval; - if (!due) { - ++deferred; - continue; - } - const auto staleness = known == base_state.end() - ? std::numeric_limits::max() - : tick - known->second.updated; - candidates.push_back({value.second, staleness, key, &value.first}); - } + const auto consider = + [&](const Key& key, const std::pair, std::uint64_t>& value) { + const auto known = base_state.find(key); + if (known != base_state.end() && known->second.record == value.first) + return; // The client already holds this value. + const bool due = known == base_state.end() || value.second <= near_radius || + tick - known->second.updated >= far_interval; + if (!due) { + ++deferred; + return; + } + const auto staleness = known == base_state.end() + ? std::numeric_limits::max() + : tick - known->second.updated; + candidates.push_back({value.second, staleness, key, &value.first}); + }; + if (shared != nullptr) + for (const auto& [key, value] : *shared) + consider(key, value); + for (const auto& [key, value] : own) + consider(key, value); std::ranges::sort(candidates, [](const Candidate& a, const Candidate& b) { return std::tie(a.distance, b.staleness, a.key) < std::tie(b.distance, a.staleness, b.key); diff --git a/tests/network_replication.cpp b/tests/network_replication.cpp index 3efa9aa..b24efcd 100644 --- a/tests/network_replication.cpp +++ b/tests/network_replication.cpp @@ -4,6 +4,7 @@ #include "evo/network/replication.hpp" #include "network_test.hpp" #include +#include #include #include #include @@ -447,8 +448,58 @@ void shedding() { CHECK(client.latest()->environment.size() == 225 && baseline != 0); } +// M10.9d (ADR-0097): sessions with the same interest share one encoding of the environment +// per tick; each still converges on exactly its square, and another interest gets its own. +// Prints the server's time per tick for 20 sessions (a development reference). +void shared_interest() { + auto server = *ReplicationServer::create(limits()); + World world; + constexpr std::uint64_t sessions = 21; + std::vector clients; + for (std::uint64_t id = 1; id <= sessions; ++id) { + CHECK(server.open(id, 5)); + // Session 21 looks elsewhere (centre 3, 0): its own square. + CHECK(server.set_interest(id, {id == sessions ? 3 : 0, 0, slice, 7})); + clients.emplace_back(identity, 5); + } + const auto source = [&](const SessionInterest&) { return world.source(); }; + std::uint64_t tick = 0; + std::chrono::nanoseconds spent{}; + const auto step = [&] { + const auto started = std::chrono::steady_clock::now(); + const auto out = server.tick(++tick, source, {}); + spent += std::chrono::steady_clock::now() - started; + for (std::uint64_t id = 1; id <= sessions; ++id) + for (const auto& a : deliver(clients[id - 1], for_session(out, id))) + server.acknowledge(id, a); + }; + for (int i = 0; i < 6; ++i) + step(); + for (std::uint64_t id = 1; id <= sessions; ++id) { + const auto& view = clients[id - 1].latest()->environment; + const std::int64_t centre = id == sessions ? 3 : 0; + CHECK(view.size() == 225 && std::ranges::all_of(view, [&](const auto& c) { + return std::abs(c.voxel.x - centre) <= 7 && std::abs(c.voxel.y) <= 7; + })); + } + // A change reaches every session that sees it. + world.quantity[{1, 1}] = 77; + spent = {}; + for (int i = 0; i < 40; ++i) + step(); + for (auto& c : clients) + CHECK(std::ranges::any_of(c.latest()->environment, [](const auto& e) { + return e.voxel.x == 1 && e.voxel.y == 1 && e.quantity[0] == 77; + })); + std::printf("replication shared_interest sessions=%llu us_per_tick=%lld\n", + static_cast(sessions), + static_cast( + std::chrono::duration_cast(spent).count() / 40)); +} + void replication_tests() { codec_v2(); + shared_interest(); convergence_and_deltas(); shedding(); concealment(); diff --git a/tools/ops/load-drill.sh b/tools/ops/load-drill.sh index 57d521c..dfc6c4b 100644 --- a/tools/ops/load-drill.sh +++ b/tools/ops/load-drill.sh @@ -112,14 +112,17 @@ say "enabling synthetic inputs in $ns (drill-time override, restored on exit)" "${k[@]}" set env -n "$ns" deployment/evo-region REGION_INPUTS=development-v1 > /dev/null pod="$(ready_pod "$before" 240)" || fail "no Ready evo-region pod with synthetic inputs" drill_pod="$pod" -# Warm up: wait for a full governor window and a calm report (refusing=0 shed=0). +# Warm up: two consecutive calm reports (refusing=0 shed=0, about 10 s apart), so the +# governor's window no longer holds the restart's first ticks when the bots arrive (the +# second reference run met one calm report, then a refusal at the first login). calm="" for _ in $(seq 1 60); do sleep 3 - last="$("${k[@]}" logs -n "$ns" --since=15s "$pod" 2> /dev/null | grep "m10 load-report" | tail -1)" - case "$last" in *"refusing=0 shed=0"*) calm=1; break ;; esac + last2="$("${k[@]}" logs -n "$ns" --since=25s "$pod" 2> /dev/null | grep "m10 load-report" | tail -2)" + last="$(echo "$last2" | tail -1)" + [ "$(echo "$last2" | grep -c "refusing=0 shed=0")" = 2 ] && { calm=1; break; } done -[ -n "$calm" ] || fail "the region did not report calm within 3 minutes of starting: $last" +[ -n "$calm" ] || fail "the region did not report calm twice within 3 minutes of starting: $last" say "region calm: $last" say "running evo-load against $connect" set +e