Add M10.9a load measurement, governor and overload shedding
Foundation / checks (deploy) (push) Successful in 3m20s
Foundation / checks (ecs) (push) Successful in 1m59s
Foundation / checks (client) (push) Failing after 10m0s
Foundation / checks (network) (push) Successful in 13m9s
Foundation / checks (tsan) (push) Successful in 8m1s
Foundation / checks (checks) (push) Successful in 32m2s
Foundation / checks (postgres) (push) Failing after 26m39s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped

ADR-0097 (M10.9 load and abuse qualification; only evolution-dev
reference-worker runs qualify). evo-server measures each simulated tick's
work (host step plus the M10 bridge, with the session layer's share kept
separately) and feeds a portable LoadGovernor: grant logins are refused
at a 2 s p95 of 15 ms or any tick debt, replication is shed in up to
three levels at 20 ms or debt >= 2 (radius cap 7/7/5/3, far updates
x1/x2/x2/x4, no fresh baselines, own markers unaffected), and both relax
after 5 s of calm. Handoff tickets are still admitted. /metrics gains
evo_m10_* gauges when the listener is enabled.

Adds the missing ADR index entries for ADR-0094 to ADR-0096. Refreshes
the dev session window.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-07 06:34:40 -04:00
co-authored by Claude Opus 5.5
parent 265163a9a6
commit b8cb9db297
22 changed files with 676 additions and 34 deletions
+19 -5
View File
@@ -1,5 +1,6 @@
#include "evo/network/replication.hpp"
#include <algorithm>
#include <array>
#include <limits>
namespace evo::network {
@@ -224,10 +225,20 @@ ReplicationServer::tick(std::uint64_t tick,
return this->tick(tick, source, perceives, [payload](std::uint64_t) { return payload; });
}
void ReplicationServer::set_shed(std::uint8_t level) noexcept {
shed_ = std::min<std::uint8_t>(level, 3);
}
std::vector<std::pair<std::uint64_t, std::vector<SnapshotChunk>>> ReplicationServer::tick(
std::uint64_t tick, const std::function<ReplicationSource(const SessionInterest&)>& source,
const Perceives& perceives, const std::function<std::size_t(std::uint64_t)>& payload_of) {
std::vector<std::pair<std::uint64_t, std::vector<SnapshotChunk>>> out;
// Shedding (ADR-0097): radius cap, near radius and far-update multiplier per level.
constexpr std::array<std::uint16_t, 4> shed_radius{7, 7, 5, 3};
constexpr std::array<std::uint16_t, 4> shed_near{3, 3, 3, 2};
constexpr std::array<std::uint64_t, 4> shed_far{1, 2, 2, 4};
const auto near_radius = std::min(limits_.near_radius, shed_near.at(shed_));
const auto far_interval = limits_.far_interval * shed_far.at(shed_);
const auto overhead = session_header_bytes + snapshot_chunk_header_bytes;
if (tick == 0)
return out;
@@ -239,12 +250,15 @@ std::vector<std::pair<std::uint64_t, std::vector<SnapshotChunk>>> ReplicationSer
if (payload <= overhead)
continue; // No datagram channel yet (or a path too small): nothing to send.
const auto capacity = payload - overhead; // Record bytes per chunk.
const auto& in = *s.interest;
auto cached = std::ranges::find_if(sources, [&](const auto& e) { return e.first == in; });
const auto& wanted = *s.interest;
auto cached =
std::ranges::find_if(sources, [&](const auto& e) { return e.first == wanted; });
if (cached == sources.end()) {
sources.emplace_back(in, source(in));
sources.emplace_back(wanted, source(wanted));
cached = std::prev(sources.end());
}
auto in = wanted; // Encoded within the shed cap; the stored interest is unchanged.
in.radius = std::min(in.radius, shed_radius.at(shed_));
const auto& src = cached->second;
// Spatial filter, then perception: nothing outside either is ever encoded.
std::map<Key, std::pair<std::vector<std::uint8_t>, std::uint64_t>> truth;
@@ -299,8 +313,8 @@ std::vector<std::pair<std::uint64_t, std::vector<SnapshotChunk>>> ReplicationSer
const auto known = base_state.find(key);
if (known != base_state.end() && known->second.record == value.first)
continue; // The client already holds this value.
const bool due = known == base_state.end() || value.second <= limits_.near_radius ||
tick - known->second.updated >= limits_.far_interval;
const bool due = known == base_state.end() || value.second <= near_radius ||
tick - known->second.updated >= far_interval;
if (!due) {
++deferred;
continue;
+69
View File
@@ -0,0 +1,69 @@
#include "evo/server/load_governor.hpp"
#include <algorithm>
#include <utility>
namespace evo::server {
std::expected<LoadGovernor, LoadPolicyError> LoadGovernor::create(LoadPolicy p) {
if (p.relax_us == 0 || p.relax_us >= p.refuse_us || p.refuse_us > p.shed_us ||
p.shed_us > p.tick_budget_us || p.step_us == 0 || p.settle_us == 0 || p.max_shed > 3)
return std::unexpected(LoadPolicyError::invalid);
return LoadGovernor{p};
}
namespace {
using Window = std::array<std::uint64_t, load_window_ticks>;
// The first `n` samples sorted; nearest rank: the smallest sample with at least `percent` of
// the window at or below it.
Window sorted_of(const Window& w, std::size_t n) {
Window sorted{};
std::copy_n(w.begin(), n, sorted.begin());
std::sort(sorted.begin(), sorted.begin() + static_cast<std::ptrdiff_t>(n));
return sorted;
}
std::uint64_t rank(const Window& sorted, std::size_t n, std::size_t percent) {
return sorted.at((((n * percent) + 99) / 100) - 1);
}
} // namespace
bool LoadGovernor::observe(std::uint64_t work_us, std::uint64_t session_us, std::uint64_t tick_debt,
std::uint64_t now_us) {
window_.at(next_) = work_us;
session_window_.at(next_) = std::min(session_us, work_us);
next_ = (next_ + 1) % window_.size();
state_.samples = std::min(state_.samples + 1, window_.size());
const auto n = state_.samples;
const auto sorted = sorted_of(window_, n);
state_.p95_us = rank(sorted, n, 95);
state_.p99_us = rank(sorted, n, 99);
state_.max_us = sorted.at(n - 1);
const auto sessions = sorted_of(session_window_, n);
state_.session_p95_us = rank(sessions, n, 95);
state_.session_p99_us = rank(sessions, n, 99);
const auto before = std::pair{state_.refusing, state_.shed};
const bool hot = state_.p95_us >= policy_.refuse_us || tick_debt > 0;
const bool overloaded = state_.p95_us >= policy_.shed_us || tick_debt >= 2;
if (hot)
state_.refusing = true;
if (overloaded && state_.shed < policy_.max_shed &&
(!stepped_ || now_us - last_step_us_ >= policy_.step_us)) {
++state_.shed;
stepped_ = true;
last_step_us_ = now_us;
}
if (state_.p95_us >= policy_.relax_us || tick_debt > 0) {
calm_ = false;
} else if (!calm_) {
calm_ = true;
calm_since_us_ = now_us;
} else if (now_us - calm_since_us_ >= policy_.settle_us) {
if (state_.shed > 0)
--state_.shed; // One level per settle interval, then logins.
else
state_.refusing = false;
calm_since_us_ = now_us;
}
return before != std::pair{state_.refusing, state_.shed};
}
} // namespace evo::server
+18
View File
@@ -103,6 +103,24 @@ std::string render_metrics(const HostStatus& s, const OpsCounters& c) {
.append("\"} ")
.append(std::to_string(c.dropped[i]))
.append("\n");
if (c.m10_enabled) {
metric(out, "evo_m10_sessions", "gauge", "Open M10 sessions", c.m10_sessions);
metric(out, "evo_m10_refusing", "gauge", "1 while new logins are refused for load",
c.m10_load.refusing);
metric(out, "evo_m10_shed_level", "gauge", "Replication shed level (0-3)", c.m10_load.shed);
metric(out, "evo_m10_tick_work_p95_microseconds", "gauge",
"p95 per-tick region work over the last 40 ticks", c.m10_load.p95_us);
metric(out, "evo_m10_tick_work_p99_microseconds", "gauge",
"p99 per-tick region work over the last 40 ticks", c.m10_load.p99_us);
metric(out, "evo_m10_tick_work_max_microseconds", "gauge",
"Largest per-tick region work over the last 40 ticks", c.m10_load.max_us);
metric(out, "evo_m10_session_work_p95_microseconds", "gauge",
"p95 per-tick session-layer work over the last 40 ticks", c.m10_load.session_p95_us);
metric(out, "evo_m10_session_work_p99_microseconds", "gauge",
"p99 per-tick session-layer work over the last 40 ticks", c.m10_load.session_p99_us);
metric(out, "evo_m10_refused_logins_total", "counter", "Logins refused for load",
c.m10_refused_logins);
}
return out;
}
} // namespace evo::server
+15
View File
@@ -108,6 +108,8 @@ struct RegionSessions::Impl {
std::vector<Outgoing> outgoing; // Handed off, not yet known committed.
Clock::time_point next_ghosts{}, next_overdue{};
std::optional<network::SessionInterest> default_interest;
bool refuse_logins{}; // Load governor (ADR-0097).
std::uint64_t refused_load{}; // Grant logins refused for load.
HostFacts facts;
std::vector<Connection> connections;
std::vector<RegionSessionEvent> events;
@@ -216,6 +218,10 @@ struct RegionSessions::Impl {
return close(c, SessionCloseReason::rate_limited, now);
if (network::looks_like_handoff_ticket(m.grant))
return ticket_hello(c, m, now_seconds, now);
if (refuse_logins) { // Overload (ADR-0097): before signature work.
++refused_load;
return close(c, SessionCloseReason::capacity, now);
}
const auto grant = verifier.verify(m.grant);
if (!grant)
return close(c, reason_for(grant.error()), now);
@@ -774,6 +780,15 @@ std::optional<std::uint64_t> RegionSessions::holder(const network::AccountId& ac
return impl_->control.holder(account);
}
void RegionSessions::set_load(bool refuse_logins, std::uint8_t shed) {
impl_->refuse_logins = refuse_logins;
impl_->replication.set_shed(shed);
}
std::uint64_t RegionSessions::refused_for_load() const noexcept {
return impl_->refused_load;
}
std::size_t RegionSessions::sessions() const noexcept {
return impl_->control.sessions();
}