Foundation / checks (deploy) (push) Successful in 3m24s
Foundation / checks (ecs) (push) Successful in 2m0s
Foundation / checks (client) (push) Successful in 12m24s
Foundation / checks (network) (push) Successful in 13m35s
Foundation / checks (tsan) (push) Successful in 7m58s
Foundation / checks (checks) (push) Successful in 32m32s
Foundation / checks (postgres) (push) Successful in 45m25s
Foundation / postgres-tools (push) Successful in 17s
Foundation / image (push) Successful in 2m32s
Foundation / server-image (push) Successful in 4m15s
ADR-0097. The first evolution-dev load drill (image 6c96629e) did not measure capacity: the governor refused six seconds after the restart with no session (p95 over a window of only startup ticks), and the bots spent their grants retrying every 2 s. Reference numbers with one session: tick work p95 4.0 ms with 33 ms spikes, session layer p95 0.68 ms. - The governor acts on work only over a full 40-tick window; tick debt still counts at once. - evo-load backs off 10 s after a load refusal and, in bundle mode, reuses the grant the region refused before using it. - The drill waits for a calm load-report before the bots start, takes the drill pod's whole log before restoring (the live follow died), and confirms the restore against a pod other than the drill pod. Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
75 lines
3.0 KiB
C++
75 lines
3.0 KiB
C++
#include "evo/server/load_governor.hpp"
|
|
|
|
#include <algorithm>
|
|
#include <utility>
|
|
|
|
namespace evo::server {
|
|
std::expected<LoadGovernor, LoadPolicyError> LoadGovernor::create(LoadPolicy p) {
|
|
if (p.relax_us == 0 || p.relax_us >= p.refuse_us || p.refuse_us > p.shed_us ||
|
|
p.shed_us > p.tick_budget_us || p.step_us == 0 || p.settle_us == 0 || p.max_shed > 3)
|
|
return std::unexpected(LoadPolicyError::invalid);
|
|
return LoadGovernor{p};
|
|
}
|
|
|
|
namespace {
|
|
using Window = std::array<std::uint64_t, load_window_ticks>;
|
|
// The first `n` samples sorted; nearest rank: the smallest sample with at least `percent` of
|
|
// the window at or below it.
|
|
Window sorted_of(const Window& w, std::size_t n) {
|
|
Window sorted{};
|
|
std::copy_n(w.begin(), n, sorted.begin());
|
|
std::sort(sorted.begin(), sorted.begin() + static_cast<std::ptrdiff_t>(n));
|
|
return sorted;
|
|
}
|
|
std::uint64_t rank(const Window& sorted, std::size_t n, std::size_t percent) {
|
|
return sorted.at((((n * percent) + 99) / 100) - 1);
|
|
}
|
|
} // namespace
|
|
|
|
bool LoadGovernor::observe(std::uint64_t work_us, std::uint64_t session_us, std::uint64_t tick_debt,
|
|
std::uint64_t now_us) {
|
|
window_.at(next_) = work_us;
|
|
session_window_.at(next_) = std::min(session_us, work_us);
|
|
next_ = (next_ + 1) % window_.size();
|
|
state_.samples = std::min(state_.samples + 1, window_.size());
|
|
const auto n = state_.samples;
|
|
const auto sorted = sorted_of(window_, n);
|
|
state_.p95_us = rank(sorted, n, 95);
|
|
state_.p99_us = rank(sorted, n, 99);
|
|
state_.max_us = sorted.at(n - 1);
|
|
const auto sessions = sorted_of(session_window_, n);
|
|
state_.session_p95_us = rank(sessions, n, 95);
|
|
state_.session_p99_us = rank(sessions, n, 99);
|
|
|
|
const auto before = std::pair{state_.refusing, state_.shed};
|
|
// Work counts only over a full window: after a (re)start the first ticks (cold caches, the
|
|
// first commits) would otherwise be the p95 alone (seen on evolution-dev, 10.9d). Tick
|
|
// debt is acted on at once.
|
|
const bool full = n == window_.size();
|
|
const bool hot = (full && state_.p95_us >= policy_.refuse_us) || tick_debt >= refuse_debt_ticks;
|
|
const bool overloaded =
|
|
(full && state_.p95_us >= policy_.shed_us) || tick_debt >= shed_debt_ticks;
|
|
if (hot)
|
|
state_.refusing = true;
|
|
if (overloaded && state_.shed < policy_.max_shed &&
|
|
(!stepped_ || now_us - last_step_us_ >= policy_.step_us)) {
|
|
++state_.shed;
|
|
stepped_ = true;
|
|
last_step_us_ = now_us;
|
|
}
|
|
if (state_.p95_us >= policy_.relax_us || tick_debt > 0) {
|
|
calm_ = false;
|
|
} else if (!calm_) {
|
|
calm_ = true;
|
|
calm_since_us_ = now_us;
|
|
} else if (now_us - calm_since_us_ >= policy_.settle_us) {
|
|
if (state_.shed > 0)
|
|
--state_.shed; // One level per settle interval, then logins.
|
|
else
|
|
state_.refusing = false;
|
|
calm_since_us_ = now_us;
|
|
}
|
|
return before != std::pair{state_.refusing, state_.shed};
|
|
}
|
|
} // namespace evo::server
|