Add M10.9d load drill harness and serialize Mbed TLS use
Foundation / checks (deploy) (push) Successful in 3m24s
Foundation / checks (ecs) (push) Successful in 1m56s
Foundation / checks (client) (push) Successful in 12m18s
Foundation / checks (network) (push) Successful in 13m30s
Foundation / checks (tsan) (push) Successful in 8m0s
Foundation / checks (checks) (push) Successful in 32m39s
Foundation / checks (postgres) (push) Successful in 45m47s
Foundation / postgres-tools (push) Successful in 17s
Foundation / image (push) Successful in 2m30s
Foundation / server-image (push) Successful in 4m8s
Foundation / checks (deploy) (push) Successful in 3m24s
Foundation / checks (ecs) (push) Successful in 1m56s
Foundation / checks (client) (push) Successful in 12m18s
Foundation / checks (network) (push) Successful in 13m30s
Foundation / checks (tsan) (push) Successful in 8m0s
Foundation / checks (checks) (push) Successful in 32m39s
Foundation / checks (postgres) (push) Successful in 45m47s
Foundation / postgres-tools (push) Successful in 17s
Foundation / image (push) Successful in 2m30s
Foundation / server-image (push) Successful in 4m8s
ADR-0097. tools/ops/load-drill.sh (runbook docs/networking/M10_LOAD_DRILL.md) runs the evolution-dev reference qualification: synthetic inputs on for the run (the ADR-0073 drill-time override, restored on exit), bundle-mode evo-load bots through the public route, and each step's hold matched with the server's reports. evo-server logs "m10 load-report" every 10 s, since the agent cannot reach the ops port. evo-load gains --grants FILE --grants-per-bot C, exhausted=, hold_utc= and close_reasons=; evo-session-probe grants gains --accounts N. Finding: Mbed TLS/PSA are built without MBEDTLS_THREADING_C, and bots on different threads verifying their first connections at once were refused as untrusted (two per smoke run). One process-wide recursive crypto lock now guards every network-library entry point that touches them; the smoke went from 2/2 failing to 3/3 clean. quic-transport adds a concurrent-connect test. Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+107
-33
@@ -2,7 +2,8 @@
|
||||
// runs against the evolution-dev reference worker count as evidence (10.9d); local runs are
|
||||
// smoke tests and development references.
|
||||
//
|
||||
// evo-load --connect HOST:PORT --server-name NAME --ca FILE --issuer-key FILE
|
||||
// evo-load --connect HOST:PORT --server-name NAME --ca FILE
|
||||
// (--issuer-key FILE | --grants FILE --grants-per-bot C)
|
||||
// --world N --frame N --region N [--steps N[,N]...] [--hold S] [--warmup S]
|
||||
// [--ramp-ms MS] [--threads T] [--seed X] [--lifetime S] [--metrics HOST:PORT]
|
||||
// [--response-budget-ms MS] [--per-client] [--expect-established]
|
||||
@@ -14,11 +15,17 @@
|
||||
// probe marker; every 2 s it sends a probe action; every 10 s it moves its interest by a
|
||||
// cell (a fresh baseline). A closed or refused bot logs in again after 2 s with the next
|
||||
// serial.
|
||||
// M10.9d: with --grants (a bundle from `evo-session-probe grants --accounts N --count C`,
|
||||
// e.g. made where the dev issuer key lives), bot i logs in with grants i*C .. i*C+C-1 in
|
||||
// order instead of signing; a bot that used all C stays out (counted as exhausted).
|
||||
//
|
||||
// Steps: the bot count rises to each step's N (one login every --ramp-ms), waits --warmup
|
||||
// seconds, then measures for --hold seconds. Bots stay connected into the next step. Per
|
||||
// step it prints:
|
||||
// load step N=.. established=.. logins=.. drops=.. login_failures=.. refused=.. seconds=..
|
||||
// exhausted=.. hold_utc=START..END (unix seconds of the hold, to match server samples)
|
||||
// close_reasons=REASON:COUNT,... (session-v1 close reasons; without a session Close,
|
||||
// 256 + the QUIC transport cause, or 255 when unknown)
|
||||
// (drops: established sessions closed; login failures: closed before Welcome, including
|
||||
// transport refusals at the connection limit; refused: closed with capacity or
|
||||
// rate_limited, either way. Logins, drops and failures include the step's ramp and warmup;
|
||||
@@ -111,7 +118,8 @@ double ms(std::uint64_t us) {
|
||||
}
|
||||
|
||||
struct Options {
|
||||
std::string host, server_name, ca, issuer_key, metrics_host;
|
||||
std::string host, server_name, ca, issuer_key, grants_file, metrics_host;
|
||||
std::uint64_t grants_per_bot{};
|
||||
std::uint16_t port{}, metrics_port{};
|
||||
std::uint64_t world{}, frame{}, region{}, seed{1}, lifetime{900};
|
||||
std::vector<std::size_t> steps{20};
|
||||
@@ -125,6 +133,7 @@ struct Samples {
|
||||
gap_ticks;
|
||||
std::uint64_t snapshots{}, bytes{}, intents{}, refused_intents{}, logins{}, drops{},
|
||||
login_failures{}, refused{};
|
||||
std::map<unsigned, std::uint64_t> closes; // By session close reason (255: no Close).
|
||||
std::uint64_t connected_polls{}; // Polls while established (starvation is judged on these).
|
||||
};
|
||||
void append(std::vector<std::uint64_t>& to, const std::vector<std::uint64_t>& from) {
|
||||
@@ -142,6 +151,8 @@ struct Bot {
|
||||
client::InputLead lead;
|
||||
std::uint64_t last_tick{}, last_view{}, last_bytes{}, next_action_id{1};
|
||||
std::uint64_t applied{}; // Newest applied intention sequence seen in an own marker.
|
||||
std::uint64_t used{}; // Bundle grants used (bundle mode).
|
||||
std::atomic<bool> exhausted{false};
|
||||
std::map<std::uint64_t, Clock::time_point> receipt_pending, response_pending, outcome_pending;
|
||||
std::map<std::uint64_t, std::uint64_t> intent_tick;
|
||||
std::int16_t vx{}, vy{};
|
||||
@@ -153,9 +164,11 @@ struct Bot {
|
||||
|
||||
class Load {
|
||||
public:
|
||||
Load(Options o, GrantSigningKey key, std::string address)
|
||||
: o_{std::move(o)}, key_{std::move(key)}, address_{std::move(address)},
|
||||
authority_{read(o_.ca)} {
|
||||
// Signs with `key`, or logs in from `bundle` (concatenated grants, grants_per_bot each).
|
||||
Load(Options o, std::optional<GrantSigningKey> key, std::vector<SessionGrant> bundle,
|
||||
std::string address)
|
||||
: o_{std::move(o)}, key_{std::move(key)}, bundle_{std::move(bundle)},
|
||||
address_{std::move(address)}, authority_{read(o_.ca)} {
|
||||
const auto most = *std::ranges::max_element(o_.steps);
|
||||
bots_.reserve(most);
|
||||
for (std::size_t i = 0; i < most; ++i) {
|
||||
@@ -199,6 +212,11 @@ class Load {
|
||||
}
|
||||
return out;
|
||||
}
|
||||
[[nodiscard]] std::size_t exhausted(std::size_t n) const {
|
||||
return static_cast<std::size_t>(
|
||||
std::count_if(bots_.begin(), bots_.begin() + static_cast<std::ptrdiff_t>(n),
|
||||
[](const auto& b) { return b->exhausted.load(); }));
|
||||
}
|
||||
[[nodiscard]] std::size_t established(std::size_t n) const {
|
||||
return static_cast<std::size_t>(
|
||||
std::count_if(bots_.begin(), bots_.begin() + static_cast<std::ptrdiff_t>(n),
|
||||
@@ -229,15 +247,26 @@ class Load {
|
||||
}
|
||||
}
|
||||
void login(Bot& b, Clock::time_point now) {
|
||||
GrantClaims claims;
|
||||
claims.account = b.account;
|
||||
claims.serial = ++b.serial;
|
||||
claims.world = {o_.world, o_.frame};
|
||||
claims.region = o_.region;
|
||||
claims.issued_at = now_seconds();
|
||||
claims.expires_at = claims.issued_at + o_.lifetime;
|
||||
claims.grant_id = (b.serial << 16U) ^ (b.index + 1);
|
||||
const auto grant = key_.sign(claims);
|
||||
std::optional<SessionGrant> grant;
|
||||
if (key_) {
|
||||
GrantClaims claims;
|
||||
claims.account = b.account;
|
||||
claims.serial = ++b.serial;
|
||||
claims.world = {o_.world, o_.frame};
|
||||
claims.region = o_.region;
|
||||
claims.issued_at = now_seconds();
|
||||
claims.expires_at = claims.issued_at + o_.lifetime;
|
||||
claims.grant_id = (b.serial << 16U) ^ (b.index + 1);
|
||||
if (auto signed_grant = key_->sign(claims))
|
||||
grant = *signed_grant;
|
||||
} else {
|
||||
if (b.used >= o_.grants_per_bot) {
|
||||
b.exhausted = true; // Out of grants: stays out.
|
||||
b.retry_at = Clock::time_point::max();
|
||||
return;
|
||||
}
|
||||
grant = bundle_.at((b.index * o_.grants_per_bot) + b.used++);
|
||||
}
|
||||
client::RegionSessionClientConfig config;
|
||||
config.transport.server_address = address_;
|
||||
config.transport.port = o_.port;
|
||||
@@ -278,6 +307,13 @@ class Load {
|
||||
++b.s.drops;
|
||||
else
|
||||
++b.s.login_failures;
|
||||
if (const auto reason = c.close_reason();
|
||||
reason && *reason != SessionCloseReason::connection_lost)
|
||||
++b.s.closes[static_cast<unsigned>(*reason)];
|
||||
else if (const auto cause = c.transport_cause()) // No session Close: 256 + cause.
|
||||
++b.s.closes[256U + static_cast<unsigned>(*cause)];
|
||||
else
|
||||
++b.s.closes[255U];
|
||||
if (c.close_reason() == SessionCloseReason::capacity ||
|
||||
c.close_reason() == SessionCloseReason::rate_limited)
|
||||
++b.s.refused;
|
||||
@@ -406,7 +442,8 @@ class Load {
|
||||
}
|
||||
|
||||
Options o_;
|
||||
GrantSigningKey key_;
|
||||
std::optional<GrantSigningKey> key_;
|
||||
std::vector<SessionGrant> bundle_;
|
||||
std::string address_, authority_;
|
||||
std::vector<std::unique_ptr<Bot>> bots_;
|
||||
std::vector<std::mutex> locks_;
|
||||
@@ -481,9 +518,9 @@ void line(const char* name, const std::vector<std::uint64_t>& pooled,
|
||||
}
|
||||
|
||||
// One step's report; true when within budget.
|
||||
bool report(const Options& o, std::size_t n, std::size_t established,
|
||||
const std::vector<Samples>& all, double seconds, double cpu,
|
||||
const std::map<std::string, std::uint64_t>& server) {
|
||||
bool report(const Options& o, std::size_t n, std::size_t established, std::size_t exhausted,
|
||||
std::pair<std::uint64_t, std::uint64_t> hold_utc, const std::vector<Samples>& all,
|
||||
double seconds, double cpu, const std::map<std::string, std::uint64_t>& server) {
|
||||
Samples pooled;
|
||||
std::vector<std::uint64_t> response95, receipt95, outcome95, login95, correction95, rates, kib;
|
||||
for (const auto& s : all) {
|
||||
@@ -499,6 +536,8 @@ bool report(const Options& o, std::size_t n, std::size_t established,
|
||||
pooled.logins += s.logins;
|
||||
pooled.drops += s.drops;
|
||||
pooled.login_failures += s.login_failures;
|
||||
for (const auto& [why, count] : s.closes)
|
||||
pooled.closes[why] += count;
|
||||
pooled.refused += s.refused;
|
||||
response95.push_back(percentile(s.response_us, 0.95));
|
||||
receipt95.push_back(percentile(s.receipt_us, 0.95));
|
||||
@@ -511,12 +550,18 @@ bool report(const Options& o, std::size_t n, std::size_t established,
|
||||
kib.push_back(static_cast<std::uint64_t>(
|
||||
std::lround(static_cast<double>(s.bytes) / 1024.0 / std::max(seconds, 0.001))));
|
||||
}
|
||||
std::string reasons;
|
||||
for (const auto& [why, count] : pooled.closes)
|
||||
reasons += (reasons.empty() ? "" : ",") + std::to_string(why) + ":" + std::to_string(count);
|
||||
std::printf("load step N=%zu established=%zu logins=%llu drops=%llu login_failures=%llu "
|
||||
"refused=%llu seconds=%.1f\n",
|
||||
"refused=%llu seconds=%.1f exhausted=%zu hold_utc=%llu..%llu close_reasons=%s\n",
|
||||
n, established, static_cast<unsigned long long>(pooled.logins),
|
||||
static_cast<unsigned long long>(pooled.drops),
|
||||
static_cast<unsigned long long>(pooled.login_failures),
|
||||
static_cast<unsigned long long>(pooled.refused), seconds);
|
||||
static_cast<unsigned long long>(pooled.refused), seconds, exhausted,
|
||||
static_cast<unsigned long long>(hold_utc.first),
|
||||
static_cast<unsigned long long>(hold_utc.second),
|
||||
reasons.empty() ? "-" : reasons.c_str());
|
||||
line("response", pooled.response_us, response95, 1000.0, "ms");
|
||||
line("receipt", pooled.receipt_us, receipt95, 1000.0, "ms");
|
||||
line("outcome", pooled.outcome_us, outcome95, 1000.0, "ms");
|
||||
@@ -663,6 +708,10 @@ int main(int argc, char** argv) {
|
||||
o.ca = value;
|
||||
else if (flag == "--issuer-key")
|
||||
o.issuer_key = value;
|
||||
else if (flag == "--grants")
|
||||
o.grants_file = value;
|
||||
else if (n && flag == "--grants-per-bot" && *n >= 1 && *n <= 64)
|
||||
o.grants_per_bot = *n;
|
||||
else if (flag == "--steps") {
|
||||
const auto s = steps_of(value);
|
||||
ok = s.has_value();
|
||||
@@ -676,7 +725,7 @@ int main(int argc, char** argv) {
|
||||
o.region = *n;
|
||||
else if (n && flag == "--seed" && *n != 0)
|
||||
o.seed = *n;
|
||||
else if (n && flag == "--lifetime" && *n >= 60 && *n <= 3600)
|
||||
else if (n && flag == "--lifetime" && *n >= 60 && *n <= 14400)
|
||||
o.lifetime = *n;
|
||||
else if (n && flag == "--hold" && *n >= 1 && *n <= 3600)
|
||||
o.hold_s = *n;
|
||||
@@ -691,23 +740,44 @@ int main(int argc, char** argv) {
|
||||
else
|
||||
ok = false;
|
||||
}
|
||||
if (!ok || o.host.empty() || o.server_name.empty() || o.ca.empty() || o.issuer_key.empty() ||
|
||||
o.world == 0 || o.frame == 0 || o.region == 0) {
|
||||
const bool signs = !o.issuer_key.empty();
|
||||
if (!ok || o.host.empty() || o.server_name.empty() || o.ca.empty() ||
|
||||
signs == !o.grants_file.empty() || (!signs && o.grants_per_bot == 0) || o.world == 0 ||
|
||||
o.frame == 0 || o.region == 0) {
|
||||
std::fputs("usage: evo-load --connect HOST:PORT --server-name NAME --ca FILE "
|
||||
"--issuer-key FILE --world N --frame N --region N [--steps N[,N]...] "
|
||||
"(--issuer-key FILE | --grants FILE --grants-per-bot C) --world N "
|
||||
"--frame N --region N [--steps N[,N]...] "
|
||||
"[--hold S] [--warmup S] [--ramp-ms MS] [--threads T] [--seed X] "
|
||||
"[--lifetime S] [--metrics HOST:PORT] [--response-budget-ms MS] "
|
||||
"[--per-client] [--expect-established]\n",
|
||||
stderr);
|
||||
return 2;
|
||||
}
|
||||
const auto scalar = read(o.issuer_key);
|
||||
auto key = GrantSigningKey::import(
|
||||
DeploymentEnvironment::dev, 1,
|
||||
std::span{reinterpret_cast<const std::uint8_t*>(scalar.data()), scalar.size()});
|
||||
if (!key) {
|
||||
std::puts("error issuer-key");
|
||||
return 2;
|
||||
std::optional<GrantSigningKey> key;
|
||||
std::vector<SessionGrant> bundle;
|
||||
if (signs) {
|
||||
const auto scalar = read(o.issuer_key);
|
||||
auto imported = GrantSigningKey::import(
|
||||
DeploymentEnvironment::dev, 1,
|
||||
std::span{reinterpret_cast<const std::uint8_t*>(scalar.data()), scalar.size()});
|
||||
if (!imported) {
|
||||
std::puts("error issuer-key");
|
||||
return 2;
|
||||
}
|
||||
key.emplace(std::move(*imported));
|
||||
} else {
|
||||
const auto bytes = read(o.grants_file);
|
||||
const auto needed = *std::ranges::max_element(o.steps) * o.grants_per_bot;
|
||||
if (bytes.size() % session_grant_bytes != 0 ||
|
||||
bytes.size() / session_grant_bytes < needed) {
|
||||
std::printf("error grants (need %llu grants: bots x grants-per-bot)\n",
|
||||
static_cast<unsigned long long>(needed));
|
||||
return 2;
|
||||
}
|
||||
bundle.resize(bytes.size() / session_grant_bytes);
|
||||
for (std::size_t i = 0; i < bundle.size(); ++i)
|
||||
std::copy_n(bytes.begin() + static_cast<std::ptrdiff_t>(i * session_grant_bytes),
|
||||
session_grant_bytes, reinterpret_cast<char*>(bundle[i].data()));
|
||||
}
|
||||
const auto address = quic::resolve_ipv4(o.host);
|
||||
if (!address) {
|
||||
@@ -723,7 +793,7 @@ int main(int argc, char** argv) {
|
||||
bool smoke_ok = true;
|
||||
std::optional<std::size_t> sustainable, saturated;
|
||||
{
|
||||
Load load{o, std::move(*key), *address};
|
||||
Load load{o, std::move(key), std::move(bundle), *address};
|
||||
for (const auto n : o.steps) {
|
||||
load.activate(n);
|
||||
std::this_thread::sleep_for(std::chrono::seconds{o.warmup_s});
|
||||
@@ -731,6 +801,7 @@ int main(int argc, char** argv) {
|
||||
// the warmup is discarded.
|
||||
const auto warm = load.take(n);
|
||||
const auto started = Clock::now();
|
||||
const auto hold_start = now_seconds();
|
||||
const auto cpu0 = cpu_seconds();
|
||||
std::this_thread::sleep_for(std::chrono::seconds{o.hold_s});
|
||||
auto samples = load.take(n);
|
||||
@@ -739,12 +810,15 @@ int main(int argc, char** argv) {
|
||||
samples[i].logins += warm[i].logins;
|
||||
samples[i].drops += warm[i].drops;
|
||||
samples[i].login_failures += warm[i].login_failures;
|
||||
for (const auto& [why, count] : warm[i].closes)
|
||||
samples[i].closes[why] += count;
|
||||
samples[i].refused += warm[i].refused;
|
||||
}
|
||||
const auto seconds = std::chrono::duration<double>(Clock::now() - started).count();
|
||||
const auto established = load.established(n);
|
||||
const bool within =
|
||||
report(o, n, established, samples, seconds, cpu_seconds() - cpu0, scrape(o));
|
||||
report(o, n, established, load.exhausted(n), {hold_start, now_seconds()}, samples,
|
||||
seconds, cpu_seconds() - cpu0, scrape(o));
|
||||
if (within && !saturated)
|
||||
sustainable = n;
|
||||
if (!within && !saturated)
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env bash
|
||||
# M10.9d load drill (ADR-0097) for evolution-dev only: the reference-worker qualification run.
|
||||
#
|
||||
# tools/ops/load-drill.sh <tokens-dir> [--context <name>] [--steps N[,N]...] [--hold S]
|
||||
# [--warmup S] [--grants-per-bot C] [--connect HOST:PORT] [--server-name NAME]
|
||||
#
|
||||
# <tokens-dir> holds dev-ca.pem (the current dev session CA) and load-grants.bin, a bundle
|
||||
# made where the dev issuer key lives:
|
||||
# evo-session-probe grants --issuer-key issuer.key --world W --frame F --region R \
|
||||
# --account <HEX32> --accounts <most bots> --count <grants per bot> --lifetime 14400 \
|
||||
# --out load-grants.bin
|
||||
# The issuer key never reaches the machine running the drill. Requires kubectl access to
|
||||
# evolution-dev (patch Deployment/evo-region, read pods and logs), `evo-load` (EVO_LOAD,
|
||||
# default: on PATH) and python3.
|
||||
#
|
||||
# Steps: refuse unless the session window outlasts the drill; enable synthetic inputs on
|
||||
# evo-region (the drill-time override of ADR-0073, never committed) and wait for the new
|
||||
# pod to be Ready; follow its log with timestamps (the server prints `m10 load-report`
|
||||
# every 10 s); run evo-load through the public route; then restore REGION_INPUTS to the Git
|
||||
# value (also on failure) so Argo returns to Synced. The report matches each step's hold
|
||||
# (evo-load's hold_utc) with the server's reports in it and prints, per step, the client
|
||||
# verdict and the server's worst tick work, session work, governor state and tick debt.
|
||||
# Stage and World0 are never touched: the namespace is fixed.
|
||||
set -euo pipefail
|
||||
usage() {
|
||||
echo "usage: $0 <tokens-dir> [--context <name>] [--steps N[,N]...] [--hold S] [--warmup S]" \
|
||||
"[--grants-per-bot C] [--connect HOST:PORT] [--server-name NAME]" >&2
|
||||
exit 2
|
||||
}
|
||||
[ $# -ge 1 ] || usage
|
||||
tokens="$1"; shift
|
||||
k=(kubectl --request-timeout=30s)
|
||||
steps=20,24,28,32 hold=60 warmup=10 per_bot=4
|
||||
connect=void.aeternalisgames.com:7400 server_name=void.aeternalisgames.com
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--context) [ -n "${2:-}" ] || usage; k+=(--context "$2"); shift 2 ;;
|
||||
--steps) [ -n "${2:-}" ] || usage; steps="$2"; shift 2 ;;
|
||||
--hold) [ -n "${2:-}" ] || usage; hold="$2"; shift 2 ;;
|
||||
--warmup) [ -n "${2:-}" ] || usage; warmup="$2"; shift 2 ;;
|
||||
--grants-per-bot) [ -n "${2:-}" ] || usage; per_bot="$2"; shift 2 ;;
|
||||
--connect) [ -n "${2:-}" ] || usage; connect="$2"; shift 2 ;;
|
||||
--server-name) [ -n "${2:-}" ] || usage; server_name="$2"; shift 2 ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
done
|
||||
ns=evolution-dev # Fixed: qualification runs on the dev reference worker only.
|
||||
tool="${EVO_LOAD:-evo-load}"
|
||||
py="$(command -v python3 || command -v python || true)"
|
||||
for f in dev-ca.pem load-grants.bin; do
|
||||
[ -f "$tokens/$f" ] || { echo "missing $tokens/$f" >&2; exit 2; }
|
||||
done
|
||||
command -v "$tool" > /dev/null || [ -x "$tool" ] || { echo "evo-load not found (set EVO_LOAD)" >&2; exit 2; }
|
||||
[ -n "$py" ] || { echo "python3 is required" >&2; exit 2; }
|
||||
selector=app.kubernetes.io/name=evo-region
|
||||
work="$(mktemp -d)"
|
||||
chmod 700 "$work"
|
||||
follow_pid=""
|
||||
say() { echo "$(date -u +%H:%M:%S) $*"; }
|
||||
fail() { say "FAIL $*"; exit 1; }
|
||||
env_of() {
|
||||
"${k[@]}" get deployment -n "$ns" evo-region \
|
||||
-o jsonpath="{.spec.template.spec.containers[0].env[?(@.name==\"$1\")].value}"
|
||||
}
|
||||
ready_pod() { # Waits up to $2 seconds for a Ready pod other than $1.
|
||||
local old="$1" limit="${2:-240}"
|
||||
for _ in $(seq 1 $((limit * 5))); do
|
||||
for p in $("${k[@]}" get pods -n "$ns" -l "$selector" \
|
||||
-o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}'); do
|
||||
[ "$p" = "$old" ] && continue
|
||||
[ "$("${k[@]}" get pod -n "$ns" "$p" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2> /dev/null)" = True ] &&
|
||||
{ echo "$p"; return 0; }
|
||||
done
|
||||
sleep 0.2
|
||||
done
|
||||
return 1
|
||||
}
|
||||
restore() {
|
||||
set +e
|
||||
[ -n "$follow_pid" ] && kill "$follow_pid" 2> /dev/null
|
||||
say "restoring the Git configuration of evo-region (REGION_INPUTS empty)"
|
||||
local current
|
||||
current="$("${k[@]}" get pods -n "$ns" -l "$selector" -o jsonpath='{.items[0].metadata.name}' 2> /dev/null)"
|
||||
"${k[@]}" set env -n "$ns" deployment/evo-region REGION_INPUTS= > /dev/null
|
||||
ready_pod "$current" 240 > /dev/null && say "restored: evo-region Ready" || say "WARN evo-region not Ready after restore"
|
||||
say "drill files kept in $work"
|
||||
}
|
||||
|
||||
world="$(env_of REGION_WORLD)" region="$(env_of REGION_ID)" frame="$(env_of REGION_FRAME)"
|
||||
window="$(env_of REGION_SESSION_WINDOW)"
|
||||
[ -n "$world" ] && [ -n "$region" ] && [ -n "$frame" ] || fail "cannot read the region identity"
|
||||
[ -z "$(env_of REGION_INPUTS)" ] || fail "REGION_INPUTS is already set: another drill may be running"
|
||||
# The drill must end before the session window closes.
|
||||
count=$(echo "$steps" | tr ',' '\n' | wc -l)
|
||||
most=$(echo "$steps" | tr ',' '\n' | sort -n | tail -1)
|
||||
needed=$(( count * (hold + warmup + most / 10 + 5) + 600 ))
|
||||
expiry="${window##*:}"
|
||||
[ -n "$expiry" ] && [ "$expiry" -gt $(( $(date -u +%s) + needed )) ] 2> /dev/null ||
|
||||
fail "the session window ($window) closes within the drill's ${needed} s: renew it first"
|
||||
size=$(wc -c < "$tokens/load-grants.bin")
|
||||
[ $(( size / 160 )) -ge $(( most * per_bot )) ] ||
|
||||
fail "load-grants.bin holds $(( size / 160 )) grants; $most bots x $per_bot need $(( most * per_bot ))"
|
||||
image="$("${k[@]}" get deployment -n "$ns" evo-region -o jsonpath='{.spec.template.spec.containers[0].image}')"
|
||||
say "drill: world $world region $region image $image steps $steps hold ${hold}s"
|
||||
|
||||
trap restore EXIT
|
||||
before="$("${k[@]}" get pods -n "$ns" -l "$selector" -o jsonpath='{.items[0].metadata.name}')"
|
||||
say "enabling synthetic inputs in $ns (drill-time override, restored on exit)"
|
||||
"${k[@]}" set env -n "$ns" deployment/evo-region REGION_INPUTS=development-v1 > /dev/null
|
||||
pod="$(ready_pod "$before" 240)" || fail "no Ready evo-region pod with synthetic inputs"
|
||||
"${k[@]}" logs -n "$ns" -f --timestamps "$pod" > "$work/server.log" 2> /dev/null &
|
||||
follow_pid=$!
|
||||
sleep 5
|
||||
grep -q "ready 1" "$work/server.log" || say "WARN no 'ready 1' in the server log yet"
|
||||
say "running evo-load against $connect"
|
||||
set +e
|
||||
"$tool" --connect "$connect" --server-name "$server_name" --ca "$tokens/dev-ca.pem" \
|
||||
--grants "$tokens/load-grants.bin" --grants-per-bot "$per_bot" --world "$world" \
|
||||
--frame "$frame" --region "$region" --steps "$steps" --hold "$hold" --warmup "$warmup" \
|
||||
--threads 8 --per-client > "$work/load.log" 2>&1
|
||||
load_exit=$?
|
||||
set -e
|
||||
sleep 12 # One more server report after the last hold.
|
||||
grep -E "^load (step|response_ms|outcome_ms|snapshots|starved|verdict|result)" "$work/load.log" || true
|
||||
[ "$load_exit" = 0 ] || say "WARN evo-load exited $load_exit (see $work/load.log)"
|
||||
"$py" - "$work/load.log" "$work/server.log" <<'PY'
|
||||
import re, sys
|
||||
from datetime import datetime
|
||||
load, server = sys.argv[1], sys.argv[2]
|
||||
reports = []
|
||||
for line in open(server, encoding='utf-8', errors='replace'):
|
||||
m = re.match(r'(\S+) m10 load-report (.*)', line.strip())
|
||||
if not m:
|
||||
continue
|
||||
stamp = m.group(1)[:19]
|
||||
t = datetime.strptime(stamp, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=None)
|
||||
epoch = int((t - datetime(1970, 1, 1)).total_seconds())
|
||||
fields = dict(kv.split('=') for kv in m.group(2).split())
|
||||
reports.append((epoch, {k: int(v) for k, v in fields.items()}))
|
||||
print(f"drill server reports={len(reports)}")
|
||||
for line in open(load, encoding='utf-8', errors='replace'):
|
||||
m = re.match(r'load step N=(\d+) .*hold_utc=(\d+)\.\.(\d+)', line)
|
||||
if not m:
|
||||
continue
|
||||
n, t0, t1 = int(m.group(1)), int(m.group(2)), int(m.group(3))
|
||||
# A report covers the 2 s before it: only reports wholly inside the hold count.
|
||||
inside = [r for t, r in reports if t0 + 2 <= t <= t1]
|
||||
if not inside:
|
||||
print(f"drill step N={n} server_reports=0")
|
||||
continue
|
||||
worst = lambda key: max(r[key] for r in inside)
|
||||
sessions = max(1, worst('sessions'))
|
||||
print(f"drill step N={n} server_reports={len(inside)} sessions_max={worst('sessions')} "
|
||||
f"work_p99_max_ms={worst('p99_us') / 1000:.1f} work_max_ms={worst('max_us') / 1000:.1f} "
|
||||
f"session_p99_max_ms={worst('session_p99_us') / 1000:.1f} "
|
||||
f"per_session_ms={worst('session_p99_us') / 1000 / sessions:.3f} "
|
||||
f"refusing_max={worst('refusing')} shed_max={worst('shed')} "
|
||||
f"tick_debt_max={worst('tick_debt')} "
|
||||
f"server_within_budget={int(worst('p99_us') <= 25000 and worst('session_p99_us') <= 500 * sessions and worst('tick_debt') == 0)}")
|
||||
PY
|
||||
say "drill complete (evo-load exit $load_exit); restoring"
|
||||
Reference in New Issue
Block a user