Add M10.9d load drill harness and serialize Mbed TLS use
Foundation / checks (deploy) (push) Successful in 3m24s
Foundation / checks (ecs) (push) Successful in 1m56s
Foundation / checks (client) (push) Successful in 12m18s
Foundation / checks (network) (push) Successful in 13m30s
Foundation / checks (tsan) (push) Successful in 8m0s
Foundation / checks (checks) (push) Successful in 32m39s
Foundation / checks (postgres) (push) Successful in 45m47s
Foundation / postgres-tools (push) Successful in 17s
Foundation / image (push) Successful in 2m30s
Foundation / server-image (push) Successful in 4m8s
Foundation / checks (deploy) (push) Successful in 3m24s
Foundation / checks (ecs) (push) Successful in 1m56s
Foundation / checks (client) (push) Successful in 12m18s
Foundation / checks (network) (push) Successful in 13m30s
Foundation / checks (tsan) (push) Successful in 8m0s
Foundation / checks (checks) (push) Successful in 32m39s
Foundation / checks (postgres) (push) Successful in 45m47s
Foundation / postgres-tools (push) Successful in 17s
Foundation / image (push) Successful in 2m30s
Foundation / server-image (push) Successful in 4m8s
ADR-0097. tools/ops/load-drill.sh (runbook docs/networking/M10_LOAD_DRILL.md) runs the evolution-dev reference qualification: synthetic inputs on for the run (the ADR-0073 drill-time override, restored on exit), bundle-mode evo-load bots through the public route, and each step's hold matched with the server's reports. evo-server logs "m10 load-report" every 10 s, since the agent cannot reach the ops port. evo-load gains --grants FILE --grants-per-bot C, exhausted=, hold_utc= and close_reasons=; evo-session-probe grants gains --accounts N. Finding: Mbed TLS/PSA are built without MBEDTLS_THREADING_C, and bots on different threads verifying their first connections at once were refused as untrusted (two per smoke run). One process-wide recursive crypto lock now guards every network-library entry point that touches them; the smoke went from 2/2 failing to 3/3 clean. quic-transport adds a concurrent-connect test. Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -296,6 +296,7 @@ template <typename Host> class M10Bridge {
|
||||
bool changed = false;
|
||||
for (std::uint64_t i = 0; i < ticks; ++i)
|
||||
changed = governor_.observe(per_tick, session_per_tick, s.tick_debt, now_us) || changed;
|
||||
report(s, now_us);
|
||||
if (!changed || !governed_)
|
||||
return;
|
||||
const auto& load = governor_.state();
|
||||
@@ -303,6 +304,23 @@ template <typename Host> class M10Bridge {
|
||||
std::printf("m10 load refusing=%d shed=%u p95_us=%llu\n", load.refusing ? 1 : 0,
|
||||
static_cast<unsigned>(load.shed), static_cast<unsigned long long>(load.p95_us));
|
||||
}
|
||||
// M10.9d (ADR-0097): every 10 s, the load as /metrics would show it, in the log (drills
|
||||
// read it with `kubectl logs --timestamps` where the ops port is not reachable).
|
||||
void report(const server::HostStatus& s, std::uint64_t now_us) {
|
||||
if (reported_us_ != 0 && now_us - reported_us_ < 10'000'000)
|
||||
return;
|
||||
reported_us_ = now_us;
|
||||
const auto& load = governor_.state();
|
||||
std::printf("m10 load-report sessions=%zu p95_us=%llu p99_us=%llu max_us=%llu "
|
||||
"session_p95_us=%llu session_p99_us=%llu refusing=%d shed=%u tick_debt=%llu\n",
|
||||
sessions_->sessions(), static_cast<unsigned long long>(load.p95_us),
|
||||
static_cast<unsigned long long>(load.p99_us),
|
||||
static_cast<unsigned long long>(load.max_us),
|
||||
static_cast<unsigned long long>(load.session_p95_us),
|
||||
static_cast<unsigned long long>(load.session_p99_us),
|
||||
governed_ && load.refusing ? 1 : 0, governed_ ? load.shed : 0U,
|
||||
static_cast<unsigned long long>(s.tick_debt));
|
||||
}
|
||||
static const processes::EnvironmentalCommunity&
|
||||
community(const processes::EnvironmentalCommunity& c) {
|
||||
return c;
|
||||
@@ -448,6 +466,7 @@ template <typename Host> class M10Bridge {
|
||||
std::unique_ptr<server::PostgresActionLedger> actions_;
|
||||
server::LoadGovernor governor_{*server::LoadGovernor::create({})};
|
||||
bool governed_{true};
|
||||
std::uint64_t reported_us_{};
|
||||
std::chrono::nanoseconds busy_{}, session_busy_{};
|
||||
std::uint64_t measured_tick_{};
|
||||
std::unique_ptr<server::PostgresProbeLedger> probes_; // Handoff mode only.
|
||||
|
||||
@@ -15,6 +15,9 @@
|
||||
// interleaved in serial order, so a client crossing between regions always has a newer
|
||||
// serial for the next one (serials must rise world-wide). A handoff key is a key pair
|
||||
// from `keys` (issuer.key for --m10-handoff-key, issuer.pub for --m10-neighbour).
|
||||
// M10.9d (ADR-0097): --accounts N (1..256) writes the same for N accounts, account by
|
||||
// account: account i is --account with its last two bytes replaced by i + 1 (big
|
||||
// endian). This is the load bundle `evo-load --grants FILE --grants-per-bot COUNT` uses.
|
||||
// evo-session-probe run --connect HOST:PORT --server-name NAME --ca FILE --issuer-key FILE
|
||||
// --world N --frame N --region N --account HEX32 --serial N [--lifetime S]
|
||||
// [--intents N] [--actions N --action-base ID] [--query ID]... [--timeout-ms N]
|
||||
@@ -206,14 +209,14 @@ bool write_new(const std::filesystem::path& path, std::span<const std::uint8_t>
|
||||
return true;
|
||||
}
|
||||
|
||||
int grants(const RunOptions& o, const std::string& out, std::uint64_t count) {
|
||||
int grants(const RunOptions& o, const std::string& out, std::uint64_t count,
|
||||
std::uint64_t accounts) {
|
||||
const auto key = issuer(o.issuer_key);
|
||||
if (!key || !o.account) {
|
||||
std::puts("error issuer-key");
|
||||
return 2;
|
||||
}
|
||||
GrantClaims claims;
|
||||
claims.account = *o.account;
|
||||
claims.world = {o.world, o.frame};
|
||||
claims.issued_at = now_seconds();
|
||||
claims.expires_at = claims.issued_at + o.lifetime;
|
||||
@@ -221,31 +224,39 @@ int grants(const RunOptions& o, const std::string& out, std::uint64_t count) {
|
||||
std::vector<std::uint64_t> regions{o.region};
|
||||
regions.insert(regions.end(), o.more_regions.begin(), o.more_regions.end());
|
||||
if (count * regions.size() > 64) {
|
||||
std::puts("error count (at most 64 grants in all)");
|
||||
std::puts("error count (at most 64 grants per account)");
|
||||
return 2;
|
||||
}
|
||||
std::vector<std::uint8_t> bundle;
|
||||
std::uint64_t serial = first;
|
||||
for (std::uint64_t i = 0; i < count; ++i)
|
||||
for (const auto region : regions) {
|
||||
claims.region = region;
|
||||
claims.serial = serial++;
|
||||
const auto grant = sign(*key, claims);
|
||||
if (!grant) {
|
||||
std::puts("error grant");
|
||||
return 2;
|
||||
}
|
||||
bundle.insert(bundle.end(), grant->begin(), grant->end());
|
||||
for (std::uint64_t a = 0; a < accounts; ++a) {
|
||||
claims.account = *o.account;
|
||||
if (accounts > 1) { // Account a: the last two bytes are a + 1.
|
||||
claims.account.bytes.at(14) = static_cast<std::uint8_t>(((a + 1) >> 8U) & 0xFFU);
|
||||
claims.account.bytes.at(15) = static_cast<std::uint8_t>((a + 1) & 0xFFU);
|
||||
}
|
||||
std::uint64_t serial = first;
|
||||
for (std::uint64_t i = 0; i < count; ++i)
|
||||
for (const auto region : regions) {
|
||||
claims.region = region;
|
||||
claims.serial = serial++;
|
||||
const auto grant = sign(*key, claims);
|
||||
if (!grant) {
|
||||
std::puts("error grant");
|
||||
return 2;
|
||||
}
|
||||
bundle.insert(bundle.end(), grant->begin(), grant->end());
|
||||
}
|
||||
}
|
||||
count *= regions.size();
|
||||
if (!write_new(out, bundle)) {
|
||||
std::puts("error out (exists or unwritable)");
|
||||
return 1;
|
||||
}
|
||||
std::printf("wrote %llu grants serial %llu..%llu valid %llu s\n",
|
||||
static_cast<unsigned long long>(count), static_cast<unsigned long long>(first),
|
||||
static_cast<unsigned long long>(first + count - 1),
|
||||
static_cast<unsigned long long>(o.lifetime));
|
||||
std::printf(
|
||||
"wrote %llu grants serial %llu..%llu valid %llu s accounts %llu\n",
|
||||
static_cast<unsigned long long>(count) * accounts, static_cast<unsigned long long>(first),
|
||||
static_cast<unsigned long long>(first + count - 1),
|
||||
static_cast<unsigned long long>(o.lifetime), static_cast<unsigned long long>(accounts));
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -430,7 +441,7 @@ int main(int argc, char** argv) {
|
||||
RunOptions o;
|
||||
const bool bundle = !args.empty() && args[0] == "grants";
|
||||
std::string out;
|
||||
std::uint64_t count = 1;
|
||||
std::uint64_t count = 1, accounts = 1;
|
||||
if (bundle)
|
||||
o.lifetime = 300; // The region's default maximum.
|
||||
bool ok = !args.empty() && (args[0] == "run" || bundle) && args.size() % 2 == 1;
|
||||
@@ -489,26 +500,29 @@ int main(int argc, char** argv) {
|
||||
out = value;
|
||||
else if (bundle && n && flag == "--count" && *n >= 1 && *n <= 64)
|
||||
count = *n;
|
||||
else if (bundle && n && flag == "--accounts" && *n >= 1 && *n <= 256)
|
||||
accounts = *n;
|
||||
else
|
||||
ok = false;
|
||||
}
|
||||
if (bundle && ok && !o.issuer_key.empty() && o.account && o.world != 0 && o.frame != 0 &&
|
||||
o.region != 0 && !out.empty() && o.lifetime >= 60 && o.host.empty())
|
||||
return grants(o, out, count);
|
||||
return grants(o, out, count, accounts);
|
||||
const bool signs = o.grants_file.empty();
|
||||
if (bundle || !ok || o.host.empty() || o.server_name.empty() || o.ca.empty() ||
|
||||
(signs && (o.issuer_key.empty() || !o.account || o.world == 0 || o.frame == 0 ||
|
||||
o.region == 0 || o.serial == 0)) ||
|
||||
(!signs && !o.issuer_key.empty())) {
|
||||
std::fputs("usage: evo-session-probe keys --out DIR | grants --issuer-key FILE "
|
||||
"--world N --frame N --region N --account HEX32 --out FILE [--serial N] "
|
||||
"[--count 1..64] [--lifetime 60..14400] | run --connect HOST:PORT "
|
||||
"--server-name NAME --ca FILE --issuer-key FILE --world N --frame N "
|
||||
"--region N --account HEX32 --serial N [--lifetime S] [--intents N] "
|
||||
"[--actions N --action-base ID] [--query ID]... [--timeout-ms N] "
|
||||
"[--interest X,Y,Z,R] [--snapshots N] | run --connect HOST:PORT "
|
||||
"--server-name NAME --ca FILE --grants FILE [--grant N] [run options]\n",
|
||||
stderr);
|
||||
std::fputs(
|
||||
"usage: evo-session-probe keys --out DIR | grants --issuer-key FILE "
|
||||
"--world N --frame N --region N --account HEX32 --out FILE [--serial N] "
|
||||
"[--count 1..64] [--accounts 1..256] [--lifetime 60..14400] | run --connect HOST:PORT "
|
||||
"--server-name NAME --ca FILE --issuer-key FILE --world N --frame N "
|
||||
"--region N --account HEX32 --serial N [--lifetime S] [--intents N] "
|
||||
"[--actions N --action-base ID] [--query ID]... [--timeout-ms N] "
|
||||
"[--interest X,Y,Z,R] [--snapshots N] | run --connect HOST:PORT "
|
||||
"--server-name NAME --ca FILE --grants FILE [--grant N] [run options]\n",
|
||||
stderr);
|
||||
return 2;
|
||||
}
|
||||
return run(o);
|
||||
|
||||
Reference in New Issue
Block a user