An AEvolution-only Prometheus 3.13.3 and Alertmanager 0.34.1, pinned by digest and delivered by Argo (ADR-0072), scrape each region's operations Service through static targets (no API access, no RBAC); the region namespaces admit port 9100 from Prometheus only. Sixteen alert rules cover authority, lease, commits, backlog and object capacity and are proven by promtool injection tests in CI with tools copied from the same pinned images. Two new gauges expose object volume free space and the reserve. Alerts route to a null receiver until a destination is chosen; no broker is adopted and the database outbox stays. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
120 lines
4.7 KiB
C++
120 lines
4.7 KiB
C++
#include "evo/server/ops.hpp"
|
|
|
|
#include <algorithm>
|
|
#include <cstdio>
|
|
#include <limits>
|
|
#include <set>
|
|
#include <source_location>
|
|
#include <sstream>
|
|
#include <stdexcept>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
namespace {
|
|
using namespace evo::server;
|
|
void check(bool ok, std::source_location at = std::source_location::current()) {
|
|
if (!ok)
|
|
throw std::runtime_error("ops line " + std::to_string(at.line()));
|
|
}
|
|
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
|
|
|
struct Sample {
|
|
std::string name, labels, value;
|
|
};
|
|
// Parses exposition lines; every family must be introduced by HELP and TYPE.
|
|
std::vector<Sample> parse(const std::string& text) {
|
|
std::vector<Sample> out;
|
|
std::set<std::string> typed, helped;
|
|
std::istringstream in(text);
|
|
for (std::string line; std::getline(in, line);) {
|
|
CHECK(!line.empty());
|
|
if (line.rfind("# HELP ", 0) == 0) {
|
|
helped.insert(line.substr(7, line.find(' ', 7) - 7));
|
|
continue;
|
|
}
|
|
if (line.rfind("# TYPE ", 0) == 0) {
|
|
const auto name = line.substr(7, line.find(' ', 7) - 7);
|
|
const auto type = line.substr(line.rfind(' ') + 1);
|
|
CHECK(type == "gauge" || type == "counter");
|
|
CHECK(helped.contains(name) && !typed.contains(name)); // One family, documented.
|
|
typed.insert(name);
|
|
continue;
|
|
}
|
|
const auto space = line.rfind(' ');
|
|
const auto brace = line.find('{');
|
|
Sample s{line.substr(0, std::min(space, brace)), "", line.substr(space + 1)};
|
|
if (brace < space)
|
|
s.labels = line.substr(brace, space - brace);
|
|
CHECK(typed.contains(s.name));
|
|
CHECK(!s.value.empty() && s.value.find_first_not_of("0123456789") == std::string::npos);
|
|
out.push_back(s);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
void exposition_is_bounded_and_well_formed() {
|
|
HostStatus s;
|
|
s.phase = HostPhase::running;
|
|
s.ready = true;
|
|
s.epoch = 3;
|
|
s.simulated_tick = 50;
|
|
s.committed_tick = 41;
|
|
s.lease_remaining_ms = 1234;
|
|
OpsCounters c;
|
|
c.queue_capacity_inputs = 64;
|
|
c.queue_capacity_bytes = 65536;
|
|
c.dropped[2] = 5;
|
|
const auto text = render_metrics(s, c);
|
|
const auto samples = parse(text);
|
|
const std::set<std::string> label_names{"", "phase", "reason"};
|
|
std::size_t phases_on = 0;
|
|
for (const auto& x : samples) {
|
|
CHECK(x.name.rfind("evo_region_", 0) == 0);
|
|
const auto key =
|
|
x.labels.empty() ? std::string{} : x.labels.substr(1, x.labels.find('=') - 1);
|
|
CHECK(label_names.contains(key)); // No world/region/entity/request labels.
|
|
if (x.name == "evo_region_phase")
|
|
phases_on += x.value == "1";
|
|
}
|
|
CHECK(phases_on == 1);
|
|
CHECK(text.find("evo_region_phase{phase=\"running\"} 1\n") != std::string::npos);
|
|
CHECK(text.find("evo_region_ready 1\n") != std::string::npos);
|
|
CHECK(text.find("evo_region_commit_lag_ticks 9\n") != std::string::npos);
|
|
CHECK(text.find("evo_region_lease_remaining_milliseconds 1234\n") != std::string::npos);
|
|
CHECK(text.find("evo_region_dropped_total{reason=\"fenced\"} 5\n") != std::string::npos);
|
|
// Series count is fixed: 6 phases + 4 reasons + 22 unlabelled families.
|
|
CHECK(samples.size() == 6 + 4 + 22);
|
|
|
|
// Worst-case values keep the size bounded and never underflow the lag.
|
|
HostStatus big;
|
|
big.phase = HostPhase::quarantined;
|
|
constexpr auto most = std::numeric_limits<std::uint64_t>::max();
|
|
big.epoch = big.committed_tick = big.committed_sequence = big.tick_debt = most;
|
|
big.max_tick_debt = big.commits = big.store_failures = big.unknown_outcomes = most;
|
|
big.fences = big.lease_remaining_ms = most;
|
|
big.simulated_tick = 1; // Lag saturates at zero rather than wrapping.
|
|
OpsCounters bigc{most, most, most, most, most, {most, most, most, most}, most, most};
|
|
const auto large = render_metrics(big, bigc);
|
|
CHECK(large.size() <= max_metrics_bytes);
|
|
CHECK(large.find("evo_region_commit_lag_ticks 0\n") != std::string::npos);
|
|
CHECK(large.find("evo_region_ready 0\n") != std::string::npos);
|
|
CHECK(parse(large).size() == samples.size());
|
|
for (std::size_t i = 0; i < host_phase_count; ++i)
|
|
CHECK(std::string(phase_name(static_cast<HostPhase>(i))) != "unknown");
|
|
for (std::size_t i = 0; i < unacknowledged_count; ++i)
|
|
CHECK(std::string(unacknowledged_name(static_cast<Unacknowledged>(i))) != "unknown");
|
|
std::printf("ops_metrics_bytes typical=%zu worst=%zu\n", text.size(), large.size());
|
|
}
|
|
} // namespace
|
|
|
|
int main() {
|
|
try {
|
|
exposition_is_bounded_and_well_formed();
|
|
std::puts("ops contracts passed");
|
|
return 0;
|
|
} catch (const std::exception& e) {
|
|
std::fprintf(stderr, "%s\n", e.what());
|
|
return 1;
|
|
}
|
|
}
|