An AEvolution-only Prometheus 3.13.3 and Alertmanager 0.34.1, pinned by digest and delivered by Argo (ADR-0072), scrape each region's operations Service through static targets (no API access, no RBAC); the region namespaces admit port 9100 from Prometheus only. Sixteen alert rules cover authority, lease, commits, backlog and object capacity and are proven by promtool injection tests in CI with tools copied from the same pinned images. Two new gauges expose object volume free space and the reserve. Alerts route to a null receiver until a destination is chosen; no broker is adopted and the database outbox stays. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
274 lines
12 KiB
C++
274 lines
12 KiB
C++
// M7.8 operational endpoints on real evo-server processes (ADR-0066): liveness is
|
|
// never authority, readiness follows the lease, metrics are bounded, and the
|
|
// responder is bounded against idle, excess and malformed clients.
|
|
#include "evo/server/ops_http.hpp"
|
|
#include "fixtures/server_process.hpp"
|
|
#include <algorithm>
|
|
#include <arpa/inet.h>
|
|
#include <atomic>
|
|
#include <netinet/in.h>
|
|
#include <source_location>
|
|
#include <sys/socket.h>
|
|
#include <thread>
|
|
|
|
void run_region_ops_tests();
|
|
|
|
namespace {
|
|
using namespace evo::test::process;
|
|
void check(bool value, std::source_location at = std::source_location::current()) {
|
|
if (!value)
|
|
throw std::runtime_error("region ops test line " + std::to_string(at.line()));
|
|
}
|
|
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
|
|
|
int connect_local(std::uint16_t port) {
|
|
const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0);
|
|
CHECK(fd >= 0);
|
|
sockaddr_in address{};
|
|
address.sin_family = AF_INET;
|
|
address.sin_port = htons(port);
|
|
address.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
|
|
CHECK(::connect(fd, reinterpret_cast<sockaddr*>(&address), sizeof address) == 0);
|
|
timeval timeout{3, 0};
|
|
::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout);
|
|
return fd;
|
|
}
|
|
struct Reply {
|
|
int status{};
|
|
std::string body;
|
|
};
|
|
// Sends raw bytes and reads until the server closes (Connection: close).
|
|
Reply http_exchange(std::uint16_t port, const std::string& request) {
|
|
const int fd = connect_local(port);
|
|
std::size_t sent = 0;
|
|
while (sent < request.size()) {
|
|
const auto n = ::send(fd, request.data() + sent, request.size() - sent, MSG_NOSIGNAL);
|
|
if (n <= 0)
|
|
break;
|
|
sent += static_cast<std::size_t>(n);
|
|
}
|
|
std::string raw;
|
|
char chunk[4096];
|
|
for (;;) {
|
|
const auto n = ::recv(fd, chunk, sizeof chunk, 0);
|
|
if (n <= 0)
|
|
break;
|
|
raw.append(chunk, static_cast<std::size_t>(n));
|
|
}
|
|
::close(fd);
|
|
int status = 0;
|
|
CHECK(std::sscanf(raw.c_str(), "HTTP/1.1 %d", &status) == 1);
|
|
const auto body = raw.find("\r\n\r\n");
|
|
return Reply{status, body == std::string::npos ? "" : raw.substr(body + 4)};
|
|
}
|
|
Reply get(std::uint16_t port, const std::string& path) {
|
|
return http_exchange(port, "GET " + path + " HTTP/1.1\r\nHost: probe\r\n\r\n");
|
|
}
|
|
std::uint64_t metric(std::uint16_t port, const std::string& series) {
|
|
const auto reply = get(port, "/metrics");
|
|
CHECK(reply.status == 200);
|
|
const auto at = reply.body.find("\n" + series + " ");
|
|
CHECK(at != std::string::npos);
|
|
return std::stoull(reply.body.substr(at + series.size() + 2));
|
|
}
|
|
std::uint16_t ops_port(Child& server) {
|
|
CHECK(server.wait_line("ops listening ", 10s));
|
|
for (const auto& l : server.lines())
|
|
if (starts(l, "ops listening "))
|
|
return static_cast<std::uint16_t>(std::stoul(l.substr(14)));
|
|
CHECK(false);
|
|
return 0;
|
|
}
|
|
std::vector<std::string> with_ops(const ServerRegion& r, std::vector<std::string> extra = {}) {
|
|
extra.insert(extra.begin(), {"--ops-listen", "127.0.0.1:0"});
|
|
return server_args(r, std::move(extra));
|
|
}
|
|
|
|
void probes_follow_authority_and_stay_bounded() {
|
|
const auto r = provision(51, 0x0B);
|
|
Child server{with_ops(r)};
|
|
const auto port = ops_port(server);
|
|
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
|
auto live = get(port, "/livez");
|
|
auto ready = get(port, "/readyz");
|
|
CHECK(live.status == 200 && live.body == "live\n");
|
|
CHECK(ready.status == 200 && ready.body == "ready\n");
|
|
CHECK(metric(port, "evo_region_ready") == 1 && metric(port, "evo_region_lease_epoch") == 1);
|
|
CHECK(metric(port, "evo_region_lease_remaining_milliseconds") > 0);
|
|
CHECK(metric(port, "evo_region_queue_capacity_inputs") == 64 &&
|
|
metric(port, "evo_region_queue_capacity_bytes") == 65536);
|
|
// M8.5: the default reserve is 1024 MiB, and the volume reports real free space.
|
|
CHECK(metric(port, "evo_region_objects_reserve_bytes") == std::uint64_t{1024} * 1024 * 1024 &&
|
|
metric(port, "evo_region_objects_available_bytes") > 0);
|
|
auto page = get(port, "/metrics");
|
|
CHECK(page.body.find("evo_region_phase{phase=\"running\"} 1\n") != std::string::npos);
|
|
server.send(input_lines(1, 30, false));
|
|
CHECK(server.wait_line("ack " + request_hex(30) + " 30 ", 15s));
|
|
CHECK(metric(port, "evo_region_committed_tick") == 30 &&
|
|
metric(port, "evo_region_commit_lag_ticks") == 0 &&
|
|
metric(port, "evo_region_acknowledgements_total") == 30);
|
|
CHECK(metric(port, "evo_region_checkpoint_age_milliseconds") < 60000);
|
|
|
|
// Bounded, strict responder.
|
|
CHECK(http_exchange(port, "POST /readyz HTTP/1.1\r\n\r\n").status == 405);
|
|
CHECK(get(port, "/nope").status == 404);
|
|
CHECK(get(port, "/readyz?x=1").status == 404); // Exact paths only.
|
|
CHECK(http_exchange(port, "HELLO\r\n\r\n").status == 400);
|
|
CHECK(http_exchange(port, "GET /livez HTTP/1.1\r\nX: " + std::string(5000, 'a') + "\r\n\r\n")
|
|
.status == 431);
|
|
// Idle connections neither block probes nor exceed the connection bound.
|
|
std::vector<int> idle;
|
|
idle.reserve(8);
|
|
for (int i = 0; i < 8; ++i)
|
|
idle.push_back(connect_local(port));
|
|
const int excess = connect_local(port);
|
|
char byte;
|
|
// Refused at once (well before the 1 s idle deadline), never queued behind others.
|
|
const auto refused_at = Clock::now();
|
|
CHECK(::recv(excess, &byte, 1, 0) == 0);
|
|
CHECK(Clock::now() - refused_at < 500ms);
|
|
::close(excess);
|
|
std::this_thread::sleep_for(1300ms); // Idle connections expire at their deadline.
|
|
const auto started = Clock::now();
|
|
CHECK(get(port, "/livez").status == 200);
|
|
CHECK(Clock::now() - started < 1s);
|
|
for (const int fd : idle) {
|
|
CHECK(::recv(fd, &byte, 1, 0) == 0); // Closed by the server.
|
|
::close(fd);
|
|
}
|
|
server.signal(SIGTERM);
|
|
CHECK(server.wait_exit(20s) == 0);
|
|
}
|
|
|
|
// The database is unreachable: the process is alive but never ready.
|
|
void unavailable_database_is_live_not_ready() {
|
|
const ServerRegion r{{7, 52}, fs::path(env("PGHOST")) / "server-52"};
|
|
fs::remove_all(r.root);
|
|
fs::create_directory(r.root);
|
|
auto args = with_ops(r);
|
|
const auto at = std::ranges::find(args, "--pg-port");
|
|
CHECK(at != args.end());
|
|
*(at + 1) = "1"; // No server socket there.
|
|
Child server{args};
|
|
const auto port = ops_port(server);
|
|
std::this_thread::sleep_for(500ms);
|
|
CHECK(get(port, "/livez").status == 200);
|
|
auto ready = get(port, "/readyz");
|
|
CHECK(ready.status == 503 && ready.body == "not ready: acquiring\n");
|
|
CHECK(metric(port, "evo_region_ready") == 0 &&
|
|
metric(port, "evo_region_store_failures_total") > 0);
|
|
CHECK(get(port, "/metrics").body.find("evo_region_phase{phase=\"acquiring\"} 1\n") !=
|
|
std::string::npos);
|
|
CHECK(!server.printed("ready 1"));
|
|
server.signal(SIGTERM);
|
|
CHECK(server.wait_exit(20s) == 0);
|
|
}
|
|
|
|
// A paused process that lost its lease is alive, but readiness and metrics say it
|
|
// no longer holds authority; the successor is the only ready process.
|
|
void stale_process_is_live_not_ready() {
|
|
const auto r = provision(53, 0x0C);
|
|
Child stale{with_ops(r)};
|
|
const auto stale_port = ops_port(stale);
|
|
CHECK(stale.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
|
stale.signal(SIGSTOP);
|
|
std::this_thread::sleep_for(4500ms);
|
|
Child current{with_ops(r)};
|
|
const auto current_port = ops_port(current);
|
|
CHECK(current.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
|
stale.signal(SIGCONT);
|
|
CHECK(stale.wait_line("ready 0", 10s));
|
|
CHECK(get(stale_port, "/livez").status == 200);
|
|
CHECK(get(stale_port, "/readyz").status == 503);
|
|
CHECK(metric(stale_port, "evo_region_ready") == 0 &&
|
|
metric(stale_port, "evo_region_fences_total") == 1 &&
|
|
metric(stale_port, "evo_region_lease_remaining_milliseconds") == 0);
|
|
CHECK(get(current_port, "/readyz").status == 200 &&
|
|
metric(current_port, "evo_region_lease_epoch") == 2);
|
|
stale.signal(SIGTERM);
|
|
current.signal(SIGTERM);
|
|
CHECK(stale.wait_exit(20s) == 0 && current.wait_exit(20s) == 0);
|
|
}
|
|
|
|
// Review fix: accept work per serve() is bounded, refused connections included, so
|
|
// replenished connection pressure cannot hold the host loop; accepted connections
|
|
// are still answered while the pressure continues.
|
|
void accept_work_is_bounded_per_serve() {
|
|
using evo::server::OpsServer;
|
|
auto made = OpsServer::listen("127.0.0.1:0");
|
|
CHECK(made);
|
|
auto& ops = **made;
|
|
const auto handler = [](std::string_view) { return evo::server::OpsResponse{200, "ok\n"}; };
|
|
constexpr auto per = OpsServer::max_accepts_per_serve;
|
|
std::vector<int> clients;
|
|
clients.reserve(64);
|
|
for (int i = 0; i < 16; ++i)
|
|
clients.push_back(connect_local(ops.port())); // Queued in the listen backlog.
|
|
std::this_thread::sleep_for(50ms);
|
|
ops.serve(handler);
|
|
CHECK(ops.open_connections() == per && ops.refused() == 0);
|
|
ops.serve(handler);
|
|
CHECK(ops.open_connections() == OpsServer::max_connections && ops.refused() == 0);
|
|
// At the connection cap, each call still refuses at most the budget.
|
|
ops.serve(handler);
|
|
CHECK(ops.open_connections() == OpsServer::max_connections && ops.refused() == per);
|
|
// The first accepted client is answered while new connections keep arriving.
|
|
const std::string request = "GET /livez HTTP/1.1\r\n\r\n";
|
|
CHECK(::send(clients[0], request.data(), request.size(), MSG_NOSIGNAL) ==
|
|
static_cast<ssize_t>(request.size()));
|
|
for (int round = 0; round < 6; ++round) {
|
|
clients.push_back(connect_local(ops.port()));
|
|
const auto before = ops.refused();
|
|
ops.serve(handler);
|
|
CHECK(ops.refused() - before <= per);
|
|
}
|
|
char reply[64] = {};
|
|
CHECK(::recv(clients[0], reply, sizeof reply - 1, 0) > 0);
|
|
CHECK(std::string_view(reply).starts_with("HTTP/1.1 200"));
|
|
for (const int fd : clients)
|
|
::close(fd);
|
|
}
|
|
|
|
// A real server under sustained connection churn keeps its lease and readiness.
|
|
void connection_pressure_keeps_authority() {
|
|
const auto r = provision(54, 0x0D);
|
|
Child server{with_ops(r)};
|
|
const auto port = ops_port(server);
|
|
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
|
std::atomic<bool> stop{false};
|
|
std::thread flood([&] {
|
|
while (!stop.load()) {
|
|
const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0);
|
|
sockaddr_in address{};
|
|
address.sin_family = AF_INET;
|
|
address.sin_port = htons(port);
|
|
address.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
|
|
if (fd >= 0) {
|
|
// Refusal or backlog overflow is part of the pressure; ignore it.
|
|
[[maybe_unused]] const int connected =
|
|
::connect(fd, reinterpret_cast<sockaddr*>(&address), sizeof address);
|
|
::close(fd);
|
|
}
|
|
}
|
|
});
|
|
// Longer than the fast lease (4 s): renewals must keep happening under pressure.
|
|
server.send(input_lines(1, 60, false));
|
|
const bool acked = server.wait_line("ack " + request_hex(60) + " 60 ", 20s);
|
|
stop = true;
|
|
flood.join();
|
|
CHECK(acked);
|
|
CHECK(!server.printed("ready 0"));
|
|
CHECK(get(port, "/readyz").status == 200);
|
|
server.signal(SIGTERM);
|
|
CHECK(server.wait_exit(20s) == 0);
|
|
}
|
|
} // namespace
|
|
|
|
void run_region_ops_tests() {
|
|
accept_work_is_bounded_per_serve();
|
|
connection_pressure_keeps_authority();
|
|
probes_follow_authority_and_stay_bounded();
|
|
unavailable_database_is_live_not_ready();
|
|
stale_process_is_live_not_ready();
|
|
}
|