ADR-0066: evo-server --ops-listen serves /livez (loop responding, never authority), /readyz (200 only while the host holds the lease with a verified recovered head) and /metrics (Prometheus text: 30 fixed series for readiness, phase, lease epoch/remaining, simulated/committed ticks, commit lag, checkpoint age, tick debt, queue use and bounds, and counters; the only labels are a 6-value phase and 4-value drop reason). The dependency-free OpsServer runs on the host loop with no thread: GET on exact paths, 405/404/400/431 responses, 8 connections with excess refused on accept, 4 KiB requests, 1 s deadlines, 16 KiB responses. HostStatus/phase types moved to the portable evo::ops library with a Windows-tested renderer. evo-server now refuses to start without its required identity and connection settings. Adds the server-runtime Docker target (evo-server + pinned libpq, UID 10001, no credentials), a CI server-image job with smoke script and optional publishing, and a Kubernetes example outside Argo. Validated: Windows CTest 59/59; Linux Debug/Release/clang-tidy/ASan CTest 59/59; PostgreSQL suites under Release, clang-tidy and ASan with real process probes; two ops mutations caught. Docker build/smoke and Gitea CI not executed locally. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
194 lines
8.0 KiB
C++
194 lines
8.0 KiB
C++
// M7.8 operational endpoints on real evo-server processes (ADR-0066): liveness is
|
|
// never authority, readiness follows the lease, metrics are bounded, and the
|
|
// responder is bounded against idle, excess and malformed clients.
|
|
#include "fixtures/server_process.hpp"
|
|
#include <algorithm>
|
|
#include <arpa/inet.h>
|
|
#include <netinet/in.h>
|
|
#include <source_location>
|
|
#include <sys/socket.h>
|
|
#include <thread>
|
|
|
|
void run_region_ops_tests();
|
|
|
|
namespace {
|
|
using namespace evo::test::process;
|
|
void check(bool value, std::source_location at = std::source_location::current()) {
|
|
if (!value)
|
|
throw std::runtime_error("region ops test line " + std::to_string(at.line()));
|
|
}
|
|
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
|
|
|
int connect_local(std::uint16_t port) {
|
|
const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0);
|
|
CHECK(fd >= 0);
|
|
sockaddr_in address{};
|
|
address.sin_family = AF_INET;
|
|
address.sin_port = htons(port);
|
|
address.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
|
|
CHECK(::connect(fd, reinterpret_cast<sockaddr*>(&address), sizeof address) == 0);
|
|
timeval timeout{3, 0};
|
|
::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout);
|
|
return fd;
|
|
}
|
|
struct Reply {
|
|
int status{};
|
|
std::string body;
|
|
};
|
|
// Sends raw bytes and reads until the server closes (Connection: close).
|
|
Reply http_exchange(std::uint16_t port, const std::string& request) {
|
|
const int fd = connect_local(port);
|
|
std::size_t sent = 0;
|
|
while (sent < request.size()) {
|
|
const auto n = ::send(fd, request.data() + sent, request.size() - sent, MSG_NOSIGNAL);
|
|
if (n <= 0)
|
|
break;
|
|
sent += static_cast<std::size_t>(n);
|
|
}
|
|
std::string raw;
|
|
char chunk[4096];
|
|
for (;;) {
|
|
const auto n = ::recv(fd, chunk, sizeof chunk, 0);
|
|
if (n <= 0)
|
|
break;
|
|
raw.append(chunk, static_cast<std::size_t>(n));
|
|
}
|
|
::close(fd);
|
|
int status = 0;
|
|
CHECK(std::sscanf(raw.c_str(), "HTTP/1.1 %d", &status) == 1);
|
|
const auto body = raw.find("\r\n\r\n");
|
|
return Reply{status, body == std::string::npos ? "" : raw.substr(body + 4)};
|
|
}
|
|
Reply get(std::uint16_t port, const std::string& path) {
|
|
return http_exchange(port, "GET " + path + " HTTP/1.1\r\nHost: probe\r\n\r\n");
|
|
}
|
|
std::uint64_t metric(std::uint16_t port, const std::string& series) {
|
|
const auto reply = get(port, "/metrics");
|
|
CHECK(reply.status == 200);
|
|
const auto at = reply.body.find("\n" + series + " ");
|
|
CHECK(at != std::string::npos);
|
|
return std::stoull(reply.body.substr(at + series.size() + 2));
|
|
}
|
|
std::uint16_t ops_port(Child& server) {
|
|
CHECK(server.wait_line("ops listening ", 10s));
|
|
for (const auto& l : server.lines())
|
|
if (starts(l, "ops listening "))
|
|
return static_cast<std::uint16_t>(std::stoul(l.substr(14)));
|
|
CHECK(false);
|
|
return 0;
|
|
}
|
|
std::vector<std::string> with_ops(const ServerRegion& r, std::vector<std::string> extra = {}) {
|
|
extra.insert(extra.begin(), {"--ops-listen", "127.0.0.1:0"});
|
|
return server_args(r, std::move(extra));
|
|
}
|
|
|
|
void probes_follow_authority_and_stay_bounded() {
|
|
const auto r = provision(51, 0x0B);
|
|
Child server{with_ops(r)};
|
|
const auto port = ops_port(server);
|
|
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
|
auto live = get(port, "/livez");
|
|
auto ready = get(port, "/readyz");
|
|
CHECK(live.status == 200 && live.body == "live\n");
|
|
CHECK(ready.status == 200 && ready.body == "ready\n");
|
|
CHECK(metric(port, "evo_region_ready") == 1 && metric(port, "evo_region_lease_epoch") == 1);
|
|
CHECK(metric(port, "evo_region_lease_remaining_milliseconds") > 0);
|
|
CHECK(metric(port, "evo_region_queue_capacity_inputs") == 64 &&
|
|
metric(port, "evo_region_queue_capacity_bytes") == 65536);
|
|
auto page = get(port, "/metrics");
|
|
CHECK(page.body.find("evo_region_phase{phase=\"running\"} 1\n") != std::string::npos);
|
|
server.send(input_lines(1, 30, false));
|
|
CHECK(server.wait_line("ack " + request_hex(30) + " 30 ", 15s));
|
|
CHECK(metric(port, "evo_region_committed_tick") == 30 &&
|
|
metric(port, "evo_region_commit_lag_ticks") == 0 &&
|
|
metric(port, "evo_region_acknowledgements_total") == 30);
|
|
CHECK(metric(port, "evo_region_checkpoint_age_milliseconds") < 60000);
|
|
|
|
// Bounded, strict responder.
|
|
CHECK(http_exchange(port, "POST /readyz HTTP/1.1\r\n\r\n").status == 405);
|
|
CHECK(get(port, "/nope").status == 404);
|
|
CHECK(get(port, "/readyz?x=1").status == 404); // Exact paths only.
|
|
CHECK(http_exchange(port, "HELLO\r\n\r\n").status == 400);
|
|
CHECK(http_exchange(port, "GET /livez HTTP/1.1\r\nX: " + std::string(5000, 'a') + "\r\n\r\n")
|
|
.status == 431);
|
|
// Idle connections neither block probes nor exceed the connection bound.
|
|
std::vector<int> idle;
|
|
idle.reserve(8);
|
|
for (int i = 0; i < 8; ++i)
|
|
idle.push_back(connect_local(port));
|
|
const int excess = connect_local(port);
|
|
char byte;
|
|
// Refused at once (well before the 1 s idle deadline), never queued behind others.
|
|
const auto refused_at = Clock::now();
|
|
CHECK(::recv(excess, &byte, 1, 0) == 0);
|
|
CHECK(Clock::now() - refused_at < 500ms);
|
|
::close(excess);
|
|
std::this_thread::sleep_for(1300ms); // Idle connections expire at their deadline.
|
|
const auto started = Clock::now();
|
|
CHECK(get(port, "/livez").status == 200);
|
|
CHECK(Clock::now() - started < 1s);
|
|
for (const int fd : idle) {
|
|
CHECK(::recv(fd, &byte, 1, 0) == 0); // Closed by the server.
|
|
::close(fd);
|
|
}
|
|
server.signal(SIGTERM);
|
|
CHECK(server.wait_exit(20s) == 0);
|
|
}
|
|
|
|
// The database is unreachable: the process is alive but never ready.
|
|
void unavailable_database_is_live_not_ready() {
|
|
const ServerRegion r{{7, 52}, fs::path(env("PGHOST")) / "server-52"};
|
|
fs::remove_all(r.root);
|
|
fs::create_directory(r.root);
|
|
auto args = with_ops(r);
|
|
const auto at = std::ranges::find(args, "--pg-port");
|
|
CHECK(at != args.end());
|
|
*(at + 1) = "1"; // No server socket there.
|
|
Child server{args};
|
|
const auto port = ops_port(server);
|
|
std::this_thread::sleep_for(500ms);
|
|
CHECK(get(port, "/livez").status == 200);
|
|
auto ready = get(port, "/readyz");
|
|
CHECK(ready.status == 503 && ready.body == "not ready: acquiring\n");
|
|
CHECK(metric(port, "evo_region_ready") == 0 &&
|
|
metric(port, "evo_region_store_failures_total") > 0);
|
|
CHECK(get(port, "/metrics").body.find("evo_region_phase{phase=\"acquiring\"} 1\n") !=
|
|
std::string::npos);
|
|
CHECK(!server.printed("ready 1"));
|
|
server.signal(SIGTERM);
|
|
CHECK(server.wait_exit(20s) == 0);
|
|
}
|
|
|
|
// A paused process that lost its lease is alive, but readiness and metrics say it
|
|
// no longer holds authority; the successor is the only ready process.
|
|
void stale_process_is_live_not_ready() {
|
|
const auto r = provision(53, 0x0C);
|
|
Child stale{with_ops(r)};
|
|
const auto stale_port = ops_port(stale);
|
|
CHECK(stale.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
|
stale.signal(SIGSTOP);
|
|
std::this_thread::sleep_for(4500ms);
|
|
Child current{with_ops(r)};
|
|
const auto current_port = ops_port(current);
|
|
CHECK(current.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
|
stale.signal(SIGCONT);
|
|
CHECK(stale.wait_line("ready 0", 10s));
|
|
CHECK(get(stale_port, "/livez").status == 200);
|
|
CHECK(get(stale_port, "/readyz").status == 503);
|
|
CHECK(metric(stale_port, "evo_region_ready") == 0 &&
|
|
metric(stale_port, "evo_region_fences_total") == 1 &&
|
|
metric(stale_port, "evo_region_lease_remaining_milliseconds") == 0);
|
|
CHECK(get(current_port, "/readyz").status == 200 &&
|
|
metric(current_port, "evo_region_lease_epoch") == 2);
|
|
stale.signal(SIGTERM);
|
|
current.signal(SIGTERM);
|
|
CHECK(stale.wait_exit(20s) == 0 && current.wait_exit(20s) == 0);
|
|
}
|
|
} // namespace
|
|
|
|
void run_region_ops_tests() {
|
|
probes_follow_authority_and_stay_bounded();
|
|
unavailable_database_is_live_not_ready();
|
|
stale_process_is_live_not_ready();
|
|
}
|