// M7.8 operational endpoints on real evo-server processes (ADR-0066): liveness is // never authority, readiness follows the lease, metrics are bounded, and the // responder is bounded against idle, excess and malformed clients. #include "fixtures/server_process.hpp" #include #include #include #include #include #include void run_region_ops_tests(); namespace { using namespace evo::test::process; void check(bool value, std::source_location at = std::source_location::current()) { if (!value) throw std::runtime_error("region ops test line " + std::to_string(at.line())); } #define CHECK(...) check(static_cast(__VA_ARGS__)) int connect_local(std::uint16_t port) { const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); CHECK(fd >= 0); sockaddr_in address{}; address.sin_family = AF_INET; address.sin_port = htons(port); address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); CHECK(::connect(fd, reinterpret_cast(&address), sizeof address) == 0); timeval timeout{3, 0}; ::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout); return fd; } struct Reply { int status{}; std::string body; }; // Sends raw bytes and reads until the server closes (Connection: close). Reply http_exchange(std::uint16_t port, const std::string& request) { const int fd = connect_local(port); std::size_t sent = 0; while (sent < request.size()) { const auto n = ::send(fd, request.data() + sent, request.size() - sent, MSG_NOSIGNAL); if (n <= 0) break; sent += static_cast(n); } std::string raw; char chunk[4096]; for (;;) { const auto n = ::recv(fd, chunk, sizeof chunk, 0); if (n <= 0) break; raw.append(chunk, static_cast(n)); } ::close(fd); int status = 0; CHECK(std::sscanf(raw.c_str(), "HTTP/1.1 %d", &status) == 1); const auto body = raw.find("\r\n\r\n"); return Reply{status, body == std::string::npos ? "" : raw.substr(body + 4)}; } Reply get(std::uint16_t port, const std::string& path) { return http_exchange(port, "GET " + path + " HTTP/1.1\r\nHost: probe\r\n\r\n"); } std::uint64_t metric(std::uint16_t port, const std::string& series) { const auto reply = get(port, "/metrics"); CHECK(reply.status == 200); const auto at = reply.body.find("\n" + series + " "); CHECK(at != std::string::npos); return std::stoull(reply.body.substr(at + series.size() + 2)); } std::uint16_t ops_port(Child& server) { CHECK(server.wait_line("ops listening ", 10s)); for (const auto& l : server.lines()) if (starts(l, "ops listening ")) return static_cast(std::stoul(l.substr(14))); CHECK(false); return 0; } std::vector with_ops(const ServerRegion& r, std::vector extra = {}) { extra.insert(extra.begin(), {"--ops-listen", "127.0.0.1:0"}); return server_args(r, std::move(extra)); } void probes_follow_authority_and_stay_bounded() { const auto r = provision(51, 0x0B); Child server{with_ops(r)}; const auto port = ops_port(server); CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); auto live = get(port, "/livez"); auto ready = get(port, "/readyz"); CHECK(live.status == 200 && live.body == "live\n"); CHECK(ready.status == 200 && ready.body == "ready\n"); CHECK(metric(port, "evo_region_ready") == 1 && metric(port, "evo_region_lease_epoch") == 1); CHECK(metric(port, "evo_region_lease_remaining_milliseconds") > 0); CHECK(metric(port, "evo_region_queue_capacity_inputs") == 64 && metric(port, "evo_region_queue_capacity_bytes") == 65536); auto page = get(port, "/metrics"); CHECK(page.body.find("evo_region_phase{phase=\"running\"} 1\n") != std::string::npos); server.send(input_lines(1, 30, false)); CHECK(server.wait_line("ack " + request_hex(30) + " 30 ", 15s)); CHECK(metric(port, "evo_region_committed_tick") == 30 && metric(port, "evo_region_commit_lag_ticks") == 0 && metric(port, "evo_region_acknowledgements_total") == 30); CHECK(metric(port, "evo_region_checkpoint_age_milliseconds") < 60000); // Bounded, strict responder. CHECK(http_exchange(port, "POST /readyz HTTP/1.1\r\n\r\n").status == 405); CHECK(get(port, "/nope").status == 404); CHECK(get(port, "/readyz?x=1").status == 404); // Exact paths only. CHECK(http_exchange(port, "HELLO\r\n\r\n").status == 400); CHECK(http_exchange(port, "GET /livez HTTP/1.1\r\nX: " + std::string(5000, 'a') + "\r\n\r\n") .status == 431); // Idle connections neither block probes nor exceed the connection bound. std::vector idle; idle.reserve(8); for (int i = 0; i < 8; ++i) idle.push_back(connect_local(port)); const int excess = connect_local(port); char byte; // Refused at once (well before the 1 s idle deadline), never queued behind others. const auto refused_at = Clock::now(); CHECK(::recv(excess, &byte, 1, 0) == 0); CHECK(Clock::now() - refused_at < 500ms); ::close(excess); std::this_thread::sleep_for(1300ms); // Idle connections expire at their deadline. const auto started = Clock::now(); CHECK(get(port, "/livez").status == 200); CHECK(Clock::now() - started < 1s); for (const int fd : idle) { CHECK(::recv(fd, &byte, 1, 0) == 0); // Closed by the server. ::close(fd); } server.signal(SIGTERM); CHECK(server.wait_exit(20s) == 0); } // The database is unreachable: the process is alive but never ready. void unavailable_database_is_live_not_ready() { const ServerRegion r{{7, 52}, fs::path(env("PGHOST")) / "server-52"}; fs::remove_all(r.root); fs::create_directory(r.root); auto args = with_ops(r); const auto at = std::ranges::find(args, "--pg-port"); CHECK(at != args.end()); *(at + 1) = "1"; // No server socket there. Child server{args}; const auto port = ops_port(server); std::this_thread::sleep_for(500ms); CHECK(get(port, "/livez").status == 200); auto ready = get(port, "/readyz"); CHECK(ready.status == 503 && ready.body == "not ready: acquiring\n"); CHECK(metric(port, "evo_region_ready") == 0 && metric(port, "evo_region_store_failures_total") > 0); CHECK(get(port, "/metrics").body.find("evo_region_phase{phase=\"acquiring\"} 1\n") != std::string::npos); CHECK(!server.printed("ready 1")); server.signal(SIGTERM); CHECK(server.wait_exit(20s) == 0); } // A paused process that lost its lease is alive, but readiness and metrics say it // no longer holds authority; the successor is the only ready process. void stale_process_is_live_not_ready() { const auto r = provision(53, 0x0C); Child stale{with_ops(r)}; const auto stale_port = ops_port(stale); CHECK(stale.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); stale.signal(SIGSTOP); std::this_thread::sleep_for(4500ms); Child current{with_ops(r)}; const auto current_port = ops_port(current); CHECK(current.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); stale.signal(SIGCONT); CHECK(stale.wait_line("ready 0", 10s)); CHECK(get(stale_port, "/livez").status == 200); CHECK(get(stale_port, "/readyz").status == 503); CHECK(metric(stale_port, "evo_region_ready") == 0 && metric(stale_port, "evo_region_fences_total") == 1 && metric(stale_port, "evo_region_lease_remaining_milliseconds") == 0); CHECK(get(current_port, "/readyz").status == 200 && metric(current_port, "evo_region_lease_epoch") == 2); stale.signal(SIGTERM); current.signal(SIGTERM); CHECK(stale.wait_exit(20s) == 0 && current.wait_exit(20s) == 0); } } // namespace void run_region_ops_tests() { probes_follow_authority_and_stay_bounded(); unavailable_database_is_live_not_ready(); stale_process_is_live_not_ready(); }