// M7.8 operational endpoints on real evo-server processes (ADR-0066): liveness is // never authority, readiness follows the lease, metrics are bounded, and the // responder is bounded against idle, excess and malformed clients. #include "evo/server/ops_http.hpp" #include "fixtures/server_process.hpp" #include #include #include #include #include #include #include void run_region_ops_tests(); namespace { using namespace evo::test::process; void check(bool value, std::source_location at = std::source_location::current()) { if (!value) throw std::runtime_error("region ops test line " + std::to_string(at.line())); } #define CHECK(...) check(static_cast(__VA_ARGS__)) int connect_local(std::uint16_t port) { const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); CHECK(fd >= 0); sockaddr_in address{}; address.sin_family = AF_INET; address.sin_port = htons(port); address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); CHECK(::connect(fd, reinterpret_cast(&address), sizeof address) == 0); timeval timeout{3, 0}; ::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout); return fd; } struct Reply { int status{}; std::string body; }; // Sends raw bytes and reads until the server closes (Connection: close). Reply http_exchange(std::uint16_t port, const std::string& request) { const int fd = connect_local(port); std::size_t sent = 0; while (sent < request.size()) { const auto n = ::send(fd, request.data() + sent, request.size() - sent, MSG_NOSIGNAL); if (n <= 0) break; sent += static_cast(n); } std::string raw; char chunk[4096]; for (;;) { const auto n = ::recv(fd, chunk, sizeof chunk, 0); if (n <= 0) break; raw.append(chunk, static_cast(n)); } ::close(fd); int status = 0; CHECK(std::sscanf(raw.c_str(), "HTTP/1.1 %d", &status) == 1); const auto body = raw.find("\r\n\r\n"); return Reply{status, body == std::string::npos ? "" : raw.substr(body + 4)}; } Reply get(std::uint16_t port, const std::string& path) { return http_exchange(port, "GET " + path + " HTTP/1.1\r\nHost: probe\r\n\r\n"); } std::uint64_t metric(std::uint16_t port, const std::string& series) { const auto reply = get(port, "/metrics"); CHECK(reply.status == 200); const auto at = reply.body.find("\n" + series + " "); CHECK(at != std::string::npos); return std::stoull(reply.body.substr(at + series.size() + 2)); } std::uint16_t ops_port(Child& server) { CHECK(server.wait_line("ops listening ", 10s)); for (const auto& l : server.lines()) if (starts(l, "ops listening ")) return static_cast(std::stoul(l.substr(14))); CHECK(false); return 0; } std::vector with_ops(const ServerRegion& r, std::vector extra = {}) { extra.insert(extra.begin(), {"--ops-listen", "127.0.0.1:0"}); return server_args(r, std::move(extra)); } void probes_follow_authority_and_stay_bounded() { const auto r = provision(51, 0xA7); Child server{with_ops(r)}; const auto port = ops_port(server); CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); auto live = get(port, "/livez"); auto ready = get(port, "/readyz"); CHECK(live.status == 200 && live.body == "live\n"); CHECK(ready.status == 200 && ready.body == "ready\n"); CHECK(metric(port, "evo_region_ready") == 1 && metric(port, "evo_region_lease_epoch") == 1); CHECK(metric(port, "evo_region_lease_remaining_milliseconds") > 0); CHECK(metric(port, "evo_region_queue_capacity_inputs") == 64 && metric(port, "evo_region_queue_capacity_bytes") == 65536); // M8.5: the default reserve is 1024 MiB, and the volume reports real free space. CHECK(metric(port, "evo_region_objects_reserve_bytes") == std::uint64_t{1024} * 1024 * 1024 && metric(port, "evo_region_objects_available_bytes") > 0); auto page = get(port, "/metrics"); CHECK(page.body.find("evo_region_phase{phase=\"running\"} 1\n") != std::string::npos); server.send(input_lines(1, 30, false)); CHECK(server.wait_line("ack " + request_hex(30) + " 30 ", 15s)); CHECK(metric(port, "evo_region_committed_tick") == 30 && metric(port, "evo_region_commit_lag_ticks") == 0 && metric(port, "evo_region_acknowledgements_total") == 30); CHECK(metric(port, "evo_region_checkpoint_age_milliseconds") < 60000); // Bounded, strict responder. CHECK(http_exchange(port, "POST /readyz HTTP/1.1\r\n\r\n").status == 405); CHECK(get(port, "/nope").status == 404); CHECK(get(port, "/readyz?x=1").status == 404); // Exact paths only. CHECK(http_exchange(port, "HELLO\r\n\r\n").status == 400); CHECK(http_exchange(port, "GET /livez HTTP/1.1\r\nX: " + std::string(5000, 'a') + "\r\n\r\n") .status == 431); // Idle connections neither block probes nor exceed the connection bound. std::vector idle; idle.reserve(8); for (int i = 0; i < 8; ++i) idle.push_back(connect_local(port)); const int excess = connect_local(port); char byte; // Refused at once (well before the 1 s idle deadline), never queued behind others. const auto refused_at = Clock::now(); CHECK(::recv(excess, &byte, 1, 0) == 0); CHECK(Clock::now() - refused_at < 500ms); ::close(excess); std::this_thread::sleep_for(1300ms); // Idle connections expire at their deadline. const auto started = Clock::now(); CHECK(get(port, "/livez").status == 200); CHECK(Clock::now() - started < 1s); for (const int fd : idle) { CHECK(::recv(fd, &byte, 1, 0) == 0); // Closed by the server. ::close(fd); } server.signal(SIGTERM); CHECK(server.wait_exit(20s) == 0); } // The database is unreachable: the process is alive but never ready. void unavailable_database_is_live_not_ready() { const ServerRegion r{{7, 52}, fs::path(env("PGHOST")) / "server-52"}; fs::remove_all(r.root); fs::create_directory(r.root); auto args = with_ops(r); const auto at = std::ranges::find(args, "--pg-port"); CHECK(at != args.end()); *(at + 1) = "1"; // No server socket there. Child server{args}; const auto port = ops_port(server); std::this_thread::sleep_for(500ms); CHECK(get(port, "/livez").status == 200); auto ready = get(port, "/readyz"); CHECK(ready.status == 503 && ready.body == "not ready: acquiring\n"); CHECK(metric(port, "evo_region_ready") == 0 && metric(port, "evo_region_store_failures_total") > 0); CHECK(get(port, "/metrics").body.find("evo_region_phase{phase=\"acquiring\"} 1\n") != std::string::npos); CHECK(!server.printed("ready 1")); server.signal(SIGTERM); CHECK(server.wait_exit(20s) == 0); } // A paused process that lost its lease is alive, but readiness and metrics say it // no longer holds authority; the successor is the only ready process. void stale_process_is_live_not_ready() { const auto r = provision(53, 0x0C); Child stale{with_ops(r)}; const auto stale_port = ops_port(stale); CHECK(stale.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); stale.signal(SIGSTOP); std::this_thread::sleep_for(4500ms); Child current{with_ops(r)}; const auto current_port = ops_port(current); CHECK(current.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); stale.signal(SIGCONT); CHECK(stale.wait_line("ready 0", 10s)); CHECK(get(stale_port, "/livez").status == 200); CHECK(get(stale_port, "/readyz").status == 503); CHECK(metric(stale_port, "evo_region_ready") == 0 && metric(stale_port, "evo_region_fences_total") == 1 && metric(stale_port, "evo_region_lease_remaining_milliseconds") == 0); CHECK(get(current_port, "/readyz").status == 200 && metric(current_port, "evo_region_lease_epoch") == 2); stale.signal(SIGTERM); current.signal(SIGTERM); CHECK(stale.wait_exit(20s) == 0 && current.wait_exit(20s) == 0); } // Review fix: accept work per serve() is bounded, refused connections included, so // replenished connection pressure cannot hold the host loop; accepted connections // are still answered while the pressure continues. void accept_work_is_bounded_per_serve() { using evo::server::OpsServer; auto made = OpsServer::listen("127.0.0.1:0"); CHECK(made); auto& ops = **made; const auto handler = [](std::string_view) { return evo::server::OpsResponse{200, "ok\n"}; }; constexpr auto per = OpsServer::max_accepts_per_serve; std::vector clients; clients.reserve(64); for (int i = 0; i < 16; ++i) clients.push_back(connect_local(ops.port())); // Queued in the listen backlog. std::this_thread::sleep_for(50ms); ops.serve(handler); CHECK(ops.open_connections() == per && ops.refused() == 0); ops.serve(handler); CHECK(ops.open_connections() == OpsServer::max_connections && ops.refused() == 0); // At the connection cap, each call still refuses at most the budget. ops.serve(handler); CHECK(ops.open_connections() == OpsServer::max_connections && ops.refused() == per); // The first accepted client is answered while new connections keep arriving. const std::string request = "GET /livez HTTP/1.1\r\n\r\n"; CHECK(::send(clients[0], request.data(), request.size(), MSG_NOSIGNAL) == static_cast(request.size())); for (int round = 0; round < 6; ++round) { clients.push_back(connect_local(ops.port())); const auto before = ops.refused(); ops.serve(handler); CHECK(ops.refused() - before <= per); } char reply[64] = {}; CHECK(::recv(clients[0], reply, sizeof reply - 1, 0) > 0); CHECK(std::string_view(reply).starts_with("HTTP/1.1 200")); for (const int fd : clients) ::close(fd); } // ADR-0182: API request bodies only when asked for, complete (Content-Length, possibly over // several reads), bounded (413) and nothing else: no chunking, no body on GET, no extra bytes. void api_bodies_are_bounded_and_complete() { using evo::server::OpsServer; for (const std::size_t limit : {std::size_t{0}, std::size_t{64}}) { auto made = OpsServer::listen("127.0.0.1:0"); CHECK(made); auto& ops = **made; std::atomic stop{false}; std::thread serving([&] { while (!stop.load()) { ops.serve_api( [](const evo::server::ApiRequest& r) { return evo::server::OpsResponse{200, std::string(r.method) + ":" + std::string(r.body)}; }, limit); std::this_thread::sleep_for(1ms); } }); const auto port = ops.port(); const auto post = [&](const std::string& headers, const std::string& body) { std::string request = "POST /x HTTP/1.1\r\n"; request.append(headers).append("\r\n").append(body); return http_exchange(port, request); }; CHECK(post("", "").body == "POST:"); CHECK(post("Content-Length: 0\r\n", "").body == "POST:"); const auto hello = post("Content-Length: 5\r\n", "hello"); CHECK(limit == 0 ? hello.status == 400 : hello.status == 200 && hello.body == "POST:hello"); if (limit > 0) { // A body arriving after the headers, in a later read. const int fd = connect_local(port); const std::string first = "POST /x HTTP/1.1\r\nContent-Length: 5\r\n\r\nhe"; CHECK(::send(fd, first.data(), first.size(), MSG_NOSIGNAL) == static_cast(first.size())); std::this_thread::sleep_for(50ms); CHECK(::send(fd, "llo", 3, MSG_NOSIGNAL) == 3); std::string raw; char chunk[512]; for (ssize_t n = 0; (n = ::recv(fd, chunk, sizeof chunk, 0)) > 0;) raw.append(chunk, static_cast(n)); ::close(fd); CHECK(raw.starts_with("HTTP/1.1 200") && raw.ends_with("\r\n\r\nPOST:hello")); } CHECK(post("Content-Length: 65\r\n", std::string(65, 'a')).status == (limit == 0 ? 400 : 413)); CHECK(post("Transfer-Encoding: chunked\r\n", "5\r\nhello\r\n0\r\n\r\n").status == 400); CHECK(post("Content-Length: 2\r\n", "hixx").status == 400); CHECK(post("Content-Length: 2\r\nContent-Length: 2\r\n", "hi").status == 400); CHECK(post("Content-Length: two\r\n", "hi").status == 400); CHECK(http_exchange(port, "GET /x HTTP/1.1\r\nContent-Length: 2\r\n\r\nhi").status == 400); stop = true; serving.join(); } } // A real server under sustained connection churn keeps its lease and readiness. void connection_pressure_keeps_authority() { const auto r = provision(54, 0x0D); Child server{with_ops(r)}; const auto port = ops_port(server); CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); std::atomic stop{false}; std::thread flood([&] { while (!stop.load()) { const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); sockaddr_in address{}; address.sin_family = AF_INET; address.sin_port = htons(port); address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); if (fd >= 0) { // Refusal or backlog overflow is part of the pressure; ignore it. [[maybe_unused]] const int connected = ::connect(fd, reinterpret_cast(&address), sizeof address); ::close(fd); } } }); // Longer than the fast lease (4 s): renewals must keep happening under pressure. server.send(input_lines(1, 60, false)); const bool acked = server.wait_line("ack " + request_hex(60) + " 60 ", 20s); stop = true; flood.join(); CHECK(acked); CHECK(!server.printed("ready 0")); CHECK(get(port, "/readyz").status == 200); server.signal(SIGTERM); CHECK(server.wait_exit(20s) == 0); } } // namespace void run_region_ops_tests() { accept_work_is_bounded_per_serve(); api_bodies_are_bounded_and_complete(); connection_pressure_keeps_authority(); probes_follow_authority_and_stay_bounded(); unavailable_database_is_live_not_ready(); stale_process_is_live_not_ready(); }