Files
aevolution/tests/region_ops_tests.cpp
T
dingoandClaude Opus 5.5 7273b157d6
Foundation / checks (client) (push) Successful in 4s
Foundation / checks (checks) (push) Successful in 4s
Foundation / checks (ecs) (push) Successful in 4s
Foundation / checks (network) (push) Successful in 5s
Foundation / checks (postgres) (push) Successful in 5s
Foundation / checks (tsan) (push) Successful in 4s
Foundation / checks (deploy) (push) Successful in 4m47s
Foundation / postgres-tools (push) Successful in 29s
Foundation / server-image (push) Successful in 4m42s
Foundation / image (push) Successful in 5m55s
feat(admin): live tuning revisions adopted at recorded ticks (schema 66)
ADR-0182 section 13. The admin portal's Tuning page edits the world's
rules-13 tuning (the Git revamp file format); the admin API checks it,
stores an append-only revision and queues an adopt_tuning admin command
(kind 7: revision and rules digest) for every region of the world.

- Lifecycle: adopt_tuning replaces the region's rules after births when
  a resolver names the rules for that revision and digest; the safety
  revision cannot change live. At most one adoption per batch.
- Host: fetches the revision text, rebuilds and digests the rules before
  appending the command; publication and recovery name the rules from
  the successor state, checked against the recorded digest.
- Schema 66: tuning_revisions, admin_set_tuning_v1, load_tuning_revision_v1
  (runtime), admin_tuning_revisions_v1, admin_tuning_text_v1.
- OpsServer: optional bounded request bodies (Content-Length, 413).
- Region view: rules_sha256, so the portal shows each region's revision.
- Portal: Tuning page; baseline from the mounted cellular-revamp.tuning.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-27 19:35:09 -04:00

332 lines
14 KiB
C++

// M7.8 operational endpoints on real evo-server processes (ADR-0066): liveness is
// never authority, readiness follows the lease, metrics are bounded, and the
// responder is bounded against idle, excess and malformed clients.
#include "evo/server/ops_http.hpp"
#include "fixtures/server_process.hpp"
#include <algorithm>
#include <arpa/inet.h>
#include <atomic>
#include <netinet/in.h>
#include <source_location>
#include <sys/socket.h>
#include <thread>
void run_region_ops_tests();
namespace {
using namespace evo::test::process;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region ops test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
int connect_local(std::uint16_t port) {
const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0);
CHECK(fd >= 0);
sockaddr_in address{};
address.sin_family = AF_INET;
address.sin_port = htons(port);
address.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
CHECK(::connect(fd, reinterpret_cast<sockaddr*>(&address), sizeof address) == 0);
timeval timeout{3, 0};
::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout);
return fd;
}
struct Reply {
int status{};
std::string body;
};
// Sends raw bytes and reads until the server closes (Connection: close).
Reply http_exchange(std::uint16_t port, const std::string& request) {
const int fd = connect_local(port);
std::size_t sent = 0;
while (sent < request.size()) {
const auto n = ::send(fd, request.data() + sent, request.size() - sent, MSG_NOSIGNAL);
if (n <= 0)
break;
sent += static_cast<std::size_t>(n);
}
std::string raw;
char chunk[4096];
for (;;) {
const auto n = ::recv(fd, chunk, sizeof chunk, 0);
if (n <= 0)
break;
raw.append(chunk, static_cast<std::size_t>(n));
}
::close(fd);
int status = 0;
CHECK(std::sscanf(raw.c_str(), "HTTP/1.1 %d", &status) == 1);
const auto body = raw.find("\r\n\r\n");
return Reply{status, body == std::string::npos ? "" : raw.substr(body + 4)};
}
Reply get(std::uint16_t port, const std::string& path) {
return http_exchange(port, "GET " + path + " HTTP/1.1\r\nHost: probe\r\n\r\n");
}
std::uint64_t metric(std::uint16_t port, const std::string& series) {
const auto reply = get(port, "/metrics");
CHECK(reply.status == 200);
const auto at = reply.body.find("\n" + series + " ");
CHECK(at != std::string::npos);
return std::stoull(reply.body.substr(at + series.size() + 2));
}
std::uint16_t ops_port(Child& server) {
CHECK(server.wait_line("ops listening ", 10s));
for (const auto& l : server.lines())
if (starts(l, "ops listening "))
return static_cast<std::uint16_t>(std::stoul(l.substr(14)));
CHECK(false);
return 0;
}
std::vector<std::string> with_ops(const ServerRegion& r, std::vector<std::string> extra = {}) {
extra.insert(extra.begin(), {"--ops-listen", "127.0.0.1:0"});
return server_args(r, std::move(extra));
}
void probes_follow_authority_and_stay_bounded() {
const auto r = provision(51, 0xA7);
Child server{with_ops(r)};
const auto port = ops_port(server);
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
auto live = get(port, "/livez");
auto ready = get(port, "/readyz");
CHECK(live.status == 200 && live.body == "live\n");
CHECK(ready.status == 200 && ready.body == "ready\n");
CHECK(metric(port, "evo_region_ready") == 1 && metric(port, "evo_region_lease_epoch") == 1);
CHECK(metric(port, "evo_region_lease_remaining_milliseconds") > 0);
CHECK(metric(port, "evo_region_queue_capacity_inputs") == 64 &&
metric(port, "evo_region_queue_capacity_bytes") == 65536);
// M8.5: the default reserve is 1024 MiB, and the volume reports real free space.
CHECK(metric(port, "evo_region_objects_reserve_bytes") == std::uint64_t{1024} * 1024 * 1024 &&
metric(port, "evo_region_objects_available_bytes") > 0);
auto page = get(port, "/metrics");
CHECK(page.body.find("evo_region_phase{phase=\"running\"} 1\n") != std::string::npos);
server.send(input_lines(1, 30, false));
CHECK(server.wait_line("ack " + request_hex(30) + " 30 ", 15s));
CHECK(metric(port, "evo_region_committed_tick") == 30 &&
metric(port, "evo_region_commit_lag_ticks") == 0 &&
metric(port, "evo_region_acknowledgements_total") == 30);
CHECK(metric(port, "evo_region_checkpoint_age_milliseconds") < 60000);
// Bounded, strict responder.
CHECK(http_exchange(port, "POST /readyz HTTP/1.1\r\n\r\n").status == 405);
CHECK(get(port, "/nope").status == 404);
CHECK(get(port, "/readyz?x=1").status == 404); // Exact paths only.
CHECK(http_exchange(port, "HELLO\r\n\r\n").status == 400);
CHECK(http_exchange(port, "GET /livez HTTP/1.1\r\nX: " + std::string(5000, 'a') + "\r\n\r\n")
.status == 431);
// Idle connections neither block probes nor exceed the connection bound.
std::vector<int> idle;
idle.reserve(8);
for (int i = 0; i < 8; ++i)
idle.push_back(connect_local(port));
const int excess = connect_local(port);
char byte;
// Refused at once (well before the 1 s idle deadline), never queued behind others.
const auto refused_at = Clock::now();
CHECK(::recv(excess, &byte, 1, 0) == 0);
CHECK(Clock::now() - refused_at < 500ms);
::close(excess);
std::this_thread::sleep_for(1300ms); // Idle connections expire at their deadline.
const auto started = Clock::now();
CHECK(get(port, "/livez").status == 200);
CHECK(Clock::now() - started < 1s);
for (const int fd : idle) {
CHECK(::recv(fd, &byte, 1, 0) == 0); // Closed by the server.
::close(fd);
}
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 0);
}
// The database is unreachable: the process is alive but never ready.
void unavailable_database_is_live_not_ready() {
const ServerRegion r{{7, 52}, fs::path(env("PGHOST")) / "server-52"};
fs::remove_all(r.root);
fs::create_directory(r.root);
auto args = with_ops(r);
const auto at = std::ranges::find(args, "--pg-port");
CHECK(at != args.end());
*(at + 1) = "1"; // No server socket there.
Child server{args};
const auto port = ops_port(server);
std::this_thread::sleep_for(500ms);
CHECK(get(port, "/livez").status == 200);
auto ready = get(port, "/readyz");
CHECK(ready.status == 503 && ready.body == "not ready: acquiring\n");
CHECK(metric(port, "evo_region_ready") == 0 &&
metric(port, "evo_region_store_failures_total") > 0);
CHECK(get(port, "/metrics").body.find("evo_region_phase{phase=\"acquiring\"} 1\n") !=
std::string::npos);
CHECK(!server.printed("ready 1"));
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 0);
}
// A paused process that lost its lease is alive, but readiness and metrics say it
// no longer holds authority; the successor is the only ready process.
void stale_process_is_live_not_ready() {
const auto r = provision(53, 0x0C);
Child stale{with_ops(r)};
const auto stale_port = ops_port(stale);
CHECK(stale.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
stale.signal(SIGSTOP);
std::this_thread::sleep_for(4500ms);
Child current{with_ops(r)};
const auto current_port = ops_port(current);
CHECK(current.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
stale.signal(SIGCONT);
CHECK(stale.wait_line("ready 0", 10s));
CHECK(get(stale_port, "/livez").status == 200);
CHECK(get(stale_port, "/readyz").status == 503);
CHECK(metric(stale_port, "evo_region_ready") == 0 &&
metric(stale_port, "evo_region_fences_total") == 1 &&
metric(stale_port, "evo_region_lease_remaining_milliseconds") == 0);
CHECK(get(current_port, "/readyz").status == 200 &&
metric(current_port, "evo_region_lease_epoch") == 2);
stale.signal(SIGTERM);
current.signal(SIGTERM);
CHECK(stale.wait_exit(20s) == 0 && current.wait_exit(20s) == 0);
}
// Review fix: accept work per serve() is bounded, refused connections included, so
// replenished connection pressure cannot hold the host loop; accepted connections
// are still answered while the pressure continues.
void accept_work_is_bounded_per_serve() {
using evo::server::OpsServer;
auto made = OpsServer::listen("127.0.0.1:0");
CHECK(made);
auto& ops = **made;
const auto handler = [](std::string_view) { return evo::server::OpsResponse{200, "ok\n"}; };
constexpr auto per = OpsServer::max_accepts_per_serve;
std::vector<int> clients;
clients.reserve(64);
for (int i = 0; i < 16; ++i)
clients.push_back(connect_local(ops.port())); // Queued in the listen backlog.
std::this_thread::sleep_for(50ms);
ops.serve(handler);
CHECK(ops.open_connections() == per && ops.refused() == 0);
ops.serve(handler);
CHECK(ops.open_connections() == OpsServer::max_connections && ops.refused() == 0);
// At the connection cap, each call still refuses at most the budget.
ops.serve(handler);
CHECK(ops.open_connections() == OpsServer::max_connections && ops.refused() == per);
// The first accepted client is answered while new connections keep arriving.
const std::string request = "GET /livez HTTP/1.1\r\n\r\n";
CHECK(::send(clients[0], request.data(), request.size(), MSG_NOSIGNAL) ==
static_cast<ssize_t>(request.size()));
for (int round = 0; round < 6; ++round) {
clients.push_back(connect_local(ops.port()));
const auto before = ops.refused();
ops.serve(handler);
CHECK(ops.refused() - before <= per);
}
char reply[64] = {};
CHECK(::recv(clients[0], reply, sizeof reply - 1, 0) > 0);
CHECK(std::string_view(reply).starts_with("HTTP/1.1 200"));
for (const int fd : clients)
::close(fd);
}
// ADR-0182: API request bodies only when asked for, complete (Content-Length, possibly over
// several reads), bounded (413) and nothing else: no chunking, no body on GET, no extra bytes.
void api_bodies_are_bounded_and_complete() {
using evo::server::OpsServer;
for (const std::size_t limit : {std::size_t{0}, std::size_t{64}}) {
auto made = OpsServer::listen("127.0.0.1:0");
CHECK(made);
auto& ops = **made;
std::atomic<bool> stop{false};
std::thread serving([&] {
while (!stop.load()) {
ops.serve_api(
[](const evo::server::ApiRequest& r) {
return evo::server::OpsResponse{200, std::string(r.method) + ":" +
std::string(r.body)};
},
limit);
std::this_thread::sleep_for(1ms);
}
});
const auto port = ops.port();
const auto post = [&](const std::string& headers, const std::string& body) {
std::string request = "POST /x HTTP/1.1\r\n";
request.append(headers).append("\r\n").append(body);
return http_exchange(port, request);
};
CHECK(post("", "").body == "POST:");
CHECK(post("Content-Length: 0\r\n", "").body == "POST:");
const auto hello = post("Content-Length: 5\r\n", "hello");
CHECK(limit == 0 ? hello.status == 400 : hello.status == 200 && hello.body == "POST:hello");
if (limit > 0) {
// A body arriving after the headers, in a later read.
const int fd = connect_local(port);
const std::string first = "POST /x HTTP/1.1\r\nContent-Length: 5\r\n\r\nhe";
CHECK(::send(fd, first.data(), first.size(), MSG_NOSIGNAL) ==
static_cast<ssize_t>(first.size()));
std::this_thread::sleep_for(50ms);
CHECK(::send(fd, "llo", 3, MSG_NOSIGNAL) == 3);
std::string raw;
char chunk[512];
for (ssize_t n = 0; (n = ::recv(fd, chunk, sizeof chunk, 0)) > 0;)
raw.append(chunk, static_cast<std::size_t>(n));
::close(fd);
CHECK(raw.starts_with("HTTP/1.1 200") && raw.ends_with("\r\n\r\nPOST:hello"));
}
CHECK(post("Content-Length: 65\r\n", std::string(65, 'a')).status ==
(limit == 0 ? 400 : 413));
CHECK(post("Transfer-Encoding: chunked\r\n", "5\r\nhello\r\n0\r\n\r\n").status == 400);
CHECK(post("Content-Length: 2\r\n", "hixx").status == 400);
CHECK(post("Content-Length: 2\r\nContent-Length: 2\r\n", "hi").status == 400);
CHECK(post("Content-Length: two\r\n", "hi").status == 400);
CHECK(http_exchange(port, "GET /x HTTP/1.1\r\nContent-Length: 2\r\n\r\nhi").status == 400);
stop = true;
serving.join();
}
}
// A real server under sustained connection churn keeps its lease and readiness.
void connection_pressure_keeps_authority() {
const auto r = provision(54, 0x0D);
Child server{with_ops(r)};
const auto port = ops_port(server);
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
std::atomic<bool> stop{false};
std::thread flood([&] {
while (!stop.load()) {
const int fd = ::socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0);
sockaddr_in address{};
address.sin_family = AF_INET;
address.sin_port = htons(port);
address.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
if (fd >= 0) {
// Refusal or backlog overflow is part of the pressure; ignore it.
[[maybe_unused]] const int connected =
::connect(fd, reinterpret_cast<sockaddr*>(&address), sizeof address);
::close(fd);
}
}
});
// Longer than the fast lease (4 s): renewals must keep happening under pressure.
server.send(input_lines(1, 60, false));
const bool acked = server.wait_line("ack " + request_hex(60) + " 60 ", 20s);
stop = true;
flood.join();
CHECK(acked);
CHECK(!server.printed("ready 0"));
CHECK(get(port, "/readyz").status == 200);
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 0);
}
} // namespace
void run_region_ops_tests() {
accept_work_is_bounded_per_serve();
api_bodies_are_bounded_and_complete();
connection_pressure_keeps_authority();
probes_follow_authority_and_stay_bounded();
unavailable_database_is_live_not_ready();
stale_process_is_live_not_ready();
}