Complete M7.8 operations endpoints and region server image

ADR-0066: evo-server --ops-listen serves /livez (loop responding, never
authority), /readyz (200 only while the host holds the lease with a
verified recovered head) and /metrics (Prometheus text: 30 fixed
series for readiness, phase, lease epoch/remaining, simulated/committed
ticks, commit lag, checkpoint age, tick debt, queue use and bounds, and
counters; the only labels are a 6-value phase and 4-value drop reason).

The dependency-free OpsServer runs on the host loop with no thread:
GET on exact paths, 405/404/400/431 responses, 8 connections with excess
refused on accept, 4 KiB requests, 1 s deadlines, 16 KiB responses.
HostStatus/phase types moved to the portable evo::ops library with a
Windows-tested renderer. evo-server now refuses to start without its
required identity and connection settings.

Adds the server-runtime Docker target (evo-server + pinned libpq,
UID 10001, no credentials), a CI server-image job with smoke script and
optional publishing, and a Kubernetes example outside Argo.

Validated: Windows CTest 59/59; Linux Debug/Release/clang-tidy/ASan
CTest 59/59; PostgreSQL suites under Release, clang-tidy and ASan with
real process probes; two ops mutations caught. Docker build/smoke and
Gitea CI not executed locally.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-28 23:09:28 -04:00
co-authored by Claude Opus 5.5
parent 30411a83f9
commit 7aec68db8e
23 changed files with 1222 additions and 65 deletions
+104
View File
@@ -0,0 +1,104 @@
#include "evo/server/ops.hpp"
#include <string_view>
namespace evo::server {
const char* phase_name(HostPhase phase) noexcept {
switch (phase) {
case HostPhase::acquiring:
return "acquiring";
case HostPhase::running:
return "running";
case HostPhase::resolving:
return "resolving";
case HostPhase::draining:
return "draining";
case HostPhase::stopped:
return "stopped";
case HostPhase::quarantined:
return "quarantined";
}
return "unknown";
}
const char* unacknowledged_name(Unacknowledged reason) noexcept {
switch (reason) {
case Unacknowledged::refused:
return "refused";
case Unacknowledged::drained:
return "drained";
case Unacknowledged::fenced:
return "fenced";
case Unacknowledged::quarantined:
return "quarantined";
}
return "unknown";
}
namespace {
void metric(std::string& out, std::string_view name, std::string_view type, std::string_view help,
std::uint64_t value) {
out.append("# HELP ").append(name).append(" ").append(help).append("\n");
out.append("# TYPE ").append(name).append(" ").append(type).append("\n");
out.append(name).append(" ").append(std::to_string(value)).append("\n");
}
} // namespace
std::string render_metrics(const HostStatus& s, const OpsCounters& c) {
std::string out;
out.reserve(4096);
metric(out, "evo_region_ready", "gauge",
"1 only while this host holds current authority and may originate output", s.ready);
out.append("# HELP evo_region_phase Current host phase (one-hot)\n");
out.append("# TYPE evo_region_phase gauge\n");
for (std::size_t i = 0; i < host_phase_count; ++i) {
const auto phase = static_cast<HostPhase>(i);
out.append("evo_region_phase{phase=\"")
.append(phase_name(phase))
.append("\"} ")
.append(phase == s.phase ? "1" : "0")
.append("\n");
}
metric(out, "evo_region_lease_epoch", "gauge", "Lease epoch held or last held", s.epoch);
metric(out, "evo_region_lease_remaining_milliseconds", "gauge",
"Local time until this host must stop originating (0 without a lease)",
s.lease_remaining_ms);
metric(out, "evo_region_simulated_tick", "gauge", "Last simulated tick", s.simulated_tick);
metric(out, "evo_region_committed_tick", "gauge", "End tick of the committed head",
s.committed_tick);
metric(out, "evo_region_committed_sequence", "gauge", "Commit sequence of the head",
s.committed_sequence);
metric(out, "evo_region_commit_lag_ticks", "gauge",
"Simulated but not yet durably committed ticks",
s.simulated_tick >= s.committed_tick ? s.simulated_tick - s.committed_tick : 0);
metric(out, "evo_region_checkpoint_age_milliseconds", "gauge",
"Time since the last confirmed commit or recovery", c.checkpoint_age_ms);
metric(out, "evo_region_tick_debt", "gauge", "Due ticks not yet simulated", s.tick_debt);
metric(out, "evo_region_tick_debt_max", "gauge", "Largest tick debt since recovery",
s.max_tick_debt);
metric(out, "evo_region_queued_inputs", "gauge", "Admitted inputs awaiting a tick", s.queued);
metric(out, "evo_region_queued_bytes", "gauge", "Accounted bytes of queued inputs",
s.queued_bytes);
metric(out, "evo_region_queue_capacity_inputs", "gauge", "Input backlog bound",
c.queue_capacity_inputs);
metric(out, "evo_region_queue_capacity_bytes", "gauge", "Input backlog byte bound",
c.queue_capacity_bytes);
metric(out, "evo_region_commits_total", "counter", "Confirmed commits", s.commits);
metric(out, "evo_region_store_failures_total", "counter",
"Storage or database failures that paused progress", s.store_failures);
metric(out, "evo_region_unknown_outcomes_total", "counter",
"Commits whose outcome had to be resolved", s.unknown_outcomes);
metric(out, "evo_region_fences_total", "counter", "Times this host lost authority", s.fences);
metric(out, "evo_region_acknowledgements_total", "counter", "Final acknowledgements emitted",
c.acks);
metric(out, "evo_region_outcomes_total", "counter", "Final permanent outcomes emitted",
c.outcomes);
out.append("# HELP evo_region_dropped_total Accepted inputs that will never be acknowledged\n");
out.append("# TYPE evo_region_dropped_total counter\n");
for (std::size_t i = 0; i < unacknowledged_count; ++i)
out.append("evo_region_dropped_total{reason=\"")
.append(unacknowledged_name(static_cast<Unacknowledged>(i)))
.append("\"} ")
.append(std::to_string(c.dropped[i]))
.append("\n");
return out;
}
} // namespace evo::server
+175
View File
@@ -0,0 +1,175 @@
#include "evo/server/ops_http.hpp"
#include <arpa/inet.h>
#include <cerrno>
#include <charconv>
#include <netinet/in.h>
#include <sys/socket.h>
#include <unistd.h>
namespace evo::server {
namespace {
using Clock = std::chrono::steady_clock;
const char* reason(int status) {
switch (status) {
case 200:
return "OK";
case 400:
return "Bad Request";
case 404:
return "Not Found";
case 405:
return "Method Not Allowed";
case 431:
return "Request Header Fields Too Large";
case 503:
return "Service Unavailable";
default:
return "Internal Server Error";
}
}
std::string encode(const OpsResponse& r) {
std::string out = "HTTP/1.1 " + std::to_string(r.status) + " " + reason(r.status) +
"\r\nContent-Type: " + r.content_type +
"\r\nContent-Length: " + std::to_string(r.body.size()) +
"\r\nCache-Control: no-store\r\nConnection: close\r\n";
if (r.status == 405)
out += "Allow: GET\r\n";
return out + "\r\n" + r.body;
}
bool printable(std::string_view s) {
for (const char ch : s)
if (ch <= ' ' || ch > '~')
return false;
return true;
}
} // namespace
std::expected<std::unique_ptr<OpsServer>, OpsError> OpsServer::listen(std::string_view address) {
const auto colon = address.rfind(':');
if (colon == std::string_view::npos || colon + 1 >= address.size())
return std::unexpected{OpsError::address};
unsigned port{};
const auto digits = address.substr(colon + 1);
const auto [end, error] = std::from_chars(digits.data(), digits.data() + digits.size(), port);
if (error != std::errc{} || end != digits.data() + digits.size() || port > 65535)
return std::unexpected{OpsError::address};
auto host = std::string(address.substr(0, colon));
sockaddr_storage storage{};
socklen_t length{};
int family{};
if (host.size() >= 2 && host.front() == '[' && host.back() == ']') {
host = host.substr(1, host.size() - 2);
auto* v6 = reinterpret_cast<sockaddr_in6*>(&storage);
if (inet_pton(AF_INET6, host.c_str(), &v6->sin6_addr) != 1)
return std::unexpected{OpsError::address};
v6->sin6_family = AF_INET6;
v6->sin6_port = htons(static_cast<std::uint16_t>(port));
family = AF_INET6;
length = sizeof(sockaddr_in6);
} else {
auto* v4 = reinterpret_cast<sockaddr_in*>(&storage);
if (inet_pton(AF_INET, host.c_str(), &v4->sin_addr) != 1)
return std::unexpected{OpsError::address};
v4->sin_family = AF_INET;
v4->sin_port = htons(static_cast<std::uint16_t>(port));
family = AF_INET;
length = sizeof(sockaddr_in);
}
const int fd = ::socket(family, SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC, 0);
if (fd < 0)
return std::unexpected{OpsError::socket};
const int on = 1;
::setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof on);
if (::bind(fd, reinterpret_cast<sockaddr*>(&storage), length) != 0 || ::listen(fd, 16) != 0) {
::close(fd);
return std::unexpected{OpsError::socket};
}
sockaddr_storage bound{};
socklen_t bound_length = sizeof bound;
::getsockname(fd, reinterpret_cast<sockaddr*>(&bound), &bound_length);
const auto actual = family == AF_INET6
? ntohs(reinterpret_cast<sockaddr_in6*>(&bound)->sin6_port)
: ntohs(reinterpret_cast<sockaddr_in*>(&bound)->sin_port);
return std::unique_ptr<OpsServer>(new OpsServer(fd, actual));
}
OpsServer::~OpsServer() {
for (auto& c : connections_)
::close(c.fd);
::close(fd_);
}
void OpsServer::respond(Connection& c, const OpsHandler& handler) {
OpsResponse response;
const auto line_end = c.in.find("\r\n");
const std::string_view line = std::string_view(c.in).substr(0, line_end);
const auto first = line.find(' ');
const auto second = first == std::string_view::npos ? first : line.find(' ', first + 1);
if (second == std::string_view::npos || line.substr(second + 1).rfind("HTTP/1.", 0) != 0) {
response = {400, "bad request\n"};
} else {
const auto method = line.substr(0, first);
const auto path = line.substr(first + 1, second - first - 1);
if (method != "GET")
response = {405, "method not allowed\n"};
else if (path.empty() || path.front() != '/' || path.size() > 256 || !printable(path))
response = {400, "bad request\n"};
else
response = handler(path);
}
if (response.body.size() > max_response_bytes)
response = {500, "response too large\n"};
c.out = encode(response);
}
void OpsServer::serve(const OpsHandler& handler) {
const auto now = Clock::now();
for (;;) {
const int fd = ::accept4(fd_, nullptr, nullptr, SOCK_NONBLOCK | SOCK_CLOEXEC);
if (fd < 0)
break; // EAGAIN: nothing pending (other errors: retried next call).
if (connections_.size() >= max_connections) {
::close(fd); // Bounded: excess connections are refused, never queued.
++refused_;
continue;
}
connections_.push_back({fd, {}, {}, 0, now + connection_deadline});
}
for (auto it = connections_.begin(); it != connections_.end();) {
auto& c = *it;
bool done = false;
// Read before expiring: a complete request that waited behind a bounded host
// step is still answered; only incomplete or unsent work expires.
if (c.out.empty()) {
char chunk[1024];
const auto n = ::recv(c.fd, chunk, sizeof chunk, 0);
if (n == 0 || (n < 0 && errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR)) {
done = true;
} else if (n > 0) {
c.in.append(chunk, static_cast<std::size_t>(n));
if (c.in.size() > max_request_bytes)
c.out = encode({431, "request too large\n"});
else if (c.in.find("\r\n\r\n") != std::string::npos)
respond(c, handler);
}
}
if (!done && !c.out.empty()) {
const auto n = ::send(c.fd, c.out.data() + c.sent, c.out.size() - c.sent, MSG_NOSIGNAL);
if (n > 0)
c.sent += static_cast<std::size_t>(n);
done = c.sent == c.out.size() ||
(n < 0 && errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR);
}
// Slow client: an incomplete request expires at the deadline; a response the
// client will not take gets one more deadline, then is dropped.
if (!done && now > c.deadline && (c.out.empty() || now > c.deadline + connection_deadline))
done = true;
if (done) {
::close(c.fd);
it = connections_.erase(it);
} else {
++it;
}
}
}
} // namespace evo::server
+7
View File
@@ -386,6 +386,13 @@ template <typename State> void RegionHost<State>::refresh(SchedulerTime now) {
status_.queued_bytes = queue_.size() * tick_input_bytes;
const auto& grant = authority_.grant();
status_.epoch = grant ? grant->epoch : 0;
const auto lease_now = LeaseClock::now();
status_.lease_remaining_ms =
grant && authority_.may_originate(lease_now)
? static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
authority_.deadline() - lease_now)
.count())
: 0;
status_.committed_tick = committed_ ? committed_->head.end_tick : 0;
status_.committed_sequence = committed_ ? committed_->head.sequence : 0;
status_.simulated_tick = state_ ? Ops<State>::tick(*state_) : status_.committed_tick;