Complete M7.8 operations endpoints and region server image

ADR-0066: evo-server --ops-listen serves /livez (loop responding, never
authority), /readyz (200 only while the host holds the lease with a
verified recovered head) and /metrics (Prometheus text: 30 fixed
series for readiness, phase, lease epoch/remaining, simulated/committed
ticks, commit lag, checkpoint age, tick debt, queue use and bounds, and
counters; the only labels are a 6-value phase and 4-value drop reason).

The dependency-free OpsServer runs on the host loop with no thread:
GET on exact paths, 405/404/400/431 responses, 8 connections with excess
refused on accept, 4 KiB requests, 1 s deadlines, 16 KiB responses.
HostStatus/phase types moved to the portable evo::ops library with a
Windows-tested renderer. evo-server now refuses to start without its
required identity and connection settings.

Adds the server-runtime Docker target (evo-server + pinned libpq,
UID 10001, no credentials), a CI server-image job with smoke script and
optional publishing, and a Kubernetes example outside Argo.

Validated: Windows CTest 59/59; Linux Debug/Release/clang-tidy/ASan
CTest 59/59; PostgreSQL suites under Release, clang-tidy and ASan with
real process probes; two ops mutations caught. Docker build/smoke and
Gitea CI not executed locally.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-28 23:09:28 -04:00
co-authored by Claude Opus 5.5
parent 30411a83f9
commit 7aec68db8e
23 changed files with 1222 additions and 65 deletions
+74 -37
View File
@@ -11,9 +11,14 @@
// stats max_queued=<n> max_tick_debt=<n> commits=<n> store_failures=<n>
// Exit: 0 drained, 2 usage, 3 drain deadline passed, 4 quarantined.
//
// --ops-listen <numeric address:port> (M7.8) serves GET /livez (the loop is alive; says
// nothing about authority), /readyz (200 only with current authority) and /metrics
// (Prometheus text). It prints "ops listening <port>" once bound.
//
// --crash-at <point>:<n> (tests only, M7.7) SIGKILLs this process at the n-th
// occurrence of before-upload, before-transaction, after-commit or after-ack (right
// after a step's acknowledgements are written and flushed).
#include "evo/server/ops_http.hpp"
#include "evo/server/region_host.hpp"
#include <algorithm>
@@ -48,6 +53,7 @@ struct Options {
std::chrono::milliseconds drain{10000};
std::string crash_point;
std::uint64_t crash_ordinal{};
std::string ops_listen;
};
// Kills this process without any cleanup: the real SIGKILL, at a chosen boundary.
@@ -140,6 +146,8 @@ std::optional<Options> parse(int argc, char** argv) {
o.host.lease = {4000ms, 800ms, 400ms, 400ms, 1000ms};
else if (flag == "--lease-profile" && value == "default")
o.host.lease = {};
else if (flag == "--ops-listen")
o.ops_listen = value;
else if (flag == "--crash-at") {
const auto colon = value.find(':');
const auto ordinal =
@@ -154,6 +162,12 @@ std::optional<Options> parse(int argc, char** argv) {
} else
return std::nullopt;
}
// Every identity and connection setting is required: there is no default
// region, database, object root or build to fall back on.
if (o.host.key.world == 0 || o.host.key.region == 0 || o.host.frame == 0 || o.pg.host.empty() ||
o.pg.port.empty() || o.pg.database.empty() || o.pg.user.empty() || o.objects.empty() ||
o.host.build == persistence::Sha256Digest{})
return std::nullopt;
if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end())
o.host.retained_builds.push_back(o.host.build);
return o;
@@ -194,23 +208,6 @@ std::optional<server::TickInput> parse_input(std::string_view line) {
return in;
}
const char* phase_name(server::HostPhase p) {
switch (p) {
case server::HostPhase::acquiring:
return "acquiring";
case server::HostPhase::running:
return "running";
case server::HostPhase::resolving:
return "resolving";
case server::HostPhase::draining:
return "draining";
case server::HostPhase::stopped:
return "stopped";
case server::HostPhase::quarantined:
return "quarantined";
}
return "unknown";
}
const char* admission_name(server::AdmissionError e) {
switch (e) {
case server::AdmissionError::not_ready:
@@ -226,20 +223,6 @@ const char* admission_name(server::AdmissionError e) {
}
return "unknown";
}
const char* dropped_name(server::Unacknowledged u) {
switch (u) {
case server::Unacknowledged::refused:
return "refused";
case server::Unacknowledged::drained:
return "drained";
case server::Unacknowledged::fenced:
return "fenced";
case server::Unacknowledged::quarantined:
return "quarantined";
}
return "unknown";
}
// Reads complete, bounded lines from stdin without blocking.
class LineReader {
public:
@@ -294,6 +277,41 @@ template <typename State> int serve(Options& o) {
return 2;
}
auto& host = **made;
std::unique_ptr<server::OpsServer> ops;
if (!o.ops_listen.empty()) {
auto listening = server::OpsServer::listen(o.ops_listen);
if (!listening) {
std::puts("error ops-listen");
return 2;
}
ops = std::move(*listening);
std::printf("ops listening %u\n", static_cast<unsigned>(ops->port()));
std::fflush(stdout);
}
server::OpsCounters counters;
counters.queue_capacity_inputs = o.host.max_queued_inputs;
counters.queue_capacity_bytes = o.host.max_queued_bytes;
std::uint64_t last_sequence = 0;
auto last_checkpoint = std::chrono::steady_clock::now();
bool recovered = false;
const auto handler = [&](std::string_view path) -> server::OpsResponse {
const auto& s = host.status();
if (path == "/livez")
return {200, "live\n"};
if (path == "/readyz")
return s.ready ? server::OpsResponse{200, "ready\n"}
: server::OpsResponse{503, std::string("not ready: ") +
server::phase_name(s.phase) + "\n"};
if (path == "/metrics") {
counters.checkpoint_age_ms =
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - last_checkpoint)
.count());
return {200, server::render_metrics(s, counters),
"text/plain; version=0.0.4; charset=utf-8"};
}
return {404, "not found\n"};
};
LineReader reader;
bool input_open = true, ready = false;
std::optional<std::string> pending;
@@ -346,28 +364,46 @@ template <typename State> int serve(Options& o) {
}
const auto& s = host.step(std::chrono::steady_clock::now());
max_queued = std::max(max_queued, s.queued);
// Checkpoint age restarts at recovery and at every confirmed commit.
if ((!recovered && s.phase == server::HostPhase::running) ||
s.committed_sequence != last_sequence) {
recovered = recovered || s.phase == server::HostPhase::running;
last_sequence = s.committed_sequence;
last_checkpoint = std::chrono::steady_clock::now();
}
if (s.phase == server::HostPhase::acquiring)
recovered = false;
if (!phase || *phase != s.phase) {
phase = s.phase;
std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n", phase_name(s.phase),
static_cast<unsigned long long>(s.epoch),
std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n",
server::phase_name(s.phase), static_cast<unsigned long long>(s.epoch),
static_cast<unsigned long long>(s.committed_tick),
static_cast<unsigned long long>(s.simulated_tick));
}
set_ready(s.ready);
const auto acknowledged = host.take_acks();
counters.acks += acknowledged.size();
for (const auto& a : acknowledged)
std::printf("ack %s %llu %llu\n", text(a.request).c_str(),
static_cast<unsigned long long>(a.tick),
static_cast<unsigned long long>(a.commit_sequence));
for (const auto& x : host.take_outcomes())
const auto outcomes = host.take_outcomes();
counters.outcomes += outcomes.size();
for (const auto& x : outcomes)
std::printf("outcome %llu %u %llu %llu\n",
static_cast<unsigned long long>(x.decision.tick),
static_cast<unsigned>(x.decision.actor),
static_cast<unsigned long long>(x.decision.organism),
static_cast<unsigned long long>(x.commit_sequence));
for (const auto& d : host.take_dropped())
std::printf("dropped %s %s\n", text(d.input.request).c_str(), dropped_name(d.reason));
const auto dropped = host.take_dropped();
for (const auto& d : dropped)
++counters.dropped[static_cast<std::size_t>(d.reason)];
for (const auto& d : dropped)
std::printf("dropped %s %s\n", text(d.input.request).c_str(),
server::unacknowledged_name(d.reason));
std::fflush(stdout);
if (ops)
ops->serve(handler);
if (o.crash_point == "after-ack" && !acknowledged.empty() && ++ack_steps == o.crash_ordinal)
crash_now();
const auto finish = [&](int code) {
@@ -398,7 +434,8 @@ int main(int argc, char** argv) {
"--frame N --pg-host H --pg-port P --pg-database D --pg-user U "
"--objects DIR --build HEX64 [--retained HEX64]... [--pg-hostaddr A "
"--pg-passfile F --pg-ca F] [--flush-ms N] [--drain-ms N] "
"[--lease-profile default|fast] [--ready-file PATH]\n",
"[--lease-profile default|fast] [--ready-file PATH] "
"[--ops-listen ADDRESS:PORT]\n",
stderr);
return 2;
}