Complete M7.8 operations endpoints and region server image
ADR-0066: evo-server --ops-listen serves /livez (loop responding, never authority), /readyz (200 only while the host holds the lease with a verified recovered head) and /metrics (Prometheus text: 30 fixed series for readiness, phase, lease epoch/remaining, simulated/committed ticks, commit lag, checkpoint age, tick debt, queue use and bounds, and counters; the only labels are a 6-value phase and 4-value drop reason). The dependency-free OpsServer runs on the host loop with no thread: GET on exact paths, 405/404/400/431 responses, 8 connections with excess refused on accept, 4 KiB requests, 1 s deadlines, 16 KiB responses. HostStatus/phase types moved to the portable evo::ops library with a Windows-tested renderer. evo-server now refuses to start without its required identity and connection settings. Adds the server-runtime Docker target (evo-server + pinned libpq, UID 10001, no credentials), a CI server-image job with smoke script and optional publishing, and a Kubernetes example outside Argo. Validated: Windows CTest 59/59; Linux Debug/Release/clang-tidy/ASan CTest 59/59; PostgreSQL suites under Release, clang-tidy and ASan with real process probes; two ops mutations caught. Docker build/smoke and Gitea CI not executed locally. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+74
-37
@@ -11,9 +11,14 @@
|
||||
// stats max_queued=<n> max_tick_debt=<n> commits=<n> store_failures=<n>
|
||||
// Exit: 0 drained, 2 usage, 3 drain deadline passed, 4 quarantined.
|
||||
//
|
||||
// --ops-listen <numeric address:port> (M7.8) serves GET /livez (the loop is alive; says
|
||||
// nothing about authority), /readyz (200 only with current authority) and /metrics
|
||||
// (Prometheus text). It prints "ops listening <port>" once bound.
|
||||
//
|
||||
// --crash-at <point>:<n> (tests only, M7.7) SIGKILLs this process at the n-th
|
||||
// occurrence of before-upload, before-transaction, after-commit or after-ack (right
|
||||
// after a step's acknowledgements are written and flushed).
|
||||
#include "evo/server/ops_http.hpp"
|
||||
#include "evo/server/region_host.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
@@ -48,6 +53,7 @@ struct Options {
|
||||
std::chrono::milliseconds drain{10000};
|
||||
std::string crash_point;
|
||||
std::uint64_t crash_ordinal{};
|
||||
std::string ops_listen;
|
||||
};
|
||||
|
||||
// Kills this process without any cleanup: the real SIGKILL, at a chosen boundary.
|
||||
@@ -140,6 +146,8 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
o.host.lease = {4000ms, 800ms, 400ms, 400ms, 1000ms};
|
||||
else if (flag == "--lease-profile" && value == "default")
|
||||
o.host.lease = {};
|
||||
else if (flag == "--ops-listen")
|
||||
o.ops_listen = value;
|
||||
else if (flag == "--crash-at") {
|
||||
const auto colon = value.find(':');
|
||||
const auto ordinal =
|
||||
@@ -154,6 +162,12 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
} else
|
||||
return std::nullopt;
|
||||
}
|
||||
// Every identity and connection setting is required: there is no default
|
||||
// region, database, object root or build to fall back on.
|
||||
if (o.host.key.world == 0 || o.host.key.region == 0 || o.host.frame == 0 || o.pg.host.empty() ||
|
||||
o.pg.port.empty() || o.pg.database.empty() || o.pg.user.empty() || o.objects.empty() ||
|
||||
o.host.build == persistence::Sha256Digest{})
|
||||
return std::nullopt;
|
||||
if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end())
|
||||
o.host.retained_builds.push_back(o.host.build);
|
||||
return o;
|
||||
@@ -194,23 +208,6 @@ std::optional<server::TickInput> parse_input(std::string_view line) {
|
||||
return in;
|
||||
}
|
||||
|
||||
const char* phase_name(server::HostPhase p) {
|
||||
switch (p) {
|
||||
case server::HostPhase::acquiring:
|
||||
return "acquiring";
|
||||
case server::HostPhase::running:
|
||||
return "running";
|
||||
case server::HostPhase::resolving:
|
||||
return "resolving";
|
||||
case server::HostPhase::draining:
|
||||
return "draining";
|
||||
case server::HostPhase::stopped:
|
||||
return "stopped";
|
||||
case server::HostPhase::quarantined:
|
||||
return "quarantined";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
const char* admission_name(server::AdmissionError e) {
|
||||
switch (e) {
|
||||
case server::AdmissionError::not_ready:
|
||||
@@ -226,20 +223,6 @@ const char* admission_name(server::AdmissionError e) {
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
const char* dropped_name(server::Unacknowledged u) {
|
||||
switch (u) {
|
||||
case server::Unacknowledged::refused:
|
||||
return "refused";
|
||||
case server::Unacknowledged::drained:
|
||||
return "drained";
|
||||
case server::Unacknowledged::fenced:
|
||||
return "fenced";
|
||||
case server::Unacknowledged::quarantined:
|
||||
return "quarantined";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
// Reads complete, bounded lines from stdin without blocking.
|
||||
class LineReader {
|
||||
public:
|
||||
@@ -294,6 +277,41 @@ template <typename State> int serve(Options& o) {
|
||||
return 2;
|
||||
}
|
||||
auto& host = **made;
|
||||
std::unique_ptr<server::OpsServer> ops;
|
||||
if (!o.ops_listen.empty()) {
|
||||
auto listening = server::OpsServer::listen(o.ops_listen);
|
||||
if (!listening) {
|
||||
std::puts("error ops-listen");
|
||||
return 2;
|
||||
}
|
||||
ops = std::move(*listening);
|
||||
std::printf("ops listening %u\n", static_cast<unsigned>(ops->port()));
|
||||
std::fflush(stdout);
|
||||
}
|
||||
server::OpsCounters counters;
|
||||
counters.queue_capacity_inputs = o.host.max_queued_inputs;
|
||||
counters.queue_capacity_bytes = o.host.max_queued_bytes;
|
||||
std::uint64_t last_sequence = 0;
|
||||
auto last_checkpoint = std::chrono::steady_clock::now();
|
||||
bool recovered = false;
|
||||
const auto handler = [&](std::string_view path) -> server::OpsResponse {
|
||||
const auto& s = host.status();
|
||||
if (path == "/livez")
|
||||
return {200, "live\n"};
|
||||
if (path == "/readyz")
|
||||
return s.ready ? server::OpsResponse{200, "ready\n"}
|
||||
: server::OpsResponse{503, std::string("not ready: ") +
|
||||
server::phase_name(s.phase) + "\n"};
|
||||
if (path == "/metrics") {
|
||||
counters.checkpoint_age_ms =
|
||||
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now() - last_checkpoint)
|
||||
.count());
|
||||
return {200, server::render_metrics(s, counters),
|
||||
"text/plain; version=0.0.4; charset=utf-8"};
|
||||
}
|
||||
return {404, "not found\n"};
|
||||
};
|
||||
LineReader reader;
|
||||
bool input_open = true, ready = false;
|
||||
std::optional<std::string> pending;
|
||||
@@ -346,28 +364,46 @@ template <typename State> int serve(Options& o) {
|
||||
}
|
||||
const auto& s = host.step(std::chrono::steady_clock::now());
|
||||
max_queued = std::max(max_queued, s.queued);
|
||||
// Checkpoint age restarts at recovery and at every confirmed commit.
|
||||
if ((!recovered && s.phase == server::HostPhase::running) ||
|
||||
s.committed_sequence != last_sequence) {
|
||||
recovered = recovered || s.phase == server::HostPhase::running;
|
||||
last_sequence = s.committed_sequence;
|
||||
last_checkpoint = std::chrono::steady_clock::now();
|
||||
}
|
||||
if (s.phase == server::HostPhase::acquiring)
|
||||
recovered = false;
|
||||
if (!phase || *phase != s.phase) {
|
||||
phase = s.phase;
|
||||
std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n", phase_name(s.phase),
|
||||
static_cast<unsigned long long>(s.epoch),
|
||||
std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n",
|
||||
server::phase_name(s.phase), static_cast<unsigned long long>(s.epoch),
|
||||
static_cast<unsigned long long>(s.committed_tick),
|
||||
static_cast<unsigned long long>(s.simulated_tick));
|
||||
}
|
||||
set_ready(s.ready);
|
||||
const auto acknowledged = host.take_acks();
|
||||
counters.acks += acknowledged.size();
|
||||
for (const auto& a : acknowledged)
|
||||
std::printf("ack %s %llu %llu\n", text(a.request).c_str(),
|
||||
static_cast<unsigned long long>(a.tick),
|
||||
static_cast<unsigned long long>(a.commit_sequence));
|
||||
for (const auto& x : host.take_outcomes())
|
||||
const auto outcomes = host.take_outcomes();
|
||||
counters.outcomes += outcomes.size();
|
||||
for (const auto& x : outcomes)
|
||||
std::printf("outcome %llu %u %llu %llu\n",
|
||||
static_cast<unsigned long long>(x.decision.tick),
|
||||
static_cast<unsigned>(x.decision.actor),
|
||||
static_cast<unsigned long long>(x.decision.organism),
|
||||
static_cast<unsigned long long>(x.commit_sequence));
|
||||
for (const auto& d : host.take_dropped())
|
||||
std::printf("dropped %s %s\n", text(d.input.request).c_str(), dropped_name(d.reason));
|
||||
const auto dropped = host.take_dropped();
|
||||
for (const auto& d : dropped)
|
||||
++counters.dropped[static_cast<std::size_t>(d.reason)];
|
||||
for (const auto& d : dropped)
|
||||
std::printf("dropped %s %s\n", text(d.input.request).c_str(),
|
||||
server::unacknowledged_name(d.reason));
|
||||
std::fflush(stdout);
|
||||
if (ops)
|
||||
ops->serve(handler);
|
||||
if (o.crash_point == "after-ack" && !acknowledged.empty() && ++ack_steps == o.crash_ordinal)
|
||||
crash_now();
|
||||
const auto finish = [&](int code) {
|
||||
@@ -398,7 +434,8 @@ int main(int argc, char** argv) {
|
||||
"--frame N --pg-host H --pg-port P --pg-database D --pg-user U "
|
||||
"--objects DIR --build HEX64 [--retained HEX64]... [--pg-hostaddr A "
|
||||
"--pg-passfile F --pg-ca F] [--flush-ms N] [--drain-ms N] "
|
||||
"[--lease-profile default|fast] [--ready-file PATH]\n",
|
||||
"[--lease-profile default|fast] [--ready-file PATH] "
|
||||
"[--ops-listen ADDRESS:PORT]\n",
|
||||
stderr);
|
||||
return 2;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user