Files
aevolution/apps/evo-server/main.cpp
T
dingoandClaude Opus 5.5 b4493b3d0f Add M8.5 observability: pinned Prometheus, tested alerts, capacity metrics
An AEvolution-only Prometheus 3.13.3 and Alertmanager 0.34.1, pinned by
digest and delivered by Argo (ADR-0072), scrape each region's operations
Service through static targets (no API access, no RBAC); the region
namespaces admit port 9100 from Prometheus only. Sixteen alert rules
cover authority, lease, commits, backlog and object capacity and are
proven by promtool injection tests in CI with tools copied from the same
pinned images. Two new gauges expose object volume free space and the
reserve. Alerts route to a null receiver until a destination is chosen;
no broker is adopted and the database outbox stays.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 06:16:36 -04:00

472 lines
19 KiB
C++

// M7.6 region server process (ADR-0064). Development host for one region: tick
// inputs arrive as lines on stdin (the network gateway is later work); events are
// written as lines on stdout. Linux only; requires the PostgreSQL build.
//
// Input: input <request:32 hex> <temperature_mK> <source_enabled> <bite0..4> <terminal0..1>
// Output: phase <name> epoch=<e> committed=<t> simulated=<t>
// ready <0|1>
// rejected <request> <reason> | dropped <request> <reason>
// ack <request> <tick> <commit_sequence>
// outcome <tick> <actor> <organism> <commit_sequence>
// stats max_queued=<n> max_tick_debt=<n> commits=<n> store_failures=<n>
// Exit: 0 drained, 2 usage, 3 drain deadline passed, 4 quarantined.
//
// --pg-service <file> (M8.3) takes host, hostaddr, port, database, user and CA from
// the [evolution] section of a libpq service file; anything else in it is refused.
//
// --objects-reserve-mib <n> (M8.3, default 1024) refuses new objects that would leave
// less free space on the object volume: node-local volumes do not enforce their
// requested size. A refused upload is a store failure; nothing is acknowledged.
//
// --ops-listen <numeric address:port> (M7.8) serves GET /livez (the loop is alive; says
// nothing about authority), /readyz (200 only with current authority) and /metrics
// (Prometheus text). It prints "ops listening <port>" once bound.
//
// --crash-at <point>:<n> (tests only, M7.7) SIGKILLs this process at the n-th
// occurrence of before-upload, before-transaction, after-commit or after-ack (right
// after a step's acknowledgements are written and flushed).
#include "evo/server/ops_http.hpp"
#include "evo/server/region_host.hpp"
#include <algorithm>
#include <array>
#include <cerrno>
#include <charconv>
#include <csignal>
#include <cstdio>
#include <filesystem>
#include <fstream>
#include <optional>
#include <poll.h>
#include <signal.h>
#include <string>
#include <string_view>
#include <unistd.h>
#include <vector>
namespace {
using namespace evo;
using namespace std::chrono_literals;
volatile std::sig_atomic_t stop_requested = 0;
extern "C" void on_stop(int) {
stop_requested = 1;
}
struct Options {
std::string profile{"lineage"};
persistence::PgOptions pg;
std::filesystem::path objects, ready_file;
server::HostConfig host;
std::chrono::milliseconds drain{10000};
std::string crash_point;
std::uint64_t crash_ordinal{};
std::string ops_listen;
std::uint64_t objects_reserve_mib{1024};
};
// Kills this process without any cleanup: the real SIGKILL, at a chosen boundary.
[[noreturn]] void crash_now() {
std::fflush(stdout);
::kill(::getpid(), SIGKILL);
for (;;)
::pause();
}
std::optional<std::uint64_t> number(std::string_view text) {
std::uint64_t value{};
const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), value);
if (error != std::errc{} || end != text.data() + text.size())
return std::nullopt;
return value;
}
template <std::size_t N> std::optional<std::array<std::uint8_t, N>> hex(std::string_view text) {
if (text.size() != N * 2)
return std::nullopt;
std::array<std::uint8_t, N> out{};
for (std::size_t i = 0; i < N; ++i) {
unsigned value{};
const auto [end, error] =
std::from_chars(text.data() + (2 * i), text.data() + (2 * i) + 2, value, 16);
if (error != std::errc{} || end != text.data() + (2 * i) + 2)
return std::nullopt;
out[i] = static_cast<std::uint8_t>(value);
}
return out;
}
template <std::size_t N> std::string text(const std::array<std::uint8_t, N>& bytes) {
constexpr char digits[] = "0123456789abcdef";
std::string out;
for (const auto b : bytes) {
out.push_back(digits[b >> 4U]);
out.push_back(digits[b & 0x0FU]);
}
return out;
}
std::optional<Options> parse(int argc, char** argv) {
Options o;
o.host.retained_builds.clear();
for (int i = 1; i < argc; ++i) {
const std::string_view flag = argv[i];
if (i + 1 >= argc)
return std::nullopt;
const std::string_view value = argv[++i];
const auto n = number(value);
if (flag == "--profile" && (value == "lineage" || value == "environmental"))
o.profile = value;
else if (flag == "--world" && n)
o.host.key.world = *n;
else if (flag == "--region" && n)
o.host.key.region = *n;
else if (flag == "--frame" && n)
o.host.frame = *n;
else if (flag == "--pg-host")
o.pg.host = value;
else if (flag == "--pg-hostaddr")
o.pg.hostaddr = value;
else if (flag == "--pg-port")
o.pg.port = value;
else if (flag == "--pg-database")
o.pg.database = value;
else if (flag == "--pg-user")
o.pg.user = value;
else if (flag == "--pg-passfile")
o.pg.password_file = value;
else if (flag == "--pg-service") {
// Connection from the [evolution] section of a libpq service file (M8.3);
// the password file stays separate (--pg-passfile).
auto service = persistence::read_pg_service(std::string(value), "evolution");
if (!service)
return std::nullopt;
service->password_file = o.pg.password_file;
o.pg = *service;
} else if (flag == "--pg-ca")
o.pg.ca_file = value;
else if (flag == "--objects")
o.objects = value;
else if (flag == "--ready-file")
o.ready_file = value;
else if (flag == "--build" || flag == "--retained") {
const auto digest = hex<32>(value);
if (!digest)
return std::nullopt;
if (flag == "--build")
o.host.build = *digest;
else
o.host.retained_builds.push_back(*digest);
} else if (flag == "--flush-ms" && n && *n > 0 && *n <= 60000)
o.host.flush_interval = std::chrono::milliseconds{*n};
else if (flag == "--drain-ms" && n && *n > 0 && *n <= 600000)
o.drain = std::chrono::milliseconds{*n};
else if (flag == "--lease-profile" && value == "fast") // Development/test only.
o.host.lease = {4000ms, 800ms, 400ms, 400ms, 1000ms};
else if (flag == "--lease-profile" && value == "default")
o.host.lease = {};
else if (flag == "--ops-listen")
o.ops_listen = value;
else if (flag == "--objects-reserve-mib" && n && *n <= 1048576)
o.objects_reserve_mib = *n;
else if (flag == "--crash-at") {
const auto colon = value.find(':');
const auto ordinal =
colon == std::string_view::npos ? std::nullopt : number(value.substr(colon + 1));
const auto point = value.substr(0, colon);
if (!ordinal || *ordinal == 0 ||
(point != "before-upload" && point != "before-transaction" &&
point != "after-commit" && point != "after-ack"))
return std::nullopt;
o.crash_point = point;
o.crash_ordinal = *ordinal;
} else
return std::nullopt;
}
// Every identity and connection setting is required: there is no default
// region, database, object root or build to fall back on.
if (o.host.key.world == 0 || o.host.key.region == 0 || o.host.frame == 0 || o.pg.host.empty() ||
o.pg.port.empty() || o.pg.database.empty() || o.pg.user.empty() || o.objects.empty() ||
o.host.build == persistence::Sha256Digest{})
return std::nullopt;
if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end())
o.host.retained_builds.push_back(o.host.build);
return o;
}
std::optional<server::TickInput> parse_input(std::string_view line) {
std::array<std::string_view, 11> fields{};
std::size_t count = 0;
while (!line.empty() && count < fields.size() + 1) {
const auto start = line.find_first_not_of(' ');
if (start == std::string_view::npos)
break;
line.remove_prefix(start);
const auto end = line.find(' ');
if (count == fields.size())
return std::nullopt; // Too many fields.
fields[count++] = line.substr(0, end);
line.remove_prefix(end == std::string_view::npos ? line.size() : end);
}
if (count != fields.size() || fields[0] != "input")
return std::nullopt;
auto request = hex<16>(fields[1]);
if (!request)
return std::nullopt;
std::array<std::uint64_t, 9> v{};
for (std::size_t i = 0; i < v.size(); ++i) {
const auto n = number(fields[i + 2]);
if (!n)
return std::nullopt;
v[i] = *n;
}
server::TickInput in{*request, {}};
in.input.temperature_mK = v[0];
in.input.source_enabled = v[1];
for (std::size_t k = 0; k < 5; ++k)
in.input.bite[k] = v[2 + k];
in.input.terminal = {v[7], v[8]};
return in;
}
const char* admission_name(server::AdmissionError e) {
switch (e) {
case server::AdmissionError::not_ready:
return "not_ready";
case server::AdmissionError::draining:
return "draining";
case server::AdmissionError::full:
return "full";
case server::AdmissionError::duplicate:
return "duplicate";
case server::AdmissionError::invalid:
return "invalid";
}
return "unknown";
}
// Reads complete, bounded lines from stdin without blocking.
class LineReader {
public:
// False on EOF/error.
bool fill() {
std::array<char, 4096> chunk{};
const auto n = ::read(STDIN_FILENO, chunk.data(), chunk.size());
if (n <= 0)
return n < 0 && errno == EINTR;
buffer_.append(chunk.data(), static_cast<std::size_t>(n));
return true;
}
std::optional<std::string> next() {
const auto end = buffer_.find('\n');
if (end == std::string::npos) {
if (buffer_.size() > 512)
buffer_.clear(); // Overlong line: discarded, never partially parsed.
return std::nullopt;
}
auto line = buffer_.substr(0, end);
buffer_.erase(0, end + 1);
return line;
}
private:
std::string buffer_;
};
template <typename State> int serve(Options& o) {
if (!o.crash_point.empty() && o.crash_point != "after-ack") {
o.host.crash_hook = [point = o.crash_point, ordinal = o.crash_ordinal,
seen = std::uint64_t{0}](server::CrashPoint at) mutable {
const char* name = at == server::CrashPoint::before_upload ? "before-upload"
: at == server::CrashPoint::before_transaction ? "before-transaction"
: "after-commit";
if (point == name && ++seen == ordinal)
crash_now();
};
}
std::uint64_t ack_steps = 0;
auto objects = persistence::FilesystemObjectStore::open(
o.objects, o.host.key, persistence::max_lineage_payload_bytes, nullptr,
o.objects_reserve_mib * 1024 * 1024);
if (!objects) {
std::puts("error objects");
return 2;
}
const auto pg = o.pg;
auto made = server::RegionHost<State>::create(
o.host, [pg] { return persistence::Postgres::connect(pg, 2s); }, std::move(*objects));
if (!made) {
std::puts("error config");
return 2;
}
auto& host = **made;
std::unique_ptr<server::OpsServer> ops;
if (!o.ops_listen.empty()) {
auto listening = server::OpsServer::listen(o.ops_listen);
if (!listening) {
std::puts("error ops-listen");
return 2;
}
ops = std::move(*listening);
std::printf("ops listening %u\n", static_cast<unsigned>(ops->port()));
std::fflush(stdout);
}
server::OpsCounters counters;
counters.queue_capacity_inputs = o.host.max_queued_inputs;
counters.queue_capacity_bytes = o.host.max_queued_bytes;
counters.objects_reserve_bytes = o.objects_reserve_mib * 1024 * 1024;
std::uint64_t last_sequence = 0;
auto last_checkpoint = std::chrono::steady_clock::now();
bool recovered = false;
const auto handler = [&](std::string_view path) -> server::OpsResponse {
const auto& s = host.status();
if (path == "/livez")
return {200, "live\n"};
if (path == "/readyz")
return s.ready ? server::OpsResponse{200, "ready\n"}
: server::OpsResponse{503, std::string("not ready: ") +
server::phase_name(s.phase) + "\n"};
if (path == "/metrics") {
counters.checkpoint_age_ms =
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - last_checkpoint)
.count());
counters.objects_available_bytes = host.objects().available_bytes().value_or(0);
return {200, server::render_metrics(s, counters),
"text/plain; version=0.0.4; charset=utf-8"};
}
return {404, "not found\n"};
};
LineReader reader;
bool input_open = true, ready = false;
std::optional<std::string> pending;
std::optional<server::HostPhase> phase;
std::size_t max_queued = 0;
std::optional<std::chrono::steady_clock::time_point> drain_deadline;
const auto set_ready = [&](bool now) {
if (now == ready)
return;
ready = now;
std::printf("ready %d\n", ready ? 1 : 0);
if (!o.ready_file.empty()) {
std::error_code ignored;
if (ready)
std::ofstream(o.ready_file) << "ready\n";
else
std::filesystem::remove(o.ready_file, ignored);
}
};
for (;;) {
if (stop_requested && !drain_deadline) {
host.drain();
drain_deadline = std::chrono::steady_clock::now() + o.drain;
}
// Read only while the bounded backlog has room: stdin itself backpressures.
const bool room = host.status().queued < o.host.max_queued_inputs && !pending;
pollfd fd{STDIN_FILENO, POLLIN, 0};
if (input_open && room && !drain_deadline && ::poll(&fd, 1, 5) > 0)
input_open = reader.fill();
else if (!input_open || !room || drain_deadline)
::usleep(2000);
while (!drain_deadline) {
if (!pending)
pending = reader.next();
if (!pending)
break;
auto in = parse_input(*pending);
if (!in) {
std::puts("rejected - malformed");
pending.reset();
continue;
}
auto admitted = host.submit(*in);
if (!admitted && admitted.error() == server::AdmissionError::full)
break; // Keep the line and retry after ticks drain the backlog.
if (!admitted)
std::printf("rejected %s %s\n", text(in->request).c_str(),
admission_name(admitted.error()));
pending.reset();
}
const auto& s = host.step(std::chrono::steady_clock::now());
max_queued = std::max(max_queued, s.queued);
// Checkpoint age restarts at recovery and at every confirmed commit.
if ((!recovered && s.phase == server::HostPhase::running) ||
s.committed_sequence != last_sequence) {
recovered = recovered || s.phase == server::HostPhase::running;
last_sequence = s.committed_sequence;
last_checkpoint = std::chrono::steady_clock::now();
}
if (s.phase == server::HostPhase::acquiring)
recovered = false;
if (!phase || *phase != s.phase) {
phase = s.phase;
std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n",
server::phase_name(s.phase), static_cast<unsigned long long>(s.epoch),
static_cast<unsigned long long>(s.committed_tick),
static_cast<unsigned long long>(s.simulated_tick));
}
set_ready(s.ready);
const auto acknowledged = host.take_acks();
counters.acks += acknowledged.size();
for (const auto& a : acknowledged)
std::printf("ack %s %llu %llu\n", text(a.request).c_str(),
static_cast<unsigned long long>(a.tick),
static_cast<unsigned long long>(a.commit_sequence));
const auto outcomes = host.take_outcomes();
counters.outcomes += outcomes.size();
for (const auto& x : outcomes)
std::printf("outcome %llu %u %llu %llu\n",
static_cast<unsigned long long>(x.decision.tick),
static_cast<unsigned>(x.decision.actor),
static_cast<unsigned long long>(x.decision.organism),
static_cast<unsigned long long>(x.commit_sequence));
const auto dropped = host.take_dropped();
for (const auto& d : dropped)
++counters.dropped[static_cast<std::size_t>(d.reason)];
for (const auto& d : dropped)
std::printf("dropped %s %s\n", text(d.input.request).c_str(),
server::unacknowledged_name(d.reason));
std::fflush(stdout);
if (ops)
ops->serve(handler);
if (o.crash_point == "after-ack" && !acknowledged.empty() && ++ack_steps == o.crash_ordinal)
crash_now();
const auto finish = [&](int code) {
set_ready(false);
std::printf(
"stats max_queued=%zu max_tick_debt=%llu commits=%llu store_failures=%llu\n",
max_queued, static_cast<unsigned long long>(s.max_tick_debt),
static_cast<unsigned long long>(s.commits),
static_cast<unsigned long long>(s.store_failures));
std::printf("exit %d\n", code);
std::fflush(stdout);
return code;
};
if (s.phase == server::HostPhase::stopped)
return finish(0);
if (s.phase == server::HostPhase::quarantined)
return finish(4);
if (drain_deadline && std::chrono::steady_clock::now() > *drain_deadline)
return finish(3); // Pending/unknown work is left for the next owner to reconcile.
}
}
} // namespace
int main(int argc, char** argv) {
auto options = parse(argc, argv);
if (!options) {
std::fputs("usage: evo-server --profile lineage|environmental --world N --region N "
"--frame N --pg-host H --pg-port P --pg-database D --pg-user U "
"--objects DIR --build HEX64 [--retained HEX64]... [--pg-hostaddr A "
"--pg-passfile F --pg-ca F] [--flush-ms N] [--drain-ms N] "
"[--lease-profile default|fast] [--ready-file PATH] "
"[--ops-listen ADDRESS:PORT]\n",
stderr);
return 2;
}
struct sigaction action{};
action.sa_handler = on_stop;
sigemptyset(&action.sa_mask);
sigaction(SIGTERM, &action, nullptr);
sigaction(SIGINT, &action, nullptr);
std::signal(SIGPIPE, SIG_IGN);
return options->profile == "lineage" ? serve<processes::LineageRegion>(*options)
: serve<processes::EnvironmentalCommunity>(*options);
}