Keep the filesystem object store on a node-local Retain-class PVC per environment (ADR-0070), with the database on the external PostgreSQL VM. Because local-path does not enforce sizes, the object store now refuses new objects below a free-space reserve with a distinct full error, so a full volume publishes and acknowledges nothing. evo-world-clone gains check-store to qualify volume retention, reserve, database version/TLS and the runtime privilege boundary from inside the cluster; it now ships in the server image. Add the storage class, object claims, hand-applied store Jobs, policy rules forbidding password env vars, and the runbook. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
459 lines
19 KiB
C++
459 lines
19 KiB
C++
// M7.6 region server process (ADR-0064). Development host for one region: tick
|
|
// inputs arrive as lines on stdin (the network gateway is later work); events are
|
|
// written as lines on stdout. Linux only; requires the PostgreSQL build.
|
|
//
|
|
// Input: input <request:32 hex> <temperature_mK> <source_enabled> <bite0..4> <terminal0..1>
|
|
// Output: phase <name> epoch=<e> committed=<t> simulated=<t>
|
|
// ready <0|1>
|
|
// rejected <request> <reason> | dropped <request> <reason>
|
|
// ack <request> <tick> <commit_sequence>
|
|
// outcome <tick> <actor> <organism> <commit_sequence>
|
|
// stats max_queued=<n> max_tick_debt=<n> commits=<n> store_failures=<n>
|
|
// Exit: 0 drained, 2 usage, 3 drain deadline passed, 4 quarantined.
|
|
//
|
|
// --objects-reserve-mib <n> (M8.3, default 1024) refuses new objects that would leave
|
|
// less free space on the object volume: node-local volumes do not enforce their
|
|
// requested size. A refused upload is a store failure; nothing is acknowledged.
|
|
//
|
|
// --ops-listen <numeric address:port> (M7.8) serves GET /livez (the loop is alive; says
|
|
// nothing about authority), /readyz (200 only with current authority) and /metrics
|
|
// (Prometheus text). It prints "ops listening <port>" once bound.
|
|
//
|
|
// --crash-at <point>:<n> (tests only, M7.7) SIGKILLs this process at the n-th
|
|
// occurrence of before-upload, before-transaction, after-commit or after-ack (right
|
|
// after a step's acknowledgements are written and flushed).
|
|
#include "evo/server/ops_http.hpp"
|
|
#include "evo/server/region_host.hpp"
|
|
|
|
#include <algorithm>
|
|
#include <array>
|
|
#include <cerrno>
|
|
#include <charconv>
|
|
#include <csignal>
|
|
#include <cstdio>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <optional>
|
|
#include <poll.h>
|
|
#include <signal.h>
|
|
#include <string>
|
|
#include <string_view>
|
|
#include <unistd.h>
|
|
#include <vector>
|
|
|
|
namespace {
|
|
using namespace evo;
|
|
using namespace std::chrono_literals;
|
|
volatile std::sig_atomic_t stop_requested = 0;
|
|
extern "C" void on_stop(int) {
|
|
stop_requested = 1;
|
|
}
|
|
|
|
struct Options {
|
|
std::string profile{"lineage"};
|
|
persistence::PgOptions pg;
|
|
std::filesystem::path objects, ready_file;
|
|
server::HostConfig host;
|
|
std::chrono::milliseconds drain{10000};
|
|
std::string crash_point;
|
|
std::uint64_t crash_ordinal{};
|
|
std::string ops_listen;
|
|
std::uint64_t objects_reserve_mib{1024};
|
|
};
|
|
|
|
// Kills this process without any cleanup: the real SIGKILL, at a chosen boundary.
|
|
[[noreturn]] void crash_now() {
|
|
std::fflush(stdout);
|
|
::kill(::getpid(), SIGKILL);
|
|
for (;;)
|
|
::pause();
|
|
}
|
|
|
|
std::optional<std::uint64_t> number(std::string_view text) {
|
|
std::uint64_t value{};
|
|
const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), value);
|
|
if (error != std::errc{} || end != text.data() + text.size())
|
|
return std::nullopt;
|
|
return value;
|
|
}
|
|
template <std::size_t N> std::optional<std::array<std::uint8_t, N>> hex(std::string_view text) {
|
|
if (text.size() != N * 2)
|
|
return std::nullopt;
|
|
std::array<std::uint8_t, N> out{};
|
|
for (std::size_t i = 0; i < N; ++i) {
|
|
unsigned value{};
|
|
const auto [end, error] =
|
|
std::from_chars(text.data() + (2 * i), text.data() + (2 * i) + 2, value, 16);
|
|
if (error != std::errc{} || end != text.data() + (2 * i) + 2)
|
|
return std::nullopt;
|
|
out[i] = static_cast<std::uint8_t>(value);
|
|
}
|
|
return out;
|
|
}
|
|
template <std::size_t N> std::string text(const std::array<std::uint8_t, N>& bytes) {
|
|
constexpr char digits[] = "0123456789abcdef";
|
|
std::string out;
|
|
for (const auto b : bytes) {
|
|
out.push_back(digits[b >> 4U]);
|
|
out.push_back(digits[b & 0x0FU]);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
std::optional<Options> parse(int argc, char** argv) {
|
|
Options o;
|
|
o.host.retained_builds.clear();
|
|
for (int i = 1; i < argc; ++i) {
|
|
const std::string_view flag = argv[i];
|
|
if (i + 1 >= argc)
|
|
return std::nullopt;
|
|
const std::string_view value = argv[++i];
|
|
const auto n = number(value);
|
|
if (flag == "--profile" && (value == "lineage" || value == "environmental"))
|
|
o.profile = value;
|
|
else if (flag == "--world" && n)
|
|
o.host.key.world = *n;
|
|
else if (flag == "--region" && n)
|
|
o.host.key.region = *n;
|
|
else if (flag == "--frame" && n)
|
|
o.host.frame = *n;
|
|
else if (flag == "--pg-host")
|
|
o.pg.host = value;
|
|
else if (flag == "--pg-hostaddr")
|
|
o.pg.hostaddr = value;
|
|
else if (flag == "--pg-port")
|
|
o.pg.port = value;
|
|
else if (flag == "--pg-database")
|
|
o.pg.database = value;
|
|
else if (flag == "--pg-user")
|
|
o.pg.user = value;
|
|
else if (flag == "--pg-passfile")
|
|
o.pg.password_file = value;
|
|
else if (flag == "--pg-ca")
|
|
o.pg.ca_file = value;
|
|
else if (flag == "--objects")
|
|
o.objects = value;
|
|
else if (flag == "--ready-file")
|
|
o.ready_file = value;
|
|
else if (flag == "--build" || flag == "--retained") {
|
|
const auto digest = hex<32>(value);
|
|
if (!digest)
|
|
return std::nullopt;
|
|
if (flag == "--build")
|
|
o.host.build = *digest;
|
|
else
|
|
o.host.retained_builds.push_back(*digest);
|
|
} else if (flag == "--flush-ms" && n && *n > 0 && *n <= 60000)
|
|
o.host.flush_interval = std::chrono::milliseconds{*n};
|
|
else if (flag == "--drain-ms" && n && *n > 0 && *n <= 600000)
|
|
o.drain = std::chrono::milliseconds{*n};
|
|
else if (flag == "--lease-profile" && value == "fast") // Development/test only.
|
|
o.host.lease = {4000ms, 800ms, 400ms, 400ms, 1000ms};
|
|
else if (flag == "--lease-profile" && value == "default")
|
|
o.host.lease = {};
|
|
else if (flag == "--ops-listen")
|
|
o.ops_listen = value;
|
|
else if (flag == "--objects-reserve-mib" && n && *n <= 1048576)
|
|
o.objects_reserve_mib = *n;
|
|
else if (flag == "--crash-at") {
|
|
const auto colon = value.find(':');
|
|
const auto ordinal =
|
|
colon == std::string_view::npos ? std::nullopt : number(value.substr(colon + 1));
|
|
const auto point = value.substr(0, colon);
|
|
if (!ordinal || *ordinal == 0 ||
|
|
(point != "before-upload" && point != "before-transaction" &&
|
|
point != "after-commit" && point != "after-ack"))
|
|
return std::nullopt;
|
|
o.crash_point = point;
|
|
o.crash_ordinal = *ordinal;
|
|
} else
|
|
return std::nullopt;
|
|
}
|
|
// Every identity and connection setting is required: there is no default
|
|
// region, database, object root or build to fall back on.
|
|
if (o.host.key.world == 0 || o.host.key.region == 0 || o.host.frame == 0 || o.pg.host.empty() ||
|
|
o.pg.port.empty() || o.pg.database.empty() || o.pg.user.empty() || o.objects.empty() ||
|
|
o.host.build == persistence::Sha256Digest{})
|
|
return std::nullopt;
|
|
if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end())
|
|
o.host.retained_builds.push_back(o.host.build);
|
|
return o;
|
|
}
|
|
|
|
std::optional<server::TickInput> parse_input(std::string_view line) {
|
|
std::array<std::string_view, 11> fields{};
|
|
std::size_t count = 0;
|
|
while (!line.empty() && count < fields.size() + 1) {
|
|
const auto start = line.find_first_not_of(' ');
|
|
if (start == std::string_view::npos)
|
|
break;
|
|
line.remove_prefix(start);
|
|
const auto end = line.find(' ');
|
|
if (count == fields.size())
|
|
return std::nullopt; // Too many fields.
|
|
fields[count++] = line.substr(0, end);
|
|
line.remove_prefix(end == std::string_view::npos ? line.size() : end);
|
|
}
|
|
if (count != fields.size() || fields[0] != "input")
|
|
return std::nullopt;
|
|
auto request = hex<16>(fields[1]);
|
|
if (!request)
|
|
return std::nullopt;
|
|
std::array<std::uint64_t, 9> v{};
|
|
for (std::size_t i = 0; i < v.size(); ++i) {
|
|
const auto n = number(fields[i + 2]);
|
|
if (!n)
|
|
return std::nullopt;
|
|
v[i] = *n;
|
|
}
|
|
server::TickInput in{*request, {}};
|
|
in.input.temperature_mK = v[0];
|
|
in.input.source_enabled = v[1];
|
|
for (std::size_t k = 0; k < 5; ++k)
|
|
in.input.bite[k] = v[2 + k];
|
|
in.input.terminal = {v[7], v[8]};
|
|
return in;
|
|
}
|
|
|
|
const char* admission_name(server::AdmissionError e) {
|
|
switch (e) {
|
|
case server::AdmissionError::not_ready:
|
|
return "not_ready";
|
|
case server::AdmissionError::draining:
|
|
return "draining";
|
|
case server::AdmissionError::full:
|
|
return "full";
|
|
case server::AdmissionError::duplicate:
|
|
return "duplicate";
|
|
case server::AdmissionError::invalid:
|
|
return "invalid";
|
|
}
|
|
return "unknown";
|
|
}
|
|
// Reads complete, bounded lines from stdin without blocking.
|
|
class LineReader {
|
|
public:
|
|
// False on EOF/error.
|
|
bool fill() {
|
|
std::array<char, 4096> chunk{};
|
|
const auto n = ::read(STDIN_FILENO, chunk.data(), chunk.size());
|
|
if (n <= 0)
|
|
return n < 0 && errno == EINTR;
|
|
buffer_.append(chunk.data(), static_cast<std::size_t>(n));
|
|
return true;
|
|
}
|
|
std::optional<std::string> next() {
|
|
const auto end = buffer_.find('\n');
|
|
if (end == std::string::npos) {
|
|
if (buffer_.size() > 512)
|
|
buffer_.clear(); // Overlong line: discarded, never partially parsed.
|
|
return std::nullopt;
|
|
}
|
|
auto line = buffer_.substr(0, end);
|
|
buffer_.erase(0, end + 1);
|
|
return line;
|
|
}
|
|
|
|
private:
|
|
std::string buffer_;
|
|
};
|
|
|
|
template <typename State> int serve(Options& o) {
|
|
if (!o.crash_point.empty() && o.crash_point != "after-ack") {
|
|
o.host.crash_hook = [point = o.crash_point, ordinal = o.crash_ordinal,
|
|
seen = std::uint64_t{0}](server::CrashPoint at) mutable {
|
|
const char* name = at == server::CrashPoint::before_upload ? "before-upload"
|
|
: at == server::CrashPoint::before_transaction ? "before-transaction"
|
|
: "after-commit";
|
|
if (point == name && ++seen == ordinal)
|
|
crash_now();
|
|
};
|
|
}
|
|
std::uint64_t ack_steps = 0;
|
|
auto objects = persistence::FilesystemObjectStore::open(
|
|
o.objects, o.host.key, persistence::max_lineage_payload_bytes, nullptr,
|
|
o.objects_reserve_mib * 1024 * 1024);
|
|
if (!objects) {
|
|
std::puts("error objects");
|
|
return 2;
|
|
}
|
|
const auto pg = o.pg;
|
|
auto made = server::RegionHost<State>::create(
|
|
o.host, [pg] { return persistence::Postgres::connect(pg, 2s); }, std::move(*objects));
|
|
if (!made) {
|
|
std::puts("error config");
|
|
return 2;
|
|
}
|
|
auto& host = **made;
|
|
std::unique_ptr<server::OpsServer> ops;
|
|
if (!o.ops_listen.empty()) {
|
|
auto listening = server::OpsServer::listen(o.ops_listen);
|
|
if (!listening) {
|
|
std::puts("error ops-listen");
|
|
return 2;
|
|
}
|
|
ops = std::move(*listening);
|
|
std::printf("ops listening %u\n", static_cast<unsigned>(ops->port()));
|
|
std::fflush(stdout);
|
|
}
|
|
server::OpsCounters counters;
|
|
counters.queue_capacity_inputs = o.host.max_queued_inputs;
|
|
counters.queue_capacity_bytes = o.host.max_queued_bytes;
|
|
std::uint64_t last_sequence = 0;
|
|
auto last_checkpoint = std::chrono::steady_clock::now();
|
|
bool recovered = false;
|
|
const auto handler = [&](std::string_view path) -> server::OpsResponse {
|
|
const auto& s = host.status();
|
|
if (path == "/livez")
|
|
return {200, "live\n"};
|
|
if (path == "/readyz")
|
|
return s.ready ? server::OpsResponse{200, "ready\n"}
|
|
: server::OpsResponse{503, std::string("not ready: ") +
|
|
server::phase_name(s.phase) + "\n"};
|
|
if (path == "/metrics") {
|
|
counters.checkpoint_age_ms =
|
|
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
std::chrono::steady_clock::now() - last_checkpoint)
|
|
.count());
|
|
return {200, server::render_metrics(s, counters),
|
|
"text/plain; version=0.0.4; charset=utf-8"};
|
|
}
|
|
return {404, "not found\n"};
|
|
};
|
|
LineReader reader;
|
|
bool input_open = true, ready = false;
|
|
std::optional<std::string> pending;
|
|
std::optional<server::HostPhase> phase;
|
|
std::size_t max_queued = 0;
|
|
std::optional<std::chrono::steady_clock::time_point> drain_deadline;
|
|
const auto set_ready = [&](bool now) {
|
|
if (now == ready)
|
|
return;
|
|
ready = now;
|
|
std::printf("ready %d\n", ready ? 1 : 0);
|
|
if (!o.ready_file.empty()) {
|
|
std::error_code ignored;
|
|
if (ready)
|
|
std::ofstream(o.ready_file) << "ready\n";
|
|
else
|
|
std::filesystem::remove(o.ready_file, ignored);
|
|
}
|
|
};
|
|
for (;;) {
|
|
if (stop_requested && !drain_deadline) {
|
|
host.drain();
|
|
drain_deadline = std::chrono::steady_clock::now() + o.drain;
|
|
}
|
|
// Read only while the bounded backlog has room: stdin itself backpressures.
|
|
const bool room = host.status().queued < o.host.max_queued_inputs && !pending;
|
|
pollfd fd{STDIN_FILENO, POLLIN, 0};
|
|
if (input_open && room && !drain_deadline && ::poll(&fd, 1, 5) > 0)
|
|
input_open = reader.fill();
|
|
else if (!input_open || !room || drain_deadline)
|
|
::usleep(2000);
|
|
while (!drain_deadline) {
|
|
if (!pending)
|
|
pending = reader.next();
|
|
if (!pending)
|
|
break;
|
|
auto in = parse_input(*pending);
|
|
if (!in) {
|
|
std::puts("rejected - malformed");
|
|
pending.reset();
|
|
continue;
|
|
}
|
|
auto admitted = host.submit(*in);
|
|
if (!admitted && admitted.error() == server::AdmissionError::full)
|
|
break; // Keep the line and retry after ticks drain the backlog.
|
|
if (!admitted)
|
|
std::printf("rejected %s %s\n", text(in->request).c_str(),
|
|
admission_name(admitted.error()));
|
|
pending.reset();
|
|
}
|
|
const auto& s = host.step(std::chrono::steady_clock::now());
|
|
max_queued = std::max(max_queued, s.queued);
|
|
// Checkpoint age restarts at recovery and at every confirmed commit.
|
|
if ((!recovered && s.phase == server::HostPhase::running) ||
|
|
s.committed_sequence != last_sequence) {
|
|
recovered = recovered || s.phase == server::HostPhase::running;
|
|
last_sequence = s.committed_sequence;
|
|
last_checkpoint = std::chrono::steady_clock::now();
|
|
}
|
|
if (s.phase == server::HostPhase::acquiring)
|
|
recovered = false;
|
|
if (!phase || *phase != s.phase) {
|
|
phase = s.phase;
|
|
std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n",
|
|
server::phase_name(s.phase), static_cast<unsigned long long>(s.epoch),
|
|
static_cast<unsigned long long>(s.committed_tick),
|
|
static_cast<unsigned long long>(s.simulated_tick));
|
|
}
|
|
set_ready(s.ready);
|
|
const auto acknowledged = host.take_acks();
|
|
counters.acks += acknowledged.size();
|
|
for (const auto& a : acknowledged)
|
|
std::printf("ack %s %llu %llu\n", text(a.request).c_str(),
|
|
static_cast<unsigned long long>(a.tick),
|
|
static_cast<unsigned long long>(a.commit_sequence));
|
|
const auto outcomes = host.take_outcomes();
|
|
counters.outcomes += outcomes.size();
|
|
for (const auto& x : outcomes)
|
|
std::printf("outcome %llu %u %llu %llu\n",
|
|
static_cast<unsigned long long>(x.decision.tick),
|
|
static_cast<unsigned>(x.decision.actor),
|
|
static_cast<unsigned long long>(x.decision.organism),
|
|
static_cast<unsigned long long>(x.commit_sequence));
|
|
const auto dropped = host.take_dropped();
|
|
for (const auto& d : dropped)
|
|
++counters.dropped[static_cast<std::size_t>(d.reason)];
|
|
for (const auto& d : dropped)
|
|
std::printf("dropped %s %s\n", text(d.input.request).c_str(),
|
|
server::unacknowledged_name(d.reason));
|
|
std::fflush(stdout);
|
|
if (ops)
|
|
ops->serve(handler);
|
|
if (o.crash_point == "after-ack" && !acknowledged.empty() && ++ack_steps == o.crash_ordinal)
|
|
crash_now();
|
|
const auto finish = [&](int code) {
|
|
set_ready(false);
|
|
std::printf(
|
|
"stats max_queued=%zu max_tick_debt=%llu commits=%llu store_failures=%llu\n",
|
|
max_queued, static_cast<unsigned long long>(s.max_tick_debt),
|
|
static_cast<unsigned long long>(s.commits),
|
|
static_cast<unsigned long long>(s.store_failures));
|
|
std::printf("exit %d\n", code);
|
|
std::fflush(stdout);
|
|
return code;
|
|
};
|
|
if (s.phase == server::HostPhase::stopped)
|
|
return finish(0);
|
|
if (s.phase == server::HostPhase::quarantined)
|
|
return finish(4);
|
|
if (drain_deadline && std::chrono::steady_clock::now() > *drain_deadline)
|
|
return finish(3); // Pending/unknown work is left for the next owner to reconcile.
|
|
}
|
|
}
|
|
} // namespace
|
|
|
|
int main(int argc, char** argv) {
|
|
auto options = parse(argc, argv);
|
|
if (!options) {
|
|
std::fputs("usage: evo-server --profile lineage|environmental --world N --region N "
|
|
"--frame N --pg-host H --pg-port P --pg-database D --pg-user U "
|
|
"--objects DIR --build HEX64 [--retained HEX64]... [--pg-hostaddr A "
|
|
"--pg-passfile F --pg-ca F] [--flush-ms N] [--drain-ms N] "
|
|
"[--lease-profile default|fast] [--ready-file PATH] "
|
|
"[--ops-listen ADDRESS:PORT]\n",
|
|
stderr);
|
|
return 2;
|
|
}
|
|
struct sigaction action{};
|
|
action.sa_handler = on_stop;
|
|
sigemptyset(&action.sa_mask);
|
|
sigaction(SIGTERM, &action, nullptr);
|
|
sigaction(SIGINT, &action, nullptr);
|
|
std::signal(SIGPIPE, SIG_IGN);
|
|
return options->profile == "lineage" ? serve<processes::LineageRegion>(*options)
|
|
: serve<processes::EnvironmentalCommunity>(*options);
|
|
}
|