Files
aevolution/apps/evo-server/main.cpp
T
dingoandClaude Opus 5.5 0bf9901c7f
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 1m46s
Foundation / checks (postgres) (push) Canceled after 0s
Foundation / checks (tsan) (push) Canceled after 0s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 3m44s
Foundation / checks (client) (push) Canceled after 3m49s
Foundation / checks (network) (push) Canceled after 1m59s
Add M10.4c evo-server QUIC listener and headless session client
evo-server gains an opt-in M10 listener (--m10-*, QUIC builds only): RegionSessions
with the PostgreSQL control and action ledgers on their own connection under the
host's lease, carried actions submitted to the host and its acknowledgements and
drops turned into outcomes. evo-session-probe is a dev-only headless client. Real
process tests cover intentions, durable action outcomes, SIGKILL before and after
an action's commit, duplicate retries and the probe. The simulated watermark now
reaches every session every tick, so clients always know the server tick.
ADR-0092.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-04 09:20:55 -04:00

603 lines
26 KiB
C++

// M7.6 region server process (ADR-0064). Development host for one region: tick
// inputs arrive as lines on stdin (the network gateway is later work); events are
// written as lines on stdout. Linux only; requires the PostgreSQL build.
//
// Input: input <request:32 hex> <temperature_mK> <source_enabled> <bite0..4> <terminal0..1>
// Output: phase <name> epoch=<e> committed=<t> simulated=<t>
// ready <0|1>
// rejected <request> <reason> | dropped <request> <reason>
// ack <request> <tick> <commit_sequence>
// outcome <tick> <actor> <organism> <commit_sequence>
// stats max_queued=<n> max_tick_debt=<n> commits=<n> store_failures=<n>
// Exit: 0 drained, 2 usage, 3 drain deadline passed, 4 quarantined.
//
// --pg-service <file> (M8.3) takes host, hostaddr, port, database, user and CA from
// the [evolution] section of a libpq service file; anything else in it is refused.
//
// --objects-reserve-mib <n> (M8.3, default 1024) refuses new objects that would leave
// less free space on the object volume: node-local volumes do not enforce their
// requested size. A refused upload is a store failure; nothing is acknowledged.
//
// --ops-listen <numeric address:port> (M7.8) serves GET /livez (the loop is alive; says
// nothing about authority), /readyz (200 only with current authority) and /metrics
// (Prometheus text). It prints "ops listening <port>" once bound.
//
// --synthetic-inputs development-v1 (dev/stage drills only, M8.6, ADR-0073) replaces
// stdin with a deterministic driver: whenever the host is running it keeps the
// backlog fed with development_input(t) for the ticks after the committed head,
// restarting from the committed tick after every recovery. Request IDs are
// "SYN1" + region + tick, so acknowledgements can be checked across owners.
//
// Upgrade rehearsal on clones only (M8.9, ADR-0076), both requiring
// --synthetic-inputs: --synthetic-until <tick> stops the driver after that tick, so
// the region commits exactly up to it and then idles (inputs are never invented);
// --accelerate <2..1000> runs the region's scheduling clock that many times faster
// than real time (lease deadlines stay real). Never in a deployed environment.
//
// Dev sessions (M9.7, ADR-0083; builds with EVO_BUILD_NETWORK only): --session-listen
// <numeric address:port> --session-cert F --session-key F --session-grant-key F
// --session-grant-ledger F --session-environment dev serve the TLS 1.3 dev session
// protocol. The ledger durably records consumed grants (on a persistent volume), so a
// restarted process refuses them. They require
// --synthetic-inputs: an admitted intention rides the next synthetic input (content
// unchanged), so its final outcome is the input's durable acknowledgement, with no
// biological effect. The driver then looks ahead 8 ticks instead of 64. Prints
// "session listening <port>" and "session opened|closed|refused ...". Any environment
// other than dev is refused: the dev identity cannot exist elsewhere.
//
// --crash-at <point>:<n> (tests only, M7.7) SIGKILLs this process at the n-th
// occurrence of before-upload, before-transaction, after-commit or after-ack (right
// after a step's acknowledgements are written and flushed).
#include "evo/processes/development_opening.hpp"
#include "evo/server/ops_http.hpp"
#include "evo/server/region_host.hpp"
#include "m10_bridge.hpp"
#include "session_bridge.hpp"
#include <algorithm>
#include <array>
#include <cerrno>
#include <charconv>
#include <csignal>
#include <cstdio>
#include <filesystem>
#include <fstream>
#include <optional>
#include <poll.h>
#include <signal.h>
#include <string>
#include <string_view>
#include <unistd.h>
#include <vector>
namespace {
using namespace evo;
using namespace std::chrono_literals;
volatile std::sig_atomic_t stop_requested = 0;
extern "C" void on_stop(int) {
stop_requested = 1;
}
struct Options {
std::string profile{"lineage"};
persistence::PgOptions pg;
std::filesystem::path objects, ready_file;
server::HostConfig host;
std::chrono::milliseconds drain{10000};
std::string crash_point;
std::uint64_t crash_ordinal{};
std::string ops_listen;
bool synthetic{};
std::uint64_t synthetic_until{}; // 0: unbounded.
std::uint64_t accelerate{1};
std::uint64_t objects_reserve_mib{1024};
server_app::SessionOptions session;
server_app::M10Options m10;
};
// Kills this process without any cleanup: the real SIGKILL, at a chosen boundary.
[[noreturn]] void crash_now() {
std::fflush(stdout);
::kill(::getpid(), SIGKILL);
for (;;)
::pause();
}
std::optional<std::uint64_t> number(std::string_view text) {
std::uint64_t value{};
const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), value);
if (error != std::errc{} || end != text.data() + text.size())
return std::nullopt;
return value;
}
template <std::size_t N> std::optional<std::array<std::uint8_t, N>> hex(std::string_view text) {
if (text.size() != N * 2)
return std::nullopt;
std::array<std::uint8_t, N> out{};
for (std::size_t i = 0; i < N; ++i) {
unsigned value{};
const auto [end, error] =
std::from_chars(text.data() + (2 * i), text.data() + (2 * i) + 2, value, 16);
if (error != std::errc{} || end != text.data() + (2 * i) + 2)
return std::nullopt;
out[i] = static_cast<std::uint8_t>(value);
}
return out;
}
template <std::size_t N> std::string text(const std::array<std::uint8_t, N>& bytes) {
constexpr char digits[] = "0123456789abcdef";
std::string out;
for (const auto b : bytes) {
out.push_back(digits[b >> 4U]);
out.push_back(digits[b & 0x0FU]);
}
return out;
}
std::optional<Options> parse(int argc, char** argv) {
Options o;
o.host.retained_builds.clear();
for (int i = 1; i < argc; ++i) {
const std::string_view flag = argv[i];
if (i + 1 >= argc)
return std::nullopt;
const std::string_view value = argv[++i];
const auto n = number(value);
if (flag == "--profile" && (value == "lineage" || value == "environmental"))
o.profile = value;
else if (flag == "--world" && n)
o.host.key.world = *n;
else if (flag == "--region" && n)
o.host.key.region = *n;
else if (flag == "--frame" && n)
o.host.frame = *n;
else if (flag == "--pg-host")
o.pg.host = value;
else if (flag == "--pg-hostaddr")
o.pg.hostaddr = value;
else if (flag == "--pg-port")
o.pg.port = value;
else if (flag == "--pg-database")
o.pg.database = value;
else if (flag == "--pg-user")
o.pg.user = value;
else if (flag == "--pg-passfile")
o.pg.password_file = value;
else if (flag == "--pg-service") {
// Connection from the [evolution] section of a libpq service file (M8.3);
// the password file stays separate (--pg-passfile).
auto service = persistence::read_pg_service(std::string(value), "evolution");
if (!service)
return std::nullopt;
service->password_file = o.pg.password_file;
o.pg = *service;
} else if (flag == "--pg-ca")
o.pg.ca_file = value;
else if (flag == "--objects")
o.objects = value;
else if (flag == "--ready-file")
o.ready_file = value;
else if (flag == "--build" || flag == "--retained") {
const auto digest = hex<32>(value);
if (!digest)
return std::nullopt;
if (flag == "--build")
o.host.build = *digest;
else
o.host.retained_builds.push_back(*digest);
} else if (flag == "--flush-ms" && n && *n > 0 && *n <= 60000)
o.host.flush_interval = std::chrono::milliseconds{*n};
else if (flag == "--drain-ms" && n && *n > 0 && *n <= 600000)
o.drain = std::chrono::milliseconds{*n};
else if (flag == "--lease-profile" && value == "fast") // Development/test only.
o.host.lease = {4000ms, 800ms, 400ms, 400ms, 1000ms};
else if (flag == "--lease-profile" && value == "default")
o.host.lease = {};
else if (flag == "--ops-listen")
o.ops_listen = value;
else if (flag == "--synthetic-inputs" && value == "development-v1")
o.synthetic = true;
else if (flag == "--synthetic-until" && n && *n > 0)
o.synthetic_until = *n;
else if (flag == "--accelerate" && n && *n >= 2 && *n <= 1000)
o.accelerate = *n;
else if (flag == "--objects-reserve-mib" && n && *n <= 1048576)
o.objects_reserve_mib = *n;
else if (flag == "--session-listen")
o.session.listen = value;
else if (flag == "--session-cert")
o.session.certificate = value;
else if (flag == "--session-key")
o.session.key = value;
else if (flag == "--session-grant-key")
o.session.grant_key = value;
else if (flag == "--session-grant-ledger")
o.session.grant_ledger = value;
else if (flag == "--session-environment" &&
(value == "dev" || value == "stage" || value == "world0"))
o.session.environment = value;
else if (flag == "--m10-listen")
o.m10.listen = value;
else if (flag == "--m10-cert")
o.m10.certificate = value;
else if (flag == "--m10-key")
o.m10.key = value;
else if (flag == "--m10-issuer")
o.m10.issuer = value;
else if (flag == "--m10-environment" &&
(value == "dev" || value == "stage" || value == "world0"))
o.m10.environment = value;
else if (flag == "--crash-at") {
const auto colon = value.find(':');
const auto ordinal =
colon == std::string_view::npos ? std::nullopt : number(value.substr(colon + 1));
const auto point = value.substr(0, colon);
if (!ordinal || *ordinal == 0 ||
(point != "before-upload" && point != "before-transaction" &&
point != "after-commit" && point != "after-ack"))
return std::nullopt;
o.crash_point = point;
o.crash_ordinal = *ordinal;
} else
return std::nullopt;
}
// Every identity and connection setting is required: there is no default
// region, database, object root or build to fall back on.
if (o.host.key.world == 0 || o.host.key.region == 0 || o.host.frame == 0 || o.pg.host.empty() ||
o.pg.port.empty() || o.pg.database.empty() || o.pg.user.empty() || o.objects.empty() ||
o.host.build == persistence::Sha256Digest{})
return std::nullopt;
// Rehearsal controls exist only for the synthetic driver (clones, drills).
if ((o.synthetic_until != 0 || o.accelerate != 1) && !o.synthetic)
return std::nullopt;
// Sessions need every setting, the network build, and the synthetic carrier.
if (o.session.any() &&
(!server_app::sessions_available || !o.session.complete() || !o.synthetic))
return std::nullopt;
// M10 sessions (ADR-0091/0092) need every setting and the QUIC build.
if (o.m10.any() && (!server_app::m10_available || !o.m10.complete()))
return std::nullopt;
if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end())
o.host.retained_builds.push_back(o.host.build);
return o;
}
std::optional<server::TickInput> parse_input(std::string_view line) {
std::array<std::string_view, 11> fields{};
std::size_t count = 0;
while (!line.empty() && count < fields.size() + 1) {
const auto start = line.find_first_not_of(' ');
if (start == std::string_view::npos)
break;
line.remove_prefix(start);
const auto end = line.find(' ');
if (count == fields.size())
return std::nullopt; // Too many fields.
fields[count++] = line.substr(0, end);
line.remove_prefix(end == std::string_view::npos ? line.size() : end);
}
if (count != fields.size() || fields[0] != "input")
return std::nullopt;
auto request = hex<16>(fields[1]);
if (!request)
return std::nullopt;
std::array<std::uint64_t, 9> v{};
for (std::size_t i = 0; i < v.size(); ++i) {
const auto n = number(fields[i + 2]);
if (!n)
return std::nullopt;
v[i] = *n;
}
server::TickInput in{*request, {}};
in.input.temperature_mK = v[0];
in.input.source_enabled = v[1];
for (std::size_t k = 0; k < 5; ++k)
in.input.bite[k] = v[2 + k];
in.input.terminal = {v[7], v[8]};
return in;
}
const char* admission_name(server::AdmissionError e) {
switch (e) {
case server::AdmissionError::not_ready:
return "not_ready";
case server::AdmissionError::draining:
return "draining";
case server::AdmissionError::full:
return "full";
case server::AdmissionError::duplicate:
return "duplicate";
case server::AdmissionError::invalid:
return "invalid";
}
return "unknown";
}
// Reads complete, bounded lines from stdin without blocking.
class LineReader {
public:
// False on EOF/error.
bool fill() {
std::array<char, 4096> chunk{};
const auto n = ::read(STDIN_FILENO, chunk.data(), chunk.size());
if (n <= 0)
return n < 0 && errno == EINTR;
buffer_.append(chunk.data(), static_cast<std::size_t>(n));
return true;
}
std::optional<std::string> next() {
const auto end = buffer_.find('\n');
if (end == std::string::npos) {
if (buffer_.size() > 512)
buffer_.clear(); // Overlong line: discarded, never partially parsed.
return std::nullopt;
}
auto line = buffer_.substr(0, end);
buffer_.erase(0, end + 1);
return line;
}
private:
std::string buffer_;
};
template <typename State> int serve(Options& o) {
if (!o.crash_point.empty() && o.crash_point != "after-ack") {
o.host.crash_hook = [point = o.crash_point, ordinal = o.crash_ordinal,
seen = std::uint64_t{0}](server::CrashPoint at) mutable {
const char* name = at == server::CrashPoint::before_upload ? "before-upload"
: at == server::CrashPoint::before_transaction ? "before-transaction"
: "after-commit";
if (point == name && ++seen == ordinal)
crash_now();
};
}
std::uint64_t ack_steps = 0;
auto objects = persistence::FilesystemObjectStore::open(
o.objects, o.host.key, persistence::max_lineage_payload_bytes, nullptr,
o.objects_reserve_mib * 1024 * 1024);
if (!objects) {
std::puts("error objects");
return 2;
}
const auto pg = o.pg;
auto made = server::RegionHost<State>::create(
o.host, [pg] { return persistence::Postgres::connect(pg, 2s); }, std::move(*objects));
if (!made) {
std::puts("error config");
return 2;
}
auto& host = **made;
std::unique_ptr<server::OpsServer> ops;
if (!o.ops_listen.empty()) {
auto listening = server::OpsServer::listen(o.ops_listen);
if (!listening) {
std::puts("error ops-listen");
return 2;
}
ops = std::move(*listening);
std::printf("ops listening %u\n", static_cast<unsigned>(ops->port()));
std::fflush(stdout);
}
auto sessions = server_app::SessionBridge::open(o.session, o.host.key, o.host.frame);
if (!sessions)
return 2;
auto m10 = server_app::M10Bridge<server::RegionHost<State>>::open(o.m10, host, o.host.key,
o.host.frame, o.pg);
if (!m10)
return 2;
std::fflush(stdout);
const std::uint64_t lookahead = sessions->enabled() ? server_app::session_lookahead_ticks : 64;
server::OpsCounters counters;
counters.queue_capacity_inputs = o.host.max_queued_inputs;
counters.queue_capacity_bytes = o.host.max_queued_bytes;
counters.objects_reserve_bytes = o.objects_reserve_mib * 1024 * 1024;
std::uint64_t last_sequence = 0;
auto last_checkpoint = std::chrono::steady_clock::now();
bool recovered = false;
const auto handler = [&](std::string_view path) -> server::OpsResponse {
const auto& s = host.status();
if (path == "/livez")
return {200, "live\n"};
if (path == "/readyz")
return s.ready ? server::OpsResponse{200, "ready\n"}
: server::OpsResponse{503, std::string("not ready: ") +
server::phase_name(s.phase) + "\n"};
if (path == "/metrics") {
counters.checkpoint_age_ms =
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - last_checkpoint)
.count());
counters.objects_available_bytes = host.objects().available_bytes().value_or(0);
return {200, server::render_metrics(s, counters),
"text/plain; version=0.0.4; charset=utf-8"};
}
return {404, "not found\n"};
};
// Scheduling time; --accelerate scales it from process start (clones only).
const auto started = std::chrono::steady_clock::now();
const auto scheduling_now = [&] {
const auto real = std::chrono::steady_clock::now();
return started + (real - started) * static_cast<std::int64_t>(o.accelerate);
};
LineReader reader;
bool input_open = !o.synthetic, ready = false;
// Synthetic driver state: the next tick to submit, reset at every recovery.
std::optional<std::uint64_t> synthetic_next;
bool synthetic_running = false;
const auto synthetic_request = [&](std::uint64_t tick) {
persistence::Uuid id{'S', 'Y', 'N', '1'};
for (unsigned i = 0; i < 4; ++i)
id[4 + i] = static_cast<std::uint8_t>((o.host.key.region >> (8U * (3U - i))) & 0xFFU);
for (unsigned i = 0; i < 8; ++i)
id[8 + i] = static_cast<std::uint8_t>((tick >> (8U * (7U - i))) & 0xFFU);
return id;
};
std::optional<std::string> pending;
std::optional<server::HostPhase> phase;
std::size_t max_queued = 0;
std::optional<std::chrono::steady_clock::time_point> drain_deadline;
const auto set_ready = [&](bool now) {
if (now == ready)
return;
ready = now;
std::printf("ready %d\n", ready ? 1 : 0);
if (!o.ready_file.empty()) {
std::error_code ignored;
if (ready)
std::ofstream(o.ready_file) << "ready\n";
else
std::filesystem::remove(o.ready_file, ignored);
}
};
for (;;) {
if (stop_requested && !drain_deadline) {
m10->shutdown();
host.drain();
drain_deadline = std::chrono::steady_clock::now() + o.drain;
}
// Read only while the bounded backlog has room: stdin itself backpressures.
const bool room = host.status().queued < o.host.max_queued_inputs && !pending;
pollfd fd{STDIN_FILENO, POLLIN, 0};
if (input_open && room && !drain_deadline && ::poll(&fd, 1, 5) > 0)
input_open = reader.fill();
else if (!input_open || !room || drain_deadline)
::usleep(2000);
while (!drain_deadline) {
if (!pending)
pending = reader.next();
if (!pending)
break;
auto in = parse_input(*pending);
if (!in) {
std::puts("rejected - malformed");
pending.reset();
continue;
}
auto admitted = host.submit(*in);
if (!admitted && admitted.error() == server::AdmissionError::full)
break; // Keep the line and retry after ticks drain the backlog.
if (!admitted)
std::printf("rejected %s %s\n", text(in->request).c_str(),
admission_name(admitted.error()));
pending.reset();
}
if (o.synthetic && !drain_deadline) {
const auto& now_status = host.status();
const bool running = now_status.phase == server::HostPhase::running && now_status.ready;
if (running && !synthetic_running)
synthetic_next = now_status.simulated_tick + 1; // Resume after the head.
synthetic_running = running;
while (running && synthetic_next &&
*synthetic_next <= host.status().simulated_tick + lookahead &&
(o.synthetic_until == 0 || *synthetic_next <= o.synthetic_until)) {
// An admitted session intention may ride this input; content is unchanged.
const auto carried = sessions->carry(*synthetic_next);
const server::TickInput in{carried.value_or(synthetic_request(*synthetic_next)),
processes::development_input(*synthetic_next)};
if (!host.submit(in)) {
if (carried)
sessions->not_committed(*carried);
break; // Full or no longer ready: continue next iteration.
}
++*synthetic_next;
}
}
const auto& s = host.step(scheduling_now());
max_queued = std::max(max_queued, s.queued);
// Checkpoint age restarts at recovery and at every confirmed commit.
if ((!recovered && s.phase == server::HostPhase::running) ||
s.committed_sequence != last_sequence) {
recovered = recovered || s.phase == server::HostPhase::running;
last_sequence = s.committed_sequence;
last_checkpoint = std::chrono::steady_clock::now();
}
if (s.phase == server::HostPhase::acquiring)
recovered = false;
if (!phase || *phase != s.phase) {
phase = s.phase;
std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n",
server::phase_name(s.phase), static_cast<unsigned long long>(s.epoch),
static_cast<unsigned long long>(s.committed_tick),
static_cast<unsigned long long>(s.simulated_tick));
}
set_ready(s.ready);
const auto acknowledged = host.take_acks();
counters.acks += acknowledged.size();
for (const auto& a : acknowledged) {
std::printf("ack %s %llu %llu\n", text(a.request).c_str(),
static_cast<unsigned long long>(a.tick),
static_cast<unsigned long long>(a.commit_sequence));
sessions->committed(a.request, a.tick, a.commit_sequence);
}
const auto outcomes = host.take_outcomes();
counters.outcomes += outcomes.size();
for (const auto& x : outcomes)
std::printf("outcome %llu %u %llu %llu\n",
static_cast<unsigned long long>(x.decision.tick),
static_cast<unsigned>(x.decision.actor),
static_cast<unsigned long long>(x.decision.organism),
static_cast<unsigned long long>(x.commit_sequence));
const auto dropped = host.take_dropped();
for (const auto& d : dropped)
++counters.dropped[static_cast<std::size_t>(d.reason)];
for (const auto& d : dropped) {
std::printf("dropped %s %s\n", text(d.input.request).c_str(),
server::unacknowledged_name(d.reason));
sessions->not_committed(d.input.request);
}
std::fflush(stdout);
if (ops)
ops->serve(handler);
sessions->step(host.state(), host.status());
m10->after_step();
std::fflush(stdout);
if (o.crash_point == "after-ack" && !acknowledged.empty() && ++ack_steps == o.crash_ordinal)
crash_now();
const auto finish = [&](int code) {
set_ready(false);
std::printf(
"stats max_queued=%zu max_tick_debt=%llu commits=%llu store_failures=%llu\n",
max_queued, static_cast<unsigned long long>(s.max_tick_debt),
static_cast<unsigned long long>(s.commits),
static_cast<unsigned long long>(s.store_failures));
std::printf("exit %d\n", code);
std::fflush(stdout);
return code;
};
if (s.phase == server::HostPhase::stopped)
return finish(0);
if (s.phase == server::HostPhase::quarantined)
return finish(4);
if (drain_deadline && std::chrono::steady_clock::now() > *drain_deadline)
return finish(3); // Pending/unknown work is left for the next owner to reconcile.
}
}
} // namespace
int main(int argc, char** argv) {
auto options = parse(argc, argv);
if (!options) {
std::fputs("usage: evo-server --profile lineage|environmental --world N --region N "
"--frame N --pg-host H --pg-port P --pg-database D --pg-user U "
"--objects DIR --build HEX64 [--retained HEX64]... [--pg-hostaddr A "
"--pg-passfile F --pg-ca F] [--flush-ms N] [--drain-ms N] "
"[--lease-profile default|fast] [--ready-file PATH] "
"[--ops-listen ADDRESS:PORT] [--synthetic-inputs development-v1 "
"[--synthetic-until TICK] [--accelerate N] [--session-listen ADDRESS:PORT "
"--session-cert F --session-key F --session-grant-key F "
"--session-grant-ledger F --session-environment dev]] [--m10-listen "
"ADDRESS:PORT --m10-cert F --m10-key F --m10-issuer F "
"--m10-environment dev|stage|world0]\n",
stderr);
return 2;
}
struct sigaction action{};
action.sa_handler = on_stop;
sigemptyset(&action.sa_mask);
sigaction(SIGTERM, &action, nullptr);
sigaction(SIGINT, &action, nullptr);
std::signal(SIGPIPE, SIG_IGN);
return options->profile == "lineage" ? serve<processes::LineageRegion>(*options)
: serve<processes::EnvironmentalCommunity>(*options);
}