Files
aevolution/apps/evo-server/main.cpp
T
dingo e6ba9bd1a9
Foundation / checks (deploy) (push) Successful in 6s
Foundation / checks (checks) (push) Failing after 40s
Foundation / checks (postgres) (push) Canceled after 0s
Foundation / checks (tsan) (push) Canceled after 0s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (client) (push) Canceled after 1m12s
Foundation / checks (network) (push) Canceled after 37s
Foundation / checks (ecs) (push) Canceled after 1m9s
Test life owner routing across region restart
2026-10-11 01:34:46 -04:00

1116 lines
50 KiB
C++

// M7.6 region server process (ADR-0064). Development host for one region: tick
// inputs arrive as lines on stdin (the network gateway is later work); events are
// written as lines on stdout. Linux only; requires the PostgreSQL build.
//
// Input: input <request:32 hex> <temperature_mK> <source_enabled> <bite0..4> <terminal0..1>
// Output: phase <name> epoch=<e> committed=<t> simulated=<t>
// ready <0|1>
// rejected <request> <reason> | dropped <request> <reason>
// ack <request> <tick> <commit_sequence>
// outcome <tick> <actor> <organism> <commit_sequence>
// stats max_queued=<n> max_tick_debt=<n> commits=<n> store_failures=<n>
// Exit: 0 drained, 2 usage, 3 drain deadline passed, 4 quarantined.
//
// --pg-service <file> (M8.3) takes host, hostaddr, port, database, user and CA from
// the [evolution] section of a libpq service file; anything else in it is refused.
//
// --objects-reserve-mib <n> (M8.3, default 1024) refuses new objects that would leave
// less free space on the object volume: node-local volumes do not enforce their
// requested size. A refused upload is a store failure; nothing is acknowledged.
//
// --ops-listen <numeric address:port> (M7.8) serves GET /livez (the loop is alive; says
// nothing about authority), /readyz (200 only with current authority) and /metrics
// (Prometheus text). It prints "ops listening <port>" once bound.
//
// --synthetic-inputs development-v1 (dev/stage drills only, M8.6, ADR-0073) replaces
// stdin with a deterministic driver: whenever the host is running it keeps the
// backlog fed with development_input(t) for the ticks after the committed head,
// restarting from the committed tick after every recovery. Request IDs are
// "SYN1" + region + tick, so acknowledgements can be checked across owners.
//
// Upgrade rehearsal on clones only (M8.9, ADR-0076), both requiring
// --synthetic-inputs: --synthetic-until <tick> stops the driver after that tick, so
// the region commits exactly up to it and then idles (inputs are never invented);
// --accelerate <2..1000> runs the region's scheduling clock that many times faster
// than real time (lease deadlines stay real). Never in a deployed environment.
//
// Dev sessions (M9.7, ADR-0083; builds with EVO_BUILD_NETWORK only): --session-listen
// <numeric address:port> --session-cert F --session-key F --session-grant-key F
// --session-grant-ledger F --session-environment dev serve the TLS 1.3 dev session
// protocol. The ledger durably records consumed grants (on a persistent volume), so a
// restarted process refuses them. They require
// --synthetic-inputs: an admitted intention rides the next synthetic input (content
// unchanged), so its final outcome is the input's durable acknowledgement, with no
// biological effect. The driver then looks ahead 8 ticks instead of 64. Prints
// "session listening <port>" and "session opened|closed|refused ...". Any environment
// other than dev is refused: the dev identity cannot exist elsewhere.
//
// --crash-at <point>:<n> (tests only, M7.7) SIGKILLs this process at the n-th
// occurrence of before-upload, before-transaction, after-commit or after-ack (right
// after a step's acknowledgements are written and flushed).
// --profile primitive (M11.2d) starts the creation-only profile-3 host. It requires
// an owner-provisioned checkpoint and becomes ready only after recovery. Optional
// --primitive-* flags enable the v5 new-life listener; biology ticks and genesis
// provisioning remain disabled.
#include "evo/biology/founder.hpp"
#include "evo/processes/development_opening.hpp"
#include "evo/server/cellular_session_batch.hpp"
#include "evo/server/lifecycle/primitive_lives.hpp"
#include "evo/server/ops_http.hpp"
#include "evo/server/primitive_region_host.hpp"
#include "evo/server/region_host.hpp"
#include "m10_bridge.hpp"
#include "session_bridge.hpp"
#ifdef EVO_SERVER_M10
#include "cellular_bridge.hpp"
#include "evo/server/pg_slot.hpp"
#include "evo/server/postgres_control_ledger.hpp"
#include "evo/server/region_sessions.hpp"
#endif
#include <algorithm>
#include <array>
#include <cerrno>
#include <charconv>
#include <chrono>
#include <csignal>
#include <cstdio>
#include <filesystem>
#include <fstream>
#include <iterator>
#include <optional>
#include <poll.h>
#include <signal.h>
#include <string>
#include <string_view>
#include <thread>
#include <unistd.h>
#include <vector>
namespace {
using namespace evo;
using namespace std::chrono_literals;
volatile std::sig_atomic_t stop_requested = 0;
extern "C" void on_stop(int) {
stop_requested = 1;
}
struct Options {
std::string profile{"lineage"};
persistence::PgOptions pg;
std::filesystem::path objects, ready_file;
server::HostConfig host;
std::chrono::milliseconds drain{10000};
std::string crash_point;
std::uint64_t crash_ordinal{};
std::string ops_listen;
bool synthetic{};
std::uint64_t synthetic_until{}; // 0: unbounded.
std::uint64_t accelerate{1};
std::uint64_t objects_reserve_mib{1024};
server_app::SessionOptions session;
server_app::M10Options m10;
struct PrimitiveSessionOptions {
std::string listen, certificate, key, issuer, environment;
[[nodiscard]] bool any() const noexcept {
return !listen.empty() || !certificate.empty() || !key.empty() || !issuer.empty() ||
!environment.empty();
}
[[nodiscard]] bool complete() const noexcept {
return !listen.empty() && !certificate.empty() && !key.empty() && !issuer.empty() &&
!environment.empty();
}
} primitive_session;
};
// Kills this process without any cleanup: the real SIGKILL, at a chosen boundary.
[[noreturn]] void crash_now() {
std::fflush(stdout);
::kill(::getpid(), SIGKILL);
for (;;)
::pause();
}
std::optional<std::uint64_t> number(std::string_view text) {
std::uint64_t value{};
const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), value);
if (error != std::errc{} || end != text.data() + text.size())
return std::nullopt;
return value;
}
template <std::size_t N> std::optional<std::array<std::uint8_t, N>> hex(std::string_view text) {
if (text.size() != N * 2)
return std::nullopt;
std::array<std::uint8_t, N> out{};
for (std::size_t i = 0; i < N; ++i) {
unsigned value{};
const auto [end, error] =
std::from_chars(text.data() + (2 * i), text.data() + (2 * i) + 2, value, 16);
if (error != std::errc{} || end != text.data() + (2 * i) + 2)
return std::nullopt;
out[i] = static_cast<std::uint8_t>(value);
}
return out;
}
template <std::size_t N> std::string text(const std::array<std::uint8_t, N>& bytes) {
constexpr char digits[] = "0123456789abcdef";
std::string out;
for (const auto b : bytes) {
out.push_back(digits[b >> 4U]);
out.push_back(digits[b & 0x0FU]);
}
return out;
}
std::optional<Options> parse(int argc, char** argv) {
Options o;
o.host.retained_builds.clear();
for (int i = 1; i < argc; ++i) {
const std::string_view flag = argv[i];
if (i + 1 >= argc)
return std::nullopt;
const std::string_view value = argv[++i];
const auto n = number(value);
if (flag == "--profile" && (value == "lineage" || value == "environmental" ||
value == "primitive" || value == "cellular"))
o.profile = value;
else if (flag == "--world" && n)
o.host.key.world = *n;
else if (flag == "--region" && n)
o.host.key.region = *n;
else if (flag == "--frame" && n)
o.host.frame = *n;
else if (flag == "--pg-host")
o.pg.host = value;
else if (flag == "--pg-hostaddr")
o.pg.hostaddr = value;
else if (flag == "--pg-port")
o.pg.port = value;
else if (flag == "--pg-database")
o.pg.database = value;
else if (flag == "--pg-user")
o.pg.user = value;
else if (flag == "--pg-passfile")
o.pg.password_file = value;
else if (flag == "--pg-service") {
// Connection from the [evolution] section of a libpq service file (M8.3);
// the password file stays separate (--pg-passfile).
auto service = persistence::read_pg_service(std::string(value), "evolution");
if (!service)
return std::nullopt;
service->password_file = o.pg.password_file;
o.pg = *service;
} else if (flag == "--pg-ca")
o.pg.ca_file = value;
else if (flag == "--objects")
o.objects = value;
else if (flag == "--ready-file")
o.ready_file = value;
else if (flag == "--build" || flag == "--retained") {
const auto digest = hex<32>(value);
if (!digest)
return std::nullopt;
if (flag == "--build")
o.host.build = *digest;
else
o.host.retained_builds.push_back(*digest);
} else if (flag == "--flush-ms" && n && *n > 0 && *n <= 60000)
o.host.flush_interval = std::chrono::milliseconds{*n};
else if (flag == "--drain-ms" && n && *n > 0 && *n <= 600000)
o.drain = std::chrono::milliseconds{*n};
else if (flag == "--lease-profile" && value == "fast") // Development/test only.
o.host.lease = {4000ms, 800ms, 400ms, 400ms, 1000ms};
else if (flag == "--lease-profile" && value == "default")
o.host.lease = {};
else if (flag == "--ops-listen")
o.ops_listen = value;
else if (flag == "--synthetic-inputs" && value == "development-v1")
o.synthetic = true;
else if (flag == "--synthetic-until" && n && *n > 0)
o.synthetic_until = *n;
else if (flag == "--accelerate" && n && *n >= 2 && *n <= 1000)
o.accelerate = *n;
else if (flag == "--objects-reserve-mib" && n && *n <= 1048576)
o.objects_reserve_mib = *n;
else if (flag == "--session-listen")
o.session.listen = value;
else if (flag == "--session-cert")
o.session.certificate = value;
else if (flag == "--session-key")
o.session.key = value;
else if (flag == "--session-grant-key")
o.session.grant_key = value;
else if (flag == "--session-grant-ledger")
o.session.grant_ledger = value;
else if (flag == "--session-environment" &&
(value == "dev" || value == "stage" || value == "world0"))
o.session.environment = value;
else if (flag == "--m10-listen")
o.m10.listen = value;
else if (flag == "--m10-cert")
o.m10.certificate = value;
else if (flag == "--m10-key")
o.m10.key = value;
else if (flag == "--m10-issuer")
o.m10.issuer = value;
else if (flag == "--m10-environment" &&
(value == "dev" || value == "stage" || value == "world0"))
o.m10.environment = value;
else if (flag == "--session-window")
o.m10.session_window = value;
else if (flag == "--m10-max-grant-lifetime" && n && *n >= 60 && *n <= 14400)
o.m10.max_grant_lifetime = *n;
else if (flag == "--m10-area")
o.m10.area = value;
else if (flag == "--m10-handoff-key")
o.m10.handoff_key = value;
else if (flag == "--m10-neighbour" && o.m10.neighbours.size() < 8)
o.m10.neighbours.emplace_back(value);
else if (flag == "--life-route" && o.m10.life_routes.size() < 4096)
o.m10.life_routes.emplace_back(value);
else if (flag == "--m10-load-governor" && (value == "on" || value == "off"))
o.m10.load_governor = value == "on";
else if (flag == "--primitive-listen")
o.primitive_session.listen = value;
else if (flag == "--primitive-cert")
o.primitive_session.certificate = value;
else if (flag == "--primitive-key")
o.primitive_session.key = value;
else if (flag == "--primitive-issuer")
o.primitive_session.issuer = value;
else if (flag == "--primitive-environment" &&
(value == "dev" || value == "stage" || value == "world0"))
o.primitive_session.environment = value;
else if (flag == "--crash-at") {
const auto colon = value.find(':');
const auto ordinal =
colon == std::string_view::npos ? std::nullopt : number(value.substr(colon + 1));
const auto point = value.substr(0, colon);
if (!ordinal || *ordinal == 0 ||
(point != "before-upload" && point != "before-transaction" &&
point != "after-commit" && point != "after-ack"))
return std::nullopt;
o.crash_point = point;
o.crash_ordinal = *ordinal;
} else
return std::nullopt;
}
// Every identity and connection setting is required: there is no default
// region, database, object root or build to fall back on.
if (o.host.key.world == 0 || o.host.key.region == 0 || o.host.frame == 0 || o.pg.host.empty() ||
o.pg.port.empty() || o.pg.database.empty() || o.pg.user.empty() || o.objects.empty() ||
o.host.build == persistence::Sha256Digest{})
return std::nullopt;
// Rehearsal controls exist only for the synthetic driver (clones, drills).
if ((o.synthetic_until != 0 || o.accelerate != 1) && !o.synthetic)
return std::nullopt;
// Sessions need every setting, the network build, and the synthetic carrier.
if (o.session.any() &&
(!server_app::sessions_available || !o.session.complete() || !o.synthetic))
return std::nullopt;
// M10 sessions (ADR-0091/0092) need every setting and the QUIC build.
if (o.m10.session_any() && (!server_app::m10_available || !o.m10.complete()))
return std::nullopt;
if (!o.m10.life_routes.empty() && o.profile != "primitive" && o.profile != "cellular")
return std::nullopt;
if (!o.m10.life_routes.empty() && o.profile == "primitive" && !o.primitive_session.any())
return std::nullopt;
// Primitive profile serves only the creation-only host. Its optional listener
// uses a separate v5 contract and never enables the M9/M10 action path.
if (o.primitive_session.any() &&
(!server_app::m10_available || !o.primitive_session.complete() || o.profile != "primitive"))
return std::nullopt;
if (o.profile == "primitive" &&
(o.synthetic || !o.crash_point.empty() || o.session.any() || o.m10.session_any() ||
o.host.flush_interval != 1000ms || o.drain != 10000ms))
return std::nullopt;
if (o.profile == "cellular" &&
(!server_app::m10_available || !o.m10.complete() || o.primitive_session.any() ||
o.synthetic || !o.crash_point.empty() || o.session.any() || !o.m10.area.empty() ||
!o.m10.handoff_key.empty() || !o.m10.neighbours.empty() || o.m10.max_grant_lifetime != 0 ||
!o.m10.load_governor || o.host.flush_interval != 1000ms || o.drain != 10000ms))
return std::nullopt;
if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end())
o.host.retained_builds.push_back(o.host.build);
return o;
}
std::optional<server::TickInput> parse_input(std::string_view line) {
std::array<std::string_view, 11> fields{};
std::size_t count = 0;
while (!line.empty() && count < fields.size() + 1) {
const auto start = line.find_first_not_of(' ');
if (start == std::string_view::npos)
break;
line.remove_prefix(start);
const auto end = line.find(' ');
if (count == fields.size())
return std::nullopt; // Too many fields.
fields[count++] = line.substr(0, end);
line.remove_prefix(end == std::string_view::npos ? line.size() : end);
}
if (count != fields.size() || fields[0] != "input")
return std::nullopt;
auto request = hex<16>(fields[1]);
if (!request)
return std::nullopt;
std::array<std::uint64_t, 9> v{};
for (std::size_t i = 0; i < v.size(); ++i) {
const auto n = number(fields[i + 2]);
if (!n)
return std::nullopt;
v[i] = *n;
}
server::TickInput in{*request, {}};
in.input.temperature_mK = v[0];
in.input.source_enabled = v[1];
for (std::size_t k = 0; k < 5; ++k)
in.input.bite[k] = v[2 + k];
in.input.terminal = {v[7], v[8]};
return in;
}
const char* admission_name(server::AdmissionError e) {
switch (e) {
case server::AdmissionError::not_ready:
return "not_ready";
case server::AdmissionError::draining:
return "draining";
case server::AdmissionError::full:
return "full";
case server::AdmissionError::duplicate:
return "duplicate";
case server::AdmissionError::invalid:
return "invalid";
}
return "unknown";
}
// Reads complete, bounded lines from stdin without blocking.
class LineReader {
public:
// False on EOF/error.
bool fill() {
std::array<char, 4096> chunk{};
const auto n = ::read(STDIN_FILENO, chunk.data(), chunk.size());
if (n <= 0)
return n < 0 && errno == EINTR;
buffer_.append(chunk.data(), static_cast<std::size_t>(n));
return true;
}
std::optional<std::string> next() {
const auto end = buffer_.find('\n');
if (end == std::string::npos) {
if (buffer_.size() > 512)
buffer_.clear(); // Overlong line: discarded, never partially parsed.
return std::nullopt;
}
auto line = buffer_.substr(0, end);
buffer_.erase(0, end + 1);
return line;
}
private:
std::string buffer_;
};
template <typename State> int serve(Options& o) {
if (!o.crash_point.empty() && o.crash_point != "after-ack") {
o.host.crash_hook = [point = o.crash_point, ordinal = o.crash_ordinal,
seen = std::uint64_t{0}](server::CrashPoint at) mutable {
const char* name = at == server::CrashPoint::before_upload ? "before-upload"
: at == server::CrashPoint::before_transaction ? "before-transaction"
: "after-commit";
if (point == name && ++seen == ordinal)
crash_now();
};
}
std::uint64_t ack_steps = 0;
auto objects = persistence::FilesystemObjectStore::open(
o.objects, o.host.key, persistence::max_lineage_payload_bytes, nullptr,
o.objects_reserve_mib * 1024 * 1024);
if (!objects) {
std::puts("error objects");
return 2;
}
const auto pg = o.pg;
auto made = server::RegionHost<State>::create(
o.host, [pg] { return persistence::Postgres::connect(pg, 2s); }, std::move(*objects));
if (!made) {
std::puts("error config");
return 2;
}
auto& host = **made;
std::unique_ptr<server::OpsServer> ops;
if (!o.ops_listen.empty()) {
auto listening = server::OpsServer::listen(o.ops_listen);
if (!listening) {
std::puts("error ops-listen");
return 2;
}
ops = std::move(*listening);
std::printf("ops listening %u\n", static_cast<unsigned>(ops->port()));
std::fflush(stdout);
}
auto sessions = server_app::SessionBridge::open(o.session, o.host.key, o.host.frame);
if (!sessions)
return 2;
auto m10 = server_app::M10Bridge<server::RegionHost<State>>::open(o.m10, host, o.host.key,
o.host.frame, o.pg);
if (!m10)
return 2;
std::fflush(stdout);
const std::uint64_t lookahead = sessions->enabled() ? server_app::session_lookahead_ticks : 64;
server::OpsCounters counters;
counters.queue_capacity_inputs = o.host.max_queued_inputs;
counters.queue_capacity_bytes = o.host.max_queued_bytes;
counters.objects_reserve_bytes = o.objects_reserve_mib * 1024 * 1024;
std::uint64_t last_sequence = 0;
auto last_checkpoint = std::chrono::steady_clock::now();
bool recovered = false;
const auto handler = [&](std::string_view path) -> server::OpsResponse {
const auto& s = host.status();
if (path == "/livez")
return {200, "live\n"};
if (path == "/readyz")
return s.ready ? server::OpsResponse{200, "ready\n"}
: server::OpsResponse{503, std::string("not ready: ") +
server::phase_name(s.phase) + "\n"};
if (path == "/metrics") {
counters.checkpoint_age_ms =
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - last_checkpoint)
.count());
counters.objects_available_bytes = host.objects().available_bytes().value_or(0);
m10->metrics(counters);
return {200, server::render_metrics(s, counters),
"text/plain; version=0.0.4; charset=utf-8"};
}
return {404, "not found\n"};
};
// Scheduling time; --accelerate scales it from process start (clones only).
const auto started = std::chrono::steady_clock::now();
const auto scheduling_now = [&] {
const auto real = std::chrono::steady_clock::now();
return started + (real - started) * static_cast<std::int64_t>(o.accelerate);
};
LineReader reader;
bool input_open = !o.synthetic, ready = false;
// Synthetic driver state: the next tick to submit, reset at every recovery.
std::optional<std::uint64_t> synthetic_next;
bool synthetic_running = false;
const auto synthetic_request = [&](std::uint64_t tick) {
persistence::Uuid id{'S', 'Y', 'N', '1'};
for (unsigned i = 0; i < 4; ++i)
id[4 + i] = static_cast<std::uint8_t>((o.host.key.region >> (8U * (3U - i))) & 0xFFU);
for (unsigned i = 0; i < 8; ++i)
id[8 + i] = static_cast<std::uint8_t>((tick >> (8U * (7U - i))) & 0xFFU);
return id;
};
std::optional<std::string> pending;
std::optional<server::HostPhase> phase;
std::size_t max_queued = 0;
std::optional<std::chrono::steady_clock::time_point> drain_deadline;
const auto set_ready = [&](bool now) {
if (now == ready)
return;
ready = now;
std::printf("ready %d\n", ready ? 1 : 0);
if (!o.ready_file.empty()) {
std::error_code ignored;
if (ready)
std::ofstream(o.ready_file) << "ready\n";
else
std::filesystem::remove(o.ready_file, ignored);
}
};
for (;;) {
if (stop_requested && !drain_deadline) {
m10->shutdown();
host.drain();
drain_deadline = std::chrono::steady_clock::now() + o.drain;
}
// Read only while the bounded backlog has room: stdin itself backpressures.
const bool room = host.status().queued < o.host.max_queued_inputs && !pending;
pollfd fd{STDIN_FILENO, POLLIN, 0};
if (input_open && room && !drain_deadline && ::poll(&fd, 1, 5) > 0)
input_open = reader.fill();
else if (!input_open || !room || drain_deadline)
::usleep(2000);
while (!drain_deadline) {
if (!pending)
pending = reader.next();
if (!pending)
break;
auto in = parse_input(*pending);
if (!in) {
std::puts("rejected - malformed");
pending.reset();
continue;
}
auto admitted = host.submit(*in);
if (!admitted && admitted.error() == server::AdmissionError::full)
break; // Keep the line and retry after ticks drain the backlog.
if (!admitted)
std::printf("rejected %s %s\n", text(in->request).c_str(),
admission_name(admitted.error()));
pending.reset();
}
const bool m10_accepting = m10->accepting(
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count()));
if (o.synthetic && !drain_deadline && (!m10->enabled() || m10_accepting)) {
const auto& now_status = host.status();
const bool running = now_status.phase == server::HostPhase::running && now_status.ready;
if (running && !synthetic_running)
synthetic_next = now_status.simulated_tick + 1; // Resume after the head.
synthetic_running = running;
while (running && synthetic_next &&
*synthetic_next <= host.status().simulated_tick + lookahead &&
(o.synthetic_until == 0 || *synthetic_next <= o.synthetic_until)) {
// An admitted session intention may ride this input; content is unchanged.
const auto carried = sessions->carry(*synthetic_next);
const server::TickInput in{carried.value_or(synthetic_request(*synthetic_next)),
processes::development_input(*synthetic_next)};
if (!host.submit(in)) {
if (carried)
sessions->not_committed(*carried);
break; // Full or no longer ready: continue next iteration.
}
++*synthetic_next;
}
}
const auto step_started = std::chrono::steady_clock::now();
const auto& s = host.step(scheduling_now());
const auto host_work = std::chrono::steady_clock::now() - step_started; // ADR-0097.
max_queued = std::max(max_queued, s.queued);
// Checkpoint age restarts at recovery and at every confirmed commit.
if ((!recovered && s.phase == server::HostPhase::running) ||
s.committed_sequence != last_sequence) {
recovered = recovered || s.phase == server::HostPhase::running;
last_sequence = s.committed_sequence;
last_checkpoint = std::chrono::steady_clock::now();
}
if (s.phase == server::HostPhase::acquiring)
recovered = false;
if (!phase || *phase != s.phase) {
phase = s.phase;
std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n",
server::phase_name(s.phase), static_cast<unsigned long long>(s.epoch),
static_cast<unsigned long long>(s.committed_tick),
static_cast<unsigned long long>(s.simulated_tick));
}
set_ready(s.ready);
const auto acknowledged = host.take_acks();
counters.acks += acknowledged.size();
for (const auto& a : acknowledged) {
std::printf("ack %s %llu %llu\n", text(a.request).c_str(),
static_cast<unsigned long long>(a.tick),
static_cast<unsigned long long>(a.commit_sequence));
sessions->committed(a.request, a.tick, a.commit_sequence);
}
const auto outcomes = host.take_outcomes();
counters.outcomes += outcomes.size();
for (const auto& x : outcomes)
std::printf("outcome %llu %u %llu %llu\n",
static_cast<unsigned long long>(x.decision.tick),
static_cast<unsigned>(x.decision.actor),
static_cast<unsigned long long>(x.decision.organism),
static_cast<unsigned long long>(x.commit_sequence));
const auto dropped = host.take_dropped();
for (const auto& d : dropped)
++counters.dropped[static_cast<std::size_t>(d.reason)];
for (const auto& d : dropped) {
std::printf("dropped %s %s\n", text(d.input.request).c_str(),
server::unacknowledged_name(d.reason));
sessions->not_committed(d.input.request);
}
std::fflush(stdout);
if (ops)
ops->serve(handler);
sessions->step(host.state(), host.status());
m10->after_step(host_work);
std::fflush(stdout);
if (o.crash_point == "after-ack" && !acknowledged.empty() && ++ack_steps == o.crash_ordinal)
crash_now();
const auto finish = [&](int code) {
set_ready(false);
std::printf(
"stats max_queued=%zu max_tick_debt=%llu commits=%llu store_failures=%llu\n",
max_queued, static_cast<unsigned long long>(s.max_tick_debt),
static_cast<unsigned long long>(s.commits),
static_cast<unsigned long long>(s.store_failures));
std::printf("exit %d\n", code);
std::fflush(stdout);
return code;
};
if (s.phase == server::HostPhase::stopped)
return finish(0);
if (s.phase == server::HostPhase::quarantined)
return finish(4);
if (drain_deadline && std::chrono::steady_clock::now() > *drain_deadline)
return finish(3); // Pending/unknown work is left for the next owner to reconcile.
}
}
#ifdef EVO_SERVER_M10
namespace {
using PrimitiveWireOutcome = network::PrimitiveLifeOutcome;
PrimitiveWireOutcome primitive_wire_outcome(std::uint64_t request,
const server::PrimitiveHostReply& reply) {
using network::PrimitiveLifeResult;
PrimitiveLifeResult result = PrimitiveLifeResult::unknown;
std::uint64_t identity = 0, sequence = 0;
if (reply.kind == server::PrimitiveHostReplyKind::born && reply.published) {
result = PrimitiveLifeResult::born;
identity = reply.published->identity;
sequence = reply.published->commit_sequence;
} else if (reply.kind == server::PrimitiveHostReplyKind::pending) {
result = PrimitiveLifeResult::pending;
} else if (reply.kind == server::PrimitiveHostReplyKind::refused) {
switch (reply.reservation.refusal) {
case persistence::LifeRefusal::insufficient_funding:
result = PrimitiveLifeResult::insufficient_funding;
break;
case persistence::LifeRefusal::no_placement:
result = PrimitiveLifeResult::no_placement;
break;
case persistence::LifeRefusal::cancelled:
result = PrimitiveLifeResult::cancelled;
break;
case persistence::LifeRefusal::existing_life:
result = PrimitiveLifeResult::existing_life;
break;
case persistence::LifeRefusal::none:
result = PrimitiveLifeResult::unknown;
break;
}
}
// A reservation owned by another region is never asserted as a birth here. The current
// session can query again after it reaches the owner; region routing stays explicit.
return {request, result, identity, sequence};
}
struct PrimitiveSessionBridge {
std::unique_ptr<server::PgSlot> slot;
std::unique_ptr<server::PostgresControlLedger> ledger;
std::unique_ptr<server::RegionSessions> sessions;
};
std::optional<PrimitiveSessionBridge>
open_primitive_sessions(const Options::PrimitiveSessionOptions& options,
const server_app::M10Options& routing, server::PrimitiveRegionHost& host,
persistence::RegionKey key, std::uint64_t frame,
const persistence::PgOptions& pg) {
PrimitiveSessionBridge bridge;
if (!options.any())
return bridge;
const auto colon = options.listen.rfind(':');
std::uint16_t port{};
if (colon == std::string::npos ||
std::from_chars(options.listen.data() + colon + 1,
options.listen.data() + options.listen.size(), port)
.ec != std::errc{}) {
std::puts("error primitive-listen");
return std::nullopt;
}
std::ifstream issuer_file(options.issuer, std::ios::binary);
std::vector<char> issuer((std::istreambuf_iterator<char>(issuer_file)), {});
if (issuer.size() != network::grant_public_key_bytes) {
std::puts("error primitive-issuer");
return std::nullopt;
}
network::RegionSessionsConfig config;
config.transport.listen_address = options.listen.substr(0, colon);
config.transport.port = port;
config.transport.certificate_pem_file = options.certificate;
config.transport.key_pem_file = options.key;
config.control.environment = options.environment == "dev" ? network::DeploymentEnvironment::dev
: options.environment == "stage"
? network::DeploymentEnvironment::stage
: network::DeploymentEnvironment::world0;
config.control.world = {key.world, frame};
config.control.region = key.region;
config.control.epoch = 1;
config.control.max_sessions = 512;
config.transport.limits.max_connections = 576;
const auto life_routes = server_app::parse_life_owner_routes(routing);
if (!life_routes) {
std::puts("error primitive-life-routes");
return std::nullopt;
}
config.life_owner_routes = *life_routes;
network::GrantPublicKey issuer_key{};
std::copy(issuer.begin(), issuer.end(), issuer_key.begin());
config.issuer_keys.push_back({1, issuer_key});
bridge.slot = std::make_unique<server::PgSlot>(pg, std::chrono::seconds{2});
bridge.ledger = std::make_unique<server::PostgresControlLedger>(
*bridge.slot, key, [&host] { return host.authority(); });
config.ledger = bridge.ledger.get();
config.create_primitive_life = [&host](const network::AccountId& account,
std::uint64_t generation, std::uint64_t request) {
const auto result =
host.new_life(account.bytes, generation, request, biology::founder_life_rules);
return primitive_wire_outcome(request, result);
};
config.query_primitive_life = [&host](const network::AccountId& account,
std::uint64_t generation, std::uint64_t request) {
return primitive_wire_outcome(request, host.query_life(account.bytes, generation, request));
};
auto sessions = server::RegionSessions::create(std::move(config));
if (!sessions) {
std::puts(sessions.error() == server::RegionSessionsError::transport
? "error primitive-listen"
: "error primitive-session");
return std::nullopt;
}
bridge.sessions = std::move(*sessions);
std::printf("primitive listening %u\n", static_cast<unsigned>(bridge.sessions->port()));
return bridge;
}
} // namespace
#endif
int serve_primitive(Options& o) {
auto objects = persistence::FilesystemObjectStore::open(
o.objects, o.host.key, server::max_primitive_lives_bytes, nullptr,
o.objects_reserve_mib * 1024 * 1024);
if (!objects) {
std::puts("error objects");
return 2;
}
// The unique_ptr moves into the host, but the store object itself stays put.
const auto* objects_view = objects->get();
const auto pg = o.pg;
auto made = server::PrimitiveRegionHost::create(
{o.host.key, o.host.frame, o.host.lease, o.host.build, o.host.retained_builds},
[pg] { return persistence::Postgres::connect(pg, 2s); }, std::move(*objects));
if (!made) {
std::puts("error config");
return 2;
}
auto& host = **made;
#ifdef EVO_SERVER_M10
auto primitive_sessions =
open_primitive_sessions(o.primitive_session, o.m10, host, o.host.key, o.host.frame, o.pg);
if (!primitive_sessions)
return 2;
#endif
std::unique_ptr<server::OpsServer> ops;
if (!o.ops_listen.empty()) {
auto listening = server::OpsServer::listen(o.ops_listen);
if (!listening) {
std::puts("error ops-listen");
return 2;
}
ops = std::move(*listening);
std::printf("ops listening %u\n", static_cast<unsigned>(ops->port()));
}
server::OpsCounters counters;
counters.objects_reserve_bytes = o.objects_reserve_mib * 1024 * 1024;
bool ready = false;
std::optional<server::HostPhase> phase;
const auto set_ready = [&](bool now) {
if (ready != now) {
ready = now;
std::printf("ready %d\n", ready ? 1 : 0);
}
if (!o.ready_file.empty()) {
std::error_code ignored;
if (ready)
std::ofstream(o.ready_file) << "ready\n";
else
std::filesystem::remove(o.ready_file, ignored);
}
};
const auto finish = [&](int code) {
set_ready(false);
const auto& status = host.status();
std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n",
server::phase_name(status.phase), static_cast<unsigned long long>(status.epoch),
static_cast<unsigned long long>(status.committed_tick),
static_cast<unsigned long long>(status.simulated_tick));
std::printf("stats commits=%llu store_failures=%llu\n",
static_cast<unsigned long long>(status.commits),
static_cast<unsigned long long>(status.store_failures));
std::printf("exit %d\n", code);
std::fflush(stdout);
return code;
};
for (;;) {
if (stop_requested) {
#ifdef EVO_SERVER_M10
if (primitive_sessions->sessions)
primitive_sessions->sessions->shutdown(std::chrono::steady_clock::now());
#endif
host.drain();
return finish(0);
}
const auto& status = host.step();
#ifdef EVO_SERVER_M10
if (primitive_sessions->sessions)
primitive_sessions->sessions->poll(
{status.ready, status.epoch, status.simulated_tick},
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count()),
std::chrono::steady_clock::now());
#endif
set_ready(status.ready);
if (!phase || *phase != status.phase) {
phase = status.phase;
std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n",
server::phase_name(status.phase),
static_cast<unsigned long long>(status.epoch),
static_cast<unsigned long long>(status.committed_tick),
static_cast<unsigned long long>(status.simulated_tick));
}
if (ops) {
const auto handler = [&](std::string_view path) -> server::OpsResponse {
const auto& current = host.status();
if (path == "/livez")
return {200, "live\n"};
if (path == "/readyz")
return current.ready ? server::OpsResponse{200, "ready\n"}
: server::OpsResponse{
503, std::string("not ready: ") +
server::phase_name(current.phase) + "\n"};
if (path == "/metrics") {
counters.objects_available_bytes = objects_view->available_bytes().value_or(0);
return {200, server::render_metrics(current, counters),
"text/plain; version=0.0.4; charset=utf-8"};
}
return {404, "not found\n"};
};
ops->serve(handler);
}
std::fflush(stdout);
if (status.phase == server::HostPhase::quarantined)
return finish(4);
std::this_thread::sleep_for(5ms);
}
}
#ifdef EVO_SERVER_M10
int serve_cellular(Options& o) {
auto objects = persistence::FilesystemObjectStore::open(
o.objects, o.host.key, server::max_cellular_lives_bytes, nullptr,
o.objects_reserve_mib * 1024 * 1024);
if (!objects) {
std::puts("error objects");
return 2;
}
const auto pg = o.pg;
auto made = server::CellularRegionHost::create(
{o.host.key, o.host.frame, o.host.lease, o.host.build, o.host.retained_builds},
[pg] { return persistence::Postgres::connect(pg, 2s); }, std::move(*objects));
if (!made) {
std::puts("error config");
return 2;
}
auto& host = **made;
auto bridge = server_app::CellularBridge::open(o.m10, host, o.host.key, o.host.frame, o.pg);
if (!bridge)
return 2;
std::unique_ptr<server::OpsServer> ops;
if (!o.ops_listen.empty()) {
auto listening = server::OpsServer::listen(o.ops_listen);
if (!listening) {
std::puts("error ops-listen");
return 2;
}
ops = std::move(*listening);
std::printf("ops listening %u\n", static_cast<unsigned>(ops->port()));
}
bool ready = false;
std::optional<server::HostPhase> phase;
std::optional<std::chrono::steady_clock::time_point> next_tick;
std::vector<network::CarriedAction> awaiting_outcomes;
const auto set_ready = [&](bool now) {
if (ready != now) {
ready = now;
std::printf("ready %d\n", ready ? 1 : 0);
}
if (!o.ready_file.empty()) {
std::error_code ignored;
if (ready)
std::ofstream(o.ready_file) << "ready\n";
else
std::filesystem::remove(o.ready_file, ignored);
}
};
const auto deliver_finalized = [&] {
for (const auto& finalized : host.take_finalized_actions()) {
const auto pending = std::ranges::find_if(awaiting_outcomes, [&](const auto& action) {
return action.account.bytes == finalized.action.account &&
action.action_id == finalized.action.action_id;
});
if (pending == awaiting_outcomes.end())
continue;
const std::array committed{*pending};
bridge->committed(committed, finalized.commit_sequence);
awaiting_outcomes.erase(pending);
}
};
const auto finish = [&](int code) {
set_ready(false);
const auto& status = host.status();
std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n",
server::phase_name(status.phase), static_cast<unsigned long long>(status.epoch),
static_cast<unsigned long long>(status.committed_tick),
static_cast<unsigned long long>(status.simulated_tick));
std::printf("stats commits=%llu store_failures=%llu\n",
static_cast<unsigned long long>(status.commits),
static_cast<unsigned long long>(status.store_failures));
std::printf("exit %d\n", code);
std::fflush(stdout);
return code;
};
for (;;) {
if (stop_requested) {
bridge->shutdown();
host.drain();
return finish(0);
}
const auto& status = host.step();
const auto wall =
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count());
static_cast<void>(bridge->accepting(wall));
bridge->poll(status, wall);
deliver_finalized();
set_ready(status.ready);
if (!phase || *phase != status.phase) {
phase = status.phase;
std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n",
server::phase_name(status.phase),
static_cast<unsigned long long>(status.epoch),
static_cast<unsigned long long>(status.committed_tick),
static_cast<unsigned long long>(status.simulated_tick));
}
if (ops) {
const auto handler = [&](std::string_view path) -> server::OpsResponse {
const auto& current = host.status();
if (path == "/livez")
return {200, "live\n"};
if (path == "/readyz")
return current.ready ? server::OpsResponse{200, "ready\n"}
: server::OpsResponse{
503, std::string("not ready: ") +
server::phase_name(current.phase) + "\n"};
if (path == "/metrics")
return {200, server::render_metrics(current, {})};
return {404, "not found\n"};
};
ops->serve(handler);
}
if (status.phase == server::HostPhase::quarantined)
return finish(4);
const auto* state = host.state();
bool player_present = false;
if (status.ready && state) {
for (const auto& life : state->lives()) {
network::AccountId account;
account.bytes = life.account;
if (bridge->sessions().holder(account)) {
player_present = true;
break;
}
}
}
const bool birth_waiting = !host.queued_births().empty();
if (status.ready && state && (player_present || birth_waiting)) {
const auto now = std::chrono::steady_clock::now();
if (!next_tick)
next_tick = now + std::chrono::milliseconds{50};
if (now >= *next_tick && state->tick() != UINT64_MAX) {
const auto tick = state->tick() + 1;
std::vector<server::CellularLifeIntent> baseline;
baseline.reserve(state->lives().size());
for (const auto& life : state->lives())
baseline.push_back({life.identity.life, {}});
std::ranges::sort(baseline, {}, &server::CellularLifeIntent::life_id);
auto actions = bridge->carry(tick);
auto builder =
server::CellularSessionBatchBuilder::create(*state, host.committed()->head);
if (!builder) {
bridge->dropped(actions, true);
} else {
const auto births = host.queued_births().first(
std::min(host.queued_births().size(), biology::max_cellular_tick_lives));
const auto appended = builder->append(tick, baseline, actions, births);
if (!appended) {
bridge->dropped(actions, true);
} else {
awaiting_outcomes.insert(awaiting_outcomes.end(), actions.begin(),
actions.end());
auto published = host.publish(builder->batch(), builder->actions());
if (!published && host.status().phase != server::HostPhase::resolving) {
bridge->dropped(actions, true);
std::erase_if(awaiting_outcomes, [&](const auto& action) {
return std::ranges::find(actions, action) != actions.end();
});
}
deliver_finalized();
}
}
*next_tick = std::chrono::steady_clock::now() +
std::chrono::microseconds{static_cast<std::chrono::microseconds::rep>(
biology::cellular_motion_tick_microseconds)};
}
} else {
next_tick.reset(); // A disconnected life does not accumulate offline ticks.
}
if (const auto* committed_state = host.state(); committed_state)
bridge->replicate(*committed_state);
std::fflush(stdout);
std::this_thread::sleep_for(2ms);
}
}
#endif
} // namespace
int main(int argc, char** argv) {
auto options = parse(argc, argv);
if (!options) {
std::fputs("usage: evo-server --profile lineage|environmental|primitive|"
"cellular --world N --region N "
"--frame N --pg-host H --pg-port P --pg-database D --pg-user U "
"--objects DIR --build HEX64 [--retained HEX64]... [--pg-hostaddr A "
"--pg-passfile F --pg-ca F] [--flush-ms N] [--drain-ms N] "
"[--lease-profile default|fast] [--ready-file PATH] "
"[--primitive-listen ADDRESS:PORT --primitive-cert F --primitive-key F "
"--primitive-issuer F --primitive-environment dev|stage|world0] "
"[--ops-listen ADDRESS:PORT] [--synthetic-inputs development-v1 "
"[--synthetic-until TICK] [--accelerate N] [--session-listen ADDRESS:PORT "
"--session-cert F --session-key F --session-grant-key F "
"--session-grant-ledger F --session-environment dev]] [--m10-listen "
"ADDRESS:PORT --m10-cert F --m10-key F --m10-issuer F "
"--m10-environment dev|stage|world0 --session-window ACTIVATION:UTC_EXPIRY "
"[--m10-max-grant-lifetime 60..14400] [--m10-area X0,Y0,X1,Y1] "
"[--m10-handoff-key F [--m10-neighbour "
"REGION=X0,Y0,X1,Y1=HOST:PORT=NAME=PUBFILE]...] "
"[--life-route REGION=HOST:PORT=TLS_NAME ...] "
"[--m10-load-governor on|off]]\\n",
stderr);
return 2;
}
struct sigaction action{};
action.sa_handler = on_stop;
sigemptyset(&action.sa_mask);
sigaction(SIGTERM, &action, nullptr);
sigaction(SIGINT, &action, nullptr);
std::signal(SIGPIPE, SIG_IGN);
if (options->profile == "primitive")
return serve_primitive(*options);
#ifdef EVO_SERVER_M10
if (options->profile == "cellular")
return serve_cellular(*options);
#endif
return options->profile == "lineage" ? serve<processes::LineageRegion>(*options)
: serve<processes::EnvironmentalCommunity>(*options);
}