ADR-0065: before any readiness, recovery now verifies the head's tail. The trace object must exist with its manifest digest, its header must bind profile, region, predecessor and range; the predecessor (schema v6 load_region_commit) must hash to the digest the head names; replaying the trace from the predecessor's verified payload must reproduce every report and the exact head bytes; and trace-derived decisions must equal the head's durable decisions. Missing, corrupt or non-reproducing tails quarantine (RecoveryError::tail); nothing older or empty is served. Test-only crash seams (HostConfig::crash_hook, PublishHooks) and evo-server --crash-at deliver real SIGKILL at before-upload, before-transaction, after-commit and after-ack. The kill matrix, a SIGSTOP/SIGCONT stale owner and damaged-head startups all behave as the durability contract requires; successors verify before readiness. Validated: PostgreSQL suites under GCC Release, clang-tidy and ASan/UBSan; tail-skip and replay-equality mutations each caught. No Docker/Gitea CI, production-lease kill test or deployment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
414 lines
16 KiB
C++
414 lines
16 KiB
C++
// M7.6 region server process (ADR-0064). Development host for one region: tick
|
|
// inputs arrive as lines on stdin (the network gateway is later work); events are
|
|
// written as lines on stdout. Linux only; requires the PostgreSQL build.
|
|
//
|
|
// Input: input <request:32 hex> <temperature_mK> <source_enabled> <bite0..4> <terminal0..1>
|
|
// Output: phase <name> epoch=<e> committed=<t> simulated=<t>
|
|
// ready <0|1>
|
|
// rejected <request> <reason> | dropped <request> <reason>
|
|
// ack <request> <tick> <commit_sequence>
|
|
// outcome <tick> <actor> <organism> <commit_sequence>
|
|
// stats max_queued=<n> max_tick_debt=<n> commits=<n> store_failures=<n>
|
|
// Exit: 0 drained, 2 usage, 3 drain deadline passed, 4 quarantined.
|
|
//
|
|
// --crash-at <point>:<n> (tests only, M7.7) SIGKILLs this process at the n-th
|
|
// occurrence of before-upload, before-transaction, after-commit or after-ack (right
|
|
// after a step's acknowledgements are written and flushed).
|
|
#include "evo/server/region_host.hpp"
|
|
|
|
#include <algorithm>
|
|
#include <array>
|
|
#include <cerrno>
|
|
#include <charconv>
|
|
#include <csignal>
|
|
#include <cstdio>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <optional>
|
|
#include <poll.h>
|
|
#include <signal.h>
|
|
#include <string>
|
|
#include <string_view>
|
|
#include <unistd.h>
|
|
#include <vector>
|
|
|
|
namespace {
|
|
using namespace evo;
|
|
using namespace std::chrono_literals;
|
|
volatile std::sig_atomic_t stop_requested = 0;
|
|
extern "C" void on_stop(int) {
|
|
stop_requested = 1;
|
|
}
|
|
|
|
struct Options {
|
|
std::string profile{"lineage"};
|
|
persistence::PgOptions pg;
|
|
std::filesystem::path objects, ready_file;
|
|
server::HostConfig host;
|
|
std::chrono::milliseconds drain{10000};
|
|
std::string crash_point;
|
|
std::uint64_t crash_ordinal{};
|
|
};
|
|
|
|
// Kills this process without any cleanup: the real SIGKILL, at a chosen boundary.
|
|
[[noreturn]] void crash_now() {
|
|
std::fflush(stdout);
|
|
::kill(::getpid(), SIGKILL);
|
|
for (;;)
|
|
::pause();
|
|
}
|
|
|
|
std::optional<std::uint64_t> number(std::string_view text) {
|
|
std::uint64_t value{};
|
|
const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), value);
|
|
if (error != std::errc{} || end != text.data() + text.size())
|
|
return std::nullopt;
|
|
return value;
|
|
}
|
|
template <std::size_t N> std::optional<std::array<std::uint8_t, N>> hex(std::string_view text) {
|
|
if (text.size() != N * 2)
|
|
return std::nullopt;
|
|
std::array<std::uint8_t, N> out{};
|
|
for (std::size_t i = 0; i < N; ++i) {
|
|
unsigned value{};
|
|
const auto [end, error] =
|
|
std::from_chars(text.data() + (2 * i), text.data() + (2 * i) + 2, value, 16);
|
|
if (error != std::errc{} || end != text.data() + (2 * i) + 2)
|
|
return std::nullopt;
|
|
out[i] = static_cast<std::uint8_t>(value);
|
|
}
|
|
return out;
|
|
}
|
|
template <std::size_t N> std::string text(const std::array<std::uint8_t, N>& bytes) {
|
|
constexpr char digits[] = "0123456789abcdef";
|
|
std::string out;
|
|
for (const auto b : bytes) {
|
|
out.push_back(digits[b >> 4U]);
|
|
out.push_back(digits[b & 0x0FU]);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
std::optional<Options> parse(int argc, char** argv) {
|
|
Options o;
|
|
o.host.retained_builds.clear();
|
|
for (int i = 1; i < argc; ++i) {
|
|
const std::string_view flag = argv[i];
|
|
if (i + 1 >= argc)
|
|
return std::nullopt;
|
|
const std::string_view value = argv[++i];
|
|
const auto n = number(value);
|
|
if (flag == "--profile" && (value == "lineage" || value == "environmental"))
|
|
o.profile = value;
|
|
else if (flag == "--world" && n)
|
|
o.host.key.world = *n;
|
|
else if (flag == "--region" && n)
|
|
o.host.key.region = *n;
|
|
else if (flag == "--frame" && n)
|
|
o.host.frame = *n;
|
|
else if (flag == "--pg-host")
|
|
o.pg.host = value;
|
|
else if (flag == "--pg-hostaddr")
|
|
o.pg.hostaddr = value;
|
|
else if (flag == "--pg-port")
|
|
o.pg.port = value;
|
|
else if (flag == "--pg-database")
|
|
o.pg.database = value;
|
|
else if (flag == "--pg-user")
|
|
o.pg.user = value;
|
|
else if (flag == "--pg-passfile")
|
|
o.pg.password_file = value;
|
|
else if (flag == "--pg-ca")
|
|
o.pg.ca_file = value;
|
|
else if (flag == "--objects")
|
|
o.objects = value;
|
|
else if (flag == "--ready-file")
|
|
o.ready_file = value;
|
|
else if (flag == "--build" || flag == "--retained") {
|
|
const auto digest = hex<32>(value);
|
|
if (!digest)
|
|
return std::nullopt;
|
|
if (flag == "--build")
|
|
o.host.build = *digest;
|
|
else
|
|
o.host.retained_builds.push_back(*digest);
|
|
} else if (flag == "--flush-ms" && n && *n > 0 && *n <= 60000)
|
|
o.host.flush_interval = std::chrono::milliseconds{*n};
|
|
else if (flag == "--drain-ms" && n && *n > 0 && *n <= 600000)
|
|
o.drain = std::chrono::milliseconds{*n};
|
|
else if (flag == "--lease-profile" && value == "fast") // Development/test only.
|
|
o.host.lease = {4000ms, 800ms, 400ms, 400ms, 1000ms};
|
|
else if (flag == "--lease-profile" && value == "default")
|
|
o.host.lease = {};
|
|
else if (flag == "--crash-at") {
|
|
const auto colon = value.find(':');
|
|
const auto ordinal =
|
|
colon == std::string_view::npos ? std::nullopt : number(value.substr(colon + 1));
|
|
const auto point = value.substr(0, colon);
|
|
if (!ordinal || *ordinal == 0 ||
|
|
(point != "before-upload" && point != "before-transaction" &&
|
|
point != "after-commit" && point != "after-ack"))
|
|
return std::nullopt;
|
|
o.crash_point = point;
|
|
o.crash_ordinal = *ordinal;
|
|
} else
|
|
return std::nullopt;
|
|
}
|
|
if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end())
|
|
o.host.retained_builds.push_back(o.host.build);
|
|
return o;
|
|
}
|
|
|
|
std::optional<server::TickInput> parse_input(std::string_view line) {
|
|
std::array<std::string_view, 11> fields{};
|
|
std::size_t count = 0;
|
|
while (!line.empty() && count < fields.size() + 1) {
|
|
const auto start = line.find_first_not_of(' ');
|
|
if (start == std::string_view::npos)
|
|
break;
|
|
line.remove_prefix(start);
|
|
const auto end = line.find(' ');
|
|
if (count == fields.size())
|
|
return std::nullopt; // Too many fields.
|
|
fields[count++] = line.substr(0, end);
|
|
line.remove_prefix(end == std::string_view::npos ? line.size() : end);
|
|
}
|
|
if (count != fields.size() || fields[0] != "input")
|
|
return std::nullopt;
|
|
auto request = hex<16>(fields[1]);
|
|
if (!request)
|
|
return std::nullopt;
|
|
std::array<std::uint64_t, 9> v{};
|
|
for (std::size_t i = 0; i < v.size(); ++i) {
|
|
const auto n = number(fields[i + 2]);
|
|
if (!n)
|
|
return std::nullopt;
|
|
v[i] = *n;
|
|
}
|
|
server::TickInput in{*request, {}};
|
|
in.input.temperature_mK = v[0];
|
|
in.input.source_enabled = v[1];
|
|
for (std::size_t k = 0; k < 5; ++k)
|
|
in.input.bite[k] = v[2 + k];
|
|
in.input.terminal = {v[7], v[8]};
|
|
return in;
|
|
}
|
|
|
|
const char* phase_name(server::HostPhase p) {
|
|
switch (p) {
|
|
case server::HostPhase::acquiring:
|
|
return "acquiring";
|
|
case server::HostPhase::running:
|
|
return "running";
|
|
case server::HostPhase::resolving:
|
|
return "resolving";
|
|
case server::HostPhase::draining:
|
|
return "draining";
|
|
case server::HostPhase::stopped:
|
|
return "stopped";
|
|
case server::HostPhase::quarantined:
|
|
return "quarantined";
|
|
}
|
|
return "unknown";
|
|
}
|
|
const char* admission_name(server::AdmissionError e) {
|
|
switch (e) {
|
|
case server::AdmissionError::not_ready:
|
|
return "not_ready";
|
|
case server::AdmissionError::draining:
|
|
return "draining";
|
|
case server::AdmissionError::full:
|
|
return "full";
|
|
case server::AdmissionError::duplicate:
|
|
return "duplicate";
|
|
case server::AdmissionError::invalid:
|
|
return "invalid";
|
|
}
|
|
return "unknown";
|
|
}
|
|
const char* dropped_name(server::Unacknowledged u) {
|
|
switch (u) {
|
|
case server::Unacknowledged::refused:
|
|
return "refused";
|
|
case server::Unacknowledged::drained:
|
|
return "drained";
|
|
case server::Unacknowledged::fenced:
|
|
return "fenced";
|
|
case server::Unacknowledged::quarantined:
|
|
return "quarantined";
|
|
}
|
|
return "unknown";
|
|
}
|
|
|
|
// Reads complete, bounded lines from stdin without blocking.
|
|
class LineReader {
|
|
public:
|
|
// False on EOF/error.
|
|
bool fill() {
|
|
std::array<char, 4096> chunk{};
|
|
const auto n = ::read(STDIN_FILENO, chunk.data(), chunk.size());
|
|
if (n <= 0)
|
|
return n < 0 && errno == EINTR;
|
|
buffer_.append(chunk.data(), static_cast<std::size_t>(n));
|
|
return true;
|
|
}
|
|
std::optional<std::string> next() {
|
|
const auto end = buffer_.find('\n');
|
|
if (end == std::string::npos) {
|
|
if (buffer_.size() > 512)
|
|
buffer_.clear(); // Overlong line: discarded, never partially parsed.
|
|
return std::nullopt;
|
|
}
|
|
auto line = buffer_.substr(0, end);
|
|
buffer_.erase(0, end + 1);
|
|
return line;
|
|
}
|
|
|
|
private:
|
|
std::string buffer_;
|
|
};
|
|
|
|
template <typename State> int serve(Options& o) {
|
|
if (!o.crash_point.empty() && o.crash_point != "after-ack") {
|
|
o.host.crash_hook = [point = o.crash_point, ordinal = o.crash_ordinal,
|
|
seen = std::uint64_t{0}](server::CrashPoint at) mutable {
|
|
const char* name = at == server::CrashPoint::before_upload ? "before-upload"
|
|
: at == server::CrashPoint::before_transaction ? "before-transaction"
|
|
: "after-commit";
|
|
if (point == name && ++seen == ordinal)
|
|
crash_now();
|
|
};
|
|
}
|
|
std::uint64_t ack_steps = 0;
|
|
auto objects = persistence::FilesystemObjectStore::open(o.objects, o.host.key,
|
|
persistence::max_lineage_payload_bytes);
|
|
if (!objects) {
|
|
std::puts("error objects");
|
|
return 2;
|
|
}
|
|
const auto pg = o.pg;
|
|
auto made = server::RegionHost<State>::create(
|
|
o.host, [pg] { return persistence::Postgres::connect(pg, 2s); }, std::move(*objects));
|
|
if (!made) {
|
|
std::puts("error config");
|
|
return 2;
|
|
}
|
|
auto& host = **made;
|
|
LineReader reader;
|
|
bool input_open = true, ready = false;
|
|
std::optional<std::string> pending;
|
|
std::optional<server::HostPhase> phase;
|
|
std::size_t max_queued = 0;
|
|
std::optional<std::chrono::steady_clock::time_point> drain_deadline;
|
|
const auto set_ready = [&](bool now) {
|
|
if (now == ready)
|
|
return;
|
|
ready = now;
|
|
std::printf("ready %d\n", ready ? 1 : 0);
|
|
if (!o.ready_file.empty()) {
|
|
std::error_code ignored;
|
|
if (ready)
|
|
std::ofstream(o.ready_file) << "ready\n";
|
|
else
|
|
std::filesystem::remove(o.ready_file, ignored);
|
|
}
|
|
};
|
|
for (;;) {
|
|
if (stop_requested && !drain_deadline) {
|
|
host.drain();
|
|
drain_deadline = std::chrono::steady_clock::now() + o.drain;
|
|
}
|
|
// Read only while the bounded backlog has room: stdin itself backpressures.
|
|
const bool room = host.status().queued < o.host.max_queued_inputs && !pending;
|
|
pollfd fd{STDIN_FILENO, POLLIN, 0};
|
|
if (input_open && room && !drain_deadline && ::poll(&fd, 1, 5) > 0)
|
|
input_open = reader.fill();
|
|
else if (!input_open || !room || drain_deadline)
|
|
::usleep(2000);
|
|
while (!drain_deadline) {
|
|
if (!pending)
|
|
pending = reader.next();
|
|
if (!pending)
|
|
break;
|
|
auto in = parse_input(*pending);
|
|
if (!in) {
|
|
std::puts("rejected - malformed");
|
|
pending.reset();
|
|
continue;
|
|
}
|
|
auto admitted = host.submit(*in);
|
|
if (!admitted && admitted.error() == server::AdmissionError::full)
|
|
break; // Keep the line and retry after ticks drain the backlog.
|
|
if (!admitted)
|
|
std::printf("rejected %s %s\n", text(in->request).c_str(),
|
|
admission_name(admitted.error()));
|
|
pending.reset();
|
|
}
|
|
const auto& s = host.step(std::chrono::steady_clock::now());
|
|
max_queued = std::max(max_queued, s.queued);
|
|
if (!phase || *phase != s.phase) {
|
|
phase = s.phase;
|
|
std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n", phase_name(s.phase),
|
|
static_cast<unsigned long long>(s.epoch),
|
|
static_cast<unsigned long long>(s.committed_tick),
|
|
static_cast<unsigned long long>(s.simulated_tick));
|
|
}
|
|
set_ready(s.ready);
|
|
const auto acknowledged = host.take_acks();
|
|
for (const auto& a : acknowledged)
|
|
std::printf("ack %s %llu %llu\n", text(a.request).c_str(),
|
|
static_cast<unsigned long long>(a.tick),
|
|
static_cast<unsigned long long>(a.commit_sequence));
|
|
for (const auto& x : host.take_outcomes())
|
|
std::printf("outcome %llu %u %llu %llu\n",
|
|
static_cast<unsigned long long>(x.decision.tick),
|
|
static_cast<unsigned>(x.decision.actor),
|
|
static_cast<unsigned long long>(x.decision.organism),
|
|
static_cast<unsigned long long>(x.commit_sequence));
|
|
for (const auto& d : host.take_dropped())
|
|
std::printf("dropped %s %s\n", text(d.input.request).c_str(), dropped_name(d.reason));
|
|
std::fflush(stdout);
|
|
if (o.crash_point == "after-ack" && !acknowledged.empty() && ++ack_steps == o.crash_ordinal)
|
|
crash_now();
|
|
const auto finish = [&](int code) {
|
|
set_ready(false);
|
|
std::printf(
|
|
"stats max_queued=%zu max_tick_debt=%llu commits=%llu store_failures=%llu\n",
|
|
max_queued, static_cast<unsigned long long>(s.max_tick_debt),
|
|
static_cast<unsigned long long>(s.commits),
|
|
static_cast<unsigned long long>(s.store_failures));
|
|
std::printf("exit %d\n", code);
|
|
std::fflush(stdout);
|
|
return code;
|
|
};
|
|
if (s.phase == server::HostPhase::stopped)
|
|
return finish(0);
|
|
if (s.phase == server::HostPhase::quarantined)
|
|
return finish(4);
|
|
if (drain_deadline && std::chrono::steady_clock::now() > *drain_deadline)
|
|
return finish(3); // Pending/unknown work is left for the next owner to reconcile.
|
|
}
|
|
}
|
|
} // namespace
|
|
|
|
int main(int argc, char** argv) {
|
|
auto options = parse(argc, argv);
|
|
if (!options) {
|
|
std::fputs("usage: evo-server --profile lineage|environmental --world N --region N "
|
|
"--frame N --pg-host H --pg-port P --pg-database D --pg-user U "
|
|
"--objects DIR --build HEX64 [--retained HEX64]... [--pg-hostaddr A "
|
|
"--pg-passfile F --pg-ca F] [--flush-ms N] [--drain-ms N] "
|
|
"[--lease-profile default|fast] [--ready-file PATH]\n",
|
|
stderr);
|
|
return 2;
|
|
}
|
|
struct sigaction action{};
|
|
action.sa_handler = on_stop;
|
|
sigemptyset(&action.sa_mask);
|
|
sigaction(SIGTERM, &action, nullptr);
|
|
sigaction(SIGINT, &action, nullptr);
|
|
std::signal(SIGPIPE, SIG_IGN);
|
|
return options->profile == "lineage" ? serve<processes::LineageRegion>(*options)
|
|
: serve<processes::EnvironmentalCommunity>(*options);
|
|
}
|