// M7.6 region server process (ADR-0064). Development host for one region: tick // inputs arrive as lines on stdin (the network gateway is later work); events are // written as lines on stdout. Linux only; requires the PostgreSQL build. // // Input: input // Output: phase epoch= committed= simulated= // ready <0|1> // rejected | dropped // ack // outcome // stats max_queued= max_tick_debt= commits= store_failures= // Exit: 0 drained, 2 usage, 3 drain deadline passed, 4 quarantined. // // --pg-service (M8.3) takes host, hostaddr, port, database, user and CA from // the [evolution] section of a libpq service file; anything else in it is refused. // // --objects-reserve-mib (M8.3, default 1024) refuses new objects that would leave // less free space on the object volume: node-local volumes do not enforce their // requested size. A refused upload is a store failure; nothing is acknowledged. // // --ops-listen (M7.8) serves GET /livez (the loop is alive; says // nothing about authority), /readyz (200 only with current authority) and /metrics // (Prometheus text). It prints "ops listening " once bound. // // --synthetic-inputs development-v1 (dev/stage drills only, M8.6, ADR-0073) replaces // stdin with a deterministic driver: whenever the host is running it keeps the // backlog fed with development_input(t) for the ticks after the committed head, // restarting from the committed tick after every recovery. Request IDs are // "SYN1" + region + tick, so acknowledgements can be checked across owners. // // Upgrade rehearsal on clones only (M8.9, ADR-0076), both requiring // --synthetic-inputs: --synthetic-until stops the driver after that tick, so // the region commits exactly up to it and then idles (inputs are never invented); // --accelerate <2..1000> runs the region's scheduling clock that many times faster // than real time (lease deadlines stay real). Never in a deployed environment. // // Dev sessions (M9.7, ADR-0083; builds with EVO_BUILD_NETWORK only): --session-listen // --session-cert F --session-key F --session-grant-key F // --session-grant-ledger F --session-environment dev serve the TLS 1.3 dev session // protocol. The ledger durably records consumed grants (on a persistent volume), so a // restarted process refuses them. They require // --synthetic-inputs: an admitted intention rides the next synthetic input (content // unchanged), so its final outcome is the input's durable acknowledgement, with no // biological effect. The driver then looks ahead 8 ticks instead of 64. Prints // "session listening " and "session opened|closed|refused ...". Any environment // other than dev is refused: the dev identity cannot exist elsewhere. // // --crash-at : (tests only, M7.7) SIGKILLs this process at the n-th // occurrence of before-upload, before-transaction, after-commit or after-ack (right // after a step's acknowledgements are written and flushed). #include "evo/processes/development_opening.hpp" #include "evo/server/ops_http.hpp" #include "evo/server/region_host.hpp" #include "m10_bridge.hpp" #include "session_bridge.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace { using namespace evo; using namespace std::chrono_literals; volatile std::sig_atomic_t stop_requested = 0; extern "C" void on_stop(int) { stop_requested = 1; } struct Options { std::string profile{"lineage"}; persistence::PgOptions pg; std::filesystem::path objects, ready_file; server::HostConfig host; std::chrono::milliseconds drain{10000}; std::string crash_point; std::uint64_t crash_ordinal{}; std::string ops_listen; bool synthetic{}; std::uint64_t synthetic_until{}; // 0: unbounded. std::uint64_t accelerate{1}; std::uint64_t objects_reserve_mib{1024}; server_app::SessionOptions session; server_app::M10Options m10; }; // Kills this process without any cleanup: the real SIGKILL, at a chosen boundary. [[noreturn]] void crash_now() { std::fflush(stdout); ::kill(::getpid(), SIGKILL); for (;;) ::pause(); } std::optional number(std::string_view text) { std::uint64_t value{}; const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), value); if (error != std::errc{} || end != text.data() + text.size()) return std::nullopt; return value; } template std::optional> hex(std::string_view text) { if (text.size() != N * 2) return std::nullopt; std::array out{}; for (std::size_t i = 0; i < N; ++i) { unsigned value{}; const auto [end, error] = std::from_chars(text.data() + (2 * i), text.data() + (2 * i) + 2, value, 16); if (error != std::errc{} || end != text.data() + (2 * i) + 2) return std::nullopt; out[i] = static_cast(value); } return out; } template std::string text(const std::array& bytes) { constexpr char digits[] = "0123456789abcdef"; std::string out; for (const auto b : bytes) { out.push_back(digits[b >> 4U]); out.push_back(digits[b & 0x0FU]); } return out; } std::optional parse(int argc, char** argv) { Options o; o.host.retained_builds.clear(); for (int i = 1; i < argc; ++i) { const std::string_view flag = argv[i]; if (i + 1 >= argc) return std::nullopt; const std::string_view value = argv[++i]; const auto n = number(value); if (flag == "--profile" && (value == "lineage" || value == "environmental")) o.profile = value; else if (flag == "--world" && n) o.host.key.world = *n; else if (flag == "--region" && n) o.host.key.region = *n; else if (flag == "--frame" && n) o.host.frame = *n; else if (flag == "--pg-host") o.pg.host = value; else if (flag == "--pg-hostaddr") o.pg.hostaddr = value; else if (flag == "--pg-port") o.pg.port = value; else if (flag == "--pg-database") o.pg.database = value; else if (flag == "--pg-user") o.pg.user = value; else if (flag == "--pg-passfile") o.pg.password_file = value; else if (flag == "--pg-service") { // Connection from the [evolution] section of a libpq service file (M8.3); // the password file stays separate (--pg-passfile). auto service = persistence::read_pg_service(std::string(value), "evolution"); if (!service) return std::nullopt; service->password_file = o.pg.password_file; o.pg = *service; } else if (flag == "--pg-ca") o.pg.ca_file = value; else if (flag == "--objects") o.objects = value; else if (flag == "--ready-file") o.ready_file = value; else if (flag == "--build" || flag == "--retained") { const auto digest = hex<32>(value); if (!digest) return std::nullopt; if (flag == "--build") o.host.build = *digest; else o.host.retained_builds.push_back(*digest); } else if (flag == "--flush-ms" && n && *n > 0 && *n <= 60000) o.host.flush_interval = std::chrono::milliseconds{*n}; else if (flag == "--drain-ms" && n && *n > 0 && *n <= 600000) o.drain = std::chrono::milliseconds{*n}; else if (flag == "--lease-profile" && value == "fast") // Development/test only. o.host.lease = {4000ms, 800ms, 400ms, 400ms, 1000ms}; else if (flag == "--lease-profile" && value == "default") o.host.lease = {}; else if (flag == "--ops-listen") o.ops_listen = value; else if (flag == "--synthetic-inputs" && value == "development-v1") o.synthetic = true; else if (flag == "--synthetic-until" && n && *n > 0) o.synthetic_until = *n; else if (flag == "--accelerate" && n && *n >= 2 && *n <= 1000) o.accelerate = *n; else if (flag == "--objects-reserve-mib" && n && *n <= 1048576) o.objects_reserve_mib = *n; else if (flag == "--session-listen") o.session.listen = value; else if (flag == "--session-cert") o.session.certificate = value; else if (flag == "--session-key") o.session.key = value; else if (flag == "--session-grant-key") o.session.grant_key = value; else if (flag == "--session-grant-ledger") o.session.grant_ledger = value; else if (flag == "--session-environment" && (value == "dev" || value == "stage" || value == "world0")) o.session.environment = value; else if (flag == "--m10-listen") o.m10.listen = value; else if (flag == "--m10-cert") o.m10.certificate = value; else if (flag == "--m10-key") o.m10.key = value; else if (flag == "--m10-issuer") o.m10.issuer = value; else if (flag == "--m10-environment" && (value == "dev" || value == "stage" || value == "world0")) o.m10.environment = value; else if (flag == "--session-window") o.m10.session_window = value; else if (flag == "--m10-max-grant-lifetime" && n && *n >= 60 && *n <= 3600) o.m10.max_grant_lifetime = *n; else if (flag == "--crash-at") { const auto colon = value.find(':'); const auto ordinal = colon == std::string_view::npos ? std::nullopt : number(value.substr(colon + 1)); const auto point = value.substr(0, colon); if (!ordinal || *ordinal == 0 || (point != "before-upload" && point != "before-transaction" && point != "after-commit" && point != "after-ack")) return std::nullopt; o.crash_point = point; o.crash_ordinal = *ordinal; } else return std::nullopt; } // Every identity and connection setting is required: there is no default // region, database, object root or build to fall back on. if (o.host.key.world == 0 || o.host.key.region == 0 || o.host.frame == 0 || o.pg.host.empty() || o.pg.port.empty() || o.pg.database.empty() || o.pg.user.empty() || o.objects.empty() || o.host.build == persistence::Sha256Digest{}) return std::nullopt; // Rehearsal controls exist only for the synthetic driver (clones, drills). if ((o.synthetic_until != 0 || o.accelerate != 1) && !o.synthetic) return std::nullopt; // Sessions need every setting, the network build, and the synthetic carrier. if (o.session.any() && (!server_app::sessions_available || !o.session.complete() || !o.synthetic)) return std::nullopt; // M10 sessions (ADR-0091/0092) need every setting and the QUIC build. if (o.m10.any() && (!server_app::m10_available || !o.m10.complete())) return std::nullopt; if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end()) o.host.retained_builds.push_back(o.host.build); return o; } std::optional parse_input(std::string_view line) { std::array fields{}; std::size_t count = 0; while (!line.empty() && count < fields.size() + 1) { const auto start = line.find_first_not_of(' '); if (start == std::string_view::npos) break; line.remove_prefix(start); const auto end = line.find(' '); if (count == fields.size()) return std::nullopt; // Too many fields. fields[count++] = line.substr(0, end); line.remove_prefix(end == std::string_view::npos ? line.size() : end); } if (count != fields.size() || fields[0] != "input") return std::nullopt; auto request = hex<16>(fields[1]); if (!request) return std::nullopt; std::array v{}; for (std::size_t i = 0; i < v.size(); ++i) { const auto n = number(fields[i + 2]); if (!n) return std::nullopt; v[i] = *n; } server::TickInput in{*request, {}}; in.input.temperature_mK = v[0]; in.input.source_enabled = v[1]; for (std::size_t k = 0; k < 5; ++k) in.input.bite[k] = v[2 + k]; in.input.terminal = {v[7], v[8]}; return in; } const char* admission_name(server::AdmissionError e) { switch (e) { case server::AdmissionError::not_ready: return "not_ready"; case server::AdmissionError::draining: return "draining"; case server::AdmissionError::full: return "full"; case server::AdmissionError::duplicate: return "duplicate"; case server::AdmissionError::invalid: return "invalid"; } return "unknown"; } // Reads complete, bounded lines from stdin without blocking. class LineReader { public: // False on EOF/error. bool fill() { std::array chunk{}; const auto n = ::read(STDIN_FILENO, chunk.data(), chunk.size()); if (n <= 0) return n < 0 && errno == EINTR; buffer_.append(chunk.data(), static_cast(n)); return true; } std::optional next() { const auto end = buffer_.find('\n'); if (end == std::string::npos) { if (buffer_.size() > 512) buffer_.clear(); // Overlong line: discarded, never partially parsed. return std::nullopt; } auto line = buffer_.substr(0, end); buffer_.erase(0, end + 1); return line; } private: std::string buffer_; }; template int serve(Options& o) { if (!o.crash_point.empty() && o.crash_point != "after-ack") { o.host.crash_hook = [point = o.crash_point, ordinal = o.crash_ordinal, seen = std::uint64_t{0}](server::CrashPoint at) mutable { const char* name = at == server::CrashPoint::before_upload ? "before-upload" : at == server::CrashPoint::before_transaction ? "before-transaction" : "after-commit"; if (point == name && ++seen == ordinal) crash_now(); }; } std::uint64_t ack_steps = 0; auto objects = persistence::FilesystemObjectStore::open( o.objects, o.host.key, persistence::max_lineage_payload_bytes, nullptr, o.objects_reserve_mib * 1024 * 1024); if (!objects) { std::puts("error objects"); return 2; } const auto pg = o.pg; auto made = server::RegionHost::create( o.host, [pg] { return persistence::Postgres::connect(pg, 2s); }, std::move(*objects)); if (!made) { std::puts("error config"); return 2; } auto& host = **made; std::unique_ptr ops; if (!o.ops_listen.empty()) { auto listening = server::OpsServer::listen(o.ops_listen); if (!listening) { std::puts("error ops-listen"); return 2; } ops = std::move(*listening); std::printf("ops listening %u\n", static_cast(ops->port())); std::fflush(stdout); } auto sessions = server_app::SessionBridge::open(o.session, o.host.key, o.host.frame); if (!sessions) return 2; auto m10 = server_app::M10Bridge>::open(o.m10, host, o.host.key, o.host.frame, o.pg); if (!m10) return 2; std::fflush(stdout); const std::uint64_t lookahead = sessions->enabled() ? server_app::session_lookahead_ticks : 64; server::OpsCounters counters; counters.queue_capacity_inputs = o.host.max_queued_inputs; counters.queue_capacity_bytes = o.host.max_queued_bytes; counters.objects_reserve_bytes = o.objects_reserve_mib * 1024 * 1024; std::uint64_t last_sequence = 0; auto last_checkpoint = std::chrono::steady_clock::now(); bool recovered = false; const auto handler = [&](std::string_view path) -> server::OpsResponse { const auto& s = host.status(); if (path == "/livez") return {200, "live\n"}; if (path == "/readyz") return s.ready ? server::OpsResponse{200, "ready\n"} : server::OpsResponse{503, std::string("not ready: ") + server::phase_name(s.phase) + "\n"}; if (path == "/metrics") { counters.checkpoint_age_ms = static_cast(std::chrono::duration_cast( std::chrono::steady_clock::now() - last_checkpoint) .count()); counters.objects_available_bytes = host.objects().available_bytes().value_or(0); return {200, server::render_metrics(s, counters), "text/plain; version=0.0.4; charset=utf-8"}; } return {404, "not found\n"}; }; // Scheduling time; --accelerate scales it from process start (clones only). const auto started = std::chrono::steady_clock::now(); const auto scheduling_now = [&] { const auto real = std::chrono::steady_clock::now(); return started + (real - started) * static_cast(o.accelerate); }; LineReader reader; bool input_open = !o.synthetic, ready = false; // Synthetic driver state: the next tick to submit, reset at every recovery. std::optional synthetic_next; bool synthetic_running = false; const auto synthetic_request = [&](std::uint64_t tick) { persistence::Uuid id{'S', 'Y', 'N', '1'}; for (unsigned i = 0; i < 4; ++i) id[4 + i] = static_cast((o.host.key.region >> (8U * (3U - i))) & 0xFFU); for (unsigned i = 0; i < 8; ++i) id[8 + i] = static_cast((tick >> (8U * (7U - i))) & 0xFFU); return id; }; std::optional pending; std::optional phase; std::size_t max_queued = 0; std::optional drain_deadline; const auto set_ready = [&](bool now) { if (now == ready) return; ready = now; std::printf("ready %d\n", ready ? 1 : 0); if (!o.ready_file.empty()) { std::error_code ignored; if (ready) std::ofstream(o.ready_file) << "ready\n"; else std::filesystem::remove(o.ready_file, ignored); } }; for (;;) { if (stop_requested && !drain_deadline) { m10->shutdown(); host.drain(); drain_deadline = std::chrono::steady_clock::now() + o.drain; } // Read only while the bounded backlog has room: stdin itself backpressures. const bool room = host.status().queued < o.host.max_queued_inputs && !pending; pollfd fd{STDIN_FILENO, POLLIN, 0}; if (input_open && room && !drain_deadline && ::poll(&fd, 1, 5) > 0) input_open = reader.fill(); else if (!input_open || !room || drain_deadline) ::usleep(2000); while (!drain_deadline) { if (!pending) pending = reader.next(); if (!pending) break; auto in = parse_input(*pending); if (!in) { std::puts("rejected - malformed"); pending.reset(); continue; } auto admitted = host.submit(*in); if (!admitted && admitted.error() == server::AdmissionError::full) break; // Keep the line and retry after ticks drain the backlog. if (!admitted) std::printf("rejected %s %s\n", text(in->request).c_str(), admission_name(admitted.error())); pending.reset(); } const bool m10_accepting = m10->accepting( static_cast(std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count())); if (o.synthetic && !drain_deadline && (!m10->enabled() || m10_accepting)) { const auto& now_status = host.status(); const bool running = now_status.phase == server::HostPhase::running && now_status.ready; if (running && !synthetic_running) synthetic_next = now_status.simulated_tick + 1; // Resume after the head. synthetic_running = running; while (running && synthetic_next && *synthetic_next <= host.status().simulated_tick + lookahead && (o.synthetic_until == 0 || *synthetic_next <= o.synthetic_until)) { // An admitted session intention may ride this input; content is unchanged. const auto carried = sessions->carry(*synthetic_next); const server::TickInput in{carried.value_or(synthetic_request(*synthetic_next)), processes::development_input(*synthetic_next)}; if (!host.submit(in)) { if (carried) sessions->not_committed(*carried); break; // Full or no longer ready: continue next iteration. } ++*synthetic_next; } } const auto& s = host.step(scheduling_now()); max_queued = std::max(max_queued, s.queued); // Checkpoint age restarts at recovery and at every confirmed commit. if ((!recovered && s.phase == server::HostPhase::running) || s.committed_sequence != last_sequence) { recovered = recovered || s.phase == server::HostPhase::running; last_sequence = s.committed_sequence; last_checkpoint = std::chrono::steady_clock::now(); } if (s.phase == server::HostPhase::acquiring) recovered = false; if (!phase || *phase != s.phase) { phase = s.phase; std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n", server::phase_name(s.phase), static_cast(s.epoch), static_cast(s.committed_tick), static_cast(s.simulated_tick)); } set_ready(s.ready); const auto acknowledged = host.take_acks(); counters.acks += acknowledged.size(); for (const auto& a : acknowledged) { std::printf("ack %s %llu %llu\n", text(a.request).c_str(), static_cast(a.tick), static_cast(a.commit_sequence)); sessions->committed(a.request, a.tick, a.commit_sequence); } const auto outcomes = host.take_outcomes(); counters.outcomes += outcomes.size(); for (const auto& x : outcomes) std::printf("outcome %llu %u %llu %llu\n", static_cast(x.decision.tick), static_cast(x.decision.actor), static_cast(x.decision.organism), static_cast(x.commit_sequence)); const auto dropped = host.take_dropped(); for (const auto& d : dropped) ++counters.dropped[static_cast(d.reason)]; for (const auto& d : dropped) { std::printf("dropped %s %s\n", text(d.input.request).c_str(), server::unacknowledged_name(d.reason)); sessions->not_committed(d.input.request); } std::fflush(stdout); if (ops) ops->serve(handler); sessions->step(host.state(), host.status()); m10->after_step(); std::fflush(stdout); if (o.crash_point == "after-ack" && !acknowledged.empty() && ++ack_steps == o.crash_ordinal) crash_now(); const auto finish = [&](int code) { set_ready(false); std::printf( "stats max_queued=%zu max_tick_debt=%llu commits=%llu store_failures=%llu\n", max_queued, static_cast(s.max_tick_debt), static_cast(s.commits), static_cast(s.store_failures)); std::printf("exit %d\n", code); std::fflush(stdout); return code; }; if (s.phase == server::HostPhase::stopped) return finish(0); if (s.phase == server::HostPhase::quarantined) return finish(4); if (drain_deadline && std::chrono::steady_clock::now() > *drain_deadline) return finish(3); // Pending/unknown work is left for the next owner to reconcile. } } } // namespace int main(int argc, char** argv) { auto options = parse(argc, argv); if (!options) { std::fputs("usage: evo-server --profile lineage|environmental --world N --region N " "--frame N --pg-host H --pg-port P --pg-database D --pg-user U " "--objects DIR --build HEX64 [--retained HEX64]... [--pg-hostaddr A " "--pg-passfile F --pg-ca F] [--flush-ms N] [--drain-ms N] " "[--lease-profile default|fast] [--ready-file PATH] " "[--ops-listen ADDRESS:PORT] [--synthetic-inputs development-v1 " "[--synthetic-until TICK] [--accelerate N] [--session-listen ADDRESS:PORT " "--session-cert F --session-key F --session-grant-key F " "--session-grant-ledger F --session-environment dev]] [--m10-listen " "ADDRESS:PORT --m10-cert F --m10-key F --m10-issuer F " "--m10-environment dev|stage|world0 --session-window ACTIVATION:UTC_EXPIRY " "[--m10-max-grant-lifetime 60..3600]]\n", stderr); return 2; } struct sigaction action{}; action.sa_handler = on_stop; sigemptyset(&action.sa_mask); sigaction(SIGTERM, &action, nullptr); sigaction(SIGINT, &action, nullptr); std::signal(SIGPIPE, SIG_IGN); return options->profile == "lineage" ? serve(*options) : serve(*options); }