From d6d0408aae091deaabc07a4de93c25ce366766ab Mon Sep 17 00:00:00 2001 From: dingo Date: Mon, 28 Sep 2026 20:53:11 -0400 Subject: [PATCH] Complete M7.6 bounded authoritative region host ADR-0064: RegionHost (environmental and lineage profiles) is a single-owner, single-threaded state machine advanced by step(now): acquire the fenced lease and recover the verified head, run, resolve ambiguous commits, drain and stop, or quarantine. Admission is provisional and bounded (64 inputs, 64 KiB) with backpressure; FinalAck and FinalOutcome are emitted only after a confirmed commit, and inputs that will never be acknowledged are reported with a reason. Ticks follow the caller's scheduling clock with bounded catch-up and measured tick debt, never invented inputs. A death seals the batch and commits in the same step, so no dependent tick uses an uncommitted outcome. Storage outages pause and retry, unknown outcomes resolve by exact bytes, fencing discards unconfirmed work and re-acquires; readiness fails closed. Drain commits pending ticks and releases the lease. evo-server hosts one region with a bounded stdin input feed, stdout event lines, a readiness file and SIGTERM drain (exit 0/3/4). Validated: Windows CTest 58/58; PostgreSQL suites under GCC Release, clang-tidy and ASan/UBSan including real-process SIGTERM drain and SIGKILL with successor recovery; four host mutations each caught. No gateway, metrics, Docker/Gitea CI or deployment. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 14 +- apps/evo-server/main.cpp | 373 ++++++++++++++++++++ docs/ROADMAP.md | 13 + docs/STATUS.md | 25 ++ docs/adr/0064-region-host.md | 118 +++++++ docs/adr/README.md | 1 + docs/architecture/REGION_DURABILITY.md | 9 + docs/engineering/VALIDATION.md | 43 +++ docs/infrastructure/POSTGRESQL.md | 23 ++ include/evo/server/region_host.hpp | 170 +++++++++ src/server/region_host.cpp | 471 +++++++++++++++++++++++++ tests/postgres_tests.cpp | 6 + tests/region_host_tests.cpp | 332 +++++++++++++++++ tests/region_server_process_tests.cpp | 334 ++++++++++++++++++ tools/ci/postgres-check.sh | 5 +- 15 files changed, 1934 insertions(+), 3 deletions(-) create mode 100644 apps/evo-server/main.cpp create mode 100644 docs/adr/0064-region-host.md create mode 100644 include/evo/server/region_host.hpp create mode 100644 src/server/region_host.cpp create mode 100644 tests/region_host_tests.cpp create mode 100644 tests/region_server_process_tests.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 4d78156..73e402e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -103,11 +103,21 @@ if(EVO_BUILD_POSTGRES) target_link_libraries(evo_region_store PUBLIC evo::postgres evo::region_checkpoint PRIVATE OpenSSL::Crypto) evo_owned_target(evo_region_store) + # M7.6 authoritative region host and its process (ADR-0064). + add_library(evo_region_host STATIC src/server/region_host.cpp) + add_library(evo::region_host ALIAS evo_region_host) + target_link_libraries(evo_region_host PUBLIC evo::region_store) + evo_owned_target(evo_region_host) + add_executable(evo-server apps/evo-server/main.cpp) + target_link_libraries(evo-server PRIVATE evo::region_host) + evo_owned_target(evo-server) if(BUILD_TESTING) # Contention tests use one thread and one connection per contender. add_executable(evo-postgres-tests tests/postgres_tests.cpp tests/region_lease_store_tests.cpp - tests/region_checkpoint_store_tests.cpp) - target_link_libraries(evo-postgres-tests PRIVATE evo::region_store Threads::Threads) + tests/region_checkpoint_store_tests.cpp tests/region_host_tests.cpp + tests/region_server_process_tests.cpp) + target_link_libraries(evo-postgres-tests PRIVATE evo::region_host Threads::Threads) + add_dependencies(evo-postgres-tests evo-server) # Spawned by the process tests. evo_owned_target(evo-postgres-tests) # Intentionally not an ordinary CTest: tools/ci/postgres-check.sh owns its private DB. endif() diff --git a/apps/evo-server/main.cpp b/apps/evo-server/main.cpp new file mode 100644 index 0000000..f980e81 --- /dev/null +++ b/apps/evo-server/main.cpp @@ -0,0 +1,373 @@ +// M7.6 region server process (ADR-0064). Development host for one region: tick +// inputs arrive as lines on stdin (the network gateway is later work); events are +// written as lines on stdout. Linux only; requires the PostgreSQL build. +// +// Input: input +// Output: phase epoch= committed= simulated= +// ready <0|1> +// rejected | dropped +// ack +// outcome +// stats max_queued= max_tick_debt= commits= store_failures= +// Exit: 0 drained, 2 usage, 3 drain deadline passed, 4 quarantined. +#include "evo/server/region_host.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +using namespace evo; +using namespace std::chrono_literals; +volatile std::sig_atomic_t stop_requested = 0; +extern "C" void on_stop(int) { + stop_requested = 1; +} + +struct Options { + std::string profile{"lineage"}; + persistence::PgOptions pg; + std::filesystem::path objects, ready_file; + server::HostConfig host; + std::chrono::milliseconds drain{10000}; +}; + +std::optional number(std::string_view text) { + std::uint64_t value{}; + const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), value); + if (error != std::errc{} || end != text.data() + text.size()) + return std::nullopt; + return value; +} +template std::optional> hex(std::string_view text) { + if (text.size() != N * 2) + return std::nullopt; + std::array out{}; + for (std::size_t i = 0; i < N; ++i) { + unsigned value{}; + const auto [end, error] = + std::from_chars(text.data() + (2 * i), text.data() + (2 * i) + 2, value, 16); + if (error != std::errc{} || end != text.data() + (2 * i) + 2) + return std::nullopt; + out[i] = static_cast(value); + } + return out; +} +template std::string text(const std::array& bytes) { + constexpr char digits[] = "0123456789abcdef"; + std::string out; + for (const auto b : bytes) { + out.push_back(digits[b >> 4U]); + out.push_back(digits[b & 0x0FU]); + } + return out; +} + +std::optional parse(int argc, char** argv) { + Options o; + o.host.retained_builds.clear(); + for (int i = 1; i < argc; ++i) { + const std::string_view flag = argv[i]; + if (i + 1 >= argc) + return std::nullopt; + const std::string_view value = argv[++i]; + const auto n = number(value); + if (flag == "--profile" && (value == "lineage" || value == "environmental")) + o.profile = value; + else if (flag == "--world" && n) + o.host.key.world = *n; + else if (flag == "--region" && n) + o.host.key.region = *n; + else if (flag == "--frame" && n) + o.host.frame = *n; + else if (flag == "--pg-host") + o.pg.host = value; + else if (flag == "--pg-hostaddr") + o.pg.hostaddr = value; + else if (flag == "--pg-port") + o.pg.port = value; + else if (flag == "--pg-database") + o.pg.database = value; + else if (flag == "--pg-user") + o.pg.user = value; + else if (flag == "--pg-passfile") + o.pg.password_file = value; + else if (flag == "--pg-ca") + o.pg.ca_file = value; + else if (flag == "--objects") + o.objects = value; + else if (flag == "--ready-file") + o.ready_file = value; + else if (flag == "--build" || flag == "--retained") { + const auto digest = hex<32>(value); + if (!digest) + return std::nullopt; + if (flag == "--build") + o.host.build = *digest; + else + o.host.retained_builds.push_back(*digest); + } else if (flag == "--flush-ms" && n && *n > 0 && *n <= 60000) + o.host.flush_interval = std::chrono::milliseconds{*n}; + else if (flag == "--drain-ms" && n && *n > 0 && *n <= 600000) + o.drain = std::chrono::milliseconds{*n}; + else if (flag == "--lease-profile" && value == "fast") // Development/test only. + o.host.lease = {4000ms, 800ms, 400ms, 400ms, 1000ms}; + else if (flag == "--lease-profile" && value == "default") + o.host.lease = {}; + else + return std::nullopt; + } + if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end()) + o.host.retained_builds.push_back(o.host.build); + return o; +} + +std::optional parse_input(std::string_view line) { + std::array fields{}; + std::size_t count = 0; + while (!line.empty() && count < fields.size() + 1) { + const auto start = line.find_first_not_of(' '); + if (start == std::string_view::npos) + break; + line.remove_prefix(start); + const auto end = line.find(' '); + if (count == fields.size()) + return std::nullopt; // Too many fields. + fields[count++] = line.substr(0, end); + line.remove_prefix(end == std::string_view::npos ? line.size() : end); + } + if (count != fields.size() || fields[0] != "input") + return std::nullopt; + auto request = hex<16>(fields[1]); + if (!request) + return std::nullopt; + std::array v{}; + for (std::size_t i = 0; i < v.size(); ++i) { + const auto n = number(fields[i + 2]); + if (!n) + return std::nullopt; + v[i] = *n; + } + server::TickInput in{*request, {}}; + in.input.temperature_mK = v[0]; + in.input.source_enabled = v[1]; + for (std::size_t k = 0; k < 5; ++k) + in.input.bite[k] = v[2 + k]; + in.input.terminal = {v[7], v[8]}; + return in; +} + +const char* phase_name(server::HostPhase p) { + switch (p) { + case server::HostPhase::acquiring: + return "acquiring"; + case server::HostPhase::running: + return "running"; + case server::HostPhase::resolving: + return "resolving"; + case server::HostPhase::draining: + return "draining"; + case server::HostPhase::stopped: + return "stopped"; + case server::HostPhase::quarantined: + return "quarantined"; + } + return "unknown"; +} +const char* admission_name(server::AdmissionError e) { + switch (e) { + case server::AdmissionError::not_ready: + return "not_ready"; + case server::AdmissionError::draining: + return "draining"; + case server::AdmissionError::full: + return "full"; + case server::AdmissionError::duplicate: + return "duplicate"; + case server::AdmissionError::invalid: + return "invalid"; + } + return "unknown"; +} +const char* dropped_name(server::Unacknowledged u) { + switch (u) { + case server::Unacknowledged::refused: + return "refused"; + case server::Unacknowledged::drained: + return "drained"; + case server::Unacknowledged::fenced: + return "fenced"; + case server::Unacknowledged::quarantined: + return "quarantined"; + } + return "unknown"; +} + +// Reads complete, bounded lines from stdin without blocking. +class LineReader { + public: + // False on EOF/error. + bool fill() { + std::array chunk{}; + const auto n = ::read(STDIN_FILENO, chunk.data(), chunk.size()); + if (n <= 0) + return n < 0 && errno == EINTR; + buffer_.append(chunk.data(), static_cast(n)); + return true; + } + std::optional next() { + const auto end = buffer_.find('\n'); + if (end == std::string::npos) { + if (buffer_.size() > 512) + buffer_.clear(); // Overlong line: discarded, never partially parsed. + return std::nullopt; + } + auto line = buffer_.substr(0, end); + buffer_.erase(0, end + 1); + return line; + } + + private: + std::string buffer_; +}; + +template int serve(Options& o) { + auto objects = persistence::FilesystemObjectStore::open(o.objects, o.host.key, + persistence::max_lineage_payload_bytes); + if (!objects) { + std::puts("error objects"); + return 2; + } + const auto pg = o.pg; + auto made = server::RegionHost::create( + o.host, [pg] { return persistence::Postgres::connect(pg, 2s); }, std::move(*objects)); + if (!made) { + std::puts("error config"); + return 2; + } + auto& host = **made; + LineReader reader; + bool input_open = true, ready = false; + std::optional pending; + std::optional phase; + std::size_t max_queued = 0; + std::optional drain_deadline; + const auto set_ready = [&](bool now) { + if (now == ready) + return; + ready = now; + std::printf("ready %d\n", ready ? 1 : 0); + if (!o.ready_file.empty()) { + std::error_code ignored; + if (ready) + std::ofstream(o.ready_file) << "ready\n"; + else + std::filesystem::remove(o.ready_file, ignored); + } + }; + for (;;) { + if (stop_requested && !drain_deadline) { + host.drain(); + drain_deadline = std::chrono::steady_clock::now() + o.drain; + } + // Read only while the bounded backlog has room: stdin itself backpressures. + const bool room = host.status().queued < o.host.max_queued_inputs && !pending; + pollfd fd{STDIN_FILENO, POLLIN, 0}; + if (input_open && room && !drain_deadline && ::poll(&fd, 1, 5) > 0) + input_open = reader.fill(); + else if (!input_open || !room || drain_deadline) + ::usleep(2000); + while (!drain_deadline) { + if (!pending) + pending = reader.next(); + if (!pending) + break; + auto in = parse_input(*pending); + if (!in) { + std::puts("rejected - malformed"); + pending.reset(); + continue; + } + auto admitted = host.submit(*in); + if (!admitted && admitted.error() == server::AdmissionError::full) + break; // Keep the line and retry after ticks drain the backlog. + if (!admitted) + std::printf("rejected %s %s\n", text(in->request).c_str(), + admission_name(admitted.error())); + pending.reset(); + } + const auto& s = host.step(std::chrono::steady_clock::now()); + max_queued = std::max(max_queued, s.queued); + if (!phase || *phase != s.phase) { + phase = s.phase; + std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n", phase_name(s.phase), + static_cast(s.epoch), + static_cast(s.committed_tick), + static_cast(s.simulated_tick)); + } + set_ready(s.ready); + for (const auto& a : host.take_acks()) + std::printf("ack %s %llu %llu\n", text(a.request).c_str(), + static_cast(a.tick), + static_cast(a.commit_sequence)); + for (const auto& x : host.take_outcomes()) + std::printf("outcome %llu %u %llu %llu\n", + static_cast(x.decision.tick), + static_cast(x.decision.actor), + static_cast(x.decision.organism), + static_cast(x.commit_sequence)); + for (const auto& d : host.take_dropped()) + std::printf("dropped %s %s\n", text(d.input.request).c_str(), dropped_name(d.reason)); + std::fflush(stdout); + const auto finish = [&](int code) { + set_ready(false); + std::printf( + "stats max_queued=%zu max_tick_debt=%llu commits=%llu store_failures=%llu\n", + max_queued, static_cast(s.max_tick_debt), + static_cast(s.commits), + static_cast(s.store_failures)); + std::printf("exit %d\n", code); + std::fflush(stdout); + return code; + }; + if (s.phase == server::HostPhase::stopped) + return finish(0); + if (s.phase == server::HostPhase::quarantined) + return finish(4); + if (drain_deadline && std::chrono::steady_clock::now() > *drain_deadline) + return finish(3); // Pending/unknown work is left for the next owner to reconcile. + } +} +} // namespace + +int main(int argc, char** argv) { + auto options = parse(argc, argv); + if (!options) { + std::fputs("usage: evo-server --profile lineage|environmental --world N --region N " + "--frame N --pg-host H --pg-port P --pg-database D --pg-user U " + "--objects DIR --build HEX64 [--retained HEX64]... [--pg-hostaddr A " + "--pg-passfile F --pg-ca F] [--flush-ms N] [--drain-ms N] " + "[--lease-profile default|fast] [--ready-file PATH]\n", + stderr); + return 2; + } + struct sigaction action{}; + action.sa_handler = on_stop; + sigemptyset(&action.sa_mask); + sigaction(SIGTERM, &action, nullptr); + sigaction(SIGINT, &action, nullptr); + std::signal(SIGPIPE, SIG_IGN); + return options->profile == "lineage" ? serve(*options) + : serve(*options); +} diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index f6a6092..ffa63c7 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -554,3 +554,16 @@ M7.5 implemented and locally qualified (2026-09-28), under ADR-0063. ambiguous outcomes. - Recovery reconciles decisions with the restored state. Next: M7.6 hosts bounded authoritative ticks (apps/evo-server, region_host). + +M7.6 implemented and locally qualified (2026-09-28), under ADR-0064. +- `RegionHost` is a single-owner state machine: acquire and recover, run, + resolve, drain and stop, or quarantine. +- Admission is bounded (64 inputs, 64 KiB) with backpressure, and + acknowledgements become final only after a confirmed commit. +- Tick debt is measured with bounded catch-up. +- A death pauses the region until it is durable. +- Readiness fails closed; storage outages, unknown outcomes and fences each have + a defined recovery. +- `evo-server` passed real SIGTERM drain and SIGKILL tests, with successor + recovery. +Next: M7.7 recovery verification at every kill boundary. diff --git a/docs/STATUS.md b/docs/STATUS.md index ad03687..e1b5e2a 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -836,3 +836,28 @@ mutations were each caught. Windows and Linux ran 58/58; PostgreSQL suites passe under Release, analysis and ASan. Next: M7.6 region host. No host, broker, deployment or process-kill test exists yet. + +## M7.6 region host and server process (2026-09-28) + +ADR-0064 adds `RegionHost`, a single-owner, single-threaded state machine: +acquire and recover, run, resolve, drain and stop, or quarantine. +- Admission is provisional and bounded (64 inputs, 64 KiB) with backpressure. +- Acknowledgements and outcomes are final only after a confirmed commit. +- A death pauses the region and commits in the same step. +- Tick debt is measured, and catch-up is bounded to 8 ticks per step. +- Readiness fails closed. + +`evo-server` hosts one region using a stdin input feed and stdout event lines, +with a readiness file and SIGTERM drain. Real process tests: +- SIGTERM: 80 inputs through the 64-input backlog were drained, committed and + released. +- SIGKILL: every acknowledged tick was committed, and the successor became ready + about 3.9 s later, after the 4 s development lease expired, from the committed + head. + +Seven host scenarios pass against real PostgreSQL, and four mutations were each +caught. Windows ran 58/58; the PostgreSQL suites passed under Release, analysis +and ASan/UBSan. + +Next: M7.7 recovery verification at every kill boundary. There is no gateway, +metrics or deployment yet. diff --git a/docs/adr/0064-region-host.md b/docs/adr/0064-region-host.md new file mode 100644 index 0000000..2f1dcdf --- /dev/null +++ b/docs/adr/0064-region-host.md @@ -0,0 +1,118 @@ +# ADR-0064: Bounded authoritative region host + +## Status +Accepted for M7.6, 2026-09-28. It composes ADR-0060 (fencing), ADR-0061/0062 +(checkpoints) and ADR-0063 (batch outcomes). It adds no gateway, network listener, +metrics endpoint or deployment; those are M7.8 and later. + +## Context +The persistence APIs exist, but nothing yet owns a region over time. ADR-0058 +requires: +- one simulation owner, with simulated and committed ticks both visible; +- bounded backlogs, with backpressure before admission; +- a whole-region pause at any irreversible outcome until durable confirmation; +- bounded drain, and readiness that fails closed. + +Nothing may treat an uncommitted death as fact. + +## Decision +**`RegionHost`** (`src/server`, Linux), for `EnvironmentalCommunity` and +`LineageRegion`, is a single-threaded state machine advanced by `step(now)`: +- *acquiring*: take the fenced lease, then recover and verify the committed head. + The recovered quantum and frame must match the configuration. +- *running*: lease upkeep, bounded ticks, and commits. +- *resolving*: an ambiguous commit is resolved before anything else. +- *draining*, then *stopped*; or *quarantined*. + +All I/O runs on the caller's thread within the LeasePolicy deadlines. While a +commit runs, the region is stopped. + +**Admission** is provisional: +- The backlog holds at most 64 inputs and 64 KiB (88 bytes are accounted per + input). When it is full, the caller gets `full` and must hold its input. +- Duplicate request identities within the host are refused. +- Inputs are refused unless the host is ready. + +An input becomes final only when the commit containing its tick is confirmed. +The host then emits `FinalAck(request, tick, commit_sequence)`; permanent +outcomes are emitted as `FinalOutcome`. Inputs that will never be acknowledged +are reported as `Dropped`, with reason refused, drained, fenced or quarantined. +After an unknown outcome, "fenced" means "unknown to this host", not "not +applied". + +**Ticks**: +- Scheduling time comes from the caller, while lease deadlines use `LeaseClock` + and database time. +- The due count is (now − start) / quantum. Each step runs at most 8 catch-up + ticks, and one queued input is consumed per tick. +- An empty backlog accrues tick debt. The host never invents inputs, and debt is + measured (current and maximum) rather than skipped. + +**Commits** use `RegionBatch`: +- A death seals the batch and commits in the same step, so no dependent tick runs + on an uncommitted outcome. +- Otherwise a batch flushes after `flush_interval` (default 1 s), at 64 ticks, + or on drain. +- The commit ID is the acquisition's holder token with the next sequence number + in its last 8 bytes. It is stable across retries of one batch, and unique + because holder tokens are generated per acquisition. + +| Outcome | Host response | +|---|---| +| Storage failure (upload/verify/digest), transient SQL error | Stay paused and retry the same batch; `store_failures` counts it | +| Unknown | Enter *resolving*: resolve the exact prepared bytes. Committed → adopt the receipt. Absent → retry the same bytes while `may_begin_commit` holds. Conflict → quarantine | +| Fenced | Discard state, report unconfirmed inputs, re-acquire and recover from the head | +| Candidate, predecessor, head moved, conflict, receipt mismatch, unrestorable head | Quarantine, releasing the lease if still held | + +**Readiness** is true only when running, recovered, not draining, and +`LeaseAuthority::may_originate` holds. + +**Drain** stops admission, reports the backlog as drained, commits pending ticks, +releases the lease, then stops. A successor can therefore acquire immediately, +without waiting for expiry. + +**`evo-server`** is the development process around one host: +- Tick inputs are stdin lines. It reads only while the backlog has room, so stdin + itself backpressures. +- Events are stdout lines: phase, ready, ack, outcome, dropped, stats. +- An optional readiness file mirrors readiness. +- SIGTERM or SIGINT drain within `--drain-ms`. +- Exit codes: 0 drained, 3 drain deadline passed (pending or unknown work is left + to the next owner), 4 quarantined, 2 usage. +- `--lease-profile fast` (a 4 s lease) is for development tests only. + +## Alternatives Considered +- **A separate I/O thread**: would allow ticks during commits, but the contract + pauses the region at commits anyway, and extra threads violate single ownership + without a measured need. +- **Wall-clock-driven synthetic inputs**: would invent world input. +- **Acknowledging on apply**: would expose provisional state as final. +- **Waiting for lease expiry after a clean shutdown**: slows handover for no + safety benefit. + +## Benefits +- One place enforces pause-at-outcome, final-only-after-commit and fail-closed + readiness. +- Every failure mode maps to a defined phase. +- The process can be killed or terminated with verifiable durable results. + +## Costs +- Ticks stall during commits: about 7 ms locally per batch in Release (M7.5), and + longer if storage is slow. +- The stdin feed is development plumbing, not a network protocol. +- There are no metrics endpoints yet. + +## Risks +- Duplicate request detection is per host process, not durable. Client + deduplication across owners needs the gateway protocol. +- The killed-process test measures recovery after the development 4 s lease; the + production 20 s lease bounds unavailability accordingly. +- Clock steps (ADR-0060) still apply. + +## Consequences +- M7.7 kills processes at every publication and ACK boundary, and verifies + recovery before readiness. +- M7.8 exposes phase, readiness, tick debt, commit lag and lease safety as probes + and metrics, and packages `evo-server`. +- A gateway must send inputs with request identities and treat only `FinalAck` as + durable. diff --git a/docs/adr/README.md b/docs/adr/README.md index 343ffe7..14f36f8 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -74,3 +74,4 @@ design. No unadopted candidate is a link dependency of the proof. - [ADR-0061: Checkpoint publication](0061-checkpoint-publication.md) - accepted for M7.4a manifest v1, immutable filesystem objects, OpenSSL SHA-256 and verified recovery. - [ADR-0062: Lineage checkpoint profile](0062-lineage-checkpoint-profile.md) - accepted for M7.4b archive codec, LineageRegion closure and history-capable profile 2. - [ADR-0063: Batch outcomes](0063-batch-outcomes.md) - accepted for M7.5 manifest v2, validation trace replay, durable decisions/outbox and commit resolution. +- [ADR-0064: Region host](0064-region-host.md) - accepted for M7.6 single-owner host state machine, bounded admission, final-only-after-commit and evo-server process. diff --git a/docs/architecture/REGION_DURABILITY.md b/docs/architecture/REGION_DURABILITY.md index a492478..210b601 100644 --- a/docs/architecture/REGION_DURABILITY.md +++ b/docs/architecture/REGION_DURABILITY.md @@ -201,3 +201,12 @@ M7.5 (ADR-0063) executed its rows against real PostgreSQL: Recovery now reconciles durable deaths with the restored lifecycle, so the checkpoint and durable history must agree. Kill tests at each boundary remain M7.7. + +M7.6 (ADR-0064) executed its host rows: +- A paused stale host fences itself when renewals cannot reach the database. It + reports unconfirmed inputs as never final and re-acquires from the head. +- A death pauses the region in the same step as its commit. +- A storage outage keeps the sealed batch waiting with nothing acknowledged. +- Shutdown drains, commits and releases. +- A real SIGKILLed `evo-server` left no acknowledged tick uncommitted, and its + successor became ready only after lease expiry, from the committed head. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 0d93854..55f54c3 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -2296,3 +2296,46 @@ Only clean-to-v1 and exact-v1 restart are supported migration paths at this gate PostgreSQL suites, all passing, and the repository format check. The changes are Linux-only persistence code, SQL and tests; the portable targets and Windows are unchanged. + +## M7.6 region host - 2026-09-28 + +- Windows MSVC Release CTest passed 58/58; this change touches only the Linux + PostgreSQL CMake section and Linux sources. The repository clang-format 19 check + passed. +- WSL Linux: GCC14 Release, clang19 analysis (clang-tidy) and clang19 ASan/UBSan + builds of `evo-postgres-tests` and `evo-server` passed the full PostgreSQL suite. + Final runs: Release 3, analysis 1, ASan 3 across both verification passes. +- The first analysis build flagged unchecked optional accesses and ignored release + results in the host and server, plus two test issues. These were fixed with + explicit local checks rather than suppressions, then rebuilt and rerun in all + three configurations. +- Host scenarios (real PostgreSQL, scaled scheduling clock, 4 s test lease): + - Admission: not_ready before recovery, 64-input cap (full), duplicate and + invalid refusals, byte accounting. + - Ticks 1–19 are provisional with no acks. Tick 20's death pauses and commits + in the same step: 20 acks and one outcome (consumer, organism 11, sequence 1). + The next batch flushed by interval at tick 41. + - Starvation: debt reached 10 without invented ticks, then catch-up of 8 per step. + - Storage outage: the sealed batch waits with nothing acknowledged and debt + growing, then commits when storage recovers. + - Lease row locked: the commit becomes unknown, the host is not ready and + refuses admission, and resolution adopts the receipt after release. + - Renewals blocked past the local deadline: fence, all 8 unconfirmed inputs + reported `fenced`, then re-acquisition at a newer epoch from the committed head. + - Drain (environmental profile): backlog dropped as drained, 6 pending ticks + committed, lease released, and a successor acquired immediately at epoch 2. + - Unrestorable build and foreign frame quarantine; an invalid configuration is + refused. +- Process tests (real `evo-server` via posix_spawn, stdin and stdout pipes): + - SIGTERM after 80 inputs: exit 0, 80 acks, head 80 at sequence 8, lease + released, readiness file removed, max_queued 64. The outcome line followed + tick 20's final ack. + - SIGKILL during running: every acked tick ≤ the committed head (11/11) and the + lease was left held. The successor was ready after 3.94–3.97 s at epoch 2 with + committed=11, then drained cleanly. +- Mutations (restored and rerun green), each caught: acknowledging at simulation + time, not committing a sealed death, reporting ready while resolving, and + draining without releasing the lease. +- Not executed: Docker/Gitea CI, TSan (the host is single-threaded; libpq is + uninstrumented), remote PostgreSQL/TLS, a production 20 s lease kill test, + durable client deduplication, and a network gateway. diff --git a/docs/infrastructure/POSTGRESQL.md b/docs/infrastructure/POSTGRESQL.md index 1a5474e..0516c85 100644 --- a/docs/infrastructure/POSTGRESQL.md +++ b/docs/infrastructure/POSTGRESQL.md @@ -165,3 +165,26 @@ the existing HARBOR_PROJECT and Harbor robot secrets. Pin the reported registry digest in the example Job; do not use the simulation image. See deploy/README.md. Local Unix-socket integration is qualified; remote TLS/authentication, actual Secret mounts and container execution still require environment-specific verification. + +## Region server process (M7.6, ADR-0064) + +`evo-server` is built with the PostgreSQL build and hosts one provisioned region +as the runtime login. It is a development host with no network gateway. +Tick inputs are stdin lines: + + input + +Events are stdout lines: `phase`, `ready`, `ack`, `outcome`, `dropped` and a +final `stats`. Only `ack` and `outcome` lines are durable facts. +`--ready-file` mirrors readiness for probes. SIGTERM drains (commits pending ticks +and releases the lease) within `--drain-ms`. + +Exit codes: +- 0: drained; +- 3: the drain deadline passed, and the next owner reconciles; +- 4: quarantined; +- 2: usage error. + +Use the default 20 s lease in any shared environment. `--lease-profile fast` exists +only for local tests. The runtime credentials come from `--pg-*` flags and a +passfile; never place passwords on the command line. diff --git a/include/evo/server/region_host.hpp b/include/evo/server/region_host.hpp new file mode 100644 index 0000000..9b994b2 --- /dev/null +++ b/include/evo/server/region_host.hpp @@ -0,0 +1,170 @@ +#pragma once +#include "evo/persistence/checkpoint_store.hpp" +#include "evo/persistence/region_batch.hpp" +#include "evo/persistence/region_lease.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// M7.6 authoritative region host (ADR-0064). Linux; composes the fenced lease, +// verified recovery and batch publication. One owner, one thread: every call runs +// on the caller's thread and all I/O is bounded by the lease policy's deadlines. +// While a batch commits the region is stopped, by design (ADR-0058). +namespace evo::server { +// One tick's host input, from the (future) gateway or a local feed. +struct TickInput { + persistence::Uuid request{}; // Caller identity for acknowledgement; nonzero. + processes::EnvironmentalInput input{}; + friend bool operator==(const TickInput&, const TickInput&) = default; +}; +inline constexpr std::size_t tick_input_bytes = 16 + 72; // Canonical accounting size. + +enum class AdmissionError : std::uint8_t { + not_ready, // No current authority or not recovered: nothing is accepted. + draining, // Shutdown in progress. + full, // Backpressure: bounded backlog (inputs or bytes) is full. + duplicate, // Request identity already queued or pending in this host. + invalid, // Zero request identity. +}; +// Final: the input's tick is in a durably committed checkpoint. +struct FinalAck { + persistence::Uuid request{}; + std::uint64_t tick{}; + std::uint64_t commit_sequence{}; + friend bool operator==(const FinalAck&, const FinalAck&) = default; +}; +// Final: the permanent outcome is durable (and in the outbox). +struct FinalOutcome { + persistence::BatchDecision decision{}; + std::uint64_t commit_sequence{}; + friend bool operator==(const FinalOutcome&, const FinalOutcome&) = default; +}; +// Why an accepted input will never be acknowledged by this host. +enum class Unacknowledged : std::uint8_t { + refused, // The simulation rejected it; no tick consumed it. + drained, // Still queued at shutdown. + fenced, // Authority was lost before its tick committed; the next owner never saw it. + quarantined, +}; +struct Dropped { + TickInput input{}; + Unacknowledged reason{}; +}; + +enum class HostPhase : std::uint8_t { + acquiring, // Waiting for the lease (busy/unavailable) or recovering. + running, // Authoritative: simulating and committing. + resolving, // A commit outcome is unknown: paused until resolved. + draining, // Shutdown: committing pending work, then releasing. + stopped, // Drained (lease released when possible). + quarantined, // Fail closed: committed state or history cannot be trusted. +}; +struct HostStatus { + HostPhase phase{HostPhase::acquiring}; + bool ready{}; // Authoritative now: recovered, running, lease may originate. + std::uint64_t epoch{}; + std::uint64_t simulated_tick{}, committed_tick{}, committed_sequence{}; + std::size_t queued{}, queued_bytes{}; + std::uint64_t tick_debt{}, max_tick_debt{}; // Due minus simulated since running. + std::uint64_t commits{}, store_failures{}, unknown_outcomes{}, fences{}; +}; + +using Connect = + std::function, persistence::PgFailure>()>; +struct HostConfig { + persistence::RegionKey key{}; + std::uint64_t frame{}; + persistence::LeasePolicy lease{}; + // Region quantum; must equal the committed checkpoint's tick_microseconds. + std::chrono::microseconds tick_quantum{50000}; + std::chrono::milliseconds flush_interval{1000}; // Commit a non-empty batch this often. + std::size_t max_queued_inputs{64}; + std::size_t max_queued_bytes{65536}; + std::uint32_t max_catch_up_ticks{8}; // Per step; debt is never skipped. + persistence::Sha256Digest build{}; + std::vector retained_builds; + std::size_t restore_budget{8000000}; +}; +enum class HostConfigError : std::uint8_t { invalid }; + +// Scheduling time comes from the caller (monotonic, may be scaled in tests); lease +// deadlines always use LeaseClock (real, suspend-aware) and database time. +using SchedulerTime = std::chrono::steady_clock::time_point; + +template class RegionHost { + public: + [[nodiscard]] static std::expected, HostConfigError> + create(HostConfig config, Connect connect, + std::unique_ptr objects); + RegionHost(const RegionHost&) = delete; + RegionHost& operator=(const RegionHost&) = delete; + RegionHost(RegionHost&&) = delete; + RegionHost& operator=(RegionHost&&) = delete; + ~RegionHost(); + + // Provisional admission only; nothing is final until a FinalAck. + [[nodiscard]] std::expected submit(const TickInput& input); + // Advances the state machine once: lease upkeep, recovery, resolution, commits + // and up to max_catch_up_ticks ticks. Blocks at most for bounded I/O. + const HostStatus& step(SchedulerTime now); + // Stops admission, drops the backlog, commits pending ticks, releases the lease. + void drain(); + + [[nodiscard]] std::vector take_acks(); + [[nodiscard]] std::vector take_outcomes(); + [[nodiscard]] std::vector take_dropped(); + [[nodiscard]] const HostStatus& status() const noexcept { + return status_; + } + // Read-only view for diagnostics/tests; null unless recovered. + [[nodiscard]] const State* state() const noexcept { + return state_.get(); + } + + private: + struct Applied { + TickInput input; + std::uint64_t tick{}; + }; + RegionHost(HostConfig config, Connect connect, + std::unique_ptr objects); + bool database(); + void acquire(SchedulerTime now); + void upkeep(); + void simulate(SchedulerTime now); + void commit(SchedulerTime now); + void resolve(SchedulerTime now); + void confirmed(const persistence::PublishedBatch& batch, SchedulerTime now); + void lose_authority(Unacknowledged reason, HostPhase next); + void refresh(SchedulerTime now); + [[nodiscard]] persistence::Uuid next_commit_id() const; + + HostConfig config_; + Connect connect_; + std::unique_ptr objects_; + std::unique_ptr db_; + persistence::LeaseAuthority authority_; + std::unique_ptr state_; + std::optional> batch_; + std::optional committed_; + std::optional prepared_; + std::deque queue_; + std::vector applied_; + std::vector acks_; + std::vector outcomes_; + std::vector dropped_; + SchedulerTime origin_{}, batch_started_{}; + std::uint64_t origin_tick_{}; + bool draining_{}; + HostStatus status_{}; +}; + +extern template class RegionHost; +extern template class RegionHost; +} // namespace evo::server diff --git a/src/server/region_host.cpp b/src/server/region_host.cpp new file mode 100644 index 0000000..71f4585 --- /dev/null +++ b/src/server/region_host.cpp @@ -0,0 +1,471 @@ +#include "evo/server/region_host.hpp" +#include "evo/persistence/sha256.hpp" +#include +#include + +namespace evo::server { +namespace { +using persistence::LeaseClock; +using persistence::LeaseStoreError; +using persistence::PublicationError; +using persistence::RecoveryError; + +// Per-profile calls into the persistence layer. +template struct Ops; +template <> struct Ops { + using State = processes::EnvironmentalCommunity; + static std::uint64_t tick(const State& s) { + return s.progress().tick; + } + static std::uint64_t quantum(const State& s) { + return s.tick_microseconds(); + } + template static auto recover(A&&... a) { + return persistence::recover_environmental_checkpoint(std::forward(a)...); + } + template static auto publish(A&&... a) { + return persistence::publish_environmental_batch(std::forward(a)...); + } +}; +template <> struct Ops { + using State = processes::LineageRegion; + static std::uint64_t tick(const State& s) { + return s.community().progress().tick; + } + static std::uint64_t quantum(const State& s) { + return s.community().tick_microseconds(); + } + template static auto recover(A&&... a) { + return persistence::recover_lineage_checkpoint(std::forward(a)...); + } + template static auto publish(A&&... a) { + return persistence::publish_lineage_batch(std::forward(a)...); + } +}; + +bool valid(const HostConfig& c) { + constexpr std::size_t max_backlog_inputs = 64; + constexpr std::size_t max_backlog_bytes = 65536; + return c.key.world != 0 && c.key.region != 0 && c.frame != 0 && persistence::valid(c.lease) && + c.tick_quantum.count() > 0 && c.flush_interval.count() > 0 && c.max_queued_inputs >= 1 && + c.max_queued_inputs <= max_backlog_inputs && c.max_queued_bytes >= tick_input_bytes && + c.max_queued_bytes <= max_backlog_bytes && c.max_catch_up_ticks >= 1 && + c.max_catch_up_ticks <= 64 && c.build != persistence::Sha256Digest{} && + std::ranges::find(c.retained_builds, c.build) != c.retained_builds.end() && + c.restore_budget > 0; +} +} // namespace + +template +RegionHost::RegionHost(HostConfig config, Connect connect, + std::unique_ptr objects) + : config_(std::move(config)), connect_(std::move(connect)), objects_(std::move(objects)), + authority_(config_.lease) {} + +template RegionHost::~RegionHost() = default; + +template +std::expected>, HostConfigError> +RegionHost::create(HostConfig config, Connect connect, + std::unique_ptr objects) { + if (!valid(config) || !connect || !objects) + return std::unexpected{HostConfigError::invalid}; + return std::unique_ptr( + new RegionHost(std::move(config), std::move(connect), std::move(objects))); +} + +template +std::expected RegionHost::submit(const TickInput& input) { + if (input.request == persistence::Uuid{}) + return std::unexpected{AdmissionError::invalid}; + if (draining_) + return std::unexpected{AdmissionError::draining}; + if (!status_.ready) + return std::unexpected{AdmissionError::not_ready}; + const auto same = [&](const TickInput& x) { return x.request == input.request; }; + if (std::ranges::any_of(queue_, same) || + std::ranges::any_of(applied_, [&](const Applied& a) { return same(a.input); })) + return std::unexpected{AdmissionError::duplicate}; + if (queue_.size() >= config_.max_queued_inputs || + (queue_.size() + 1) * tick_input_bytes > config_.max_queued_bytes) + return std::unexpected{AdmissionError::full}; + queue_.push_back(input); + status_.queued = queue_.size(); + status_.queued_bytes = queue_.size() * tick_input_bytes; + return {}; +} + +template bool RegionHost::database() { + if (db_) + return true; + auto made = connect_(); + if (!made) { + ++status_.store_failures; + return false; + } + db_ = std::move(*made); + return true; +} + +template +void RegionHost::lose_authority(Unacknowledged reason, HostPhase next) { + for (auto& a : applied_) + dropped_.push_back({a.input, reason}); + for (auto& q : queue_) + dropped_.push_back({q, reason}); + applied_.clear(); + queue_.clear(); + batch_.reset(); // References state_: reset first. + state_.reset(); + committed_.reset(); + prepared_.reset(); + // Best effort: a quarantined region's lease is released so peers see it free. + const auto& grant = authority_.grant(); + if (next == HostPhase::quarantined && db_ && grant && + authority_.may_originate(LeaseClock::now())) { + [[maybe_unused]] const auto released = persistence::release_region_lease( + *db_, config_.key, *grant, config_.lease.renew_timeout); + } + authority_.fence(); + if (reason == Unacknowledged::fenced) + ++status_.fences; + status_.phase = draining_ && next == HostPhase::acquiring ? HostPhase::stopped : next; +} + +template void RegionHost::acquire(SchedulerTime now) { + if (!database()) + return; + // A grant survives a transient recovery failure; only re-acquire without one. + if (!authority_.may_originate(LeaseClock::now())) { + const auto sent = LeaseClock::now(); + auto grant = persistence::acquire_region_lease(*db_, config_.key, config_.lease.duration, + config_.lease.renew_timeout); + if (!grant) { + if (grant.error().kind == LeaseStoreError::not_provisioned) + status_.phase = HostPhase::quarantined; + else if (grant.error().kind != LeaseStoreError::busy) + db_.reset(); + return; + } + if (grant->frame != config_.frame) { + status_.phase = HostPhase::quarantined; + return; + } + if (!authority_.acquired(*grant, sent, LeaseClock::now())) + return; // Reply too late to use; the lease expires unused. + } + auto recovered = Ops::recover(*db_, *objects_, config_.key, config_.retained_builds, + config_.restore_budget, config_.lease.commit_timeout); + if (!recovered) { + if (recovered.error().kind == RecoveryError::store) { + db_.reset(); // Transient: keep the grant and retry recovery. + ++status_.store_failures; + return; + } + lose_authority(Unacknowledged::quarantined, HostPhase::quarantined); + return; + } + if (Ops::quantum(*recovered->state) != + static_cast(config_.tick_quantum.count())) { + lose_authority(Unacknowledged::quarantined, HostPhase::quarantined); + return; + } + state_ = std::move(recovered->state); + committed_ = std::move(recovered->committed); + batch_.emplace(*state_); + origin_ = now; + origin_tick_ = Ops::tick(*state_); + status_.max_tick_debt = 0; + status_.phase = draining_ ? HostPhase::draining : HostPhase::running; +} + +template void RegionHost::upkeep() { + const auto state = authority_.state(LeaseClock::now()); + if (state == persistence::LeaseState::fenced) { + lose_authority(Unacknowledged::fenced, HostPhase::acquiring); + return; + } + if (state != persistence::LeaseState::renew_due || !database()) + return; + const auto grant = *authority_.grant(); + const auto sent = LeaseClock::now(); + auto renewed = persistence::renew_region_lease(*db_, config_.key, grant, config_.lease.duration, + config_.lease.renew_timeout); + if (renewed) { + if (!authority_.renewed(grant.epoch, *renewed, sent, LeaseClock::now())) + lose_authority(Unacknowledged::fenced, HostPhase::acquiring); + return; + } + switch (renewed.error().kind) { + case LeaseStoreError::unknown: + // It may have shortened or extended nothing: keep the earlier deadline. + authority_.renewal_unknown(config_.lease.duration, sent); + db_.reset(); + ++status_.store_failures; + return; + case LeaseStoreError::fenced: + lose_authority(Unacknowledged::fenced, HostPhase::acquiring); + return; + default: + db_.reset(); + ++status_.store_failures; + return; + } +} + +template persistence::Uuid RegionHost::next_commit_id() const { + // Unique: the DB-generated holder token names this acquisition; the + // sequence names the link. Stable across retries of the same batch. + const auto& grant = authority_.grant(); + if (!grant || !committed_) + return {}; + auto id = grant->holder; + const auto sequence = committed_->head.sequence + 1; + for (unsigned i = 0; i < 8; ++i) + id[8 + i] = static_cast((sequence >> (8U * i)) & 0xFFU); + return id; +} + +template +void RegionHost::confirmed(const persistence::PublishedBatch& published, SchedulerTime now) { + const auto& head = published.committed.head; + if (!batch_ || !batch_->confirm(head.end_tick)) { + lose_authority(Unacknowledged::quarantined, HostPhase::quarantined); + return; + } + committed_ = published.committed; + for (const auto& a : applied_) + acks_.push_back({a.input.request, a.tick, head.sequence}); + applied_.clear(); + for (const auto& d : published.decisions) + outcomes_.push_back({d, head.sequence}); + prepared_.reset(); + batch_started_ = now; + ++status_.commits; + status_.phase = draining_ ? HostPhase::draining : HostPhase::running; +} + +template void RegionHost::commit(SchedulerTime now) { + if (!authority_.may_begin_commit(LeaseClock::now())) + return; // Renewal first; upkeep fences if the deadline passes. + if (!database()) + return; + const auto& grant = authority_.grant(); + if (!grant || !committed_ || !batch_) + return; + const auto commit_id = next_commit_id(); + batch_->seal(); + auto published = + Ops::publish(*db_, *objects_, *grant, *committed_, *batch_, commit_id, config_.build, + config_.restore_budget, config_.lease.commit_timeout); + if (published) { + confirmed(*published, now); + return; + } + const auto& f = published.error(); + switch (f.kind) { + case PublicationError::upload: + case PublicationError::verify: + case PublicationError::digest: + ++status_.store_failures; // Nothing sent: retry the same batch later. + return; + case PublicationError::unknown: + prepared_ = f.prepared; + db_.reset(); + ++status_.unknown_outcomes; + status_.phase = HostPhase::resolving; + return; + case PublicationError::rejected: + if (f.store.kind == LeaseStoreError::fenced) { + lose_authority(Unacknowledged::fenced, HostPhase::acquiring); + return; + } + if (f.store.kind == LeaseStoreError::store || f.store.kind == LeaseStoreError::invalid) { + // SQL/connection error before any mutation: retry with the same bytes. + db_.reset(); + ++status_.store_failures; + return; + } + [[fallthrough]]; + default: // candidate, predecessor, receipt, head moved, conflicts: fail closed. + lose_authority(Unacknowledged::quarantined, HostPhase::quarantined); + return; + } +} + +template void RegionHost::resolve(SchedulerTime now) { + if (!database() || !prepared_ || !batch_) + return; + const auto& prepared = *prepared_; + auto resolved = persistence::resolve_prepared(*db_, prepared, config_.lease.commit_timeout); + if (!resolved) { + db_.reset(); + ++status_.store_failures; + return; + } + const auto adopt = [&](const persistence::RegionHead& receipt) { + auto decisions = persistence::decode_decisions(prepared.decisions, batch_->begin_tick(), + batch_->end_tick()); + const auto digest = persistence::sha256(prepared.manifest); + if (!decisions || !digest || *digest != receipt.digest || + receipt.end_tick != batch_->end_tick()) { + lose_authority(Unacknowledged::quarantined, HostPhase::quarantined); + return; + } + confirmed({{receipt, prepared.manifest}, *decisions}, now); + }; + switch (resolved->status) { + case persistence::Resolution::committed: + adopt(resolved->receipt); + return; + case persistence::Resolution::conflict: + lose_authority(Unacknowledged::quarantined, HostPhase::quarantined); + return; + case persistence::Resolution::absent: + break; + } + // Absent now: retry exactly the same bytes, only while still authorized. + if (!authority_.may_begin_commit(LeaseClock::now())) + return; + auto retried = persistence::retry_prepared(*db_, prepared, config_.lease.commit_timeout); + if (retried) { + adopt(*retried); + return; + } + switch (retried.error().kind) { + case LeaseStoreError::fenced: + lose_authority(Unacknowledged::fenced, HostPhase::acquiring); + return; + case LeaseStoreError::unknown: + case LeaseStoreError::store: + db_.reset(); + ++status_.store_failures; + return; + default: + lose_authority(Unacknowledged::quarantined, HostPhase::quarantined); + return; + } +} + +template void RegionHost::simulate(SchedulerTime now) { + const auto elapsed = std::chrono::duration_cast(now - origin_); + const auto due = elapsed.count() <= 0 + ? std::uint64_t{0} + : static_cast(elapsed / config_.tick_quantum); + if (!batch_ || !state_) + return; + auto& batch = *batch_; + const auto& state = *state_; + for (std::uint32_t k = 0; k < config_.max_catch_up_ticks && !batch.sealed(); ++k) { + if (Ops::tick(state) - origin_tick_ >= due || queue_.empty() || + !authority_.may_originate(LeaseClock::now())) + break; // Not due, starved (debt accrues; inputs are never invented) or unauthorized. + const auto input = queue_.front(); + queue_.pop_front(); + if (batch.steps().empty()) + batch_started_ = now; + auto report = batch.advance(input.input); + if (!report) { + dropped_.push_back({input, Unacknowledged::refused}); + continue; + } + applied_.push_back({input, report->tick}); + } +} + +template void RegionHost::refresh(SchedulerTime now) { + status_.queued = queue_.size(); + status_.queued_bytes = queue_.size() * tick_input_bytes; + const auto& grant = authority_.grant(); + status_.epoch = grant ? grant->epoch : 0; + status_.committed_tick = committed_ ? committed_->head.end_tick : 0; + status_.committed_sequence = committed_ ? committed_->head.sequence : 0; + status_.simulated_tick = state_ ? Ops::tick(*state_) : status_.committed_tick; + if (state_ && status_.phase == HostPhase::running) { + const auto elapsed = std::chrono::duration_cast(now - origin_); + const auto due = elapsed.count() <= 0 + ? std::uint64_t{0} + : static_cast(elapsed / config_.tick_quantum); + const auto done = Ops::tick(*state_) - origin_tick_; + status_.tick_debt = due > done ? due - done : 0; + status_.max_tick_debt = std::max(status_.max_tick_debt, status_.tick_debt); + } + status_.ready = status_.phase == HostPhase::running && state_ && !draining_ && + authority_.may_originate(LeaseClock::now()); +} + +template const HostStatus& RegionHost::step(SchedulerTime now) { + switch (status_.phase) { + case HostPhase::acquiring: + acquire(now); + break; + case HostPhase::resolving: + upkeep(); + if (status_.phase == HostPhase::resolving) + resolve(now); + break; + case HostPhase::running: + case HostPhase::draining: { + upkeep(); + if (status_.phase != HostPhase::running && status_.phase != HostPhase::draining) + break; + if (!draining_) + simulate(now); + if (!batch_) + break; // Running always has a batch; defensive. + const bool flush = + !batch_->steps().empty() && + (batch_->sealed() || draining_ || now - batch_started_ >= config_.flush_interval); + if (flush) + commit(now); // May lose authority and reset the batch. + if (draining_ && status_.phase == HostPhase::draining && batch_ && + batch_->steps().empty()) { + const auto& grant = authority_.grant(); + if (database() && grant && authority_.may_originate(LeaseClock::now())) { + // Best effort: an unreleased lease simply expires. + [[maybe_unused]] const auto released = persistence::release_region_lease( + *db_, config_.key, *grant, config_.lease.renew_timeout); + } + batch_.reset(); + state_.reset(); + authority_.fence(); + db_.reset(); + status_.phase = HostPhase::stopped; + } + break; + } + case HostPhase::stopped: + case HostPhase::quarantined: + break; + } + refresh(now); + return status_; +} + +template void RegionHost::drain() { + if (draining_) + return; + draining_ = true; + for (auto& q : queue_) + dropped_.push_back({q, Unacknowledged::drained}); + queue_.clear(); + if (status_.phase == HostPhase::running) + status_.phase = HostPhase::draining; + else if (status_.phase == HostPhase::acquiring) + status_.phase = HostPhase::stopped; + status_.ready = false; +} + +template std::vector RegionHost::take_acks() { + return std::exchange(acks_, {}); +} +template std::vector RegionHost::take_outcomes() { + return std::exchange(outcomes_, {}); +} +template std::vector RegionHost::take_dropped() { + return std::exchange(dropped_, {}); +} + +template class RegionHost; +template class RegionHost; +} // namespace evo::server diff --git a/tests/postgres_tests.cpp b/tests/postgres_tests.cpp index f3d2bf9..a21ab46 100644 --- a/tests/postgres_tests.cpp +++ b/tests/postgres_tests.cpp @@ -5,6 +5,8 @@ #include void run_region_lease_store_tests(); void run_region_checkpoint_store_tests(); +void run_region_host_tests(); +void run_region_server_process_tests(); namespace { using namespace evo::persistence; using namespace std::chrono_literals; @@ -75,6 +77,10 @@ int main() { std::puts("region lease fencing integration passed"); run_region_checkpoint_store_tests(); std::puts("region checkpoint publication integration passed"); + run_region_host_tests(); + std::puts("region host integration passed"); + run_region_server_process_tests(); + std::puts("region server process integration passed"); return 0; } catch (const std::exception& e) { std::fprintf(stderr, "%s\n", e.what()); diff --git a/tests/region_host_tests.cpp b/tests/region_host_tests.cpp new file mode 100644 index 0000000..e851afd --- /dev/null +++ b/tests/region_host_tests.cpp @@ -0,0 +1,332 @@ +// Real PostgreSQL M7.6 region host behavior (ADR-0064). Run by tools/ci/postgres-check.sh. +#include "evo/server/region_host.hpp" +#include "fixtures/environmental_region.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +void run_region_host_tests(); + +namespace { +using namespace evo::persistence; +using namespace evo::server; +using namespace std::chrono_literals; +using evo::processes::EnvironmentalCommunity; +using evo::processes::LineageRegion; +namespace fs = std::filesystem; +void check(bool value, std::source_location at = std::source_location::current()) { + if (!value) + throw std::runtime_error("region host test line " + std::to_string(at.line())); +} +#define CHECK(...) check(static_cast(__VA_ARGS__)) +std::string env(const char* key) { + const auto* value = std::getenv(key); + if (!value) + throw std::runtime_error("missing test environment"); + return value; +} +PgOptions options(const std::string& user) { + return {env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}}; +} +std::unique_ptr connect_as(const std::string& user) { + auto made = Postgres::connect(options(user), 5s); + CHECK(made); + return std::move(*made); +} +Connect runtime_connect() { + return [] { return Postgres::connect(options(env("EVO_PG_RUNTIME_USER")), 2s); }; +} +template std::array filled(std::uint8_t value) { + std::array out{}; + out.fill(value); + return out; +} +const Sha256Digest build = filled<32>(0xB1); +// Fast lease for tests: 4 s, renew every 0.8 s, 0.4 s margin/renew deadline, 1 s commits. +const LeasePolicy fast{4000ms, 800ms, 400ms, 400ms, 1000ms}; +TickInput input(std::uint64_t tick, bool consumer_dies = false) { + Uuid request{}; + request[0] = 0x9A; + for (unsigned i = 0; i < 8; ++i) + request[8 + i] = static_cast((tick >> (8U * i)) & 0xFFU); + return {request, evo::test::region_input(tick, consumer_dies)}; +} + +struct Region { + RegionKey key{7, 21}; + fs::path root; +}; +// Provisions a fresh region (owner) and returns its key and object root. +template Region provisioned(std::uint64_t region, std::uint8_t genesis) { + Region r{{7, 20 + region}, fs::path(env("PGHOST")) / ("host-" + std::to_string(region))}; + fs::remove_all(r.root); + fs::create_directory(r.root); + fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace); + auto owner = connect_as(env("PGUSER")); + CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " + "(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING", + {}, 5s)); + auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes); + CHECK(objects); + if constexpr (std::is_same_v) + CHECK(provision_lineage_region(*owner, **objects, r.key, evo::test::region_identity.frame, + *evo::test::lineage_region(), filled<16>(genesis), build, + 8000000, 5s)); + else + CHECK(provision_environmental_region( + *owner, **objects, r.key, evo::test::region_identity.frame, + *evo::test::region_community(), filled<16>(genesis), build, 8000000, 5s)); + return r; +} +HostConfig config(const Region& r) { + HostConfig c; + c.key = r.key; + c.frame = evo::test::region_identity.frame; + c.lease = fast; + c.build = build; + c.retained_builds = {build}; + return c; +} +template +std::unique_ptr> host(const Region& r, HostConfig c, + ObjectStoreFaults* faults = nullptr) { + auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes, faults); + CHECK(objects); + auto made = RegionHost::create(std::move(c), runtime_connect(), std::move(*objects)); + CHECK(made); + return std::move(*made); +} +// Steps until `done` holds, advancing scheduling time by `advance` each step. +template +const HostStatus& step_until(Host& h, SchedulerTime& t, std::chrono::microseconds advance, + Done done, int limit = 400) { + for (int i = 0; i < limit; ++i) { + t += advance; + const auto& s = h.step(t); + if (done(s)) + return s; + std::this_thread::sleep_for(10ms); + } + CHECK(false); + return h.status(); +} + +// Admission, provisional ticks, pause at a permanent outcome and final acknowledgement. +void acknowledgements_follow_commits() { + const auto r = provisioned(1, 0x3A); + auto h = host(r, config(r)); + CHECK(h->submit(input(1)).error() == AdmissionError::not_ready); + SchedulerTime t = std::chrono::steady_clock::now(); + auto s = h->step(t); + CHECK(s.phase == HostPhase::running && s.ready && s.committed_tick == 0 && s.epoch == 1); + for (std::uint64_t tick = 1; tick <= 64; ++tick) + CHECK(h->submit(input(tick, true))); + CHECK(h->submit(input(65)).error() == AdmissionError::full); // Backlog of 64. + CHECK(h->submit(input(1)).error() == AdmissionError::duplicate); + CHECK(h->submit({{}, {}}).error() == AdmissionError::invalid); + CHECK(h->status().queued_bytes == 64 * tick_input_bytes); + for (std::uint64_t tick = 1; tick <= 19; ++tick) { + t += 50ms; + s = h->step(t); + // Simulated ahead of committed; nothing is final yet. + CHECK(s.simulated_tick == tick && s.committed_tick == 0 && s.ready); + CHECK(h->take_acks().empty() && h->take_outcomes().empty()); + } + // Tick 20 kills the consumer: the region pauses and commits in the same step, + // so no dependent tick ever runs on the uncommitted death. + t += 50ms; + s = h->step(t); + CHECK(s.simulated_tick == 20 && s.committed_tick == 20 && s.committed_sequence == 1); + const auto outcomes = h->take_outcomes(); + CHECK(outcomes.size() == 1 && outcomes[0].commit_sequence == 1 && + outcomes[0].decision == BatchDecision{20, Actor::consumer, Outcome::death, 11}); + auto acks = h->take_acks(); + CHECK(acks.size() == 20); + for (std::uint64_t i = 0; i < 20; ++i) + CHECK(acks[i] == FinalAck{input(i + 1).request, i + 1, 1}); + // The next batch flushes after the configured interval, not per tick. + bool provisional = false; + s = step_until(*h, t, 50ms, [&](const HostStatus& x) { + provisional = provisional || (x.simulated_tick > x.committed_tick); + return x.committed_sequence == 2; + }); + CHECK(provisional && s.simulated_tick == s.committed_tick && s.committed_tick > 30); + acks = h->take_acks(); + CHECK(!acks.empty() && acks.back().tick == s.committed_tick && acks.front().tick == 21); + CHECK(std::ranges::all_of(acks, [](const FinalAck& a) { return a.commit_sequence == 2; })); + CHECK(h->take_dropped().empty()); + std::printf("host_commits=%llu committed_tick=%llu\n", + static_cast(s.commits), + static_cast(s.committed_tick)); +} + +// No inputs, no invented ticks: debt accrues and is repaid with bounded catch-up. +void starvation_accrues_bounded_debt() { + const auto r = provisioned(2, 0x3B); + auto h = host(r, config(r)); + SchedulerTime t = std::chrono::steady_clock::now(); + CHECK(h->step(t).ready); + for (int i = 0; i < 10; ++i) { + t += 50ms; + CHECK(h->step(t).simulated_tick == 0); + } + CHECK(h->status().tick_debt == 10); + for (std::uint64_t tick = 1; tick <= 20; ++tick) + CHECK(h->submit(input(tick))); + auto s = h->step(t); + CHECK(s.simulated_tick == 8 && s.tick_debt == 2); // At most 8 per step. + s = h->step(t); + CHECK(s.simulated_tick == 10 && s.tick_debt == 0 && s.max_tick_debt == 10); + CHECK(s.queued == 10); +} + +// Object store outage: the sealed batch waits, nothing is acknowledged, and the +// region resumes once storage is durable again. +void storage_outage_pauses_the_region() { + const auto r = provisioned(3, 0x41); + ObjectStoreFaults faults; + auto h = host(r, config(r), &faults); + SchedulerTime t = std::chrono::steady_clock::now(); + CHECK(h->step(t).ready); + for (std::uint64_t tick = 1; tick <= 30; ++tick) + CHECK(h->submit(input(tick, true))); + step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 19; }); + faults.fail_directory_syncs = 1000; + for (int i = 0; i < 5; ++i) { + t += 50ms; + const auto& s = h->step(t); + CHECK(s.simulated_tick == 20 && s.committed_tick == 0); // Paused at the death. + } + CHECK(h->status().store_failures >= 5 && h->take_acks().empty() && h->take_outcomes().empty()); + CHECK(h->status().tick_debt == 4); + faults.fail_directory_syncs = 0; + t += 50ms; + auto s = h->step(t); + CHECK(s.committed_tick == 20 && h->take_acks().size() == 20 && h->take_outcomes().size() == 1); + s = h->step(t); // Catch-up after the outage is bounded but continues. + CHECK(s.simulated_tick > 20 && s.simulated_tick <= 28); +} + +// Holds the lease row lock to make database calls time out. +struct RowLock { + std::unique_ptr db = connect_as(env("PGUSER")); + explicit RowLock(RegionKey key) { + const std::array p{std::to_string(key.world), std::to_string(key.region)}; + CHECK(db->query("BEGIN", {}, 5s)); + CHECK(db->query("SELECT 1 FROM evolution.region_leases WHERE world_id = $1::evolution.u64 " + "AND region_id = $2::evolution.u64 FOR UPDATE", + p, 5s)); + } + void release() { + CHECK(db->query("ROLLBACK", {}, 5s)); + } +}; + +// Unknown commit outcome: not ready, admission refused, resolved by exact bytes. +void unknown_outcome_is_resolved_before_continuing() { + const auto r = provisioned(4, 0x3D); + auto h = host(r, config(r)); + SchedulerTime t = std::chrono::steady_clock::now(); + CHECK(h->step(t).ready); + for (std::uint64_t tick = 1; tick <= 25; ++tick) + CHECK(h->submit(input(tick, true))); + step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 19; }); + RowLock lock{r.key}; + t += 50ms; + auto s = h->step(t); // The death seals; the commit cannot finish in time. + CHECK(s.phase == HostPhase::resolving && !s.ready && s.unknown_outcomes == 1); + CHECK(h->submit(input(26)).error() == AdmissionError::not_ready); + CHECK(h->take_outcomes().empty() && h->take_acks().empty()); + lock.release(); + s = step_until(*h, t, 0ms, [](const HostStatus& x) { return x.phase == HostPhase::running; }); + CHECK(s.committed_tick == 20 && s.ready); + CHECK(h->take_outcomes().size() == 1 && h->take_acks().size() == 20); +} + +// Renewals that cannot reach the database fence the host locally; unconfirmed +// work is reported as never final, and authority is re-acquired from the head. +void renewal_failure_fences_and_recovers() { + const auto r = provisioned(5, 0x3E); + auto h = host(r, config(r)); + SchedulerTime t = std::chrono::steady_clock::now(); + CHECK(h->step(t).ready); + for (std::uint64_t tick = 1; tick <= 8; ++tick) + CHECK(h->submit(input(tick))); + step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 5; }); + CHECK(h->status().committed_tick == 0); + const auto epoch = h->status().epoch; + std::optional lock{std::in_place, r.key}; + step_until(*h, t, 0ms, [](const HostStatus& x) { return x.phase != HostPhase::running; }, 1000); + CHECK(!h->status().ready && h->status().fences == 1 && h->state() == nullptr); + const auto dropped = h->take_dropped(); + CHECK(dropped.size() == 8 && std::ranges::all_of(dropped, [](const Dropped& d) { + return d.reason == Unacknowledged::fenced; + })); + CHECK(h->take_acks().empty()); + lock->release(); + lock.reset(); + const auto& s = step_until( + *h, t, 0ms, [](const HostStatus& x) { return x.phase == HostPhase::running; }, 1000); + CHECK(s.epoch > epoch && s.committed_tick == 0 && s.simulated_tick == 0 && s.ready); +} + +// Drain: backlog dropped, pending ticks committed, lease released for an +// immediate successor (environmental-v1 profile). +void drain_commits_and_hands_over() { + const auto r = provisioned(6, 0x3F); + auto h = host(r, config(r)); + SchedulerTime t = std::chrono::steady_clock::now(); + CHECK(h->step(t).ready); + for (std::uint64_t tick = 1; tick <= 9; ++tick) + CHECK(h->submit(input(tick))); + step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 6; }); + h->drain(); + CHECK(!h->status().ready && h->submit(input(10)).error() == AdmissionError::draining); + auto dropped = h->take_dropped(); + CHECK(dropped.size() == 3 && dropped[0].reason == Unacknowledged::drained); + const auto& s = h->step(t); + CHECK(s.phase == HostPhase::stopped && s.committed_tick == 6); + CHECK(h->take_acks().size() == 6); + auto successor = host(r, config(r)); + const auto& next = successor->step(t); + CHECK(next.phase == HostPhase::running && next.committed_tick == 6 && next.epoch == 2); + successor->drain(); + CHECK(successor->step(t).phase == HostPhase::stopped); +} + +// A committed head this host cannot restore quarantines; it is never replaced. +void unrestorable_head_quarantines() { + const auto r = provisioned(7, 0x40); + auto c = config(r); + c.build = filled<32>(0xB2); + c.retained_builds = {c.build}; // Not the build that produced the head. + auto h = host(r, c); + const auto& s = h->step(std::chrono::steady_clock::now()); + CHECK(s.phase == HostPhase::quarantined && !s.ready && h->state() == nullptr); + CHECK(h->submit(input(1)).error() == AdmissionError::not_ready); + // Wrong frame and invalid configuration are refused too. + auto wrong = config(r); + wrong.frame = 10; + CHECK(host(r, wrong)->step(std::chrono::steady_clock::now()).phase == + HostPhase::quarantined); + auto invalid = config(r); + invalid.retained_builds = {}; + auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes); + CHECK(!RegionHost::create(invalid, runtime_connect(), std::move(*objects))); +} +} // namespace + +void run_region_host_tests() { + acknowledgements_follow_commits(); + starvation_accrues_bounded_debt(); + storage_outage_pauses_the_region(); + unknown_outcome_is_resolved_before_continuing(); + renewal_failure_fences_and_recovers(); + drain_commits_and_hands_over(); + unrestorable_head_quarantines(); +} diff --git a/tests/region_server_process_tests.cpp b/tests/region_server_process_tests.cpp new file mode 100644 index 0000000..52e5ffd --- /dev/null +++ b/tests/region_server_process_tests.cpp @@ -0,0 +1,334 @@ +// Real process tests for evo-server (M7.6, ADR-0064): SIGTERM drain and SIGKILL +// with recovery by a successor process. Run by tools/ci/postgres-check.sh, which +// sets EVO_SERVER_BIN. +#include "evo/persistence/checkpoint_store.hpp" +#include "fixtures/environmental_region.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern char** environ; +void run_region_server_process_tests(); + +namespace { +using namespace evo::persistence; +using namespace std::chrono_literals; +namespace fs = std::filesystem; +using Clock = std::chrono::steady_clock; +void check(bool value, std::source_location at = std::source_location::current()) { + if (!value) + throw std::runtime_error("region server process test line " + std::to_string(at.line())); +} +#define CHECK(...) check(static_cast(__VA_ARGS__)) +std::string env(const char* key) { + const auto* value = std::getenv(key); + if (!value) + throw std::runtime_error("missing test environment"); + return value; +} +std::unique_ptr owner() { + auto made = Postgres::connect( + {env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("PGUSER"), {}, {}}, 5s); + CHECK(made); + return std::move(*made); +} +std::string hex32(std::uint64_t tick) { + char out[33]; + std::snprintf(out, sizeof out, "9b000000000000000000000000%06llx", + static_cast(tick)); + return out; +} +std::string input_line(std::uint64_t tick, bool consumer_dies) { + const auto in = evo::test::region_input(tick, consumer_dies); + std::string line = "input " + hex32(tick) + " " + std::to_string(in.temperature_mK) + " " + + std::to_string(in.source_enabled); + for (const auto b : in.bite) + line += " " + std::to_string(b); + for (const auto t : in.terminal) + line += " " + std::to_string(t); + return line + "\n"; +} + +class Child { + public: + explicit Child(const std::vector& args) { + int to_child[2], from_child[2]; + CHECK(::pipe(to_child) == 0 && ::pipe(from_child) == 0); + posix_spawn_file_actions_t actions; + posix_spawn_file_actions_init(&actions); + posix_spawn_file_actions_adddup2(&actions, to_child[0], STDIN_FILENO); + posix_spawn_file_actions_adddup2(&actions, from_child[1], STDOUT_FILENO); + posix_spawn_file_actions_addclose(&actions, to_child[1]); + posix_spawn_file_actions_addclose(&actions, from_child[0]); + std::vector argv; + argv.reserve(args.size() + 1); + for (const auto& a : args) + argv.push_back(const_cast(a.c_str())); + argv.push_back(nullptr); + CHECK(posix_spawn(&pid_, argv[0], &actions, nullptr, argv.data(), environ) == 0); + posix_spawn_file_actions_destroy(&actions); + ::close(to_child[0]); + ::close(from_child[1]); + in_ = to_child[1]; + out_ = from_child[0]; + } + Child(const Child&) = delete; + Child& operator=(const Child&) = delete; + ~Child() { + if (pid_ > 0) { + ::kill(pid_, SIGKILL); + ::waitpid(pid_, nullptr, 0); + } + ::close(in_); + ::close(out_); + } + void send(const std::string& text) { + std::size_t done = 0; + while (done < text.size()) { + const auto n = ::write(in_, text.data() + done, text.size() - done); + CHECK(n > 0 || errno == EINTR); + if (n > 0) + done += static_cast(n); + } + } + // Reads lines until one satisfies `want`; every line is kept in lines(). + template bool wait_for(Want want, std::chrono::milliseconds timeout) { + const auto deadline = Clock::now() + timeout; + for (std::size_t seen = 0;;) { + for (; seen < lines_.size(); ++seen) + if (want(lines_[seen])) + return true; + const auto left = + std::chrono::duration_cast(deadline - Clock::now()); + if (left.count() <= 0 || !read_some(static_cast(left.count()))) + return false; + } + } + void signal(int sig) { + CHECK(::kill(pid_, sig) == 0); + } + // Exit code, or 128+signal; drains remaining output. + int wait_exit(std::chrono::milliseconds timeout) { + const auto deadline = Clock::now() + timeout; + int status = 0; + for (;;) { + read_some(20); + const auto r = ::waitpid(pid_, &status, WNOHANG); + if (r == pid_) + break; + CHECK(Clock::now() < deadline); + } + while (read_some(20)) { + } + pid_ = -1; + return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status); + } + [[nodiscard]] const std::vector& lines() const noexcept { + return lines_; + } + + private: + bool read_some(int timeout_ms) { + pollfd fd{out_, POLLIN, 0}; + if (::poll(&fd, 1, timeout_ms) <= 0) + return false; + char chunk[4096]; + const auto n = ::read(out_, chunk, sizeof chunk); + if (n <= 0) + return false; + buffer_.append(chunk, static_cast(n)); + for (auto end = buffer_.find('\n'); end != std::string::npos; end = buffer_.find('\n')) { + lines_.push_back(buffer_.substr(0, end)); + buffer_.erase(0, end + 1); + } + return true; + } + pid_t pid_{-1}; + int in_{-1}, out_{-1}; + std::string buffer_; + std::vector lines_; +}; + +struct Region { + RegionKey key; + fs::path root; +}; +Region provision(std::uint64_t region, std::uint8_t genesis) { + Region r{{7, region}, fs::path(env("PGHOST")) / ("server-" + std::to_string(region))}; + fs::remove_all(r.root); + fs::create_directory(r.root); + fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace); + auto db = owner(); + CHECK(db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " + "(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING", + {}, 5s)); + auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes); + CHECK(objects); + Sha256Digest build{}; + build.fill(0xB1); + Uuid commit{}; + commit.fill(genesis); + CHECK(provision_lineage_region(*db, **objects, r.key, evo::test::region_identity.frame, + *evo::test::lineage_region(), commit, build, 8000000, 5s)); + return r; +} +std::vector server_args(const Region& r) { + return {env("EVO_SERVER_BIN"), + "--profile", + "lineage", + "--world", + "7", + "--region", + std::to_string(r.key.region), + "--frame", + std::to_string(evo::test::region_identity.frame), + "--pg-host", + env("PGHOST"), + "--pg-port", + env("PGPORT"), + "--pg-database", + env("PGDATABASE"), + "--pg-user", + env("EVO_PG_RUNTIME_USER"), + "--objects", + r.root.string(), + "--build", + "b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1", + "--flush-ms", + "500", + "--lease-profile", + "fast", + "--ready-file", + (r.root / "ready").string()}; +} +// (tick, commit_sequence) of every ack line. +std::vector> acks(const Child& c) { + std::vector> out; + for (const auto& line : c.lines()) { + unsigned long long tick = 0, sequence = 0; + char request[40]; + if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &tick, &sequence) == 3) + out.emplace_back(tick, sequence); + } + return out; +} +std::string head_of(RegionKey key) { + const std::array p{std::to_string(key.region)}; + auto rows = owner()->query("SELECT h.end_tick::text || ':' || h.commit_sequence::text || ':' " + "|| coalesce(l.holder::text, 'released') FROM " + "evolution.region_heads h JOIN evolution.region_leases l USING " + "(world_id, region_id) WHERE world_id = 7 AND region_id = " + "$1::evolution.u64", + p, 5s); + CHECK(rows && rows->size() == 1); + const auto& value = (*rows)[0][0]; + CHECK(value.has_value()); + return value.value_or(""); +} +std::uint64_t head_tick(RegionKey key) { + return std::stoull(head_of(key).substr(0, head_of(key).find(':'))); +} +bool starts(const std::string& line, std::string_view prefix) { + return line.starts_with(prefix); +} + +// SIGTERM: stop admission, commit pending ticks, release the lease, exit 0. Every +// acknowledged tick is committed, the backlog never exceeds its bound, and the +// permanent outcome was announced only after its commit. +void sigterm_drains_and_releases() { + const auto r = provision(31, 0x42); + Child server{server_args(r)}; + CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); + CHECK(fs::exists(r.root / "ready")); + std::string burst; + for (std::uint64_t tick = 1; tick <= 80; ++tick) + burst += input_line(tick, true); + server.send(burst); // More than the 64-input backlog: stdin backpressures. + CHECK(server.wait_for( + [](const std::string& l) { return starts(l, "ack " + hex32(80) + " 80 "); }, 30s)); + server.signal(SIGTERM); + CHECK(server.wait_exit(20s) == 0); + const auto& lines = server.lines(); + CHECK(std::ranges::find(lines, "exit 0") != lines.end()); + CHECK(std::ranges::find_if(lines, [](const std::string& l) { + return starts(l, "phase stopped"); + }) != lines.end()); + unsigned long long max_queued = 999; + for (const auto& l : lines) + std::sscanf(l.c_str(), "stats max_queued=%llu", &max_queued); + CHECK(max_queued <= 64); + const auto all = acks(server); + CHECK(all.size() == 80 && all.back().first == 80); + // The death at tick 20 is announced only with its commit: after the final + // acknowledgement of tick 20, which the host emits only after confirmation. + const auto outcome = std::ranges::find_if( + lines, [](const std::string& l) { return starts(l, "outcome 20 1 11 "); }); + const auto ack20 = std::ranges::find_if( + lines, [](const std::string& l) { return starts(l, "ack " + hex32(20) + " 20 "); }); + CHECK(outcome != lines.end() && ack20 != lines.end() && ack20 < outcome); + CHECK(head_of(r.key).starts_with("80:") && head_of(r.key).ends_with(":released")); + CHECK(!fs::exists(r.root / "ready")); + std::printf("server_sigterm commits=%s acks=%zu max_queued=%llu\n", head_of(r.key).c_str(), + all.size(), max_queued); +} + +// SIGKILL: no drain. Nothing acknowledged is lost, nothing unacknowledged was +// claimed, and a successor becomes ready only after the database lease expires. +void sigkill_recovers_through_a_successor() { + const auto r = provision(32, 0x43); + std::uint64_t acked_max = 0; + { + Child server{server_args(r)}; + CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); + std::string burst; + for (std::uint64_t tick = 1; tick <= 60; ++tick) + burst += input_line(tick, false); + server.send(burst); + CHECK(server.wait_for([](const std::string& l) { return starts(l, "ack "); }, 15s)); + std::this_thread::sleep_for(300ms); // Ticks run past the last commit. + server.signal(SIGKILL); + CHECK(server.wait_exit(10s) == 128 + SIGKILL); + for (const auto& [tick, sequence] : acks(server)) { + static_cast(sequence); + acked_max = std::max(acked_max, tick); + } + } + const auto committed = head_tick(r.key); + CHECK(acked_max > 0 && committed >= acked_max); // No acknowledged tick was lost. + CHECK(!head_of(r.key).ends_with(":released")); // The killed holder never released. + const auto started = Clock::now(); + Child successor{server_args(r)}; + CHECK(successor.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); + const auto to_ready = + std::chrono::duration_cast(Clock::now() - started).count(); + const auto running = std::ranges::find_if( + successor.lines(), [](const std::string& l) { return starts(l, "phase running"); }); + CHECK(running != successor.lines().end()); + CHECK(running->find("committed=" + std::to_string(committed) + " ") != std::string::npos); + CHECK(running->find("epoch=2 ") != std::string::npos); + successor.signal(SIGTERM); + CHECK(successor.wait_exit(20s) == 0); + CHECK(head_tick(r.key) == committed && head_of(r.key).ends_with(":released")); + std::printf("server_sigkill acked_max=%llu committed=%llu successor_ready_ms=%lld\n", + static_cast(acked_max), + static_cast(committed), static_cast(to_ready)); +} +} // namespace + +void run_region_server_process_tests() { + sigterm_drains_and_releases(); + sigkill_recovers_through_a_successor(); +} diff --git a/tools/ci/postgres-check.sh b/tools/ci/postgres-check.sh index de93066..5602209 100644 --- a/tools/ci/postgres-check.sh +++ b/tools/ci/postgres-check.sh @@ -17,7 +17,10 @@ psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_runtime_test NOLOGIN' \ export EVO_PG_RUNTIME_ROLE=evo_runtime_test EVO_PG_RUNTIME_USER=evo_runtime_login bash "$root/tools/migrate-postgres.sh" bash "$root/tools/migrate-postgres.sh" # exact supported upgrade/restart is a no-op -"${1:?path to evo-postgres-tests}" +# Process tests spawn the server built beside the test binary. +export EVO_SERVER_BIN +EVO_SERVER_BIN="$(dirname "${1:?path to evo-postgres-tests}")/evo-server" +"$1" v1_sha="$(sha256sum "$root/migrations/0001_region_bootstrap.sql")" v1_sha="${v1_sha%% *}" # Migration failure must not publish a partial schema.