diff --git a/CMakeLists.txt b/CMakeLists.txt index 4d78156..73e402e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -103,11 +103,21 @@ if(EVO_BUILD_POSTGRES) target_link_libraries(evo_region_store PUBLIC evo::postgres evo::region_checkpoint PRIVATE OpenSSL::Crypto) evo_owned_target(evo_region_store) + # M7.6 authoritative region host and its process (ADR-0064). + add_library(evo_region_host STATIC src/server/region_host.cpp) + add_library(evo::region_host ALIAS evo_region_host) + target_link_libraries(evo_region_host PUBLIC evo::region_store) + evo_owned_target(evo_region_host) + add_executable(evo-server apps/evo-server/main.cpp) + target_link_libraries(evo-server PRIVATE evo::region_host) + evo_owned_target(evo-server) if(BUILD_TESTING) # Contention tests use one thread and one connection per contender. add_executable(evo-postgres-tests tests/postgres_tests.cpp tests/region_lease_store_tests.cpp - tests/region_checkpoint_store_tests.cpp) - target_link_libraries(evo-postgres-tests PRIVATE evo::region_store Threads::Threads) + tests/region_checkpoint_store_tests.cpp tests/region_host_tests.cpp + tests/region_server_process_tests.cpp) + target_link_libraries(evo-postgres-tests PRIVATE evo::region_host Threads::Threads) + add_dependencies(evo-postgres-tests evo-server) # Spawned by the process tests. evo_owned_target(evo-postgres-tests) # Intentionally not an ordinary CTest: tools/ci/postgres-check.sh owns its private DB. endif() diff --git a/apps/evo-server/main.cpp b/apps/evo-server/main.cpp new file mode 100644 index 0000000..f980e81 --- /dev/null +++ b/apps/evo-server/main.cpp @@ -0,0 +1,373 @@ +// M7.6 region server process (ADR-0064). Development host for one region: tick +// inputs arrive as lines on stdin (the network gateway is later work); events are +// written as lines on stdout. Linux only; requires the PostgreSQL build. +// +// Input: input +// Output: phase epoch= committed= simulated= +// ready <0|1> +// rejected | dropped +// ack +// outcome +// stats max_queued= max_tick_debt= commits= store_failures= +// Exit: 0 drained, 2 usage, 3 drain deadline passed, 4 quarantined. +#include "evo/server/region_host.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +using namespace evo; +using namespace std::chrono_literals; +volatile std::sig_atomic_t stop_requested = 0; +extern "C" void on_stop(int) { + stop_requested = 1; +} + +struct Options { + std::string profile{"lineage"}; + persistence::PgOptions pg; + std::filesystem::path objects, ready_file; + server::HostConfig host; + std::chrono::milliseconds drain{10000}; +}; + +std::optional number(std::string_view text) { + std::uint64_t value{}; + const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), value); + if (error != std::errc{} || end != text.data() + text.size()) + return std::nullopt; + return value; +} +template std::optional> hex(std::string_view text) { + if (text.size() != N * 2) + return std::nullopt; + std::array out{}; + for (std::size_t i = 0; i < N; ++i) { + unsigned value{}; + const auto [end, error] = + std::from_chars(text.data() + (2 * i), text.data() + (2 * i) + 2, value, 16); + if (error != std::errc{} || end != text.data() + (2 * i) + 2) + return std::nullopt; + out[i] = static_cast(value); + } + return out; +} +template std::string text(const std::array& bytes) { + constexpr char digits[] = "0123456789abcdef"; + std::string out; + for (const auto b : bytes) { + out.push_back(digits[b >> 4U]); + out.push_back(digits[b & 0x0FU]); + } + return out; +} + +std::optional parse(int argc, char** argv) { + Options o; + o.host.retained_builds.clear(); + for (int i = 1; i < argc; ++i) { + const std::string_view flag = argv[i]; + if (i + 1 >= argc) + return std::nullopt; + const std::string_view value = argv[++i]; + const auto n = number(value); + if (flag == "--profile" && (value == "lineage" || value == "environmental")) + o.profile = value; + else if (flag == "--world" && n) + o.host.key.world = *n; + else if (flag == "--region" && n) + o.host.key.region = *n; + else if (flag == "--frame" && n) + o.host.frame = *n; + else if (flag == "--pg-host") + o.pg.host = value; + else if (flag == "--pg-hostaddr") + o.pg.hostaddr = value; + else if (flag == "--pg-port") + o.pg.port = value; + else if (flag == "--pg-database") + o.pg.database = value; + else if (flag == "--pg-user") + o.pg.user = value; + else if (flag == "--pg-passfile") + o.pg.password_file = value; + else if (flag == "--pg-ca") + o.pg.ca_file = value; + else if (flag == "--objects") + o.objects = value; + else if (flag == "--ready-file") + o.ready_file = value; + else if (flag == "--build" || flag == "--retained") { + const auto digest = hex<32>(value); + if (!digest) + return std::nullopt; + if (flag == "--build") + o.host.build = *digest; + else + o.host.retained_builds.push_back(*digest); + } else if (flag == "--flush-ms" && n && *n > 0 && *n <= 60000) + o.host.flush_interval = std::chrono::milliseconds{*n}; + else if (flag == "--drain-ms" && n && *n > 0 && *n <= 600000) + o.drain = std::chrono::milliseconds{*n}; + else if (flag == "--lease-profile" && value == "fast") // Development/test only. + o.host.lease = {4000ms, 800ms, 400ms, 400ms, 1000ms}; + else if (flag == "--lease-profile" && value == "default") + o.host.lease = {}; + else + return std::nullopt; + } + if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end()) + o.host.retained_builds.push_back(o.host.build); + return o; +} + +std::optional parse_input(std::string_view line) { + std::array fields{}; + std::size_t count = 0; + while (!line.empty() && count < fields.size() + 1) { + const auto start = line.find_first_not_of(' '); + if (start == std::string_view::npos) + break; + line.remove_prefix(start); + const auto end = line.find(' '); + if (count == fields.size()) + return std::nullopt; // Too many fields. + fields[count++] = line.substr(0, end); + line.remove_prefix(end == std::string_view::npos ? line.size() : end); + } + if (count != fields.size() || fields[0] != "input") + return std::nullopt; + auto request = hex<16>(fields[1]); + if (!request) + return std::nullopt; + std::array v{}; + for (std::size_t i = 0; i < v.size(); ++i) { + const auto n = number(fields[i + 2]); + if (!n) + return std::nullopt; + v[i] = *n; + } + server::TickInput in{*request, {}}; + in.input.temperature_mK = v[0]; + in.input.source_enabled = v[1]; + for (std::size_t k = 0; k < 5; ++k) + in.input.bite[k] = v[2 + k]; + in.input.terminal = {v[7], v[8]}; + return in; +} + +const char* phase_name(server::HostPhase p) { + switch (p) { + case server::HostPhase::acquiring: + return "acquiring"; + case server::HostPhase::running: + return "running"; + case server::HostPhase::resolving: + return "resolving"; + case server::HostPhase::draining: + return "draining"; + case server::HostPhase::stopped: + return "stopped"; + case server::HostPhase::quarantined: + return "quarantined"; + } + return "unknown"; +} +const char* admission_name(server::AdmissionError e) { + switch (e) { + case server::AdmissionError::not_ready: + return "not_ready"; + case server::AdmissionError::draining: + return "draining"; + case server::AdmissionError::full: + return "full"; + case server::AdmissionError::duplicate: + return "duplicate"; + case server::AdmissionError::invalid: + return "invalid"; + } + return "unknown"; +} +const char* dropped_name(server::Unacknowledged u) { + switch (u) { + case server::Unacknowledged::refused: + return "refused"; + case server::Unacknowledged::drained: + return "drained"; + case server::Unacknowledged::fenced: + return "fenced"; + case server::Unacknowledged::quarantined: + return "quarantined"; + } + return "unknown"; +} + +// Reads complete, bounded lines from stdin without blocking. +class LineReader { + public: + // False on EOF/error. + bool fill() { + std::array chunk{}; + const auto n = ::read(STDIN_FILENO, chunk.data(), chunk.size()); + if (n <= 0) + return n < 0 && errno == EINTR; + buffer_.append(chunk.data(), static_cast(n)); + return true; + } + std::optional next() { + const auto end = buffer_.find('\n'); + if (end == std::string::npos) { + if (buffer_.size() > 512) + buffer_.clear(); // Overlong line: discarded, never partially parsed. + return std::nullopt; + } + auto line = buffer_.substr(0, end); + buffer_.erase(0, end + 1); + return line; + } + + private: + std::string buffer_; +}; + +template int serve(Options& o) { + auto objects = persistence::FilesystemObjectStore::open(o.objects, o.host.key, + persistence::max_lineage_payload_bytes); + if (!objects) { + std::puts("error objects"); + return 2; + } + const auto pg = o.pg; + auto made = server::RegionHost::create( + o.host, [pg] { return persistence::Postgres::connect(pg, 2s); }, std::move(*objects)); + if (!made) { + std::puts("error config"); + return 2; + } + auto& host = **made; + LineReader reader; + bool input_open = true, ready = false; + std::optional pending; + std::optional phase; + std::size_t max_queued = 0; + std::optional drain_deadline; + const auto set_ready = [&](bool now) { + if (now == ready) + return; + ready = now; + std::printf("ready %d\n", ready ? 1 : 0); + if (!o.ready_file.empty()) { + std::error_code ignored; + if (ready) + std::ofstream(o.ready_file) << "ready\n"; + else + std::filesystem::remove(o.ready_file, ignored); + } + }; + for (;;) { + if (stop_requested && !drain_deadline) { + host.drain(); + drain_deadline = std::chrono::steady_clock::now() + o.drain; + } + // Read only while the bounded backlog has room: stdin itself backpressures. + const bool room = host.status().queued < o.host.max_queued_inputs && !pending; + pollfd fd{STDIN_FILENO, POLLIN, 0}; + if (input_open && room && !drain_deadline && ::poll(&fd, 1, 5) > 0) + input_open = reader.fill(); + else if (!input_open || !room || drain_deadline) + ::usleep(2000); + while (!drain_deadline) { + if (!pending) + pending = reader.next(); + if (!pending) + break; + auto in = parse_input(*pending); + if (!in) { + std::puts("rejected - malformed"); + pending.reset(); + continue; + } + auto admitted = host.submit(*in); + if (!admitted && admitted.error() == server::AdmissionError::full) + break; // Keep the line and retry after ticks drain the backlog. + if (!admitted) + std::printf("rejected %s %s\n", text(in->request).c_str(), + admission_name(admitted.error())); + pending.reset(); + } + const auto& s = host.step(std::chrono::steady_clock::now()); + max_queued = std::max(max_queued, s.queued); + if (!phase || *phase != s.phase) { + phase = s.phase; + std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n", phase_name(s.phase), + static_cast(s.epoch), + static_cast(s.committed_tick), + static_cast(s.simulated_tick)); + } + set_ready(s.ready); + for (const auto& a : host.take_acks()) + std::printf("ack %s %llu %llu\n", text(a.request).c_str(), + static_cast(a.tick), + static_cast(a.commit_sequence)); + for (const auto& x : host.take_outcomes()) + std::printf("outcome %llu %u %llu %llu\n", + static_cast(x.decision.tick), + static_cast(x.decision.actor), + static_cast(x.decision.organism), + static_cast(x.commit_sequence)); + for (const auto& d : host.take_dropped()) + std::printf("dropped %s %s\n", text(d.input.request).c_str(), dropped_name(d.reason)); + std::fflush(stdout); + const auto finish = [&](int code) { + set_ready(false); + std::printf( + "stats max_queued=%zu max_tick_debt=%llu commits=%llu store_failures=%llu\n", + max_queued, static_cast(s.max_tick_debt), + static_cast(s.commits), + static_cast(s.store_failures)); + std::printf("exit %d\n", code); + std::fflush(stdout); + return code; + }; + if (s.phase == server::HostPhase::stopped) + return finish(0); + if (s.phase == server::HostPhase::quarantined) + return finish(4); + if (drain_deadline && std::chrono::steady_clock::now() > *drain_deadline) + return finish(3); // Pending/unknown work is left for the next owner to reconcile. + } +} +} // namespace + +int main(int argc, char** argv) { + auto options = parse(argc, argv); + if (!options) { + std::fputs("usage: evo-server --profile lineage|environmental --world N --region N " + "--frame N --pg-host H --pg-port P --pg-database D --pg-user U " + "--objects DIR --build HEX64 [--retained HEX64]... [--pg-hostaddr A " + "--pg-passfile F --pg-ca F] [--flush-ms N] [--drain-ms N] " + "[--lease-profile default|fast] [--ready-file PATH]\n", + stderr); + return 2; + } + struct sigaction action{}; + action.sa_handler = on_stop; + sigemptyset(&action.sa_mask); + sigaction(SIGTERM, &action, nullptr); + sigaction(SIGINT, &action, nullptr); + std::signal(SIGPIPE, SIG_IGN); + return options->profile == "lineage" ? serve(*options) + : serve(*options); +} diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index f6a6092..ffa63c7 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -554,3 +554,16 @@ M7.5 implemented and locally qualified (2026-09-28), under ADR-0063. ambiguous outcomes. - Recovery reconciles decisions with the restored state. Next: M7.6 hosts bounded authoritative ticks (apps/evo-server, region_host). + +M7.6 implemented and locally qualified (2026-09-28), under ADR-0064. +- `RegionHost` is a single-owner state machine: acquire and recover, run, + resolve, drain and stop, or quarantine. +- Admission is bounded (64 inputs, 64 KiB) with backpressure, and + acknowledgements become final only after a confirmed commit. +- Tick debt is measured with bounded catch-up. +- A death pauses the region until it is durable. +- Readiness fails closed; storage outages, unknown outcomes and fences each have + a defined recovery. +- `evo-server` passed real SIGTERM drain and SIGKILL tests, with successor + recovery. +Next: M7.7 recovery verification at every kill boundary. diff --git a/docs/STATUS.md b/docs/STATUS.md index ad03687..e1b5e2a 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -836,3 +836,28 @@ mutations were each caught. Windows and Linux ran 58/58; PostgreSQL suites passe under Release, analysis and ASan. Next: M7.6 region host. No host, broker, deployment or process-kill test exists yet. + +## M7.6 region host and server process (2026-09-28) + +ADR-0064 adds `RegionHost`, a single-owner, single-threaded state machine: +acquire and recover, run, resolve, drain and stop, or quarantine. +- Admission is provisional and bounded (64 inputs, 64 KiB) with backpressure. +- Acknowledgements and outcomes are final only after a confirmed commit. +- A death pauses the region and commits in the same step. +- Tick debt is measured, and catch-up is bounded to 8 ticks per step. +- Readiness fails closed. + +`evo-server` hosts one region using a stdin input feed and stdout event lines, +with a readiness file and SIGTERM drain. Real process tests: +- SIGTERM: 80 inputs through the 64-input backlog were drained, committed and + released. +- SIGKILL: every acknowledged tick was committed, and the successor became ready + about 3.9 s later, after the 4 s development lease expired, from the committed + head. + +Seven host scenarios pass against real PostgreSQL, and four mutations were each +caught. Windows ran 58/58; the PostgreSQL suites passed under Release, analysis +and ASan/UBSan. + +Next: M7.7 recovery verification at every kill boundary. There is no gateway, +metrics or deployment yet. diff --git a/docs/adr/0064-region-host.md b/docs/adr/0064-region-host.md new file mode 100644 index 0000000..2f1dcdf --- /dev/null +++ b/docs/adr/0064-region-host.md @@ -0,0 +1,118 @@ +# ADR-0064: Bounded authoritative region host + +## Status +Accepted for M7.6, 2026-09-28. It composes ADR-0060 (fencing), ADR-0061/0062 +(checkpoints) and ADR-0063 (batch outcomes). It adds no gateway, network listener, +metrics endpoint or deployment; those are M7.8 and later. + +## Context +The persistence APIs exist, but nothing yet owns a region over time. ADR-0058 +requires: +- one simulation owner, with simulated and committed ticks both visible; +- bounded backlogs, with backpressure before admission; +- a whole-region pause at any irreversible outcome until durable confirmation; +- bounded drain, and readiness that fails closed. + +Nothing may treat an uncommitted death as fact. + +## Decision +**`RegionHost`** (`src/server`, Linux), for `EnvironmentalCommunity` and +`LineageRegion`, is a single-threaded state machine advanced by `step(now)`: +- *acquiring*: take the fenced lease, then recover and verify the committed head. + The recovered quantum and frame must match the configuration. +- *running*: lease upkeep, bounded ticks, and commits. +- *resolving*: an ambiguous commit is resolved before anything else. +- *draining*, then *stopped*; or *quarantined*. + +All I/O runs on the caller's thread within the LeasePolicy deadlines. While a +commit runs, the region is stopped. + +**Admission** is provisional: +- The backlog holds at most 64 inputs and 64 KiB (88 bytes are accounted per + input). When it is full, the caller gets `full` and must hold its input. +- Duplicate request identities within the host are refused. +- Inputs are refused unless the host is ready. + +An input becomes final only when the commit containing its tick is confirmed. +The host then emits `FinalAck(request, tick, commit_sequence)`; permanent +outcomes are emitted as `FinalOutcome`. Inputs that will never be acknowledged +are reported as `Dropped`, with reason refused, drained, fenced or quarantined. +After an unknown outcome, "fenced" means "unknown to this host", not "not +applied". + +**Ticks**: +- Scheduling time comes from the caller, while lease deadlines use `LeaseClock` + and database time. +- The due count is (now − start) / quantum. Each step runs at most 8 catch-up + ticks, and one queued input is consumed per tick. +- An empty backlog accrues tick debt. The host never invents inputs, and debt is + measured (current and maximum) rather than skipped. + +**Commits** use `RegionBatch`: +- A death seals the batch and commits in the same step, so no dependent tick runs + on an uncommitted outcome. +- Otherwise a batch flushes after `flush_interval` (default 1 s), at 64 ticks, + or on drain. +- The commit ID is the acquisition's holder token with the next sequence number + in its last 8 bytes. It is stable across retries of one batch, and unique + because holder tokens are generated per acquisition. + +| Outcome | Host response | +|---|---| +| Storage failure (upload/verify/digest), transient SQL error | Stay paused and retry the same batch; `store_failures` counts it | +| Unknown | Enter *resolving*: resolve the exact prepared bytes. Committed → adopt the receipt. Absent → retry the same bytes while `may_begin_commit` holds. Conflict → quarantine | +| Fenced | Discard state, report unconfirmed inputs, re-acquire and recover from the head | +| Candidate, predecessor, head moved, conflict, receipt mismatch, unrestorable head | Quarantine, releasing the lease if still held | + +**Readiness** is true only when running, recovered, not draining, and +`LeaseAuthority::may_originate` holds. + +**Drain** stops admission, reports the backlog as drained, commits pending ticks, +releases the lease, then stops. A successor can therefore acquire immediately, +without waiting for expiry. + +**`evo-server`** is the development process around one host: +- Tick inputs are stdin lines. It reads only while the backlog has room, so stdin + itself backpressures. +- Events are stdout lines: phase, ready, ack, outcome, dropped, stats. +- An optional readiness file mirrors readiness. +- SIGTERM or SIGINT drain within `--drain-ms`. +- Exit codes: 0 drained, 3 drain deadline passed (pending or unknown work is left + to the next owner), 4 quarantined, 2 usage. +- `--lease-profile fast` (a 4 s lease) is for development tests only. + +## Alternatives Considered +- **A separate I/O thread**: would allow ticks during commits, but the contract + pauses the region at commits anyway, and extra threads violate single ownership + without a measured need. +- **Wall-clock-driven synthetic inputs**: would invent world input. +- **Acknowledging on apply**: would expose provisional state as final. +- **Waiting for lease expiry after a clean shutdown**: slows handover for no + safety benefit. + +## Benefits +- One place enforces pause-at-outcome, final-only-after-commit and fail-closed + readiness. +- Every failure mode maps to a defined phase. +- The process can be killed or terminated with verifiable durable results. + +## Costs +- Ticks stall during commits: about 7 ms locally per batch in Release (M7.5), and + longer if storage is slow. +- The stdin feed is development plumbing, not a network protocol. +- There are no metrics endpoints yet. + +## Risks +- Duplicate request detection is per host process, not durable. Client + deduplication across owners needs the gateway protocol. +- The killed-process test measures recovery after the development 4 s lease; the + production 20 s lease bounds unavailability accordingly. +- Clock steps (ADR-0060) still apply. + +## Consequences +- M7.7 kills processes at every publication and ACK boundary, and verifies + recovery before readiness. +- M7.8 exposes phase, readiness, tick debt, commit lag and lease safety as probes + and metrics, and packages `evo-server`. +- A gateway must send inputs with request identities and treat only `FinalAck` as + durable. diff --git a/docs/adr/README.md b/docs/adr/README.md index 343ffe7..14f36f8 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -74,3 +74,4 @@ design. No unadopted candidate is a link dependency of the proof. - [ADR-0061: Checkpoint publication](0061-checkpoint-publication.md) - accepted for M7.4a manifest v1, immutable filesystem objects, OpenSSL SHA-256 and verified recovery. - [ADR-0062: Lineage checkpoint profile](0062-lineage-checkpoint-profile.md) - accepted for M7.4b archive codec, LineageRegion closure and history-capable profile 2. - [ADR-0063: Batch outcomes](0063-batch-outcomes.md) - accepted for M7.5 manifest v2, validation trace replay, durable decisions/outbox and commit resolution. +- [ADR-0064: Region host](0064-region-host.md) - accepted for M7.6 single-owner host state machine, bounded admission, final-only-after-commit and evo-server process. diff --git a/docs/architecture/REGION_DURABILITY.md b/docs/architecture/REGION_DURABILITY.md index a492478..210b601 100644 --- a/docs/architecture/REGION_DURABILITY.md +++ b/docs/architecture/REGION_DURABILITY.md @@ -201,3 +201,12 @@ M7.5 (ADR-0063) executed its rows against real PostgreSQL: Recovery now reconciles durable deaths with the restored lifecycle, so the checkpoint and durable history must agree. Kill tests at each boundary remain M7.7. + +M7.6 (ADR-0064) executed its host rows: +- A paused stale host fences itself when renewals cannot reach the database. It + reports unconfirmed inputs as never final and re-acquires from the head. +- A death pauses the region in the same step as its commit. +- A storage outage keeps the sealed batch waiting with nothing acknowledged. +- Shutdown drains, commits and releases. +- A real SIGKILLed `evo-server` left no acknowledged tick uncommitted, and its + successor became ready only after lease expiry, from the committed head. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 0d93854..55f54c3 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -2296,3 +2296,46 @@ Only clean-to-v1 and exact-v1 restart are supported migration paths at this gate PostgreSQL suites, all passing, and the repository format check. The changes are Linux-only persistence code, SQL and tests; the portable targets and Windows are unchanged. + +## M7.6 region host - 2026-09-28 + +- Windows MSVC Release CTest passed 58/58; this change touches only the Linux + PostgreSQL CMake section and Linux sources. The repository clang-format 19 check + passed. +- WSL Linux: GCC14 Release, clang19 analysis (clang-tidy) and clang19 ASan/UBSan + builds of `evo-postgres-tests` and `evo-server` passed the full PostgreSQL suite. + Final runs: Release 3, analysis 1, ASan 3 across both verification passes. +- The first analysis build flagged unchecked optional accesses and ignored release + results in the host and server, plus two test issues. These were fixed with + explicit local checks rather than suppressions, then rebuilt and rerun in all + three configurations. +- Host scenarios (real PostgreSQL, scaled scheduling clock, 4 s test lease): + - Admission: not_ready before recovery, 64-input cap (full), duplicate and + invalid refusals, byte accounting. + - Ticks 1–19 are provisional with no acks. Tick 20's death pauses and commits + in the same step: 20 acks and one outcome (consumer, organism 11, sequence 1). + The next batch flushed by interval at tick 41. + - Starvation: debt reached 10 without invented ticks, then catch-up of 8 per step. + - Storage outage: the sealed batch waits with nothing acknowledged and debt + growing, then commits when storage recovers. + - Lease row locked: the commit becomes unknown, the host is not ready and + refuses admission, and resolution adopts the receipt after release. + - Renewals blocked past the local deadline: fence, all 8 unconfirmed inputs + reported `fenced`, then re-acquisition at a newer epoch from the committed head. + - Drain (environmental profile): backlog dropped as drained, 6 pending ticks + committed, lease released, and a successor acquired immediately at epoch 2. + - Unrestorable build and foreign frame quarantine; an invalid configuration is + refused. +- Process tests (real `evo-server` via posix_spawn, stdin and stdout pipes): + - SIGTERM after 80 inputs: exit 0, 80 acks, head 80 at sequence 8, lease + released, readiness file removed, max_queued 64. The outcome line followed + tick 20's final ack. + - SIGKILL during running: every acked tick ≤ the committed head (11/11) and the + lease was left held. The successor was ready after 3.94–3.97 s at epoch 2 with + committed=11, then drained cleanly. +- Mutations (restored and rerun green), each caught: acknowledging at simulation + time, not committing a sealed death, reporting ready while resolving, and + draining without releasing the lease. +- Not executed: Docker/Gitea CI, TSan (the host is single-threaded; libpq is + uninstrumented), remote PostgreSQL/TLS, a production 20 s lease kill test, + durable client deduplication, and a network gateway. diff --git a/docs/infrastructure/POSTGRESQL.md b/docs/infrastructure/POSTGRESQL.md index 1a5474e..0516c85 100644 --- a/docs/infrastructure/POSTGRESQL.md +++ b/docs/infrastructure/POSTGRESQL.md @@ -165,3 +165,26 @@ the existing HARBOR_PROJECT and Harbor robot secrets. Pin the reported registry digest in the example Job; do not use the simulation image. See deploy/README.md. Local Unix-socket integration is qualified; remote TLS/authentication, actual Secret mounts and container execution still require environment-specific verification. + +## Region server process (M7.6, ADR-0064) + +`evo-server` is built with the PostgreSQL build and hosts one provisioned region +as the runtime login. It is a development host with no network gateway. +Tick inputs are stdin lines: + + input + +Events are stdout lines: `phase`, `ready`, `ack`, `outcome`, `dropped` and a +final `stats`. Only `ack` and `outcome` lines are durable facts. +`--ready-file` mirrors readiness for probes. SIGTERM drains (commits pending ticks +and releases the lease) within `--drain-ms`. + +Exit codes: +- 0: drained; +- 3: the drain deadline passed, and the next owner reconciles; +- 4: quarantined; +- 2: usage error. + +Use the default 20 s lease in any shared environment. `--lease-profile fast` exists +only for local tests. The runtime credentials come from `--pg-*` flags and a +passfile; never place passwords on the command line. diff --git a/include/evo/server/region_host.hpp b/include/evo/server/region_host.hpp new file mode 100644 index 0000000..9b994b2 --- /dev/null +++ b/include/evo/server/region_host.hpp @@ -0,0 +1,170 @@ +#pragma once +#include "evo/persistence/checkpoint_store.hpp" +#include "evo/persistence/region_batch.hpp" +#include "evo/persistence/region_lease.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// M7.6 authoritative region host (ADR-0064). Linux; composes the fenced lease, +// verified recovery and batch publication. One owner, one thread: every call runs +// on the caller's thread and all I/O is bounded by the lease policy's deadlines. +// While a batch commits the region is stopped, by design (ADR-0058). +namespace evo::server { +// One tick's host input, from the (future) gateway or a local feed. +struct TickInput { + persistence::Uuid request{}; // Caller identity for acknowledgement; nonzero. + processes::EnvironmentalInput input{}; + friend bool operator==(const TickInput&, const TickInput&) = default; +}; +inline constexpr std::size_t tick_input_bytes = 16 + 72; // Canonical accounting size. + +enum class AdmissionError : std::uint8_t { + not_ready, // No current authority or not recovered: nothing is accepted. + draining, // Shutdown in progress. + full, // Backpressure: bounded backlog (inputs or bytes) is full. + duplicate, // Request identity already queued or pending in this host. + invalid, // Zero request identity. +}; +// Final: the input's tick is in a durably committed checkpoint. +struct FinalAck { + persistence::Uuid request{}; + std::uint64_t tick{}; + std::uint64_t commit_sequence{}; + friend bool operator==(const FinalAck&, const FinalAck&) = default; +}; +// Final: the permanent outcome is durable (and in the outbox). +struct FinalOutcome { + persistence::BatchDecision decision{}; + std::uint64_t commit_sequence{}; + friend bool operator==(const FinalOutcome&, const FinalOutcome&) = default; +}; +// Why an accepted input will never be acknowledged by this host. +enum class Unacknowledged : std::uint8_t { + refused, // The simulation rejected it; no tick consumed it. + drained, // Still queued at shutdown. + fenced, // Authority was lost before its tick committed; the next owner never saw it. + quarantined, +}; +struct Dropped { + TickInput input{}; + Unacknowledged reason{}; +}; + +enum class HostPhase : std::uint8_t { + acquiring, // Waiting for the lease (busy/unavailable) or recovering. + running, // Authoritative: simulating and committing. + resolving, // A commit outcome is unknown: paused until resolved. + draining, // Shutdown: committing pending work, then releasing. + stopped, // Drained (lease released when possible). + quarantined, // Fail closed: committed state or history cannot be trusted. +}; +struct HostStatus { + HostPhase phase{HostPhase::acquiring}; + bool ready{}; // Authoritative now: recovered, running, lease may originate. + std::uint64_t epoch{}; + std::uint64_t simulated_tick{}, committed_tick{}, committed_sequence{}; + std::size_t queued{}, queued_bytes{}; + std::uint64_t tick_debt{}, max_tick_debt{}; // Due minus simulated since running. + std::uint64_t commits{}, store_failures{}, unknown_outcomes{}, fences{}; +}; + +using Connect = + std::function, persistence::PgFailure>()>; +struct HostConfig { + persistence::RegionKey key{}; + std::uint64_t frame{}; + persistence::LeasePolicy lease{}; + // Region quantum; must equal the committed checkpoint's tick_microseconds. + std::chrono::microseconds tick_quantum{50000}; + std::chrono::milliseconds flush_interval{1000}; // Commit a non-empty batch this often. + std::size_t max_queued_inputs{64}; + std::size_t max_queued_bytes{65536}; + std::uint32_t max_catch_up_ticks{8}; // Per step; debt is never skipped. + persistence::Sha256Digest build{}; + std::vector retained_builds; + std::size_t restore_budget{8000000}; +}; +enum class HostConfigError : std::uint8_t { invalid }; + +// Scheduling time comes from the caller (monotonic, may be scaled in tests); lease +// deadlines always use LeaseClock (real, suspend-aware) and database time. +using SchedulerTime = std::chrono::steady_clock::time_point; + +template class RegionHost { + public: + [[nodiscard]] static std::expected, HostConfigError> + create(HostConfig config, Connect connect, + std::unique_ptr objects); + RegionHost(const RegionHost&) = delete; + RegionHost& operator=(const RegionHost&) = delete; + RegionHost(RegionHost&&) = delete; + RegionHost& operator=(RegionHost&&) = delete; + ~RegionHost(); + + // Provisional admission only; nothing is final until a FinalAck. + [[nodiscard]] std::expected submit(const TickInput& input); + // Advances the state machine once: lease upkeep, recovery, resolution, commits + // and up to max_catch_up_ticks ticks. Blocks at most for bounded I/O. + const HostStatus& step(SchedulerTime now); + // Stops admission, drops the backlog, commits pending ticks, releases the lease. + void drain(); + + [[nodiscard]] std::vector take_acks(); + [[nodiscard]] std::vector take_outcomes(); + [[nodiscard]] std::vector take_dropped(); + [[nodiscard]] const HostStatus& status() const noexcept { + return status_; + } + // Read-only view for diagnostics/tests; null unless recovered. + [[nodiscard]] const State* state() const noexcept { + return state_.get(); + } + + private: + struct Applied { + TickInput input; + std::uint64_t tick{}; + }; + RegionHost(HostConfig config, Connect connect, + std::unique_ptr objects); + bool database(); + void acquire(SchedulerTime now); + void upkeep(); + void simulate(SchedulerTime now); + void commit(SchedulerTime now); + void resolve(SchedulerTime now); + void confirmed(const persistence::PublishedBatch& batch, SchedulerTime now); + void lose_authority(Unacknowledged reason, HostPhase next); + void refresh(SchedulerTime now); + [[nodiscard]] persistence::Uuid next_commit_id() const; + + HostConfig config_; + Connect connect_; + std::unique_ptr objects_; + std::unique_ptr db_; + persistence::LeaseAuthority authority_; + std::unique_ptr state_; + std::optional> batch_; + std::optional committed_; + std::optional prepared_; + std::deque queue_; + std::vector applied_; + std::vector acks_; + std::vector outcomes_; + std::vector dropped_; + SchedulerTime origin_{}, batch_started_{}; + std::uint64_t origin_tick_{}; + bool draining_{}; + HostStatus status_{}; +}; + +extern template class RegionHost; +extern template class RegionHost; +} // namespace evo::server diff --git a/src/server/region_host.cpp b/src/server/region_host.cpp new file mode 100644 index 0000000..71f4585 --- /dev/null +++ b/src/server/region_host.cpp @@ -0,0 +1,471 @@ +#include "evo/server/region_host.hpp" +#include "evo/persistence/sha256.hpp" +#include +#include + +namespace evo::server { +namespace { +using persistence::LeaseClock; +using persistence::LeaseStoreError; +using persistence::PublicationError; +using persistence::RecoveryError; + +// Per-profile calls into the persistence layer. +template struct Ops; +template <> struct Ops { + using State = processes::EnvironmentalCommunity; + static std::uint64_t tick(const State& s) { + return s.progress().tick; + } + static std::uint64_t quantum(const State& s) { + return s.tick_microseconds(); + } + template static auto recover(A&&... a) { + return persistence::recover_environmental_checkpoint(std::forward(a)...); + } + template static auto publish(A&&... a) { + return persistence::publish_environmental_batch(std::forward(a)...); + } +}; +template <> struct Ops { + using State = processes::LineageRegion; + static std::uint64_t tick(const State& s) { + return s.community().progress().tick; + } + static std::uint64_t quantum(const State& s) { + return s.community().tick_microseconds(); + } + template static auto recover(A&&... a) { + return persistence::recover_lineage_checkpoint(std::forward(a)...); + } + template static auto publish(A&&... a) { + return persistence::publish_lineage_batch(std::forward(a)...); + } +}; + +bool valid(const HostConfig& c) { + constexpr std::size_t max_backlog_inputs = 64; + constexpr std::size_t max_backlog_bytes = 65536; + return c.key.world != 0 && c.key.region != 0 && c.frame != 0 && persistence::valid(c.lease) && + c.tick_quantum.count() > 0 && c.flush_interval.count() > 0 && c.max_queued_inputs >= 1 && + c.max_queued_inputs <= max_backlog_inputs && c.max_queued_bytes >= tick_input_bytes && + c.max_queued_bytes <= max_backlog_bytes && c.max_catch_up_ticks >= 1 && + c.max_catch_up_ticks <= 64 && c.build != persistence::Sha256Digest{} && + std::ranges::find(c.retained_builds, c.build) != c.retained_builds.end() && + c.restore_budget > 0; +} +} // namespace + +template +RegionHost::RegionHost(HostConfig config, Connect connect, + std::unique_ptr objects) + : config_(std::move(config)), connect_(std::move(connect)), objects_(std::move(objects)), + authority_(config_.lease) {} + +template RegionHost::~RegionHost() = default; + +template +std::expected>, HostConfigError> +RegionHost::create(HostConfig config, Connect connect, + std::unique_ptr objects) { + if (!valid(config) || !connect || !objects) + return std::unexpected{HostConfigError::invalid}; + return std::unique_ptr( + new RegionHost(std::move(config), std::move(connect), std::move(objects))); +} + +template +std::expected RegionHost::submit(const TickInput& input) { + if (input.request == persistence::Uuid{}) + return std::unexpected{AdmissionError::invalid}; + if (draining_) + return std::unexpected{AdmissionError::draining}; + if (!status_.ready) + return std::unexpected{AdmissionError::not_ready}; + const auto same = [&](const TickInput& x) { return x.request == input.request; }; + if (std::ranges::any_of(queue_, same) || + std::ranges::any_of(applied_, [&](const Applied& a) { return same(a.input); })) + return std::unexpected{AdmissionError::duplicate}; + if (queue_.size() >= config_.max_queued_inputs || + (queue_.size() + 1) * tick_input_bytes > config_.max_queued_bytes) + return std::unexpected{AdmissionError::full}; + queue_.push_back(input); + status_.queued = queue_.size(); + status_.queued_bytes = queue_.size() * tick_input_bytes; + return {}; +} + +template bool RegionHost::database() { + if (db_) + return true; + auto made = connect_(); + if (!made) { + ++status_.store_failures; + return false; + } + db_ = std::move(*made); + return true; +} + +template +void RegionHost::lose_authority(Unacknowledged reason, HostPhase next) { + for (auto& a : applied_) + dropped_.push_back({a.input, reason}); + for (auto& q : queue_) + dropped_.push_back({q, reason}); + applied_.clear(); + queue_.clear(); + batch_.reset(); // References state_: reset first. + state_.reset(); + committed_.reset(); + prepared_.reset(); + // Best effort: a quarantined region's lease is released so peers see it free. + const auto& grant = authority_.grant(); + if (next == HostPhase::quarantined && db_ && grant && + authority_.may_originate(LeaseClock::now())) { + [[maybe_unused]] const auto released = persistence::release_region_lease( + *db_, config_.key, *grant, config_.lease.renew_timeout); + } + authority_.fence(); + if (reason == Unacknowledged::fenced) + ++status_.fences; + status_.phase = draining_ && next == HostPhase::acquiring ? HostPhase::stopped : next; +} + +template void RegionHost::acquire(SchedulerTime now) { + if (!database()) + return; + // A grant survives a transient recovery failure; only re-acquire without one. + if (!authority_.may_originate(LeaseClock::now())) { + const auto sent = LeaseClock::now(); + auto grant = persistence::acquire_region_lease(*db_, config_.key, config_.lease.duration, + config_.lease.renew_timeout); + if (!grant) { + if (grant.error().kind == LeaseStoreError::not_provisioned) + status_.phase = HostPhase::quarantined; + else if (grant.error().kind != LeaseStoreError::busy) + db_.reset(); + return; + } + if (grant->frame != config_.frame) { + status_.phase = HostPhase::quarantined; + return; + } + if (!authority_.acquired(*grant, sent, LeaseClock::now())) + return; // Reply too late to use; the lease expires unused. + } + auto recovered = Ops::recover(*db_, *objects_, config_.key, config_.retained_builds, + config_.restore_budget, config_.lease.commit_timeout); + if (!recovered) { + if (recovered.error().kind == RecoveryError::store) { + db_.reset(); // Transient: keep the grant and retry recovery. + ++status_.store_failures; + return; + } + lose_authority(Unacknowledged::quarantined, HostPhase::quarantined); + return; + } + if (Ops::quantum(*recovered->state) != + static_cast(config_.tick_quantum.count())) { + lose_authority(Unacknowledged::quarantined, HostPhase::quarantined); + return; + } + state_ = std::move(recovered->state); + committed_ = std::move(recovered->committed); + batch_.emplace(*state_); + origin_ = now; + origin_tick_ = Ops::tick(*state_); + status_.max_tick_debt = 0; + status_.phase = draining_ ? HostPhase::draining : HostPhase::running; +} + +template void RegionHost::upkeep() { + const auto state = authority_.state(LeaseClock::now()); + if (state == persistence::LeaseState::fenced) { + lose_authority(Unacknowledged::fenced, HostPhase::acquiring); + return; + } + if (state != persistence::LeaseState::renew_due || !database()) + return; + const auto grant = *authority_.grant(); + const auto sent = LeaseClock::now(); + auto renewed = persistence::renew_region_lease(*db_, config_.key, grant, config_.lease.duration, + config_.lease.renew_timeout); + if (renewed) { + if (!authority_.renewed(grant.epoch, *renewed, sent, LeaseClock::now())) + lose_authority(Unacknowledged::fenced, HostPhase::acquiring); + return; + } + switch (renewed.error().kind) { + case LeaseStoreError::unknown: + // It may have shortened or extended nothing: keep the earlier deadline. + authority_.renewal_unknown(config_.lease.duration, sent); + db_.reset(); + ++status_.store_failures; + return; + case LeaseStoreError::fenced: + lose_authority(Unacknowledged::fenced, HostPhase::acquiring); + return; + default: + db_.reset(); + ++status_.store_failures; + return; + } +} + +template persistence::Uuid RegionHost::next_commit_id() const { + // Unique: the DB-generated holder token names this acquisition; the + // sequence names the link. Stable across retries of the same batch. + const auto& grant = authority_.grant(); + if (!grant || !committed_) + return {}; + auto id = grant->holder; + const auto sequence = committed_->head.sequence + 1; + for (unsigned i = 0; i < 8; ++i) + id[8 + i] = static_cast((sequence >> (8U * i)) & 0xFFU); + return id; +} + +template +void RegionHost::confirmed(const persistence::PublishedBatch& published, SchedulerTime now) { + const auto& head = published.committed.head; + if (!batch_ || !batch_->confirm(head.end_tick)) { + lose_authority(Unacknowledged::quarantined, HostPhase::quarantined); + return; + } + committed_ = published.committed; + for (const auto& a : applied_) + acks_.push_back({a.input.request, a.tick, head.sequence}); + applied_.clear(); + for (const auto& d : published.decisions) + outcomes_.push_back({d, head.sequence}); + prepared_.reset(); + batch_started_ = now; + ++status_.commits; + status_.phase = draining_ ? HostPhase::draining : HostPhase::running; +} + +template void RegionHost::commit(SchedulerTime now) { + if (!authority_.may_begin_commit(LeaseClock::now())) + return; // Renewal first; upkeep fences if the deadline passes. + if (!database()) + return; + const auto& grant = authority_.grant(); + if (!grant || !committed_ || !batch_) + return; + const auto commit_id = next_commit_id(); + batch_->seal(); + auto published = + Ops::publish(*db_, *objects_, *grant, *committed_, *batch_, commit_id, config_.build, + config_.restore_budget, config_.lease.commit_timeout); + if (published) { + confirmed(*published, now); + return; + } + const auto& f = published.error(); + switch (f.kind) { + case PublicationError::upload: + case PublicationError::verify: + case PublicationError::digest: + ++status_.store_failures; // Nothing sent: retry the same batch later. + return; + case PublicationError::unknown: + prepared_ = f.prepared; + db_.reset(); + ++status_.unknown_outcomes; + status_.phase = HostPhase::resolving; + return; + case PublicationError::rejected: + if (f.store.kind == LeaseStoreError::fenced) { + lose_authority(Unacknowledged::fenced, HostPhase::acquiring); + return; + } + if (f.store.kind == LeaseStoreError::store || f.store.kind == LeaseStoreError::invalid) { + // SQL/connection error before any mutation: retry with the same bytes. + db_.reset(); + ++status_.store_failures; + return; + } + [[fallthrough]]; + default: // candidate, predecessor, receipt, head moved, conflicts: fail closed. + lose_authority(Unacknowledged::quarantined, HostPhase::quarantined); + return; + } +} + +template void RegionHost::resolve(SchedulerTime now) { + if (!database() || !prepared_ || !batch_) + return; + const auto& prepared = *prepared_; + auto resolved = persistence::resolve_prepared(*db_, prepared, config_.lease.commit_timeout); + if (!resolved) { + db_.reset(); + ++status_.store_failures; + return; + } + const auto adopt = [&](const persistence::RegionHead& receipt) { + auto decisions = persistence::decode_decisions(prepared.decisions, batch_->begin_tick(), + batch_->end_tick()); + const auto digest = persistence::sha256(prepared.manifest); + if (!decisions || !digest || *digest != receipt.digest || + receipt.end_tick != batch_->end_tick()) { + lose_authority(Unacknowledged::quarantined, HostPhase::quarantined); + return; + } + confirmed({{receipt, prepared.manifest}, *decisions}, now); + }; + switch (resolved->status) { + case persistence::Resolution::committed: + adopt(resolved->receipt); + return; + case persistence::Resolution::conflict: + lose_authority(Unacknowledged::quarantined, HostPhase::quarantined); + return; + case persistence::Resolution::absent: + break; + } + // Absent now: retry exactly the same bytes, only while still authorized. + if (!authority_.may_begin_commit(LeaseClock::now())) + return; + auto retried = persistence::retry_prepared(*db_, prepared, config_.lease.commit_timeout); + if (retried) { + adopt(*retried); + return; + } + switch (retried.error().kind) { + case LeaseStoreError::fenced: + lose_authority(Unacknowledged::fenced, HostPhase::acquiring); + return; + case LeaseStoreError::unknown: + case LeaseStoreError::store: + db_.reset(); + ++status_.store_failures; + return; + default: + lose_authority(Unacknowledged::quarantined, HostPhase::quarantined); + return; + } +} + +template void RegionHost::simulate(SchedulerTime now) { + const auto elapsed = std::chrono::duration_cast(now - origin_); + const auto due = elapsed.count() <= 0 + ? std::uint64_t{0} + : static_cast(elapsed / config_.tick_quantum); + if (!batch_ || !state_) + return; + auto& batch = *batch_; + const auto& state = *state_; + for (std::uint32_t k = 0; k < config_.max_catch_up_ticks && !batch.sealed(); ++k) { + if (Ops::tick(state) - origin_tick_ >= due || queue_.empty() || + !authority_.may_originate(LeaseClock::now())) + break; // Not due, starved (debt accrues; inputs are never invented) or unauthorized. + const auto input = queue_.front(); + queue_.pop_front(); + if (batch.steps().empty()) + batch_started_ = now; + auto report = batch.advance(input.input); + if (!report) { + dropped_.push_back({input, Unacknowledged::refused}); + continue; + } + applied_.push_back({input, report->tick}); + } +} + +template void RegionHost::refresh(SchedulerTime now) { + status_.queued = queue_.size(); + status_.queued_bytes = queue_.size() * tick_input_bytes; + const auto& grant = authority_.grant(); + status_.epoch = grant ? grant->epoch : 0; + status_.committed_tick = committed_ ? committed_->head.end_tick : 0; + status_.committed_sequence = committed_ ? committed_->head.sequence : 0; + status_.simulated_tick = state_ ? Ops::tick(*state_) : status_.committed_tick; + if (state_ && status_.phase == HostPhase::running) { + const auto elapsed = std::chrono::duration_cast(now - origin_); + const auto due = elapsed.count() <= 0 + ? std::uint64_t{0} + : static_cast(elapsed / config_.tick_quantum); + const auto done = Ops::tick(*state_) - origin_tick_; + status_.tick_debt = due > done ? due - done : 0; + status_.max_tick_debt = std::max(status_.max_tick_debt, status_.tick_debt); + } + status_.ready = status_.phase == HostPhase::running && state_ && !draining_ && + authority_.may_originate(LeaseClock::now()); +} + +template const HostStatus& RegionHost::step(SchedulerTime now) { + switch (status_.phase) { + case HostPhase::acquiring: + acquire(now); + break; + case HostPhase::resolving: + upkeep(); + if (status_.phase == HostPhase::resolving) + resolve(now); + break; + case HostPhase::running: + case HostPhase::draining: { + upkeep(); + if (status_.phase != HostPhase::running && status_.phase != HostPhase::draining) + break; + if (!draining_) + simulate(now); + if (!batch_) + break; // Running always has a batch; defensive. + const bool flush = + !batch_->steps().empty() && + (batch_->sealed() || draining_ || now - batch_started_ >= config_.flush_interval); + if (flush) + commit(now); // May lose authority and reset the batch. + if (draining_ && status_.phase == HostPhase::draining && batch_ && + batch_->steps().empty()) { + const auto& grant = authority_.grant(); + if (database() && grant && authority_.may_originate(LeaseClock::now())) { + // Best effort: an unreleased lease simply expires. + [[maybe_unused]] const auto released = persistence::release_region_lease( + *db_, config_.key, *grant, config_.lease.renew_timeout); + } + batch_.reset(); + state_.reset(); + authority_.fence(); + db_.reset(); + status_.phase = HostPhase::stopped; + } + break; + } + case HostPhase::stopped: + case HostPhase::quarantined: + break; + } + refresh(now); + return status_; +} + +template void RegionHost::drain() { + if (draining_) + return; + draining_ = true; + for (auto& q : queue_) + dropped_.push_back({q, Unacknowledged::drained}); + queue_.clear(); + if (status_.phase == HostPhase::running) + status_.phase = HostPhase::draining; + else if (status_.phase == HostPhase::acquiring) + status_.phase = HostPhase::stopped; + status_.ready = false; +} + +template std::vector RegionHost::take_acks() { + return std::exchange(acks_, {}); +} +template std::vector RegionHost::take_outcomes() { + return std::exchange(outcomes_, {}); +} +template std::vector RegionHost::take_dropped() { + return std::exchange(dropped_, {}); +} + +template class RegionHost; +template class RegionHost; +} // namespace evo::server diff --git a/tests/postgres_tests.cpp b/tests/postgres_tests.cpp index f3d2bf9..a21ab46 100644 --- a/tests/postgres_tests.cpp +++ b/tests/postgres_tests.cpp @@ -5,6 +5,8 @@ #include void run_region_lease_store_tests(); void run_region_checkpoint_store_tests(); +void run_region_host_tests(); +void run_region_server_process_tests(); namespace { using namespace evo::persistence; using namespace std::chrono_literals; @@ -75,6 +77,10 @@ int main() { std::puts("region lease fencing integration passed"); run_region_checkpoint_store_tests(); std::puts("region checkpoint publication integration passed"); + run_region_host_tests(); + std::puts("region host integration passed"); + run_region_server_process_tests(); + std::puts("region server process integration passed"); return 0; } catch (const std::exception& e) { std::fprintf(stderr, "%s\n", e.what()); diff --git a/tests/region_host_tests.cpp b/tests/region_host_tests.cpp new file mode 100644 index 0000000..e851afd --- /dev/null +++ b/tests/region_host_tests.cpp @@ -0,0 +1,332 @@ +// Real PostgreSQL M7.6 region host behavior (ADR-0064). Run by tools/ci/postgres-check.sh. +#include "evo/server/region_host.hpp" +#include "fixtures/environmental_region.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +void run_region_host_tests(); + +namespace { +using namespace evo::persistence; +using namespace evo::server; +using namespace std::chrono_literals; +using evo::processes::EnvironmentalCommunity; +using evo::processes::LineageRegion; +namespace fs = std::filesystem; +void check(bool value, std::source_location at = std::source_location::current()) { + if (!value) + throw std::runtime_error("region host test line " + std::to_string(at.line())); +} +#define CHECK(...) check(static_cast(__VA_ARGS__)) +std::string env(const char* key) { + const auto* value = std::getenv(key); + if (!value) + throw std::runtime_error("missing test environment"); + return value; +} +PgOptions options(const std::string& user) { + return {env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}}; +} +std::unique_ptr connect_as(const std::string& user) { + auto made = Postgres::connect(options(user), 5s); + CHECK(made); + return std::move(*made); +} +Connect runtime_connect() { + return [] { return Postgres::connect(options(env("EVO_PG_RUNTIME_USER")), 2s); }; +} +template std::array filled(std::uint8_t value) { + std::array out{}; + out.fill(value); + return out; +} +const Sha256Digest build = filled<32>(0xB1); +// Fast lease for tests: 4 s, renew every 0.8 s, 0.4 s margin/renew deadline, 1 s commits. +const LeasePolicy fast{4000ms, 800ms, 400ms, 400ms, 1000ms}; +TickInput input(std::uint64_t tick, bool consumer_dies = false) { + Uuid request{}; + request[0] = 0x9A; + for (unsigned i = 0; i < 8; ++i) + request[8 + i] = static_cast((tick >> (8U * i)) & 0xFFU); + return {request, evo::test::region_input(tick, consumer_dies)}; +} + +struct Region { + RegionKey key{7, 21}; + fs::path root; +}; +// Provisions a fresh region (owner) and returns its key and object root. +template Region provisioned(std::uint64_t region, std::uint8_t genesis) { + Region r{{7, 20 + region}, fs::path(env("PGHOST")) / ("host-" + std::to_string(region))}; + fs::remove_all(r.root); + fs::create_directory(r.root); + fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace); + auto owner = connect_as(env("PGUSER")); + CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " + "(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING", + {}, 5s)); + auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes); + CHECK(objects); + if constexpr (std::is_same_v) + CHECK(provision_lineage_region(*owner, **objects, r.key, evo::test::region_identity.frame, + *evo::test::lineage_region(), filled<16>(genesis), build, + 8000000, 5s)); + else + CHECK(provision_environmental_region( + *owner, **objects, r.key, evo::test::region_identity.frame, + *evo::test::region_community(), filled<16>(genesis), build, 8000000, 5s)); + return r; +} +HostConfig config(const Region& r) { + HostConfig c; + c.key = r.key; + c.frame = evo::test::region_identity.frame; + c.lease = fast; + c.build = build; + c.retained_builds = {build}; + return c; +} +template +std::unique_ptr> host(const Region& r, HostConfig c, + ObjectStoreFaults* faults = nullptr) { + auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes, faults); + CHECK(objects); + auto made = RegionHost::create(std::move(c), runtime_connect(), std::move(*objects)); + CHECK(made); + return std::move(*made); +} +// Steps until `done` holds, advancing scheduling time by `advance` each step. +template +const HostStatus& step_until(Host& h, SchedulerTime& t, std::chrono::microseconds advance, + Done done, int limit = 400) { + for (int i = 0; i < limit; ++i) { + t += advance; + const auto& s = h.step(t); + if (done(s)) + return s; + std::this_thread::sleep_for(10ms); + } + CHECK(false); + return h.status(); +} + +// Admission, provisional ticks, pause at a permanent outcome and final acknowledgement. +void acknowledgements_follow_commits() { + const auto r = provisioned(1, 0x3A); + auto h = host(r, config(r)); + CHECK(h->submit(input(1)).error() == AdmissionError::not_ready); + SchedulerTime t = std::chrono::steady_clock::now(); + auto s = h->step(t); + CHECK(s.phase == HostPhase::running && s.ready && s.committed_tick == 0 && s.epoch == 1); + for (std::uint64_t tick = 1; tick <= 64; ++tick) + CHECK(h->submit(input(tick, true))); + CHECK(h->submit(input(65)).error() == AdmissionError::full); // Backlog of 64. + CHECK(h->submit(input(1)).error() == AdmissionError::duplicate); + CHECK(h->submit({{}, {}}).error() == AdmissionError::invalid); + CHECK(h->status().queued_bytes == 64 * tick_input_bytes); + for (std::uint64_t tick = 1; tick <= 19; ++tick) { + t += 50ms; + s = h->step(t); + // Simulated ahead of committed; nothing is final yet. + CHECK(s.simulated_tick == tick && s.committed_tick == 0 && s.ready); + CHECK(h->take_acks().empty() && h->take_outcomes().empty()); + } + // Tick 20 kills the consumer: the region pauses and commits in the same step, + // so no dependent tick ever runs on the uncommitted death. + t += 50ms; + s = h->step(t); + CHECK(s.simulated_tick == 20 && s.committed_tick == 20 && s.committed_sequence == 1); + const auto outcomes = h->take_outcomes(); + CHECK(outcomes.size() == 1 && outcomes[0].commit_sequence == 1 && + outcomes[0].decision == BatchDecision{20, Actor::consumer, Outcome::death, 11}); + auto acks = h->take_acks(); + CHECK(acks.size() == 20); + for (std::uint64_t i = 0; i < 20; ++i) + CHECK(acks[i] == FinalAck{input(i + 1).request, i + 1, 1}); + // The next batch flushes after the configured interval, not per tick. + bool provisional = false; + s = step_until(*h, t, 50ms, [&](const HostStatus& x) { + provisional = provisional || (x.simulated_tick > x.committed_tick); + return x.committed_sequence == 2; + }); + CHECK(provisional && s.simulated_tick == s.committed_tick && s.committed_tick > 30); + acks = h->take_acks(); + CHECK(!acks.empty() && acks.back().tick == s.committed_tick && acks.front().tick == 21); + CHECK(std::ranges::all_of(acks, [](const FinalAck& a) { return a.commit_sequence == 2; })); + CHECK(h->take_dropped().empty()); + std::printf("host_commits=%llu committed_tick=%llu\n", + static_cast(s.commits), + static_cast(s.committed_tick)); +} + +// No inputs, no invented ticks: debt accrues and is repaid with bounded catch-up. +void starvation_accrues_bounded_debt() { + const auto r = provisioned(2, 0x3B); + auto h = host(r, config(r)); + SchedulerTime t = std::chrono::steady_clock::now(); + CHECK(h->step(t).ready); + for (int i = 0; i < 10; ++i) { + t += 50ms; + CHECK(h->step(t).simulated_tick == 0); + } + CHECK(h->status().tick_debt == 10); + for (std::uint64_t tick = 1; tick <= 20; ++tick) + CHECK(h->submit(input(tick))); + auto s = h->step(t); + CHECK(s.simulated_tick == 8 && s.tick_debt == 2); // At most 8 per step. + s = h->step(t); + CHECK(s.simulated_tick == 10 && s.tick_debt == 0 && s.max_tick_debt == 10); + CHECK(s.queued == 10); +} + +// Object store outage: the sealed batch waits, nothing is acknowledged, and the +// region resumes once storage is durable again. +void storage_outage_pauses_the_region() { + const auto r = provisioned(3, 0x41); + ObjectStoreFaults faults; + auto h = host(r, config(r), &faults); + SchedulerTime t = std::chrono::steady_clock::now(); + CHECK(h->step(t).ready); + for (std::uint64_t tick = 1; tick <= 30; ++tick) + CHECK(h->submit(input(tick, true))); + step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 19; }); + faults.fail_directory_syncs = 1000; + for (int i = 0; i < 5; ++i) { + t += 50ms; + const auto& s = h->step(t); + CHECK(s.simulated_tick == 20 && s.committed_tick == 0); // Paused at the death. + } + CHECK(h->status().store_failures >= 5 && h->take_acks().empty() && h->take_outcomes().empty()); + CHECK(h->status().tick_debt == 4); + faults.fail_directory_syncs = 0; + t += 50ms; + auto s = h->step(t); + CHECK(s.committed_tick == 20 && h->take_acks().size() == 20 && h->take_outcomes().size() == 1); + s = h->step(t); // Catch-up after the outage is bounded but continues. + CHECK(s.simulated_tick > 20 && s.simulated_tick <= 28); +} + +// Holds the lease row lock to make database calls time out. +struct RowLock { + std::unique_ptr db = connect_as(env("PGUSER")); + explicit RowLock(RegionKey key) { + const std::array p{std::to_string(key.world), std::to_string(key.region)}; + CHECK(db->query("BEGIN", {}, 5s)); + CHECK(db->query("SELECT 1 FROM evolution.region_leases WHERE world_id = $1::evolution.u64 " + "AND region_id = $2::evolution.u64 FOR UPDATE", + p, 5s)); + } + void release() { + CHECK(db->query("ROLLBACK", {}, 5s)); + } +}; + +// Unknown commit outcome: not ready, admission refused, resolved by exact bytes. +void unknown_outcome_is_resolved_before_continuing() { + const auto r = provisioned(4, 0x3D); + auto h = host(r, config(r)); + SchedulerTime t = std::chrono::steady_clock::now(); + CHECK(h->step(t).ready); + for (std::uint64_t tick = 1; tick <= 25; ++tick) + CHECK(h->submit(input(tick, true))); + step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 19; }); + RowLock lock{r.key}; + t += 50ms; + auto s = h->step(t); // The death seals; the commit cannot finish in time. + CHECK(s.phase == HostPhase::resolving && !s.ready && s.unknown_outcomes == 1); + CHECK(h->submit(input(26)).error() == AdmissionError::not_ready); + CHECK(h->take_outcomes().empty() && h->take_acks().empty()); + lock.release(); + s = step_until(*h, t, 0ms, [](const HostStatus& x) { return x.phase == HostPhase::running; }); + CHECK(s.committed_tick == 20 && s.ready); + CHECK(h->take_outcomes().size() == 1 && h->take_acks().size() == 20); +} + +// Renewals that cannot reach the database fence the host locally; unconfirmed +// work is reported as never final, and authority is re-acquired from the head. +void renewal_failure_fences_and_recovers() { + const auto r = provisioned(5, 0x3E); + auto h = host(r, config(r)); + SchedulerTime t = std::chrono::steady_clock::now(); + CHECK(h->step(t).ready); + for (std::uint64_t tick = 1; tick <= 8; ++tick) + CHECK(h->submit(input(tick))); + step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 5; }); + CHECK(h->status().committed_tick == 0); + const auto epoch = h->status().epoch; + std::optional lock{std::in_place, r.key}; + step_until(*h, t, 0ms, [](const HostStatus& x) { return x.phase != HostPhase::running; }, 1000); + CHECK(!h->status().ready && h->status().fences == 1 && h->state() == nullptr); + const auto dropped = h->take_dropped(); + CHECK(dropped.size() == 8 && std::ranges::all_of(dropped, [](const Dropped& d) { + return d.reason == Unacknowledged::fenced; + })); + CHECK(h->take_acks().empty()); + lock->release(); + lock.reset(); + const auto& s = step_until( + *h, t, 0ms, [](const HostStatus& x) { return x.phase == HostPhase::running; }, 1000); + CHECK(s.epoch > epoch && s.committed_tick == 0 && s.simulated_tick == 0 && s.ready); +} + +// Drain: backlog dropped, pending ticks committed, lease released for an +// immediate successor (environmental-v1 profile). +void drain_commits_and_hands_over() { + const auto r = provisioned(6, 0x3F); + auto h = host(r, config(r)); + SchedulerTime t = std::chrono::steady_clock::now(); + CHECK(h->step(t).ready); + for (std::uint64_t tick = 1; tick <= 9; ++tick) + CHECK(h->submit(input(tick))); + step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 6; }); + h->drain(); + CHECK(!h->status().ready && h->submit(input(10)).error() == AdmissionError::draining); + auto dropped = h->take_dropped(); + CHECK(dropped.size() == 3 && dropped[0].reason == Unacknowledged::drained); + const auto& s = h->step(t); + CHECK(s.phase == HostPhase::stopped && s.committed_tick == 6); + CHECK(h->take_acks().size() == 6); + auto successor = host(r, config(r)); + const auto& next = successor->step(t); + CHECK(next.phase == HostPhase::running && next.committed_tick == 6 && next.epoch == 2); + successor->drain(); + CHECK(successor->step(t).phase == HostPhase::stopped); +} + +// A committed head this host cannot restore quarantines; it is never replaced. +void unrestorable_head_quarantines() { + const auto r = provisioned(7, 0x40); + auto c = config(r); + c.build = filled<32>(0xB2); + c.retained_builds = {c.build}; // Not the build that produced the head. + auto h = host(r, c); + const auto& s = h->step(std::chrono::steady_clock::now()); + CHECK(s.phase == HostPhase::quarantined && !s.ready && h->state() == nullptr); + CHECK(h->submit(input(1)).error() == AdmissionError::not_ready); + // Wrong frame and invalid configuration are refused too. + auto wrong = config(r); + wrong.frame = 10; + CHECK(host(r, wrong)->step(std::chrono::steady_clock::now()).phase == + HostPhase::quarantined); + auto invalid = config(r); + invalid.retained_builds = {}; + auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes); + CHECK(!RegionHost::create(invalid, runtime_connect(), std::move(*objects))); +} +} // namespace + +void run_region_host_tests() { + acknowledgements_follow_commits(); + starvation_accrues_bounded_debt(); + storage_outage_pauses_the_region(); + unknown_outcome_is_resolved_before_continuing(); + renewal_failure_fences_and_recovers(); + drain_commits_and_hands_over(); + unrestorable_head_quarantines(); +} diff --git a/tests/region_server_process_tests.cpp b/tests/region_server_process_tests.cpp new file mode 100644 index 0000000..52e5ffd --- /dev/null +++ b/tests/region_server_process_tests.cpp @@ -0,0 +1,334 @@ +// Real process tests for evo-server (M7.6, ADR-0064): SIGTERM drain and SIGKILL +// with recovery by a successor process. Run by tools/ci/postgres-check.sh, which +// sets EVO_SERVER_BIN. +#include "evo/persistence/checkpoint_store.hpp" +#include "fixtures/environmental_region.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern char** environ; +void run_region_server_process_tests(); + +namespace { +using namespace evo::persistence; +using namespace std::chrono_literals; +namespace fs = std::filesystem; +using Clock = std::chrono::steady_clock; +void check(bool value, std::source_location at = std::source_location::current()) { + if (!value) + throw std::runtime_error("region server process test line " + std::to_string(at.line())); +} +#define CHECK(...) check(static_cast(__VA_ARGS__)) +std::string env(const char* key) { + const auto* value = std::getenv(key); + if (!value) + throw std::runtime_error("missing test environment"); + return value; +} +std::unique_ptr owner() { + auto made = Postgres::connect( + {env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("PGUSER"), {}, {}}, 5s); + CHECK(made); + return std::move(*made); +} +std::string hex32(std::uint64_t tick) { + char out[33]; + std::snprintf(out, sizeof out, "9b000000000000000000000000%06llx", + static_cast(tick)); + return out; +} +std::string input_line(std::uint64_t tick, bool consumer_dies) { + const auto in = evo::test::region_input(tick, consumer_dies); + std::string line = "input " + hex32(tick) + " " + std::to_string(in.temperature_mK) + " " + + std::to_string(in.source_enabled); + for (const auto b : in.bite) + line += " " + std::to_string(b); + for (const auto t : in.terminal) + line += " " + std::to_string(t); + return line + "\n"; +} + +class Child { + public: + explicit Child(const std::vector& args) { + int to_child[2], from_child[2]; + CHECK(::pipe(to_child) == 0 && ::pipe(from_child) == 0); + posix_spawn_file_actions_t actions; + posix_spawn_file_actions_init(&actions); + posix_spawn_file_actions_adddup2(&actions, to_child[0], STDIN_FILENO); + posix_spawn_file_actions_adddup2(&actions, from_child[1], STDOUT_FILENO); + posix_spawn_file_actions_addclose(&actions, to_child[1]); + posix_spawn_file_actions_addclose(&actions, from_child[0]); + std::vector argv; + argv.reserve(args.size() + 1); + for (const auto& a : args) + argv.push_back(const_cast(a.c_str())); + argv.push_back(nullptr); + CHECK(posix_spawn(&pid_, argv[0], &actions, nullptr, argv.data(), environ) == 0); + posix_spawn_file_actions_destroy(&actions); + ::close(to_child[0]); + ::close(from_child[1]); + in_ = to_child[1]; + out_ = from_child[0]; + } + Child(const Child&) = delete; + Child& operator=(const Child&) = delete; + ~Child() { + if (pid_ > 0) { + ::kill(pid_, SIGKILL); + ::waitpid(pid_, nullptr, 0); + } + ::close(in_); + ::close(out_); + } + void send(const std::string& text) { + std::size_t done = 0; + while (done < text.size()) { + const auto n = ::write(in_, text.data() + done, text.size() - done); + CHECK(n > 0 || errno == EINTR); + if (n > 0) + done += static_cast(n); + } + } + // Reads lines until one satisfies `want`; every line is kept in lines(). + template bool wait_for(Want want, std::chrono::milliseconds timeout) { + const auto deadline = Clock::now() + timeout; + for (std::size_t seen = 0;;) { + for (; seen < lines_.size(); ++seen) + if (want(lines_[seen])) + return true; + const auto left = + std::chrono::duration_cast(deadline - Clock::now()); + if (left.count() <= 0 || !read_some(static_cast(left.count()))) + return false; + } + } + void signal(int sig) { + CHECK(::kill(pid_, sig) == 0); + } + // Exit code, or 128+signal; drains remaining output. + int wait_exit(std::chrono::milliseconds timeout) { + const auto deadline = Clock::now() + timeout; + int status = 0; + for (;;) { + read_some(20); + const auto r = ::waitpid(pid_, &status, WNOHANG); + if (r == pid_) + break; + CHECK(Clock::now() < deadline); + } + while (read_some(20)) { + } + pid_ = -1; + return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status); + } + [[nodiscard]] const std::vector& lines() const noexcept { + return lines_; + } + + private: + bool read_some(int timeout_ms) { + pollfd fd{out_, POLLIN, 0}; + if (::poll(&fd, 1, timeout_ms) <= 0) + return false; + char chunk[4096]; + const auto n = ::read(out_, chunk, sizeof chunk); + if (n <= 0) + return false; + buffer_.append(chunk, static_cast(n)); + for (auto end = buffer_.find('\n'); end != std::string::npos; end = buffer_.find('\n')) { + lines_.push_back(buffer_.substr(0, end)); + buffer_.erase(0, end + 1); + } + return true; + } + pid_t pid_{-1}; + int in_{-1}, out_{-1}; + std::string buffer_; + std::vector lines_; +}; + +struct Region { + RegionKey key; + fs::path root; +}; +Region provision(std::uint64_t region, std::uint8_t genesis) { + Region r{{7, region}, fs::path(env("PGHOST")) / ("server-" + std::to_string(region))}; + fs::remove_all(r.root); + fs::create_directory(r.root); + fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace); + auto db = owner(); + CHECK(db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " + "(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING", + {}, 5s)); + auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes); + CHECK(objects); + Sha256Digest build{}; + build.fill(0xB1); + Uuid commit{}; + commit.fill(genesis); + CHECK(provision_lineage_region(*db, **objects, r.key, evo::test::region_identity.frame, + *evo::test::lineage_region(), commit, build, 8000000, 5s)); + return r; +} +std::vector server_args(const Region& r) { + return {env("EVO_SERVER_BIN"), + "--profile", + "lineage", + "--world", + "7", + "--region", + std::to_string(r.key.region), + "--frame", + std::to_string(evo::test::region_identity.frame), + "--pg-host", + env("PGHOST"), + "--pg-port", + env("PGPORT"), + "--pg-database", + env("PGDATABASE"), + "--pg-user", + env("EVO_PG_RUNTIME_USER"), + "--objects", + r.root.string(), + "--build", + "b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1", + "--flush-ms", + "500", + "--lease-profile", + "fast", + "--ready-file", + (r.root / "ready").string()}; +} +// (tick, commit_sequence) of every ack line. +std::vector> acks(const Child& c) { + std::vector> out; + for (const auto& line : c.lines()) { + unsigned long long tick = 0, sequence = 0; + char request[40]; + if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &tick, &sequence) == 3) + out.emplace_back(tick, sequence); + } + return out; +} +std::string head_of(RegionKey key) { + const std::array p{std::to_string(key.region)}; + auto rows = owner()->query("SELECT h.end_tick::text || ':' || h.commit_sequence::text || ':' " + "|| coalesce(l.holder::text, 'released') FROM " + "evolution.region_heads h JOIN evolution.region_leases l USING " + "(world_id, region_id) WHERE world_id = 7 AND region_id = " + "$1::evolution.u64", + p, 5s); + CHECK(rows && rows->size() == 1); + const auto& value = (*rows)[0][0]; + CHECK(value.has_value()); + return value.value_or(""); +} +std::uint64_t head_tick(RegionKey key) { + return std::stoull(head_of(key).substr(0, head_of(key).find(':'))); +} +bool starts(const std::string& line, std::string_view prefix) { + return line.starts_with(prefix); +} + +// SIGTERM: stop admission, commit pending ticks, release the lease, exit 0. Every +// acknowledged tick is committed, the backlog never exceeds its bound, and the +// permanent outcome was announced only after its commit. +void sigterm_drains_and_releases() { + const auto r = provision(31, 0x42); + Child server{server_args(r)}; + CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); + CHECK(fs::exists(r.root / "ready")); + std::string burst; + for (std::uint64_t tick = 1; tick <= 80; ++tick) + burst += input_line(tick, true); + server.send(burst); // More than the 64-input backlog: stdin backpressures. + CHECK(server.wait_for( + [](const std::string& l) { return starts(l, "ack " + hex32(80) + " 80 "); }, 30s)); + server.signal(SIGTERM); + CHECK(server.wait_exit(20s) == 0); + const auto& lines = server.lines(); + CHECK(std::ranges::find(lines, "exit 0") != lines.end()); + CHECK(std::ranges::find_if(lines, [](const std::string& l) { + return starts(l, "phase stopped"); + }) != lines.end()); + unsigned long long max_queued = 999; + for (const auto& l : lines) + std::sscanf(l.c_str(), "stats max_queued=%llu", &max_queued); + CHECK(max_queued <= 64); + const auto all = acks(server); + CHECK(all.size() == 80 && all.back().first == 80); + // The death at tick 20 is announced only with its commit: after the final + // acknowledgement of tick 20, which the host emits only after confirmation. + const auto outcome = std::ranges::find_if( + lines, [](const std::string& l) { return starts(l, "outcome 20 1 11 "); }); + const auto ack20 = std::ranges::find_if( + lines, [](const std::string& l) { return starts(l, "ack " + hex32(20) + " 20 "); }); + CHECK(outcome != lines.end() && ack20 != lines.end() && ack20 < outcome); + CHECK(head_of(r.key).starts_with("80:") && head_of(r.key).ends_with(":released")); + CHECK(!fs::exists(r.root / "ready")); + std::printf("server_sigterm commits=%s acks=%zu max_queued=%llu\n", head_of(r.key).c_str(), + all.size(), max_queued); +} + +// SIGKILL: no drain. Nothing acknowledged is lost, nothing unacknowledged was +// claimed, and a successor becomes ready only after the database lease expires. +void sigkill_recovers_through_a_successor() { + const auto r = provision(32, 0x43); + std::uint64_t acked_max = 0; + { + Child server{server_args(r)}; + CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); + std::string burst; + for (std::uint64_t tick = 1; tick <= 60; ++tick) + burst += input_line(tick, false); + server.send(burst); + CHECK(server.wait_for([](const std::string& l) { return starts(l, "ack "); }, 15s)); + std::this_thread::sleep_for(300ms); // Ticks run past the last commit. + server.signal(SIGKILL); + CHECK(server.wait_exit(10s) == 128 + SIGKILL); + for (const auto& [tick, sequence] : acks(server)) { + static_cast(sequence); + acked_max = std::max(acked_max, tick); + } + } + const auto committed = head_tick(r.key); + CHECK(acked_max > 0 && committed >= acked_max); // No acknowledged tick was lost. + CHECK(!head_of(r.key).ends_with(":released")); // The killed holder never released. + const auto started = Clock::now(); + Child successor{server_args(r)}; + CHECK(successor.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); + const auto to_ready = + std::chrono::duration_cast(Clock::now() - started).count(); + const auto running = std::ranges::find_if( + successor.lines(), [](const std::string& l) { return starts(l, "phase running"); }); + CHECK(running != successor.lines().end()); + CHECK(running->find("committed=" + std::to_string(committed) + " ") != std::string::npos); + CHECK(running->find("epoch=2 ") != std::string::npos); + successor.signal(SIGTERM); + CHECK(successor.wait_exit(20s) == 0); + CHECK(head_tick(r.key) == committed && head_of(r.key).ends_with(":released")); + std::printf("server_sigkill acked_max=%llu committed=%llu successor_ready_ms=%lld\n", + static_cast(acked_max), + static_cast(committed), static_cast(to_ready)); +} +} // namespace + +void run_region_server_process_tests() { + sigterm_drains_and_releases(); + sigkill_recovers_through_a_successor(); +} diff --git a/tools/ci/postgres-check.sh b/tools/ci/postgres-check.sh index de93066..5602209 100644 --- a/tools/ci/postgres-check.sh +++ b/tools/ci/postgres-check.sh @@ -17,7 +17,10 @@ psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_runtime_test NOLOGIN' \ export EVO_PG_RUNTIME_ROLE=evo_runtime_test EVO_PG_RUNTIME_USER=evo_runtime_login bash "$root/tools/migrate-postgres.sh" bash "$root/tools/migrate-postgres.sh" # exact supported upgrade/restart is a no-op -"${1:?path to evo-postgres-tests}" +# Process tests spawn the server built beside the test binary. +export EVO_SERVER_BIN +EVO_SERVER_BIN="$(dirname "${1:?path to evo-postgres-tests}")/evo-server" +"$1" v1_sha="$(sha256sum "$root/migrations/0001_region_bootstrap.sql")" v1_sha="${v1_sha%% *}" # Migration failure must not publish a partial schema.