Complete M7.6 bounded authoritative region host

ADR-0064: RegionHost<State> (environmental and lineage profiles) is a
single-owner, single-threaded state machine advanced by step(now):
acquire the fenced lease and recover the verified head, run, resolve
ambiguous commits, drain and stop, or quarantine. Admission is
provisional and bounded (64 inputs, 64 KiB) with backpressure; FinalAck
and FinalOutcome are emitted only after a confirmed commit, and inputs
that will never be acknowledged are reported with a reason. Ticks follow
the caller's scheduling clock with bounded catch-up and measured tick
debt, never invented inputs. A death seals the batch and commits in the
same step, so no dependent tick uses an uncommitted outcome. Storage
outages pause and retry, unknown outcomes resolve by exact bytes,
fencing discards unconfirmed work and re-acquires; readiness fails
closed. Drain commits pending ticks and releases the lease.

evo-server hosts one region with a bounded stdin input feed, stdout
event lines, a readiness file and SIGTERM drain (exit 0/3/4).

Validated: Windows CTest 58/58; PostgreSQL suites under GCC Release,
clang-tidy and ASan/UBSan including real-process SIGTERM drain and
SIGKILL with successor recovery; four host mutations each caught. No
gateway, metrics, Docker/Gitea CI or deployment.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-28 20:53:11 -04:00
co-authored by Claude Opus 5.5
parent ab4bf5355f
commit d6d0408aae
15 changed files with 1934 additions and 3 deletions
+12 -2
View File
@@ -103,11 +103,21 @@ if(EVO_BUILD_POSTGRES)
target_link_libraries(evo_region_store PUBLIC evo::postgres evo::region_checkpoint
PRIVATE OpenSSL::Crypto)
evo_owned_target(evo_region_store)
# M7.6 authoritative region host and its process (ADR-0064).
add_library(evo_region_host STATIC src/server/region_host.cpp)
add_library(evo::region_host ALIAS evo_region_host)
target_link_libraries(evo_region_host PUBLIC evo::region_store)
evo_owned_target(evo_region_host)
add_executable(evo-server apps/evo-server/main.cpp)
target_link_libraries(evo-server PRIVATE evo::region_host)
evo_owned_target(evo-server)
if(BUILD_TESTING)
# Contention tests use one thread and one connection per contender.
add_executable(evo-postgres-tests tests/postgres_tests.cpp tests/region_lease_store_tests.cpp
tests/region_checkpoint_store_tests.cpp)
target_link_libraries(evo-postgres-tests PRIVATE evo::region_store Threads::Threads)
tests/region_checkpoint_store_tests.cpp tests/region_host_tests.cpp
tests/region_server_process_tests.cpp)
target_link_libraries(evo-postgres-tests PRIVATE evo::region_host Threads::Threads)
add_dependencies(evo-postgres-tests evo-server) # Spawned by the process tests.
evo_owned_target(evo-postgres-tests)
# Intentionally not an ordinary CTest: tools/ci/postgres-check.sh owns its private DB.
endif()
+373
View File
@@ -0,0 +1,373 @@
// M7.6 region server process (ADR-0064). Development host for one region: tick
// inputs arrive as lines on stdin (the network gateway is later work); events are
// written as lines on stdout. Linux only; requires the PostgreSQL build.
//
// Input: input <request:32 hex> <temperature_mK> <source_enabled> <bite0..4> <terminal0..1>
// Output: phase <name> epoch=<e> committed=<t> simulated=<t>
// ready <0|1>
// rejected <request> <reason> | dropped <request> <reason>
// ack <request> <tick> <commit_sequence>
// outcome <tick> <actor> <organism> <commit_sequence>
// stats max_queued=<n> max_tick_debt=<n> commits=<n> store_failures=<n>
// Exit: 0 drained, 2 usage, 3 drain deadline passed, 4 quarantined.
#include "evo/server/region_host.hpp"
#include <algorithm>
#include <array>
#include <cerrno>
#include <charconv>
#include <csignal>
#include <cstdio>
#include <filesystem>
#include <fstream>
#include <optional>
#include <poll.h>
#include <string>
#include <string_view>
#include <unistd.h>
#include <vector>
namespace {
using namespace evo;
using namespace std::chrono_literals;
volatile std::sig_atomic_t stop_requested = 0;
extern "C" void on_stop(int) {
stop_requested = 1;
}
struct Options {
std::string profile{"lineage"};
persistence::PgOptions pg;
std::filesystem::path objects, ready_file;
server::HostConfig host;
std::chrono::milliseconds drain{10000};
};
std::optional<std::uint64_t> number(std::string_view text) {
std::uint64_t value{};
const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), value);
if (error != std::errc{} || end != text.data() + text.size())
return std::nullopt;
return value;
}
template <std::size_t N> std::optional<std::array<std::uint8_t, N>> hex(std::string_view text) {
if (text.size() != N * 2)
return std::nullopt;
std::array<std::uint8_t, N> out{};
for (std::size_t i = 0; i < N; ++i) {
unsigned value{};
const auto [end, error] =
std::from_chars(text.data() + (2 * i), text.data() + (2 * i) + 2, value, 16);
if (error != std::errc{} || end != text.data() + (2 * i) + 2)
return std::nullopt;
out[i] = static_cast<std::uint8_t>(value);
}
return out;
}
template <std::size_t N> std::string text(const std::array<std::uint8_t, N>& bytes) {
constexpr char digits[] = "0123456789abcdef";
std::string out;
for (const auto b : bytes) {
out.push_back(digits[b >> 4U]);
out.push_back(digits[b & 0x0FU]);
}
return out;
}
std::optional<Options> parse(int argc, char** argv) {
Options o;
o.host.retained_builds.clear();
for (int i = 1; i < argc; ++i) {
const std::string_view flag = argv[i];
if (i + 1 >= argc)
return std::nullopt;
const std::string_view value = argv[++i];
const auto n = number(value);
if (flag == "--profile" && (value == "lineage" || value == "environmental"))
o.profile = value;
else if (flag == "--world" && n)
o.host.key.world = *n;
else if (flag == "--region" && n)
o.host.key.region = *n;
else if (flag == "--frame" && n)
o.host.frame = *n;
else if (flag == "--pg-host")
o.pg.host = value;
else if (flag == "--pg-hostaddr")
o.pg.hostaddr = value;
else if (flag == "--pg-port")
o.pg.port = value;
else if (flag == "--pg-database")
o.pg.database = value;
else if (flag == "--pg-user")
o.pg.user = value;
else if (flag == "--pg-passfile")
o.pg.password_file = value;
else if (flag == "--pg-ca")
o.pg.ca_file = value;
else if (flag == "--objects")
o.objects = value;
else if (flag == "--ready-file")
o.ready_file = value;
else if (flag == "--build" || flag == "--retained") {
const auto digest = hex<32>(value);
if (!digest)
return std::nullopt;
if (flag == "--build")
o.host.build = *digest;
else
o.host.retained_builds.push_back(*digest);
} else if (flag == "--flush-ms" && n && *n > 0 && *n <= 60000)
o.host.flush_interval = std::chrono::milliseconds{*n};
else if (flag == "--drain-ms" && n && *n > 0 && *n <= 600000)
o.drain = std::chrono::milliseconds{*n};
else if (flag == "--lease-profile" && value == "fast") // Development/test only.
o.host.lease = {4000ms, 800ms, 400ms, 400ms, 1000ms};
else if (flag == "--lease-profile" && value == "default")
o.host.lease = {};
else
return std::nullopt;
}
if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end())
o.host.retained_builds.push_back(o.host.build);
return o;
}
std::optional<server::TickInput> parse_input(std::string_view line) {
std::array<std::string_view, 11> fields{};
std::size_t count = 0;
while (!line.empty() && count < fields.size() + 1) {
const auto start = line.find_first_not_of(' ');
if (start == std::string_view::npos)
break;
line.remove_prefix(start);
const auto end = line.find(' ');
if (count == fields.size())
return std::nullopt; // Too many fields.
fields[count++] = line.substr(0, end);
line.remove_prefix(end == std::string_view::npos ? line.size() : end);
}
if (count != fields.size() || fields[0] != "input")
return std::nullopt;
auto request = hex<16>(fields[1]);
if (!request)
return std::nullopt;
std::array<std::uint64_t, 9> v{};
for (std::size_t i = 0; i < v.size(); ++i) {
const auto n = number(fields[i + 2]);
if (!n)
return std::nullopt;
v[i] = *n;
}
server::TickInput in{*request, {}};
in.input.temperature_mK = v[0];
in.input.source_enabled = v[1];
for (std::size_t k = 0; k < 5; ++k)
in.input.bite[k] = v[2 + k];
in.input.terminal = {v[7], v[8]};
return in;
}
const char* phase_name(server::HostPhase p) {
switch (p) {
case server::HostPhase::acquiring:
return "acquiring";
case server::HostPhase::running:
return "running";
case server::HostPhase::resolving:
return "resolving";
case server::HostPhase::draining:
return "draining";
case server::HostPhase::stopped:
return "stopped";
case server::HostPhase::quarantined:
return "quarantined";
}
return "unknown";
}
const char* admission_name(server::AdmissionError e) {
switch (e) {
case server::AdmissionError::not_ready:
return "not_ready";
case server::AdmissionError::draining:
return "draining";
case server::AdmissionError::full:
return "full";
case server::AdmissionError::duplicate:
return "duplicate";
case server::AdmissionError::invalid:
return "invalid";
}
return "unknown";
}
const char* dropped_name(server::Unacknowledged u) {
switch (u) {
case server::Unacknowledged::refused:
return "refused";
case server::Unacknowledged::drained:
return "drained";
case server::Unacknowledged::fenced:
return "fenced";
case server::Unacknowledged::quarantined:
return "quarantined";
}
return "unknown";
}
// Reads complete, bounded lines from stdin without blocking.
class LineReader {
public:
// False on EOF/error.
bool fill() {
std::array<char, 4096> chunk{};
const auto n = ::read(STDIN_FILENO, chunk.data(), chunk.size());
if (n <= 0)
return n < 0 && errno == EINTR;
buffer_.append(chunk.data(), static_cast<std::size_t>(n));
return true;
}
std::optional<std::string> next() {
const auto end = buffer_.find('\n');
if (end == std::string::npos) {
if (buffer_.size() > 512)
buffer_.clear(); // Overlong line: discarded, never partially parsed.
return std::nullopt;
}
auto line = buffer_.substr(0, end);
buffer_.erase(0, end + 1);
return line;
}
private:
std::string buffer_;
};
template <typename State> int serve(Options& o) {
auto objects = persistence::FilesystemObjectStore::open(o.objects, o.host.key,
persistence::max_lineage_payload_bytes);
if (!objects) {
std::puts("error objects");
return 2;
}
const auto pg = o.pg;
auto made = server::RegionHost<State>::create(
o.host, [pg] { return persistence::Postgres::connect(pg, 2s); }, std::move(*objects));
if (!made) {
std::puts("error config");
return 2;
}
auto& host = **made;
LineReader reader;
bool input_open = true, ready = false;
std::optional<std::string> pending;
std::optional<server::HostPhase> phase;
std::size_t max_queued = 0;
std::optional<std::chrono::steady_clock::time_point> drain_deadline;
const auto set_ready = [&](bool now) {
if (now == ready)
return;
ready = now;
std::printf("ready %d\n", ready ? 1 : 0);
if (!o.ready_file.empty()) {
std::error_code ignored;
if (ready)
std::ofstream(o.ready_file) << "ready\n";
else
std::filesystem::remove(o.ready_file, ignored);
}
};
for (;;) {
if (stop_requested && !drain_deadline) {
host.drain();
drain_deadline = std::chrono::steady_clock::now() + o.drain;
}
// Read only while the bounded backlog has room: stdin itself backpressures.
const bool room = host.status().queued < o.host.max_queued_inputs && !pending;
pollfd fd{STDIN_FILENO, POLLIN, 0};
if (input_open && room && !drain_deadline && ::poll(&fd, 1, 5) > 0)
input_open = reader.fill();
else if (!input_open || !room || drain_deadline)
::usleep(2000);
while (!drain_deadline) {
if (!pending)
pending = reader.next();
if (!pending)
break;
auto in = parse_input(*pending);
if (!in) {
std::puts("rejected - malformed");
pending.reset();
continue;
}
auto admitted = host.submit(*in);
if (!admitted && admitted.error() == server::AdmissionError::full)
break; // Keep the line and retry after ticks drain the backlog.
if (!admitted)
std::printf("rejected %s %s\n", text(in->request).c_str(),
admission_name(admitted.error()));
pending.reset();
}
const auto& s = host.step(std::chrono::steady_clock::now());
max_queued = std::max(max_queued, s.queued);
if (!phase || *phase != s.phase) {
phase = s.phase;
std::printf("phase %s epoch=%llu committed=%llu simulated=%llu\n", phase_name(s.phase),
static_cast<unsigned long long>(s.epoch),
static_cast<unsigned long long>(s.committed_tick),
static_cast<unsigned long long>(s.simulated_tick));
}
set_ready(s.ready);
for (const auto& a : host.take_acks())
std::printf("ack %s %llu %llu\n", text(a.request).c_str(),
static_cast<unsigned long long>(a.tick),
static_cast<unsigned long long>(a.commit_sequence));
for (const auto& x : host.take_outcomes())
std::printf("outcome %llu %u %llu %llu\n",
static_cast<unsigned long long>(x.decision.tick),
static_cast<unsigned>(x.decision.actor),
static_cast<unsigned long long>(x.decision.organism),
static_cast<unsigned long long>(x.commit_sequence));
for (const auto& d : host.take_dropped())
std::printf("dropped %s %s\n", text(d.input.request).c_str(), dropped_name(d.reason));
std::fflush(stdout);
const auto finish = [&](int code) {
set_ready(false);
std::printf(
"stats max_queued=%zu max_tick_debt=%llu commits=%llu store_failures=%llu\n",
max_queued, static_cast<unsigned long long>(s.max_tick_debt),
static_cast<unsigned long long>(s.commits),
static_cast<unsigned long long>(s.store_failures));
std::printf("exit %d\n", code);
std::fflush(stdout);
return code;
};
if (s.phase == server::HostPhase::stopped)
return finish(0);
if (s.phase == server::HostPhase::quarantined)
return finish(4);
if (drain_deadline && std::chrono::steady_clock::now() > *drain_deadline)
return finish(3); // Pending/unknown work is left for the next owner to reconcile.
}
}
} // namespace
int main(int argc, char** argv) {
auto options = parse(argc, argv);
if (!options) {
std::fputs("usage: evo-server --profile lineage|environmental --world N --region N "
"--frame N --pg-host H --pg-port P --pg-database D --pg-user U "
"--objects DIR --build HEX64 [--retained HEX64]... [--pg-hostaddr A "
"--pg-passfile F --pg-ca F] [--flush-ms N] [--drain-ms N] "
"[--lease-profile default|fast] [--ready-file PATH]\n",
stderr);
return 2;
}
struct sigaction action{};
action.sa_handler = on_stop;
sigemptyset(&action.sa_mask);
sigaction(SIGTERM, &action, nullptr);
sigaction(SIGINT, &action, nullptr);
std::signal(SIGPIPE, SIG_IGN);
return options->profile == "lineage" ? serve<processes::LineageRegion>(*options)
: serve<processes::EnvironmentalCommunity>(*options);
}
+13
View File
@@ -554,3 +554,16 @@ M7.5 implemented and locally qualified (2026-09-28), under ADR-0063.
ambiguous outcomes.
- Recovery reconciles decisions with the restored state.
Next: M7.6 hosts bounded authoritative ticks (apps/evo-server, region_host).
M7.6 implemented and locally qualified (2026-09-28), under ADR-0064.
- `RegionHost` is a single-owner state machine: acquire and recover, run,
resolve, drain and stop, or quarantine.
- Admission is bounded (64 inputs, 64 KiB) with backpressure, and
acknowledgements become final only after a confirmed commit.
- Tick debt is measured with bounded catch-up.
- A death pauses the region until it is durable.
- Readiness fails closed; storage outages, unknown outcomes and fences each have
a defined recovery.
- `evo-server` passed real SIGTERM drain and SIGKILL tests, with successor
recovery.
Next: M7.7 recovery verification at every kill boundary.
+25
View File
@@ -836,3 +836,28 @@ mutations were each caught. Windows and Linux ran 58/58; PostgreSQL suites passe
under Release, analysis and ASan.
Next: M7.6 region host. No host, broker, deployment or process-kill test exists yet.
## M7.6 region host and server process (2026-09-28)
ADR-0064 adds `RegionHost`, a single-owner, single-threaded state machine:
acquire and recover, run, resolve, drain and stop, or quarantine.
- Admission is provisional and bounded (64 inputs, 64 KiB) with backpressure.
- Acknowledgements and outcomes are final only after a confirmed commit.
- A death pauses the region and commits in the same step.
- Tick debt is measured, and catch-up is bounded to 8 ticks per step.
- Readiness fails closed.
`evo-server` hosts one region using a stdin input feed and stdout event lines,
with a readiness file and SIGTERM drain. Real process tests:
- SIGTERM: 80 inputs through the 64-input backlog were drained, committed and
released.
- SIGKILL: every acknowledged tick was committed, and the successor became ready
about 3.9 s later, after the 4 s development lease expired, from the committed
head.
Seven host scenarios pass against real PostgreSQL, and four mutations were each
caught. Windows ran 58/58; the PostgreSQL suites passed under Release, analysis
and ASan/UBSan.
Next: M7.7 recovery verification at every kill boundary. There is no gateway,
metrics or deployment yet.
+118
View File
@@ -0,0 +1,118 @@
# ADR-0064: Bounded authoritative region host
## Status
Accepted for M7.6, 2026-09-28. It composes ADR-0060 (fencing), ADR-0061/0062
(checkpoints) and ADR-0063 (batch outcomes). It adds no gateway, network listener,
metrics endpoint or deployment; those are M7.8 and later.
## Context
The persistence APIs exist, but nothing yet owns a region over time. ADR-0058
requires:
- one simulation owner, with simulated and committed ticks both visible;
- bounded backlogs, with backpressure before admission;
- a whole-region pause at any irreversible outcome until durable confirmation;
- bounded drain, and readiness that fails closed.
Nothing may treat an uncommitted death as fact.
## Decision
**`RegionHost<State>`** (`src/server`, Linux), for `EnvironmentalCommunity` and
`LineageRegion`, is a single-threaded state machine advanced by `step(now)`:
- *acquiring*: take the fenced lease, then recover and verify the committed head.
The recovered quantum and frame must match the configuration.
- *running*: lease upkeep, bounded ticks, and commits.
- *resolving*: an ambiguous commit is resolved before anything else.
- *draining*, then *stopped*; or *quarantined*.
All I/O runs on the caller's thread within the LeasePolicy deadlines. While a
commit runs, the region is stopped.
**Admission** is provisional:
- The backlog holds at most 64 inputs and 64 KiB (88 bytes are accounted per
input). When it is full, the caller gets `full` and must hold its input.
- Duplicate request identities within the host are refused.
- Inputs are refused unless the host is ready.
An input becomes final only when the commit containing its tick is confirmed.
The host then emits `FinalAck(request, tick, commit_sequence)`; permanent
outcomes are emitted as `FinalOutcome`. Inputs that will never be acknowledged
are reported as `Dropped`, with reason refused, drained, fenced or quarantined.
After an unknown outcome, "fenced" means "unknown to this host", not "not
applied".
**Ticks**:
- Scheduling time comes from the caller, while lease deadlines use `LeaseClock`
and database time.
- The due count is (now − start) / quantum. Each step runs at most 8 catch-up
ticks, and one queued input is consumed per tick.
- An empty backlog accrues tick debt. The host never invents inputs, and debt is
measured (current and maximum) rather than skipped.
**Commits** use `RegionBatch`:
- A death seals the batch and commits in the same step, so no dependent tick runs
on an uncommitted outcome.
- Otherwise a batch flushes after `flush_interval` (default 1 s), at 64 ticks,
or on drain.
- The commit ID is the acquisition's holder token with the next sequence number
in its last 8 bytes. It is stable across retries of one batch, and unique
because holder tokens are generated per acquisition.
| Outcome | Host response |
|---|---|
| Storage failure (upload/verify/digest), transient SQL error | Stay paused and retry the same batch; `store_failures` counts it |
| Unknown | Enter *resolving*: resolve the exact prepared bytes. Committed → adopt the receipt. Absent → retry the same bytes while `may_begin_commit` holds. Conflict → quarantine |
| Fenced | Discard state, report unconfirmed inputs, re-acquire and recover from the head |
| Candidate, predecessor, head moved, conflict, receipt mismatch, unrestorable head | Quarantine, releasing the lease if still held |
**Readiness** is true only when running, recovered, not draining, and
`LeaseAuthority::may_originate` holds.
**Drain** stops admission, reports the backlog as drained, commits pending ticks,
releases the lease, then stops. A successor can therefore acquire immediately,
without waiting for expiry.
**`evo-server`** is the development process around one host:
- Tick inputs are stdin lines. It reads only while the backlog has room, so stdin
itself backpressures.
- Events are stdout lines: phase, ready, ack, outcome, dropped, stats.
- An optional readiness file mirrors readiness.
- SIGTERM or SIGINT drain within `--drain-ms`.
- Exit codes: 0 drained, 3 drain deadline passed (pending or unknown work is left
to the next owner), 4 quarantined, 2 usage.
- `--lease-profile fast` (a 4 s lease) is for development tests only.
## Alternatives Considered
- **A separate I/O thread**: would allow ticks during commits, but the contract
pauses the region at commits anyway, and extra threads violate single ownership
without a measured need.
- **Wall-clock-driven synthetic inputs**: would invent world input.
- **Acknowledging on apply**: would expose provisional state as final.
- **Waiting for lease expiry after a clean shutdown**: slows handover for no
safety benefit.
## Benefits
- One place enforces pause-at-outcome, final-only-after-commit and fail-closed
readiness.
- Every failure mode maps to a defined phase.
- The process can be killed or terminated with verifiable durable results.
## Costs
- Ticks stall during commits: about 7 ms locally per batch in Release (M7.5), and
longer if storage is slow.
- The stdin feed is development plumbing, not a network protocol.
- There are no metrics endpoints yet.
## Risks
- Duplicate request detection is per host process, not durable. Client
deduplication across owners needs the gateway protocol.
- The killed-process test measures recovery after the development 4 s lease; the
production 20 s lease bounds unavailability accordingly.
- Clock steps (ADR-0060) still apply.
## Consequences
- M7.7 kills processes at every publication and ACK boundary, and verifies
recovery before readiness.
- M7.8 exposes phase, readiness, tick debt, commit lag and lease safety as probes
and metrics, and packages `evo-server`.
- A gateway must send inputs with request identities and treat only `FinalAck` as
durable.
+1
View File
@@ -74,3 +74,4 @@ design. No unadopted candidate is a link dependency of the proof.
- [ADR-0061: Checkpoint publication](0061-checkpoint-publication.md) - accepted for M7.4a manifest v1, immutable filesystem objects, OpenSSL SHA-256 and verified recovery.
- [ADR-0062: Lineage checkpoint profile](0062-lineage-checkpoint-profile.md) - accepted for M7.4b archive codec, LineageRegion closure and history-capable profile 2.
- [ADR-0063: Batch outcomes](0063-batch-outcomes.md) - accepted for M7.5 manifest v2, validation trace replay, durable decisions/outbox and commit resolution.
- [ADR-0064: Region host](0064-region-host.md) - accepted for M7.6 single-owner host state machine, bounded admission, final-only-after-commit and evo-server process.
+9
View File
@@ -201,3 +201,12 @@ M7.5 (ADR-0063) executed its rows against real PostgreSQL:
Recovery now reconciles durable deaths with the restored lifecycle, so the
checkpoint and durable history must agree. Kill tests at each boundary remain
M7.7.
M7.6 (ADR-0064) executed its host rows:
- A paused stale host fences itself when renewals cannot reach the database. It
reports unconfirmed inputs as never final and re-acquires from the head.
- A death pauses the region in the same step as its commit.
- A storage outage keeps the sealed batch waiting with nothing acknowledged.
- Shutdown drains, commits and releases.
- A real SIGKILLed `evo-server` left no acknowledged tick uncommitted, and its
successor became ready only after lease expiry, from the committed head.
+43
View File
@@ -2296,3 +2296,46 @@ Only clean-to-v1 and exact-v1 restart are supported migration paths at this gate
PostgreSQL suites, all passing, and the repository format check. The changes
are Linux-only persistence code, SQL and tests; the portable targets and
Windows are unchanged.
## M7.6 region host - 2026-09-28
- Windows MSVC Release CTest passed 58/58; this change touches only the Linux
PostgreSQL CMake section and Linux sources. The repository clang-format 19 check
passed.
- WSL Linux: GCC14 Release, clang19 analysis (clang-tidy) and clang19 ASan/UBSan
builds of `evo-postgres-tests` and `evo-server` passed the full PostgreSQL suite.
Final runs: Release 3, analysis 1, ASan 3 across both verification passes.
- The first analysis build flagged unchecked optional accesses and ignored release
results in the host and server, plus two test issues. These were fixed with
explicit local checks rather than suppressions, then rebuilt and rerun in all
three configurations.
- Host scenarios (real PostgreSQL, scaled scheduling clock, 4 s test lease):
- Admission: not_ready before recovery, 64-input cap (full), duplicate and
invalid refusals, byte accounting.
- Ticks 1–19 are provisional with no acks. Tick 20's death pauses and commits
in the same step: 20 acks and one outcome (consumer, organism 11, sequence 1).
The next batch flushed by interval at tick 41.
- Starvation: debt reached 10 without invented ticks, then catch-up of 8 per step.
- Storage outage: the sealed batch waits with nothing acknowledged and debt
growing, then commits when storage recovers.
- Lease row locked: the commit becomes unknown, the host is not ready and
refuses admission, and resolution adopts the receipt after release.
- Renewals blocked past the local deadline: fence, all 8 unconfirmed inputs
reported `fenced`, then re-acquisition at a newer epoch from the committed head.
- Drain (environmental profile): backlog dropped as drained, 6 pending ticks
committed, lease released, and a successor acquired immediately at epoch 2.
- Unrestorable build and foreign frame quarantine; an invalid configuration is
refused.
- Process tests (real `evo-server` via posix_spawn, stdin and stdout pipes):
- SIGTERM after 80 inputs: exit 0, 80 acks, head 80 at sequence 8, lease
released, readiness file removed, max_queued 64. The outcome line followed
tick 20's final ack.
- SIGKILL during running: every acked tick ≤ the committed head (11/11) and the
lease was left held. The successor was ready after 3.94–3.97 s at epoch 2 with
committed=11, then drained cleanly.
- Mutations (restored and rerun green), each caught: acknowledging at simulation
time, not committing a sealed death, reporting ready while resolving, and
draining without releasing the lease.
- Not executed: Docker/Gitea CI, TSan (the host is single-threaded; libpq is
uninstrumented), remote PostgreSQL/TLS, a production 20 s lease kill test,
durable client deduplication, and a network gateway.
+23
View File
@@ -165,3 +165,26 @@ the existing HARBOR_PROJECT and Harbor robot secrets. Pin the reported registry
digest in the example Job; do not use the simulation image. See deploy/README.md.
Local Unix-socket integration is qualified; remote TLS/authentication, actual Secret
mounts and container execution still require environment-specific verification.
## Region server process (M7.6, ADR-0064)
`evo-server` is built with the PostgreSQL build and hosts one provisioned region
as the runtime login. It is a development host with no network gateway.
Tick inputs are stdin lines:
input <request:32 hex> <temperature_mK> <source_enabled> <bite0..4> <terminal0..1>
Events are stdout lines: `phase`, `ready`, `ack`, `outcome`, `dropped` and a
final `stats`. Only `ack` and `outcome` lines are durable facts.
`--ready-file` mirrors readiness for probes. SIGTERM drains (commits pending ticks
and releases the lease) within `--drain-ms`.
Exit codes:
- 0: drained;
- 3: the drain deadline passed, and the next owner reconciles;
- 4: quarantined;
- 2: usage error.
Use the default 20 s lease in any shared environment. `--lease-profile fast` exists
only for local tests. The runtime credentials come from `--pg-*` flags and a
passfile; never place passwords on the command line.
+170
View File
@@ -0,0 +1,170 @@
#pragma once
#include "evo/persistence/checkpoint_store.hpp"
#include "evo/persistence/region_batch.hpp"
#include "evo/persistence/region_lease.hpp"
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <deque>
#include <expected>
#include <functional>
#include <memory>
#include <optional>
#include <vector>
// M7.6 authoritative region host (ADR-0064). Linux; composes the fenced lease,
// verified recovery and batch publication. One owner, one thread: every call runs
// on the caller's thread and all I/O is bounded by the lease policy's deadlines.
// While a batch commits the region is stopped, by design (ADR-0058).
namespace evo::server {
// One tick's host input, from the (future) gateway or a local feed.
struct TickInput {
persistence::Uuid request{}; // Caller identity for acknowledgement; nonzero.
processes::EnvironmentalInput input{};
friend bool operator==(const TickInput&, const TickInput&) = default;
};
inline constexpr std::size_t tick_input_bytes = 16 + 72; // Canonical accounting size.
enum class AdmissionError : std::uint8_t {
not_ready, // No current authority or not recovered: nothing is accepted.
draining, // Shutdown in progress.
full, // Backpressure: bounded backlog (inputs or bytes) is full.
duplicate, // Request identity already queued or pending in this host.
invalid, // Zero request identity.
};
// Final: the input's tick is in a durably committed checkpoint.
struct FinalAck {
persistence::Uuid request{};
std::uint64_t tick{};
std::uint64_t commit_sequence{};
friend bool operator==(const FinalAck&, const FinalAck&) = default;
};
// Final: the permanent outcome is durable (and in the outbox).
struct FinalOutcome {
persistence::BatchDecision decision{};
std::uint64_t commit_sequence{};
friend bool operator==(const FinalOutcome&, const FinalOutcome&) = default;
};
// Why an accepted input will never be acknowledged by this host.
enum class Unacknowledged : std::uint8_t {
refused, // The simulation rejected it; no tick consumed it.
drained, // Still queued at shutdown.
fenced, // Authority was lost before its tick committed; the next owner never saw it.
quarantined,
};
struct Dropped {
TickInput input{};
Unacknowledged reason{};
};
enum class HostPhase : std::uint8_t {
acquiring, // Waiting for the lease (busy/unavailable) or recovering.
running, // Authoritative: simulating and committing.
resolving, // A commit outcome is unknown: paused until resolved.
draining, // Shutdown: committing pending work, then releasing.
stopped, // Drained (lease released when possible).
quarantined, // Fail closed: committed state or history cannot be trusted.
};
struct HostStatus {
HostPhase phase{HostPhase::acquiring};
bool ready{}; // Authoritative now: recovered, running, lease may originate.
std::uint64_t epoch{};
std::uint64_t simulated_tick{}, committed_tick{}, committed_sequence{};
std::size_t queued{}, queued_bytes{};
std::uint64_t tick_debt{}, max_tick_debt{}; // Due minus simulated since running.
std::uint64_t commits{}, store_failures{}, unknown_outcomes{}, fences{};
};
using Connect =
std::function<std::expected<std::unique_ptr<persistence::Postgres>, persistence::PgFailure>()>;
struct HostConfig {
persistence::RegionKey key{};
std::uint64_t frame{};
persistence::LeasePolicy lease{};
// Region quantum; must equal the committed checkpoint's tick_microseconds.
std::chrono::microseconds tick_quantum{50000};
std::chrono::milliseconds flush_interval{1000}; // Commit a non-empty batch this often.
std::size_t max_queued_inputs{64};
std::size_t max_queued_bytes{65536};
std::uint32_t max_catch_up_ticks{8}; // Per step; debt is never skipped.
persistence::Sha256Digest build{};
std::vector<persistence::Sha256Digest> retained_builds;
std::size_t restore_budget{8000000};
};
enum class HostConfigError : std::uint8_t { invalid };
// Scheduling time comes from the caller (monotonic, may be scaled in tests); lease
// deadlines always use LeaseClock (real, suspend-aware) and database time.
using SchedulerTime = std::chrono::steady_clock::time_point;
template <typename State> class RegionHost {
public:
[[nodiscard]] static std::expected<std::unique_ptr<RegionHost>, HostConfigError>
create(HostConfig config, Connect connect,
std::unique_ptr<persistence::FilesystemObjectStore> objects);
RegionHost(const RegionHost&) = delete;
RegionHost& operator=(const RegionHost&) = delete;
RegionHost(RegionHost&&) = delete;
RegionHost& operator=(RegionHost&&) = delete;
~RegionHost();
// Provisional admission only; nothing is final until a FinalAck.
[[nodiscard]] std::expected<void, AdmissionError> submit(const TickInput& input);
// Advances the state machine once: lease upkeep, recovery, resolution, commits
// and up to max_catch_up_ticks ticks. Blocks at most for bounded I/O.
const HostStatus& step(SchedulerTime now);
// Stops admission, drops the backlog, commits pending ticks, releases the lease.
void drain();
[[nodiscard]] std::vector<FinalAck> take_acks();
[[nodiscard]] std::vector<FinalOutcome> take_outcomes();
[[nodiscard]] std::vector<Dropped> take_dropped();
[[nodiscard]] const HostStatus& status() const noexcept {
return status_;
}
// Read-only view for diagnostics/tests; null unless recovered.
[[nodiscard]] const State* state() const noexcept {
return state_.get();
}
private:
struct Applied {
TickInput input;
std::uint64_t tick{};
};
RegionHost(HostConfig config, Connect connect,
std::unique_ptr<persistence::FilesystemObjectStore> objects);
bool database();
void acquire(SchedulerTime now);
void upkeep();
void simulate(SchedulerTime now);
void commit(SchedulerTime now);
void resolve(SchedulerTime now);
void confirmed(const persistence::PublishedBatch& batch, SchedulerTime now);
void lose_authority(Unacknowledged reason, HostPhase next);
void refresh(SchedulerTime now);
[[nodiscard]] persistence::Uuid next_commit_id() const;
HostConfig config_;
Connect connect_;
std::unique_ptr<persistence::FilesystemObjectStore> objects_;
std::unique_ptr<persistence::Postgres> db_;
persistence::LeaseAuthority authority_;
std::unique_ptr<State> state_;
std::optional<persistence::RegionBatch<State>> batch_;
std::optional<persistence::CommittedCheckpoint> committed_;
std::optional<persistence::PreparedCommit> prepared_;
std::deque<TickInput> queue_;
std::vector<Applied> applied_;
std::vector<FinalAck> acks_;
std::vector<FinalOutcome> outcomes_;
std::vector<Dropped> dropped_;
SchedulerTime origin_{}, batch_started_{};
std::uint64_t origin_tick_{};
bool draining_{};
HostStatus status_{};
};
extern template class RegionHost<processes::EnvironmentalCommunity>;
extern template class RegionHost<processes::LineageRegion>;
} // namespace evo::server
+471
View File
@@ -0,0 +1,471 @@
#include "evo/server/region_host.hpp"
#include "evo/persistence/sha256.hpp"
#include <algorithm>
#include <utility>
namespace evo::server {
namespace {
using persistence::LeaseClock;
using persistence::LeaseStoreError;
using persistence::PublicationError;
using persistence::RecoveryError;
// Per-profile calls into the persistence layer.
template <typename State> struct Ops;
template <> struct Ops<processes::EnvironmentalCommunity> {
using State = processes::EnvironmentalCommunity;
static std::uint64_t tick(const State& s) {
return s.progress().tick;
}
static std::uint64_t quantum(const State& s) {
return s.tick_microseconds();
}
template <typename... A> static auto recover(A&&... a) {
return persistence::recover_environmental_checkpoint(std::forward<A>(a)...);
}
template <typename... A> static auto publish(A&&... a) {
return persistence::publish_environmental_batch(std::forward<A>(a)...);
}
};
template <> struct Ops<processes::LineageRegion> {
using State = processes::LineageRegion;
static std::uint64_t tick(const State& s) {
return s.community().progress().tick;
}
static std::uint64_t quantum(const State& s) {
return s.community().tick_microseconds();
}
template <typename... A> static auto recover(A&&... a) {
return persistence::recover_lineage_checkpoint(std::forward<A>(a)...);
}
template <typename... A> static auto publish(A&&... a) {
return persistence::publish_lineage_batch(std::forward<A>(a)...);
}
};
bool valid(const HostConfig& c) {
constexpr std::size_t max_backlog_inputs = 64;
constexpr std::size_t max_backlog_bytes = 65536;
return c.key.world != 0 && c.key.region != 0 && c.frame != 0 && persistence::valid(c.lease) &&
c.tick_quantum.count() > 0 && c.flush_interval.count() > 0 && c.max_queued_inputs >= 1 &&
c.max_queued_inputs <= max_backlog_inputs && c.max_queued_bytes >= tick_input_bytes &&
c.max_queued_bytes <= max_backlog_bytes && c.max_catch_up_ticks >= 1 &&
c.max_catch_up_ticks <= 64 && c.build != persistence::Sha256Digest{} &&
std::ranges::find(c.retained_builds, c.build) != c.retained_builds.end() &&
c.restore_budget > 0;
}
} // namespace
template <typename State>
RegionHost<State>::RegionHost(HostConfig config, Connect connect,
std::unique_ptr<persistence::FilesystemObjectStore> objects)
: config_(std::move(config)), connect_(std::move(connect)), objects_(std::move(objects)),
authority_(config_.lease) {}
template <typename State> RegionHost<State>::~RegionHost() = default;
template <typename State>
std::expected<std::unique_ptr<RegionHost<State>>, HostConfigError>
RegionHost<State>::create(HostConfig config, Connect connect,
std::unique_ptr<persistence::FilesystemObjectStore> objects) {
if (!valid(config) || !connect || !objects)
return std::unexpected{HostConfigError::invalid};
return std::unique_ptr<RegionHost>(
new RegionHost(std::move(config), std::move(connect), std::move(objects)));
}
template <typename State>
std::expected<void, AdmissionError> RegionHost<State>::submit(const TickInput& input) {
if (input.request == persistence::Uuid{})
return std::unexpected{AdmissionError::invalid};
if (draining_)
return std::unexpected{AdmissionError::draining};
if (!status_.ready)
return std::unexpected{AdmissionError::not_ready};
const auto same = [&](const TickInput& x) { return x.request == input.request; };
if (std::ranges::any_of(queue_, same) ||
std::ranges::any_of(applied_, [&](const Applied& a) { return same(a.input); }))
return std::unexpected{AdmissionError::duplicate};
if (queue_.size() >= config_.max_queued_inputs ||
(queue_.size() + 1) * tick_input_bytes > config_.max_queued_bytes)
return std::unexpected{AdmissionError::full};
queue_.push_back(input);
status_.queued = queue_.size();
status_.queued_bytes = queue_.size() * tick_input_bytes;
return {};
}
template <typename State> bool RegionHost<State>::database() {
if (db_)
return true;
auto made = connect_();
if (!made) {
++status_.store_failures;
return false;
}
db_ = std::move(*made);
return true;
}
template <typename State>
void RegionHost<State>::lose_authority(Unacknowledged reason, HostPhase next) {
for (auto& a : applied_)
dropped_.push_back({a.input, reason});
for (auto& q : queue_)
dropped_.push_back({q, reason});
applied_.clear();
queue_.clear();
batch_.reset(); // References state_: reset first.
state_.reset();
committed_.reset();
prepared_.reset();
// Best effort: a quarantined region's lease is released so peers see it free.
const auto& grant = authority_.grant();
if (next == HostPhase::quarantined && db_ && grant &&
authority_.may_originate(LeaseClock::now())) {
[[maybe_unused]] const auto released = persistence::release_region_lease(
*db_, config_.key, *grant, config_.lease.renew_timeout);
}
authority_.fence();
if (reason == Unacknowledged::fenced)
++status_.fences;
status_.phase = draining_ && next == HostPhase::acquiring ? HostPhase::stopped : next;
}
template <typename State> void RegionHost<State>::acquire(SchedulerTime now) {
if (!database())
return;
// A grant survives a transient recovery failure; only re-acquire without one.
if (!authority_.may_originate(LeaseClock::now())) {
const auto sent = LeaseClock::now();
auto grant = persistence::acquire_region_lease(*db_, config_.key, config_.lease.duration,
config_.lease.renew_timeout);
if (!grant) {
if (grant.error().kind == LeaseStoreError::not_provisioned)
status_.phase = HostPhase::quarantined;
else if (grant.error().kind != LeaseStoreError::busy)
db_.reset();
return;
}
if (grant->frame != config_.frame) {
status_.phase = HostPhase::quarantined;
return;
}
if (!authority_.acquired(*grant, sent, LeaseClock::now()))
return; // Reply too late to use; the lease expires unused.
}
auto recovered = Ops<State>::recover(*db_, *objects_, config_.key, config_.retained_builds,
config_.restore_budget, config_.lease.commit_timeout);
if (!recovered) {
if (recovered.error().kind == RecoveryError::store) {
db_.reset(); // Transient: keep the grant and retry recovery.
++status_.store_failures;
return;
}
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
return;
}
if (Ops<State>::quantum(*recovered->state) !=
static_cast<std::uint64_t>(config_.tick_quantum.count())) {
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
return;
}
state_ = std::move(recovered->state);
committed_ = std::move(recovered->committed);
batch_.emplace(*state_);
origin_ = now;
origin_tick_ = Ops<State>::tick(*state_);
status_.max_tick_debt = 0;
status_.phase = draining_ ? HostPhase::draining : HostPhase::running;
}
template <typename State> void RegionHost<State>::upkeep() {
const auto state = authority_.state(LeaseClock::now());
if (state == persistence::LeaseState::fenced) {
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
return;
}
if (state != persistence::LeaseState::renew_due || !database())
return;
const auto grant = *authority_.grant();
const auto sent = LeaseClock::now();
auto renewed = persistence::renew_region_lease(*db_, config_.key, grant, config_.lease.duration,
config_.lease.renew_timeout);
if (renewed) {
if (!authority_.renewed(grant.epoch, *renewed, sent, LeaseClock::now()))
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
return;
}
switch (renewed.error().kind) {
case LeaseStoreError::unknown:
// It may have shortened or extended nothing: keep the earlier deadline.
authority_.renewal_unknown(config_.lease.duration, sent);
db_.reset();
++status_.store_failures;
return;
case LeaseStoreError::fenced:
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
return;
default:
db_.reset();
++status_.store_failures;
return;
}
}
template <typename State> persistence::Uuid RegionHost<State>::next_commit_id() const {
// Unique: the DB-generated holder token names this acquisition; the
// sequence names the link. Stable across retries of the same batch.
const auto& grant = authority_.grant();
if (!grant || !committed_)
return {};
auto id = grant->holder;
const auto sequence = committed_->head.sequence + 1;
for (unsigned i = 0; i < 8; ++i)
id[8 + i] = static_cast<std::uint8_t>((sequence >> (8U * i)) & 0xFFU);
return id;
}
template <typename State>
void RegionHost<State>::confirmed(const persistence::PublishedBatch& published, SchedulerTime now) {
const auto& head = published.committed.head;
if (!batch_ || !batch_->confirm(head.end_tick)) {
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
return;
}
committed_ = published.committed;
for (const auto& a : applied_)
acks_.push_back({a.input.request, a.tick, head.sequence});
applied_.clear();
for (const auto& d : published.decisions)
outcomes_.push_back({d, head.sequence});
prepared_.reset();
batch_started_ = now;
++status_.commits;
status_.phase = draining_ ? HostPhase::draining : HostPhase::running;
}
template <typename State> void RegionHost<State>::commit(SchedulerTime now) {
if (!authority_.may_begin_commit(LeaseClock::now()))
return; // Renewal first; upkeep fences if the deadline passes.
if (!database())
return;
const auto& grant = authority_.grant();
if (!grant || !committed_ || !batch_)
return;
const auto commit_id = next_commit_id();
batch_->seal();
auto published =
Ops<State>::publish(*db_, *objects_, *grant, *committed_, *batch_, commit_id, config_.build,
config_.restore_budget, config_.lease.commit_timeout);
if (published) {
confirmed(*published, now);
return;
}
const auto& f = published.error();
switch (f.kind) {
case PublicationError::upload:
case PublicationError::verify:
case PublicationError::digest:
++status_.store_failures; // Nothing sent: retry the same batch later.
return;
case PublicationError::unknown:
prepared_ = f.prepared;
db_.reset();
++status_.unknown_outcomes;
status_.phase = HostPhase::resolving;
return;
case PublicationError::rejected:
if (f.store.kind == LeaseStoreError::fenced) {
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
return;
}
if (f.store.kind == LeaseStoreError::store || f.store.kind == LeaseStoreError::invalid) {
// SQL/connection error before any mutation: retry with the same bytes.
db_.reset();
++status_.store_failures;
return;
}
[[fallthrough]];
default: // candidate, predecessor, receipt, head moved, conflicts: fail closed.
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
return;
}
}
template <typename State> void RegionHost<State>::resolve(SchedulerTime now) {
if (!database() || !prepared_ || !batch_)
return;
const auto& prepared = *prepared_;
auto resolved = persistence::resolve_prepared(*db_, prepared, config_.lease.commit_timeout);
if (!resolved) {
db_.reset();
++status_.store_failures;
return;
}
const auto adopt = [&](const persistence::RegionHead& receipt) {
auto decisions = persistence::decode_decisions(prepared.decisions, batch_->begin_tick(),
batch_->end_tick());
const auto digest = persistence::sha256(prepared.manifest);
if (!decisions || !digest || *digest != receipt.digest ||
receipt.end_tick != batch_->end_tick()) {
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
return;
}
confirmed({{receipt, prepared.manifest}, *decisions}, now);
};
switch (resolved->status) {
case persistence::Resolution::committed:
adopt(resolved->receipt);
return;
case persistence::Resolution::conflict:
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
return;
case persistence::Resolution::absent:
break;
}
// Absent now: retry exactly the same bytes, only while still authorized.
if (!authority_.may_begin_commit(LeaseClock::now()))
return;
auto retried = persistence::retry_prepared(*db_, prepared, config_.lease.commit_timeout);
if (retried) {
adopt(*retried);
return;
}
switch (retried.error().kind) {
case LeaseStoreError::fenced:
lose_authority(Unacknowledged::fenced, HostPhase::acquiring);
return;
case LeaseStoreError::unknown:
case LeaseStoreError::store:
db_.reset();
++status_.store_failures;
return;
default:
lose_authority(Unacknowledged::quarantined, HostPhase::quarantined);
return;
}
}
template <typename State> void RegionHost<State>::simulate(SchedulerTime now) {
const auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(now - origin_);
const auto due = elapsed.count() <= 0
? std::uint64_t{0}
: static_cast<std::uint64_t>(elapsed / config_.tick_quantum);
if (!batch_ || !state_)
return;
auto& batch = *batch_;
const auto& state = *state_;
for (std::uint32_t k = 0; k < config_.max_catch_up_ticks && !batch.sealed(); ++k) {
if (Ops<State>::tick(state) - origin_tick_ >= due || queue_.empty() ||
!authority_.may_originate(LeaseClock::now()))
break; // Not due, starved (debt accrues; inputs are never invented) or unauthorized.
const auto input = queue_.front();
queue_.pop_front();
if (batch.steps().empty())
batch_started_ = now;
auto report = batch.advance(input.input);
if (!report) {
dropped_.push_back({input, Unacknowledged::refused});
continue;
}
applied_.push_back({input, report->tick});
}
}
template <typename State> void RegionHost<State>::refresh(SchedulerTime now) {
status_.queued = queue_.size();
status_.queued_bytes = queue_.size() * tick_input_bytes;
const auto& grant = authority_.grant();
status_.epoch = grant ? grant->epoch : 0;
status_.committed_tick = committed_ ? committed_->head.end_tick : 0;
status_.committed_sequence = committed_ ? committed_->head.sequence : 0;
status_.simulated_tick = state_ ? Ops<State>::tick(*state_) : status_.committed_tick;
if (state_ && status_.phase == HostPhase::running) {
const auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(now - origin_);
const auto due = elapsed.count() <= 0
? std::uint64_t{0}
: static_cast<std::uint64_t>(elapsed / config_.tick_quantum);
const auto done = Ops<State>::tick(*state_) - origin_tick_;
status_.tick_debt = due > done ? due - done : 0;
status_.max_tick_debt = std::max(status_.max_tick_debt, status_.tick_debt);
}
status_.ready = status_.phase == HostPhase::running && state_ && !draining_ &&
authority_.may_originate(LeaseClock::now());
}
template <typename State> const HostStatus& RegionHost<State>::step(SchedulerTime now) {
switch (status_.phase) {
case HostPhase::acquiring:
acquire(now);
break;
case HostPhase::resolving:
upkeep();
if (status_.phase == HostPhase::resolving)
resolve(now);
break;
case HostPhase::running:
case HostPhase::draining: {
upkeep();
if (status_.phase != HostPhase::running && status_.phase != HostPhase::draining)
break;
if (!draining_)
simulate(now);
if (!batch_)
break; // Running always has a batch; defensive.
const bool flush =
!batch_->steps().empty() &&
(batch_->sealed() || draining_ || now - batch_started_ >= config_.flush_interval);
if (flush)
commit(now); // May lose authority and reset the batch.
if (draining_ && status_.phase == HostPhase::draining && batch_ &&
batch_->steps().empty()) {
const auto& grant = authority_.grant();
if (database() && grant && authority_.may_originate(LeaseClock::now())) {
// Best effort: an unreleased lease simply expires.
[[maybe_unused]] const auto released = persistence::release_region_lease(
*db_, config_.key, *grant, config_.lease.renew_timeout);
}
batch_.reset();
state_.reset();
authority_.fence();
db_.reset();
status_.phase = HostPhase::stopped;
}
break;
}
case HostPhase::stopped:
case HostPhase::quarantined:
break;
}
refresh(now);
return status_;
}
template <typename State> void RegionHost<State>::drain() {
if (draining_)
return;
draining_ = true;
for (auto& q : queue_)
dropped_.push_back({q, Unacknowledged::drained});
queue_.clear();
if (status_.phase == HostPhase::running)
status_.phase = HostPhase::draining;
else if (status_.phase == HostPhase::acquiring)
status_.phase = HostPhase::stopped;
status_.ready = false;
}
template <typename State> std::vector<FinalAck> RegionHost<State>::take_acks() {
return std::exchange(acks_, {});
}
template <typename State> std::vector<FinalOutcome> RegionHost<State>::take_outcomes() {
return std::exchange(outcomes_, {});
}
template <typename State> std::vector<Dropped> RegionHost<State>::take_dropped() {
return std::exchange(dropped_, {});
}
template class RegionHost<processes::EnvironmentalCommunity>;
template class RegionHost<processes::LineageRegion>;
} // namespace evo::server
+6
View File
@@ -5,6 +5,8 @@
#include <stdexcept>
void run_region_lease_store_tests();
void run_region_checkpoint_store_tests();
void run_region_host_tests();
void run_region_server_process_tests();
namespace {
using namespace evo::persistence;
using namespace std::chrono_literals;
@@ -75,6 +77,10 @@ int main() {
std::puts("region lease fencing integration passed");
run_region_checkpoint_store_tests();
std::puts("region checkpoint publication integration passed");
run_region_host_tests();
std::puts("region host integration passed");
run_region_server_process_tests();
std::puts("region server process integration passed");
return 0;
} catch (const std::exception& e) {
std::fprintf(stderr, "%s\n", e.what());
+332
View File
@@ -0,0 +1,332 @@
// Real PostgreSQL M7.6 region host behavior (ADR-0064). Run by tools/ci/postgres-check.sh.
#include "evo/server/region_host.hpp"
#include "fixtures/environmental_region.hpp"
#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <filesystem>
#include <source_location>
#include <stdexcept>
#include <string>
#include <thread>
void run_region_host_tests();
namespace {
using namespace evo::persistence;
using namespace evo::server;
using namespace std::chrono_literals;
using evo::processes::EnvironmentalCommunity;
using evo::processes::LineageRegion;
namespace fs = std::filesystem;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region host test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
std::string env(const char* key) {
const auto* value = std::getenv(key);
if (!value)
throw std::runtime_error("missing test environment");
return value;
}
PgOptions options(const std::string& user) {
return {env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}};
}
std::unique_ptr<Postgres> connect_as(const std::string& user) {
auto made = Postgres::connect(options(user), 5s);
CHECK(made);
return std::move(*made);
}
Connect runtime_connect() {
return [] { return Postgres::connect(options(env("EVO_PG_RUNTIME_USER")), 2s); };
}
template <std::size_t N> std::array<std::uint8_t, N> filled(std::uint8_t value) {
std::array<std::uint8_t, N> out{};
out.fill(value);
return out;
}
const Sha256Digest build = filled<32>(0xB1);
// Fast lease for tests: 4 s, renew every 0.8 s, 0.4 s margin/renew deadline, 1 s commits.
const LeasePolicy fast{4000ms, 800ms, 400ms, 400ms, 1000ms};
TickInput input(std::uint64_t tick, bool consumer_dies = false) {
Uuid request{};
request[0] = 0x9A;
for (unsigned i = 0; i < 8; ++i)
request[8 + i] = static_cast<std::uint8_t>((tick >> (8U * i)) & 0xFFU);
return {request, evo::test::region_input(tick, consumer_dies)};
}
struct Region {
RegionKey key{7, 21};
fs::path root;
};
// Provisions a fresh region (owner) and returns its key and object root.
template <typename State> Region provisioned(std::uint64_t region, std::uint8_t genesis) {
Region r{{7, 20 + region}, fs::path(env("PGHOST")) / ("host-" + std::to_string(region))};
fs::remove_all(r.root);
fs::create_directory(r.root);
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
auto owner = connect_as(env("PGUSER"));
CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
{}, 5s));
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
CHECK(objects);
if constexpr (std::is_same_v<State, LineageRegion>)
CHECK(provision_lineage_region(*owner, **objects, r.key, evo::test::region_identity.frame,
*evo::test::lineage_region(), filled<16>(genesis), build,
8000000, 5s));
else
CHECK(provision_environmental_region(
*owner, **objects, r.key, evo::test::region_identity.frame,
*evo::test::region_community(), filled<16>(genesis), build, 8000000, 5s));
return r;
}
HostConfig config(const Region& r) {
HostConfig c;
c.key = r.key;
c.frame = evo::test::region_identity.frame;
c.lease = fast;
c.build = build;
c.retained_builds = {build};
return c;
}
template <typename State>
std::unique_ptr<RegionHost<State>> host(const Region& r, HostConfig c,
ObjectStoreFaults* faults = nullptr) {
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes, faults);
CHECK(objects);
auto made = RegionHost<State>::create(std::move(c), runtime_connect(), std::move(*objects));
CHECK(made);
return std::move(*made);
}
// Steps until `done` holds, advancing scheduling time by `advance` each step.
template <typename Host, typename Done>
const HostStatus& step_until(Host& h, SchedulerTime& t, std::chrono::microseconds advance,
Done done, int limit = 400) {
for (int i = 0; i < limit; ++i) {
t += advance;
const auto& s = h.step(t);
if (done(s))
return s;
std::this_thread::sleep_for(10ms);
}
CHECK(false);
return h.status();
}
// Admission, provisional ticks, pause at a permanent outcome and final acknowledgement.
void acknowledgements_follow_commits() {
const auto r = provisioned<LineageRegion>(1, 0x3A);
auto h = host<LineageRegion>(r, config(r));
CHECK(h->submit(input(1)).error() == AdmissionError::not_ready);
SchedulerTime t = std::chrono::steady_clock::now();
auto s = h->step(t);
CHECK(s.phase == HostPhase::running && s.ready && s.committed_tick == 0 && s.epoch == 1);
for (std::uint64_t tick = 1; tick <= 64; ++tick)
CHECK(h->submit(input(tick, true)));
CHECK(h->submit(input(65)).error() == AdmissionError::full); // Backlog of 64.
CHECK(h->submit(input(1)).error() == AdmissionError::duplicate);
CHECK(h->submit({{}, {}}).error() == AdmissionError::invalid);
CHECK(h->status().queued_bytes == 64 * tick_input_bytes);
for (std::uint64_t tick = 1; tick <= 19; ++tick) {
t += 50ms;
s = h->step(t);
// Simulated ahead of committed; nothing is final yet.
CHECK(s.simulated_tick == tick && s.committed_tick == 0 && s.ready);
CHECK(h->take_acks().empty() && h->take_outcomes().empty());
}
// Tick 20 kills the consumer: the region pauses and commits in the same step,
// so no dependent tick ever runs on the uncommitted death.
t += 50ms;
s = h->step(t);
CHECK(s.simulated_tick == 20 && s.committed_tick == 20 && s.committed_sequence == 1);
const auto outcomes = h->take_outcomes();
CHECK(outcomes.size() == 1 && outcomes[0].commit_sequence == 1 &&
outcomes[0].decision == BatchDecision{20, Actor::consumer, Outcome::death, 11});
auto acks = h->take_acks();
CHECK(acks.size() == 20);
for (std::uint64_t i = 0; i < 20; ++i)
CHECK(acks[i] == FinalAck{input(i + 1).request, i + 1, 1});
// The next batch flushes after the configured interval, not per tick.
bool provisional = false;
s = step_until(*h, t, 50ms, [&](const HostStatus& x) {
provisional = provisional || (x.simulated_tick > x.committed_tick);
return x.committed_sequence == 2;
});
CHECK(provisional && s.simulated_tick == s.committed_tick && s.committed_tick > 30);
acks = h->take_acks();
CHECK(!acks.empty() && acks.back().tick == s.committed_tick && acks.front().tick == 21);
CHECK(std::ranges::all_of(acks, [](const FinalAck& a) { return a.commit_sequence == 2; }));
CHECK(h->take_dropped().empty());
std::printf("host_commits=%llu committed_tick=%llu\n",
static_cast<unsigned long long>(s.commits),
static_cast<unsigned long long>(s.committed_tick));
}
// No inputs, no invented ticks: debt accrues and is repaid with bounded catch-up.
void starvation_accrues_bounded_debt() {
const auto r = provisioned<LineageRegion>(2, 0x3B);
auto h = host<LineageRegion>(r, config(r));
SchedulerTime t = std::chrono::steady_clock::now();
CHECK(h->step(t).ready);
for (int i = 0; i < 10; ++i) {
t += 50ms;
CHECK(h->step(t).simulated_tick == 0);
}
CHECK(h->status().tick_debt == 10);
for (std::uint64_t tick = 1; tick <= 20; ++tick)
CHECK(h->submit(input(tick)));
auto s = h->step(t);
CHECK(s.simulated_tick == 8 && s.tick_debt == 2); // At most 8 per step.
s = h->step(t);
CHECK(s.simulated_tick == 10 && s.tick_debt == 0 && s.max_tick_debt == 10);
CHECK(s.queued == 10);
}
// Object store outage: the sealed batch waits, nothing is acknowledged, and the
// region resumes once storage is durable again.
void storage_outage_pauses_the_region() {
const auto r = provisioned<LineageRegion>(3, 0x41);
ObjectStoreFaults faults;
auto h = host<LineageRegion>(r, config(r), &faults);
SchedulerTime t = std::chrono::steady_clock::now();
CHECK(h->step(t).ready);
for (std::uint64_t tick = 1; tick <= 30; ++tick)
CHECK(h->submit(input(tick, true)));
step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 19; });
faults.fail_directory_syncs = 1000;
for (int i = 0; i < 5; ++i) {
t += 50ms;
const auto& s = h->step(t);
CHECK(s.simulated_tick == 20 && s.committed_tick == 0); // Paused at the death.
}
CHECK(h->status().store_failures >= 5 && h->take_acks().empty() && h->take_outcomes().empty());
CHECK(h->status().tick_debt == 4);
faults.fail_directory_syncs = 0;
t += 50ms;
auto s = h->step(t);
CHECK(s.committed_tick == 20 && h->take_acks().size() == 20 && h->take_outcomes().size() == 1);
s = h->step(t); // Catch-up after the outage is bounded but continues.
CHECK(s.simulated_tick > 20 && s.simulated_tick <= 28);
}
// Holds the lease row lock to make database calls time out.
struct RowLock {
std::unique_ptr<Postgres> db = connect_as(env("PGUSER"));
explicit RowLock(RegionKey key) {
const std::array<std::string, 2> p{std::to_string(key.world), std::to_string(key.region)};
CHECK(db->query("BEGIN", {}, 5s));
CHECK(db->query("SELECT 1 FROM evolution.region_leases WHERE world_id = $1::evolution.u64 "
"AND region_id = $2::evolution.u64 FOR UPDATE",
p, 5s));
}
void release() {
CHECK(db->query("ROLLBACK", {}, 5s));
}
};
// Unknown commit outcome: not ready, admission refused, resolved by exact bytes.
void unknown_outcome_is_resolved_before_continuing() {
const auto r = provisioned<LineageRegion>(4, 0x3D);
auto h = host<LineageRegion>(r, config(r));
SchedulerTime t = std::chrono::steady_clock::now();
CHECK(h->step(t).ready);
for (std::uint64_t tick = 1; tick <= 25; ++tick)
CHECK(h->submit(input(tick, true)));
step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 19; });
RowLock lock{r.key};
t += 50ms;
auto s = h->step(t); // The death seals; the commit cannot finish in time.
CHECK(s.phase == HostPhase::resolving && !s.ready && s.unknown_outcomes == 1);
CHECK(h->submit(input(26)).error() == AdmissionError::not_ready);
CHECK(h->take_outcomes().empty() && h->take_acks().empty());
lock.release();
s = step_until(*h, t, 0ms, [](const HostStatus& x) { return x.phase == HostPhase::running; });
CHECK(s.committed_tick == 20 && s.ready);
CHECK(h->take_outcomes().size() == 1 && h->take_acks().size() == 20);
}
// Renewals that cannot reach the database fence the host locally; unconfirmed
// work is reported as never final, and authority is re-acquired from the head.
void renewal_failure_fences_and_recovers() {
const auto r = provisioned<LineageRegion>(5, 0x3E);
auto h = host<LineageRegion>(r, config(r));
SchedulerTime t = std::chrono::steady_clock::now();
CHECK(h->step(t).ready);
for (std::uint64_t tick = 1; tick <= 8; ++tick)
CHECK(h->submit(input(tick)));
step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 5; });
CHECK(h->status().committed_tick == 0);
const auto epoch = h->status().epoch;
std::optional<RowLock> lock{std::in_place, r.key};
step_until(*h, t, 0ms, [](const HostStatus& x) { return x.phase != HostPhase::running; }, 1000);
CHECK(!h->status().ready && h->status().fences == 1 && h->state() == nullptr);
const auto dropped = h->take_dropped();
CHECK(dropped.size() == 8 && std::ranges::all_of(dropped, [](const Dropped& d) {
return d.reason == Unacknowledged::fenced;
}));
CHECK(h->take_acks().empty());
lock->release();
lock.reset();
const auto& s = step_until(
*h, t, 0ms, [](const HostStatus& x) { return x.phase == HostPhase::running; }, 1000);
CHECK(s.epoch > epoch && s.committed_tick == 0 && s.simulated_tick == 0 && s.ready);
}
// Drain: backlog dropped, pending ticks committed, lease released for an
// immediate successor (environmental-v1 profile).
void drain_commits_and_hands_over() {
const auto r = provisioned<EnvironmentalCommunity>(6, 0x3F);
auto h = host<EnvironmentalCommunity>(r, config(r));
SchedulerTime t = std::chrono::steady_clock::now();
CHECK(h->step(t).ready);
for (std::uint64_t tick = 1; tick <= 9; ++tick)
CHECK(h->submit(input(tick)));
step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 6; });
h->drain();
CHECK(!h->status().ready && h->submit(input(10)).error() == AdmissionError::draining);
auto dropped = h->take_dropped();
CHECK(dropped.size() == 3 && dropped[0].reason == Unacknowledged::drained);
const auto& s = h->step(t);
CHECK(s.phase == HostPhase::stopped && s.committed_tick == 6);
CHECK(h->take_acks().size() == 6);
auto successor = host<EnvironmentalCommunity>(r, config(r));
const auto& next = successor->step(t);
CHECK(next.phase == HostPhase::running && next.committed_tick == 6 && next.epoch == 2);
successor->drain();
CHECK(successor->step(t).phase == HostPhase::stopped);
}
// A committed head this host cannot restore quarantines; it is never replaced.
void unrestorable_head_quarantines() {
const auto r = provisioned<LineageRegion>(7, 0x40);
auto c = config(r);
c.build = filled<32>(0xB2);
c.retained_builds = {c.build}; // Not the build that produced the head.
auto h = host<LineageRegion>(r, c);
const auto& s = h->step(std::chrono::steady_clock::now());
CHECK(s.phase == HostPhase::quarantined && !s.ready && h->state() == nullptr);
CHECK(h->submit(input(1)).error() == AdmissionError::not_ready);
// Wrong frame and invalid configuration are refused too.
auto wrong = config(r);
wrong.frame = 10;
CHECK(host<LineageRegion>(r, wrong)->step(std::chrono::steady_clock::now()).phase ==
HostPhase::quarantined);
auto invalid = config(r);
invalid.retained_builds = {};
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
CHECK(!RegionHost<LineageRegion>::create(invalid, runtime_connect(), std::move(*objects)));
}
} // namespace
void run_region_host_tests() {
acknowledgements_follow_commits();
starvation_accrues_bounded_debt();
storage_outage_pauses_the_region();
unknown_outcome_is_resolved_before_continuing();
renewal_failure_fences_and_recovers();
drain_commits_and_hands_over();
unrestorable_head_quarantines();
}
+334
View File
@@ -0,0 +1,334 @@
// Real process tests for evo-server (M7.6, ADR-0064): SIGTERM drain and SIGKILL
// with recovery by a successor process. Run by tools/ci/postgres-check.sh, which
// sets EVO_SERVER_BIN.
#include "evo/persistence/checkpoint_store.hpp"
#include "fixtures/environmental_region.hpp"
#include <algorithm>
#include <cerrno>
#include <chrono>
#include <csignal>
#include <cstdio>
#include <cstdlib>
#include <filesystem>
#include <poll.h>
#include <source_location>
#include <spawn.h>
#include <stdexcept>
#include <string>
#include <sys/wait.h>
#include <thread>
#include <unistd.h>
#include <vector>
extern char** environ;
void run_region_server_process_tests();
namespace {
using namespace evo::persistence;
using namespace std::chrono_literals;
namespace fs = std::filesystem;
using Clock = std::chrono::steady_clock;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region server process test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
std::string env(const char* key) {
const auto* value = std::getenv(key);
if (!value)
throw std::runtime_error("missing test environment");
return value;
}
std::unique_ptr<Postgres> owner() {
auto made = Postgres::connect(
{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("PGUSER"), {}, {}}, 5s);
CHECK(made);
return std::move(*made);
}
std::string hex32(std::uint64_t tick) {
char out[33];
std::snprintf(out, sizeof out, "9b000000000000000000000000%06llx",
static_cast<unsigned long long>(tick));
return out;
}
std::string input_line(std::uint64_t tick, bool consumer_dies) {
const auto in = evo::test::region_input(tick, consumer_dies);
std::string line = "input " + hex32(tick) + " " + std::to_string(in.temperature_mK) + " " +
std::to_string(in.source_enabled);
for (const auto b : in.bite)
line += " " + std::to_string(b);
for (const auto t : in.terminal)
line += " " + std::to_string(t);
return line + "\n";
}
class Child {
public:
explicit Child(const std::vector<std::string>& args) {
int to_child[2], from_child[2];
CHECK(::pipe(to_child) == 0 && ::pipe(from_child) == 0);
posix_spawn_file_actions_t actions;
posix_spawn_file_actions_init(&actions);
posix_spawn_file_actions_adddup2(&actions, to_child[0], STDIN_FILENO);
posix_spawn_file_actions_adddup2(&actions, from_child[1], STDOUT_FILENO);
posix_spawn_file_actions_addclose(&actions, to_child[1]);
posix_spawn_file_actions_addclose(&actions, from_child[0]);
std::vector<char*> argv;
argv.reserve(args.size() + 1);
for (const auto& a : args)
argv.push_back(const_cast<char*>(a.c_str()));
argv.push_back(nullptr);
CHECK(posix_spawn(&pid_, argv[0], &actions, nullptr, argv.data(), environ) == 0);
posix_spawn_file_actions_destroy(&actions);
::close(to_child[0]);
::close(from_child[1]);
in_ = to_child[1];
out_ = from_child[0];
}
Child(const Child&) = delete;
Child& operator=(const Child&) = delete;
~Child() {
if (pid_ > 0) {
::kill(pid_, SIGKILL);
::waitpid(pid_, nullptr, 0);
}
::close(in_);
::close(out_);
}
void send(const std::string& text) {
std::size_t done = 0;
while (done < text.size()) {
const auto n = ::write(in_, text.data() + done, text.size() - done);
CHECK(n > 0 || errno == EINTR);
if (n > 0)
done += static_cast<std::size_t>(n);
}
}
// Reads lines until one satisfies `want`; every line is kept in lines().
template <typename Want> bool wait_for(Want want, std::chrono::milliseconds timeout) {
const auto deadline = Clock::now() + timeout;
for (std::size_t seen = 0;;) {
for (; seen < lines_.size(); ++seen)
if (want(lines_[seen]))
return true;
const auto left =
std::chrono::duration_cast<std::chrono::milliseconds>(deadline - Clock::now());
if (left.count() <= 0 || !read_some(static_cast<int>(left.count())))
return false;
}
}
void signal(int sig) {
CHECK(::kill(pid_, sig) == 0);
}
// Exit code, or 128+signal; drains remaining output.
int wait_exit(std::chrono::milliseconds timeout) {
const auto deadline = Clock::now() + timeout;
int status = 0;
for (;;) {
read_some(20);
const auto r = ::waitpid(pid_, &status, WNOHANG);
if (r == pid_)
break;
CHECK(Clock::now() < deadline);
}
while (read_some(20)) {
}
pid_ = -1;
return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status);
}
[[nodiscard]] const std::vector<std::string>& lines() const noexcept {
return lines_;
}
private:
bool read_some(int timeout_ms) {
pollfd fd{out_, POLLIN, 0};
if (::poll(&fd, 1, timeout_ms) <= 0)
return false;
char chunk[4096];
const auto n = ::read(out_, chunk, sizeof chunk);
if (n <= 0)
return false;
buffer_.append(chunk, static_cast<std::size_t>(n));
for (auto end = buffer_.find('\n'); end != std::string::npos; end = buffer_.find('\n')) {
lines_.push_back(buffer_.substr(0, end));
buffer_.erase(0, end + 1);
}
return true;
}
pid_t pid_{-1};
int in_{-1}, out_{-1};
std::string buffer_;
std::vector<std::string> lines_;
};
struct Region {
RegionKey key;
fs::path root;
};
Region provision(std::uint64_t region, std::uint8_t genesis) {
Region r{{7, region}, fs::path(env("PGHOST")) / ("server-" + std::to_string(region))};
fs::remove_all(r.root);
fs::create_directory(r.root);
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
auto db = owner();
CHECK(db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
{}, 5s));
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
CHECK(objects);
Sha256Digest build{};
build.fill(0xB1);
Uuid commit{};
commit.fill(genesis);
CHECK(provision_lineage_region(*db, **objects, r.key, evo::test::region_identity.frame,
*evo::test::lineage_region(), commit, build, 8000000, 5s));
return r;
}
std::vector<std::string> server_args(const Region& r) {
return {env("EVO_SERVER_BIN"),
"--profile",
"lineage",
"--world",
"7",
"--region",
std::to_string(r.key.region),
"--frame",
std::to_string(evo::test::region_identity.frame),
"--pg-host",
env("PGHOST"),
"--pg-port",
env("PGPORT"),
"--pg-database",
env("PGDATABASE"),
"--pg-user",
env("EVO_PG_RUNTIME_USER"),
"--objects",
r.root.string(),
"--build",
"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1",
"--flush-ms",
"500",
"--lease-profile",
"fast",
"--ready-file",
(r.root / "ready").string()};
}
// (tick, commit_sequence) of every ack line.
std::vector<std::pair<std::uint64_t, std::uint64_t>> acks(const Child& c) {
std::vector<std::pair<std::uint64_t, std::uint64_t>> out;
for (const auto& line : c.lines()) {
unsigned long long tick = 0, sequence = 0;
char request[40];
if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &tick, &sequence) == 3)
out.emplace_back(tick, sequence);
}
return out;
}
std::string head_of(RegionKey key) {
const std::array<std::string, 1> p{std::to_string(key.region)};
auto rows = owner()->query("SELECT h.end_tick::text || ':' || h.commit_sequence::text || ':' "
"|| coalesce(l.holder::text, 'released') FROM "
"evolution.region_heads h JOIN evolution.region_leases l USING "
"(world_id, region_id) WHERE world_id = 7 AND region_id = "
"$1::evolution.u64",
p, 5s);
CHECK(rows && rows->size() == 1);
const auto& value = (*rows)[0][0];
CHECK(value.has_value());
return value.value_or("");
}
std::uint64_t head_tick(RegionKey key) {
return std::stoull(head_of(key).substr(0, head_of(key).find(':')));
}
bool starts(const std::string& line, std::string_view prefix) {
return line.starts_with(prefix);
}
// SIGTERM: stop admission, commit pending ticks, release the lease, exit 0. Every
// acknowledged tick is committed, the backlog never exceeds its bound, and the
// permanent outcome was announced only after its commit.
void sigterm_drains_and_releases() {
const auto r = provision(31, 0x42);
Child server{server_args(r)};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
CHECK(fs::exists(r.root / "ready"));
std::string burst;
for (std::uint64_t tick = 1; tick <= 80; ++tick)
burst += input_line(tick, true);
server.send(burst); // More than the 64-input backlog: stdin backpressures.
CHECK(server.wait_for(
[](const std::string& l) { return starts(l, "ack " + hex32(80) + " 80 "); }, 30s));
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 0);
const auto& lines = server.lines();
CHECK(std::ranges::find(lines, "exit 0") != lines.end());
CHECK(std::ranges::find_if(lines, [](const std::string& l) {
return starts(l, "phase stopped");
}) != lines.end());
unsigned long long max_queued = 999;
for (const auto& l : lines)
std::sscanf(l.c_str(), "stats max_queued=%llu", &max_queued);
CHECK(max_queued <= 64);
const auto all = acks(server);
CHECK(all.size() == 80 && all.back().first == 80);
// The death at tick 20 is announced only with its commit: after the final
// acknowledgement of tick 20, which the host emits only after confirmation.
const auto outcome = std::ranges::find_if(
lines, [](const std::string& l) { return starts(l, "outcome 20 1 11 "); });
const auto ack20 = std::ranges::find_if(
lines, [](const std::string& l) { return starts(l, "ack " + hex32(20) + " 20 "); });
CHECK(outcome != lines.end() && ack20 != lines.end() && ack20 < outcome);
CHECK(head_of(r.key).starts_with("80:") && head_of(r.key).ends_with(":released"));
CHECK(!fs::exists(r.root / "ready"));
std::printf("server_sigterm commits=%s acks=%zu max_queued=%llu\n", head_of(r.key).c_str(),
all.size(), max_queued);
}
// SIGKILL: no drain. Nothing acknowledged is lost, nothing unacknowledged was
// claimed, and a successor becomes ready only after the database lease expires.
void sigkill_recovers_through_a_successor() {
const auto r = provision(32, 0x43);
std::uint64_t acked_max = 0;
{
Child server{server_args(r)};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
std::string burst;
for (std::uint64_t tick = 1; tick <= 60; ++tick)
burst += input_line(tick, false);
server.send(burst);
CHECK(server.wait_for([](const std::string& l) { return starts(l, "ack "); }, 15s));
std::this_thread::sleep_for(300ms); // Ticks run past the last commit.
server.signal(SIGKILL);
CHECK(server.wait_exit(10s) == 128 + SIGKILL);
for (const auto& [tick, sequence] : acks(server)) {
static_cast<void>(sequence);
acked_max = std::max(acked_max, tick);
}
}
const auto committed = head_tick(r.key);
CHECK(acked_max > 0 && committed >= acked_max); // No acknowledged tick was lost.
CHECK(!head_of(r.key).ends_with(":released")); // The killed holder never released.
const auto started = Clock::now();
Child successor{server_args(r)};
CHECK(successor.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
const auto to_ready =
std::chrono::duration_cast<std::chrono::milliseconds>(Clock::now() - started).count();
const auto running = std::ranges::find_if(
successor.lines(), [](const std::string& l) { return starts(l, "phase running"); });
CHECK(running != successor.lines().end());
CHECK(running->find("committed=" + std::to_string(committed) + " ") != std::string::npos);
CHECK(running->find("epoch=2 ") != std::string::npos);
successor.signal(SIGTERM);
CHECK(successor.wait_exit(20s) == 0);
CHECK(head_tick(r.key) == committed && head_of(r.key).ends_with(":released"));
std::printf("server_sigkill acked_max=%llu committed=%llu successor_ready_ms=%lld\n",
static_cast<unsigned long long>(acked_max),
static_cast<unsigned long long>(committed), static_cast<long long>(to_ready));
}
} // namespace
void run_region_server_process_tests() {
sigterm_drains_and_releases();
sigkill_recovers_through_a_successor();
}
+4 -1
View File
@@ -17,7 +17,10 @@ psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_runtime_test NOLOGIN' \
export EVO_PG_RUNTIME_ROLE=evo_runtime_test EVO_PG_RUNTIME_USER=evo_runtime_login
bash "$root/tools/migrate-postgres.sh"
bash "$root/tools/migrate-postgres.sh" # exact supported upgrade/restart is a no-op
"${1:?path to evo-postgres-tests}"
# Process tests spawn the server built beside the test binary.
export EVO_SERVER_BIN
EVO_SERVER_BIN="$(dirname "${1:?path to evo-postgres-tests}")/evo-server"
"$1"
v1_sha="$(sha256sum "$root/migrations/0001_region_bootstrap.sql")"
v1_sha="${v1_sha%% *}"
# Migration failure must not publish a partial schema.