Complete M7.6 bounded authoritative region host

ADR-0064: RegionHost<State> (environmental and lineage profiles) is a
single-owner, single-threaded state machine advanced by step(now):
acquire the fenced lease and recover the verified head, run, resolve
ambiguous commits, drain and stop, or quarantine. Admission is
provisional and bounded (64 inputs, 64 KiB) with backpressure; FinalAck
and FinalOutcome are emitted only after a confirmed commit, and inputs
that will never be acknowledged are reported with a reason. Ticks follow
the caller's scheduling clock with bounded catch-up and measured tick
debt, never invented inputs. A death seals the batch and commits in the
same step, so no dependent tick uses an uncommitted outcome. Storage
outages pause and retry, unknown outcomes resolve by exact bytes,
fencing discards unconfirmed work and re-acquires; readiness fails
closed. Drain commits pending ticks and releases the lease.

evo-server hosts one region with a bounded stdin input feed, stdout
event lines, a readiness file and SIGTERM drain (exit 0/3/4).

Validated: Windows CTest 58/58; PostgreSQL suites under GCC Release,
clang-tidy and ASan/UBSan including real-process SIGTERM drain and
SIGKILL with successor recovery; four host mutations each caught. No
gateway, metrics, Docker/Gitea CI or deployment.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-28 20:53:11 -04:00
co-authored by Claude Opus 5.5
parent ab4bf5355f
commit d6d0408aae
15 changed files with 1934 additions and 3 deletions
+334
View File
@@ -0,0 +1,334 @@
// Real process tests for evo-server (M7.6, ADR-0064): SIGTERM drain and SIGKILL
// with recovery by a successor process. Run by tools/ci/postgres-check.sh, which
// sets EVO_SERVER_BIN.
#include "evo/persistence/checkpoint_store.hpp"
#include "fixtures/environmental_region.hpp"
#include <algorithm>
#include <cerrno>
#include <chrono>
#include <csignal>
#include <cstdio>
#include <cstdlib>
#include <filesystem>
#include <poll.h>
#include <source_location>
#include <spawn.h>
#include <stdexcept>
#include <string>
#include <sys/wait.h>
#include <thread>
#include <unistd.h>
#include <vector>
extern char** environ;
void run_region_server_process_tests();
namespace {
using namespace evo::persistence;
using namespace std::chrono_literals;
namespace fs = std::filesystem;
using Clock = std::chrono::steady_clock;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region server process test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
std::string env(const char* key) {
const auto* value = std::getenv(key);
if (!value)
throw std::runtime_error("missing test environment");
return value;
}
std::unique_ptr<Postgres> owner() {
auto made = Postgres::connect(
{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("PGUSER"), {}, {}}, 5s);
CHECK(made);
return std::move(*made);
}
std::string hex32(std::uint64_t tick) {
char out[33];
std::snprintf(out, sizeof out, "9b000000000000000000000000%06llx",
static_cast<unsigned long long>(tick));
return out;
}
std::string input_line(std::uint64_t tick, bool consumer_dies) {
const auto in = evo::test::region_input(tick, consumer_dies);
std::string line = "input " + hex32(tick) + " " + std::to_string(in.temperature_mK) + " " +
std::to_string(in.source_enabled);
for (const auto b : in.bite)
line += " " + std::to_string(b);
for (const auto t : in.terminal)
line += " " + std::to_string(t);
return line + "\n";
}
class Child {
public:
explicit Child(const std::vector<std::string>& args) {
int to_child[2], from_child[2];
CHECK(::pipe(to_child) == 0 && ::pipe(from_child) == 0);
posix_spawn_file_actions_t actions;
posix_spawn_file_actions_init(&actions);
posix_spawn_file_actions_adddup2(&actions, to_child[0], STDIN_FILENO);
posix_spawn_file_actions_adddup2(&actions, from_child[1], STDOUT_FILENO);
posix_spawn_file_actions_addclose(&actions, to_child[1]);
posix_spawn_file_actions_addclose(&actions, from_child[0]);
std::vector<char*> argv;
argv.reserve(args.size() + 1);
for (const auto& a : args)
argv.push_back(const_cast<char*>(a.c_str()));
argv.push_back(nullptr);
CHECK(posix_spawn(&pid_, argv[0], &actions, nullptr, argv.data(), environ) == 0);
posix_spawn_file_actions_destroy(&actions);
::close(to_child[0]);
::close(from_child[1]);
in_ = to_child[1];
out_ = from_child[0];
}
Child(const Child&) = delete;
Child& operator=(const Child&) = delete;
~Child() {
if (pid_ > 0) {
::kill(pid_, SIGKILL);
::waitpid(pid_, nullptr, 0);
}
::close(in_);
::close(out_);
}
void send(const std::string& text) {
std::size_t done = 0;
while (done < text.size()) {
const auto n = ::write(in_, text.data() + done, text.size() - done);
CHECK(n > 0 || errno == EINTR);
if (n > 0)
done += static_cast<std::size_t>(n);
}
}
// Reads lines until one satisfies `want`; every line is kept in lines().
template <typename Want> bool wait_for(Want want, std::chrono::milliseconds timeout) {
const auto deadline = Clock::now() + timeout;
for (std::size_t seen = 0;;) {
for (; seen < lines_.size(); ++seen)
if (want(lines_[seen]))
return true;
const auto left =
std::chrono::duration_cast<std::chrono::milliseconds>(deadline - Clock::now());
if (left.count() <= 0 || !read_some(static_cast<int>(left.count())))
return false;
}
}
void signal(int sig) {
CHECK(::kill(pid_, sig) == 0);
}
// Exit code, or 128+signal; drains remaining output.
int wait_exit(std::chrono::milliseconds timeout) {
const auto deadline = Clock::now() + timeout;
int status = 0;
for (;;) {
read_some(20);
const auto r = ::waitpid(pid_, &status, WNOHANG);
if (r == pid_)
break;
CHECK(Clock::now() < deadline);
}
while (read_some(20)) {
}
pid_ = -1;
return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status);
}
[[nodiscard]] const std::vector<std::string>& lines() const noexcept {
return lines_;
}
private:
bool read_some(int timeout_ms) {
pollfd fd{out_, POLLIN, 0};
if (::poll(&fd, 1, timeout_ms) <= 0)
return false;
char chunk[4096];
const auto n = ::read(out_, chunk, sizeof chunk);
if (n <= 0)
return false;
buffer_.append(chunk, static_cast<std::size_t>(n));
for (auto end = buffer_.find('\n'); end != std::string::npos; end = buffer_.find('\n')) {
lines_.push_back(buffer_.substr(0, end));
buffer_.erase(0, end + 1);
}
return true;
}
pid_t pid_{-1};
int in_{-1}, out_{-1};
std::string buffer_;
std::vector<std::string> lines_;
};
struct Region {
RegionKey key;
fs::path root;
};
Region provision(std::uint64_t region, std::uint8_t genesis) {
Region r{{7, region}, fs::path(env("PGHOST")) / ("server-" + std::to_string(region))};
fs::remove_all(r.root);
fs::create_directory(r.root);
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
auto db = owner();
CHECK(db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
{}, 5s));
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
CHECK(objects);
Sha256Digest build{};
build.fill(0xB1);
Uuid commit{};
commit.fill(genesis);
CHECK(provision_lineage_region(*db, **objects, r.key, evo::test::region_identity.frame,
*evo::test::lineage_region(), commit, build, 8000000, 5s));
return r;
}
std::vector<std::string> server_args(const Region& r) {
return {env("EVO_SERVER_BIN"),
"--profile",
"lineage",
"--world",
"7",
"--region",
std::to_string(r.key.region),
"--frame",
std::to_string(evo::test::region_identity.frame),
"--pg-host",
env("PGHOST"),
"--pg-port",
env("PGPORT"),
"--pg-database",
env("PGDATABASE"),
"--pg-user",
env("EVO_PG_RUNTIME_USER"),
"--objects",
r.root.string(),
"--build",
"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1",
"--flush-ms",
"500",
"--lease-profile",
"fast",
"--ready-file",
(r.root / "ready").string()};
}
// (tick, commit_sequence) of every ack line.
std::vector<std::pair<std::uint64_t, std::uint64_t>> acks(const Child& c) {
std::vector<std::pair<std::uint64_t, std::uint64_t>> out;
for (const auto& line : c.lines()) {
unsigned long long tick = 0, sequence = 0;
char request[40];
if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &tick, &sequence) == 3)
out.emplace_back(tick, sequence);
}
return out;
}
std::string head_of(RegionKey key) {
const std::array<std::string, 1> p{std::to_string(key.region)};
auto rows = owner()->query("SELECT h.end_tick::text || ':' || h.commit_sequence::text || ':' "
"|| coalesce(l.holder::text, 'released') FROM "
"evolution.region_heads h JOIN evolution.region_leases l USING "
"(world_id, region_id) WHERE world_id = 7 AND region_id = "
"$1::evolution.u64",
p, 5s);
CHECK(rows && rows->size() == 1);
const auto& value = (*rows)[0][0];
CHECK(value.has_value());
return value.value_or("");
}
std::uint64_t head_tick(RegionKey key) {
return std::stoull(head_of(key).substr(0, head_of(key).find(':')));
}
bool starts(const std::string& line, std::string_view prefix) {
return line.starts_with(prefix);
}
// SIGTERM: stop admission, commit pending ticks, release the lease, exit 0. Every
// acknowledged tick is committed, the backlog never exceeds its bound, and the
// permanent outcome was announced only after its commit.
void sigterm_drains_and_releases() {
const auto r = provision(31, 0x42);
Child server{server_args(r)};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
CHECK(fs::exists(r.root / "ready"));
std::string burst;
for (std::uint64_t tick = 1; tick <= 80; ++tick)
burst += input_line(tick, true);
server.send(burst); // More than the 64-input backlog: stdin backpressures.
CHECK(server.wait_for(
[](const std::string& l) { return starts(l, "ack " + hex32(80) + " 80 "); }, 30s));
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 0);
const auto& lines = server.lines();
CHECK(std::ranges::find(lines, "exit 0") != lines.end());
CHECK(std::ranges::find_if(lines, [](const std::string& l) {
return starts(l, "phase stopped");
}) != lines.end());
unsigned long long max_queued = 999;
for (const auto& l : lines)
std::sscanf(l.c_str(), "stats max_queued=%llu", &max_queued);
CHECK(max_queued <= 64);
const auto all = acks(server);
CHECK(all.size() == 80 && all.back().first == 80);
// The death at tick 20 is announced only with its commit: after the final
// acknowledgement of tick 20, which the host emits only after confirmation.
const auto outcome = std::ranges::find_if(
lines, [](const std::string& l) { return starts(l, "outcome 20 1 11 "); });
const auto ack20 = std::ranges::find_if(
lines, [](const std::string& l) { return starts(l, "ack " + hex32(20) + " 20 "); });
CHECK(outcome != lines.end() && ack20 != lines.end() && ack20 < outcome);
CHECK(head_of(r.key).starts_with("80:") && head_of(r.key).ends_with(":released"));
CHECK(!fs::exists(r.root / "ready"));
std::printf("server_sigterm commits=%s acks=%zu max_queued=%llu\n", head_of(r.key).c_str(),
all.size(), max_queued);
}
// SIGKILL: no drain. Nothing acknowledged is lost, nothing unacknowledged was
// claimed, and a successor becomes ready only after the database lease expires.
void sigkill_recovers_through_a_successor() {
const auto r = provision(32, 0x43);
std::uint64_t acked_max = 0;
{
Child server{server_args(r)};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
std::string burst;
for (std::uint64_t tick = 1; tick <= 60; ++tick)
burst += input_line(tick, false);
server.send(burst);
CHECK(server.wait_for([](const std::string& l) { return starts(l, "ack "); }, 15s));
std::this_thread::sleep_for(300ms); // Ticks run past the last commit.
server.signal(SIGKILL);
CHECK(server.wait_exit(10s) == 128 + SIGKILL);
for (const auto& [tick, sequence] : acks(server)) {
static_cast<void>(sequence);
acked_max = std::max(acked_max, tick);
}
}
const auto committed = head_tick(r.key);
CHECK(acked_max > 0 && committed >= acked_max); // No acknowledged tick was lost.
CHECK(!head_of(r.key).ends_with(":released")); // The killed holder never released.
const auto started = Clock::now();
Child successor{server_args(r)};
CHECK(successor.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
const auto to_ready =
std::chrono::duration_cast<std::chrono::milliseconds>(Clock::now() - started).count();
const auto running = std::ranges::find_if(
successor.lines(), [](const std::string& l) { return starts(l, "phase running"); });
CHECK(running != successor.lines().end());
CHECK(running->find("committed=" + std::to_string(committed) + " ") != std::string::npos);
CHECK(running->find("epoch=2 ") != std::string::npos);
successor.signal(SIGTERM);
CHECK(successor.wait_exit(20s) == 0);
CHECK(head_tick(r.key) == committed && head_of(r.key).ends_with(":released"));
std::printf("server_sigkill acked_max=%llu committed=%llu successor_ready_ms=%lld\n",
static_cast<unsigned long long>(acked_max),
static_cast<unsigned long long>(committed), static_cast<long long>(to_ready));
}
} // namespace
void run_region_server_process_tests() {
sigterm_drains_and_releases();
sigkill_recovers_through_a_successor();
}