Complete M7.7 recovery verification before readiness

ADR-0065: before any readiness, recovery now verifies the head's tail.
The trace object must exist with its manifest digest, its header must
bind profile, region, predecessor and range; the predecessor (schema v6
load_region_commit) must hash to the digest the head names; replaying
the trace from the predecessor's verified payload must reproduce every
report and the exact head bytes; and trace-derived decisions must equal
the head's durable decisions. Missing, corrupt or non-reproducing tails
quarantine (RecoveryError::tail); nothing older or empty is served.

Test-only crash seams (HostConfig::crash_hook, PublishHooks) and
evo-server --crash-at deliver real SIGKILL at before-upload,
before-transaction, after-commit and after-ack. The kill matrix,
a SIGSTOP/SIGCONT stale owner and damaged-head startups all behave as
the durability contract requires; successors verify before readiness.

Validated: PostgreSQL suites under GCC Release, clang-tidy and
ASan/UBSan; tail-skip and replay-equality mutations each caught. No
Docker/Gitea CI, production-lease kill test or deployment.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-28 22:31:40 -04:00
co-authored by Claude Opus 5.5
parent d6d0408aae
commit 30411a83f9
24 changed files with 990 additions and 284 deletions
+21 -260
View File
@@ -1,249 +1,20 @@
// Real process tests for evo-server (M7.6, ADR-0064): SIGTERM drain and SIGKILL
// with recovery by a successor process. Run by tools/ci/postgres-check.sh, which
// sets EVO_SERVER_BIN.
#include "evo/persistence/checkpoint_store.hpp"
#include "fixtures/environmental_region.hpp"
#include "fixtures/server_process.hpp"
#include <algorithm>
#include <cerrno>
#include <chrono>
#include <csignal>
#include <cstdio>
#include <cstdlib>
#include <filesystem>
#include <poll.h>
#include <source_location>
#include <spawn.h>
#include <stdexcept>
#include <string>
#include <sys/wait.h>
#include <thread>
#include <unistd.h>
#include <vector>
extern char** environ;
void run_region_server_process_tests();
namespace {
using namespace evo::persistence;
using namespace std::chrono_literals;
namespace fs = std::filesystem;
using Clock = std::chrono::steady_clock;
using namespace evo::test::process;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region server process test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
std::string env(const char* key) {
const auto* value = std::getenv(key);
if (!value)
throw std::runtime_error("missing test environment");
return value;
}
std::unique_ptr<Postgres> owner() {
auto made = Postgres::connect(
{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("PGUSER"), {}, {}}, 5s);
CHECK(made);
return std::move(*made);
}
std::string hex32(std::uint64_t tick) {
char out[33];
std::snprintf(out, sizeof out, "9b000000000000000000000000%06llx",
static_cast<unsigned long long>(tick));
return out;
}
std::string input_line(std::uint64_t tick, bool consumer_dies) {
const auto in = evo::test::region_input(tick, consumer_dies);
std::string line = "input " + hex32(tick) + " " + std::to_string(in.temperature_mK) + " " +
std::to_string(in.source_enabled);
for (const auto b : in.bite)
line += " " + std::to_string(b);
for (const auto t : in.terminal)
line += " " + std::to_string(t);
return line + "\n";
}
class Child {
public:
explicit Child(const std::vector<std::string>& args) {
int to_child[2], from_child[2];
CHECK(::pipe(to_child) == 0 && ::pipe(from_child) == 0);
posix_spawn_file_actions_t actions;
posix_spawn_file_actions_init(&actions);
posix_spawn_file_actions_adddup2(&actions, to_child[0], STDIN_FILENO);
posix_spawn_file_actions_adddup2(&actions, from_child[1], STDOUT_FILENO);
posix_spawn_file_actions_addclose(&actions, to_child[1]);
posix_spawn_file_actions_addclose(&actions, from_child[0]);
std::vector<char*> argv;
argv.reserve(args.size() + 1);
for (const auto& a : args)
argv.push_back(const_cast<char*>(a.c_str()));
argv.push_back(nullptr);
CHECK(posix_spawn(&pid_, argv[0], &actions, nullptr, argv.data(), environ) == 0);
posix_spawn_file_actions_destroy(&actions);
::close(to_child[0]);
::close(from_child[1]);
in_ = to_child[1];
out_ = from_child[0];
}
Child(const Child&) = delete;
Child& operator=(const Child&) = delete;
~Child() {
if (pid_ > 0) {
::kill(pid_, SIGKILL);
::waitpid(pid_, nullptr, 0);
}
::close(in_);
::close(out_);
}
void send(const std::string& text) {
std::size_t done = 0;
while (done < text.size()) {
const auto n = ::write(in_, text.data() + done, text.size() - done);
CHECK(n > 0 || errno == EINTR);
if (n > 0)
done += static_cast<std::size_t>(n);
}
}
// Reads lines until one satisfies `want`; every line is kept in lines().
template <typename Want> bool wait_for(Want want, std::chrono::milliseconds timeout) {
const auto deadline = Clock::now() + timeout;
for (std::size_t seen = 0;;) {
for (; seen < lines_.size(); ++seen)
if (want(lines_[seen]))
return true;
const auto left =
std::chrono::duration_cast<std::chrono::milliseconds>(deadline - Clock::now());
if (left.count() <= 0 || !read_some(static_cast<int>(left.count())))
return false;
}
}
void signal(int sig) {
CHECK(::kill(pid_, sig) == 0);
}
// Exit code, or 128+signal; drains remaining output.
int wait_exit(std::chrono::milliseconds timeout) {
const auto deadline = Clock::now() + timeout;
int status = 0;
for (;;) {
read_some(20);
const auto r = ::waitpid(pid_, &status, WNOHANG);
if (r == pid_)
break;
CHECK(Clock::now() < deadline);
}
while (read_some(20)) {
}
pid_ = -1;
return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status);
}
[[nodiscard]] const std::vector<std::string>& lines() const noexcept {
return lines_;
}
private:
bool read_some(int timeout_ms) {
pollfd fd{out_, POLLIN, 0};
if (::poll(&fd, 1, timeout_ms) <= 0)
return false;
char chunk[4096];
const auto n = ::read(out_, chunk, sizeof chunk);
if (n <= 0)
return false;
buffer_.append(chunk, static_cast<std::size_t>(n));
for (auto end = buffer_.find('\n'); end != std::string::npos; end = buffer_.find('\n')) {
lines_.push_back(buffer_.substr(0, end));
buffer_.erase(0, end + 1);
}
return true;
}
pid_t pid_{-1};
int in_{-1}, out_{-1};
std::string buffer_;
std::vector<std::string> lines_;
};
struct Region {
RegionKey key;
fs::path root;
};
Region provision(std::uint64_t region, std::uint8_t genesis) {
Region r{{7, region}, fs::path(env("PGHOST")) / ("server-" + std::to_string(region))};
fs::remove_all(r.root);
fs::create_directory(r.root);
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
auto db = owner();
CHECK(db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
{}, 5s));
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
CHECK(objects);
Sha256Digest build{};
build.fill(0xB1);
Uuid commit{};
commit.fill(genesis);
CHECK(provision_lineage_region(*db, **objects, r.key, evo::test::region_identity.frame,
*evo::test::lineage_region(), commit, build, 8000000, 5s));
return r;
}
std::vector<std::string> server_args(const Region& r) {
return {env("EVO_SERVER_BIN"),
"--profile",
"lineage",
"--world",
"7",
"--region",
std::to_string(r.key.region),
"--frame",
std::to_string(evo::test::region_identity.frame),
"--pg-host",
env("PGHOST"),
"--pg-port",
env("PGPORT"),
"--pg-database",
env("PGDATABASE"),
"--pg-user",
env("EVO_PG_RUNTIME_USER"),
"--objects",
r.root.string(),
"--build",
"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1",
"--flush-ms",
"500",
"--lease-profile",
"fast",
"--ready-file",
(r.root / "ready").string()};
}
// (tick, commit_sequence) of every ack line.
std::vector<std::pair<std::uint64_t, std::uint64_t>> acks(const Child& c) {
std::vector<std::pair<std::uint64_t, std::uint64_t>> out;
for (const auto& line : c.lines()) {
unsigned long long tick = 0, sequence = 0;
char request[40];
if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &tick, &sequence) == 3)
out.emplace_back(tick, sequence);
}
return out;
}
std::string head_of(RegionKey key) {
const std::array<std::string, 1> p{std::to_string(key.region)};
auto rows = owner()->query("SELECT h.end_tick::text || ':' || h.commit_sequence::text || ':' "
"|| coalesce(l.holder::text, 'released') FROM "
"evolution.region_heads h JOIN evolution.region_leases l USING "
"(world_id, region_id) WHERE world_id = 7 AND region_id = "
"$1::evolution.u64",
p, 5s);
CHECK(rows && rows->size() == 1);
const auto& value = (*rows)[0][0];
CHECK(value.has_value());
return value.value_or("");
}
std::uint64_t head_tick(RegionKey key) {
return std::stoull(head_of(key).substr(0, head_of(key).find(':')));
}
bool starts(const std::string& line, std::string_view prefix) {
return line.starts_with(prefix);
}
// SIGTERM: stop admission, commit pending ticks, release the lease, exit 0. Every
// acknowledged tick is committed, the backlog never exceeds its bound, and the
@@ -253,19 +24,12 @@ void sigterm_drains_and_releases() {
Child server{server_args(r)};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
CHECK(fs::exists(r.root / "ready"));
std::string burst;
for (std::uint64_t tick = 1; tick <= 80; ++tick)
burst += input_line(tick, true);
server.send(burst); // More than the 64-input backlog: stdin backpressures.
CHECK(server.wait_for(
[](const std::string& l) { return starts(l, "ack " + hex32(80) + " 80 "); }, 30s));
server.send(input_lines(1, 80, true)); // More than the 64-input backlog: stdin backpressures.
CHECK(server.wait_line("ack " + request_hex(80) + " 80 ", 30s));
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 0);
const auto& lines = server.lines();
CHECK(std::ranges::find(lines, "exit 0") != lines.end());
CHECK(std::ranges::find_if(lines, [](const std::string& l) {
return starts(l, "phase stopped");
}) != lines.end());
CHECK(std::ranges::find(lines, "exit 0") != lines.end() && server.printed("phase stopped"));
unsigned long long max_queued = 999;
for (const auto& l : lines)
std::sscanf(l.c_str(), "stats max_queued=%llu", &max_queued);
@@ -277,12 +41,14 @@ void sigterm_drains_and_releases() {
const auto outcome = std::ranges::find_if(
lines, [](const std::string& l) { return starts(l, "outcome 20 1 11 "); });
const auto ack20 = std::ranges::find_if(
lines, [](const std::string& l) { return starts(l, "ack " + hex32(20) + " 20 "); });
lines, [](const std::string& l) { return starts(l, "ack " + request_hex(20) + " 20 "); });
CHECK(outcome != lines.end() && ack20 != lines.end() && ack20 < outcome);
CHECK(head_of(r.key).starts_with("80:") && head_of(r.key).ends_with(":released"));
const auto head = head_of(r.key);
CHECK(head.tick == 80 && head.released);
CHECK(!fs::exists(r.root / "ready"));
std::printf("server_sigterm commits=%s acks=%zu max_queued=%llu\n", head_of(r.key).c_str(),
all.size(), max_queued);
std::printf("server_sigterm head=%llu sequence=%llu acks=%zu max_queued=%llu\n",
static_cast<unsigned long long>(head.tick),
static_cast<unsigned long long>(head.sequence), all.size(), max_queued);
}
// SIGKILL: no drain. Nothing acknowledged is lost, nothing unacknowledged was
@@ -293,22 +59,16 @@ void sigkill_recovers_through_a_successor() {
{
Child server{server_args(r)};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
std::string burst;
for (std::uint64_t tick = 1; tick <= 60; ++tick)
burst += input_line(tick, false);
server.send(burst);
CHECK(server.wait_for([](const std::string& l) { return starts(l, "ack "); }, 15s));
server.send(input_lines(1, 60, false));
CHECK(server.wait_line("ack ", 15s));
std::this_thread::sleep_for(300ms); // Ticks run past the last commit.
server.signal(SIGKILL);
CHECK(server.wait_exit(10s) == 128 + SIGKILL);
for (const auto& [tick, sequence] : acks(server)) {
static_cast<void>(sequence);
acked_max = std::max(acked_max, tick);
}
acked_max = max_acked(server);
}
const auto committed = head_tick(r.key);
CHECK(acked_max > 0 && committed >= acked_max); // No acknowledged tick was lost.
CHECK(!head_of(r.key).ends_with(":released")); // The killed holder never released.
const auto committed = head_of(r.key);
CHECK(acked_max > 0 && committed.tick >= acked_max); // No acknowledged tick was lost.
CHECK(!committed.released); // The killed holder never released.
const auto started = Clock::now();
Child successor{server_args(r)};
CHECK(successor.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
@@ -317,14 +77,15 @@ void sigkill_recovers_through_a_successor() {
const auto running = std::ranges::find_if(
successor.lines(), [](const std::string& l) { return starts(l, "phase running"); });
CHECK(running != successor.lines().end());
CHECK(running->find("committed=" + std::to_string(committed) + " ") != std::string::npos);
CHECK(running->find("committed=" + std::to_string(committed.tick) + " ") != std::string::npos);
CHECK(running->find("epoch=2 ") != std::string::npos);
successor.signal(SIGTERM);
CHECK(successor.wait_exit(20s) == 0);
CHECK(head_tick(r.key) == committed && head_of(r.key).ends_with(":released"));
const auto after = head_of(r.key);
CHECK(after.tick == committed.tick && after.released);
std::printf("server_sigkill acked_max=%llu committed=%llu successor_ready_ms=%lld\n",
static_cast<unsigned long long>(acked_max),
static_cast<unsigned long long>(committed), static_cast<long long>(to_ready));
static_cast<unsigned long long>(committed.tick), static_cast<long long>(to_ready));
}
} // namespace