Files
aevolution/tests/region_server_process_tests.cpp
T
dingoandClaude Opus 5.5 30411a83f9 Complete M7.7 recovery verification before readiness
ADR-0065: before any readiness, recovery now verifies the head's tail.
The trace object must exist with its manifest digest, its header must
bind profile, region, predecessor and range; the predecessor (schema v6
load_region_commit) must hash to the digest the head names; replaying
the trace from the predecessor's verified payload must reproduce every
report and the exact head bytes; and trace-derived decisions must equal
the head's durable decisions. Missing, corrupt or non-reproducing tails
quarantine (RecoveryError::tail); nothing older or empty is served.

Test-only crash seams (HostConfig::crash_hook, PublishHooks) and
evo-server --crash-at deliver real SIGKILL at before-upload,
before-transaction, after-commit and after-ack. The kill matrix,
a SIGSTOP/SIGCONT stale owner and damaged-head startups all behave as
the durability contract requires; successors verify before readiness.

Validated: PostgreSQL suites under GCC Release, clang-tidy and
ASan/UBSan; tail-skip and replay-equality mutations each caught. No
Docker/Gitea CI, production-lease kill test or deployment.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-28 22:31:40 -04:00

96 lines
4.6 KiB
C++

// Real process tests for evo-server (M7.6, ADR-0064): SIGTERM drain and SIGKILL
// with recovery by a successor process. Run by tools/ci/postgres-check.sh, which
// sets EVO_SERVER_BIN.
#include "fixtures/server_process.hpp"
#include <algorithm>
#include <source_location>
#include <thread>
void run_region_server_process_tests();
namespace {
using namespace evo::test::process;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region server process test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
// SIGTERM: stop admission, commit pending ticks, release the lease, exit 0. Every
// acknowledged tick is committed, the backlog never exceeds its bound, and the
// permanent outcome was announced only after its commit.
void sigterm_drains_and_releases() {
const auto r = provision(31, 0x42);
Child server{server_args(r)};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
CHECK(fs::exists(r.root / "ready"));
server.send(input_lines(1, 80, true)); // More than the 64-input backlog: stdin backpressures.
CHECK(server.wait_line("ack " + request_hex(80) + " 80 ", 30s));
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 0);
const auto& lines = server.lines();
CHECK(std::ranges::find(lines, "exit 0") != lines.end() && server.printed("phase stopped"));
unsigned long long max_queued = 999;
for (const auto& l : lines)
std::sscanf(l.c_str(), "stats max_queued=%llu", &max_queued);
CHECK(max_queued <= 64);
const auto all = acks(server);
CHECK(all.size() == 80 && all.back().first == 80);
// The death at tick 20 is announced only with its commit: after the final
// acknowledgement of tick 20, which the host emits only after confirmation.
const auto outcome = std::ranges::find_if(
lines, [](const std::string& l) { return starts(l, "outcome 20 1 11 "); });
const auto ack20 = std::ranges::find_if(
lines, [](const std::string& l) { return starts(l, "ack " + request_hex(20) + " 20 "); });
CHECK(outcome != lines.end() && ack20 != lines.end() && ack20 < outcome);
const auto head = head_of(r.key);
CHECK(head.tick == 80 && head.released);
CHECK(!fs::exists(r.root / "ready"));
std::printf("server_sigterm head=%llu sequence=%llu acks=%zu max_queued=%llu\n",
static_cast<unsigned long long>(head.tick),
static_cast<unsigned long long>(head.sequence), all.size(), max_queued);
}
// SIGKILL: no drain. Nothing acknowledged is lost, nothing unacknowledged was
// claimed, and a successor becomes ready only after the database lease expires.
void sigkill_recovers_through_a_successor() {
const auto r = provision(32, 0x43);
std::uint64_t acked_max = 0;
{
Child server{server_args(r)};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
server.send(input_lines(1, 60, false));
CHECK(server.wait_line("ack ", 15s));
std::this_thread::sleep_for(300ms); // Ticks run past the last commit.
server.signal(SIGKILL);
CHECK(server.wait_exit(10s) == 128 + SIGKILL);
acked_max = max_acked(server);
}
const auto committed = head_of(r.key);
CHECK(acked_max > 0 && committed.tick >= acked_max); // No acknowledged tick was lost.
CHECK(!committed.released); // The killed holder never released.
const auto started = Clock::now();
Child successor{server_args(r)};
CHECK(successor.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
const auto to_ready =
std::chrono::duration_cast<std::chrono::milliseconds>(Clock::now() - started).count();
const auto running = std::ranges::find_if(
successor.lines(), [](const std::string& l) { return starts(l, "phase running"); });
CHECK(running != successor.lines().end());
CHECK(running->find("committed=" + std::to_string(committed.tick) + " ") != std::string::npos);
CHECK(running->find("epoch=2 ") != std::string::npos);
successor.signal(SIGTERM);
CHECK(successor.wait_exit(20s) == 0);
const auto after = head_of(r.key);
CHECK(after.tick == committed.tick && after.released);
std::printf("server_sigkill acked_max=%llu committed=%llu successor_ready_ms=%lld\n",
static_cast<unsigned long long>(acked_max),
static_cast<unsigned long long>(committed.tick), static_cast<long long>(to_ready));
}
} // namespace
void run_region_server_process_tests() {
sigterm_drains_and_releases();
sigkill_recovers_through_a_successor();
}