ADR-0065: before any readiness, recovery now verifies the head's tail. The trace object must exist with its manifest digest, its header must bind profile, region, predecessor and range; the predecessor (schema v6 load_region_commit) must hash to the digest the head names; replaying the trace from the predecessor's verified payload must reproduce every report and the exact head bytes; and trace-derived decisions must equal the head's durable decisions. Missing, corrupt or non-reproducing tails quarantine (RecoveryError::tail); nothing older or empty is served. Test-only crash seams (HostConfig::crash_hook, PublishHooks) and evo-server --crash-at deliver real SIGKILL at before-upload, before-transaction, after-commit and after-ack. The kill matrix, a SIGSTOP/SIGCONT stale owner and damaged-head startups all behave as the durability contract requires; successors verify before readiness. Validated: PostgreSQL suites under GCC Release, clang-tidy and ASan/UBSan; tail-skip and replay-equality mutations each caught. No Docker/Gitea CI, production-lease kill test or deployment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
96 lines
4.6 KiB
C++
96 lines
4.6 KiB
C++
// Real process tests for evo-server (M7.6, ADR-0064): SIGTERM drain and SIGKILL
|
|
// with recovery by a successor process. Run by tools/ci/postgres-check.sh, which
|
|
// sets EVO_SERVER_BIN.
|
|
#include "fixtures/server_process.hpp"
|
|
#include <algorithm>
|
|
#include <source_location>
|
|
#include <thread>
|
|
|
|
void run_region_server_process_tests();
|
|
|
|
namespace {
|
|
using namespace evo::test::process;
|
|
void check(bool value, std::source_location at = std::source_location::current()) {
|
|
if (!value)
|
|
throw std::runtime_error("region server process test line " + std::to_string(at.line()));
|
|
}
|
|
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
|
|
|
// SIGTERM: stop admission, commit pending ticks, release the lease, exit 0. Every
|
|
// acknowledged tick is committed, the backlog never exceeds its bound, and the
|
|
// permanent outcome was announced only after its commit.
|
|
void sigterm_drains_and_releases() {
|
|
const auto r = provision(31, 0x42);
|
|
Child server{server_args(r)};
|
|
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
|
CHECK(fs::exists(r.root / "ready"));
|
|
server.send(input_lines(1, 80, true)); // More than the 64-input backlog: stdin backpressures.
|
|
CHECK(server.wait_line("ack " + request_hex(80) + " 80 ", 30s));
|
|
server.signal(SIGTERM);
|
|
CHECK(server.wait_exit(20s) == 0);
|
|
const auto& lines = server.lines();
|
|
CHECK(std::ranges::find(lines, "exit 0") != lines.end() && server.printed("phase stopped"));
|
|
unsigned long long max_queued = 999;
|
|
for (const auto& l : lines)
|
|
std::sscanf(l.c_str(), "stats max_queued=%llu", &max_queued);
|
|
CHECK(max_queued <= 64);
|
|
const auto all = acks(server);
|
|
CHECK(all.size() == 80 && all.back().first == 80);
|
|
// The death at tick 20 is announced only with its commit: after the final
|
|
// acknowledgement of tick 20, which the host emits only after confirmation.
|
|
const auto outcome = std::ranges::find_if(
|
|
lines, [](const std::string& l) { return starts(l, "outcome 20 1 11 "); });
|
|
const auto ack20 = std::ranges::find_if(
|
|
lines, [](const std::string& l) { return starts(l, "ack " + request_hex(20) + " 20 "); });
|
|
CHECK(outcome != lines.end() && ack20 != lines.end() && ack20 < outcome);
|
|
const auto head = head_of(r.key);
|
|
CHECK(head.tick == 80 && head.released);
|
|
CHECK(!fs::exists(r.root / "ready"));
|
|
std::printf("server_sigterm head=%llu sequence=%llu acks=%zu max_queued=%llu\n",
|
|
static_cast<unsigned long long>(head.tick),
|
|
static_cast<unsigned long long>(head.sequence), all.size(), max_queued);
|
|
}
|
|
|
|
// SIGKILL: no drain. Nothing acknowledged is lost, nothing unacknowledged was
|
|
// claimed, and a successor becomes ready only after the database lease expires.
|
|
void sigkill_recovers_through_a_successor() {
|
|
const auto r = provision(32, 0x43);
|
|
std::uint64_t acked_max = 0;
|
|
{
|
|
Child server{server_args(r)};
|
|
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
|
server.send(input_lines(1, 60, false));
|
|
CHECK(server.wait_line("ack ", 15s));
|
|
std::this_thread::sleep_for(300ms); // Ticks run past the last commit.
|
|
server.signal(SIGKILL);
|
|
CHECK(server.wait_exit(10s) == 128 + SIGKILL);
|
|
acked_max = max_acked(server);
|
|
}
|
|
const auto committed = head_of(r.key);
|
|
CHECK(acked_max > 0 && committed.tick >= acked_max); // No acknowledged tick was lost.
|
|
CHECK(!committed.released); // The killed holder never released.
|
|
const auto started = Clock::now();
|
|
Child successor{server_args(r)};
|
|
CHECK(successor.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
|
const auto to_ready =
|
|
std::chrono::duration_cast<std::chrono::milliseconds>(Clock::now() - started).count();
|
|
const auto running = std::ranges::find_if(
|
|
successor.lines(), [](const std::string& l) { return starts(l, "phase running"); });
|
|
CHECK(running != successor.lines().end());
|
|
CHECK(running->find("committed=" + std::to_string(committed.tick) + " ") != std::string::npos);
|
|
CHECK(running->find("epoch=2 ") != std::string::npos);
|
|
successor.signal(SIGTERM);
|
|
CHECK(successor.wait_exit(20s) == 0);
|
|
const auto after = head_of(r.key);
|
|
CHECK(after.tick == committed.tick && after.released);
|
|
std::printf("server_sigkill acked_max=%llu committed=%llu successor_ready_ms=%lld\n",
|
|
static_cast<unsigned long long>(acked_max),
|
|
static_cast<unsigned long long>(committed.tick), static_cast<long long>(to_ready));
|
|
}
|
|
} // namespace
|
|
|
|
void run_region_server_process_tests() {
|
|
sigterm_drains_and_releases();
|
|
sigkill_recovers_through_a_successor();
|
|
}
|