// Real process tests for evo-server (M7.6, ADR-0064): SIGTERM drain and SIGKILL // with recovery by a successor process. Run by tools/ci/postgres-check.sh, which // sets EVO_SERVER_BIN. #include "fixtures/server_process.hpp" #include #include #include void run_region_server_process_tests(); namespace { using namespace evo::test::process; void check(bool value, std::source_location at = std::source_location::current()) { if (!value) throw std::runtime_error("region server process test line " + std::to_string(at.line())); } #define CHECK(...) check(static_cast(__VA_ARGS__)) // SIGTERM: stop admission, commit pending ticks, release the lease, exit 0. Every // acknowledged tick is committed, the backlog never exceeds its bound, and the // permanent outcome was announced only after its commit. void sigterm_drains_and_releases() { const auto r = provision(31, 0x42); Child server{server_args(r)}; CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); CHECK(fs::exists(r.root / "ready")); server.send(input_lines(1, 80, true)); // More than the 64-input backlog: stdin backpressures. CHECK(server.wait_line("ack " + request_hex(80) + " 80 ", 30s)); server.signal(SIGTERM); CHECK(server.wait_exit(20s) == 0); const auto& lines = server.lines(); CHECK(std::ranges::find(lines, "exit 0") != lines.end() && server.printed("phase stopped")); unsigned long long max_queued = 999; for (const auto& l : lines) std::sscanf(l.c_str(), "stats max_queued=%llu", &max_queued); CHECK(max_queued <= 64); const auto all = acks(server); CHECK(all.size() == 80 && all.back().first == 80); // The death at tick 20 is announced only with its commit: after the final // acknowledgement of tick 20, which the host emits only after confirmation. const auto outcome = std::ranges::find_if( lines, [](const std::string& l) { return starts(l, "outcome 20 1 11 "); }); const auto ack20 = std::ranges::find_if( lines, [](const std::string& l) { return starts(l, "ack " + request_hex(20) + " 20 "); }); CHECK(outcome != lines.end() && ack20 != lines.end() && ack20 < outcome); const auto head = head_of(r.key); CHECK(head.tick == 80 && head.released); CHECK(!fs::exists(r.root / "ready")); std::printf("server_sigterm head=%llu sequence=%llu acks=%zu max_queued=%llu\n", static_cast(head.tick), static_cast(head.sequence), all.size(), max_queued); } // SIGKILL: no drain. Nothing acknowledged is lost, nothing unacknowledged was // claimed, and a successor becomes ready only after the database lease expires. void sigkill_recovers_through_a_successor() { const auto r = provision(32, 0x43); std::uint64_t acked_max = 0; { Child server{server_args(r)}; CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); server.send(input_lines(1, 60, false)); CHECK(server.wait_line("ack ", 15s)); std::this_thread::sleep_for(300ms); // Ticks run past the last commit. server.signal(SIGKILL); CHECK(server.wait_exit(10s) == 128 + SIGKILL); acked_max = max_acked(server); } const auto committed = head_of(r.key); CHECK(acked_max > 0 && committed.tick >= acked_max); // No acknowledged tick was lost. CHECK(!committed.released); // The killed holder never released. const auto started = Clock::now(); Child successor{server_args(r)}; CHECK(successor.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); const auto to_ready = std::chrono::duration_cast(Clock::now() - started).count(); const auto running = std::ranges::find_if( successor.lines(), [](const std::string& l) { return starts(l, "phase running"); }); CHECK(running != successor.lines().end()); CHECK(running->find("committed=" + std::to_string(committed.tick) + " ") != std::string::npos); CHECK(running->find("epoch=2 ") != std::string::npos); successor.signal(SIGTERM); CHECK(successor.wait_exit(20s) == 0); const auto after = head_of(r.key); CHECK(after.tick == committed.tick && after.released); std::printf("server_sigkill acked_max=%llu committed=%llu successor_ready_ms=%lld\n", static_cast(acked_max), static_cast(committed.tick), static_cast(to_ready)); } } // namespace void run_region_server_process_tests() { sigterm_drains_and_releases(); sigkill_recovers_through_a_successor(); }