Complete M7.7 recovery verification before readiness
ADR-0065: before any readiness, recovery now verifies the head's tail. The trace object must exist with its manifest digest, its header must bind profile, region, predecessor and range; the predecessor (schema v6 load_region_commit) must hash to the digest the head names; replaying the trace from the predecessor's verified payload must reproduce every report and the exact head bytes; and trace-derived decisions must equal the head's durable decisions. Missing, corrupt or non-reproducing tails quarantine (RecoveryError::tail); nothing older or empty is served. Test-only crash seams (HostConfig::crash_hook, PublishHooks) and evo-server --crash-at deliver real SIGKILL at before-upload, before-transaction, after-commit and after-ack. The kill matrix, a SIGSTOP/SIGCONT stale owner and damaged-head startups all behave as the durability contract requires; successors verify before readiness. Validated: PostgreSQL suites under GCC Release, clang-tidy and ASan/UBSan; tail-skip and replay-equality mutations each caught. No Docker/Gitea CI, production-lease kill test or deployment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,249 +1,20 @@
|
||||
// Real process tests for evo-server (M7.6, ADR-0064): SIGTERM drain and SIGKILL
|
||||
// with recovery by a successor process. Run by tools/ci/postgres-check.sh, which
|
||||
// sets EVO_SERVER_BIN.
|
||||
#include "evo/persistence/checkpoint_store.hpp"
|
||||
#include "fixtures/environmental_region.hpp"
|
||||
#include "fixtures/server_process.hpp"
|
||||
#include <algorithm>
|
||||
#include <cerrno>
|
||||
#include <chrono>
|
||||
#include <csignal>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <filesystem>
|
||||
#include <poll.h>
|
||||
#include <source_location>
|
||||
#include <spawn.h>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <sys/wait.h>
|
||||
#include <thread>
|
||||
#include <unistd.h>
|
||||
#include <vector>
|
||||
|
||||
extern char** environ;
|
||||
void run_region_server_process_tests();
|
||||
|
||||
namespace {
|
||||
using namespace evo::persistence;
|
||||
using namespace std::chrono_literals;
|
||||
namespace fs = std::filesystem;
|
||||
using Clock = std::chrono::steady_clock;
|
||||
using namespace evo::test::process;
|
||||
void check(bool value, std::source_location at = std::source_location::current()) {
|
||||
if (!value)
|
||||
throw std::runtime_error("region server process test line " + std::to_string(at.line()));
|
||||
}
|
||||
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
||||
std::string env(const char* key) {
|
||||
const auto* value = std::getenv(key);
|
||||
if (!value)
|
||||
throw std::runtime_error("missing test environment");
|
||||
return value;
|
||||
}
|
||||
std::unique_ptr<Postgres> owner() {
|
||||
auto made = Postgres::connect(
|
||||
{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("PGUSER"), {}, {}}, 5s);
|
||||
CHECK(made);
|
||||
return std::move(*made);
|
||||
}
|
||||
std::string hex32(std::uint64_t tick) {
|
||||
char out[33];
|
||||
std::snprintf(out, sizeof out, "9b000000000000000000000000%06llx",
|
||||
static_cast<unsigned long long>(tick));
|
||||
return out;
|
||||
}
|
||||
std::string input_line(std::uint64_t tick, bool consumer_dies) {
|
||||
const auto in = evo::test::region_input(tick, consumer_dies);
|
||||
std::string line = "input " + hex32(tick) + " " + std::to_string(in.temperature_mK) + " " +
|
||||
std::to_string(in.source_enabled);
|
||||
for (const auto b : in.bite)
|
||||
line += " " + std::to_string(b);
|
||||
for (const auto t : in.terminal)
|
||||
line += " " + std::to_string(t);
|
||||
return line + "\n";
|
||||
}
|
||||
|
||||
class Child {
|
||||
public:
|
||||
explicit Child(const std::vector<std::string>& args) {
|
||||
int to_child[2], from_child[2];
|
||||
CHECK(::pipe(to_child) == 0 && ::pipe(from_child) == 0);
|
||||
posix_spawn_file_actions_t actions;
|
||||
posix_spawn_file_actions_init(&actions);
|
||||
posix_spawn_file_actions_adddup2(&actions, to_child[0], STDIN_FILENO);
|
||||
posix_spawn_file_actions_adddup2(&actions, from_child[1], STDOUT_FILENO);
|
||||
posix_spawn_file_actions_addclose(&actions, to_child[1]);
|
||||
posix_spawn_file_actions_addclose(&actions, from_child[0]);
|
||||
std::vector<char*> argv;
|
||||
argv.reserve(args.size() + 1);
|
||||
for (const auto& a : args)
|
||||
argv.push_back(const_cast<char*>(a.c_str()));
|
||||
argv.push_back(nullptr);
|
||||
CHECK(posix_spawn(&pid_, argv[0], &actions, nullptr, argv.data(), environ) == 0);
|
||||
posix_spawn_file_actions_destroy(&actions);
|
||||
::close(to_child[0]);
|
||||
::close(from_child[1]);
|
||||
in_ = to_child[1];
|
||||
out_ = from_child[0];
|
||||
}
|
||||
Child(const Child&) = delete;
|
||||
Child& operator=(const Child&) = delete;
|
||||
~Child() {
|
||||
if (pid_ > 0) {
|
||||
::kill(pid_, SIGKILL);
|
||||
::waitpid(pid_, nullptr, 0);
|
||||
}
|
||||
::close(in_);
|
||||
::close(out_);
|
||||
}
|
||||
void send(const std::string& text) {
|
||||
std::size_t done = 0;
|
||||
while (done < text.size()) {
|
||||
const auto n = ::write(in_, text.data() + done, text.size() - done);
|
||||
CHECK(n > 0 || errno == EINTR);
|
||||
if (n > 0)
|
||||
done += static_cast<std::size_t>(n);
|
||||
}
|
||||
}
|
||||
// Reads lines until one satisfies `want`; every line is kept in lines().
|
||||
template <typename Want> bool wait_for(Want want, std::chrono::milliseconds timeout) {
|
||||
const auto deadline = Clock::now() + timeout;
|
||||
for (std::size_t seen = 0;;) {
|
||||
for (; seen < lines_.size(); ++seen)
|
||||
if (want(lines_[seen]))
|
||||
return true;
|
||||
const auto left =
|
||||
std::chrono::duration_cast<std::chrono::milliseconds>(deadline - Clock::now());
|
||||
if (left.count() <= 0 || !read_some(static_cast<int>(left.count())))
|
||||
return false;
|
||||
}
|
||||
}
|
||||
void signal(int sig) {
|
||||
CHECK(::kill(pid_, sig) == 0);
|
||||
}
|
||||
// Exit code, or 128+signal; drains remaining output.
|
||||
int wait_exit(std::chrono::milliseconds timeout) {
|
||||
const auto deadline = Clock::now() + timeout;
|
||||
int status = 0;
|
||||
for (;;) {
|
||||
read_some(20);
|
||||
const auto r = ::waitpid(pid_, &status, WNOHANG);
|
||||
if (r == pid_)
|
||||
break;
|
||||
CHECK(Clock::now() < deadline);
|
||||
}
|
||||
while (read_some(20)) {
|
||||
}
|
||||
pid_ = -1;
|
||||
return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status);
|
||||
}
|
||||
[[nodiscard]] const std::vector<std::string>& lines() const noexcept {
|
||||
return lines_;
|
||||
}
|
||||
|
||||
private:
|
||||
bool read_some(int timeout_ms) {
|
||||
pollfd fd{out_, POLLIN, 0};
|
||||
if (::poll(&fd, 1, timeout_ms) <= 0)
|
||||
return false;
|
||||
char chunk[4096];
|
||||
const auto n = ::read(out_, chunk, sizeof chunk);
|
||||
if (n <= 0)
|
||||
return false;
|
||||
buffer_.append(chunk, static_cast<std::size_t>(n));
|
||||
for (auto end = buffer_.find('\n'); end != std::string::npos; end = buffer_.find('\n')) {
|
||||
lines_.push_back(buffer_.substr(0, end));
|
||||
buffer_.erase(0, end + 1);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
pid_t pid_{-1};
|
||||
int in_{-1}, out_{-1};
|
||||
std::string buffer_;
|
||||
std::vector<std::string> lines_;
|
||||
};
|
||||
|
||||
struct Region {
|
||||
RegionKey key;
|
||||
fs::path root;
|
||||
};
|
||||
Region provision(std::uint64_t region, std::uint8_t genesis) {
|
||||
Region r{{7, region}, fs::path(env("PGHOST")) / ("server-" + std::to_string(region))};
|
||||
fs::remove_all(r.root);
|
||||
fs::create_directory(r.root);
|
||||
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
|
||||
auto db = owner();
|
||||
CHECK(db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
||||
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
|
||||
{}, 5s));
|
||||
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
|
||||
CHECK(objects);
|
||||
Sha256Digest build{};
|
||||
build.fill(0xB1);
|
||||
Uuid commit{};
|
||||
commit.fill(genesis);
|
||||
CHECK(provision_lineage_region(*db, **objects, r.key, evo::test::region_identity.frame,
|
||||
*evo::test::lineage_region(), commit, build, 8000000, 5s));
|
||||
return r;
|
||||
}
|
||||
std::vector<std::string> server_args(const Region& r) {
|
||||
return {env("EVO_SERVER_BIN"),
|
||||
"--profile",
|
||||
"lineage",
|
||||
"--world",
|
||||
"7",
|
||||
"--region",
|
||||
std::to_string(r.key.region),
|
||||
"--frame",
|
||||
std::to_string(evo::test::region_identity.frame),
|
||||
"--pg-host",
|
||||
env("PGHOST"),
|
||||
"--pg-port",
|
||||
env("PGPORT"),
|
||||
"--pg-database",
|
||||
env("PGDATABASE"),
|
||||
"--pg-user",
|
||||
env("EVO_PG_RUNTIME_USER"),
|
||||
"--objects",
|
||||
r.root.string(),
|
||||
"--build",
|
||||
"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1",
|
||||
"--flush-ms",
|
||||
"500",
|
||||
"--lease-profile",
|
||||
"fast",
|
||||
"--ready-file",
|
||||
(r.root / "ready").string()};
|
||||
}
|
||||
// (tick, commit_sequence) of every ack line.
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> acks(const Child& c) {
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> out;
|
||||
for (const auto& line : c.lines()) {
|
||||
unsigned long long tick = 0, sequence = 0;
|
||||
char request[40];
|
||||
if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &tick, &sequence) == 3)
|
||||
out.emplace_back(tick, sequence);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
std::string head_of(RegionKey key) {
|
||||
const std::array<std::string, 1> p{std::to_string(key.region)};
|
||||
auto rows = owner()->query("SELECT h.end_tick::text || ':' || h.commit_sequence::text || ':' "
|
||||
"|| coalesce(l.holder::text, 'released') FROM "
|
||||
"evolution.region_heads h JOIN evolution.region_leases l USING "
|
||||
"(world_id, region_id) WHERE world_id = 7 AND region_id = "
|
||||
"$1::evolution.u64",
|
||||
p, 5s);
|
||||
CHECK(rows && rows->size() == 1);
|
||||
const auto& value = (*rows)[0][0];
|
||||
CHECK(value.has_value());
|
||||
return value.value_or("");
|
||||
}
|
||||
std::uint64_t head_tick(RegionKey key) {
|
||||
return std::stoull(head_of(key).substr(0, head_of(key).find(':')));
|
||||
}
|
||||
bool starts(const std::string& line, std::string_view prefix) {
|
||||
return line.starts_with(prefix);
|
||||
}
|
||||
|
||||
// SIGTERM: stop admission, commit pending ticks, release the lease, exit 0. Every
|
||||
// acknowledged tick is committed, the backlog never exceeds its bound, and the
|
||||
@@ -253,19 +24,12 @@ void sigterm_drains_and_releases() {
|
||||
Child server{server_args(r)};
|
||||
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
||||
CHECK(fs::exists(r.root / "ready"));
|
||||
std::string burst;
|
||||
for (std::uint64_t tick = 1; tick <= 80; ++tick)
|
||||
burst += input_line(tick, true);
|
||||
server.send(burst); // More than the 64-input backlog: stdin backpressures.
|
||||
CHECK(server.wait_for(
|
||||
[](const std::string& l) { return starts(l, "ack " + hex32(80) + " 80 "); }, 30s));
|
||||
server.send(input_lines(1, 80, true)); // More than the 64-input backlog: stdin backpressures.
|
||||
CHECK(server.wait_line("ack " + request_hex(80) + " 80 ", 30s));
|
||||
server.signal(SIGTERM);
|
||||
CHECK(server.wait_exit(20s) == 0);
|
||||
const auto& lines = server.lines();
|
||||
CHECK(std::ranges::find(lines, "exit 0") != lines.end());
|
||||
CHECK(std::ranges::find_if(lines, [](const std::string& l) {
|
||||
return starts(l, "phase stopped");
|
||||
}) != lines.end());
|
||||
CHECK(std::ranges::find(lines, "exit 0") != lines.end() && server.printed("phase stopped"));
|
||||
unsigned long long max_queued = 999;
|
||||
for (const auto& l : lines)
|
||||
std::sscanf(l.c_str(), "stats max_queued=%llu", &max_queued);
|
||||
@@ -277,12 +41,14 @@ void sigterm_drains_and_releases() {
|
||||
const auto outcome = std::ranges::find_if(
|
||||
lines, [](const std::string& l) { return starts(l, "outcome 20 1 11 "); });
|
||||
const auto ack20 = std::ranges::find_if(
|
||||
lines, [](const std::string& l) { return starts(l, "ack " + hex32(20) + " 20 "); });
|
||||
lines, [](const std::string& l) { return starts(l, "ack " + request_hex(20) + " 20 "); });
|
||||
CHECK(outcome != lines.end() && ack20 != lines.end() && ack20 < outcome);
|
||||
CHECK(head_of(r.key).starts_with("80:") && head_of(r.key).ends_with(":released"));
|
||||
const auto head = head_of(r.key);
|
||||
CHECK(head.tick == 80 && head.released);
|
||||
CHECK(!fs::exists(r.root / "ready"));
|
||||
std::printf("server_sigterm commits=%s acks=%zu max_queued=%llu\n", head_of(r.key).c_str(),
|
||||
all.size(), max_queued);
|
||||
std::printf("server_sigterm head=%llu sequence=%llu acks=%zu max_queued=%llu\n",
|
||||
static_cast<unsigned long long>(head.tick),
|
||||
static_cast<unsigned long long>(head.sequence), all.size(), max_queued);
|
||||
}
|
||||
|
||||
// SIGKILL: no drain. Nothing acknowledged is lost, nothing unacknowledged was
|
||||
@@ -293,22 +59,16 @@ void sigkill_recovers_through_a_successor() {
|
||||
{
|
||||
Child server{server_args(r)};
|
||||
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
||||
std::string burst;
|
||||
for (std::uint64_t tick = 1; tick <= 60; ++tick)
|
||||
burst += input_line(tick, false);
|
||||
server.send(burst);
|
||||
CHECK(server.wait_for([](const std::string& l) { return starts(l, "ack "); }, 15s));
|
||||
server.send(input_lines(1, 60, false));
|
||||
CHECK(server.wait_line("ack ", 15s));
|
||||
std::this_thread::sleep_for(300ms); // Ticks run past the last commit.
|
||||
server.signal(SIGKILL);
|
||||
CHECK(server.wait_exit(10s) == 128 + SIGKILL);
|
||||
for (const auto& [tick, sequence] : acks(server)) {
|
||||
static_cast<void>(sequence);
|
||||
acked_max = std::max(acked_max, tick);
|
||||
}
|
||||
acked_max = max_acked(server);
|
||||
}
|
||||
const auto committed = head_tick(r.key);
|
||||
CHECK(acked_max > 0 && committed >= acked_max); // No acknowledged tick was lost.
|
||||
CHECK(!head_of(r.key).ends_with(":released")); // The killed holder never released.
|
||||
const auto committed = head_of(r.key);
|
||||
CHECK(acked_max > 0 && committed.tick >= acked_max); // No acknowledged tick was lost.
|
||||
CHECK(!committed.released); // The killed holder never released.
|
||||
const auto started = Clock::now();
|
||||
Child successor{server_args(r)};
|
||||
CHECK(successor.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
||||
@@ -317,14 +77,15 @@ void sigkill_recovers_through_a_successor() {
|
||||
const auto running = std::ranges::find_if(
|
||||
successor.lines(), [](const std::string& l) { return starts(l, "phase running"); });
|
||||
CHECK(running != successor.lines().end());
|
||||
CHECK(running->find("committed=" + std::to_string(committed) + " ") != std::string::npos);
|
||||
CHECK(running->find("committed=" + std::to_string(committed.tick) + " ") != std::string::npos);
|
||||
CHECK(running->find("epoch=2 ") != std::string::npos);
|
||||
successor.signal(SIGTERM);
|
||||
CHECK(successor.wait_exit(20s) == 0);
|
||||
CHECK(head_tick(r.key) == committed && head_of(r.key).ends_with(":released"));
|
||||
const auto after = head_of(r.key);
|
||||
CHECK(after.tick == committed.tick && after.released);
|
||||
std::printf("server_sigkill acked_max=%llu committed=%llu successor_ready_ms=%lld\n",
|
||||
static_cast<unsigned long long>(acked_max),
|
||||
static_cast<unsigned long long>(committed), static_cast<long long>(to_ready));
|
||||
static_cast<unsigned long long>(committed.tick), static_cast<long long>(to_ready));
|
||||
}
|
||||
} // namespace
|
||||
|
||||
|
||||
Reference in New Issue
Block a user