ADR-0065: before any readiness, recovery now verifies the head's tail. The trace object must exist with its manifest digest, its header must bind profile, region, predecessor and range; the predecessor (schema v6 load_region_commit) must hash to the digest the head names; replaying the trace from the predecessor's verified payload must reproduce every report and the exact head bytes; and trace-derived decisions must equal the head's durable decisions. Missing, corrupt or non-reproducing tails quarantine (RecoveryError::tail); nothing older or empty is served. Test-only crash seams (HostConfig::crash_hook, PublishHooks) and evo-server --crash-at deliver real SIGKILL at before-upload, before-transaction, after-commit and after-ack. The kill matrix, a SIGSTOP/SIGCONT stale owner and damaged-head startups all behave as the durability contract requires; successors verify before readiness. Validated: PostgreSQL suites under GCC Release, clang-tidy and ASan/UBSan; tail-skip and replay-equality mutations each caught. No Docker/Gitea CI, production-lease kill test or deployment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
298 lines
11 KiB
C++
298 lines
11 KiB
C++
#pragma once
|
|
// Shared real-process helpers for evo-server tests (M7.6/M7.7). Linux only.
|
|
#include "environmental_region.hpp"
|
|
#include "evo/persistence/checkpoint_store.hpp"
|
|
#include <array>
|
|
#include <cerrno>
|
|
#include <chrono>
|
|
#include <csignal>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <filesystem>
|
|
#include <memory>
|
|
#include <poll.h>
|
|
#include <spawn.h>
|
|
#include <stdexcept>
|
|
#include <string>
|
|
#include <string_view>
|
|
#include <sys/wait.h>
|
|
#include <unistd.h>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
extern char** environ;
|
|
|
|
namespace evo::test::process {
|
|
using namespace std::chrono_literals;
|
|
namespace fs = std::filesystem;
|
|
using Clock = std::chrono::steady_clock;
|
|
|
|
inline void expect(bool ok, const char* what) {
|
|
if (!ok)
|
|
throw std::runtime_error(std::string("server fixture: ") + what);
|
|
}
|
|
inline std::string env(const char* key) {
|
|
const auto* value = std::getenv(key);
|
|
expect(value != nullptr, "missing test environment");
|
|
return value;
|
|
}
|
|
inline std::unique_ptr<persistence::Postgres> owner() {
|
|
auto made = persistence::Postgres::connect(
|
|
{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("PGUSER"), {}, {}}, 5s);
|
|
expect(made.has_value(), "owner connection");
|
|
return std::move(*made);
|
|
}
|
|
// Request identity: a tag byte (one per writer) and the tick.
|
|
inline std::string request_hex(std::uint64_t tick, std::uint8_t tag = 0x9B) {
|
|
char out[33];
|
|
std::snprintf(out, sizeof out, "%02x000000000000000000000000%06llx", tag,
|
|
static_cast<unsigned long long>(tick));
|
|
return out;
|
|
}
|
|
inline std::string input_line(std::uint64_t tick, bool consumer_dies, std::uint8_t tag = 0x9B) {
|
|
const auto in = region_input(tick, consumer_dies);
|
|
std::string line = "input " + request_hex(tick, tag) + " " + std::to_string(in.temperature_mK) +
|
|
" " + std::to_string(in.source_enabled);
|
|
for (const auto b : in.bite)
|
|
line += " " + std::to_string(b);
|
|
for (const auto t : in.terminal)
|
|
line += " " + std::to_string(t);
|
|
return line + "\n";
|
|
}
|
|
inline std::string input_lines(std::uint64_t first, std::uint64_t last, bool deaths,
|
|
std::uint8_t tag = 0x9B) {
|
|
std::string out;
|
|
for (auto tick = first; tick <= last; ++tick)
|
|
out += input_line(tick, deaths, tag);
|
|
return out;
|
|
}
|
|
inline bool starts(const std::string& line, std::string_view prefix) {
|
|
return line.starts_with(prefix);
|
|
}
|
|
|
|
class Child {
|
|
public:
|
|
explicit Child(const std::vector<std::string>& args) {
|
|
int to_child[2], from_child[2];
|
|
expect(::pipe(to_child) == 0 && ::pipe(from_child) == 0, "pipes");
|
|
posix_spawn_file_actions_t actions;
|
|
posix_spawn_file_actions_init(&actions);
|
|
posix_spawn_file_actions_adddup2(&actions, to_child[0], STDIN_FILENO);
|
|
posix_spawn_file_actions_adddup2(&actions, from_child[1], STDOUT_FILENO);
|
|
posix_spawn_file_actions_addclose(&actions, to_child[1]);
|
|
posix_spawn_file_actions_addclose(&actions, from_child[0]);
|
|
std::vector<char*> argv;
|
|
argv.reserve(args.size() + 1);
|
|
for (const auto& a : args)
|
|
argv.push_back(const_cast<char*>(a.c_str()));
|
|
argv.push_back(nullptr);
|
|
expect(posix_spawn(&pid_, argv[0], &actions, nullptr, argv.data(), environ) == 0, "spawn");
|
|
posix_spawn_file_actions_destroy(&actions);
|
|
::close(to_child[0]);
|
|
::close(from_child[1]);
|
|
in_ = to_child[1];
|
|
out_ = from_child[0];
|
|
}
|
|
Child(const Child&) = delete;
|
|
Child& operator=(const Child&) = delete;
|
|
~Child() {
|
|
if (pid_ > 0) {
|
|
::kill(pid_, SIGKILL);
|
|
::waitpid(pid_, nullptr, 0);
|
|
}
|
|
::close(in_);
|
|
::close(out_);
|
|
}
|
|
void send(const std::string& text) {
|
|
std::size_t done = 0;
|
|
while (done < text.size()) {
|
|
const auto n = ::write(in_, text.data() + done, text.size() - done);
|
|
expect(n > 0 || errno == EINTR, "write to child");
|
|
if (n > 0)
|
|
done += static_cast<std::size_t>(n);
|
|
}
|
|
}
|
|
// Reads lines until one satisfies `want`; every line is kept in lines().
|
|
template <typename Want> bool wait_for(Want want, std::chrono::milliseconds timeout) {
|
|
const auto deadline = Clock::now() + timeout;
|
|
for (std::size_t seen = 0;;) {
|
|
for (; seen < lines_.size(); ++seen)
|
|
if (want(lines_[seen]))
|
|
return true;
|
|
const auto left =
|
|
std::chrono::duration_cast<std::chrono::milliseconds>(deadline - Clock::now());
|
|
if (left.count() <= 0 || !read_some(static_cast<int>(left.count())))
|
|
return false;
|
|
}
|
|
}
|
|
bool wait_line(std::string_view prefix, std::chrono::milliseconds timeout) {
|
|
return wait_for([prefix](const std::string& l) { return starts(l, prefix); }, timeout);
|
|
}
|
|
void signal(int sig) {
|
|
expect(::kill(pid_, sig) == 0, "signal");
|
|
}
|
|
// Exit code, or 128+signal; drains remaining output.
|
|
int wait_exit(std::chrono::milliseconds timeout) {
|
|
const auto deadline = Clock::now() + timeout;
|
|
int status = 0;
|
|
for (;;) {
|
|
read_some(20);
|
|
const auto r = ::waitpid(pid_, &status, WNOHANG);
|
|
if (r == pid_)
|
|
break;
|
|
expect(Clock::now() < deadline, "child exit deadline");
|
|
}
|
|
while (read_some(20)) {
|
|
}
|
|
pid_ = -1;
|
|
return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status);
|
|
}
|
|
[[nodiscard]] const std::vector<std::string>& lines() const noexcept {
|
|
return lines_;
|
|
}
|
|
[[nodiscard]] bool printed(std::string_view prefix) const {
|
|
for (const auto& l : lines_)
|
|
if (starts(l, prefix))
|
|
return true;
|
|
return false;
|
|
}
|
|
|
|
private:
|
|
bool read_some(int timeout_ms) {
|
|
pollfd fd{out_, POLLIN, 0};
|
|
if (::poll(&fd, 1, timeout_ms) <= 0)
|
|
return false;
|
|
char chunk[4096];
|
|
const auto n = ::read(out_, chunk, sizeof chunk);
|
|
if (n <= 0)
|
|
return false;
|
|
buffer_.append(chunk, static_cast<std::size_t>(n));
|
|
for (auto end = buffer_.find('\n'); end != std::string::npos; end = buffer_.find('\n')) {
|
|
lines_.push_back(buffer_.substr(0, end));
|
|
buffer_.erase(0, end + 1);
|
|
}
|
|
return true;
|
|
}
|
|
pid_t pid_{-1};
|
|
int in_{-1}, out_{-1};
|
|
std::string buffer_;
|
|
std::vector<std::string> lines_;
|
|
};
|
|
|
|
struct ServerRegion {
|
|
persistence::RegionKey key;
|
|
fs::path root;
|
|
// The region's object namespace directory.
|
|
[[nodiscard]] fs::path objects() const {
|
|
return root / ("w" + std::to_string(key.world)) / ("r" + std::to_string(key.region));
|
|
}
|
|
};
|
|
// A fresh lineage region (world 7) provisioned by the owner.
|
|
inline ServerRegion provision(std::uint64_t region, std::uint8_t genesis) {
|
|
ServerRegion r{{7, region}, fs::path(env("PGHOST")) / ("server-" + std::to_string(region))};
|
|
fs::remove_all(r.root);
|
|
fs::create_directory(r.root);
|
|
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
|
|
auto db = owner();
|
|
expect(db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
|
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
|
|
{}, 5s)
|
|
.has_value(),
|
|
"retain build");
|
|
auto objects = persistence::FilesystemObjectStore::open(r.root, r.key,
|
|
persistence::max_lineage_payload_bytes);
|
|
expect(objects.has_value(), "object store");
|
|
persistence::Sha256Digest build{};
|
|
build.fill(0xB1);
|
|
persistence::Uuid commit{};
|
|
commit.fill(genesis);
|
|
expect(persistence::provision_lineage_region(*db, **objects, r.key, region_identity.frame,
|
|
*lineage_region(), commit, build, 8000000, 5s)
|
|
.has_value(),
|
|
"provision");
|
|
return r;
|
|
}
|
|
inline std::vector<std::string> server_args(const ServerRegion& r,
|
|
std::vector<std::string> extra = {}) {
|
|
std::vector<std::string> args{
|
|
env("EVO_SERVER_BIN"),
|
|
"--profile",
|
|
"lineage",
|
|
"--world",
|
|
std::to_string(r.key.world),
|
|
"--region",
|
|
std::to_string(r.key.region),
|
|
"--frame",
|
|
std::to_string(region_identity.frame),
|
|
"--pg-host",
|
|
env("PGHOST"),
|
|
"--pg-port",
|
|
env("PGPORT"),
|
|
"--pg-database",
|
|
env("PGDATABASE"),
|
|
"--pg-user",
|
|
env("EVO_PG_RUNTIME_USER"),
|
|
"--objects",
|
|
r.root.string(),
|
|
"--build",
|
|
"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1",
|
|
"--flush-ms",
|
|
"500",
|
|
"--lease-profile",
|
|
"fast",
|
|
"--ready-file",
|
|
(r.root / "ready").string()};
|
|
args.insert(args.end(), extra.begin(), extra.end());
|
|
return args;
|
|
}
|
|
// (tick, commit_sequence) of every ack line.
|
|
inline std::vector<std::pair<std::uint64_t, std::uint64_t>> acks(const Child& c) {
|
|
std::vector<std::pair<std::uint64_t, std::uint64_t>> out;
|
|
for (const auto& line : c.lines()) {
|
|
unsigned long long tick = 0, sequence = 0;
|
|
char request[40];
|
|
if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &tick, &sequence) == 3)
|
|
out.emplace_back(tick, sequence);
|
|
}
|
|
return out;
|
|
}
|
|
inline std::uint64_t max_acked(const Child& c) {
|
|
std::uint64_t most = 0;
|
|
for (const auto& [tick, sequence] : acks(c))
|
|
most = std::max(most, tick);
|
|
return most;
|
|
}
|
|
struct Head {
|
|
std::uint64_t tick{}, sequence{};
|
|
bool released{};
|
|
};
|
|
inline Head head_of(persistence::RegionKey key) {
|
|
const std::array<std::string, 2> p{std::to_string(key.world), std::to_string(key.region)};
|
|
auto rows = owner()->query(
|
|
"SELECT h.end_tick::text, h.commit_sequence::text, (l.holder IS NULL)::text FROM "
|
|
"evolution.region_heads h JOIN evolution.region_leases l USING (world_id, region_id) "
|
|
"WHERE world_id = $1::evolution.u64 AND region_id = $2::evolution.u64",
|
|
p, 5s);
|
|
expect(rows && rows->size() == 1 && (*rows)[0].size() == 3, "head row");
|
|
const auto& row = (*rows)[0];
|
|
expect(row[0] && row[1] && row[2], "head values");
|
|
return {std::stoull(row[0].value_or("0")), std::stoull(row[1].value_or("0")),
|
|
row[2].value_or("") == "true"};
|
|
}
|
|
inline std::size_t decisions_of(persistence::RegionKey key) {
|
|
const std::array<std::string, 2> p{std::to_string(key.world), std::to_string(key.region)};
|
|
auto rows = owner()->query("SELECT count(*)::text FROM evolution.region_decisions WHERE "
|
|
"world_id = $1::evolution.u64 AND region_id = $2::evolution.u64",
|
|
p, 5s);
|
|
expect(rows && rows->size() == 1, "decision count");
|
|
return std::stoull((*rows)[0][0].value_or("0"));
|
|
}
|
|
// Object files in the region namespace (temporaries excluded).
|
|
inline std::size_t object_count(const ServerRegion& r) {
|
|
std::size_t n = 0;
|
|
for (const auto& entry : fs::directory_iterator(r.objects()))
|
|
n += entry.path().filename().string().starts_with(".tmp-") ? 0 : 1;
|
|
return n;
|
|
}
|
|
} // namespace evo::test::process
|