Files
aevolution/tests/region_load_tests.cpp
dingoandClaude Opus 5.5 3f08a1d221 Add M8.6 synthetic load driver and fault drill
A drill-only evo-server --synthetic-inputs development-v1 driver feeds
deterministic inputs from the committed head after every recovery, with
request IDs that match acknowledgements across owners (ADR-0073); it is
off in Git and the policy check keeps it off. fault-drill.sh loads dev
or stage and runs repeated forced kills (RTO, zero lost
acknowledgements), a full object volume (nothing acknowledged), an
empty volume (quarantine, never a fresh world) and an operator-driven
database outage (stale epoch never resumes), restoring the Git
configuration on exit. Process tests prove contiguous acknowledgement
across a SIGKILL and exact resumption after the committed head.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 10:52:01 -04:00

105 lines
4.4 KiB
C++

// M8.6 synthetic load (ADR-0073) on real evo-server processes: the deterministic
// driver keeps a region busy across a death and a SIGKILL, every acknowledgement is
// durable, and the next owner resumes exactly after the committed head with no gap,
// repeat or lost acknowledgement. Run by tools/ci/postgres-check.sh.
#include "fixtures/server_process.hpp"
#include <algorithm>
#include <cstdio>
#include <source_location>
void run_region_load_tests();
namespace {
using namespace evo::test::process;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region load test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
struct Ack {
std::string request;
std::uint64_t tick{}, sequence{};
};
std::vector<Ack> ack_lines(const Child& c) {
std::vector<Ack> out;
for (const auto& line : c.lines()) {
char request[40];
unsigned long long tick = 0, sequence = 0;
if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &tick, &sequence) == 3)
out.push_back({request, tick, sequence});
}
return out;
}
std::string synthetic_request(std::uint64_t region, std::uint64_t tick) {
char out[33];
std::snprintf(out, sizeof out, "53594e31%08llx%016llx",
static_cast<unsigned long long>(region & 0xFFFFFFFFU),
static_cast<unsigned long long>(tick));
return out;
}
bool acked_through(const Child& c, std::uint64_t tick) {
const auto acks = ack_lines(c);
return std::ranges::any_of(acks, [&](const Ack& a) { return a.tick >= tick; });
}
// Acks are contiguous from `first`, in tick order, each with its deterministic ID.
void contiguous_from(const std::vector<Ack>& acks, std::uint64_t first, std::uint64_t region) {
CHECK(!acks.empty() && acks.front().tick == first);
for (std::size_t i = 0; i < acks.size(); ++i) {
CHECK(acks[i].tick == first + i);
CHECK(acks[i].request == synthetic_request(region, acks[i].tick));
}
}
void synthetic_load_survives_a_kill() {
const auto r = provision(87, 0xE6);
const auto args = server_args(r, {"--synthetic-inputs", "development-v1"});
std::uint64_t committed = 0;
{
Child first{args};
CHECK(first.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
CHECK(first.wait_for([&](const std::string&) { return acked_through(first, 45); }, 30s));
CHECK(first.printed("outcome 20 1 11 ")); // The consumer's death, made durable.
first.signal(SIGKILL); // Mid-stream: inputs queued, a batch open.
CHECK(first.wait_exit(20s) == 128 + SIGKILL);
const auto acks = ack_lines(first);
contiguous_from(acks, 1, r.key.region);
const auto head = head_of(r.key);
committed = head.tick;
CHECK(acks.back().tick <= committed); // Nothing acknowledged was lost.
CHECK(decisions_of(r.key) == 1);
}
// The next owner waits out the lease, then resumes right after the head.
Child second{args};
CHECK(second.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
CHECK(second.printed("phase running epoch=2 committed=" + std::to_string(committed) + " "));
CHECK(second.wait_for([&](const std::string&) { return acked_through(second, committed + 30); },
30s));
second.signal(SIGTERM);
CHECK(second.wait_exit(20s) == 0);
const auto acks = ack_lines(second);
contiguous_from(acks, committed + 1, r.key.region);
const auto head = head_of(r.key);
CHECK(acks.back().tick <= head.tick && head.released);
CHECK(decisions_of(r.key) == 1); // The death is never repeated.
}
// Stdin is not read in synthetic mode: typed inputs cannot interleave with the driver.
void synthetic_mode_ignores_stdin() {
const auto r = provision(88, 0xE5);
Child server{server_args(r, {"--synthetic-inputs", "development-v1"})};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
server.send(input_line(1, false, 0x77));
CHECK(server.wait_for([&](const std::string&) { return acked_through(server, 10); }, 30s));
for (const auto& a : ack_lines(server))
CHECK(a.request.starts_with("53594e31"));
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 0);
}
} // namespace
void run_region_load_tests() {
synthetic_load_survives_a_kill();
synthetic_mode_ignores_stdin();
}