A drill-only evo-server --synthetic-inputs development-v1 driver feeds deterministic inputs from the committed head after every recovery, with request IDs that match acknowledgements across owners (ADR-0073); it is off in Git and the policy check keeps it off. fault-drill.sh loads dev or stage and runs repeated forced kills (RTO, zero lost acknowledgements), a full object volume (nothing acknowledged), an empty volume (quarantine, never a fresh world) and an operator-driven database outage (stale epoch never resumes), restoring the Git configuration on exit. Process tests prove contiguous acknowledgement across a SIGKILL and exact resumption after the committed head. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
105 lines
4.4 KiB
C++
105 lines
4.4 KiB
C++
// M8.6 synthetic load (ADR-0073) on real evo-server processes: the deterministic
|
|
// driver keeps a region busy across a death and a SIGKILL, every acknowledgement is
|
|
// durable, and the next owner resumes exactly after the committed head with no gap,
|
|
// repeat or lost acknowledgement. Run by tools/ci/postgres-check.sh.
|
|
#include "fixtures/server_process.hpp"
|
|
#include <algorithm>
|
|
#include <cstdio>
|
|
#include <source_location>
|
|
|
|
void run_region_load_tests();
|
|
|
|
namespace {
|
|
using namespace evo::test::process;
|
|
void check(bool value, std::source_location at = std::source_location::current()) {
|
|
if (!value)
|
|
throw std::runtime_error("region load test line " + std::to_string(at.line()));
|
|
}
|
|
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
|
|
|
struct Ack {
|
|
std::string request;
|
|
std::uint64_t tick{}, sequence{};
|
|
};
|
|
std::vector<Ack> ack_lines(const Child& c) {
|
|
std::vector<Ack> out;
|
|
for (const auto& line : c.lines()) {
|
|
char request[40];
|
|
unsigned long long tick = 0, sequence = 0;
|
|
if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &tick, &sequence) == 3)
|
|
out.push_back({request, tick, sequence});
|
|
}
|
|
return out;
|
|
}
|
|
std::string synthetic_request(std::uint64_t region, std::uint64_t tick) {
|
|
char out[33];
|
|
std::snprintf(out, sizeof out, "53594e31%08llx%016llx",
|
|
static_cast<unsigned long long>(region & 0xFFFFFFFFU),
|
|
static_cast<unsigned long long>(tick));
|
|
return out;
|
|
}
|
|
bool acked_through(const Child& c, std::uint64_t tick) {
|
|
const auto acks = ack_lines(c);
|
|
return std::ranges::any_of(acks, [&](const Ack& a) { return a.tick >= tick; });
|
|
}
|
|
// Acks are contiguous from `first`, in tick order, each with its deterministic ID.
|
|
void contiguous_from(const std::vector<Ack>& acks, std::uint64_t first, std::uint64_t region) {
|
|
CHECK(!acks.empty() && acks.front().tick == first);
|
|
for (std::size_t i = 0; i < acks.size(); ++i) {
|
|
CHECK(acks[i].tick == first + i);
|
|
CHECK(acks[i].request == synthetic_request(region, acks[i].tick));
|
|
}
|
|
}
|
|
|
|
void synthetic_load_survives_a_kill() {
|
|
const auto r = provision(87, 0xE6);
|
|
const auto args = server_args(r, {"--synthetic-inputs", "development-v1"});
|
|
std::uint64_t committed = 0;
|
|
{
|
|
Child first{args};
|
|
CHECK(first.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
|
CHECK(first.wait_for([&](const std::string&) { return acked_through(first, 45); }, 30s));
|
|
CHECK(first.printed("outcome 20 1 11 ")); // The consumer's death, made durable.
|
|
first.signal(SIGKILL); // Mid-stream: inputs queued, a batch open.
|
|
CHECK(first.wait_exit(20s) == 128 + SIGKILL);
|
|
const auto acks = ack_lines(first);
|
|
contiguous_from(acks, 1, r.key.region);
|
|
const auto head = head_of(r.key);
|
|
committed = head.tick;
|
|
CHECK(acks.back().tick <= committed); // Nothing acknowledged was lost.
|
|
CHECK(decisions_of(r.key) == 1);
|
|
}
|
|
// The next owner waits out the lease, then resumes right after the head.
|
|
Child second{args};
|
|
CHECK(second.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
|
CHECK(second.printed("phase running epoch=2 committed=" + std::to_string(committed) + " "));
|
|
CHECK(second.wait_for([&](const std::string&) { return acked_through(second, committed + 30); },
|
|
30s));
|
|
second.signal(SIGTERM);
|
|
CHECK(second.wait_exit(20s) == 0);
|
|
const auto acks = ack_lines(second);
|
|
contiguous_from(acks, committed + 1, r.key.region);
|
|
const auto head = head_of(r.key);
|
|
CHECK(acks.back().tick <= head.tick && head.released);
|
|
CHECK(decisions_of(r.key) == 1); // The death is never repeated.
|
|
}
|
|
|
|
// Stdin is not read in synthetic mode: typed inputs cannot interleave with the driver.
|
|
void synthetic_mode_ignores_stdin() {
|
|
const auto r = provision(88, 0xE5);
|
|
Child server{server_args(r, {"--synthetic-inputs", "development-v1"})};
|
|
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
|
server.send(input_line(1, false, 0x77));
|
|
CHECK(server.wait_for([&](const std::string&) { return acked_through(server, 10); }, 30s));
|
|
for (const auto& a : ack_lines(server))
|
|
CHECK(a.request.starts_with("53594e31"));
|
|
server.signal(SIGTERM);
|
|
CHECK(server.wait_exit(20s) == 0);
|
|
}
|
|
} // namespace
|
|
|
|
void run_region_load_tests() {
|
|
synthetic_load_survives_a_kill();
|
|
synthetic_mode_ignores_stdin();
|
|
}
|