Files
aevolution/tests/region_recovery_tests.cpp
T
dingoandClaude Opus 5.5 f59cca2c3b Complete M7.9b upgrade barrier and durability qualification
Schema v8 binds authority to builds (ADR-0068): acquisition names the
host build and only retained, unretired builds may hold a lease; no
commit may name a retired build; upgrade_barrier retires a build and
fences every holder in one transaction, while build-less pre-v8 hosts
keep working until the first barrier. Tests cover a writer surviving a
failed and then a real online migration, old-writer fencing on every
path, whole-history recovery by the new build, and the M7 qualification
fixture (19.6 s SIGKILL-to-ready under the production lease, no lost
acknowledgements).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-29 07:31:04 -04:00

277 lines
12 KiB
C++

// M7.7 recovery before readiness (ADR-0065): real evo-server processes killed at
// every publication/acknowledgement boundary, a resumed stale owner, damaged heads
// that must never serve, and tail verification. Run by tools/ci/postgres-check.sh.
#include "evo/persistence/sha256.hpp"
#include "fixtures/server_process.hpp"
#include <algorithm>
#include <fstream>
#include <source_location>
#include <thread>
void run_region_recovery_tests();
namespace {
using namespace evo::test::process;
using namespace evo::persistence;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region recovery test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
std::string hex(std::span<const std::uint8_t> bytes) {
constexpr char digits[] = "0123456789abcdef";
std::string out;
for (const auto b : bytes) {
out.push_back(digits[b >> 4U]);
out.push_back(digits[b & 0x0FU]);
}
return out;
}
std::vector<char> read_file(const fs::path& p) {
std::ifstream in(p, std::ios::binary);
return {std::istreambuf_iterator<char>(in), {}};
}
void write_file(const fs::path& p, const std::vector<char>& bytes) {
fs::remove(p);
std::ofstream(p, std::ios::binary)
.write(bytes.data(), static_cast<std::streamsize>(bytes.size()));
fs::permissions(p, fs::perms::owner_read | fs::perms::owner_write, fs::perm_options::replace);
}
CheckpointManifest head_manifest(RegionKey key) {
auto db = owner();
auto committed = load_region_checkpoint(*db, key, 5s);
CHECK(committed);
auto m = decode_manifest(committed->manifest);
CHECK(m);
return *m;
}
// Successor: waits for readiness, requires it resumed from the committed head, drains.
void successor_resumes(const ServerRegion& r, std::uint64_t epoch) {
const auto head = head_of(r.key);
Child next{server_args(r)};
CHECK(next.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
CHECK(next.printed("phase running epoch=" + std::to_string(epoch) +
" committed=" + std::to_string(head.tick) + " "));
next.signal(SIGTERM);
CHECK(next.wait_exit(20s) == 0);
const auto after = head_of(r.key);
CHECK(after.tick == head.tick && after.sequence == head.sequence && after.released);
}
struct Boundary {
const char* point;
int ordinal;
std::uint64_t region;
std::uint8_t genesis;
};
// Inputs 1..40 with the consumer's death at tick 20: commit 1 is the death batch
// (ticks 1..20), commit 2 the next interval flush. The process is SIGKILLed at the
// chosen boundary; durable state, acknowledgements and orphans must match it.
void kill_at(const Boundary& b) {
const auto r = provision(b.region, b.genesis);
std::uint64_t acked = 0;
bool outcome = false;
{
// A long flush interval makes commit 1 exactly the death batch.
Child server{server_args(r, {"--flush-ms", "5000", "--crash-at",
std::string(b.point) + ":" + std::to_string(b.ordinal)})};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
server.send(input_lines(1, 40, true));
CHECK(server.wait_exit(30s) == 128 + SIGKILL);
acked = max_acked(server);
outcome = server.printed("outcome 20 1 11 1");
}
const auto head = head_of(r.key);
const auto decisions = decisions_of(r.key);
const auto objects = object_count(r); // Genesis payload plus 2 per uploaded batch.
const std::string point = b.point;
CHECK(!head.released); // Killed holders never release; successors wait for expiry.
CHECK(acked <= head.tick); // Never an acknowledgement beyond the committed head.
if (b.ordinal == 1) {
// The death batch never committed: nothing final, the death is not a fact.
CHECK(point == "before-transaction");
CHECK(head.sequence == 0 && head.tick == 0 && acked == 0 && decisions == 0 && !outcome);
CHECK(objects == 3); // Verified orphans only.
} else if (point == "before-upload" || point == "before-transaction") {
CHECK(head.sequence == 1 && head.tick == 20 && acked == 20 && decisions == 1 && outcome);
CHECK(objects == (point == "before-upload" ? 3U : 5U));
} else if (point == "after-commit") {
// Durable but never acknowledged: the lost acks are the safe direction.
CHECK(head.sequence == 2 && head.tick > 20 && acked == 20 && decisions == 1 && outcome);
} else {
CHECK(point == "after-ack");
CHECK(head.sequence == 2 && acked == head.tick && decisions == 1 && outcome);
}
std::printf("kill_at %s:%d head=%llu/%llu acked=%llu objects=%zu\n", b.point, b.ordinal,
static_cast<unsigned long long>(head.sequence),
static_cast<unsigned long long>(head.tick), static_cast<unsigned long long>(acked),
objects);
successor_resumes(r, 2); // Verified head, tail and history before readiness.
}
// A paused (SIGSTOP) owner loses its lease; after SIGCONT it fences itself, refuses
// admission and never acknowledges past its own commits, while the successor's
// chain continues untouched.
void resumed_stale_owner_is_fenced() {
const auto r = provision(46, 0x62);
Child stale{server_args(r)};
CHECK(stale.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
stale.send(input_lines(1, 10, false, 0xA1));
CHECK(stale.wait_line("ack " + request_hex(10, 0xA1) + " 10 ", 15s));
stale.signal(SIGSTOP);
std::this_thread::sleep_for(4500ms); // Longer than the 4 s development lease.
Child owner_now{server_args(r)};
CHECK(owner_now.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
CHECK(owner_now.printed("phase running epoch=2 committed=10 "));
owner_now.send(input_lines(11, 20, false, 0xB1));
CHECK(owner_now.wait_line("ack " + request_hex(20, 0xB1) + " 20 ", 15s));
stale.signal(SIGCONT);
CHECK(stale.wait_line("ready 0", 10s));
stale.send(input_lines(11, 12, false, 0xA2));
CHECK(stale.wait_line("rejected " + request_hex(11, 0xA2) + " not_ready", 10s));
CHECK(max_acked(stale) == 10);
const auto head = head_of(r.key);
CHECK(head.tick >= 20 && head.sequence >= 2 && !head.released);
stale.signal(SIGTERM);
CHECK(stale.wait_exit(20s) == 0); // It never regained authority: nothing to release.
owner_now.signal(SIGTERM);
CHECK(owner_now.wait_exit(20s) == 0);
CHECK(head_of(r.key).released && head_of(r.key).tick == head.tick);
}
// A committed head whose payload or tail cannot be verified is never served, and
// no older or empty world replaces it.
void damaged_heads_never_become_ready() {
const auto r = provision(47, 0x63);
{
Child server{server_args(r)};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
server.send(input_lines(1, 10, false));
CHECK(server.wait_line("ack " + request_hex(10) + " 10 ", 15s));
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 0);
}
const auto head = head_of(r.key);
const auto m = head_manifest(r.key);
const auto payload = r.objects() / hex(m.payload_sha256);
const auto trace = r.objects() / hex(m.batch.trace_sha256);
const auto quarantined = [&] {
Child server{server_args(r)};
const auto code = server.wait_exit(20s);
return code == 4 && server.printed("phase quarantined") && !server.printed("ready 1");
};
const auto payload_bytes = read_file(payload);
auto flipped = payload_bytes;
flipped[flipped.size() / 2] = static_cast<char>(flipped[flipped.size() / 2] ^ 1);
write_file(payload, flipped);
CHECK(quarantined());
write_file(payload, payload_bytes);
const auto trace_bytes = read_file(trace);
fs::remove(trace);
CHECK(quarantined());
write_file(trace, trace_bytes);
const auto after = head_of(r.key);
CHECK(after.tick == head.tick && after.sequence == head.sequence); // Nothing replaced it.
successor_resumes(r, 4); // Epochs 2 and 3 were taken by the quarantined attempts.
}
// In-process: recovery replays the head's trace from its predecessor before any
// readiness; missing, damaged or non-reproducing evidence quarantines (tail).
void tail_verification_before_readiness() {
const auto r = provision(48, 0x01);
auto runtime = Postgres::connect(
{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("EVO_PG_RUNTIME_USER"), {}, {}},
5s);
CHECK(runtime);
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
CHECK(objects);
Sha256Digest build{};
build.fill(0xB1);
const std::array<Sha256Digest, 1> retained{build};
const auto commit = [](std::uint8_t v) {
Uuid id{};
id.fill(v);
return id;
};
auto lease = acquire_region_lease(**runtime, r.key, build, 20000ms, 5s);
CHECK(lease);
auto head = load_region_checkpoint(**runtime, r.key, 5s);
CHECK(head);
auto state = evo::test::lineage_region();
evo::persistence::RegionBatch<evo::processes::LineageRegion> batch{*state};
for (int n = 0; n < 2; ++n) {
for (int i = 0; i < 10; ++i)
CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1)));
batch.seal();
auto published =
publish_lineage_batch(**runtime, **objects, *lease, *head, batch,
commit(static_cast<std::uint8_t>(2 + n)), build, 8000000, 5s);
CHECK(published && batch.confirm(published->committed.head.end_tick));
head = published->committed;
}
const auto recover = [&] {
return recover_lineage_checkpoint(**runtime, **objects, r.key, retained, 8000000, 5s);
};
CHECK(recover());
const auto m = *decode_manifest(head->manifest);
const auto trace = r.objects() / hex(m.batch.trace_sha256);
const auto trace_bytes = read_file(trace);
const auto refused = [&](RecoveryError kind) {
auto rec = recover();
return !rec && rec.error().kind == kind;
};
fs::remove(trace);
CHECK(refused(RecoveryError::tail));
auto damaged = trace_bytes;
damaged[200] = static_cast<char>(damaged[200] ^ 1);
write_file(trace, damaged);
CHECK(refused(RecoveryError::tail));
write_file(trace, trace_bytes);
// The predecessor's payload is needed to replay the head: its loss quarantines.
auto first = load_region_commit(**runtime, r.key, 1, 5s);
CHECK(first);
const auto previous = *decode_manifest(first->manifest);
const auto base = r.objects() / hex(previous.payload_sha256);
const auto base_bytes = read_file(base);
fs::remove(base);
CHECK(refused(RecoveryError::tail));
write_file(base, base_bytes);
CHECK(recover());
// Evidence that does not reproduce the payload: a trace of different inputs over
// the same range, published by bypassing the host (the database cannot tell).
for (int i = 0; i < 10; ++i)
CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1)));
auto alternate = recover();
CHECK(alternate);
evo::persistence::RegionBatch<evo::processes::LineageRegion> other{*alternate->state};
for (int i = 0; i < 10; ++i) {
auto in = evo::test::region_input(other.end_tick() + 1);
in.source_enabled = 1 - in.source_enabled; // Not what actually happened.
CHECK(other.advance(in));
}
auto payload = state->snapshot();
auto forged = encode_trace({2, r.key, m.commit, head->head.digest, m.end_tick, m.end_tick + 10},
other.steps());
CHECK(payload && forged && (*objects)->put(*payload) && (*objects)->put(*forged));
auto manifest = successor_manifest(m, head->head.digest, commit(5), *lease, m.end_tick + 10,
build, payload->size(), *sha256(*payload),
{forged->size(), *sha256(*forged), 0, {}});
CHECK(manifest);
CHECK(publish_region_checkpoint(**runtime, *encode_manifest(*manifest), {}, 5s));
CHECK(refused(RecoveryError::tail));
CHECK(release_region_lease(**runtime, r.key, *lease, 5s));
}
} // namespace
void run_region_recovery_tests() {
for (const auto& b :
{Boundary{"before-transaction", 1, 41, 0x5D}, Boundary{"before-upload", 2, 42, 0x5E},
Boundary{"before-transaction", 2, 43, 0x5F}, Boundary{"after-commit", 2, 44, 0x60},
Boundary{"after-ack", 2, 45, 0x61}})
kill_at(b);
resumed_stale_owner_is_fenced();
damaged_heads_never_become_ready();
tail_verification_before_readiness();
}