// M7.7 recovery before readiness (ADR-0065): real evo-server processes killed at // every publication/acknowledgement boundary, a resumed stale owner, damaged heads // that must never serve, and tail verification. Run by tools/ci/postgres-check.sh. #include "evo/persistence/sha256.hpp" #include "fixtures/server_process.hpp" #include #include #include #include void run_region_recovery_tests(); namespace { using namespace evo::test::process; using namespace evo::persistence; void check(bool value, std::source_location at = std::source_location::current()) { if (!value) throw std::runtime_error("region recovery test line " + std::to_string(at.line())); } #define CHECK(...) check(static_cast(__VA_ARGS__)) std::string hex(std::span bytes) { constexpr char digits[] = "0123456789abcdef"; std::string out; for (const auto b : bytes) { out.push_back(digits[b >> 4U]); out.push_back(digits[b & 0x0FU]); } return out; } std::vector read_file(const fs::path& p) { std::ifstream in(p, std::ios::binary); return {std::istreambuf_iterator(in), {}}; } void write_file(const fs::path& p, const std::vector& bytes) { fs::remove(p); std::ofstream(p, std::ios::binary) .write(bytes.data(), static_cast(bytes.size())); fs::permissions(p, fs::perms::owner_read | fs::perms::owner_write, fs::perm_options::replace); } CheckpointManifest head_manifest(RegionKey key) { auto db = owner(); auto committed = load_region_checkpoint(*db, key, 5s); CHECK(committed); auto m = decode_manifest(committed->manifest); CHECK(m); return *m; } // Successor: waits for readiness, requires it resumed from the committed head, drains. void successor_resumes(const ServerRegion& r, std::uint64_t epoch) { const auto head = head_of(r.key); Child next{server_args(r)}; CHECK(next.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); CHECK(next.printed("phase running epoch=" + std::to_string(epoch) + " committed=" + std::to_string(head.tick) + " ")); next.signal(SIGTERM); CHECK(next.wait_exit(20s) == 0); const auto after = head_of(r.key); CHECK(after.tick == head.tick && after.sequence == head.sequence && after.released); } struct Boundary { const char* point; int ordinal; std::uint64_t region; std::uint8_t genesis; }; // Inputs 1..40 with the consumer's death at tick 20: commit 1 is the death batch // (ticks 1..20), commit 2 the next interval flush. The process is SIGKILLed at the // chosen boundary; durable state, acknowledgements and orphans must match it. void kill_at(const Boundary& b) { const auto r = provision(b.region, b.genesis); std::uint64_t acked = 0; bool outcome = false; { // A long flush interval makes commit 1 exactly the death batch. Child server{server_args(r, {"--flush-ms", "5000", "--crash-at", std::string(b.point) + ":" + std::to_string(b.ordinal)})}; CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); server.send(input_lines(1, 40, true)); CHECK(server.wait_exit(30s) == 128 + SIGKILL); acked = max_acked(server); outcome = server.printed("outcome 20 1 11 1"); } const auto head = head_of(r.key); const auto decisions = decisions_of(r.key); const auto objects = object_count(r); // Genesis payload plus 2 per uploaded batch. const std::string point = b.point; CHECK(!head.released); // Killed holders never release; successors wait for expiry. CHECK(acked <= head.tick); // Never an acknowledgement beyond the committed head. if (b.ordinal == 1) { // The death batch never committed: nothing final, the death is not a fact. CHECK(point == "before-transaction"); CHECK(head.sequence == 0 && head.tick == 0 && acked == 0 && decisions == 0 && !outcome); CHECK(objects == 3); // Verified orphans only. } else if (point == "before-upload" || point == "before-transaction") { CHECK(head.sequence == 1 && head.tick == 20 && acked == 20 && decisions == 1 && outcome); CHECK(objects == (point == "before-upload" ? 3U : 5U)); } else if (point == "after-commit") { // Durable but never acknowledged: the lost acks are the safe direction. CHECK(head.sequence == 2 && head.tick > 20 && acked == 20 && decisions == 1 && outcome); } else { CHECK(point == "after-ack"); CHECK(head.sequence == 2 && acked == head.tick && decisions == 1 && outcome); } std::printf("kill_at %s:%d head=%llu/%llu acked=%llu objects=%zu\n", b.point, b.ordinal, static_cast(head.sequence), static_cast(head.tick), static_cast(acked), objects); successor_resumes(r, 2); // Verified head, tail and history before readiness. } // A paused (SIGSTOP) owner loses its lease; after SIGCONT it fences itself, refuses // admission and never acknowledges past its own commits, while the successor's // chain continues untouched. void resumed_stale_owner_is_fenced() { const auto r = provision(46, 0x62); Child stale{server_args(r)}; CHECK(stale.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); stale.send(input_lines(1, 10, false, 0xA1)); CHECK(stale.wait_line("ack " + request_hex(10, 0xA1) + " 10 ", 15s)); stale.signal(SIGSTOP); std::this_thread::sleep_for(4500ms); // Longer than the 4 s development lease. Child owner_now{server_args(r)}; CHECK(owner_now.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); CHECK(owner_now.printed("phase running epoch=2 committed=10 ")); owner_now.send(input_lines(11, 20, false, 0xB1)); CHECK(owner_now.wait_line("ack " + request_hex(20, 0xB1) + " 20 ", 15s)); stale.signal(SIGCONT); CHECK(stale.wait_line("ready 0", 10s)); stale.send(input_lines(11, 12, false, 0xA2)); CHECK(stale.wait_line("rejected " + request_hex(11, 0xA2) + " not_ready", 10s)); CHECK(max_acked(stale) == 10); const auto head = head_of(r.key); CHECK(head.tick >= 20 && head.sequence >= 2 && !head.released); stale.signal(SIGTERM); CHECK(stale.wait_exit(20s) == 0); // It never regained authority: nothing to release. owner_now.signal(SIGTERM); CHECK(owner_now.wait_exit(20s) == 0); CHECK(head_of(r.key).released && head_of(r.key).tick == head.tick); } // A committed head whose payload or tail cannot be verified is never served, and // no older or empty world replaces it. void damaged_heads_never_become_ready() { const auto r = provision(47, 0x63); { Child server{server_args(r)}; CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); server.send(input_lines(1, 10, false)); CHECK(server.wait_line("ack " + request_hex(10) + " 10 ", 15s)); server.signal(SIGTERM); CHECK(server.wait_exit(20s) == 0); } const auto head = head_of(r.key); const auto m = head_manifest(r.key); const auto payload = r.objects() / hex(m.payload_sha256); const auto trace = r.objects() / hex(m.batch.trace_sha256); const auto quarantined = [&] { Child server{server_args(r)}; const auto code = server.wait_exit(20s); return code == 4 && server.printed("phase quarantined") && !server.printed("ready 1"); }; const auto payload_bytes = read_file(payload); auto flipped = payload_bytes; flipped[flipped.size() / 2] = static_cast(flipped[flipped.size() / 2] ^ 1); write_file(payload, flipped); CHECK(quarantined()); write_file(payload, payload_bytes); const auto trace_bytes = read_file(trace); fs::remove(trace); CHECK(quarantined()); write_file(trace, trace_bytes); const auto after = head_of(r.key); CHECK(after.tick == head.tick && after.sequence == head.sequence); // Nothing replaced it. successor_resumes(r, 4); // Epochs 2 and 3 were taken by the quarantined attempts. } // In-process: recovery replays the head's trace from its predecessor before any // readiness; missing, damaged or non-reproducing evidence quarantines (tail). void tail_verification_before_readiness() { const auto r = provision(48, 0x01); auto runtime = Postgres::connect( {env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("EVO_PG_RUNTIME_USER"), {}, {}}, 5s); CHECK(runtime); auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes); CHECK(objects); Sha256Digest build{}; build.fill(0xB1); const std::array retained{build}; const auto commit = [](std::uint8_t v) { Uuid id{}; id.fill(v); return id; }; auto lease = acquire_region_lease(**runtime, r.key, build, 20000ms, 5s); CHECK(lease); auto head = load_region_checkpoint(**runtime, r.key, 5s); CHECK(head); auto state = evo::test::lineage_region(); evo::persistence::RegionBatch batch{*state}; for (int n = 0; n < 2; ++n) { for (int i = 0; i < 10; ++i) CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1))); batch.seal(); auto published = publish_lineage_batch(**runtime, **objects, *lease, *head, batch, commit(static_cast(2 + n)), build, 8000000, 5s); CHECK(published && batch.confirm(published->committed.head.end_tick)); head = published->committed; } const auto recover = [&] { return recover_lineage_checkpoint(**runtime, **objects, r.key, retained, 8000000, 5s); }; CHECK(recover()); const auto m = *decode_manifest(head->manifest); const auto trace = r.objects() / hex(m.batch.trace_sha256); const auto trace_bytes = read_file(trace); const auto refused = [&](RecoveryError kind) { auto rec = recover(); return !rec && rec.error().kind == kind; }; fs::remove(trace); CHECK(refused(RecoveryError::tail)); auto damaged = trace_bytes; damaged[200] = static_cast(damaged[200] ^ 1); write_file(trace, damaged); CHECK(refused(RecoveryError::tail)); write_file(trace, trace_bytes); // The predecessor's payload is needed to replay the head: its loss quarantines. auto first = load_region_commit(**runtime, r.key, 1, 5s); CHECK(first); const auto previous = *decode_manifest(first->manifest); const auto base = r.objects() / hex(previous.payload_sha256); const auto base_bytes = read_file(base); fs::remove(base); CHECK(refused(RecoveryError::tail)); write_file(base, base_bytes); CHECK(recover()); // Evidence that does not reproduce the payload: a trace of different inputs over // the same range, published by bypassing the host (the database cannot tell). for (int i = 0; i < 10; ++i) CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1))); auto alternate = recover(); CHECK(alternate); evo::persistence::RegionBatch other{*alternate->state}; for (int i = 0; i < 10; ++i) { auto in = evo::test::region_input(other.end_tick() + 1); in.source_enabled = 1 - in.source_enabled; // Not what actually happened. CHECK(other.advance(in)); } auto payload = state->snapshot(); auto forged = encode_trace({2, r.key, m.commit, head->head.digest, m.end_tick, m.end_tick + 10}, other.steps()); CHECK(payload && forged && (*objects)->put(*payload) && (*objects)->put(*forged)); auto manifest = successor_manifest(m, head->head.digest, commit(5), *lease, m.end_tick + 10, build, payload->size(), *sha256(*payload), {forged->size(), *sha256(*forged), 0, {}}); CHECK(manifest); CHECK(publish_region_checkpoint(**runtime, *encode_manifest(*manifest), {}, 5s)); CHECK(refused(RecoveryError::tail)); CHECK(release_region_lease(**runtime, r.key, *lease, 5s)); } } // namespace void run_region_recovery_tests() { for (const auto& b : {Boundary{"before-transaction", 1, 41, 0x5D}, Boundary{"before-upload", 2, 42, 0x5E}, Boundary{"before-transaction", 2, 43, 0x5F}, Boundary{"after-commit", 2, 44, 0x60}, Boundary{"after-ack", 2, 45, 0x61}}) kill_at(b); resumed_stale_owner_is_fenced(); damaged_heads_never_become_ready(); tail_verification_before_readiness(); }