Schema v8 binds authority to builds (ADR-0068): acquisition names the host build and only retained, unretired builds may hold a lease; no commit may name a retired build; upgrade_barrier retires a build and fences every holder in one transaction, while build-less pre-v8 hosts keep working until the first barrier. Tests cover a writer surviving a failed and then a real online migration, old-writer fencing on every path, whole-history recovery by the new build, and the M7 qualification fixture (19.6 s SIGKILL-to-ready under the production lease, no lost acknowledgements). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
277 lines
12 KiB
C++
277 lines
12 KiB
C++
// M7.7 recovery before readiness (ADR-0065): real evo-server processes killed at
|
|
// every publication/acknowledgement boundary, a resumed stale owner, damaged heads
|
|
// that must never serve, and tail verification. Run by tools/ci/postgres-check.sh.
|
|
#include "evo/persistence/sha256.hpp"
|
|
#include "fixtures/server_process.hpp"
|
|
#include <algorithm>
|
|
#include <fstream>
|
|
#include <source_location>
|
|
#include <thread>
|
|
|
|
void run_region_recovery_tests();
|
|
|
|
namespace {
|
|
using namespace evo::test::process;
|
|
using namespace evo::persistence;
|
|
void check(bool value, std::source_location at = std::source_location::current()) {
|
|
if (!value)
|
|
throw std::runtime_error("region recovery test line " + std::to_string(at.line()));
|
|
}
|
|
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
|
std::string hex(std::span<const std::uint8_t> bytes) {
|
|
constexpr char digits[] = "0123456789abcdef";
|
|
std::string out;
|
|
for (const auto b : bytes) {
|
|
out.push_back(digits[b >> 4U]);
|
|
out.push_back(digits[b & 0x0FU]);
|
|
}
|
|
return out;
|
|
}
|
|
std::vector<char> read_file(const fs::path& p) {
|
|
std::ifstream in(p, std::ios::binary);
|
|
return {std::istreambuf_iterator<char>(in), {}};
|
|
}
|
|
void write_file(const fs::path& p, const std::vector<char>& bytes) {
|
|
fs::remove(p);
|
|
std::ofstream(p, std::ios::binary)
|
|
.write(bytes.data(), static_cast<std::streamsize>(bytes.size()));
|
|
fs::permissions(p, fs::perms::owner_read | fs::perms::owner_write, fs::perm_options::replace);
|
|
}
|
|
CheckpointManifest head_manifest(RegionKey key) {
|
|
auto db = owner();
|
|
auto committed = load_region_checkpoint(*db, key, 5s);
|
|
CHECK(committed);
|
|
auto m = decode_manifest(committed->manifest);
|
|
CHECK(m);
|
|
return *m;
|
|
}
|
|
// Successor: waits for readiness, requires it resumed from the committed head, drains.
|
|
void successor_resumes(const ServerRegion& r, std::uint64_t epoch) {
|
|
const auto head = head_of(r.key);
|
|
Child next{server_args(r)};
|
|
CHECK(next.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
|
CHECK(next.printed("phase running epoch=" + std::to_string(epoch) +
|
|
" committed=" + std::to_string(head.tick) + " "));
|
|
next.signal(SIGTERM);
|
|
CHECK(next.wait_exit(20s) == 0);
|
|
const auto after = head_of(r.key);
|
|
CHECK(after.tick == head.tick && after.sequence == head.sequence && after.released);
|
|
}
|
|
|
|
struct Boundary {
|
|
const char* point;
|
|
int ordinal;
|
|
std::uint64_t region;
|
|
std::uint8_t genesis;
|
|
};
|
|
// Inputs 1..40 with the consumer's death at tick 20: commit 1 is the death batch
|
|
// (ticks 1..20), commit 2 the next interval flush. The process is SIGKILLed at the
|
|
// chosen boundary; durable state, acknowledgements and orphans must match it.
|
|
void kill_at(const Boundary& b) {
|
|
const auto r = provision(b.region, b.genesis);
|
|
std::uint64_t acked = 0;
|
|
bool outcome = false;
|
|
{
|
|
// A long flush interval makes commit 1 exactly the death batch.
|
|
Child server{server_args(r, {"--flush-ms", "5000", "--crash-at",
|
|
std::string(b.point) + ":" + std::to_string(b.ordinal)})};
|
|
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
|
server.send(input_lines(1, 40, true));
|
|
CHECK(server.wait_exit(30s) == 128 + SIGKILL);
|
|
acked = max_acked(server);
|
|
outcome = server.printed("outcome 20 1 11 1");
|
|
}
|
|
const auto head = head_of(r.key);
|
|
const auto decisions = decisions_of(r.key);
|
|
const auto objects = object_count(r); // Genesis payload plus 2 per uploaded batch.
|
|
const std::string point = b.point;
|
|
CHECK(!head.released); // Killed holders never release; successors wait for expiry.
|
|
CHECK(acked <= head.tick); // Never an acknowledgement beyond the committed head.
|
|
if (b.ordinal == 1) {
|
|
// The death batch never committed: nothing final, the death is not a fact.
|
|
CHECK(point == "before-transaction");
|
|
CHECK(head.sequence == 0 && head.tick == 0 && acked == 0 && decisions == 0 && !outcome);
|
|
CHECK(objects == 3); // Verified orphans only.
|
|
} else if (point == "before-upload" || point == "before-transaction") {
|
|
CHECK(head.sequence == 1 && head.tick == 20 && acked == 20 && decisions == 1 && outcome);
|
|
CHECK(objects == (point == "before-upload" ? 3U : 5U));
|
|
} else if (point == "after-commit") {
|
|
// Durable but never acknowledged: the lost acks are the safe direction.
|
|
CHECK(head.sequence == 2 && head.tick > 20 && acked == 20 && decisions == 1 && outcome);
|
|
} else {
|
|
CHECK(point == "after-ack");
|
|
CHECK(head.sequence == 2 && acked == head.tick && decisions == 1 && outcome);
|
|
}
|
|
std::printf("kill_at %s:%d head=%llu/%llu acked=%llu objects=%zu\n", b.point, b.ordinal,
|
|
static_cast<unsigned long long>(head.sequence),
|
|
static_cast<unsigned long long>(head.tick), static_cast<unsigned long long>(acked),
|
|
objects);
|
|
successor_resumes(r, 2); // Verified head, tail and history before readiness.
|
|
}
|
|
|
|
// A paused (SIGSTOP) owner loses its lease; after SIGCONT it fences itself, refuses
|
|
// admission and never acknowledges past its own commits, while the successor's
|
|
// chain continues untouched.
|
|
void resumed_stale_owner_is_fenced() {
|
|
const auto r = provision(46, 0x62);
|
|
Child stale{server_args(r)};
|
|
CHECK(stale.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
|
stale.send(input_lines(1, 10, false, 0xA1));
|
|
CHECK(stale.wait_line("ack " + request_hex(10, 0xA1) + " 10 ", 15s));
|
|
stale.signal(SIGSTOP);
|
|
std::this_thread::sleep_for(4500ms); // Longer than the 4 s development lease.
|
|
Child owner_now{server_args(r)};
|
|
CHECK(owner_now.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
|
CHECK(owner_now.printed("phase running epoch=2 committed=10 "));
|
|
owner_now.send(input_lines(11, 20, false, 0xB1));
|
|
CHECK(owner_now.wait_line("ack " + request_hex(20, 0xB1) + " 20 ", 15s));
|
|
stale.signal(SIGCONT);
|
|
CHECK(stale.wait_line("ready 0", 10s));
|
|
stale.send(input_lines(11, 12, false, 0xA2));
|
|
CHECK(stale.wait_line("rejected " + request_hex(11, 0xA2) + " not_ready", 10s));
|
|
CHECK(max_acked(stale) == 10);
|
|
const auto head = head_of(r.key);
|
|
CHECK(head.tick >= 20 && head.sequence >= 2 && !head.released);
|
|
stale.signal(SIGTERM);
|
|
CHECK(stale.wait_exit(20s) == 0); // It never regained authority: nothing to release.
|
|
owner_now.signal(SIGTERM);
|
|
CHECK(owner_now.wait_exit(20s) == 0);
|
|
CHECK(head_of(r.key).released && head_of(r.key).tick == head.tick);
|
|
}
|
|
|
|
// A committed head whose payload or tail cannot be verified is never served, and
|
|
// no older or empty world replaces it.
|
|
void damaged_heads_never_become_ready() {
|
|
const auto r = provision(47, 0x63);
|
|
{
|
|
Child server{server_args(r)};
|
|
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
|
server.send(input_lines(1, 10, false));
|
|
CHECK(server.wait_line("ack " + request_hex(10) + " 10 ", 15s));
|
|
server.signal(SIGTERM);
|
|
CHECK(server.wait_exit(20s) == 0);
|
|
}
|
|
const auto head = head_of(r.key);
|
|
const auto m = head_manifest(r.key);
|
|
const auto payload = r.objects() / hex(m.payload_sha256);
|
|
const auto trace = r.objects() / hex(m.batch.trace_sha256);
|
|
const auto quarantined = [&] {
|
|
Child server{server_args(r)};
|
|
const auto code = server.wait_exit(20s);
|
|
return code == 4 && server.printed("phase quarantined") && !server.printed("ready 1");
|
|
};
|
|
const auto payload_bytes = read_file(payload);
|
|
auto flipped = payload_bytes;
|
|
flipped[flipped.size() / 2] = static_cast<char>(flipped[flipped.size() / 2] ^ 1);
|
|
write_file(payload, flipped);
|
|
CHECK(quarantined());
|
|
write_file(payload, payload_bytes);
|
|
const auto trace_bytes = read_file(trace);
|
|
fs::remove(trace);
|
|
CHECK(quarantined());
|
|
write_file(trace, trace_bytes);
|
|
const auto after = head_of(r.key);
|
|
CHECK(after.tick == head.tick && after.sequence == head.sequence); // Nothing replaced it.
|
|
successor_resumes(r, 4); // Epochs 2 and 3 were taken by the quarantined attempts.
|
|
}
|
|
|
|
// In-process: recovery replays the head's trace from its predecessor before any
|
|
// readiness; missing, damaged or non-reproducing evidence quarantines (tail).
|
|
void tail_verification_before_readiness() {
|
|
const auto r = provision(48, 0x01);
|
|
auto runtime = Postgres::connect(
|
|
{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("EVO_PG_RUNTIME_USER"), {}, {}},
|
|
5s);
|
|
CHECK(runtime);
|
|
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
|
|
CHECK(objects);
|
|
Sha256Digest build{};
|
|
build.fill(0xB1);
|
|
const std::array<Sha256Digest, 1> retained{build};
|
|
const auto commit = [](std::uint8_t v) {
|
|
Uuid id{};
|
|
id.fill(v);
|
|
return id;
|
|
};
|
|
auto lease = acquire_region_lease(**runtime, r.key, build, 20000ms, 5s);
|
|
CHECK(lease);
|
|
auto head = load_region_checkpoint(**runtime, r.key, 5s);
|
|
CHECK(head);
|
|
auto state = evo::test::lineage_region();
|
|
evo::persistence::RegionBatch<evo::processes::LineageRegion> batch{*state};
|
|
for (int n = 0; n < 2; ++n) {
|
|
for (int i = 0; i < 10; ++i)
|
|
CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1)));
|
|
batch.seal();
|
|
auto published =
|
|
publish_lineage_batch(**runtime, **objects, *lease, *head, batch,
|
|
commit(static_cast<std::uint8_t>(2 + n)), build, 8000000, 5s);
|
|
CHECK(published && batch.confirm(published->committed.head.end_tick));
|
|
head = published->committed;
|
|
}
|
|
const auto recover = [&] {
|
|
return recover_lineage_checkpoint(**runtime, **objects, r.key, retained, 8000000, 5s);
|
|
};
|
|
CHECK(recover());
|
|
const auto m = *decode_manifest(head->manifest);
|
|
const auto trace = r.objects() / hex(m.batch.trace_sha256);
|
|
const auto trace_bytes = read_file(trace);
|
|
const auto refused = [&](RecoveryError kind) {
|
|
auto rec = recover();
|
|
return !rec && rec.error().kind == kind;
|
|
};
|
|
fs::remove(trace);
|
|
CHECK(refused(RecoveryError::tail));
|
|
auto damaged = trace_bytes;
|
|
damaged[200] = static_cast<char>(damaged[200] ^ 1);
|
|
write_file(trace, damaged);
|
|
CHECK(refused(RecoveryError::tail));
|
|
write_file(trace, trace_bytes);
|
|
// The predecessor's payload is needed to replay the head: its loss quarantines.
|
|
auto first = load_region_commit(**runtime, r.key, 1, 5s);
|
|
CHECK(first);
|
|
const auto previous = *decode_manifest(first->manifest);
|
|
const auto base = r.objects() / hex(previous.payload_sha256);
|
|
const auto base_bytes = read_file(base);
|
|
fs::remove(base);
|
|
CHECK(refused(RecoveryError::tail));
|
|
write_file(base, base_bytes);
|
|
CHECK(recover());
|
|
|
|
// Evidence that does not reproduce the payload: a trace of different inputs over
|
|
// the same range, published by bypassing the host (the database cannot tell).
|
|
for (int i = 0; i < 10; ++i)
|
|
CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1)));
|
|
auto alternate = recover();
|
|
CHECK(alternate);
|
|
evo::persistence::RegionBatch<evo::processes::LineageRegion> other{*alternate->state};
|
|
for (int i = 0; i < 10; ++i) {
|
|
auto in = evo::test::region_input(other.end_tick() + 1);
|
|
in.source_enabled = 1 - in.source_enabled; // Not what actually happened.
|
|
CHECK(other.advance(in));
|
|
}
|
|
auto payload = state->snapshot();
|
|
auto forged = encode_trace({2, r.key, m.commit, head->head.digest, m.end_tick, m.end_tick + 10},
|
|
other.steps());
|
|
CHECK(payload && forged && (*objects)->put(*payload) && (*objects)->put(*forged));
|
|
auto manifest = successor_manifest(m, head->head.digest, commit(5), *lease, m.end_tick + 10,
|
|
build, payload->size(), *sha256(*payload),
|
|
{forged->size(), *sha256(*forged), 0, {}});
|
|
CHECK(manifest);
|
|
CHECK(publish_region_checkpoint(**runtime, *encode_manifest(*manifest), {}, 5s));
|
|
CHECK(refused(RecoveryError::tail));
|
|
CHECK(release_region_lease(**runtime, r.key, *lease, 5s));
|
|
}
|
|
} // namespace
|
|
|
|
void run_region_recovery_tests() {
|
|
for (const auto& b :
|
|
{Boundary{"before-transaction", 1, 41, 0x5D}, Boundary{"before-upload", 2, 42, 0x5E},
|
|
Boundary{"before-transaction", 2, 43, 0x5F}, Boundary{"after-commit", 2, 44, 0x60},
|
|
Boundary{"after-ack", 2, 45, 0x61}})
|
|
kill_at(b);
|
|
resumed_stale_owner_is_fenced();
|
|
damaged_heads_never_become_ready();
|
|
tail_verification_before_readiness();
|
|
}
|