Files
aevolution/tests/region_checkpoint_store_tests.cpp
T
dingoandClaude Opus 5.5 5d4f59f901 Implement M7.4a checkpoint publication and verified recovery
ADR-0061 publishes environmental-v1 full checkpoints. Manifest v1 is a
fixed 280-byte record (identity, chain, lease, versions, build, zero
RNG/history declarations, durability policy, payload length/SHA-256)
that PostgreSQL parses and digests itself. Schema v3 adds region_commits
with a digest CHECK and a composite head FK, owner-managed retained
builds, fenced publish/load functions and owner-only genesis; it drops
the bare v2 head kernel and fails closed while v2 heads exist.

Linux evo::region_store adds OpenSSL SHA-256, an immutable
content-addressed filesystem object store (temp+fsync+link, no
overwrite/delete, O_NOFOLLOW exact-length verified reads), and
publication (validate -> upload -> read back -> fenced transaction ->
receipt check) and recovery (re-digest, allowlist, verify, full restore).
Portable evo::region_checkpoint holds the codec and payload validation.

Validated: Windows CTest 56/56; WSL Release, clang-tidy and ASan/UBSan
private-PostgreSQL suites; four SQL and one host mutation each caught.
M7.4 stays open for 7.4b (lineage archive codec and history binding).
No Docker/Gitea CI, remote store, process kill or deployment.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-28 15:25:27 -04:00

358 lines
18 KiB
C++

// Real PostgreSQL + filesystem M7.4 checkpoint publication/recovery (ADR-0061).
#include "evo/persistence/checkpoint_store.hpp"
#include "evo/persistence/sha256.hpp"
#include "fixtures/environmental_region.hpp"
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <source_location>
#include <stdexcept>
#include <string>
#include <sys/stat.h>
#include <unistd.h>
void run_region_checkpoint_store_tests();
namespace {
using namespace evo::persistence;
using namespace std::chrono_literals;
namespace fs = std::filesystem;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region checkpoint store test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
std::string env(const char* key) {
const auto* value = std::getenv(key);
if (!value)
throw std::runtime_error("missing test environment");
return value;
}
std::unique_ptr<Postgres> connect_as(const std::string& user) {
PgOptions options{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}};
auto made = Postgres::connect(options, 5s);
CHECK(made);
return std::move(*made);
}
template <std::size_t N> std::array<std::uint8_t, N> filled(std::uint8_t value) {
std::array<std::uint8_t, N> out{};
out.fill(value);
return out;
}
std::string hex(std::span<const std::uint8_t> bytes) {
constexpr char digits[] = "0123456789abcdef";
std::string out;
for (const auto b : bytes) {
out.push_back(digits[b >> 4U]);
out.push_back(digits[b & 0x0FU]);
}
return out;
}
std::vector<std::uint8_t> bytes_of(std::string_view s) {
return {s.begin(), s.end()};
}
// Private scratch root; the CI script runs each binary under its own /tmp dir.
fs::path scratch(const char* name) {
const auto root = fs::path(env("PGHOST")) / name;
fs::remove_all(root);
fs::create_directory(root);
fs::permissions(root, fs::perms::owner_all, fs::perm_options::replace);
return root;
}
const Sha256Digest build = filled<32>(0xB1);
constexpr std::size_t budget = 8000000;
void sha256_vectors_and_database_agreement() {
// FIPS 180-2 / NIST example vectors.
CHECK(hex(*sha256({})) == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
CHECK(hex(*sha256(bytes_of("abc"))) ==
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
CHECK(hex(*sha256(bytes_of("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"))) ==
"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1");
// Independent implementation: PostgreSQL computes the commit digest.
auto db = connect_as(env("PGUSER"));
std::vector<std::uint8_t> data(2000); // Hex parameter stays within the 4KiB adapter bound.
for (std::size_t i = 0; i < data.size(); ++i)
data[i] = static_cast<std::uint8_t>((i * 131U) ^ (i >> 3U));
const std::array<std::string, 1> p{"\\x" + hex(data)};
auto rows = db->query("SELECT encode(sha256($1::bytea), 'hex')", p, 5s);
CHECK(rows && (*rows)[0][0] == hex(*sha256(data)));
}
void object_store_is_immutable_and_verified() {
const auto root = scratch("objects");
CHECK(!FilesystemObjectStore::open(root / "absent", {1, 1}, 1024));
CHECK(!FilesystemObjectStore::open(root, {0, 1}, 1024));
fs::create_directory_symlink(root, root.parent_path() / "objects-link");
CHECK(!FilesystemObjectStore::open(root.parent_path() / "objects-link", {1, 1}, 1024));
fs::remove(root.parent_path() / "objects-link");
auto store = FilesystemObjectStore::open(root, {1, 2}, 1024);
CHECK(store);
const auto dir = (*store)->directory();
CHECK(dir == root / "w1" / "r2");
CHECK((fs::status(dir).permissions() & fs::perms::all) == fs::perms::owner_all);
const auto payload = bytes_of("immutable payload bytes");
auto put = (*store)->put(payload);
CHECK(put && put->bytes == payload.size() && put->sha256 == *sha256(payload));
auto again = (*store)->put(payload); // Idempotent for identical bytes.
CHECK(again && *again == *put);
auto got = (*store)->get(*put);
CHECK(got && *got == payload);
const auto file = dir / hex(put->sha256);
CHECK((fs::status(file).permissions() & fs::perms::all) == fs::perms::owner_read ||
(fs::status(file).permissions() & fs::perms::all) ==
(fs::perms::owner_read | fs::perms::owner_write));
CHECK((*store)->put({}).error() == ObjectError::invalid);
CHECK((*store)->put(std::vector<std::uint8_t>(1025)).error() == ObjectError::bounds);
CHECK((*store)->get({put->sha256, 1025}).error() == ObjectError::bounds);
CHECK((*store)->get({filled<32>(7), 5}).error() == ObjectError::missing);
CHECK((*store)->get({put->sha256, put->bytes - 1}).error() == ObjectError::corrupt);
// A crash mid-upload leaves only a temporary name no key can resolve.
std::ofstream(dir / (".tmp-" + hex(filled<32>(9)) + "-1-0")) << "partial";
CHECK((*store)->get({filled<32>(9), 7}).error() == ObjectError::missing);
// Corruption is detected, and never repaired by overwriting.
{
std::fstream f(file, std::ios::in | std::ios::out | std::ios::binary);
f.seekp(3);
f.put('X');
}
CHECK((*store)->get(*put).error() == ObjectError::corrupt);
CHECK((*store)->put(payload).error() == ObjectError::corrupt);
fs::remove(file);
fs::create_symlink(dir / "elsewhere", file);
CHECK((*store)->get(*put).error() == ObjectError::corrupt);
fs::remove(file);
CHECK((*store)->put(payload) && (*store)->get(*put));
fs::resize_file(file, payload.size() + 1);
CHECK((*store)->get(*put).error() == ObjectError::corrupt);
fs::remove(file);
if (::geteuid() != 0) { // Permission faults cannot be simulated as root.
fs::permissions(dir, fs::perms::owner_read | fs::perms::owner_exec,
fs::perm_options::replace);
CHECK((*store)->put(payload).error() == ObjectError::io);
fs::permissions(dir, fs::perms::owner_all, fs::perm_options::replace);
}
}
struct Region {
RegionKey key{7, 3};
std::unique_ptr<Postgres> owner, runtime;
std::unique_ptr<FilesystemObjectStore> objects;
};
Region provisioned(const char* name, std::uint64_t region, std::uint8_t genesis_seed) {
Region r;
r.key.region = region;
r.owner = connect_as(env("PGUSER"));
r.runtime = connect_as(env("EVO_PG_RUNTIME_USER"));
auto objects =
FilesystemObjectStore::open(scratch(name), r.key, max_environmental_payload_bytes);
CHECK(objects);
r.objects = std::move(*objects);
CHECK(r.owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
{}, 5s));
auto opening = evo::test::region_community();
auto genesis = provision_environmental_region(*r.owner, *r.objects, r.key,
evo::test::region_identity.frame, *opening,
filled<16>(genesis_seed), build, budget, 5s);
CHECK(genesis && genesis->head.sequence == 0);
return r;
}
std::size_t count_objects(const fs::path& dir) {
std::size_t n = 0;
for (const auto& entry : fs::directory_iterator(dir))
n += entry.path().filename().string().starts_with(".tmp-") ? 0 : 1;
return n;
}
void publish_recover_and_continue() {
auto r = provisioned("publish", 3, 0xC0);
const std::array<Sha256Digest, 1> retained{build};
auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s);
CHECK(lease);
// Genesis recovers to the exact opening state.
auto recovered =
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s);
CHECK(recovered && recovered->state->progress().tick == 0);
auto head = recovered->committed;
// Genesis cannot be re-provisioned or published over.
auto opening = evo::test::region_community();
CHECK(!provision_environmental_region(*r.owner, *r.objects, r.key, 9, *opening,
filled<16>(0xC9), build, budget, 5s));
auto original = evo::test::region_community();
std::uint8_t seed = 0xC1;
for (int batch = 0; batch < 3; ++batch) {
evo::test::advance_region(*original, batch == 1 ? 64 : 16);
const auto started = std::chrono::steady_clock::now();
auto published = publish_environmental_checkpoint(
*r.runtime, *r.objects, *lease, head, *original, filled<16>(seed++), build, budget, 5s);
const auto us = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now() - started)
.count();
CHECK(published && published->head.sequence == head.head.sequence + 1);
CHECK(published->head.end_tick == original->progress().tick);
std::printf(
"checkpoint_publish_us batch=%d ticks=%llu payload_bytes=%llu us=%lld\n", batch,
static_cast<unsigned long long>(published->head.end_tick - head.head.end_tick),
static_cast<unsigned long long>(decode_manifest(published->manifest)->payload_bytes),
static_cast<long long>(us));
head = std::move(*published);
}
// 65 ticks is more than one batch: refused before upload.
const auto before = count_objects(r.objects->directory());
evo::test::advance_region(*original, 65);
auto oversized = publish_environmental_checkpoint(
*r.runtime, *r.objects, *lease, head, *original, filled<16>(0xCA), build, budget, 5s);
CHECK(!oversized && oversized.error().kind == PublicationError::candidate);
CHECK(count_objects(r.objects->directory()) == before);
// A new process recovers the committed head; its trajectory matches the original's.
auto second = FilesystemObjectStore::open(r.objects->directory().parent_path().parent_path(),
r.key, max_environmental_payload_bytes);
CHECK(second);
auto restored =
recover_environmental_checkpoint(*r.runtime, **second, r.key, retained, budget, 5s);
CHECK(restored && restored->committed.head == head.head);
CHECK(restored->state->progress().tick == 96);
auto reference = evo::test::region_community();
evo::test::advance_region(*reference, 96);
CHECK(reference->snapshot() == restored->state->snapshot());
for (std::uint64_t tick = 97; tick <= 140; ++tick) {
auto a = reference->advance(tick, evo::test::region_input(tick, true));
auto b = restored->state->advance(tick, evo::test::region_input(tick, true));
CHECK(a && b && *a == *b);
}
CHECK(reference->snapshot() == restored->state->snapshot());
// Retained-build allowlist: a host without this build cannot restore it.
const std::array<Sha256Digest, 1> other{filled<32>(0xB2)};
auto foreign =
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, other, budget, 5s);
CHECK(!foreign && foreign.error().kind == RecoveryError::unsupported);
// Unretained build is refused by the database; the upload stays an orphan.
auto next = evo::test::region_community();
evo::test::advance_region(*next, 100);
auto refused = publish_environmental_checkpoint(*r.runtime, *r.objects, *lease, head, *next,
filled<16>(0xCB), filled<32>(0xB2), budget, 5s);
CHECK(!refused && refused.error().kind == PublicationError::rejected &&
refused.error().store.kind == LeaseStoreError::unsupported);
CHECK(release_region_lease(*r.runtime, r.key, *lease, 5s));
}
void failures_leave_the_previous_head() {
auto r = provisioned("failures", 4, 0xD0);
const std::array<Sha256Digest, 1> retained{build};
auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s);
CHECK(lease);
auto head = load_region_checkpoint(*r.runtime, r.key, 5s);
CHECK(head);
auto state = evo::test::region_community();
evo::test::advance_region(*state, 8);
auto first = publish_environmental_checkpoint(*r.runtime, *r.objects, *lease, *head, *state,
filled<16>(0xD1), build, budget, 5s);
CHECK(first);
const auto manifest = *decode_manifest(first->manifest);
const auto object = r.objects->directory() / hex(manifest.payload_sha256);
// Upload failure: no manifest is sent; the head does not move.
evo::test::advance_region(*state, 8);
if (::geteuid() != 0) {
fs::permissions(r.objects->directory(), fs::perms::owner_read | fs::perms::owner_exec,
fs::perm_options::replace);
auto upload = publish_environmental_checkpoint(*r.runtime, *r.objects, *lease, *first,
*state, filled<16>(0xD2), build, budget, 5s);
fs::permissions(r.objects->directory(), fs::perms::owner_all, fs::perm_options::replace);
CHECK(!upload && upload.error().kind == PublicationError::upload);
CHECK(load_region_head(*r.runtime, r.key, 5s)->sequence == 1);
}
// Manifest transaction failure after a verified upload: fenced writer, orphan retained.
auto stale = *lease;
stale.holder[0] ^= 1U;
auto fenced = publish_environmental_checkpoint(*r.runtime, *r.objects, stale, *first, *state,
filled<16>(0xD3), build, budget, 5s);
CHECK(!fenced && fenced.error().kind == PublicationError::rejected &&
fenced.error().store.kind == LeaseStoreError::fenced);
auto orphan_bytes = state->snapshot();
auto orphan = r.objects->get({*sha256(*orphan_bytes), orphan_bytes->size()});
CHECK(orphan && *orphan == *orphan_bytes); // Uploaded, verified, unreferenced.
auto recovered =
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s);
CHECK(recovered && recovered->state->progress().tick == 8); // Never the orphan.
// Missing and corrupt committed payloads quarantine; no older head is selected.
const auto saved = r.objects->get({manifest.payload_sha256, manifest.payload_bytes});
CHECK(saved);
fs::remove(object);
auto missing =
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s);
CHECK(!missing && missing.error().kind == RecoveryError::missing_payload);
CHECK(r.objects->put(*saved));
fs::permissions(object, fs::perms::owner_all, fs::perm_options::replace);
{
std::fstream f(object, std::ios::in | std::ios::out | std::ios::binary);
f.seekp(100);
f.put(static_cast<char>((*saved)[100] ^ 1U));
}
auto corrupt =
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s);
CHECK(!corrupt && corrupt.error().kind == RecoveryError::corrupt_payload);
fs::remove(object);
CHECK(r.objects->put(*saved));
CHECK(recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s));
// The database refuses tampering with committed manifests even by the owner.
auto tamper = r.owner->query("UPDATE evolution.region_commits SET manifest = "
"overlay(manifest placing '\\x02'::bytea from 121 for 1) "
"WHERE world_id = 7 AND region_id = 4 AND commit_sequence = 1",
{}, 5s);
CHECK(!tamper && std::string(tamper.error().sqlstate.data()) == "23514");
auto repoint = r.owner->query("UPDATE evolution.region_heads SET digest = sha256('\\x00') "
"WHERE world_id = 7 AND region_id = 4",
{}, 5s);
CHECK(!repoint && std::string(repoint.error().sqlstate.data()) == "23503");
// Defense in depth: even if that constraint were lost, recovery re-digests the
// manifest and quarantines. Restores the constraint before continuing.
CHECK(r.owner->query("ALTER TABLE evolution.region_commits DROP CONSTRAINT "
"region_commits_manifest_digest",
{}, 5s));
// Flip a payload-digest byte: only the manifest digest check can notice it.
CHECK(r.owner->query("UPDATE evolution.region_commits SET manifest = "
"set_byte(manifest, 248, get_byte(manifest, 248) # 1) "
"WHERE world_id = 7 AND region_id = 4 AND commit_sequence = 1",
{}, 5s));
auto tampered =
recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s);
CHECK(!tampered && tampered.error().kind == RecoveryError::manifest);
const std::array<std::string, 1> original{"\\x" + hex(first->manifest)};
CHECK(r.owner->query("UPDATE evolution.region_commits SET manifest = $1::bytea "
"WHERE world_id = 7 AND region_id = 4 AND commit_sequence = 1",
original, 5s));
CHECK(r.owner->query("ALTER TABLE evolution.region_commits ADD CONSTRAINT "
"region_commits_manifest_digest CHECK (commit_digest = sha256(manifest))",
{}, 5s));
CHECK(recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s));
// Recovery does not create genesis for an unknown region.
auto none =
recover_environmental_checkpoint(*r.runtime, *r.objects, {7, 99}, retained, budget, 5s);
CHECK(!none && none.error().kind == RecoveryError::not_provisioned);
}
} // namespace
void run_region_checkpoint_store_tests() {
sha256_vectors_and_database_agreement();
object_store_is_immutable_and_verified();
publish_recover_and_continue();
failures_leave_the_previous_head();
}