// Real PostgreSQL + filesystem M7.4 checkpoint publication/recovery (ADR-0061). #include "evo/persistence/checkpoint_store.hpp" #include "evo/persistence/sha256.hpp" #include "fixtures/environmental_region.hpp" #include #include #include #include #include #include #include #include #include #include void run_region_checkpoint_store_tests(); namespace { using namespace evo::persistence; using namespace std::chrono_literals; namespace fs = std::filesystem; void check(bool value, std::source_location at = std::source_location::current()) { if (!value) throw std::runtime_error("region checkpoint store test line " + std::to_string(at.line())); } #define CHECK(...) check(static_cast(__VA_ARGS__)) std::string env(const char* key) { const auto* value = std::getenv(key); if (!value) throw std::runtime_error("missing test environment"); return value; } std::unique_ptr connect_as(const std::string& user) { PgOptions options{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}}; auto made = Postgres::connect(options, 5s); CHECK(made); return std::move(*made); } template std::array filled(std::uint8_t value) { std::array out{}; out.fill(value); return out; } std::string hex(std::span bytes) { constexpr char digits[] = "0123456789abcdef"; std::string out; for (const auto b : bytes) { out.push_back(digits[b >> 4U]); out.push_back(digits[b & 0x0FU]); } return out; } std::vector bytes_of(std::string_view s) { return {s.begin(), s.end()}; } // Private scratch root; the CI script runs each binary under its own /tmp dir. fs::path scratch(const char* name) { const auto root = fs::path(env("PGHOST")) / name; fs::remove_all(root); fs::create_directory(root); fs::permissions(root, fs::perms::owner_all, fs::perm_options::replace); return root; } const Sha256Digest build = filled<32>(0xB1); constexpr std::size_t budget = 8000000; void sha256_vectors_and_database_agreement() { // FIPS 180-2 / NIST example vectors. CHECK(hex(*sha256({})) == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); CHECK(hex(*sha256(bytes_of("abc"))) == "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); CHECK(hex(*sha256(bytes_of("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"))) == "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); // Independent implementation: PostgreSQL computes the commit digest. auto db = connect_as(env("PGUSER")); std::vector data(2000); // Hex parameter stays within the 4KiB adapter bound. for (std::size_t i = 0; i < data.size(); ++i) data[i] = static_cast((i * 131U) ^ (i >> 3U)); const std::array p{"\\x" + hex(data)}; auto rows = db->query("SELECT encode(sha256($1::bytea), 'hex')", p, 5s); CHECK(rows && (*rows)[0][0] == hex(*sha256(data))); } void object_store_is_immutable_and_verified() { const auto root = scratch("objects"); CHECK(!FilesystemObjectStore::open(root / "absent", {1, 1}, 1024)); CHECK(!FilesystemObjectStore::open(root, {0, 1}, 1024)); fs::create_directory_symlink(root, root.parent_path() / "objects-link"); CHECK(!FilesystemObjectStore::open(root.parent_path() / "objects-link", {1, 1}, 1024)); fs::remove(root.parent_path() / "objects-link"); auto store = FilesystemObjectStore::open(root, {1, 2}, 1024); CHECK(store); const auto dir = (*store)->directory(); CHECK(dir == root / "w1" / "r2"); CHECK((fs::status(dir).permissions() & fs::perms::all) == fs::perms::owner_all); const auto payload = bytes_of("immutable payload bytes"); auto put = (*store)->put(payload); CHECK(put && put->bytes == payload.size() && put->sha256 == *sha256(payload)); auto again = (*store)->put(payload); // Idempotent for identical bytes. CHECK(again && *again == *put); auto got = (*store)->get(*put); CHECK(got && *got == payload); const auto file = dir / hex(put->sha256); CHECK((fs::status(file).permissions() & fs::perms::all) == fs::perms::owner_read || (fs::status(file).permissions() & fs::perms::all) == (fs::perms::owner_read | fs::perms::owner_write)); CHECK((*store)->put({}).error() == ObjectError::invalid); CHECK((*store)->put(std::vector(1025)).error() == ObjectError::bounds); CHECK((*store)->get({put->sha256, 1025}).error() == ObjectError::bounds); CHECK((*store)->get({filled<32>(7), 5}).error() == ObjectError::missing); CHECK((*store)->get({put->sha256, put->bytes - 1}).error() == ObjectError::corrupt); // A crash mid-upload leaves only a temporary name no key can resolve. std::ofstream(dir / (".tmp-" + hex(filled<32>(9)) + "-1-0")) << "partial"; CHECK((*store)->get({filled<32>(9), 7}).error() == ObjectError::missing); // Corruption is detected, and never repaired by overwriting. { std::fstream f(file, std::ios::in | std::ios::out | std::ios::binary); f.seekp(3); f.put('X'); } CHECK((*store)->get(*put).error() == ObjectError::corrupt); CHECK((*store)->put(payload).error() == ObjectError::corrupt); fs::remove(file); fs::create_symlink(dir / "elsewhere", file); CHECK((*store)->get(*put).error() == ObjectError::corrupt); fs::remove(file); CHECK((*store)->put(payload) && (*store)->get(*put)); fs::resize_file(file, payload.size() + 1); CHECK((*store)->get(*put).error() == ObjectError::corrupt); fs::remove(file); if (::geteuid() != 0) { // Permission faults cannot be simulated as root. fs::permissions(dir, fs::perms::owner_read | fs::perms::owner_exec, fs::perm_options::replace); CHECK((*store)->put(payload).error() == ObjectError::io); fs::permissions(dir, fs::perms::owner_all, fs::perm_options::replace); } } struct Region { RegionKey key{7, 3}; std::unique_ptr owner, runtime; std::unique_ptr objects; }; Region provisioned(const char* name, std::uint64_t region, std::uint8_t genesis_seed) { Region r; r.key.region = region; r.owner = connect_as(env("PGUSER")); r.runtime = connect_as(env("EVO_PG_RUNTIME_USER")); auto objects = FilesystemObjectStore::open(scratch(name), r.key, max_environmental_payload_bytes); CHECK(objects); r.objects = std::move(*objects); CHECK(r.owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " "(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING", {}, 5s)); auto opening = evo::test::region_community(); auto genesis = provision_environmental_region(*r.owner, *r.objects, r.key, evo::test::region_identity.frame, *opening, filled<16>(genesis_seed), build, budget, 5s); CHECK(genesis && genesis->head.sequence == 0); return r; } std::size_t count_objects(const fs::path& dir) { std::size_t n = 0; for (const auto& entry : fs::directory_iterator(dir)) n += entry.path().filename().string().starts_with(".tmp-") ? 0 : 1; return n; } void publish_recover_and_continue() { auto r = provisioned("publish", 3, 0xC0); const std::array retained{build}; auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s); CHECK(lease); // Genesis recovers to the exact opening state. auto recovered = recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s); CHECK(recovered && recovered->state->progress().tick == 0); auto head = recovered->committed; // Genesis cannot be re-provisioned or published over. auto opening = evo::test::region_community(); CHECK(!provision_environmental_region(*r.owner, *r.objects, r.key, 9, *opening, filled<16>(0xC9), build, budget, 5s)); auto original = evo::test::region_community(); std::uint8_t seed = 0xC1; for (int batch = 0; batch < 3; ++batch) { evo::test::advance_region(*original, batch == 1 ? 64 : 16); const auto started = std::chrono::steady_clock::now(); auto published = publish_environmental_checkpoint( *r.runtime, *r.objects, *lease, head, *original, filled<16>(seed++), build, budget, 5s); const auto us = std::chrono::duration_cast( std::chrono::steady_clock::now() - started) .count(); CHECK(published && published->head.sequence == head.head.sequence + 1); CHECK(published->head.end_tick == original->progress().tick); std::printf( "checkpoint_publish_us batch=%d ticks=%llu payload_bytes=%llu us=%lld\n", batch, static_cast(published->head.end_tick - head.head.end_tick), static_cast(decode_manifest(published->manifest)->payload_bytes), static_cast(us)); head = std::move(*published); } // 65 ticks is more than one batch: refused before upload. const auto before = count_objects(r.objects->directory()); evo::test::advance_region(*original, 65); auto oversized = publish_environmental_checkpoint( *r.runtime, *r.objects, *lease, head, *original, filled<16>(0xCA), build, budget, 5s); CHECK(!oversized && oversized.error().kind == PublicationError::candidate); CHECK(count_objects(r.objects->directory()) == before); // A new process recovers the committed head; its trajectory matches the original's. auto second = FilesystemObjectStore::open(r.objects->directory().parent_path().parent_path(), r.key, max_environmental_payload_bytes); CHECK(second); auto restored = recover_environmental_checkpoint(*r.runtime, **second, r.key, retained, budget, 5s); CHECK(restored && restored->committed.head == head.head); CHECK(restored->state->progress().tick == 96); auto reference = evo::test::region_community(); evo::test::advance_region(*reference, 96); CHECK(reference->snapshot() == restored->state->snapshot()); for (std::uint64_t tick = 97; tick <= 140; ++tick) { auto a = reference->advance(tick, evo::test::region_input(tick, true)); auto b = restored->state->advance(tick, evo::test::region_input(tick, true)); CHECK(a && b && *a == *b); } CHECK(reference->snapshot() == restored->state->snapshot()); // Retained-build allowlist: a host without this build cannot restore it. const std::array other{filled<32>(0xB2)}; auto foreign = recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, other, budget, 5s); CHECK(!foreign && foreign.error().kind == RecoveryError::unsupported); // Unretained build is refused by the database; the upload stays an orphan. auto next = evo::test::region_community(); evo::test::advance_region(*next, 100); auto refused = publish_environmental_checkpoint(*r.runtime, *r.objects, *lease, head, *next, filled<16>(0xCB), filled<32>(0xB2), budget, 5s); CHECK(!refused && refused.error().kind == PublicationError::rejected && refused.error().store.kind == LeaseStoreError::unsupported); CHECK(release_region_lease(*r.runtime, r.key, *lease, 5s)); } void failures_leave_the_previous_head() { auto r = provisioned("failures", 4, 0xD0); const std::array retained{build}; auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s); CHECK(lease); auto head = load_region_checkpoint(*r.runtime, r.key, 5s); CHECK(head); auto state = evo::test::region_community(); evo::test::advance_region(*state, 8); auto first = publish_environmental_checkpoint(*r.runtime, *r.objects, *lease, *head, *state, filled<16>(0xD1), build, budget, 5s); CHECK(first); const auto manifest = *decode_manifest(first->manifest); const auto object = r.objects->directory() / hex(manifest.payload_sha256); // Upload failure: no manifest is sent; the head does not move. evo::test::advance_region(*state, 8); if (::geteuid() != 0) { fs::permissions(r.objects->directory(), fs::perms::owner_read | fs::perms::owner_exec, fs::perm_options::replace); auto upload = publish_environmental_checkpoint(*r.runtime, *r.objects, *lease, *first, *state, filled<16>(0xD2), build, budget, 5s); fs::permissions(r.objects->directory(), fs::perms::owner_all, fs::perm_options::replace); CHECK(!upload && upload.error().kind == PublicationError::upload); CHECK(load_region_head(*r.runtime, r.key, 5s)->sequence == 1); } // Manifest transaction failure after a verified upload: fenced writer, orphan retained. auto stale = *lease; stale.holder[0] ^= 1U; auto fenced = publish_environmental_checkpoint(*r.runtime, *r.objects, stale, *first, *state, filled<16>(0xD3), build, budget, 5s); CHECK(!fenced && fenced.error().kind == PublicationError::rejected && fenced.error().store.kind == LeaseStoreError::fenced); auto orphan_bytes = state->snapshot(); auto orphan = r.objects->get({*sha256(*orphan_bytes), orphan_bytes->size()}); CHECK(orphan && *orphan == *orphan_bytes); // Uploaded, verified, unreferenced. auto recovered = recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s); CHECK(recovered && recovered->state->progress().tick == 8); // Never the orphan. // Missing and corrupt committed payloads quarantine; no older head is selected. const auto saved = r.objects->get({manifest.payload_sha256, manifest.payload_bytes}); CHECK(saved); fs::remove(object); auto missing = recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s); CHECK(!missing && missing.error().kind == RecoveryError::missing_payload); CHECK(r.objects->put(*saved)); fs::permissions(object, fs::perms::owner_all, fs::perm_options::replace); { std::fstream f(object, std::ios::in | std::ios::out | std::ios::binary); f.seekp(100); f.put(static_cast((*saved)[100] ^ 1U)); } auto corrupt = recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s); CHECK(!corrupt && corrupt.error().kind == RecoveryError::corrupt_payload); fs::remove(object); CHECK(r.objects->put(*saved)); CHECK(recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s)); // The database refuses tampering with committed manifests even by the owner. auto tamper = r.owner->query("UPDATE evolution.region_commits SET manifest = " "overlay(manifest placing '\\x02'::bytea from 121 for 1) " "WHERE world_id = 7 AND region_id = 4 AND commit_sequence = 1", {}, 5s); CHECK(!tamper && std::string(tamper.error().sqlstate.data()) == "23514"); auto repoint = r.owner->query("UPDATE evolution.region_heads SET digest = sha256('\\x00') " "WHERE world_id = 7 AND region_id = 4", {}, 5s); CHECK(!repoint && std::string(repoint.error().sqlstate.data()) == "23503"); // Defense in depth: even if that constraint were lost, recovery re-digests the // manifest and quarantines. Restores the constraint before continuing. CHECK(r.owner->query("ALTER TABLE evolution.region_commits DROP CONSTRAINT " "region_commits_manifest_digest", {}, 5s)); // Flip a payload-digest byte: only the manifest digest check can notice it. CHECK(r.owner->query("UPDATE evolution.region_commits SET manifest = " "set_byte(manifest, 248, get_byte(manifest, 248) # 1) " "WHERE world_id = 7 AND region_id = 4 AND commit_sequence = 1", {}, 5s)); auto tampered = recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s); CHECK(!tampered && tampered.error().kind == RecoveryError::manifest); const std::array original{"\\x" + hex(first->manifest)}; CHECK(r.owner->query("UPDATE evolution.region_commits SET manifest = $1::bytea " "WHERE world_id = 7 AND region_id = 4 AND commit_sequence = 1", original, 5s)); CHECK(r.owner->query("ALTER TABLE evolution.region_commits ADD CONSTRAINT " "region_commits_manifest_digest CHECK (commit_digest = sha256(manifest))", {}, 5s)); CHECK(recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s)); // Recovery does not create genesis for an unknown region. auto none = recover_environmental_checkpoint(*r.runtime, *r.objects, {7, 99}, retained, budget, 5s); CHECK(!none && none.error().kind == RecoveryError::not_provisioned); } } // namespace void run_region_checkpoint_store_tests() { sha256_vectors_and_database_agreement(); object_store_is_immutable_and_verified(); publish_recover_and_continue(); failures_leave_the_previous_head(); }