// Real PostgreSQL + filesystem M7.4 checkpoint publication/recovery (ADR-0061). #include "evo/persistence/checkpoint_store.hpp" #include "evo/persistence/sha256.hpp" #include "fixtures/environmental_region.hpp" #include #include #include #include #include #include #include #include #include #include #include #include void run_region_checkpoint_store_tests(); namespace { using namespace evo::persistence; using namespace std::chrono_literals; namespace fs = std::filesystem; void check(bool value, std::source_location at = std::source_location::current()) { if (!value) throw std::runtime_error("region checkpoint store test line " + std::to_string(at.line())); } #define CHECK(...) check(static_cast(__VA_ARGS__)) std::string env(const char* key) { const auto* value = std::getenv(key); if (!value) throw std::runtime_error("missing test environment"); return value; } std::unique_ptr connect_as(const std::string& user) { PgOptions options{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}}; auto made = Postgres::connect(options, 5s); CHECK(made); return std::move(*made); } template std::array filled(std::uint8_t value) { std::array out{}; out.fill(value); return out; } std::string hex(std::span bytes) { constexpr char digits[] = "0123456789abcdef"; std::string out; for (const auto b : bytes) { out.push_back(digits[b >> 4U]); out.push_back(digits[b & 0x0FU]); } return out; } std::vector bytes_of(std::string_view s) { return {s.begin(), s.end()}; } // Private scratch root; the CI script runs each binary under its own /tmp dir. fs::path scratch(const char* name) { const auto root = fs::path(env("PGHOST")) / name; fs::remove_all(root); fs::create_directory(root); fs::permissions(root, fs::perms::owner_all, fs::perm_options::replace); return root; } const Sha256Digest build = filled<32>(0xB1); constexpr std::size_t budget = 8000000; void sha256_vectors_and_database_agreement() { // FIPS 180-2 / NIST example vectors. CHECK(hex(*sha256({})) == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); CHECK(hex(*sha256(bytes_of("abc"))) == "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); CHECK(hex(*sha256(bytes_of("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"))) == "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); // Independent implementation: PostgreSQL computes the commit digest. auto db = connect_as(env("PGUSER")); std::vector data(2000); // Hex parameter stays within the 4KiB adapter bound. for (std::size_t i = 0; i < data.size(); ++i) data[i] = static_cast((i * 131U) ^ (i >> 3U)); const std::array p{"\\x" + hex(data)}; auto rows = db->query("SELECT encode(sha256($1::bytea), 'hex')", p, 5s); CHECK(rows && (*rows)[0][0] == hex(*sha256(data))); } void object_store_is_immutable_and_verified() { const auto root = scratch("objects"); CHECK(!FilesystemObjectStore::open(root / "absent", {1, 1}, 1024)); CHECK(!FilesystemObjectStore::open(root, {0, 1}, 1024)); fs::create_directory_symlink(root, root.parent_path() / "objects-link"); CHECK(!FilesystemObjectStore::open(root.parent_path() / "objects-link", {1, 1}, 1024)); fs::remove(root.parent_path() / "objects-link"); auto store = FilesystemObjectStore::open(root, {1, 2}, 1024); CHECK(store); const auto dir = (*store)->directory(); CHECK(dir == root / "w1" / "r2"); CHECK((fs::status(dir).permissions() & fs::perms::all) == fs::perms::owner_all); const auto payload = bytes_of("immutable payload bytes"); auto put = (*store)->put(payload); CHECK(put && put->bytes == payload.size() && put->sha256 == *sha256(payload)); auto again = (*store)->put(payload); // Idempotent for identical bytes. CHECK(again && *again == *put); auto got = (*store)->get(*put); CHECK(got && *got == payload); const auto file = dir / hex(put->sha256); CHECK((fs::status(file).permissions() & fs::perms::all) == fs::perms::owner_read || (fs::status(file).permissions() & fs::perms::all) == (fs::perms::owner_read | fs::perms::owner_write)); CHECK((*store)->put({}).error() == ObjectError::invalid); CHECK((*store)->put(std::vector(1025)).error() == ObjectError::bounds); CHECK((*store)->get({put->sha256, 1025}).error() == ObjectError::bounds); CHECK((*store)->get({filled<32>(7), 5}).error() == ObjectError::missing); CHECK((*store)->get({put->sha256, put->bytes - 1}).error() == ObjectError::corrupt); // A crash mid-upload leaves only a temporary name no key can resolve. std::ofstream(dir / (".tmp-" + hex(filled<32>(9)) + "-1-0")) << "partial"; CHECK((*store)->get({filled<32>(9), 7}).error() == ObjectError::missing); // Corruption is detected, and never repaired by overwriting. { std::fstream f(file, std::ios::in | std::ios::out | std::ios::binary); f.seekp(3); f.put('X'); } CHECK((*store)->get(*put).error() == ObjectError::corrupt); CHECK((*store)->put(payload).error() == ObjectError::corrupt); fs::remove(file); fs::create_symlink(dir / "elsewhere", file); CHECK((*store)->get(*put).error() == ObjectError::corrupt); fs::remove(file); CHECK((*store)->put(payload) && (*store)->get(*put)); fs::resize_file(file, payload.size() + 1); CHECK((*store)->get(*put).error() == ObjectError::corrupt); fs::remove(file); if (::geteuid() != 0) { // Permission faults cannot be simulated as root. fs::permissions(dir, fs::perms::owner_read | fs::perms::owner_exec, fs::perm_options::replace); CHECK((*store)->put(payload).error() == ObjectError::io); fs::permissions(dir, fs::perms::owner_all, fs::perm_options::replace); } } // Review P1: a name made visible by an install whose directory sync failed is not // durable. Neither a retry that finds it nor a namespace reopen may report success // until a directory sync actually succeeds. void directory_sync_failures_are_never_acknowledged() { const auto root = scratch("syncs"); ObjectStoreFaults faults; faults.fail_directory_syncs = 1; // Creating w1 under the root. CHECK(FilesystemObjectStore::open(root, {1, 5}, 1024, &faults).error() == ObjectError::io); CHECK(fs::exists(root / "w1")); // Created, but its entry is unconfirmed. faults.fail_directory_syncs = 1; // Reopen finds it and must sync again. CHECK(FilesystemObjectStore::open(root, {1, 5}, 1024, &faults).error() == ObjectError::io); auto store = FilesystemObjectStore::open(root, {1, 5}, 1024, &faults); CHECK(store && faults.directory_syncs == 2); // root->w1 and w1->r5. const auto payload = bytes_of("durable only after a directory sync succeeds"); const VerifiedObject object{*sha256(payload), payload.size()}; faults.fail_directory_syncs = 1; CHECK((*store)->put(payload).error() == ObjectError::io); CHECK((*store)->get(object)); // Visible, yet never acknowledged. faults.fail_directory_syncs = 3; // Retries while the directory still cannot sync. for (int attempt = 0; attempt < 3; ++attempt) CHECK((*store)->put(payload).error() == ObjectError::io); CHECK(faults.directory_syncs == 2); auto retried = (*store)->put(payload); CHECK(retried && *retried == object && faults.directory_syncs == 3); // Every reuse re-establishes durability rather than trusting the visible name. CHECK((*store)->put(payload) && faults.directory_syncs == 4); } struct Region { RegionKey key{7, 3}; std::unique_ptr owner, runtime; std::unique_ptr objects; }; Region provisioned(const char* name, std::uint64_t region, std::uint8_t genesis_seed) { Region r; r.key.region = region; r.owner = connect_as(env("PGUSER")); r.runtime = connect_as(env("EVO_PG_RUNTIME_USER")); auto objects = FilesystemObjectStore::open(scratch(name), r.key, max_environmental_payload_bytes); CHECK(objects); r.objects = std::move(*objects); CHECK(r.owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " "(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING", {}, 5s)); auto opening = evo::test::region_community(); auto genesis = provision_environmental_region(*r.owner, *r.objects, r.key, evo::test::region_identity.frame, *opening, filled<16>(genesis_seed), build, budget, 5s); CHECK(genesis && genesis->head.sequence == 0); return r; } std::size_t count_objects(const fs::path& dir) { std::size_t n = 0; for (const auto& entry : fs::directory_iterator(dir)) n += entry.path().filename().string().starts_with(".tmp-") ? 0 : 1; return n; } using EnvBatch = RegionBatch; using LineageBatch = RegionBatch; // Runs up to `ticks` more ticks (stopping early if an outcome seals it), then flushes. template void run_batch(Batch& b, std::uint64_t ticks, bool deaths = false) { for (std::uint64_t i = 0; i < ticks && !b.sealed(); ++i) CHECK(b.advance(evo::test::region_input(b.end_tick() + 1, deaths))); b.seal(); } void publish_recover_and_continue() { auto r = provisioned("publish", 3, 0xC0); const std::array retained{build}; auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s); CHECK(lease); // Genesis recovers to the exact opening state. auto recovered = recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s); CHECK(recovered && recovered->state->progress().tick == 0 && recovered->decisions.empty()); auto head = recovered->committed; // Genesis cannot be re-provisioned or published over. auto opening = evo::test::region_community(); CHECK(!provision_environmental_region(*r.owner, *r.objects, r.key, 9, *opening, filled<16>(0xC9), build, budget, 5s)); auto original = evo::test::region_community(); EnvBatch batch{*original}; // An unsealed batch is still being simulated: refused before any upload. CHECK(batch.advance(evo::test::region_input(1))); const auto before = count_objects(r.objects->directory()); auto open = publish_environmental_batch(*r.runtime, *r.objects, *lease, head, batch, filled<16>(0xCA), build, budget, 5s); CHECK(!open && open.error().kind == PublicationError::candidate); CHECK(count_objects(r.objects->directory()) == before); std::uint8_t seed = 0xC1; for (int n = 0; n < 3; ++n) { run_batch(batch, n == 1 ? 64 : 15); const auto started = std::chrono::steady_clock::now(); auto published = publish_environmental_batch(*r.runtime, *r.objects, *lease, head, batch, filled<16>(seed++), build, budget, 5s); const auto us = std::chrono::duration_cast( std::chrono::steady_clock::now() - started) .count(); CHECK(published && published->committed.head.sequence == head.head.sequence + 1); CHECK(published->committed.head.end_tick == original->progress().tick); CHECK(published->decisions.empty()); const auto m = *decode_manifest(published->committed.manifest); CHECK(m.manifest_version == 2 && m.batch.trace_bytes == trace_bytes_for(m.end_tick - m.begin_tick)); std::printf("batch_publish_us batch=%d ticks=%llu payload_bytes=%llu us=%lld\n", n, static_cast(m.end_tick - m.begin_tick), static_cast(m.payload_bytes), static_cast(us)); CHECK(batch.confirm(m.end_tick)); head = std::move(published->committed); // The trace object is retained, bound to this link of the chain. auto trace = r.objects->get({m.batch.trace_sha256, m.batch.trace_bytes}); auto decoded = trace ? decode_trace(*trace) : std::unexpected{BatchError::format}; CHECK(decoded && decoded->header.begin_tick == m.begin_tick && decoded->header.predecessor_digest == m.predecessor_digest); } // State advanced outside the recorded batch cannot be published: the replay of // the recorded inputs no longer reproduces it. auto drifted = evo::test::region_community(); EnvBatch drift{*drifted}; run_batch(drift, 3); CHECK(drifted->advance(4, evo::test::region_input(4))); auto diverged = publish_environmental_batch(*r.runtime, *r.objects, *lease, recovered->committed, drift, filled<16>(0xCD), build, budget, 5s); CHECK(!diverged && diverged.error().kind == PublicationError::candidate); // A batch replayed from the wrong predecessor cannot reproduce its state. run_batch(batch, 4); auto stale = publish_environmental_batch(*r.runtime, *r.objects, *lease, recovered->committed, batch, filled<16>(0xCB), build, budget, 5s); CHECK(!stale && stale.error().kind == PublicationError::candidate); // A new process recovers the committed head; its trajectory matches the original's. auto second = FilesystemObjectStore::open(r.objects->directory().parent_path().parent_path(), r.key, max_environmental_payload_bytes); CHECK(second); auto restored = recover_environmental_checkpoint(*r.runtime, **second, r.key, retained, budget, 5s); CHECK(restored && restored->committed.head == head.head); CHECK(restored->state->progress().tick == 95); auto reference = evo::test::region_community(); evo::test::advance_region(*reference, 95); CHECK(reference->snapshot() == restored->state->snapshot()); for (std::uint64_t tick = 96; tick <= 140; ++tick) { auto a = reference->advance(tick, evo::test::region_input(tick, true)); auto b = restored->state->advance(tick, evo::test::region_input(tick, true)); CHECK(a && b && *a == *b); } CHECK(reference->snapshot() == restored->state->snapshot()); // Retained-build allowlist: a host without this build cannot restore it. const std::array other{filled<32>(0xB2)}; auto foreign = recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, other, budget, 5s); CHECK(!foreign && foreign.error().kind == RecoveryError::unsupported); // Unretained build is refused by the database; the uploads stay orphans. auto refused = publish_environmental_batch(*r.runtime, *r.objects, *lease, head, batch, filled<16>(0xCC), filled<32>(0xB2), budget, 5s); CHECK(!refused && refused.error().kind == PublicationError::rejected && refused.error().store.kind == LeaseStoreError::unsupported); CHECK(release_region_lease(*r.runtime, r.key, *lease, 5s)); } void failures_leave_the_previous_head() { auto r = provisioned("failures", 4, 0xD0); const std::array retained{build}; auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s); CHECK(lease); auto head = load_region_checkpoint(*r.runtime, r.key, 5s); CHECK(head); auto state = evo::test::region_community(); EnvBatch batch{*state}; run_batch(batch, 8); auto first = publish_environmental_batch(*r.runtime, *r.objects, *lease, *head, batch, filled<16>(0xD1), build, budget, 5s); CHECK(first && batch.confirm(8)); const auto manifest = *decode_manifest(first->committed.manifest); const auto object = r.objects->directory() / hex(manifest.payload_sha256); // Upload failure: no manifest is sent; the head does not move. run_batch(batch, 8); if (::geteuid() != 0) { fs::permissions(r.objects->directory(), fs::perms::owner_read | fs::perms::owner_exec, fs::perm_options::replace); auto upload = publish_environmental_batch(*r.runtime, *r.objects, *lease, first->committed, batch, filled<16>(0xD2), build, budget, 5s); fs::permissions(r.objects->directory(), fs::perms::owner_all, fs::perm_options::replace); CHECK(!upload && upload.error().kind == PublicationError::upload); CHECK(load_region_head(*r.runtime, r.key, 5s)->sequence == 1); } // Manifest transaction failure after verified uploads: fenced writer, orphans retained. auto stale = *lease; stale.holder[0] ^= 1U; auto fenced = publish_environmental_batch(*r.runtime, *r.objects, stale, first->committed, batch, filled<16>(0xD3), build, budget, 5s); CHECK(!fenced && fenced.error().kind == PublicationError::rejected && fenced.error().store.kind == LeaseStoreError::fenced); CHECK(fenced.error().prepared.commit == filled<16>(0xD3)); auto orphan_bytes = state->snapshot(); auto orphan = r.objects->get({*sha256(*orphan_bytes), orphan_bytes->size()}); CHECK(orphan && *orphan == *orphan_bytes); // Uploaded, verified, unreferenced. auto recovered = recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s); CHECK(recovered && recovered->state->progress().tick == 8); // Never the orphan. // Missing and corrupt committed payloads quarantine; no older head is selected. const auto saved = r.objects->get({manifest.payload_sha256, manifest.payload_bytes}); CHECK(saved); fs::remove(object); auto missing = recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s); CHECK(!missing && missing.error().kind == RecoveryError::missing_payload); // Staging replays from the committed predecessor bytes; without them it stops. auto no_base = publish_environmental_batch(*r.runtime, *r.objects, *lease, first->committed, batch, filled<16>(0xD4), build, budget, 5s); CHECK(!no_base && no_base.error().kind == PublicationError::predecessor && no_base.error().object == ObjectError::missing); CHECK(r.objects->put(*saved)); fs::permissions(object, fs::perms::owner_all, fs::perm_options::replace); { std::fstream f(object, std::ios::in | std::ios::out | std::ios::binary); f.seekp(100); f.put(static_cast((*saved)[100] ^ 1U)); } auto corrupt = recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s); CHECK(!corrupt && corrupt.error().kind == RecoveryError::corrupt_payload); fs::remove(object); CHECK(r.objects->put(*saved)); CHECK(recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s)); // Committed history is immutable even to the owner (trigger), and the head // cannot be repointed (composite foreign key). auto tamper = r.owner->query("UPDATE evolution.region_commits SET manifest = " "overlay(manifest placing '\\x02'::bytea from 121 for 1) " "WHERE world_id = 7 AND region_id = 4 AND commit_sequence = 1", {}, 5s); CHECK(!tamper && std::string(tamper.error().sqlstate.data()) == "EVL09"); auto erase = r.owner->query("DELETE FROM evolution.region_commits WHERE world_id = 7 AND " "region_id = 4 AND commit_sequence = 1", {}, 5s); CHECK(!erase && std::string(erase.error().sqlstate.data()) == "EVL09"); auto repoint = r.owner->query("UPDATE evolution.region_heads SET digest = sha256('\\x00') " "WHERE world_id = 7 AND region_id = 4", {}, 5s); CHECK(!repoint && std::string(repoint.error().sqlstate.data()) == "23503"); // Defense in depth: even with the trigger and digest constraint removed, recovery // re-digests the manifest and quarantines. Both guards are restored afterwards. CHECK(r.owner->query("ALTER TABLE evolution.region_commits DISABLE TRIGGER " "region_commits_immutable", {}, 5s)); CHECK(r.owner->query("ALTER TABLE evolution.region_commits DROP CONSTRAINT " "region_commits_manifest_digest", {}, 5s)); // Flip a payload-digest byte: only the manifest digest check can notice it. CHECK(r.owner->query("UPDATE evolution.region_commits SET manifest = " "set_byte(manifest, 248, get_byte(manifest, 248) # 1) " "WHERE world_id = 7 AND region_id = 4 AND commit_sequence = 1", {}, 5s)); auto tampered = recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s); CHECK(!tampered && tampered.error().kind == RecoveryError::manifest); const std::array original{"\\x" + hex(first->committed.manifest)}; CHECK(r.owner->query("UPDATE evolution.region_commits SET manifest = $1::bytea " "WHERE world_id = 7 AND region_id = 4 AND commit_sequence = 1", original, 5s)); CHECK(r.owner->query("ALTER TABLE evolution.region_commits ADD CONSTRAINT " "region_commits_manifest_digest CHECK (commit_digest = sha256(manifest))", {}, 5s)); CHECK(r.owner->query("ALTER TABLE evolution.region_commits ENABLE TRIGGER " "region_commits_immutable", {}, 5s)); CHECK(recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s)); // Recovery does not create genesis for an unknown region. auto none = recover_environmental_checkpoint(*r.runtime, *r.objects, {7, 99}, retained, budget, 5s); CHECK(!none && none.error().kind == RecoveryError::not_provisioned); } // History-capable profile: deaths commit with the state that caused them, the // archive survives recovery, and a region never switches profile. void lineage_publish_and_recover() { const RegionKey key{7, 6}; auto owner = connect_as(env("PGUSER")); auto runtime = connect_as(env("EVO_PG_RUNTIME_USER")); auto objects = FilesystemObjectStore::open(scratch("lineage"), key, max_lineage_payload_bytes); CHECK(objects); CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " "(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING", {}, 5s)); auto opening = evo::test::lineage_region(); auto genesis = provision_lineage_region(*owner, **objects, key, evo::test::region_identity.frame, *opening, filled<16>(0xE0), build, budget, 5s); CHECK(genesis); CHECK(decode_manifest(genesis->manifest)->profile == CheckpointProfile::environmental_lineage_v1); auto lease = acquire_region_lease(*runtime, key, 20000ms, 5s); CHECK(lease); auto region = evo::test::lineage_region(); LineageBatch batch{*region}; run_batch(batch, 64, true); // The consumer's death at tick 20 seals the batch there. CHECK(batch.end_tick() == 20); auto first = publish_lineage_batch(*runtime, **objects, *lease, *genesis, batch, filled<16>(0xE1), build, budget, 5s); CHECK(first && first->committed.head.end_tick == 20); CHECK(first->decisions.size() == 1 && first->decisions[0] == BatchDecision{20, Actor::consumer, Outcome::death, 11}); CHECK(batch.confirm(20)); // The environmental path refuses a lineage predecessor before uploading. auto plain = evo::test::region_community(); EnvBatch plain_batch{*plain}; run_batch(plain_batch, 30); auto crossed = publish_environmental_batch(*runtime, **objects, *lease, first->committed, plain_batch, filled<16>(0xE2), build, budget, 5s); CHECK(!crossed && crossed.error().kind == PublicationError::candidate); // The database refuses a profile switch even when the manifest is well formed. auto switched = *decode_manifest(first->committed.manifest); auto next = successor_manifest(switched, first->committed.head.digest, filled<16>(0xE3), *lease, 30, build, 100, filled<32>(0xA2), {trace_bytes_for(10), filled<32>(0x7C), 0, {}}); CHECK(next); next->profile = CheckpointProfile::environmental_v1; next->history_profile = 0; auto switched_bytes = encode_manifest(*next); CHECK(switched_bytes); auto refused = publish_region_checkpoint(*runtime, *switched_bytes, {}, 5s); CHECK(!refused && refused.error().kind == LeaseStoreError::unsupported); const std::array retained{build}; auto recovered = recover_lineage_checkpoint(*runtime, **objects, key, retained, budget, 5s); CHECK(recovered && recovered->committed.head == first->committed.head); CHECK(recovered->decisions.size() == 1 && recovered->decisions[0].commit_sequence == 1); const auto consumer = recovered->state->archive().organism(11); CHECK(consumer && !consumer->alive && consumer->died.value == 20); CHECK(recovered->state->archive().ancestry(2)->size() == 2); // Retained ancestry. CHECK(recovered->state->snapshot() == region->snapshot()); LineageBatch resumed{*recovered->state}; for (std::uint64_t tick = 21; tick <= 60; ++tick) { auto a = region->advance(tick, evo::test::region_input(tick)); auto b = resumed.advance(evo::test::region_input(tick)); CHECK(a && b && *a == *b); } CHECK(recovered->state->snapshot() == region->snapshot()); resumed.seal(); auto second = publish_lineage_batch(*runtime, **objects, *lease, recovered->committed, resumed, filled<16>(0xE4), build, budget, 5s); CHECK(second && second->committed.head.sequence == 2 && second->committed.head.end_tick == 60 && second->decisions.empty()); // A host restoring the other profile refuses rather than misreading the payload. auto wrong = recover_environmental_checkpoint(*runtime, **objects, key, retained, budget, 5s); CHECK(!wrong && wrong.error().kind == RecoveryError::unsupported); CHECK(release_region_lease(*runtime, key, *lease, 5s)); } std::string runtime_user() { return env("EVO_PG_RUNTIME_USER"); } // Polls resolution on a fresh connection until the prepared commit is committed. ResolvedCommit await_resolution(const PreparedCommit& prepared, Resolution wanted) { auto db = connect_as(runtime_user()); for (int attempt = 0; attempt < 200; ++attempt) { auto resolved = resolve_prepared(*db, prepared, 5s); CHECK(resolved); if (resolved->status == wanted) return *resolved; std::this_thread::sleep_for(25ms); } CHECK(false); return {}; } // Holds the lease row lock so a publication's outcome becomes ambiguous to its client. struct LeaseRowLock { std::unique_ptr db; LeaseRowLock(RegionKey key) : db(connect_as(env("PGUSER"))) { const std::array p{std::to_string(key.world), std::to_string(key.region)}; CHECK(db->query("BEGIN", {}, 5s)); CHECK(db->query("SELECT 1 FROM evolution.region_leases WHERE world_id = $1::evolution.u64 " "AND region_id = $2::evolution.u64 FOR UPDATE", p, 5s)); } // Terminates backends of the runtime login queued behind this lock. void kill_waiters() { const std::array p{runtime_user()}; CHECK(db->query("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE " "usename = $1 AND wait_event_type = 'Lock'", p, 5s)); } void release() { CHECK(db->query("ROLLBACK", {}, 5s)); } }; // M7.5 durability: permanent outcomes commit atomically with their checkpoint, // ambiguity is resolved by exact bytes, and nothing is ever applied twice. void durable_outcomes_and_ambiguity() { auto r = provisioned("outcomes", 7, 0xF0); const std::array retained{build}; auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s); CHECK(lease); auto genesis = load_region_checkpoint(*r.runtime, r.key, 5s); CHECK(genesis); auto state = evo::test::region_community(); EnvBatch batch{*state}; run_batch(batch, 64, true); // Sealed by the consumer's death at tick 20. CHECK(batch.end_tick() == 20); auto died = publish_environmental_batch(*r.runtime, *r.objects, *lease, *genesis, batch, filled<16>(0xF1), build, budget, 5s); CHECK(died && died->decisions.size() == 1 && died->decisions[0] == BatchDecision{20, Actor::consumer, Outcome::death, 0}); CHECK(batch.confirm(20)); const CommittedDecision final_death{1, died->decisions[0]}; auto decisions = load_region_decisions(*r.runtime, r.key, 5s); CHECK(decisions && decisions->size() == 1 && (*decisions)[0] == final_death); // Outbox: at-least-once delivery, idempotent acknowledgement, no simulation effect. auto pending = pending_region_outbox(*r.runtime, r.key, 64, 5s); CHECK(pending && pending->size() == 1 && (*pending)[0] == final_death); CHECK(pending_region_outbox(*r.runtime, r.key, 64, 5s)->size() == 1); // Redelivered. auto ack = acknowledge_region_outbox(*r.runtime, r.key, died->decisions[0], 5s); CHECK(ack && *ack); ack = acknowledge_region_outbox(*r.runtime, r.key, died->decisions[0], 5s); CHECK(ack && !*ack); CHECK(pending_region_outbox(*r.runtime, r.key, 64, 5s)->empty()); CHECK(!pending_region_outbox(*r.runtime, r.key, 65, 5s)); const BatchDecision never{21, Actor::producer, Outcome::death, 0}; CHECK(!acknowledge_region_outbox(*r.runtime, r.key, never, 5s)); // Commit succeeded but the response was lost: the exact retry returns the same // receipt and records nothing twice; resolution confirms it under any epoch. auto death_block = encode_decisions(died->decisions, 0, 20); CHECK(death_block); const PreparedCommit lost{r.key, filled<16>(0xF1), died->committed.manifest, *death_block}; auto retried = retry_prepared(*r.runtime, lost, 5s); CHECK(retried && *retried == died->committed.head); auto resolved = resolve_prepared(*r.runtime, lost, 5s); CHECK(resolved && resolved->status == Resolution::committed && resolved->receipt == died->committed.head); CHECK(load_region_decisions(*r.runtime, r.key, 5s)->size() == 1); // Changed bytes under the same ID conflict, whether manifest or decisions differ. auto changed = lost; changed.manifest[288] ^= 1U; // Trace digest. CHECK(retry_prepared(*r.runtime, changed, 5s).error().kind == LeaseStoreError::commit_conflict); CHECK(resolve_prepared(*r.runtime, changed, 5s)->status == Resolution::conflict); auto undecided = lost; undecided.decisions.clear(); // No longer what the manifest digests. CHECK(retry_prepared(*r.runtime, undecided, 5s).error().kind == LeaseStoreError::invalid); CHECK(resolve_prepared(*r.runtime, undecided, 5s)->status == Resolution::conflict); auto elsewhere = lost; elsewhere.key.region = 3; CHECK(resolve_prepared(*r.runtime, elsewhere, 5s)->status == Resolution::conflict); auto unknown_id = lost; unknown_id.commit = filled<16>(0xFE); CHECK(resolve_prepared(*r.runtime, unknown_id, 5s)->status == Resolution::absent); // The committed dead stay dead: a later batch cannot finalize the death again. const auto head = died->committed; const std::array again{BatchDecision{25, Actor::consumer, Outcome::death, 0}}; auto again_bytes = encode_decisions(again, 20, 30); CHECK(again_bytes); auto relapse = successor_manifest( *decode_manifest(head.manifest), head.head.digest, filled<16>(0xF2), *lease, 30, build, 100, filled<32>(0xA2), {trace_bytes_for(10), filled<32>(0x7C), 1, *sha256(*again_bytes)}); CHECK(relapse); auto relapse_bytes = encode_manifest(*relapse); auto repeated = publish_region_checkpoint(*r.runtime, *relapse_bytes, *again_bytes, 5s); CHECK(!repeated && repeated.error().kind == LeaseStoreError::outcome_final); // A block of the right size that is not the one the manifest digests is refused. auto forged_block = *again_bytes; forged_block[0] ^= 1U; // Tick 25 -> 24: still well formed, different bytes. CHECK(publish_region_checkpoint(*r.runtime, *relapse_bytes, forged_block, 5s).error().kind == LeaseStoreError::invalid); // Partial or reordered decision blocks are refused whole; nothing is recorded. CHECK(publish_region_checkpoint(*r.runtime, *relapse_bytes, {}, 5s).error().kind == LeaseStoreError::invalid); const std::array two{BatchDecision{25, Actor::producer, Outcome::death, 0}, BatchDecision{24, Actor::consumer, Outcome::death, 0}}; std::vector reordered(two.size() * decision_bytes); auto ordered = encode_decisions(std::array{two[1], two[0]}, 20, 30); CHECK(ordered); std::copy_n(ordered->begin() + 24, 24, reordered.begin()); std::copy_n(ordered->begin(), 24, reordered.begin() + 24); auto reorder = *relapse; reorder.batch = {trace_bytes_for(10), filled<32>(0x7C), 2, *sha256(reordered)}; CHECK(publish_region_checkpoint(*r.runtime, *encode_manifest(reorder), reordered, 5s) .error() .kind == LeaseStoreError::invalid); // A gap: a link from an older predecessor is not the committed head's successor. auto gap = successor_manifest(*decode_manifest(genesis->manifest), genesis->head.digest, filled<16>(0xF3), *lease, 30, build, 100, filled<32>(0xA2), {trace_bytes_for(30), filled<32>(0x7C), 0, {}}); CHECK(publish_region_checkpoint(*r.runtime, *encode_manifest(*gap), {}, 5s).error().kind == LeaseStoreError::head_mismatch); CHECK(load_region_head(*r.runtime, r.key, 5s)->sequence == 1); CHECK(load_region_decisions(*r.runtime, r.key, 5s)->size() == 1); // Keep authority comfortably valid for the ambiguity cases (sanitizer builds are slow). CHECK(renew_region_lease(*r.runtime, r.key, *lease, 20000ms, 5s)); // Ambiguous outcome that did commit: the client times out while queued, the // server finishes after the lock is released. Resolution adopts the receipt. run_batch(batch, 10); std::optional lock{std::in_place, r.key}; auto ambiguous_db = connect_as(runtime_user()); auto ambiguous = publish_environmental_batch(*ambiguous_db, *r.objects, *lease, head, batch, filled<16>(0xF4), build, budget, 300ms); CHECK(!ambiguous && ambiguous.error().kind == PublicationError::unknown); CHECK(batch.sealed()); // Still paused: never assume rollback. lock->release(); lock.reset(); const auto committed = await_resolution(ambiguous.error().prepared, Resolution::committed); CHECK(committed.receipt.sequence == 2 && committed.receipt.end_tick == 30); auto adopted = retry_prepared(*r.runtime, ambiguous.error().prepared, 5s); CHECK(adopted && *adopted == committed.receipt); CHECK(batch.confirm(30)); // Ambiguous outcome that did not commit: the orphaned request is terminated // before it reaches the row. Absent, so the same bytes are retried while authorized. auto after = load_region_checkpoint(*r.runtime, r.key, 5s); CHECK(after); run_batch(batch, 5); lock.emplace(r.key); auto orphan_db = connect_as(runtime_user()); auto orphaned = publish_environmental_batch(*orphan_db, *r.objects, *lease, *after, batch, filled<16>(0xF5), build, budget, 300ms); CHECK(!orphaned && orphaned.error().kind == PublicationError::unknown); lock->kill_waiters(); lock->release(); lock.reset(); std::this_thread::sleep_for(200ms); CHECK(resolve_prepared(*r.runtime, orphaned.error().prepared, 5s)->status == Resolution::absent); auto recommitted = retry_prepared(*r.runtime, orphaned.error().prepared, 5s); CHECK(recommitted && recommitted->sequence == 3 && recommitted->end_tick == 35); CHECK(batch.confirm(35)); // Recovery reconciles durable outcomes with the restored lifecycle. auto recovered = recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s); CHECK(recovered && recovered->decisions.size() == 1 && recovered->decisions[0] == final_death && recovered->state->progress().consumer_terminal == 1); CHECK(recovered->state->snapshot() == state->snapshot()); // A durable death the state does not show is a history disagreement: quarantine. CHECK(r.owner->query("INSERT INTO evolution.region_decisions(world_id, region_id, outcome, " "actor, tick, organism, commit_sequence) VALUES (7, 7, 1, 0, 30, 0, 2)", {}, 5s)); auto disagreed = recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s); CHECK(!disagreed && disagreed.error().kind == RecoveryError::history); auto undo = r.owner->query("DELETE FROM evolution.region_decisions WHERE world_id = 7 AND " "region_id = 7 AND actor = 0", {}, 5s); CHECK(!undo && std::string(undo.error().sqlstate.data()) == "EVL09"); // Stays quarantined. CHECK(release_region_lease(*r.runtime, r.key, *lease, 5s)); } // Publication cannot acknowledge a checkpoint whose objects are not yet durable. void publication_waits_for_directory_durability() { auto r = provisioned("sync-publish", 9, 0xA8); ObjectStoreFaults faults; auto faulty = FilesystemObjectStore::open(r.objects->directory().parent_path().parent_path(), r.key, max_environmental_payload_bytes, &faults); CHECK(faulty); auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s); CHECK(lease); auto genesis = load_region_checkpoint(*r.runtime, r.key, 5s); CHECK(genesis); auto state = evo::test::region_community(); EnvBatch batch{*state}; run_batch(batch, 8); for (const std::uint32_t failures : {1U, 1U, 2U}) { faults.fail_directory_syncs = failures; auto attempt = publish_environmental_batch(*r.runtime, **faulty, *lease, *genesis, batch, filled<16>(0xA9), build, budget, 5s); CHECK(!attempt && attempt.error().kind == PublicationError::upload && attempt.error().object == ObjectError::io); CHECK(load_region_head(*r.runtime, r.key, 5s)->sequence == 0); // Nothing sent. } faults.fail_directory_syncs = 0; auto published = publish_environmental_batch(*r.runtime, **faulty, *lease, *genesis, batch, filled<16>(0xA9), build, budget, 5s); CHECK(published && published->committed.head.sequence == 1); CHECK(release_region_lease(*r.runtime, r.key, *lease, 5s)); } // Review P1: a committed death with no durable decision (what an accepted pre-v5 // upgrade would leave, or a host that skipped derivation) is never served as a // recovered region. The database cannot see payload deaths; recovery must. void undecided_death_is_quarantined() { auto r = provisioned("undecided", 10, 0xB6); auto lease = acquire_region_lease(*r.runtime, r.key, 20000ms, 5s); CHECK(lease); auto genesis = load_region_checkpoint(*r.runtime, r.key, 5s); CHECK(genesis); auto state = evo::test::region_community(); EnvBatch batch{*state}; run_batch(batch, 64, true); // The consumer dies at tick 20. CHECK(batch.end_tick() == 20 && state->progress().consumer_terminal == 1); const auto previous = *decode_manifest(genesis->manifest); auto payload = state->snapshot(); auto trace = encode_trace({1, r.key, previous.commit, genesis->head.digest, 0, 20}, batch.steps()); CHECK(payload && trace && r.objects->put(*payload) && r.objects->put(*trace)); auto manifest = successor_manifest(previous, genesis->head.digest, filled<16>(0xB7), *lease, 20, build, payload->size(), *sha256(*payload), {trace->size(), *sha256(*trace), 0, {}}); CHECK(manifest); auto bytes = encode_manifest(*manifest); CHECK(bytes && publish_region_checkpoint(*r.runtime, *bytes, {}, 5s)); const std::array retained{build}; auto recovered = recover_environmental_checkpoint(*r.runtime, *r.objects, r.key, retained, budget, 5s); CHECK(!recovered && recovered.error().kind == RecoveryError::history); CHECK(release_region_lease(*r.runtime, r.key, *lease, 5s)); } } // namespace void run_region_checkpoint_store_tests() { sha256_vectors_and_database_agreement(); object_store_is_immutable_and_verified(); directory_sync_failures_are_never_acknowledged(); publish_recover_and_continue(); failures_leave_the_previous_head(); lineage_publish_and_recover(); durable_outcomes_and_ambiguity(); publication_waits_for_directory_durability(); undecided_death_is_quarantined(); }