// Real PostgreSQL M7.3 fencing checks (ADR-0060), publishing M7.4 manifests // (ADR-0061) with synthetic payload digests. Run by tools/ci/postgres-check.sh. #include "evo/persistence/region_checkpoint.hpp" #include "evo/persistence/region_lease_store.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include void run_region_lease_store_tests(); namespace { using namespace evo::persistence; using namespace std::chrono_literals; using std::chrono::milliseconds; void check(bool value, std::source_location at = std::source_location::current()) { if (!value) throw std::runtime_error("region lease store test line " + std::to_string(at.line())); } #define CHECK(...) check(static_cast(__VA_ARGS__)) std::string env(const char* key) { const auto* value = std::getenv(key); if (!value) throw std::runtime_error("missing test environment"); return value; } std::unique_ptr connect_as(const std::string& user) { PgOptions options{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}}; auto made = Postgres::connect(options, 5s); CHECK(made); return std::move(*made); } std::unique_ptr owner() { return connect_as(env("PGUSER")); } std::unique_ptr runtime() { return connect_as(env("EVO_PG_RUNTIME_USER")); } Uuid id(std::uint8_t seed) { Uuid out{}; out.fill(seed); return out; } Sha256Digest digest(std::uint8_t seed) { Sha256Digest out{}; out.fill(seed); return out; } const Sha256Digest build = digest(0xB1); void retain_build(Postgres& db) { CHECK(db.query("INSERT INTO evolution.retained_builds(build_identity) VALUES " "(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING", {}, 5s)); } std::string bytea(std::span bytes) { constexpr char digits[] = "0123456789abcdef"; std::string out = "\\x"; for (const auto b : bytes) { out.push_back(digits[b >> 4U]); out.push_back(digits[b & 0x0FU]); } return out; } // Genesis commit ID bytes are the region number; frame is 7. Payload digests are // synthetic: the database records them and never reads the object store. RegionKey provision(Postgres& db, std::uint64_t region) { retain_build(db); const RegionKey key{1, region}; auto g = genesis_manifest(key, 7, id(static_cast(region)), 50000, build, 100, digest(0xA0)); CHECK(g); auto bytes = encode_manifest(*g); CHECK(bytes); const std::array p{bytea(*bytes)}; CHECK(db.query("SELECT evolution.provision_region_checkpoint($1::bytea)", p, 5s)); return key; } CommittedCheckpoint committed(Postgres& db, RegionKey key) { auto c = load_region_checkpoint(db, key, 5s); CHECK(c); return std::move(*c); } // Successor fields set directly, so tests can send links the local builder refuses. CheckpointManifest link(const CommittedCheckpoint& from, const LeaseGrant& lease, std::uint8_t commit, std::uint64_t end_tick) { auto m = decode_manifest(from.manifest); CHECK(m); m->commit = id(commit); m->sequence = from.head.sequence + 1; m->predecessor_commit = from.head.commit; m->predecessor_digest = from.head.digest; m->begin_tick = from.head.end_tick; m->end_tick = end_tick; m->holder = lease.holder; m->epoch = lease.epoch; m->payload_sha256 = digest(0xA1); // Synthetic trace evidence of the right size; no decisions. m->batch = {trace_bytes_for(end_tick - from.head.end_tick), digest(0x7C), 0, {}}; return *m; } std::expected publish(Postgres& db, std::vector bytes) { auto head = publish_region_checkpoint(db, bytes, {}, 5s); if (!head) return std::unexpected{head.error()}; return CommittedCheckpoint{*head, std::move(bytes)}; } std::expected publish(Postgres& db, const CheckpointManifest& m) { auto bytes = encode_manifest(m); CHECK(bytes); return publish(db, std::move(*bytes)); } // Encodes a valid link, then overwrites raw bytes the encoder would refuse. std::vector patched(const CheckpointManifest& m, std::size_t offset, std::span value) { auto bytes = encode_manifest(m); CHECK(bytes); std::ranges::copy(value, bytes->begin() + static_cast(offset)); return std::move(*bytes); } // Expiry is decided by database time, which is the server's wall clock. A local // sleep does not prove it: a VM can step that clock back (WSL2 was measured // stepping ~0.5 s every ~32 s). Wait on the database's own judgement instead. // Owner-only read; works inside a transaction that holds the row lock. void wait_until_expired(Postgres& owner_db, RegionKey key) { const std::array region{std::to_string(key.region)}; for (int attempt = 0; attempt < 400; ++attempt) { auto rows = owner_db.query("SELECT (clock_timestamp() >= expires_at)::text FROM " "evolution.region_leases WHERE world_id = 1 AND region_id = " "$1::evolution.u64", region, 5s); CHECK(rows && rows->size() == 1); if ((*rows)[0][0] == "true") return; std::this_thread::sleep_for(50ms); } CHECK(false); // Not expired after 20 s of database time polling. } bool failed(const auto& result, LeaseStoreError kind) { return !result && result.error().kind == kind; } std::string sqlstate(const std::expected& result) { return result ? std::string{} : std::string(result.error().sqlstate.data()); } void runtime_cannot_bypass_the_guard() { auto admin = owner(); const auto key = provision(*admin, 10); auto db = runtime(); for (const auto* sql : { "UPDATE evolution.region_leases SET holder = NULL", "UPDATE evolution.region_heads SET commit_sequence = 5", "DELETE FROM evolution.region_heads", "INSERT INTO evolution.region_leases(world_id,region_id,frame_id) VALUES (9,9,9)", "SELECT * FROM evolution.region_leases", "SELECT * FROM evolution.region_heads", "SELECT evolution.provision_region_checkpoint('\\x00')", "SELECT * FROM evolution.region_commits", "INSERT INTO evolution.retained_builds VALUES ('\\x00')", "SELECT evolution.parse_checkpoint_manifest('\\x00')", "SELECT evolution.check_region_lease(1, 10, gen_random_uuid(), 1)", "CREATE FUNCTION evolution.bypass() RETURNS int LANGUAGE sql AS 'SELECT 1'", "CREATE TABLE evolution.bypass(x int)", }) CHECK(sqlstate(db->query(sql, {}, 5s)) == "42501"); const std::array owner_name{env("PGUSER")}; CHECK(!db->query("SELECT pg_catalog.set_config('role', $1, false)", owner_name, 5s)); // A SECURITY DEFINER function ignores a hostile caller search_path. CHECK(db->query("SET search_path = pg_temp, public", {}, 5s)); CHECK(db->query("CREATE TEMP TABLE region_leases(x int)", {}, 5s)); auto lease = acquire_region_lease(*db, key, 20000ms, 5s); CHECK(lease && lease->epoch == 1 && lease->frame == 7); CHECK(release_region_lease(*db, key, *lease, 5s)); } void acquire_renew_release_and_publish() { auto admin = owner(); const auto key = provision(*admin, 20); auto db = runtime(); CHECK( failed(acquire_region_lease(*db, {1, 999}, 20000ms, 5s), LeaseStoreError::not_provisioned)); CHECK(failed(acquire_region_lease(*db, key, 999ms, 5s), LeaseStoreError::invalid)); CHECK(failed(acquire_region_lease(*db, key, 60001ms, 5s), LeaseStoreError::invalid)); auto lease = acquire_region_lease(*db, key, 20000ms, 5s); CHECK(lease && lease->epoch == 1 && lease->duration == 20000ms && lease->holder != Uuid{}); CHECK(failed(acquire_region_lease(*db, key, 20000ms, 5s), LeaseStoreError::busy)); auto renewed = renew_region_lease(*db, key, *lease, 20000ms, 5s); CHECK(renewed && *renewed == 20000ms); auto forged = *lease; forged.holder[0] ^= 1U; CHECK(failed(renew_region_lease(*db, key, forged, 20000ms, 5s), LeaseStoreError::fenced)); auto stale_epoch = *lease; stale_epoch.epoch = 2; CHECK(failed(renew_region_lease(*db, key, stale_epoch, 20000ms, 5s), LeaseStoreError::fenced)); const auto genesis = committed(*db, key); CHECK(genesis.head.sequence == 0 && genesis.head.end_tick == 0 && genesis.head.frame == 7); CHECK(genesis.head.commit == id(20) && genesis.head.digest != Sha256Digest{}); const std::array tick65{65}; CHECK(failed(publish(*db, patched(link(genesis, *lease, 21, 64), 120, tick65)), LeaseStoreError::invalid)); const auto reused = id(20); CHECK(failed(publish(*db, patched(link(genesis, *lease, 21, 1), 40, reused)), LeaseStoreError::invalid)); const auto first = link(genesis, *lease, 21, 64); auto head = publish(*db, first); CHECK(head && head->head.sequence == 1 && head->head.end_tick == 64 && head->head.commit == id(21)); CHECK(failed(publish(*db, link(*head, forged, 22, 65)), LeaseStoreError::fenced)); // Exact retry after an applied publication returns the original receipt (M7.5). auto retried = publish(*db, first); CHECK(retried && retried->head == head->head); auto current = committed(*db, key); CHECK(current.head == head->head && current.manifest == head->manifest); auto changed = link(*head, *lease, 22, 65); changed.predecessor_digest[31] ^= 1U; CHECK(failed(publish(*db, changed), LeaseStoreError::head_mismatch)); const std::array tick64{64}; CHECK(failed(publish(*db, patched(link(*head, *lease, 22, 65), 120, tick64)), LeaseStoreError::invalid)); auto other_quantum = link(*head, *lease, 22, 65); other_quantum.tick_microseconds = 25000; // Fixed per region; changing it is a migration. CHECK(failed(publish(*db, other_quantum), LeaseStoreError::unsupported)); auto unretained = link(*head, *lease, 22, 65); unretained.build = digest(0xB2); CHECK(failed(publish(*db, unretained), LeaseStoreError::unsupported)); auto future = link(*head, *lease, 22, 65); future.versions.rules = 3; CHECK(failed(publish(*db, future), LeaseStoreError::unsupported)); // Commit IDs are globally unique: region 10's genesis ID cannot be reused here. CHECK(failed(publish(*db, link(*head, *lease, 10, 65)), LeaseStoreError::commit_conflict)); CHECK(release_region_lease(*db, key, *lease, 5s)); CHECK(failed(release_region_lease(*db, key, *lease, 5s), LeaseStoreError::fenced)); CHECK(failed(renew_region_lease(*db, key, *lease, 20000ms, 5s), LeaseStoreError::fenced)); CHECK(failed(publish(*db, link(*head, *lease, 22, 65)), LeaseStoreError::fenced)); auto next = acquire_region_lease(*db, key, 20000ms, 5s); CHECK(next && next->epoch == 2 && next->holder != lease->holder); auto final_head = publish(*db, link(*head, *next, 22, 65)); CHECK(final_head && final_head->head.sequence == 2); } void database_time_expiry_and_paused_writer() { auto admin = owner(); const auto key = provision(*admin, 30); auto worker_a = runtime(); auto worker_b = runtime(); const LeasePolicy fast{1000ms, 200ms, 100ms, 100ms, 300ms}; LeaseAuthority a{fast}; auto sent = LeaseClock::now(); auto lease_a = acquire_region_lease(*worker_a, key, fast.duration, 5s); CHECK(lease_a && a.acquired(*lease_a, sent, LeaseClock::now())); CHECK(failed(acquire_region_lease(*worker_b, key, 20000ms, 5s), LeaseStoreError::busy)); const auto genesis = committed(*worker_a, key); std::this_thread::sleep_for(1200ms); // Worker A is paused past its lease. CHECK(a.state(LeaseClock::now()) == LeaseState::fenced); // Expiry is decided by database time; the successor acquires a newer epoch. wait_until_expired(*admin, key); auto lease_b = acquire_region_lease(*worker_b, key, 20000ms, 5s); CHECK(lease_b && lease_b->epoch == lease_a->epoch + 1); auto head_b = publish(*worker_b, link(genesis, *lease_b, 31, 3)); CHECK(head_b); // A resumes and ignores its local fence: every database path rejects it. CHECK(failed(renew_region_lease(*worker_a, key, *lease_a, fast.duration, 5s), LeaseStoreError::fenced)); CHECK(failed(publish(*worker_a, link(genesis, *lease_a, 32, 1)), LeaseStoreError::fenced)); CHECK(failed(publish(*worker_a, link(*head_b, *lease_a, 32, 4)), LeaseStoreError::fenced)); CHECK(failed(release_region_lease(*worker_a, key, *lease_a, 5s), LeaseStoreError::fenced)); auto head = load_region_head(*worker_a, key, 5s); CHECK(head && *head == head_b->head); // Expired renewal cannot revive a lease even without a successor. CHECK(release_region_lease(*worker_b, key, *lease_b, 5s)); auto lease_c = acquire_region_lease(*worker_a, key, 1000ms, 5s); CHECK(lease_c && lease_c->epoch == lease_b->epoch + 1); wait_until_expired(*admin, key); CHECK( failed(renew_region_lease(*worker_a, key, *lease_c, 1000ms, 5s), LeaseStoreError::fenced)); } // A writer queued on the lease row lock is judged by the clock after the wait, // not its statement start time, so a lease that expired while queued is fenced. void queued_writer_uses_post_lock_database_time() { auto admin = owner(); const auto key = provision(*admin, 40); auto worker = runtime(); auto lease = acquire_region_lease(*worker, key, 1000ms, 5s); CHECK(lease); const auto genesis = committed(*worker, key); auto blocker = owner(); const std::array region{"40"}; CHECK(blocker->query("BEGIN", {}, 5s)); CHECK(blocker->query("SELECT 1 FROM evolution.region_leases WHERE world_id = 1 AND " "region_id = $1::evolution.u64 FOR UPDATE", region, 5s)); std::expected queued = std::unexpected{LeaseStoreFailure{}}; std::thread writer([&] { queued = publish(*worker, link(genesis, *lease, 41, 1)); }); std::this_thread::sleep_for(300ms); // Let the writer queue behind the lock. wait_until_expired(*blocker, key); // Still holding the row lock. CHECK(blocker->query("ROLLBACK", {}, 5s)); writer.join(); CHECK(failed(queued, LeaseStoreError::fenced)); auto head = load_region_head(*worker, key, 5s); CHECK(head && *head == genesis.head); } // Partition: the renewal's outcome is unknown and the connection is gone. The // holder keeps its earlier deadline and fences itself before a successor starts. void partitioned_renewal_fences_locally() { auto admin = owner(); const auto key = provision(*admin, 50); auto worker = runtime(); const LeasePolicy fast{1000ms, 200ms, 100ms, 100ms, 300ms}; LeaseAuthority authority{fast}; const auto sent = LeaseClock::now(); auto lease = acquire_region_lease(*worker, key, fast.duration, 5s); CHECK(lease && authority.acquired(*lease, sent, LeaseClock::now())); auto blocker = owner(); const std::array region{"50"}; CHECK(blocker->query("BEGIN", {}, 5s)); CHECK(blocker->query("SELECT 1 FROM evolution.region_leases WHERE world_id = 1 AND " "region_id = $1::evolution.u64 FOR UPDATE", region, 5s)); const auto renew_sent = LeaseClock::now(); auto renewal = renew_region_lease(*worker, key, *lease, fast.duration, fast.renew_timeout); CHECK(failed(renewal, LeaseStoreError::unknown)); authority.renewal_unknown(fast.duration, renew_sent); const auto local_deadline = authority.deadline(); CHECK(local_deadline <= sent + fast.duration - fast.safety_margin); std::this_thread::sleep_for(1300ms); CHECK(!authority.may_originate(LeaseClock::now())); // Release the orphaned renewal only after database expiry, so it must be refused. wait_until_expired(*blocker, key); CHECK(blocker->query("ROLLBACK", {}, 5s)); // The orphaned renewal reached the row only after database expiry: rejected. auto successor = runtime(); auto next = acquire_region_lease(*successor, key, 20000ms, 5s); CHECK(next && next->epoch == lease->epoch + 1); } void contenders_serialize_on_the_database() { auto admin = owner(); const auto key = provision(*admin, 60); std::array, 2> workers{runtime(), runtime()}; std::uint64_t previous_epoch = 0; constexpr int rounds = 64; for (int round = 0; round < rounds; ++round) { std::array, 2> results{ std::unexpected{LeaseStoreFailure{}}, std::unexpected{LeaseStoreFailure{}}}; std::latch start{2}; std::vector threads; threads.reserve(2); for (std::size_t i = 0; i < 2; ++i) threads.emplace_back([&, i] { start.arrive_and_wait(); results[i] = acquire_region_lease(*workers[i], key, 20000ms, 5s); }); for (auto& t : threads) t.join(); const auto winners = std::count_if(results.begin(), results.end(), [](const auto& r) { return r.has_value(); }); CHECK(winners == 1); const std::size_t w = results[0] ? 0 : 1; CHECK(failed(results[1 - w], LeaseStoreError::busy)); CHECK(results[w]->epoch == previous_epoch + 1); previous_epoch = results[w]->epoch; CHECK(release_region_lease(*workers[w], key, *results[w], 5s)); } // The same holder racing two commits from the same predecessor: exactly one wins. auto lease = acquire_region_lease(*workers[0], key, 20000ms, 5s); CHECK(lease); auto head = committed(*workers[0], key); for (std::uint8_t round = 0; round < 32; ++round) { std::array, 2> results{ std::unexpected{LeaseStoreFailure{}}, std::unexpected{LeaseStoreFailure{}}}; std::latch start{2}; std::vector threads; threads.reserve(2); for (std::size_t i = 0; i < 2; ++i) threads.emplace_back([&, i] { const auto seed = static_cast(100 + (round * 2) + i); start.arrive_and_wait(); results[i] = publish(*workers[i], link(head, *lease, seed, head.head.end_tick + 1)); }); for (auto& t : threads) t.join(); const std::size_t w = results[0] ? 0 : 1; CHECK(results[w] && failed(results[1 - w], LeaseStoreError::head_mismatch)); CHECK(results[w]->head.sequence == head.head.sequence + 1); head = *results[w]; } auto last = load_region_head(*workers[1], key, 5s); CHECK(last && *last == head.head && last->sequence == 32); } void exhaustion_fails_closed() { auto admin = owner(); const auto key = provision(*admin, 70); CHECK(admin->query("UPDATE evolution.region_leases SET epoch = 18446744073709551615 " "WHERE world_id = 1 AND region_id = 70", {}, 5s)); auto db = runtime(); CHECK(failed(acquire_region_lease(*db, key, 20000ms, 5s), LeaseStoreError::exhausted)); const auto other = provision(*admin, 71); auto lease = acquire_region_lease(*db, other, 20000ms, 5s); CHECK(lease); // Owner-only fixture: a chain link at the last sequence (the head FK requires it). CHECK(admin->query( "INSERT INTO evolution.region_commits(world_id, region_id, commit_sequence, commit_id, " "commit_digest, manifest, begin_tick, end_tick, published_epoch, tick_microseconds, " "versions, build_identity, payload_bytes, payload_sha256) SELECT world_id, region_id, " "18446744073709551615, '000000ff-0000-0000-0000-000000000000', commit_digest, manifest, 0, " "5, 1, tick_microseconds, versions, build_identity, payload_bytes, payload_sha256 FROM " "evolution.region_commits WHERE world_id = 1 AND region_id = 71", {}, 5s)); CHECK(admin->query("UPDATE evolution.region_heads SET commit_sequence = 18446744073709551615, " "commit_id = '000000ff-0000-0000-0000-000000000000', end_tick = 5, " "published_epoch = 1 WHERE world_id = 1 AND region_id = 71", {}, 5s)); const auto last = committed(*db, other); CHECK(last.head.sequence == 18446744073709551615ULL); auto beyond = link(last, *lease, 72, 6); beyond.sequence = 18446744073709551615ULL; // Any value: exhaustion is checked first. CHECK(failed(publish(*db, beyond), LeaseStoreError::exhausted)); } // The SQL parser's per-profile payload caps match the C++ constants exactly. void payload_caps_follow_the_profile() { auto admin = owner(); retain_build(*admin); const auto provision_raw = [&](std::vector bytes) { const std::array p{bytea(bytes)}; return admin->query("SELECT evolution.provision_region_checkpoint($1::bytea)", p, 5s); }; auto lineage = genesis_manifest({1, 90}, 7, id(90), 50000, build, max_lineage_payload_bytes, digest(0xA0), CheckpointProfile::environmental_lineage_v1); CHECK(lineage); CHECK(provision_raw(*encode_manifest(*lineage))); lineage->key.region = 91; lineage->commit = id(91); const std::array over_lineage{0x71, 0x20, 0x41, 0, 0, 0, 0, 0}; // 4268145 CHECK(sqlstate(provision_raw(patched(*lineage, 240, over_lineage))) == "22023"); auto plain = genesis_manifest({1, 92}, 7, id(92), 50000, build, max_environmental_payload_bytes, digest(0xA0)); CHECK(plain); const std::array over_plain{0x01, 0x00, 0x10, 0, 0, 0, 0, 0}; // 1 MiB + 1 CHECK(sqlstate(provision_raw(patched(*plain, 240, over_plain))) == "22023"); // Profile and history profile must match: a lineage profile claiming no history. const std::array no_history{0}; CHECK(sqlstate(provision_raw(patched(*lineage, 232, no_history))) == "EVL06"); } struct Summary { std::vector samples; void print(const char* name) { std::sort(samples.begin(), samples.end()); const auto pick = [&](std::size_t percent) { return samples[std::min(samples.size() - 1, samples.size() * percent / 100)]; }; std::printf("lease_latency_us %s n=%zu p50=%lld p99=%lld max=%lld\n", name, samples.size(), pick(50), pick(99), samples.back()); } }; template long long timed(F&& f) { const auto begin = std::chrono::steady_clock::now(); f(); return std::chrono::duration_cast(std::chrono::steady_clock::now() - begin) .count(); } // Local Unix-socket round trips only; remote TLS latency must be measured in place. void measure_round_trips() { auto admin = owner(); const auto key = provision(*admin, 80); auto db = runtime(); constexpr int n = 200; Summary acquire, renew, advance, release; auto head = committed(*db, key); for (int i = 0; i < n; ++i) { std::expected lease = std::unexpected{LeaseStoreFailure{}}; acquire.samples.push_back( timed([&] { lease = acquire_region_lease(*db, key, 20000ms, 5s); })); CHECK(lease); renew.samples.push_back( timed([&] { CHECK(renew_region_lease(*db, key, *lease, 20000ms, 5s)); })); auto m = link(head, *lease, 0, head.head.end_tick + 64); m.commit[0] = 0xAA; // Unique across all tests: 0xAA, index, zeros. m.commit[1] = static_cast(i); advance.samples.push_back(timed([&] { auto next = publish(*db, m); CHECK(next); head = std::move(*next); })); release.samples.push_back( timed([&] { CHECK(release_region_lease(*db, key, *lease, 5s)); })); } acquire.print("acquire"); renew.print("renew"); advance.print("publish_manifest"); release.print("release"); } } // namespace void run_region_lease_store_tests() { runtime_cannot_bypass_the_guard(); acquire_renew_release_and_publish(); database_time_expiry_and_paused_writer(); queued_writer_uses_post_lock_database_time(); partitioned_renewal_fences_locally(); contenders_serialize_on_the_database(); exhaustion_fails_closed(); payload_caps_follow_the_profile(); measure_round_trips(); }