// Real PostgreSQL M10.3c account control checks (ADR-0091, schema v10): durable grant // serials across regions and restarts, cross-region takeover with a wait, fencing by // generation, region lease and expiry, and contention. World 9100 is used only here. // Run by tools/ci/postgres-check.sh. #include "evo/persistence/account_control_store.hpp" #include "evo/persistence/region_checkpoint.hpp" #include "evo/persistence/region_lease_store.hpp" #ifdef EVO_POSTGRES_SESSIONS #include "evo/server/postgres_control_ledger.hpp" #endif #include #include #include #include #include #include #include #include #include #include #include #include #include void run_account_control_store_tests(); namespace { using namespace evo::persistence; using namespace std::chrono_literals; using std::chrono::milliseconds; void check(bool value, std::source_location at = std::source_location::current()) { if (!value) throw std::runtime_error("account control store test line " + std::to_string(at.line())); } #define CHECK(...) check(static_cast(__VA_ARGS__)) constexpr std::uint64_t world = 9100; std::string env(const char* key) { const auto* value = std::getenv(key); if (!value) throw std::runtime_error("missing test environment"); return value; } std::unique_ptr connect_as(const std::string& user) { PgOptions options{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}}; auto made = Postgres::connect(options, 5s); CHECK(made); return std::move(*made); } Uuid id(std::uint8_t seed) { Uuid out{}; out.fill(seed); return out; } Sha256Digest digest(std::uint8_t seed) { Sha256Digest out{}; out.fill(seed); return out; } std::string bytea(std::span bytes) { constexpr char digits[] = "0123456789abcdef"; std::string out = "\\x"; for (const auto b : bytes) { out.push_back(digits[b >> 4U]); out.push_back(digits[b & 0x0FU]); } return out; } const Sha256Digest build = digest(0xB1); RegionKey provision(Postgres& db, std::uint64_t region) { CHECK(db.query("INSERT INTO evolution.retained_builds(build_identity) VALUES " "(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING", {}, 5s)); const RegionKey key{world, region}; auto commit = id(0xAC); // Commit ids are unique across the database: tag this test's. commit.back() = static_cast(region); auto g = genesis_manifest(key, 7, commit, 50000, build, 100, digest(0xA0)); CHECK(g); auto bytes = encode_manifest(*g); CHECK(bytes); const std::array p{bytea(*bytes)}; const auto provisioned = db.query("SELECT evolution.provision_region_checkpoint($1::bytea)", p, 5s); if (!provisioned) std::fprintf(stderr, "provision sqlstate %s\n", provisioned.error().sqlstate.data()); CHECK(provisioned); return key; } LeaseGrant acquire(Postgres& db, RegionKey key) { auto lease = acquire_region_lease(db, key, build, 20000ms, 5s); CHECK(lease); return *lease; } bool failed(const auto& result, ControlStoreError kind) { return !result && result.error().kind == kind; } // Database-time expiry of one account's control (owner read), as for region leases. void wait_until_expired(Postgres& owner_db, const Uuid& account) { const std::array p{bytea(account)}; for (int attempt = 0; attempt < 400; ++attempt) { auto rows = owner_db.query( "SELECT (clock_timestamp() >= expires_at)::text FROM evolution.account_controls " "WHERE world_id = 9100 AND account_id = encode($1::bytea, 'hex')::uuid", p, 5s); CHECK(rows && rows->size() == 1); if ((*rows)[0][0] == "true") return; std::this_thread::sleep_for(50ms); } CHECK(false); } void serials_takeover_and_fencing() { auto admin = connect_as(env("PGUSER")); const auto a = provision(*admin, 1); const auto b = provision(*admin, 2); auto db = connect_as(env("EVO_PG_RUNTIME_USER")); const auto lease_a = acquire(*db, a); const auto lease_b = acquire(*db, b); const auto alice = id(0x31); // The runtime reaches the table only through the functions. CHECK(!db->query("SELECT count(*) FROM evolution.account_controls", {}, 5s)); auto first = claim_account_control(*db, a, lease_a, alice, 5, 1, 20000ms, 5s); CHECK(first && first->generation == 1 && first->wait == 0ms && first->granted == 20000ms); CHECK(failed(claim_account_control(*db, a, lease_a, alice, 5, 2, 20000ms, 5s), ControlStoreError::stale_grant)); // Replay. CHECK(failed(claim_account_control(*db, a, lease_a, alice, 4, 2, 20000ms, 5s), ControlStoreError::stale_grant)); // Older grant. // Same region takeover: no wait; the old generation is fenced. auto second = claim_account_control(*db, a, lease_a, alice, 6, 2, 20000ms, 5s); CHECK(second && second->generation == 2 && second->wait == 0ms); CHECK(failed(renew_account_control(*db, a, lease_a, alice, 1, 20000ms, 5s), ControlStoreError::fenced)); CHECK(renew_account_control(*db, a, lease_a, alice, 2, 20000ms, 5s) == 20000ms); // Another region with a newer grant takes control but must wait out region A's. auto moved = claim_account_control(*db, b, lease_b, alice, 7, 1, 20000ms, 5s); CHECK(moved && moved->generation == 3 && moved->wait > 15000ms && moved->wait <= 20000ms); CHECK(failed(renew_account_control(*db, a, lease_a, alice, 2, 20000ms, 5s), ControlStoreError::fenced)); CHECK(release_account_control(*db, a, lease_a, alice, 2, 5s) == false); // Region B's renewal never shortens the waited-out expiry. CHECK(renew_account_control(*db, b, lease_b, alice, 3, 1000ms, 5s) == 1000ms); CHECK(release_account_control(*db, b, lease_b, alice, 3, 5s) == true); CHECK(failed(renew_account_control(*db, b, lease_b, alice, 3, 20000ms, 5s), ControlStoreError::fenced)); // Released control is free: the next claim waits for nobody; the serial stays durable. CHECK(failed(claim_account_control(*db, a, lease_a, alice, 7, 3, 20000ms, 5s), ControlStoreError::stale_grant)); auto back = claim_account_control(*db, a, lease_a, alice, 8, 3, 20000ms, 5s); CHECK(back && back->wait == 0ms); // Restart: a new process takes region A's lease under a new epoch. Serials survive, // the old epoch's control needs no wait, and the old lease can do nothing. CHECK(release_region_lease(*db, a, lease_a, 5s)); const auto restarted = acquire(*db, a); CHECK(restarted.epoch > lease_a.epoch); CHECK(failed(claim_account_control(*db, a, restarted, alice, 8, 1, 20000ms, 5s), ControlStoreError::stale_grant)); auto resumed = claim_account_control(*db, a, restarted, alice, 9, 1, 20000ms, 5s); CHECK(resumed && resumed->wait == 0ms); CHECK(failed(claim_account_control(*db, a, lease_a, alice, 10, 4, 20000ms, 5s), ControlStoreError::fenced)); CHECK(failed(renew_account_control(*db, a, lease_a, alice, resumed->generation, 20000ms, 5s), ControlStoreError::fenced)); // Expiry is final, and an expired holder is not waited for. const auto bob = id(0x32); auto brief = claim_account_control(*db, a, restarted, bob, 1, 2, 1000ms, 5s); CHECK(brief && brief->wait == 0ms); wait_until_expired(*admin, bob); CHECK(failed(renew_account_control(*db, a, restarted, bob, brief->generation, 20000ms, 5s), ControlStoreError::fenced)); auto elsewhere = claim_account_control(*db, b, lease_b, bob, 2, 2, 20000ms, 5s); CHECK(elsewhere && elsewhere->wait == 0ms); // Invalid requests change nothing. CHECK(failed(claim_account_control(*db, a, restarted, id(0x33), 1, 1, 500ms, 5s), ControlStoreError::invalid)); CHECK(failed(claim_account_control(*db, a, restarted, id(0x33), 0, 1, 20000ms, 5s), ControlStoreError::invalid)); CHECK(failed(claim_account_control(*db, a, restarted, id(0x33), 1, 0, 20000ms, 5s), ControlStoreError::invalid)); CHECK(failed(claim_account_control(*db, a, restarted, Uuid{}, 1, 1, 20000ms, 5s), ControlStoreError::invalid)); CHECK(claim_account_control(*db, a, restarted, id(0x33), 1, 1, 20000ms, 5s)); // Contention: two regions present the same newest serial at once; exactly one wins. const auto carol = id(0x34); auto first_db = connect_as(env("EVO_PG_RUNTIME_USER")); auto second_db = connect_as(env("EVO_PG_RUNTIME_USER")); std::latch start{2}; std::expected ra, rb; std::thread ta{[&] { start.arrive_and_wait(); ra = claim_account_control(*first_db, a, restarted, carol, 20, 7, 20000ms, 5s); }}; std::thread tb{[&] { start.arrive_and_wait(); rb = claim_account_control(*second_db, b, lease_b, carol, 20, 7, 20000ms, 5s); }}; ta.join(); tb.join(); CHECK(ra.has_value() != rb.has_value()); CHECK(failed(ra.has_value() ? rb : ra, ControlStoreError::stale_grant)); #ifdef EVO_POSTGRES_SESSIONS // The session layer's ledger over these functions. std::optional current = restarted; evo::server::PgSlot slot{PgOptions{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("EVO_PG_RUNTIME_USER"), {}, {}}, 5s}; evo::server::PostgresControlLedger ledger{slot, a, [&] { return current; }}; evo::network::AccountId dave; dave.bytes = id(0x35); auto claimed = ledger.claim(dave, 1, 9); CHECK(claimed && claimed->wait == 0ms && claimed->granted == 20000ms); CHECK(!ledger.claim(dave, 1, 9) && ledger.claim(dave, 1, 9).error() == evo::network::ControlLedgerError::stale_grant); CHECK(ledger.renew(dave, claimed->generation) == 20000ms); ledger.release(dave, claimed->generation); CHECK(!ledger.renew(dave, claimed->generation) && ledger.renew(dave, claimed->generation).error() == evo::network::ControlLedgerError::fenced); current.reset(); // The host lost its lease: nothing is granted or renewed. CHECK(!ledger.claim(dave, 2, 9) && ledger.claim(dave, 2, 9).error() == evo::network::ControlLedgerError::fenced); #endif CHECK(release_region_lease(*db, a, restarted, 5s)); CHECK(release_region_lease(*db, b, lease_b, 5s)); } } // namespace void run_account_control_store_tests() { serials_takeover_and_fencing(); std::puts("account control integration passed"); }