// Real PostgreSQL M10.8a probe handoff checks (ADR-0096, schema v12): ownership and // fencing, the transfer state machine with retries at every step, source and destination // restarts (new epochs) at every transition, stale epochs, commit/abort races, crossing back, // failed destinations and partitions, and the destination's account control handoff, with // unique-owner and conservation checks after each scenario. World 9200 is used only here. Run by // tools/ci/postgres-check.sh. #include "evo/persistence/account_control_store.hpp" #include "evo/persistence/probe_store.hpp" #include "evo/persistence/region_checkpoint.hpp" #include "evo/persistence/region_lease_store.hpp" #ifdef EVO_POSTGRES_SESSIONS #include "evo/server/postgres_handoff_ledger.hpp" #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include void run_probe_handoff_store_tests(); namespace { using namespace evo::persistence; using namespace std::chrono_literals; void check(bool value, std::source_location at = std::source_location::current()) { if (!value) throw std::runtime_error("probe handoff store test line " + std::to_string(at.line())); } #define CHECK(...) check(static_cast(__VA_ARGS__)) constexpr std::uint64_t world = 9200; constexpr std::uint64_t inventory = 1000; // EXPERIMENT test inventory per probe. std::string env(const char* key) { const auto* value = std::getenv(key); if (!value) throw std::runtime_error("missing test environment"); return value; } std::unique_ptr connect_as(const std::string& user) { PgOptions options{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}}; auto made = Postgres::connect(options, 5s); CHECK(made); return std::move(*made); } Uuid id(std::uint8_t seed, std::uint8_t tag = 0) { Uuid out{}; out.fill(seed); out.back() = tag; return out; } std::string bytea(std::span bytes) { constexpr char digits[] = "0123456789abcdef"; std::string out = "\\x"; for (const auto b : bytes) { out.push_back(digits[b >> 4U]); out.push_back(digits[b & 0x0FU]); } return out; } Sha256Digest digest(std::uint8_t seed) { Sha256Digest out{}; out.fill(seed); return out; } const Sha256Digest build = digest(0xB1); RegionKey provision(Postgres& db, std::uint64_t region) { CHECK(db.query("INSERT INTO evolution.retained_builds(build_identity) VALUES " "(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING", {}, 5s)); const RegionKey key{world, region}; auto commit = id(0xAD, static_cast(region)); // Unique commit ids per test. auto g = genesis_manifest(key, 7, commit, 50000, build, 100, digest(0xA0)); CHECK(g); auto bytes = encode_manifest(*g); CHECK(bytes); const std::array p{bytea(*bytes)}; CHECK(db.query("SELECT evolution.provision_region_checkpoint($1::bytea)", p, 5s)); return key; } LeaseGrant acquire(Postgres& db, RegionKey key) { auto lease = acquire_region_lease(db, key, build, 20000ms, 5s); CHECK(lease); return *lease; } // A region process "restart": its lease is released and a new epoch acquired. LeaseGrant restart(Postgres& db, RegionKey key, const LeaseGrant& lease) { CHECK(release_region_lease(db, key, lease, 5s)); const auto next = acquire(db, key); CHECK(next.epoch > lease.epoch); return next; } bool failed(const auto& result, ProbeStoreError kind) { return !result && result.error().kind == kind; } TransferState state_of(const auto& result) { if (!result) throw std::runtime_error("probe handoff store: missing transfer"); return result->state; } // World-wide invariants, read as the owner: one row per probe (primary key), the test // inventory conserved, and a probe frozen exactly when it has one open transfer, which // names its owner as the source. void invariants(Postgres& admin, std::uint64_t probes) { auto rows = admin.query( "SELECT count(*)::text, coalesce(sum(inventory), 0)::text FROM evolution.probe_records " "WHERE world_id = 9200", {}, 5s); CHECK(rows && rows->size() == 1); CHECK((*rows)[0][0] == std::to_string(probes)); CHECK((*rows)[0][1] == std::to_string(probes * inventory)); rows = admin.query( "SELECT count(*)::text FROM evolution.probe_records r WHERE r.world_id = 9200 AND " "(r.transfer_id IS NOT NULL) <> EXISTS (SELECT 1 FROM evolution.probe_transfers t " "WHERE t.world_id = r.world_id AND t.transfer_id = r.transfer_id AND t.state IN " "('prepared', 'staged') AND t.source_region = r.region_id AND t.account_id = " "r.account_id)", {}, 5s); CHECK(rows && rows->size() == 1 && (*rows)[0][0] == "0"); rows = admin.query("SELECT count(*)::text FROM evolution.probe_transfers t WHERE " "t.world_id = 9200 AND t.state IN ('prepared', 'staged') AND NOT EXISTS " "(SELECT 1 FROM evolution.probe_records r WHERE r.world_id = t.world_id " "AND r.account_id = t.account_id AND r.transfer_id = t.transfer_id)", {}, 5s); CHECK(rows && rows->size() == 1 && (*rows)[0][0] == "0"); } struct World { std::unique_ptr admin = connect_as(env("PGUSER")); std::unique_ptr db = connect_as(env("EVO_PG_RUNTIME_USER")); RegionKey a = provision(*admin, 1), b = provision(*admin, 2), c = provision(*admin, 3); LeaseGrant la = acquire(*db, a), lb = acquire(*db, b), lc = acquire(*db, c); std::uint64_t probes{}; ProbeRecord open(const RegionKey& key, const LeaseGrant& lease, const Uuid& account, std::int64_t x = 0) { const auto r = open_probe(*db, key, lease, account, x, 0, -7, inventory, 5s); CHECK(r); return *r; } }; void ownership_and_happy_path(World& w) { auto& db = *w.db; // The runtime reaches the tables only through the functions. CHECK(!db.query("SELECT count(*) FROM evolution.probe_records", {}, 5s)); CHECK(!db.query("SELECT count(*) FROM evolution.probe_transfers", {}, 5s)); const auto alice = id(0x41); // Created owned by A; B sees A as owner and cannot act; reopening changes nothing. auto r = w.open(w.a, w.la, alice, 500'000); ++w.probes; CHECK(r.region == 1 && r.generation == 1 && r.x == 500'000 && r.inventory == inventory && !r.transfer); const auto seen = w.open(w.b, w.lb, alice, 9); CHECK(seen.region == 1 && seen.x == 500'000); CHECK(save_probe(db, w.a, w.la, alice, 1, 600'000, 0, -7, 5s)); CHECK(failed(save_probe(db, w.b, w.lb, alice, 1, 0, 0, -7, 5s), ProbeStoreError::fenced)); CHECK(failed(save_probe(db, w.a, w.la, alice, 2, 0, 0, -7, 5s), ProbeStoreError::fenced)); invariants(*w.admin, w.probes); // Arguments: a transfer to itself or to an unprovisioned region is refused. const auto t1 = id(0x71, 1); CHECK(failed(prepare_probe_transfer(db, w.a, w.la, alice, 1, t1, 1, 0, 0, -7, 5s), ProbeStoreError::invalid)); CHECK(failed(prepare_probe_transfer(db, w.a, w.la, alice, 1, t1, 99, 0, 0, -7, 5s), ProbeStoreError::invalid)); CHECK(failed(stage_probe_transfer(db, w.b, w.lb, t1, 5s), ProbeStoreError::unknown_transfer)); // Prepare: frozen at its final state; nothing else may change it; retries are answered. auto p = prepare_probe_transfer(db, w.a, w.la, alice, 1, t1, 2, 700'000, 10, -7, 5s); CHECK(p && p->state == TransferState::prepared && p->source == 1 && p->destination == 2 && p->x == 700'000 && p->inventory == inventory && p->account == alice); CHECK(prepare_probe_transfer(db, w.a, w.la, alice, 1, t1, 2, 1, 1, 1, 5s) == *p); CHECK(failed(save_probe(db, w.a, w.la, alice, 1, 0, 0, -7, 5s), ProbeStoreError::fenced)); CHECK(failed(prepare_probe_transfer(db, w.a, w.la, alice, 1, id(0x71, 2), 3, 0, 0, -7, 5s), ProbeStoreError::fenced)); // One open transfer per probe. CHECK(w.open(w.a, w.la, alice).transfer == t1); invariants(*w.admin, w.probes); // Only the named destination stages and commits; a commit before staging is refused. CHECK(failed(stage_probe_transfer(db, w.c, w.lc, t1, 5s), ProbeStoreError::fenced)); CHECK(failed(commit_probe_transfer(db, w.b, w.lb, t1, 5s), ProbeStoreError::not_staged)); CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t1, 5s)) == TransferState::staged); CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t1, 5s)) == TransferState::staged); CHECK(failed(commit_probe_transfer(db, w.c, w.lc, t1, 5s), ProbeStoreError::fenced)); // The one switch: owner B, next generation, the frozen final state, same inventory. const auto committed = commit_probe_transfer(db, w.b, w.lb, t1, 5s); CHECK(state_of(committed) == TransferState::committed); r = w.open(w.b, w.lb, alice); CHECK(r.region == 2 && r.generation == 2 && r.x == 700'000 && r.y == 10 && r.inventory == inventory && !r.transfer); // Every retry and late call returns the committed outcome and changes nothing. CHECK(commit_probe_transfer(db, w.b, w.lb, t1, 5s) == *committed); CHECK(stage_probe_transfer(db, w.b, w.lb, t1, 5s) == *committed); CHECK(prepare_probe_transfer(db, w.a, w.la, alice, 1, t1, 2, 0, 0, 0, 5s) == *committed); CHECK(abort_probe_transfer(db, w.a, w.la, t1, 5s) == *committed); const auto found = find_probe_transfer(db, w.c, w.lc, t1, 5s); CHECK(found && *found == std::optional{*committed}); const auto missing = find_probe_transfer(db, w.c, w.lc, id(0x7F), 5s); CHECK(missing && !missing->has_value()); // The old owner is fenced; the new one acts. CHECK(failed(save_probe(db, w.a, w.la, alice, 1, 0, 0, -7, 5s), ProbeStoreError::fenced)); CHECK(save_probe(db, w.b, w.lb, alice, 2, 800'000, 10, -7, 5s)); // A transfer id belongs to one account and source. const auto bob = id(0x42); w.open(w.a, w.la, bob); ++w.probes; CHECK(failed(prepare_probe_transfer(db, w.a, w.la, bob, 1, t1, 2, 0, 0, -7, 5s), ProbeStoreError::fenced)); // Crossing back: B -> A with a new transfer, next generation. const auto t3 = id(0x71, 3); CHECK(state_of(prepare_probe_transfer(db, w.b, w.lb, alice, 2, t3, 1, 800'000, 10, -7, 5s)) == TransferState::prepared); CHECK(state_of(stage_probe_transfer(db, w.a, w.la, t3, 5s)) == TransferState::staged); CHECK(state_of(commit_probe_transfer(db, w.a, w.la, t3, 5s)) == TransferState::committed); r = w.open(w.a, w.la, alice); CHECK(r.region == 1 && r.generation == 3 && r.inventory == inventory); invariants(*w.admin, w.probes); } // A crash (restart: new epoch) at each transition, on either side. void restarts(World& w) { auto& db = *w.db; const auto carol = id(0x43); w.open(w.a, w.la, carol); ++w.probes; // 1. Source crashes after prepare. Its new epoch sees the probe frozen (it must not act) // and the transfer proceeds without it. const auto t = id(0x72, 1); CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, carol, 1, t, 2, 1, 2, -7, 5s)) == TransferState::prepared); w.la = restart(db, w.a, w.la); const auto frozen = w.open(w.a, w.la, carol); CHECK(frozen.region == 1 && frozen.transfer == t); CHECK(failed(save_probe(db, w.a, w.la, carol, 1, 0, 0, -7, 5s), ProbeStoreError::fenced)); // The new source epoch may still answer retries of its own prepare. CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, carol, 1, t, 2, 1, 2, -7, 5s)) == TransferState::prepared); CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::staged); // 2. Destination crashes after staging: its new epoch must stage again before committing; // the old epoch is fenced everywhere. const auto old_b = w.lb; w.lb = restart(db, w.b, w.lb); CHECK(failed(commit_probe_transfer(db, w.b, w.lb, t, 5s), ProbeStoreError::not_staged)); CHECK(failed(commit_probe_transfer(db, w.b, old_b, t, 5s), ProbeStoreError::fenced)); CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::staged); // 3. Destination crashes after its commit, before observing it: the retry answers. CHECK(state_of(commit_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::committed); w.lb = restart(db, w.b, w.lb); CHECK(state_of(commit_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::committed); const auto r = w.open(w.b, w.lb, carol); CHECK(r.region == 2 && r.generation == 2 && r.x == 1 && r.y == 2 && !r.transfer); invariants(*w.admin, w.probes); } // Failed destination, abort rules, partition, and a commit/abort race. void aborts_and_races(World& w) { auto& db = *w.db; const auto dave = id(0x44); w.open(w.a, w.la, dave); ++w.probes; // The destination never stages: the source aborts and resumes, durably. const auto t1 = id(0x73, 1); CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, dave, 1, t1, 2, 5, 5, -7, 5s)) == TransferState::prepared); CHECK(failed(abort_probe_transfer(db, w.c, w.lc, t1, 5s), ProbeStoreError::fenced)); CHECK(state_of(abort_probe_transfer(db, w.a, w.la, t1, 5s)) == TransferState::aborted); CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t1, 5s)) == TransferState::aborted); CHECK(state_of(commit_probe_transfer(db, w.b, w.lb, t1, 5s)) == TransferState::aborted); auto r = w.open(w.a, w.la, dave); CHECK(r.region == 1 && r.generation == 1 && !r.transfer && r.x == 5); CHECK(save_probe(db, w.a, w.la, dave, 1, 6, 6, -7, 5s)); // Active again at its source. // The destination aborts a staged transfer it cannot accept. const auto t2 = id(0x73, 2); CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, dave, 1, t2, 2, 6, 6, -7, 5s)) == TransferState::prepared); CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t2, 5s)) == TransferState::staged); CHECK(state_of(abort_probe_transfer(db, w.b, w.lb, t2, 5s)) == TransferState::aborted); CHECK(!w.open(w.a, w.la, dave).transfer); // Partition: the source's lease lapses (it stops) while the destination commits; on // recovery the source finds the probe owned elsewhere and the transfer committed. const auto t3 = id(0x73, 3); CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, dave, 1, t3, 2, 7, 7, -7, 5s)) == TransferState::prepared); CHECK(release_region_lease(db, w.a, w.la, 5s)); // Cut off: no source authority at all. CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t3, 5s)) == TransferState::staged); CHECK(state_of(commit_probe_transfer(db, w.b, w.lb, t3, 5s)) == TransferState::committed); w.la = acquire(db, w.a); CHECK(state_of(find_probe_transfer(db, w.a, w.la, t3, 5s).value_or(std::nullopt)) == TransferState::committed); r = w.open(w.a, w.la, dave); CHECK(r.region == 2 && r.generation == 2); CHECK(failed(save_probe(db, w.a, w.la, dave, 2, 0, 0, -7, 5s), ProbeStoreError::fenced)); invariants(*w.admin, w.probes); // Race: the destination commits while the source aborts, on separate connections. // Exactly one wins, both observe the same outcome, and the owner agrees with it. auto other = connect_as(env("EVO_PG_RUNTIME_USER")); int commits = 0, aborts = 0; for (std::uint8_t round = 0; round < 24; ++round) { const auto account = id(0x50, round); w.open(w.a, w.la, account); ++w.probes; const auto t = id(0x74, round); CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, account, 1, t, 2, 1, 1, -7, 5s)) == TransferState::prepared); CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::staged); std::latch go{2}; std::optional committed, aborted; std::thread racer{[&] { go.arrive_and_wait(); committed = state_of(commit_probe_transfer(*other, w.b, w.lb, t, 5s)); }}; go.arrive_and_wait(); aborted = state_of(abort_probe_transfer(db, w.a, w.la, t, 5s)); racer.join(); CHECK(committed == aborted); const auto owner = w.open(w.a, w.la, account); if (committed == TransferState::committed) { ++commits; CHECK(owner.region == 2 && owner.generation == 2 && !owner.transfer); } else { ++aborts; CHECK(committed == TransferState::aborted && owner.region == 1 && owner.generation == 1 && !owner.transfer); } } CHECK(commits + aborts == 24); std::printf("probe handoff race commits=%d aborts=%d\n", commits, aborts); invariants(*w.admin, w.probes); } // The destination takes the account's durable control on the strength of a transfer // committed to it; nothing else (another region, an open or aborted transfer) qualifies. void control_handoff(World& w) { auto& db = *w.db; const auto eve = id(0x45); const auto none = id(0x46); w.open(w.a, w.la, eve); w.open(w.a, w.la, none); w.probes += 2; const auto claim = claim_account_control(db, w.a, w.la, eve, 1, 11, 20000ms, 5s); CHECK(claim && claim->generation == 1); const auto t = id(0x75, 1); CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, eve, 1, t, 2, 0, 0, -7, 5s)) == TransferState::prepared); const auto refused = [&](const RegionKey& key, const LeaseGrant& lease, const Uuid& account) { const auto r = handoff_account_control(db, key, lease, account, t, 21, 20000ms, 5s); return !r && r.error().kind == ControlStoreError::fenced; }; CHECK(refused(w.b, w.lb, eve)); // Not committed yet. CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::staged); CHECK(state_of(commit_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::committed); CHECK(refused(w.c, w.lc, eve)); // Not the destination. CHECK(refused(w.b, w.lb, none)); // Not this transfer's account. // The source still holds unexpired control: the destination must outlive it. const auto waited = handoff_account_control(db, w.b, w.lb, eve, t, 21, 20000ms, 5s); CHECK(waited && waited->generation == 2 && waited->wait > 0ms); // After the source released (the normal path), there is nothing to wait for. CHECK(!release_account_control(db, w.a, w.la, eve, 1, 5s).value_or(true)); // Moved on. CHECK(release_account_control(db, w.b, w.lb, eve, 2, 5s).value_or(false)); const auto again = handoff_account_control(db, w.b, w.lb, eve, t, 22, 20000ms, 5s); CHECK(again && again->generation == 3 && again->wait == 0ms); CHECK(release_account_control(db, w.b, w.lb, eve, 3, 5s).value_or(false)); invariants(*w.admin, w.probes); } #ifdef EVO_POSTGRES_SESSIONS // The session layer's HandoffLedger over these functions: stage, commit and control in one // admission; idempotent on retry; refused for aborted, foreign or unknown transfers. void handoff_ledger(World& w) { auto& db = *w.db; const auto frank = id(0x47); w.open(w.a, w.la, frank); ++w.probes; CHECK(claim_account_control(db, w.a, w.la, frank, 1, 31, 20000ms, 5s)); const auto t = id(0x76, 1); CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, frank, 1, t, 2, 3, 4, -7, 5s)) == TransferState::prepared); CHECK(release_account_control(db, w.a, w.la, frank, 1, 5s).value_or(false)); std::optional current = w.lb; evo::server::PgSlot slot{PgOptions{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("EVO_PG_RUNTIME_USER"), {}, {}}, 5s}; evo::server::PostgresHandoffLedger ledger{slot, w.b, [&] { return current; }}; evo::network::TicketClaims ticket; ticket.transfer = t; ticket.account.bytes = frank; ticket.source = 1; ticket.destination = 2; const auto admitted = ledger.admit(ticket, 41); CHECK(admitted && admitted->generation == 2 && admitted->wait == 0ms); const auto r = w.open(w.b, w.lb, frank); CHECK(r.region == 2 && r.generation == 2 && r.x == 3 && !r.transfer); // A retried login after an undecided outcome converges (control moves on again). const auto again = ledger.admit(ticket, 42); CHECK(again && again->generation == 3); // Refusals: another account, an unknown transfer, an aborted transfer; no lease. auto wrong = ticket; wrong.account.bytes = id(0x48); CHECK(!ledger.admit(wrong, 43) && ledger.admit(wrong, 43).error() == evo::network::HandoffLedgerError::refused); auto unknown = ticket; unknown.transfer = id(0x76, 9); CHECK(ledger.admit(unknown, 44).error() == evo::network::HandoffLedgerError::refused); const auto t2 = id(0x76, 2); CHECK(state_of(prepare_probe_transfer(db, w.b, w.lb, frank, 2, t2, 1, 3, 4, -7, 5s)) == TransferState::prepared); CHECK(state_of(abort_probe_transfer(db, w.b, w.lb, t2, 5s)) == TransferState::aborted); auto aborted = ticket; aborted.transfer = t2; aborted.source = 2; evo::server::PostgresHandoffLedger at_a{slot, w.a, [&] { return std::optional{w.la}; }}; CHECK(at_a.admit(aborted, 45).error() == evo::network::HandoffLedgerError::refused); current.reset(); CHECK(ledger.admit(ticket, 46).error() == evo::network::HandoffLedgerError::fenced); CHECK(release_account_control(db, w.b, w.lb, frank, 3, 5s).value_or(false)); invariants(*w.admin, w.probes); } #endif } // namespace void run_probe_handoff_store_tests() { World w; ownership_and_happy_path(w); restarts(w); aborts_and_races(w); control_handoff(w); #ifdef EVO_POSTGRES_SESSIONS handoff_ledger(w); #endif CHECK(release_region_lease(*w.db, w.a, w.la, 5s)); CHECK(release_region_lease(*w.db, w.b, w.lb, 5s)); CHECK(release_region_lease(*w.db, w.c, w.lc, 5s)); std::puts("probe handoff integration passed"); }