Files
aevolution/tests/probe_handoff_store_tests.cpp
dingoandClaude Opus 5.5 265163a9a6
Foundation / checks (deploy) (push) Successful in 3m20s
Foundation / checks (ecs) (push) Successful in 1m57s
Foundation / checks (client) (push) Failing after 10m4s
Foundation / checks (network) (push) Successful in 13m3s
Foundation / checks (tsan) (push) Successful in 7m53s
Foundation / checks (checks) (push) Successful in 32m18s
Foundation / checks (postgres) (push) Successful in 43m26s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
Add M10.8d two-process handoff fault matrix and recovery fixes
ADR-0096. Two real evo-server processes on PostgreSQL (regions 7/161 and
7/162, adjacent areas) run crossings and crossings back, SIGKILL of the
source after prepare and of the destination after stage and after commit,
duplicate and stale ticket retries, a client that never arrives and a
destination that is down. After every scenario SQL checks one owner per
probe, the conserved inventory and frozen-iff-open-transfer.

The matrix found two recovery gaps, now fixed:
- a source restarted after a prepare left the probe frozen for good (its
  abort deadline was in memory): schema v14 overdue_probe_transfers and a
  session-layer sweep abort open outgoing transfers past the ticket
  lifetime plus 10 s by the database clock;
- a ticket replayed at a restarted destination after the probe moved on
  took control back: admission of a committed transfer now requires the
  record as that commit left it.

M10.8 is complete. Refreshes the dev session window.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 23:17:49 -04:00

484 lines
24 KiB
C++

// Real PostgreSQL M10.8a probe handoff checks (ADR-0096, schema v12): ownership and
// fencing, the transfer state machine with retries at every step, source and destination
// restarts (new epochs) at every transition, stale epochs, commit/abort races, crossing back,
// failed destinations and partitions, and the destination's account control handoff, with
// unique-owner and conservation checks after each scenario. World 9200 is used only here. Run by
// tools/ci/postgres-check.sh.
#include "evo/persistence/account_control_store.hpp"
#include "evo/persistence/probe_store.hpp"
#include "evo/persistence/region_checkpoint.hpp"
#include "evo/persistence/region_lease_store.hpp"
#ifdef EVO_POSTGRES_SESSIONS
#include "evo/server/postgres_handoff_ledger.hpp"
#endif
#include <algorithm>
#include <array>
#include <atomic>
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <expected>
#include <latch>
#include <memory>
#include <optional>
#include <source_location>
#include <span>
#include <stdexcept>
#include <string>
#include <thread>
void run_probe_handoff_store_tests();
namespace {
using namespace evo::persistence;
using namespace std::chrono_literals;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("probe handoff store test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
constexpr std::uint64_t world = 9200;
constexpr std::uint64_t inventory = 1000; // EXPERIMENT test inventory per probe.
std::string env(const char* key) {
const auto* value = std::getenv(key);
if (!value)
throw std::runtime_error("missing test environment");
return value;
}
std::unique_ptr<Postgres> connect_as(const std::string& user) {
PgOptions options{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}};
auto made = Postgres::connect(options, 5s);
CHECK(made);
return std::move(*made);
}
Uuid id(std::uint8_t seed, std::uint8_t tag = 0) {
Uuid out{};
out.fill(seed);
out.back() = tag;
return out;
}
std::string bytea(std::span<const std::uint8_t> bytes) {
constexpr char digits[] = "0123456789abcdef";
std::string out = "\\x";
for (const auto b : bytes) {
out.push_back(digits[b >> 4U]);
out.push_back(digits[b & 0x0FU]);
}
return out;
}
Sha256Digest digest(std::uint8_t seed) {
Sha256Digest out{};
out.fill(seed);
return out;
}
const Sha256Digest build = digest(0xB1);
RegionKey provision(Postgres& db, std::uint64_t region) {
CHECK(db.query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
{}, 5s));
const RegionKey key{world, region};
auto commit = id(0xAD, static_cast<std::uint8_t>(region)); // Unique commit ids per test.
auto g = genesis_manifest(key, 7, commit, 50000, build, 100, digest(0xA0));
CHECK(g);
auto bytes = encode_manifest(*g);
CHECK(bytes);
const std::array<std::string, 1> p{bytea(*bytes)};
CHECK(db.query("SELECT evolution.provision_region_checkpoint($1::bytea)", p, 5s));
return key;
}
LeaseGrant acquire(Postgres& db, RegionKey key) {
auto lease = acquire_region_lease(db, key, build, 20000ms, 5s);
CHECK(lease);
return *lease;
}
// A region process "restart": its lease is released and a new epoch acquired.
LeaseGrant restart(Postgres& db, RegionKey key, const LeaseGrant& lease) {
CHECK(release_region_lease(db, key, lease, 5s));
const auto next = acquire(db, key);
CHECK(next.epoch > lease.epoch);
return next;
}
bool failed(const auto& result, ProbeStoreError kind) {
return !result && result.error().kind == kind;
}
TransferState state_of(const auto& result) {
if (!result)
throw std::runtime_error("probe handoff store: missing transfer");
return result->state;
}
// World-wide invariants, read as the owner: one row per probe (primary key), the test
// inventory conserved, and a probe frozen exactly when it has one open transfer, which
// names its owner as the source.
void invariants(Postgres& admin, std::uint64_t probes) {
auto rows = admin.query(
"SELECT count(*)::text, coalesce(sum(inventory), 0)::text FROM evolution.probe_records "
"WHERE world_id = 9200",
{}, 5s);
CHECK(rows && rows->size() == 1);
CHECK((*rows)[0][0] == std::to_string(probes));
CHECK((*rows)[0][1] == std::to_string(probes * inventory));
rows = admin.query(
"SELECT count(*)::text FROM evolution.probe_records r WHERE r.world_id = 9200 AND "
"(r.transfer_id IS NOT NULL) <> EXISTS (SELECT 1 FROM evolution.probe_transfers t "
"WHERE t.world_id = r.world_id AND t.transfer_id = r.transfer_id AND t.state IN "
"('prepared', 'staged') AND t.source_region = r.region_id AND t.account_id = "
"r.account_id)",
{}, 5s);
CHECK(rows && rows->size() == 1 && (*rows)[0][0] == "0");
rows = admin.query("SELECT count(*)::text FROM evolution.probe_transfers t WHERE "
"t.world_id = 9200 AND t.state IN ('prepared', 'staged') AND NOT EXISTS "
"(SELECT 1 FROM evolution.probe_records r WHERE r.world_id = t.world_id "
"AND r.account_id = t.account_id AND r.transfer_id = t.transfer_id)",
{}, 5s);
CHECK(rows && rows->size() == 1 && (*rows)[0][0] == "0");
}
struct World {
std::unique_ptr<Postgres> admin = connect_as(env("PGUSER"));
std::unique_ptr<Postgres> db = connect_as(env("EVO_PG_RUNTIME_USER"));
RegionKey a = provision(*admin, 1), b = provision(*admin, 2), c = provision(*admin, 3);
LeaseGrant la = acquire(*db, a), lb = acquire(*db, b), lc = acquire(*db, c);
std::uint64_t probes{};
ProbeRecord open(const RegionKey& key, const LeaseGrant& lease, const Uuid& account,
std::int64_t x = 0) {
const auto r = open_probe(*db, key, lease, account, x, 0, -7, inventory, 5s);
CHECK(r);
return *r;
}
};
void ownership_and_happy_path(World& w) {
auto& db = *w.db;
// The runtime reaches the tables only through the functions.
CHECK(!db.query("SELECT count(*) FROM evolution.probe_records", {}, 5s));
CHECK(!db.query("SELECT count(*) FROM evolution.probe_transfers", {}, 5s));
const auto alice = id(0x41);
// Created owned by A; B sees A as owner and cannot act; reopening changes nothing.
auto r = w.open(w.a, w.la, alice, 500'000);
++w.probes;
CHECK(r.region == 1 && r.generation == 1 && r.x == 500'000 && r.inventory == inventory &&
!r.transfer);
const auto seen = w.open(w.b, w.lb, alice, 9);
CHECK(seen.region == 1 && seen.x == 500'000);
CHECK(save_probe(db, w.a, w.la, alice, 1, 600'000, 0, -7, 5s));
CHECK(failed(save_probe(db, w.b, w.lb, alice, 1, 0, 0, -7, 5s), ProbeStoreError::fenced));
CHECK(failed(save_probe(db, w.a, w.la, alice, 2, 0, 0, -7, 5s), ProbeStoreError::fenced));
invariants(*w.admin, w.probes);
// Arguments: a transfer to itself or to an unprovisioned region is refused.
const auto t1 = id(0x71, 1);
CHECK(failed(prepare_probe_transfer(db, w.a, w.la, alice, 1, t1, 1, 0, 0, -7, 5s),
ProbeStoreError::invalid));
CHECK(failed(prepare_probe_transfer(db, w.a, w.la, alice, 1, t1, 99, 0, 0, -7, 5s),
ProbeStoreError::invalid));
CHECK(failed(stage_probe_transfer(db, w.b, w.lb, t1, 5s), ProbeStoreError::unknown_transfer));
// Prepare: frozen at its final state; nothing else may change it; retries are answered.
auto p = prepare_probe_transfer(db, w.a, w.la, alice, 1, t1, 2, 700'000, 10, -7, 5s);
CHECK(p && p->state == TransferState::prepared && p->source == 1 && p->destination == 2 &&
p->x == 700'000 && p->inventory == inventory && p->account == alice);
CHECK(prepare_probe_transfer(db, w.a, w.la, alice, 1, t1, 2, 1, 1, 1, 5s) == *p);
CHECK(failed(save_probe(db, w.a, w.la, alice, 1, 0, 0, -7, 5s), ProbeStoreError::fenced));
CHECK(failed(prepare_probe_transfer(db, w.a, w.la, alice, 1, id(0x71, 2), 3, 0, 0, -7, 5s),
ProbeStoreError::fenced)); // One open transfer per probe.
CHECK(w.open(w.a, w.la, alice).transfer == t1);
invariants(*w.admin, w.probes);
// Only the named destination stages and commits; a commit before staging is refused.
CHECK(failed(stage_probe_transfer(db, w.c, w.lc, t1, 5s), ProbeStoreError::fenced));
CHECK(failed(commit_probe_transfer(db, w.b, w.lb, t1, 5s), ProbeStoreError::not_staged));
CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t1, 5s)) == TransferState::staged);
CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t1, 5s)) == TransferState::staged);
CHECK(failed(commit_probe_transfer(db, w.c, w.lc, t1, 5s), ProbeStoreError::fenced));
// Schema v13: the destination sees the open transfer addressed to it (for a ghost).
const auto incoming = incoming_probe_transfers(db, w.b, w.lb, 5s);
CHECK(incoming && incoming->size() == 1 && incoming->front().transfer == t1 &&
incoming->front().account == alice && incoming->front().source == 1 &&
incoming->front().x == 700'000);
CHECK(incoming_probe_transfers(db, w.c, w.lc, 5s)
.value_or(std::vector<IncomingTransfer>{1})
.empty());
// The one switch: owner B, next generation, the frozen final state, same inventory.
const auto committed = commit_probe_transfer(db, w.b, w.lb, t1, 5s);
CHECK(state_of(committed) == TransferState::committed);
r = w.open(w.b, w.lb, alice);
CHECK(r.region == 2 && r.generation == 2 && r.x == 700'000 && r.y == 10 &&
r.inventory == inventory && !r.transfer);
// Every retry and late call returns the committed outcome and changes nothing.
CHECK(commit_probe_transfer(db, w.b, w.lb, t1, 5s) == *committed);
CHECK(stage_probe_transfer(db, w.b, w.lb, t1, 5s) == *committed);
CHECK(prepare_probe_transfer(db, w.a, w.la, alice, 1, t1, 2, 0, 0, 0, 5s) == *committed);
CHECK(abort_probe_transfer(db, w.a, w.la, t1, 5s) == *committed);
CHECK(incoming_probe_transfers(db, w.b, w.lb, 5s)
.value_or(std::vector<IncomingTransfer>{1})
.empty()); // Decided transfers are no longer incoming.
const auto found = find_probe_transfer(db, w.c, w.lc, t1, 5s);
CHECK(found && *found == std::optional{*committed});
const auto missing = find_probe_transfer(db, w.c, w.lc, id(0x7F), 5s);
CHECK(missing && !missing->has_value());
// The old owner is fenced; the new one acts.
CHECK(failed(save_probe(db, w.a, w.la, alice, 1, 0, 0, -7, 5s), ProbeStoreError::fenced));
CHECK(save_probe(db, w.b, w.lb, alice, 2, 800'000, 10, -7, 5s));
// A transfer id belongs to one account and source.
const auto bob = id(0x42);
w.open(w.a, w.la, bob);
++w.probes;
CHECK(failed(prepare_probe_transfer(db, w.a, w.la, bob, 1, t1, 2, 0, 0, -7, 5s),
ProbeStoreError::fenced));
// Crossing back: B -> A with a new transfer, next generation.
const auto t3 = id(0x71, 3);
CHECK(state_of(prepare_probe_transfer(db, w.b, w.lb, alice, 2, t3, 1, 800'000, 10, -7, 5s)) ==
TransferState::prepared);
CHECK(state_of(stage_probe_transfer(db, w.a, w.la, t3, 5s)) == TransferState::staged);
CHECK(state_of(commit_probe_transfer(db, w.a, w.la, t3, 5s)) == TransferState::committed);
r = w.open(w.a, w.la, alice);
CHECK(r.region == 1 && r.generation == 3 && r.inventory == inventory);
invariants(*w.admin, w.probes);
}
// A crash (restart: new epoch) at each transition, on either side.
void restarts(World& w) {
auto& db = *w.db;
const auto carol = id(0x43);
w.open(w.a, w.la, carol);
++w.probes;
// 1. Source crashes after prepare. Its new epoch sees the probe frozen (it must not act)
// and the transfer proceeds without it.
const auto t = id(0x72, 1);
CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, carol, 1, t, 2, 1, 2, -7, 5s)) ==
TransferState::prepared);
w.la = restart(db, w.a, w.la);
const auto frozen = w.open(w.a, w.la, carol);
CHECK(frozen.region == 1 && frozen.transfer == t);
CHECK(failed(save_probe(db, w.a, w.la, carol, 1, 0, 0, -7, 5s), ProbeStoreError::fenced));
// The new source epoch may still answer retries of its own prepare.
CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, carol, 1, t, 2, 1, 2, -7, 5s)) ==
TransferState::prepared);
CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::staged);
// 2. Destination crashes after staging: its new epoch must stage again before committing;
// the old epoch is fenced everywhere.
const auto old_b = w.lb;
w.lb = restart(db, w.b, w.lb);
CHECK(failed(commit_probe_transfer(db, w.b, w.lb, t, 5s), ProbeStoreError::not_staged));
CHECK(failed(commit_probe_transfer(db, w.b, old_b, t, 5s), ProbeStoreError::fenced));
CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::staged);
// 3. Destination crashes after its commit, before observing it: the retry answers.
CHECK(state_of(commit_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::committed);
w.lb = restart(db, w.b, w.lb);
CHECK(state_of(commit_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::committed);
const auto r = w.open(w.b, w.lb, carol);
CHECK(r.region == 2 && r.generation == 2 && r.x == 1 && r.y == 2 && !r.transfer);
invariants(*w.admin, w.probes);
}
// Failed destination, abort rules, partition, and a commit/abort race.
void aborts_and_races(World& w) {
auto& db = *w.db;
const auto dave = id(0x44);
w.open(w.a, w.la, dave);
++w.probes;
// The destination never stages: the source aborts and resumes, durably.
const auto t1 = id(0x73, 1);
CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, dave, 1, t1, 2, 5, 5, -7, 5s)) ==
TransferState::prepared);
CHECK(failed(abort_probe_transfer(db, w.c, w.lc, t1, 5s), ProbeStoreError::fenced));
CHECK(state_of(abort_probe_transfer(db, w.a, w.la, t1, 5s)) == TransferState::aborted);
CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t1, 5s)) == TransferState::aborted);
CHECK(state_of(commit_probe_transfer(db, w.b, w.lb, t1, 5s)) == TransferState::aborted);
auto r = w.open(w.a, w.la, dave);
CHECK(r.region == 1 && r.generation == 1 && !r.transfer && r.x == 5);
CHECK(save_probe(db, w.a, w.la, dave, 1, 6, 6, -7, 5s)); // Active again at its source.
// The destination aborts a staged transfer it cannot accept.
const auto t2 = id(0x73, 2);
CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, dave, 1, t2, 2, 6, 6, -7, 5s)) ==
TransferState::prepared);
CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t2, 5s)) == TransferState::staged);
CHECK(state_of(abort_probe_transfer(db, w.b, w.lb, t2, 5s)) == TransferState::aborted);
CHECK(!w.open(w.a, w.la, dave).transfer);
// Partition: the source's lease lapses (it stops) while the destination commits; on
// recovery the source finds the probe owned elsewhere and the transfer committed.
const auto t3 = id(0x73, 3);
CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, dave, 1, t3, 2, 7, 7, -7, 5s)) ==
TransferState::prepared);
CHECK(release_region_lease(db, w.a, w.la, 5s)); // Cut off: no source authority at all.
CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t3, 5s)) == TransferState::staged);
CHECK(state_of(commit_probe_transfer(db, w.b, w.lb, t3, 5s)) == TransferState::committed);
w.la = acquire(db, w.a);
CHECK(state_of(find_probe_transfer(db, w.a, w.la, t3, 5s).value_or(std::nullopt)) ==
TransferState::committed);
r = w.open(w.a, w.la, dave);
CHECK(r.region == 2 && r.generation == 2);
CHECK(failed(save_probe(db, w.a, w.la, dave, 2, 0, 0, -7, 5s), ProbeStoreError::fenced));
invariants(*w.admin, w.probes);
// Race: the destination commits while the source aborts, on separate connections.
// Exactly one wins, both observe the same outcome, and the owner agrees with it.
auto other = connect_as(env("EVO_PG_RUNTIME_USER"));
int commits = 0, aborts = 0;
for (std::uint8_t round = 0; round < 24; ++round) {
const auto account = id(0x50, round);
w.open(w.a, w.la, account);
++w.probes;
const auto t = id(0x74, round);
CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, account, 1, t, 2, 1, 1, -7, 5s)) ==
TransferState::prepared);
CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::staged);
std::latch go{2};
std::optional<TransferState> committed, aborted;
std::thread racer{[&] {
go.arrive_and_wait();
committed = state_of(commit_probe_transfer(*other, w.b, w.lb, t, 5s));
}};
go.arrive_and_wait();
aborted = state_of(abort_probe_transfer(db, w.a, w.la, t, 5s));
racer.join();
CHECK(committed == aborted);
const auto owner = w.open(w.a, w.la, account);
if (committed == TransferState::committed) {
++commits;
CHECK(owner.region == 2 && owner.generation == 2 && !owner.transfer);
} else {
++aborts;
CHECK(committed == TransferState::aborted && owner.region == 1 &&
owner.generation == 1 && !owner.transfer);
}
}
CHECK(commits + aborts == 24);
std::printf("probe handoff race commits=%d aborts=%d\n", commits, aborts);
invariants(*w.admin, w.probes);
}
// The destination takes the account's durable control on the strength of a transfer
// committed to it; nothing else (another region, an open or aborted transfer) qualifies.
void control_handoff(World& w) {
auto& db = *w.db;
const auto eve = id(0x45);
const auto none = id(0x46);
w.open(w.a, w.la, eve);
w.open(w.a, w.la, none);
w.probes += 2;
const auto claim = claim_account_control(db, w.a, w.la, eve, 1, 11, 20000ms, 5s);
CHECK(claim && claim->generation == 1);
const auto t = id(0x75, 1);
CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, eve, 1, t, 2, 0, 0, -7, 5s)) ==
TransferState::prepared);
const auto refused = [&](const RegionKey& key, const LeaseGrant& lease, const Uuid& account) {
const auto r = handoff_account_control(db, key, lease, account, t, 21, 20000ms, 5s);
return !r && r.error().kind == ControlStoreError::fenced;
};
CHECK(refused(w.b, w.lb, eve)); // Not committed yet.
CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::staged);
CHECK(state_of(commit_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::committed);
CHECK(refused(w.c, w.lc, eve)); // Not the destination.
CHECK(refused(w.b, w.lb, none)); // Not this transfer's account.
// The source still holds unexpired control: the destination must outlive it.
const auto waited = handoff_account_control(db, w.b, w.lb, eve, t, 21, 20000ms, 5s);
CHECK(waited && waited->generation == 2 && waited->wait > 0ms);
// After the source released (the normal path), there is nothing to wait for.
CHECK(!release_account_control(db, w.a, w.la, eve, 1, 5s).value_or(true)); // Moved on.
CHECK(release_account_control(db, w.b, w.lb, eve, 2, 5s).value_or(false));
const auto again = handoff_account_control(db, w.b, w.lb, eve, t, 22, 20000ms, 5s);
CHECK(again && again->generation == 3 && again->wait == 0ms);
CHECK(release_account_control(db, w.b, w.lb, eve, 3, 5s).value_or(false));
invariants(*w.admin, w.probes);
}
#ifdef EVO_POSTGRES_SESSIONS
// The session layer's HandoffLedger over these functions: stage, commit and control in one
// admission; idempotent on retry; refused for aborted, foreign or unknown transfers.
void handoff_ledger(World& w) {
auto& db = *w.db;
const auto frank = id(0x47);
w.open(w.a, w.la, frank);
++w.probes;
CHECK(claim_account_control(db, w.a, w.la, frank, 1, 31, 20000ms, 5s));
const auto t = id(0x76, 1);
CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, frank, 1, t, 2, 3, 4, -7, 5s)) ==
TransferState::prepared);
CHECK(release_account_control(db, w.a, w.la, frank, 1, 5s).value_or(false));
std::optional<LeaseGrant> current = w.lb;
evo::server::PgSlot slot{PgOptions{env("PGHOST"),
{},
env("PGPORT"),
env("PGDATABASE"),
env("EVO_PG_RUNTIME_USER"),
{},
{}},
5s};
evo::server::PostgresHandoffLedger ledger{slot, w.b, [&] { return current; }};
evo::network::TicketClaims ticket;
ticket.transfer = t;
ticket.account.bytes = frank;
ticket.source = 1;
ticket.destination = 2;
const auto admitted = ledger.admit(ticket, 41);
CHECK(admitted && admitted->lease.generation == 2 && admitted->lease.wait == 0ms &&
admitted->generation == 2 && admitted->x == 3 && admitted->y == 4);
const auto r = w.open(w.b, w.lb, frank);
CHECK(r.region == 2 && r.generation == 2 && r.x == 3 && !r.transfer);
// A retried login after an undecided outcome converges (control moves on again).
const auto again = ledger.admit(ticket, 42);
CHECK(again && again->lease.generation == 3);
// Refusals: another account, an unknown transfer, an aborted transfer; no lease.
auto wrong = ticket;
wrong.account.bytes = id(0x48);
CHECK(!ledger.admit(wrong, 43) &&
ledger.admit(wrong, 43).error() == evo::network::HandoffLedgerError::refused);
auto unknown = ticket;
unknown.transfer = id(0x76, 9);
CHECK(ledger.admit(unknown, 44).error() == evo::network::HandoffLedgerError::refused);
const auto t2 = id(0x76, 2);
CHECK(state_of(prepare_probe_transfer(db, w.b, w.lb, frank, 2, t2, 1, 3, 4, -7, 5s)) ==
TransferState::prepared);
// M10.8d: once the probe moved on (here frozen for the next transfer), a retry of the
// committed transfer's ticket no longer admits (nor takes control).
CHECK(ledger.admit(ticket, 47).error() == evo::network::HandoffLedgerError::refused);
CHECK(state_of(abort_probe_transfer(db, w.b, w.lb, t2, 5s)) == TransferState::aborted);
auto aborted = ticket;
aborted.transfer = t2;
aborted.source = 2;
evo::server::PostgresHandoffLedger at_a{slot, w.a, [&] { return std::optional{w.la}; }};
CHECK(at_a.admit(aborted, 45).error() == evo::network::HandoffLedgerError::refused);
current.reset();
CHECK(ledger.admit(ticket, 46).error() == evo::network::HandoffLedgerError::fenced);
CHECK(release_account_control(db, w.b, w.lb, frank, 3, 5s).value_or(false));
invariants(*w.admin, w.probes);
}
#endif
// Schema v14: a source's open outgoing transfers older than the given age, by the database
// clock, for recovery aborts; never another region's, never decided ones.
void overdue(World& w) {
auto& db = *w.db;
const auto gina = id(0x49);
w.open(w.a, w.la, gina);
++w.probes;
const auto t = id(0x77, 1);
CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, gina, 1, t, 2, 5, 6, -7, 5s)) ==
TransferState::prepared);
const auto has = [&](RegionKey key, const LeaseGrant& lease, std::chrono::seconds age) {
const auto found = overdue_probe_transfers(db, key, lease, age, 5s);
CHECK(found.has_value());
return std::ranges::find(*found, t) != found->end();
};
CHECK(!has(w.a, w.la, 60s));
std::this_thread::sleep_for(1100ms);
CHECK(has(w.a, w.la, 1s) && !has(w.b, w.lb, 1s));
CHECK(failed(overdue_probe_transfers(db, w.a, w.la, 0s, 5s), ProbeStoreError::invalid));
CHECK(state_of(abort_probe_transfer(db, w.a, w.la, t, 5s)) == TransferState::aborted);
CHECK(!has(w.a, w.la, 1s));
invariants(*w.admin, w.probes);
}
} // namespace
void run_probe_handoff_store_tests() {
World w;
ownership_and_happy_path(w);
restarts(w);
aborts_and_races(w);
control_handoff(w);
overdue(w);
#ifdef EVO_POSTGRES_SESSIONS
handoff_ledger(w);
#endif
CHECK(release_region_lease(*w.db, w.a, w.la, 5s));
CHECK(release_region_lease(*w.db, w.b, w.lb, 5s));
CHECK(release_region_lease(*w.db, w.c, w.lc, 5s));
std::puts("probe handoff integration passed");
}