Foundation / checks (deploy) (push) Successful in 3m29s
Foundation / checks (ecs) (push) Successful in 2m0s
Foundation / checks (client) (push) Successful in 12m51s
Foundation / checks (network) (push) Successful in 13m0s
Foundation / checks (tsan) (push) Successful in 7m50s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 32m4s
Foundation / checks (postgres) (push) Canceled after 19m13s
ADR-0096. Session protocol v4 (v3 is deployed and frozen) adds HandoffOffer, SessionRedirect, close reasons handed_off and moved, and ghost marker records. Sessions use the durable probe record: logins open it where it was saved (redirected when it lives elsewhere), and session ends save it. A marker crossing into a neighbour's area is frozen, prepared, its control released and a signed ticket offered; uncommitted transfers are aborted after the ticket's lifetime. The destination shows arriving probes as ghosts (schema v13 incoming_probe_transfers) and opens the marker where it crossed. Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
456 lines
23 KiB
C++
456 lines
23 KiB
C++
// Real PostgreSQL M10.8a probe handoff checks (ADR-0096, schema v12): ownership and
|
|
// fencing, the transfer state machine with retries at every step, source and destination
|
|
// restarts (new epochs) at every transition, stale epochs, commit/abort races, crossing back,
|
|
// failed destinations and partitions, and the destination's account control handoff, with
|
|
// unique-owner and conservation checks after each scenario. World 9200 is used only here. Run by
|
|
// tools/ci/postgres-check.sh.
|
|
#include "evo/persistence/account_control_store.hpp"
|
|
#include "evo/persistence/probe_store.hpp"
|
|
#include "evo/persistence/region_checkpoint.hpp"
|
|
#include "evo/persistence/region_lease_store.hpp"
|
|
#ifdef EVO_POSTGRES_SESSIONS
|
|
#include "evo/server/postgres_handoff_ledger.hpp"
|
|
#endif
|
|
#include <array>
|
|
#include <atomic>
|
|
#include <chrono>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <expected>
|
|
#include <latch>
|
|
#include <memory>
|
|
#include <optional>
|
|
#include <source_location>
|
|
#include <span>
|
|
#include <stdexcept>
|
|
#include <string>
|
|
#include <thread>
|
|
|
|
void run_probe_handoff_store_tests();
|
|
|
|
namespace {
|
|
using namespace evo::persistence;
|
|
using namespace std::chrono_literals;
|
|
void check(bool value, std::source_location at = std::source_location::current()) {
|
|
if (!value)
|
|
throw std::runtime_error("probe handoff store test line " + std::to_string(at.line()));
|
|
}
|
|
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
|
constexpr std::uint64_t world = 9200;
|
|
constexpr std::uint64_t inventory = 1000; // EXPERIMENT test inventory per probe.
|
|
|
|
std::string env(const char* key) {
|
|
const auto* value = std::getenv(key);
|
|
if (!value)
|
|
throw std::runtime_error("missing test environment");
|
|
return value;
|
|
}
|
|
std::unique_ptr<Postgres> connect_as(const std::string& user) {
|
|
PgOptions options{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}};
|
|
auto made = Postgres::connect(options, 5s);
|
|
CHECK(made);
|
|
return std::move(*made);
|
|
}
|
|
Uuid id(std::uint8_t seed, std::uint8_t tag = 0) {
|
|
Uuid out{};
|
|
out.fill(seed);
|
|
out.back() = tag;
|
|
return out;
|
|
}
|
|
std::string bytea(std::span<const std::uint8_t> bytes) {
|
|
constexpr char digits[] = "0123456789abcdef";
|
|
std::string out = "\\x";
|
|
for (const auto b : bytes) {
|
|
out.push_back(digits[b >> 4U]);
|
|
out.push_back(digits[b & 0x0FU]);
|
|
}
|
|
return out;
|
|
}
|
|
Sha256Digest digest(std::uint8_t seed) {
|
|
Sha256Digest out{};
|
|
out.fill(seed);
|
|
return out;
|
|
}
|
|
const Sha256Digest build = digest(0xB1);
|
|
RegionKey provision(Postgres& db, std::uint64_t region) {
|
|
CHECK(db.query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
|
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
|
|
{}, 5s));
|
|
const RegionKey key{world, region};
|
|
auto commit = id(0xAD, static_cast<std::uint8_t>(region)); // Unique commit ids per test.
|
|
auto g = genesis_manifest(key, 7, commit, 50000, build, 100, digest(0xA0));
|
|
CHECK(g);
|
|
auto bytes = encode_manifest(*g);
|
|
CHECK(bytes);
|
|
const std::array<std::string, 1> p{bytea(*bytes)};
|
|
CHECK(db.query("SELECT evolution.provision_region_checkpoint($1::bytea)", p, 5s));
|
|
return key;
|
|
}
|
|
LeaseGrant acquire(Postgres& db, RegionKey key) {
|
|
auto lease = acquire_region_lease(db, key, build, 20000ms, 5s);
|
|
CHECK(lease);
|
|
return *lease;
|
|
}
|
|
// A region process "restart": its lease is released and a new epoch acquired.
|
|
LeaseGrant restart(Postgres& db, RegionKey key, const LeaseGrant& lease) {
|
|
CHECK(release_region_lease(db, key, lease, 5s));
|
|
const auto next = acquire(db, key);
|
|
CHECK(next.epoch > lease.epoch);
|
|
return next;
|
|
}
|
|
bool failed(const auto& result, ProbeStoreError kind) {
|
|
return !result && result.error().kind == kind;
|
|
}
|
|
TransferState state_of(const auto& result) {
|
|
if (!result)
|
|
throw std::runtime_error("probe handoff store: missing transfer");
|
|
return result->state;
|
|
}
|
|
|
|
// World-wide invariants, read as the owner: one row per probe (primary key), the test
|
|
// inventory conserved, and a probe frozen exactly when it has one open transfer, which
|
|
// names its owner as the source.
|
|
void invariants(Postgres& admin, std::uint64_t probes) {
|
|
auto rows = admin.query(
|
|
"SELECT count(*)::text, coalesce(sum(inventory), 0)::text FROM evolution.probe_records "
|
|
"WHERE world_id = 9200",
|
|
{}, 5s);
|
|
CHECK(rows && rows->size() == 1);
|
|
CHECK((*rows)[0][0] == std::to_string(probes));
|
|
CHECK((*rows)[0][1] == std::to_string(probes * inventory));
|
|
rows = admin.query(
|
|
"SELECT count(*)::text FROM evolution.probe_records r WHERE r.world_id = 9200 AND "
|
|
"(r.transfer_id IS NOT NULL) <> EXISTS (SELECT 1 FROM evolution.probe_transfers t "
|
|
"WHERE t.world_id = r.world_id AND t.transfer_id = r.transfer_id AND t.state IN "
|
|
"('prepared', 'staged') AND t.source_region = r.region_id AND t.account_id = "
|
|
"r.account_id)",
|
|
{}, 5s);
|
|
CHECK(rows && rows->size() == 1 && (*rows)[0][0] == "0");
|
|
rows = admin.query("SELECT count(*)::text FROM evolution.probe_transfers t WHERE "
|
|
"t.world_id = 9200 AND t.state IN ('prepared', 'staged') AND NOT EXISTS "
|
|
"(SELECT 1 FROM evolution.probe_records r WHERE r.world_id = t.world_id "
|
|
"AND r.account_id = t.account_id AND r.transfer_id = t.transfer_id)",
|
|
{}, 5s);
|
|
CHECK(rows && rows->size() == 1 && (*rows)[0][0] == "0");
|
|
}
|
|
|
|
struct World {
|
|
std::unique_ptr<Postgres> admin = connect_as(env("PGUSER"));
|
|
std::unique_ptr<Postgres> db = connect_as(env("EVO_PG_RUNTIME_USER"));
|
|
RegionKey a = provision(*admin, 1), b = provision(*admin, 2), c = provision(*admin, 3);
|
|
LeaseGrant la = acquire(*db, a), lb = acquire(*db, b), lc = acquire(*db, c);
|
|
std::uint64_t probes{};
|
|
ProbeRecord open(const RegionKey& key, const LeaseGrant& lease, const Uuid& account,
|
|
std::int64_t x = 0) {
|
|
const auto r = open_probe(*db, key, lease, account, x, 0, -7, inventory, 5s);
|
|
CHECK(r);
|
|
return *r;
|
|
}
|
|
};
|
|
|
|
void ownership_and_happy_path(World& w) {
|
|
auto& db = *w.db;
|
|
// The runtime reaches the tables only through the functions.
|
|
CHECK(!db.query("SELECT count(*) FROM evolution.probe_records", {}, 5s));
|
|
CHECK(!db.query("SELECT count(*) FROM evolution.probe_transfers", {}, 5s));
|
|
const auto alice = id(0x41);
|
|
// Created owned by A; B sees A as owner and cannot act; reopening changes nothing.
|
|
auto r = w.open(w.a, w.la, alice, 500'000);
|
|
++w.probes;
|
|
CHECK(r.region == 1 && r.generation == 1 && r.x == 500'000 && r.inventory == inventory &&
|
|
!r.transfer);
|
|
const auto seen = w.open(w.b, w.lb, alice, 9);
|
|
CHECK(seen.region == 1 && seen.x == 500'000);
|
|
CHECK(save_probe(db, w.a, w.la, alice, 1, 600'000, 0, -7, 5s));
|
|
CHECK(failed(save_probe(db, w.b, w.lb, alice, 1, 0, 0, -7, 5s), ProbeStoreError::fenced));
|
|
CHECK(failed(save_probe(db, w.a, w.la, alice, 2, 0, 0, -7, 5s), ProbeStoreError::fenced));
|
|
invariants(*w.admin, w.probes);
|
|
// Arguments: a transfer to itself or to an unprovisioned region is refused.
|
|
const auto t1 = id(0x71, 1);
|
|
CHECK(failed(prepare_probe_transfer(db, w.a, w.la, alice, 1, t1, 1, 0, 0, -7, 5s),
|
|
ProbeStoreError::invalid));
|
|
CHECK(failed(prepare_probe_transfer(db, w.a, w.la, alice, 1, t1, 99, 0, 0, -7, 5s),
|
|
ProbeStoreError::invalid));
|
|
CHECK(failed(stage_probe_transfer(db, w.b, w.lb, t1, 5s), ProbeStoreError::unknown_transfer));
|
|
// Prepare: frozen at its final state; nothing else may change it; retries are answered.
|
|
auto p = prepare_probe_transfer(db, w.a, w.la, alice, 1, t1, 2, 700'000, 10, -7, 5s);
|
|
CHECK(p && p->state == TransferState::prepared && p->source == 1 && p->destination == 2 &&
|
|
p->x == 700'000 && p->inventory == inventory && p->account == alice);
|
|
CHECK(prepare_probe_transfer(db, w.a, w.la, alice, 1, t1, 2, 1, 1, 1, 5s) == *p);
|
|
CHECK(failed(save_probe(db, w.a, w.la, alice, 1, 0, 0, -7, 5s), ProbeStoreError::fenced));
|
|
CHECK(failed(prepare_probe_transfer(db, w.a, w.la, alice, 1, id(0x71, 2), 3, 0, 0, -7, 5s),
|
|
ProbeStoreError::fenced)); // One open transfer per probe.
|
|
CHECK(w.open(w.a, w.la, alice).transfer == t1);
|
|
invariants(*w.admin, w.probes);
|
|
// Only the named destination stages and commits; a commit before staging is refused.
|
|
CHECK(failed(stage_probe_transfer(db, w.c, w.lc, t1, 5s), ProbeStoreError::fenced));
|
|
CHECK(failed(commit_probe_transfer(db, w.b, w.lb, t1, 5s), ProbeStoreError::not_staged));
|
|
CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t1, 5s)) == TransferState::staged);
|
|
CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t1, 5s)) == TransferState::staged);
|
|
CHECK(failed(commit_probe_transfer(db, w.c, w.lc, t1, 5s), ProbeStoreError::fenced));
|
|
// Schema v13: the destination sees the open transfer addressed to it (for a ghost).
|
|
const auto incoming = incoming_probe_transfers(db, w.b, w.lb, 5s);
|
|
CHECK(incoming && incoming->size() == 1 && incoming->front().transfer == t1 &&
|
|
incoming->front().account == alice && incoming->front().source == 1 &&
|
|
incoming->front().x == 700'000);
|
|
CHECK(incoming_probe_transfers(db, w.c, w.lc, 5s)
|
|
.value_or(std::vector<IncomingTransfer>{1})
|
|
.empty());
|
|
// The one switch: owner B, next generation, the frozen final state, same inventory.
|
|
const auto committed = commit_probe_transfer(db, w.b, w.lb, t1, 5s);
|
|
CHECK(state_of(committed) == TransferState::committed);
|
|
r = w.open(w.b, w.lb, alice);
|
|
CHECK(r.region == 2 && r.generation == 2 && r.x == 700'000 && r.y == 10 &&
|
|
r.inventory == inventory && !r.transfer);
|
|
// Every retry and late call returns the committed outcome and changes nothing.
|
|
CHECK(commit_probe_transfer(db, w.b, w.lb, t1, 5s) == *committed);
|
|
CHECK(stage_probe_transfer(db, w.b, w.lb, t1, 5s) == *committed);
|
|
CHECK(prepare_probe_transfer(db, w.a, w.la, alice, 1, t1, 2, 0, 0, 0, 5s) == *committed);
|
|
CHECK(abort_probe_transfer(db, w.a, w.la, t1, 5s) == *committed);
|
|
CHECK(incoming_probe_transfers(db, w.b, w.lb, 5s)
|
|
.value_or(std::vector<IncomingTransfer>{1})
|
|
.empty()); // Decided transfers are no longer incoming.
|
|
const auto found = find_probe_transfer(db, w.c, w.lc, t1, 5s);
|
|
CHECK(found && *found == std::optional{*committed});
|
|
const auto missing = find_probe_transfer(db, w.c, w.lc, id(0x7F), 5s);
|
|
CHECK(missing && !missing->has_value());
|
|
// The old owner is fenced; the new one acts.
|
|
CHECK(failed(save_probe(db, w.a, w.la, alice, 1, 0, 0, -7, 5s), ProbeStoreError::fenced));
|
|
CHECK(save_probe(db, w.b, w.lb, alice, 2, 800'000, 10, -7, 5s));
|
|
// A transfer id belongs to one account and source.
|
|
const auto bob = id(0x42);
|
|
w.open(w.a, w.la, bob);
|
|
++w.probes;
|
|
CHECK(failed(prepare_probe_transfer(db, w.a, w.la, bob, 1, t1, 2, 0, 0, -7, 5s),
|
|
ProbeStoreError::fenced));
|
|
// Crossing back: B -> A with a new transfer, next generation.
|
|
const auto t3 = id(0x71, 3);
|
|
CHECK(state_of(prepare_probe_transfer(db, w.b, w.lb, alice, 2, t3, 1, 800'000, 10, -7, 5s)) ==
|
|
TransferState::prepared);
|
|
CHECK(state_of(stage_probe_transfer(db, w.a, w.la, t3, 5s)) == TransferState::staged);
|
|
CHECK(state_of(commit_probe_transfer(db, w.a, w.la, t3, 5s)) == TransferState::committed);
|
|
r = w.open(w.a, w.la, alice);
|
|
CHECK(r.region == 1 && r.generation == 3 && r.inventory == inventory);
|
|
invariants(*w.admin, w.probes);
|
|
}
|
|
|
|
// A crash (restart: new epoch) at each transition, on either side.
|
|
void restarts(World& w) {
|
|
auto& db = *w.db;
|
|
const auto carol = id(0x43);
|
|
w.open(w.a, w.la, carol);
|
|
++w.probes;
|
|
// 1. Source crashes after prepare. Its new epoch sees the probe frozen (it must not act)
|
|
// and the transfer proceeds without it.
|
|
const auto t = id(0x72, 1);
|
|
CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, carol, 1, t, 2, 1, 2, -7, 5s)) ==
|
|
TransferState::prepared);
|
|
w.la = restart(db, w.a, w.la);
|
|
const auto frozen = w.open(w.a, w.la, carol);
|
|
CHECK(frozen.region == 1 && frozen.transfer == t);
|
|
CHECK(failed(save_probe(db, w.a, w.la, carol, 1, 0, 0, -7, 5s), ProbeStoreError::fenced));
|
|
// The new source epoch may still answer retries of its own prepare.
|
|
CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, carol, 1, t, 2, 1, 2, -7, 5s)) ==
|
|
TransferState::prepared);
|
|
CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::staged);
|
|
// 2. Destination crashes after staging: its new epoch must stage again before committing;
|
|
// the old epoch is fenced everywhere.
|
|
const auto old_b = w.lb;
|
|
w.lb = restart(db, w.b, w.lb);
|
|
CHECK(failed(commit_probe_transfer(db, w.b, w.lb, t, 5s), ProbeStoreError::not_staged));
|
|
CHECK(failed(commit_probe_transfer(db, w.b, old_b, t, 5s), ProbeStoreError::fenced));
|
|
CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::staged);
|
|
// 3. Destination crashes after its commit, before observing it: the retry answers.
|
|
CHECK(state_of(commit_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::committed);
|
|
w.lb = restart(db, w.b, w.lb);
|
|
CHECK(state_of(commit_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::committed);
|
|
const auto r = w.open(w.b, w.lb, carol);
|
|
CHECK(r.region == 2 && r.generation == 2 && r.x == 1 && r.y == 2 && !r.transfer);
|
|
invariants(*w.admin, w.probes);
|
|
}
|
|
|
|
// Failed destination, abort rules, partition, and a commit/abort race.
|
|
void aborts_and_races(World& w) {
|
|
auto& db = *w.db;
|
|
const auto dave = id(0x44);
|
|
w.open(w.a, w.la, dave);
|
|
++w.probes;
|
|
// The destination never stages: the source aborts and resumes, durably.
|
|
const auto t1 = id(0x73, 1);
|
|
CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, dave, 1, t1, 2, 5, 5, -7, 5s)) ==
|
|
TransferState::prepared);
|
|
CHECK(failed(abort_probe_transfer(db, w.c, w.lc, t1, 5s), ProbeStoreError::fenced));
|
|
CHECK(state_of(abort_probe_transfer(db, w.a, w.la, t1, 5s)) == TransferState::aborted);
|
|
CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t1, 5s)) == TransferState::aborted);
|
|
CHECK(state_of(commit_probe_transfer(db, w.b, w.lb, t1, 5s)) == TransferState::aborted);
|
|
auto r = w.open(w.a, w.la, dave);
|
|
CHECK(r.region == 1 && r.generation == 1 && !r.transfer && r.x == 5);
|
|
CHECK(save_probe(db, w.a, w.la, dave, 1, 6, 6, -7, 5s)); // Active again at its source.
|
|
// The destination aborts a staged transfer it cannot accept.
|
|
const auto t2 = id(0x73, 2);
|
|
CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, dave, 1, t2, 2, 6, 6, -7, 5s)) ==
|
|
TransferState::prepared);
|
|
CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t2, 5s)) == TransferState::staged);
|
|
CHECK(state_of(abort_probe_transfer(db, w.b, w.lb, t2, 5s)) == TransferState::aborted);
|
|
CHECK(!w.open(w.a, w.la, dave).transfer);
|
|
// Partition: the source's lease lapses (it stops) while the destination commits; on
|
|
// recovery the source finds the probe owned elsewhere and the transfer committed.
|
|
const auto t3 = id(0x73, 3);
|
|
CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, dave, 1, t3, 2, 7, 7, -7, 5s)) ==
|
|
TransferState::prepared);
|
|
CHECK(release_region_lease(db, w.a, w.la, 5s)); // Cut off: no source authority at all.
|
|
CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t3, 5s)) == TransferState::staged);
|
|
CHECK(state_of(commit_probe_transfer(db, w.b, w.lb, t3, 5s)) == TransferState::committed);
|
|
w.la = acquire(db, w.a);
|
|
CHECK(state_of(find_probe_transfer(db, w.a, w.la, t3, 5s).value_or(std::nullopt)) ==
|
|
TransferState::committed);
|
|
r = w.open(w.a, w.la, dave);
|
|
CHECK(r.region == 2 && r.generation == 2);
|
|
CHECK(failed(save_probe(db, w.a, w.la, dave, 2, 0, 0, -7, 5s), ProbeStoreError::fenced));
|
|
invariants(*w.admin, w.probes);
|
|
// Race: the destination commits while the source aborts, on separate connections.
|
|
// Exactly one wins, both observe the same outcome, and the owner agrees with it.
|
|
auto other = connect_as(env("EVO_PG_RUNTIME_USER"));
|
|
int commits = 0, aborts = 0;
|
|
for (std::uint8_t round = 0; round < 24; ++round) {
|
|
const auto account = id(0x50, round);
|
|
w.open(w.a, w.la, account);
|
|
++w.probes;
|
|
const auto t = id(0x74, round);
|
|
CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, account, 1, t, 2, 1, 1, -7, 5s)) ==
|
|
TransferState::prepared);
|
|
CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::staged);
|
|
std::latch go{2};
|
|
std::optional<TransferState> committed, aborted;
|
|
std::thread racer{[&] {
|
|
go.arrive_and_wait();
|
|
committed = state_of(commit_probe_transfer(*other, w.b, w.lb, t, 5s));
|
|
}};
|
|
go.arrive_and_wait();
|
|
aborted = state_of(abort_probe_transfer(db, w.a, w.la, t, 5s));
|
|
racer.join();
|
|
CHECK(committed == aborted);
|
|
const auto owner = w.open(w.a, w.la, account);
|
|
if (committed == TransferState::committed) {
|
|
++commits;
|
|
CHECK(owner.region == 2 && owner.generation == 2 && !owner.transfer);
|
|
} else {
|
|
++aborts;
|
|
CHECK(committed == TransferState::aborted && owner.region == 1 &&
|
|
owner.generation == 1 && !owner.transfer);
|
|
}
|
|
}
|
|
CHECK(commits + aborts == 24);
|
|
std::printf("probe handoff race commits=%d aborts=%d\n", commits, aborts);
|
|
invariants(*w.admin, w.probes);
|
|
}
|
|
// The destination takes the account's durable control on the strength of a transfer
|
|
// committed to it; nothing else (another region, an open or aborted transfer) qualifies.
|
|
void control_handoff(World& w) {
|
|
auto& db = *w.db;
|
|
const auto eve = id(0x45);
|
|
const auto none = id(0x46);
|
|
w.open(w.a, w.la, eve);
|
|
w.open(w.a, w.la, none);
|
|
w.probes += 2;
|
|
const auto claim = claim_account_control(db, w.a, w.la, eve, 1, 11, 20000ms, 5s);
|
|
CHECK(claim && claim->generation == 1);
|
|
const auto t = id(0x75, 1);
|
|
CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, eve, 1, t, 2, 0, 0, -7, 5s)) ==
|
|
TransferState::prepared);
|
|
const auto refused = [&](const RegionKey& key, const LeaseGrant& lease, const Uuid& account) {
|
|
const auto r = handoff_account_control(db, key, lease, account, t, 21, 20000ms, 5s);
|
|
return !r && r.error().kind == ControlStoreError::fenced;
|
|
};
|
|
CHECK(refused(w.b, w.lb, eve)); // Not committed yet.
|
|
CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::staged);
|
|
CHECK(state_of(commit_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::committed);
|
|
CHECK(refused(w.c, w.lc, eve)); // Not the destination.
|
|
CHECK(refused(w.b, w.lb, none)); // Not this transfer's account.
|
|
// The source still holds unexpired control: the destination must outlive it.
|
|
const auto waited = handoff_account_control(db, w.b, w.lb, eve, t, 21, 20000ms, 5s);
|
|
CHECK(waited && waited->generation == 2 && waited->wait > 0ms);
|
|
// After the source released (the normal path), there is nothing to wait for.
|
|
CHECK(!release_account_control(db, w.a, w.la, eve, 1, 5s).value_or(true)); // Moved on.
|
|
CHECK(release_account_control(db, w.b, w.lb, eve, 2, 5s).value_or(false));
|
|
const auto again = handoff_account_control(db, w.b, w.lb, eve, t, 22, 20000ms, 5s);
|
|
CHECK(again && again->generation == 3 && again->wait == 0ms);
|
|
CHECK(release_account_control(db, w.b, w.lb, eve, 3, 5s).value_or(false));
|
|
invariants(*w.admin, w.probes);
|
|
}
|
|
#ifdef EVO_POSTGRES_SESSIONS
|
|
// The session layer's HandoffLedger over these functions: stage, commit and control in one
|
|
// admission; idempotent on retry; refused for aborted, foreign or unknown transfers.
|
|
void handoff_ledger(World& w) {
|
|
auto& db = *w.db;
|
|
const auto frank = id(0x47);
|
|
w.open(w.a, w.la, frank);
|
|
++w.probes;
|
|
CHECK(claim_account_control(db, w.a, w.la, frank, 1, 31, 20000ms, 5s));
|
|
const auto t = id(0x76, 1);
|
|
CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, frank, 1, t, 2, 3, 4, -7, 5s)) ==
|
|
TransferState::prepared);
|
|
CHECK(release_account_control(db, w.a, w.la, frank, 1, 5s).value_or(false));
|
|
std::optional<LeaseGrant> current = w.lb;
|
|
evo::server::PgSlot slot{PgOptions{env("PGHOST"),
|
|
{},
|
|
env("PGPORT"),
|
|
env("PGDATABASE"),
|
|
env("EVO_PG_RUNTIME_USER"),
|
|
{},
|
|
{}},
|
|
5s};
|
|
evo::server::PostgresHandoffLedger ledger{slot, w.b, [&] { return current; }};
|
|
evo::network::TicketClaims ticket;
|
|
ticket.transfer = t;
|
|
ticket.account.bytes = frank;
|
|
ticket.source = 1;
|
|
ticket.destination = 2;
|
|
const auto admitted = ledger.admit(ticket, 41);
|
|
CHECK(admitted && admitted->lease.generation == 2 && admitted->lease.wait == 0ms &&
|
|
admitted->generation == 2 && admitted->x == 3 && admitted->y == 4);
|
|
const auto r = w.open(w.b, w.lb, frank);
|
|
CHECK(r.region == 2 && r.generation == 2 && r.x == 3 && !r.transfer);
|
|
// A retried login after an undecided outcome converges (control moves on again).
|
|
const auto again = ledger.admit(ticket, 42);
|
|
CHECK(again && again->lease.generation == 3);
|
|
// Refusals: another account, an unknown transfer, an aborted transfer; no lease.
|
|
auto wrong = ticket;
|
|
wrong.account.bytes = id(0x48);
|
|
CHECK(!ledger.admit(wrong, 43) &&
|
|
ledger.admit(wrong, 43).error() == evo::network::HandoffLedgerError::refused);
|
|
auto unknown = ticket;
|
|
unknown.transfer = id(0x76, 9);
|
|
CHECK(ledger.admit(unknown, 44).error() == evo::network::HandoffLedgerError::refused);
|
|
const auto t2 = id(0x76, 2);
|
|
CHECK(state_of(prepare_probe_transfer(db, w.b, w.lb, frank, 2, t2, 1, 3, 4, -7, 5s)) ==
|
|
TransferState::prepared);
|
|
CHECK(state_of(abort_probe_transfer(db, w.b, w.lb, t2, 5s)) == TransferState::aborted);
|
|
auto aborted = ticket;
|
|
aborted.transfer = t2;
|
|
aborted.source = 2;
|
|
evo::server::PostgresHandoffLedger at_a{slot, w.a, [&] { return std::optional{w.la}; }};
|
|
CHECK(at_a.admit(aborted, 45).error() == evo::network::HandoffLedgerError::refused);
|
|
current.reset();
|
|
CHECK(ledger.admit(ticket, 46).error() == evo::network::HandoffLedgerError::fenced);
|
|
CHECK(release_account_control(db, w.b, w.lb, frank, 3, 5s).value_or(false));
|
|
invariants(*w.admin, w.probes);
|
|
}
|
|
#endif
|
|
} // namespace
|
|
|
|
void run_probe_handoff_store_tests() {
|
|
World w;
|
|
ownership_and_happy_path(w);
|
|
restarts(w);
|
|
aborts_and_races(w);
|
|
control_handoff(w);
|
|
#ifdef EVO_POSTGRES_SESSIONS
|
|
handoff_ledger(w);
|
|
#endif
|
|
CHECK(release_region_lease(*w.db, w.a, w.la, 5s));
|
|
CHECK(release_region_lease(*w.db, w.b, w.lb, 5s));
|
|
CHECK(release_region_lease(*w.db, w.c, w.lc, 5s));
|
|
std::puts("probe handoff integration passed");
|
|
}
|