Files
aevolution/tests/account_control_store_tests.cpp
T
dingoandClaude Opus 5.5 890c57a77f
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 1m50s
Foundation / checks (network) (push) Successful in 9m8s
Foundation / checks (client) (push) Successful in 12m10s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (postgres) (push) Canceled after 7m33s
Foundation / checks (checks) (push) Canceled after 18m49s
Foundation / checks (tsan) (push) Canceled after 6m38s
Add M10.3c durable per-world account control (schema v10)
Migration v10 keeps, per world and account, the highest accepted grant serial and
the region session in control with a fencing generation and database-time expiry,
reachable only through claim/renew/release functions under the caller's region
lease. Serials survive restarts and apply across regions; a cross-region takeover
returns a wait so control never overlaps. RegionSessions takes durable control via
a ControlLedger before admitting a session, delays Welcome by the wait, renews, and
closes sessions whose control is fenced or cannot be confirmed in time.
PostgresControlLedger implements the ledger. ADR-0091.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-03 21:50:21 -04:00

236 lines
10 KiB
C++

// Real PostgreSQL M10.3c account control checks (ADR-0091, schema v10): durable grant
// serials across regions and restarts, cross-region takeover with a wait, fencing by
// generation, region lease and expiry, and contention. World 9100 is used only here.
// Run by tools/ci/postgres-check.sh.
#include "evo/persistence/account_control_store.hpp"
#include "evo/persistence/region_checkpoint.hpp"
#include "evo/persistence/region_lease_store.hpp"
#ifdef EVO_POSTGRES_SESSIONS
#include "evo/server/postgres_control_ledger.hpp"
#endif
#include <array>
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <expected>
#include <latch>
#include <memory>
#include <optional>
#include <source_location>
#include <span>
#include <stdexcept>
#include <string>
#include <thread>
void run_account_control_store_tests();
namespace {
using namespace evo::persistence;
using namespace std::chrono_literals;
using std::chrono::milliseconds;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("account control store test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
constexpr std::uint64_t world = 9100;
std::string env(const char* key) {
const auto* value = std::getenv(key);
if (!value)
throw std::runtime_error("missing test environment");
return value;
}
std::unique_ptr<Postgres> connect_as(const std::string& user) {
PgOptions options{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}};
auto made = Postgres::connect(options, 5s);
CHECK(made);
return std::move(*made);
}
Uuid id(std::uint8_t seed) {
Uuid out{};
out.fill(seed);
return out;
}
Sha256Digest digest(std::uint8_t seed) {
Sha256Digest out{};
out.fill(seed);
return out;
}
std::string bytea(std::span<const std::uint8_t> bytes) {
constexpr char digits[] = "0123456789abcdef";
std::string out = "\\x";
for (const auto b : bytes) {
out.push_back(digits[b >> 4U]);
out.push_back(digits[b & 0x0FU]);
}
return out;
}
const Sha256Digest build = digest(0xB1);
RegionKey provision(Postgres& db, std::uint64_t region) {
CHECK(db.query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
{}, 5s));
const RegionKey key{world, region};
auto commit = id(0xAC); // Commit ids are unique across the database: tag this test's.
commit.back() = static_cast<std::uint8_t>(region);
auto g = genesis_manifest(key, 7, commit, 50000, build, 100, digest(0xA0));
CHECK(g);
auto bytes = encode_manifest(*g);
CHECK(bytes);
const std::array<std::string, 1> p{bytea(*bytes)};
const auto provisioned =
db.query("SELECT evolution.provision_region_checkpoint($1::bytea)", p, 5s);
if (!provisioned)
std::fprintf(stderr, "provision sqlstate %s\n", provisioned.error().sqlstate.data());
CHECK(provisioned);
return key;
}
LeaseGrant acquire(Postgres& db, RegionKey key) {
auto lease = acquire_region_lease(db, key, build, 20000ms, 5s);
CHECK(lease);
return *lease;
}
bool failed(const auto& result, ControlStoreError kind) {
return !result && result.error().kind == kind;
}
// Database-time expiry of one account's control (owner read), as for region leases.
void wait_until_expired(Postgres& owner_db, const Uuid& account) {
const std::array<std::string, 1> p{bytea(account)};
for (int attempt = 0; attempt < 400; ++attempt) {
auto rows = owner_db.query(
"SELECT (clock_timestamp() >= expires_at)::text FROM evolution.account_controls "
"WHERE world_id = 9100 AND account_id = encode($1::bytea, 'hex')::uuid",
p, 5s);
CHECK(rows && rows->size() == 1);
if ((*rows)[0][0] == "true")
return;
std::this_thread::sleep_for(50ms);
}
CHECK(false);
}
void serials_takeover_and_fencing() {
auto admin = connect_as(env("PGUSER"));
const auto a = provision(*admin, 1);
const auto b = provision(*admin, 2);
auto db = connect_as(env("EVO_PG_RUNTIME_USER"));
const auto lease_a = acquire(*db, a);
const auto lease_b = acquire(*db, b);
const auto alice = id(0x31);
// The runtime reaches the table only through the functions.
CHECK(!db->query("SELECT count(*) FROM evolution.account_controls", {}, 5s));
auto first = claim_account_control(*db, a, lease_a, alice, 5, 1, 20000ms, 5s);
CHECK(first && first->generation == 1 && first->wait == 0ms && first->granted == 20000ms);
CHECK(failed(claim_account_control(*db, a, lease_a, alice, 5, 2, 20000ms, 5s),
ControlStoreError::stale_grant)); // Replay.
CHECK(failed(claim_account_control(*db, a, lease_a, alice, 4, 2, 20000ms, 5s),
ControlStoreError::stale_grant)); // Older grant.
// Same region takeover: no wait; the old generation is fenced.
auto second = claim_account_control(*db, a, lease_a, alice, 6, 2, 20000ms, 5s);
CHECK(second && second->generation == 2 && second->wait == 0ms);
CHECK(failed(renew_account_control(*db, a, lease_a, alice, 1, 20000ms, 5s),
ControlStoreError::fenced));
CHECK(renew_account_control(*db, a, lease_a, alice, 2, 20000ms, 5s) == 20000ms);
// Another region with a newer grant takes control but must wait out region A's.
auto moved = claim_account_control(*db, b, lease_b, alice, 7, 1, 20000ms, 5s);
CHECK(moved && moved->generation == 3 && moved->wait > 15000ms && moved->wait <= 20000ms);
CHECK(failed(renew_account_control(*db, a, lease_a, alice, 2, 20000ms, 5s),
ControlStoreError::fenced));
CHECK(release_account_control(*db, a, lease_a, alice, 2, 5s) == false);
// Region B's renewal never shortens the waited-out expiry.
CHECK(renew_account_control(*db, b, lease_b, alice, 3, 1000ms, 5s) == 1000ms);
CHECK(release_account_control(*db, b, lease_b, alice, 3, 5s) == true);
CHECK(failed(renew_account_control(*db, b, lease_b, alice, 3, 20000ms, 5s),
ControlStoreError::fenced));
// Released control is free: the next claim waits for nobody; the serial stays durable.
CHECK(failed(claim_account_control(*db, a, lease_a, alice, 7, 3, 20000ms, 5s),
ControlStoreError::stale_grant));
auto back = claim_account_control(*db, a, lease_a, alice, 8, 3, 20000ms, 5s);
CHECK(back && back->wait == 0ms);
// Restart: a new process takes region A's lease under a new epoch. Serials survive,
// the old epoch's control needs no wait, and the old lease can do nothing.
CHECK(release_region_lease(*db, a, lease_a, 5s));
const auto restarted = acquire(*db, a);
CHECK(restarted.epoch > lease_a.epoch);
CHECK(failed(claim_account_control(*db, a, restarted, alice, 8, 1, 20000ms, 5s),
ControlStoreError::stale_grant));
auto resumed = claim_account_control(*db, a, restarted, alice, 9, 1, 20000ms, 5s);
CHECK(resumed && resumed->wait == 0ms);
CHECK(failed(claim_account_control(*db, a, lease_a, alice, 10, 4, 20000ms, 5s),
ControlStoreError::fenced));
CHECK(failed(renew_account_control(*db, a, lease_a, alice, resumed->generation, 20000ms, 5s),
ControlStoreError::fenced));
// Expiry is final, and an expired holder is not waited for.
const auto bob = id(0x32);
auto brief = claim_account_control(*db, a, restarted, bob, 1, 2, 1000ms, 5s);
CHECK(brief && brief->wait == 0ms);
wait_until_expired(*admin, bob);
CHECK(failed(renew_account_control(*db, a, restarted, bob, brief->generation, 20000ms, 5s),
ControlStoreError::fenced));
auto elsewhere = claim_account_control(*db, b, lease_b, bob, 2, 2, 20000ms, 5s);
CHECK(elsewhere && elsewhere->wait == 0ms);
// Invalid requests change nothing.
CHECK(failed(claim_account_control(*db, a, restarted, id(0x33), 1, 1, 500ms, 5s),
ControlStoreError::invalid));
CHECK(failed(claim_account_control(*db, a, restarted, id(0x33), 0, 1, 20000ms, 5s),
ControlStoreError::invalid));
CHECK(failed(claim_account_control(*db, a, restarted, id(0x33), 1, 0, 20000ms, 5s),
ControlStoreError::invalid));
CHECK(failed(claim_account_control(*db, a, restarted, Uuid{}, 1, 1, 20000ms, 5s),
ControlStoreError::invalid));
CHECK(claim_account_control(*db, a, restarted, id(0x33), 1, 1, 20000ms, 5s));
// Contention: two regions present the same newest serial at once; exactly one wins.
const auto carol = id(0x34);
auto first_db = connect_as(env("EVO_PG_RUNTIME_USER"));
auto second_db = connect_as(env("EVO_PG_RUNTIME_USER"));
std::latch start{2};
std::expected<AccountControlClaim, ControlStoreFailure> ra, rb;
std::thread ta{[&] {
start.arrive_and_wait();
ra = claim_account_control(*first_db, a, restarted, carol, 20, 7, 20000ms, 5s);
}};
std::thread tb{[&] {
start.arrive_and_wait();
rb = claim_account_control(*second_db, b, lease_b, carol, 20, 7, 20000ms, 5s);
}};
ta.join();
tb.join();
CHECK(ra.has_value() != rb.has_value());
CHECK(failed(ra.has_value() ? rb : ra, ControlStoreError::stale_grant));
#ifdef EVO_POSTGRES_SESSIONS
// The session layer's ledger over these functions.
std::optional<LeaseGrant> current = restarted;
evo::server::PostgresControlLedger ledger{*db, a, [&] { return current; }};
evo::network::AccountId dave;
dave.bytes = id(0x35);
auto claimed = ledger.claim(dave, 1, 9);
CHECK(claimed && claimed->wait == 0ms && claimed->granted == 20000ms);
CHECK(!ledger.claim(dave, 1, 9) &&
ledger.claim(dave, 1, 9).error() == evo::network::ControlLedgerError::stale_grant);
CHECK(ledger.renew(dave, claimed->generation) == 20000ms);
ledger.release(dave, claimed->generation);
CHECK(!ledger.renew(dave, claimed->generation) &&
ledger.renew(dave, claimed->generation).error() ==
evo::network::ControlLedgerError::fenced);
current.reset(); // The host lost its lease: nothing is granted or renewed.
CHECK(!ledger.claim(dave, 2, 9) &&
ledger.claim(dave, 2, 9).error() == evo::network::ControlLedgerError::fenced);
#endif
CHECK(release_region_lease(*db, a, restarted, 5s));
CHECK(release_region_lease(*db, b, lease_b, 5s));
}
} // namespace
void run_account_control_store_tests() {
serials_takeover_and_fencing();
std::puts("account control integration passed");
}