Files
aevolution/tests/account_control_store_tests.cpp
dingoandClaude Opus 5.5 0bf9901c7f
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 1m46s
Foundation / checks (postgres) (push) Canceled after 0s
Foundation / checks (tsan) (push) Canceled after 0s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 3m44s
Foundation / checks (client) (push) Canceled after 3m49s
Foundation / checks (network) (push) Canceled after 1m59s
Add M10.4c evo-server QUIC listener and headless session client
evo-server gains an opt-in M10 listener (--m10-*, QUIC builds only): RegionSessions
with the PostgreSQL control and action ledgers on their own connection under the
host's lease, carried actions submitted to the host and its acknowledgements and
drops turned into outcomes. evo-session-probe is a dev-only headless client. Real
process tests cover intentions, durable action outcomes, SIGKILL before and after
an action's commit, duplicate retries and the probe. The simulated watermark now
reaches every session every tick, so clients always know the server tick.
ADR-0092.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-04 09:20:55 -04:00

244 lines
11 KiB
C++

// Real PostgreSQL M10.3c account control checks (ADR-0091, schema v10): durable grant
// serials across regions and restarts, cross-region takeover with a wait, fencing by
// generation, region lease and expiry, and contention. World 9100 is used only here.
// Run by tools/ci/postgres-check.sh.
#include "evo/persistence/account_control_store.hpp"
#include "evo/persistence/region_checkpoint.hpp"
#include "evo/persistence/region_lease_store.hpp"
#ifdef EVO_POSTGRES_SESSIONS
#include "evo/server/postgres_control_ledger.hpp"
#endif
#include <array>
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <expected>
#include <latch>
#include <memory>
#include <optional>
#include <source_location>
#include <span>
#include <stdexcept>
#include <string>
#include <thread>
void run_account_control_store_tests();
namespace {
using namespace evo::persistence;
using namespace std::chrono_literals;
using std::chrono::milliseconds;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("account control store test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
constexpr std::uint64_t world = 9100;
std::string env(const char* key) {
const auto* value = std::getenv(key);
if (!value)
throw std::runtime_error("missing test environment");
return value;
}
std::unique_ptr<Postgres> connect_as(const std::string& user) {
PgOptions options{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}};
auto made = Postgres::connect(options, 5s);
CHECK(made);
return std::move(*made);
}
Uuid id(std::uint8_t seed) {
Uuid out{};
out.fill(seed);
return out;
}
Sha256Digest digest(std::uint8_t seed) {
Sha256Digest out{};
out.fill(seed);
return out;
}
std::string bytea(std::span<const std::uint8_t> bytes) {
constexpr char digits[] = "0123456789abcdef";
std::string out = "\\x";
for (const auto b : bytes) {
out.push_back(digits[b >> 4U]);
out.push_back(digits[b & 0x0FU]);
}
return out;
}
const Sha256Digest build = digest(0xB1);
RegionKey provision(Postgres& db, std::uint64_t region) {
CHECK(db.query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
{}, 5s));
const RegionKey key{world, region};
auto commit = id(0xAC); // Commit ids are unique across the database: tag this test's.
commit.back() = static_cast<std::uint8_t>(region);
auto g = genesis_manifest(key, 7, commit, 50000, build, 100, digest(0xA0));
CHECK(g);
auto bytes = encode_manifest(*g);
CHECK(bytes);
const std::array<std::string, 1> p{bytea(*bytes)};
const auto provisioned =
db.query("SELECT evolution.provision_region_checkpoint($1::bytea)", p, 5s);
if (!provisioned)
std::fprintf(stderr, "provision sqlstate %s\n", provisioned.error().sqlstate.data());
CHECK(provisioned);
return key;
}
LeaseGrant acquire(Postgres& db, RegionKey key) {
auto lease = acquire_region_lease(db, key, build, 20000ms, 5s);
CHECK(lease);
return *lease;
}
bool failed(const auto& result, ControlStoreError kind) {
return !result && result.error().kind == kind;
}
// Database-time expiry of one account's control (owner read), as for region leases.
void wait_until_expired(Postgres& owner_db, const Uuid& account) {
const std::array<std::string, 1> p{bytea(account)};
for (int attempt = 0; attempt < 400; ++attempt) {
auto rows = owner_db.query(
"SELECT (clock_timestamp() >= expires_at)::text FROM evolution.account_controls "
"WHERE world_id = 9100 AND account_id = encode($1::bytea, 'hex')::uuid",
p, 5s);
CHECK(rows && rows->size() == 1);
if ((*rows)[0][0] == "true")
return;
std::this_thread::sleep_for(50ms);
}
CHECK(false);
}
void serials_takeover_and_fencing() {
auto admin = connect_as(env("PGUSER"));
const auto a = provision(*admin, 1);
const auto b = provision(*admin, 2);
auto db = connect_as(env("EVO_PG_RUNTIME_USER"));
const auto lease_a = acquire(*db, a);
const auto lease_b = acquire(*db, b);
const auto alice = id(0x31);
// The runtime reaches the table only through the functions.
CHECK(!db->query("SELECT count(*) FROM evolution.account_controls", {}, 5s));
auto first = claim_account_control(*db, a, lease_a, alice, 5, 1, 20000ms, 5s);
CHECK(first && first->generation == 1 && first->wait == 0ms && first->granted == 20000ms);
CHECK(failed(claim_account_control(*db, a, lease_a, alice, 5, 2, 20000ms, 5s),
ControlStoreError::stale_grant)); // Replay.
CHECK(failed(claim_account_control(*db, a, lease_a, alice, 4, 2, 20000ms, 5s),
ControlStoreError::stale_grant)); // Older grant.
// Same region takeover: no wait; the old generation is fenced.
auto second = claim_account_control(*db, a, lease_a, alice, 6, 2, 20000ms, 5s);
CHECK(second && second->generation == 2 && second->wait == 0ms);
CHECK(failed(renew_account_control(*db, a, lease_a, alice, 1, 20000ms, 5s),
ControlStoreError::fenced));
CHECK(renew_account_control(*db, a, lease_a, alice, 2, 20000ms, 5s) == 20000ms);
// Another region with a newer grant takes control but must wait out region A's.
auto moved = claim_account_control(*db, b, lease_b, alice, 7, 1, 20000ms, 5s);
CHECK(moved && moved->generation == 3 && moved->wait > 15000ms && moved->wait <= 20000ms);
CHECK(failed(renew_account_control(*db, a, lease_a, alice, 2, 20000ms, 5s),
ControlStoreError::fenced));
CHECK(release_account_control(*db, a, lease_a, alice, 2, 5s) == false);
// Region B's renewal never shortens the waited-out expiry.
CHECK(renew_account_control(*db, b, lease_b, alice, 3, 1000ms, 5s) == 1000ms);
CHECK(release_account_control(*db, b, lease_b, alice, 3, 5s) == true);
CHECK(failed(renew_account_control(*db, b, lease_b, alice, 3, 20000ms, 5s),
ControlStoreError::fenced));
// Released control is free: the next claim waits for nobody; the serial stays durable.
CHECK(failed(claim_account_control(*db, a, lease_a, alice, 7, 3, 20000ms, 5s),
ControlStoreError::stale_grant));
auto back = claim_account_control(*db, a, lease_a, alice, 8, 3, 20000ms, 5s);
CHECK(back && back->wait == 0ms);
// Restart: a new process takes region A's lease under a new epoch. Serials survive,
// the old epoch's control needs no wait, and the old lease can do nothing.
CHECK(release_region_lease(*db, a, lease_a, 5s));
const auto restarted = acquire(*db, a);
CHECK(restarted.epoch > lease_a.epoch);
CHECK(failed(claim_account_control(*db, a, restarted, alice, 8, 1, 20000ms, 5s),
ControlStoreError::stale_grant));
auto resumed = claim_account_control(*db, a, restarted, alice, 9, 1, 20000ms, 5s);
CHECK(resumed && resumed->wait == 0ms);
CHECK(failed(claim_account_control(*db, a, lease_a, alice, 10, 4, 20000ms, 5s),
ControlStoreError::fenced));
CHECK(failed(renew_account_control(*db, a, lease_a, alice, resumed->generation, 20000ms, 5s),
ControlStoreError::fenced));
// Expiry is final, and an expired holder is not waited for.
const auto bob = id(0x32);
auto brief = claim_account_control(*db, a, restarted, bob, 1, 2, 1000ms, 5s);
CHECK(brief && brief->wait == 0ms);
wait_until_expired(*admin, bob);
CHECK(failed(renew_account_control(*db, a, restarted, bob, brief->generation, 20000ms, 5s),
ControlStoreError::fenced));
auto elsewhere = claim_account_control(*db, b, lease_b, bob, 2, 2, 20000ms, 5s);
CHECK(elsewhere && elsewhere->wait == 0ms);
// Invalid requests change nothing.
CHECK(failed(claim_account_control(*db, a, restarted, id(0x33), 1, 1, 500ms, 5s),
ControlStoreError::invalid));
CHECK(failed(claim_account_control(*db, a, restarted, id(0x33), 0, 1, 20000ms, 5s),
ControlStoreError::invalid));
CHECK(failed(claim_account_control(*db, a, restarted, id(0x33), 1, 0, 20000ms, 5s),
ControlStoreError::invalid));
CHECK(failed(claim_account_control(*db, a, restarted, Uuid{}, 1, 1, 20000ms, 5s),
ControlStoreError::invalid));
CHECK(claim_account_control(*db, a, restarted, id(0x33), 1, 1, 20000ms, 5s));
// Contention: two regions present the same newest serial at once; exactly one wins.
const auto carol = id(0x34);
auto first_db = connect_as(env("EVO_PG_RUNTIME_USER"));
auto second_db = connect_as(env("EVO_PG_RUNTIME_USER"));
std::latch start{2};
std::expected<AccountControlClaim, ControlStoreFailure> ra, rb;
std::thread ta{[&] {
start.arrive_and_wait();
ra = claim_account_control(*first_db, a, restarted, carol, 20, 7, 20000ms, 5s);
}};
std::thread tb{[&] {
start.arrive_and_wait();
rb = claim_account_control(*second_db, b, lease_b, carol, 20, 7, 20000ms, 5s);
}};
ta.join();
tb.join();
CHECK(ra.has_value() != rb.has_value());
CHECK(failed(ra.has_value() ? rb : ra, ControlStoreError::stale_grant));
#ifdef EVO_POSTGRES_SESSIONS
// The session layer's ledger over these functions.
std::optional<LeaseGrant> current = restarted;
evo::server::PgSlot slot{PgOptions{env("PGHOST"),
{},
env("PGPORT"),
env("PGDATABASE"),
env("EVO_PG_RUNTIME_USER"),
{},
{}},
5s};
evo::server::PostgresControlLedger ledger{slot, a, [&] { return current; }};
evo::network::AccountId dave;
dave.bytes = id(0x35);
auto claimed = ledger.claim(dave, 1, 9);
CHECK(claimed && claimed->wait == 0ms && claimed->granted == 20000ms);
CHECK(!ledger.claim(dave, 1, 9) &&
ledger.claim(dave, 1, 9).error() == evo::network::ControlLedgerError::stale_grant);
CHECK(ledger.renew(dave, claimed->generation) == 20000ms);
ledger.release(dave, claimed->generation);
CHECK(!ledger.renew(dave, claimed->generation) &&
ledger.renew(dave, claimed->generation).error() ==
evo::network::ControlLedgerError::fenced);
current.reset(); // The host lost its lease: nothing is granted or renewed.
CHECK(!ledger.claim(dave, 2, 9) &&
ledger.claim(dave, 2, 9).error() == evo::network::ControlLedgerError::fenced);
#endif
CHECK(release_region_lease(*db, a, restarted, 5s));
CHECK(release_region_lease(*db, b, lease_b, 5s));
}
} // namespace
void run_account_control_store_tests() {
serials_takeover_and_fencing();
std::puts("account control integration passed");
}