Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 1m50s
Foundation / checks (network) (push) Successful in 9m8s
Foundation / checks (client) (push) Successful in 12m10s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (postgres) (push) Canceled after 7m33s
Foundation / checks (checks) (push) Canceled after 18m49s
Foundation / checks (tsan) (push) Canceled after 6m38s
Migration v10 keeps, per world and account, the highest accepted grant serial and the region session in control with a fencing generation and database-time expiry, reachable only through claim/renew/release functions under the caller's region lease. Serials survive restarts and apply across regions; a cross-region takeover returns a wait so control never overlaps. RegionSessions takes durable control via a ControlLedger before admitting a session, delays Welcome by the wait, renews, and closes sessions whose control is fenced or cannot be confirmed in time. PostgresControlLedger implements the ledger. ADR-0091. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
236 lines
10 KiB
C++
236 lines
10 KiB
C++
// Real PostgreSQL M10.3c account control checks (ADR-0091, schema v10): durable grant
|
|
// serials across regions and restarts, cross-region takeover with a wait, fencing by
|
|
// generation, region lease and expiry, and contention. World 9100 is used only here.
|
|
// Run by tools/ci/postgres-check.sh.
|
|
#include "evo/persistence/account_control_store.hpp"
|
|
#include "evo/persistence/region_checkpoint.hpp"
|
|
#include "evo/persistence/region_lease_store.hpp"
|
|
#ifdef EVO_POSTGRES_SESSIONS
|
|
#include "evo/server/postgres_control_ledger.hpp"
|
|
#endif
|
|
#include <array>
|
|
#include <chrono>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <expected>
|
|
#include <latch>
|
|
#include <memory>
|
|
#include <optional>
|
|
#include <source_location>
|
|
#include <span>
|
|
#include <stdexcept>
|
|
#include <string>
|
|
#include <thread>
|
|
|
|
void run_account_control_store_tests();
|
|
|
|
namespace {
|
|
using namespace evo::persistence;
|
|
using namespace std::chrono_literals;
|
|
using std::chrono::milliseconds;
|
|
void check(bool value, std::source_location at = std::source_location::current()) {
|
|
if (!value)
|
|
throw std::runtime_error("account control store test line " + std::to_string(at.line()));
|
|
}
|
|
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
|
constexpr std::uint64_t world = 9100;
|
|
|
|
std::string env(const char* key) {
|
|
const auto* value = std::getenv(key);
|
|
if (!value)
|
|
throw std::runtime_error("missing test environment");
|
|
return value;
|
|
}
|
|
std::unique_ptr<Postgres> connect_as(const std::string& user) {
|
|
PgOptions options{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}};
|
|
auto made = Postgres::connect(options, 5s);
|
|
CHECK(made);
|
|
return std::move(*made);
|
|
}
|
|
Uuid id(std::uint8_t seed) {
|
|
Uuid out{};
|
|
out.fill(seed);
|
|
return out;
|
|
}
|
|
Sha256Digest digest(std::uint8_t seed) {
|
|
Sha256Digest out{};
|
|
out.fill(seed);
|
|
return out;
|
|
}
|
|
std::string bytea(std::span<const std::uint8_t> bytes) {
|
|
constexpr char digits[] = "0123456789abcdef";
|
|
std::string out = "\\x";
|
|
for (const auto b : bytes) {
|
|
out.push_back(digits[b >> 4U]);
|
|
out.push_back(digits[b & 0x0FU]);
|
|
}
|
|
return out;
|
|
}
|
|
const Sha256Digest build = digest(0xB1);
|
|
RegionKey provision(Postgres& db, std::uint64_t region) {
|
|
CHECK(db.query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
|
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
|
|
{}, 5s));
|
|
const RegionKey key{world, region};
|
|
auto commit = id(0xAC); // Commit ids are unique across the database: tag this test's.
|
|
commit.back() = static_cast<std::uint8_t>(region);
|
|
auto g = genesis_manifest(key, 7, commit, 50000, build, 100, digest(0xA0));
|
|
CHECK(g);
|
|
auto bytes = encode_manifest(*g);
|
|
CHECK(bytes);
|
|
const std::array<std::string, 1> p{bytea(*bytes)};
|
|
const auto provisioned =
|
|
db.query("SELECT evolution.provision_region_checkpoint($1::bytea)", p, 5s);
|
|
if (!provisioned)
|
|
std::fprintf(stderr, "provision sqlstate %s\n", provisioned.error().sqlstate.data());
|
|
CHECK(provisioned);
|
|
return key;
|
|
}
|
|
LeaseGrant acquire(Postgres& db, RegionKey key) {
|
|
auto lease = acquire_region_lease(db, key, build, 20000ms, 5s);
|
|
CHECK(lease);
|
|
return *lease;
|
|
}
|
|
bool failed(const auto& result, ControlStoreError kind) {
|
|
return !result && result.error().kind == kind;
|
|
}
|
|
// Database-time expiry of one account's control (owner read), as for region leases.
|
|
void wait_until_expired(Postgres& owner_db, const Uuid& account) {
|
|
const std::array<std::string, 1> p{bytea(account)};
|
|
for (int attempt = 0; attempt < 400; ++attempt) {
|
|
auto rows = owner_db.query(
|
|
"SELECT (clock_timestamp() >= expires_at)::text FROM evolution.account_controls "
|
|
"WHERE world_id = 9100 AND account_id = encode($1::bytea, 'hex')::uuid",
|
|
p, 5s);
|
|
CHECK(rows && rows->size() == 1);
|
|
if ((*rows)[0][0] == "true")
|
|
return;
|
|
std::this_thread::sleep_for(50ms);
|
|
}
|
|
CHECK(false);
|
|
}
|
|
|
|
void serials_takeover_and_fencing() {
|
|
auto admin = connect_as(env("PGUSER"));
|
|
const auto a = provision(*admin, 1);
|
|
const auto b = provision(*admin, 2);
|
|
auto db = connect_as(env("EVO_PG_RUNTIME_USER"));
|
|
const auto lease_a = acquire(*db, a);
|
|
const auto lease_b = acquire(*db, b);
|
|
const auto alice = id(0x31);
|
|
|
|
// The runtime reaches the table only through the functions.
|
|
CHECK(!db->query("SELECT count(*) FROM evolution.account_controls", {}, 5s));
|
|
|
|
auto first = claim_account_control(*db, a, lease_a, alice, 5, 1, 20000ms, 5s);
|
|
CHECK(first && first->generation == 1 && first->wait == 0ms && first->granted == 20000ms);
|
|
CHECK(failed(claim_account_control(*db, a, lease_a, alice, 5, 2, 20000ms, 5s),
|
|
ControlStoreError::stale_grant)); // Replay.
|
|
CHECK(failed(claim_account_control(*db, a, lease_a, alice, 4, 2, 20000ms, 5s),
|
|
ControlStoreError::stale_grant)); // Older grant.
|
|
// Same region takeover: no wait; the old generation is fenced.
|
|
auto second = claim_account_control(*db, a, lease_a, alice, 6, 2, 20000ms, 5s);
|
|
CHECK(second && second->generation == 2 && second->wait == 0ms);
|
|
CHECK(failed(renew_account_control(*db, a, lease_a, alice, 1, 20000ms, 5s),
|
|
ControlStoreError::fenced));
|
|
CHECK(renew_account_control(*db, a, lease_a, alice, 2, 20000ms, 5s) == 20000ms);
|
|
// Another region with a newer grant takes control but must wait out region A's.
|
|
auto moved = claim_account_control(*db, b, lease_b, alice, 7, 1, 20000ms, 5s);
|
|
CHECK(moved && moved->generation == 3 && moved->wait > 15000ms && moved->wait <= 20000ms);
|
|
CHECK(failed(renew_account_control(*db, a, lease_a, alice, 2, 20000ms, 5s),
|
|
ControlStoreError::fenced));
|
|
CHECK(release_account_control(*db, a, lease_a, alice, 2, 5s) == false);
|
|
// Region B's renewal never shortens the waited-out expiry.
|
|
CHECK(renew_account_control(*db, b, lease_b, alice, 3, 1000ms, 5s) == 1000ms);
|
|
CHECK(release_account_control(*db, b, lease_b, alice, 3, 5s) == true);
|
|
CHECK(failed(renew_account_control(*db, b, lease_b, alice, 3, 20000ms, 5s),
|
|
ControlStoreError::fenced));
|
|
// Released control is free: the next claim waits for nobody; the serial stays durable.
|
|
CHECK(failed(claim_account_control(*db, a, lease_a, alice, 7, 3, 20000ms, 5s),
|
|
ControlStoreError::stale_grant));
|
|
auto back = claim_account_control(*db, a, lease_a, alice, 8, 3, 20000ms, 5s);
|
|
CHECK(back && back->wait == 0ms);
|
|
|
|
// Restart: a new process takes region A's lease under a new epoch. Serials survive,
|
|
// the old epoch's control needs no wait, and the old lease can do nothing.
|
|
CHECK(release_region_lease(*db, a, lease_a, 5s));
|
|
const auto restarted = acquire(*db, a);
|
|
CHECK(restarted.epoch > lease_a.epoch);
|
|
CHECK(failed(claim_account_control(*db, a, restarted, alice, 8, 1, 20000ms, 5s),
|
|
ControlStoreError::stale_grant));
|
|
auto resumed = claim_account_control(*db, a, restarted, alice, 9, 1, 20000ms, 5s);
|
|
CHECK(resumed && resumed->wait == 0ms);
|
|
CHECK(failed(claim_account_control(*db, a, lease_a, alice, 10, 4, 20000ms, 5s),
|
|
ControlStoreError::fenced));
|
|
CHECK(failed(renew_account_control(*db, a, lease_a, alice, resumed->generation, 20000ms, 5s),
|
|
ControlStoreError::fenced));
|
|
|
|
// Expiry is final, and an expired holder is not waited for.
|
|
const auto bob = id(0x32);
|
|
auto brief = claim_account_control(*db, a, restarted, bob, 1, 2, 1000ms, 5s);
|
|
CHECK(brief && brief->wait == 0ms);
|
|
wait_until_expired(*admin, bob);
|
|
CHECK(failed(renew_account_control(*db, a, restarted, bob, brief->generation, 20000ms, 5s),
|
|
ControlStoreError::fenced));
|
|
auto elsewhere = claim_account_control(*db, b, lease_b, bob, 2, 2, 20000ms, 5s);
|
|
CHECK(elsewhere && elsewhere->wait == 0ms);
|
|
|
|
// Invalid requests change nothing.
|
|
CHECK(failed(claim_account_control(*db, a, restarted, id(0x33), 1, 1, 500ms, 5s),
|
|
ControlStoreError::invalid));
|
|
CHECK(failed(claim_account_control(*db, a, restarted, id(0x33), 0, 1, 20000ms, 5s),
|
|
ControlStoreError::invalid));
|
|
CHECK(failed(claim_account_control(*db, a, restarted, id(0x33), 1, 0, 20000ms, 5s),
|
|
ControlStoreError::invalid));
|
|
CHECK(failed(claim_account_control(*db, a, restarted, Uuid{}, 1, 1, 20000ms, 5s),
|
|
ControlStoreError::invalid));
|
|
CHECK(claim_account_control(*db, a, restarted, id(0x33), 1, 1, 20000ms, 5s));
|
|
|
|
// Contention: two regions present the same newest serial at once; exactly one wins.
|
|
const auto carol = id(0x34);
|
|
auto first_db = connect_as(env("EVO_PG_RUNTIME_USER"));
|
|
auto second_db = connect_as(env("EVO_PG_RUNTIME_USER"));
|
|
std::latch start{2};
|
|
std::expected<AccountControlClaim, ControlStoreFailure> ra, rb;
|
|
std::thread ta{[&] {
|
|
start.arrive_and_wait();
|
|
ra = claim_account_control(*first_db, a, restarted, carol, 20, 7, 20000ms, 5s);
|
|
}};
|
|
std::thread tb{[&] {
|
|
start.arrive_and_wait();
|
|
rb = claim_account_control(*second_db, b, lease_b, carol, 20, 7, 20000ms, 5s);
|
|
}};
|
|
ta.join();
|
|
tb.join();
|
|
CHECK(ra.has_value() != rb.has_value());
|
|
CHECK(failed(ra.has_value() ? rb : ra, ControlStoreError::stale_grant));
|
|
|
|
#ifdef EVO_POSTGRES_SESSIONS
|
|
// The session layer's ledger over these functions.
|
|
std::optional<LeaseGrant> current = restarted;
|
|
evo::server::PostgresControlLedger ledger{*db, a, [&] { return current; }};
|
|
evo::network::AccountId dave;
|
|
dave.bytes = id(0x35);
|
|
auto claimed = ledger.claim(dave, 1, 9);
|
|
CHECK(claimed && claimed->wait == 0ms && claimed->granted == 20000ms);
|
|
CHECK(!ledger.claim(dave, 1, 9) &&
|
|
ledger.claim(dave, 1, 9).error() == evo::network::ControlLedgerError::stale_grant);
|
|
CHECK(ledger.renew(dave, claimed->generation) == 20000ms);
|
|
ledger.release(dave, claimed->generation);
|
|
CHECK(!ledger.renew(dave, claimed->generation) &&
|
|
ledger.renew(dave, claimed->generation).error() ==
|
|
evo::network::ControlLedgerError::fenced);
|
|
current.reset(); // The host lost its lease: nothing is granted or renewed.
|
|
CHECK(!ledger.claim(dave, 2, 9) &&
|
|
ledger.claim(dave, 2, 9).error() == evo::network::ControlLedgerError::fenced);
|
|
#endif
|
|
CHECK(release_region_lease(*db, a, restarted, 5s));
|
|
CHECK(release_region_lease(*db, b, lease_b, 5s));
|
|
}
|
|
} // namespace
|
|
|
|
void run_account_control_store_tests() {
|
|
serials_takeover_and_fencing();
|
|
std::puts("account control integration passed");
|
|
}
|