Files
aevolution/tests/region_lease_store_tests.cpp
T
dingoandClaude Opus 5.5 2496649a9b Complete M7.3 fenced region ownership
ADR-0060 and schema v2 add database-time region leases and a fenced head
advance. Acquire/renew/release/advance lock the lease row, check holder,
epoch and expiry with clock_timestamp() after the lock wait, and the
advance compares the exact predecessor. Holders are DB-generated tokens;
epochs never wrap or repeat. The runtime group role gets EXECUTE on five
SECURITY DEFINER functions only - no table, provisioning or CREATE rights.
Migration history admits clean->v2, v1->v2 and a v2 no-op, and refuses
privileged runtime roles.

Host side: portable LeaseAuthority fences from send-time deadlines on
CLOCK_BOOTTIME, keeps the earlier deadline on unknown renewal and stays
fenced until a newer epoch. Selected timings: 20s lease, 5s renew, 2s
margin, 2s renew and 5s commit deadlines. Linux evo::postgres wraps the
functions with typed LeaseStoreError results.

Validated: Windows CTest 55/55; WSL Release, clang-tidy and ASan/UBSan
private-PostgreSQL suites (contenders, expiry, paused/queued stale
writers, unknown renewal, exhaustion, bypass denial, upgrade path).
Three SQL mutations were each caught. Docker/Gitea CI, remote TLS and
cluster runs were not executed. No director, host or deployment added.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-28 11:14:32 -04:00

414 lines
19 KiB
C++

// Real PostgreSQL M7.3 fencing checks (ADR-0060). Run by tools/ci/postgres-check.sh.
#include "evo/persistence/region_lease_store.hpp"
#include <algorithm>
#include <array>
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <latch>
#include <memory>
#include <source_location>
#include <stdexcept>
#include <string>
#include <thread>
#include <vector>
void run_region_lease_store_tests();
namespace {
using namespace evo::persistence;
using namespace std::chrono_literals;
using std::chrono::milliseconds;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region lease store test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
std::string env(const char* key) {
const auto* value = std::getenv(key);
if (!value)
throw std::runtime_error("missing test environment");
return value;
}
std::unique_ptr<Postgres> connect_as(const std::string& user) {
PgOptions options{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}};
auto made = Postgres::connect(options, 5s);
CHECK(made);
return std::move(*made);
}
std::unique_ptr<Postgres> owner() {
return connect_as(env("PGUSER"));
}
std::unique_ptr<Postgres> runtime() {
return connect_as(env("EVO_PG_RUNTIME_USER"));
}
Uuid id(std::uint8_t seed) {
Uuid out{};
out.fill(seed);
return out;
}
Sha256Digest digest(std::uint8_t seed) {
Sha256Digest out{};
out.fill(seed);
return out;
}
std::string hex_text(std::uint8_t seed, std::size_t bytes) {
constexpr char digits[] = "0123456789abcdef";
std::string out;
for (std::size_t i = 0; i < bytes; ++i) {
out.push_back(digits[seed >> 4U]);
out.push_back(digits[seed & 0x0FU]);
}
return out;
}
std::string uuid_literal(std::uint8_t seed) {
const auto h = hex_text(seed, 16);
return h.substr(0, 8) + '-' + h.substr(8, 4) + '-' + h.substr(12, 4) + '-' + h.substr(16, 4) +
'-' + h.substr(20);
}
// Genesis commit/digest bytes are the region number; frame is 7.
RegionKey provision(Postgres& db, std::uint64_t region) {
const auto seed = static_cast<std::uint8_t>(region);
const std::array<std::string, 4> p{std::to_string(region), uuid_literal(seed),
"\\x" + hex_text(seed, 32), "7"};
CHECK(db.query("SELECT evolution.provision_region(1, $1::evolution.u64, $4::evolution.u64, "
"$2::uuid, $3::bytea)",
p, 5s));
return {1, region};
}
bool failed(const auto& result, LeaseStoreError kind) {
return !result && result.error().kind == kind;
}
std::string sqlstate(const std::expected<PgRows, PgFailure>& result) {
return result ? std::string{} : std::string(result.error().sqlstate.data());
}
void runtime_cannot_bypass_the_guard() {
auto admin = owner();
const auto key = provision(*admin, 10);
auto db = runtime();
for (const auto* sql : {
"UPDATE evolution.region_leases SET holder = NULL",
"UPDATE evolution.region_heads SET commit_sequence = 5",
"DELETE FROM evolution.region_heads",
"INSERT INTO evolution.region_leases(world_id,region_id,frame_id) VALUES (9,9,9)",
"SELECT * FROM evolution.region_leases",
"SELECT * FROM evolution.region_heads",
"SELECT evolution.provision_region(1, 11, 7, gen_random_uuid(), '\\x00')",
"SELECT evolution.check_region_lease(1, 10, gen_random_uuid(), 1)",
"CREATE FUNCTION evolution.bypass() RETURNS int LANGUAGE sql AS 'SELECT 1'",
"CREATE TABLE evolution.bypass(x int)",
})
CHECK(sqlstate(db->query(sql, {}, 5s)) == "42501");
const std::array<std::string, 1> owner_name{env("PGUSER")};
CHECK(!db->query("SELECT pg_catalog.set_config('role', $1, false)", owner_name, 5s));
// A SECURITY DEFINER function ignores a hostile caller search_path.
CHECK(db->query("SET search_path = pg_temp, public", {}, 5s));
CHECK(db->query("CREATE TEMP TABLE region_leases(x int)", {}, 5s));
auto lease = acquire_region_lease(*db, key, 20000ms, 5s);
CHECK(lease && lease->epoch == 1 && lease->frame == 7);
CHECK(release_region_lease(*db, key, *lease, 5s));
}
void acquire_renew_release_and_advance() {
auto admin = owner();
const auto key = provision(*admin, 20);
auto db = runtime();
CHECK(
failed(acquire_region_lease(*db, {1, 999}, 20000ms, 5s), LeaseStoreError::not_provisioned));
CHECK(failed(acquire_region_lease(*db, key, 999ms, 5s), LeaseStoreError::invalid));
CHECK(failed(acquire_region_lease(*db, key, 60001ms, 5s), LeaseStoreError::invalid));
auto lease = acquire_region_lease(*db, key, 20000ms, 5s);
CHECK(lease && lease->epoch == 1 && lease->duration == 20000ms && lease->holder != Uuid{});
CHECK(failed(acquire_region_lease(*db, key, 20000ms, 5s), LeaseStoreError::busy));
auto renewed = renew_region_lease(*db, key, *lease, 20000ms, 5s);
CHECK(renewed && *renewed == 20000ms);
auto forged = *lease;
forged.holder[0] ^= 1U;
CHECK(failed(renew_region_lease(*db, key, forged, 20000ms, 5s), LeaseStoreError::fenced));
auto stale_epoch = *lease;
stale_epoch.epoch = 2;
CHECK(failed(renew_region_lease(*db, key, stale_epoch, 20000ms, 5s), LeaseStoreError::fenced));
auto genesis = load_region_head(*db, key, 5s);
CHECK(genesis && genesis->sequence == 0 && genesis->end_tick == 0 && genesis->frame == 7);
CHECK(genesis->commit == id(20) && genesis->digest == digest(20));
CHECK(failed(advance_region_head(*db, key, *lease, *genesis, {id(21), digest(21), 65}, 5s),
LeaseStoreError::invalid));
CHECK(failed(advance_region_head(*db, key, *lease, *genesis, {id(20), digest(21), 1}, 5s),
LeaseStoreError::invalid));
const HeadAdvance first{id(21), digest(21), 64};
auto head = advance_region_head(*db, key, *lease, *genesis, first, 5s);
CHECK(head && head->sequence == 1 && head->end_tick == 64 && head->commit == id(21));
CHECK(failed(advance_region_head(*db, key, forged, *head, {id(22), digest(22), 65}, 5s),
LeaseStoreError::fenced));
// Exact retry after an applied advance: mismatch, then the head proves it committed.
CHECK(failed(advance_region_head(*db, key, *lease, *genesis, first, 5s),
LeaseStoreError::head_mismatch));
auto current = load_region_head(*db, key, 5s);
CHECK(current && *current == *head);
auto changed = *head;
changed.digest[31] ^= 1U;
CHECK(failed(advance_region_head(*db, key, *lease, changed, {id(22), digest(22), 65}, 5s),
LeaseStoreError::head_mismatch));
CHECK(failed(advance_region_head(*db, key, *lease, *head, {id(22), digest(22), 64}, 5s),
LeaseStoreError::invalid));
CHECK(release_region_lease(*db, key, *lease, 5s));
CHECK(failed(release_region_lease(*db, key, *lease, 5s), LeaseStoreError::fenced));
CHECK(failed(renew_region_lease(*db, key, *lease, 20000ms, 5s), LeaseStoreError::fenced));
CHECK(failed(advance_region_head(*db, key, *lease, *head, {id(22), digest(22), 65}, 5s),
LeaseStoreError::fenced));
auto next = acquire_region_lease(*db, key, 20000ms, 5s);
CHECK(next && next->epoch == 2 && next->holder != lease->holder);
auto final_head = advance_region_head(*db, key, *next, *head, {id(22), digest(22), 65}, 5s);
CHECK(final_head && final_head->sequence == 2);
}
void database_time_expiry_and_paused_writer() {
auto admin = owner();
const auto key = provision(*admin, 30);
auto worker_a = runtime();
auto worker_b = runtime();
const LeasePolicy fast{1000ms, 200ms, 100ms, 100ms, 300ms};
LeaseAuthority a{fast};
auto sent = LeaseClock::now();
auto lease_a = acquire_region_lease(*worker_a, key, fast.duration, 5s);
CHECK(lease_a && a.acquired(*lease_a, sent, LeaseClock::now()));
CHECK(failed(acquire_region_lease(*worker_b, key, 20000ms, 5s), LeaseStoreError::busy));
auto genesis = load_region_head(*worker_a, key, 5s);
CHECK(genesis);
std::this_thread::sleep_for(1200ms); // Worker A is paused past its lease.
CHECK(a.state(LeaseClock::now()) == LeaseState::fenced);
// Expiry is decided by database time; the successor acquires a newer epoch.
auto lease_b = acquire_region_lease(*worker_b, key, 20000ms, 5s);
CHECK(lease_b && lease_b->epoch == lease_a->epoch + 1);
auto head_b =
advance_region_head(*worker_b, key, *lease_b, *genesis, {id(31), digest(31), 3}, 5s);
CHECK(head_b);
// A resumes and ignores its local fence: every database path rejects it.
CHECK(failed(renew_region_lease(*worker_a, key, *lease_a, fast.duration, 5s),
LeaseStoreError::fenced));
CHECK(
failed(advance_region_head(*worker_a, key, *lease_a, *genesis, {id(32), digest(32), 1}, 5s),
LeaseStoreError::fenced));
CHECK(
failed(advance_region_head(*worker_a, key, *lease_a, *head_b, {id(32), digest(32), 4}, 5s),
LeaseStoreError::fenced));
CHECK(failed(release_region_lease(*worker_a, key, *lease_a, 5s), LeaseStoreError::fenced));
auto head = load_region_head(*worker_a, key, 5s);
CHECK(head && *head == *head_b);
// Expired renewal cannot revive a lease even without a successor.
CHECK(release_region_lease(*worker_b, key, *lease_b, 5s));
auto lease_c = acquire_region_lease(*worker_a, key, 1000ms, 5s);
CHECK(lease_c && lease_c->epoch == lease_b->epoch + 1);
std::this_thread::sleep_for(1200ms);
CHECK(
failed(renew_region_lease(*worker_a, key, *lease_c, 1000ms, 5s), LeaseStoreError::fenced));
}
// A writer queued on the lease row lock is judged by the clock after the wait,
// not its statement start time, so a lease that expired while queued is fenced.
void queued_writer_uses_post_lock_database_time() {
auto admin = owner();
const auto key = provision(*admin, 40);
auto worker = runtime();
auto lease = acquire_region_lease(*worker, key, 1000ms, 5s);
auto genesis = load_region_head(*worker, key, 5s);
CHECK(lease && genesis);
auto blocker = owner();
const std::array<std::string, 1> region{"40"};
CHECK(blocker->query("BEGIN", {}, 5s));
CHECK(blocker->query("SELECT 1 FROM evolution.region_leases WHERE world_id = 1 AND "
"region_id = $1::evolution.u64 FOR UPDATE",
region, 5s));
std::expected<RegionHead, LeaseStoreFailure> queued = std::unexpected{LeaseStoreFailure{}};
std::thread writer([&] {
queued = advance_region_head(*worker, key, *lease, *genesis, {id(41), digest(41), 1}, 5s);
});
std::this_thread::sleep_for(1300ms);
CHECK(blocker->query("ROLLBACK", {}, 5s));
writer.join();
CHECK(failed(queued, LeaseStoreError::fenced));
auto head = load_region_head(*worker, key, 5s);
CHECK(head && *head == *genesis);
}
// Partition: the renewal's outcome is unknown and the connection is gone. The
// holder keeps its earlier deadline and fences itself before a successor starts.
void partitioned_renewal_fences_locally() {
auto admin = owner();
const auto key = provision(*admin, 50);
auto worker = runtime();
const LeasePolicy fast{1000ms, 200ms, 100ms, 100ms, 300ms};
LeaseAuthority authority{fast};
const auto sent = LeaseClock::now();
auto lease = acquire_region_lease(*worker, key, fast.duration, 5s);
CHECK(lease && authority.acquired(*lease, sent, LeaseClock::now()));
auto blocker = owner();
const std::array<std::string, 1> region{"50"};
CHECK(blocker->query("BEGIN", {}, 5s));
CHECK(blocker->query("SELECT 1 FROM evolution.region_leases WHERE world_id = 1 AND "
"region_id = $1::evolution.u64 FOR UPDATE",
region, 5s));
const auto renew_sent = LeaseClock::now();
auto renewal = renew_region_lease(*worker, key, *lease, fast.duration, fast.renew_timeout);
CHECK(failed(renewal, LeaseStoreError::unknown));
authority.renewal_unknown(fast.duration, renew_sent);
const auto local_deadline = authority.deadline();
CHECK(local_deadline <= sent + fast.duration - fast.safety_margin);
std::this_thread::sleep_for(1300ms);
CHECK(!authority.may_originate(LeaseClock::now()));
CHECK(blocker->query("ROLLBACK", {}, 5s));
// The orphaned renewal reached the row only after database expiry: rejected.
auto successor = runtime();
auto next = acquire_region_lease(*successor, key, 20000ms, 5s);
CHECK(next && next->epoch == lease->epoch + 1);
}
void contenders_serialize_on_the_database() {
auto admin = owner();
const auto key = provision(*admin, 60);
std::array<std::unique_ptr<Postgres>, 2> workers{runtime(), runtime()};
std::uint64_t previous_epoch = 0;
constexpr int rounds = 64;
for (int round = 0; round < rounds; ++round) {
std::array<std::expected<LeaseGrant, LeaseStoreFailure>, 2> results{
std::unexpected{LeaseStoreFailure{}}, std::unexpected{LeaseStoreFailure{}}};
std::latch start{2};
std::vector<std::thread> threads;
threads.reserve(2);
for (std::size_t i = 0; i < 2; ++i)
threads.emplace_back([&, i] {
start.arrive_and_wait();
results[i] = acquire_region_lease(*workers[i], key, 20000ms, 5s);
});
for (auto& t : threads)
t.join();
const auto winners = std::count_if(results.begin(), results.end(),
[](const auto& r) { return r.has_value(); });
CHECK(winners == 1);
const std::size_t w = results[0] ? 0 : 1;
CHECK(failed(results[1 - w], LeaseStoreError::busy));
CHECK(results[w]->epoch == previous_epoch + 1);
previous_epoch = results[w]->epoch;
CHECK(release_region_lease(*workers[w], key, *results[w], 5s));
}
// The same holder racing two commits from the same predecessor: exactly one wins.
auto lease = acquire_region_lease(*workers[0], key, 20000ms, 5s);
CHECK(lease);
auto head = load_region_head(*workers[0], key, 5s);
CHECK(head);
for (std::uint8_t round = 0; round < 32; ++round) {
std::array<std::expected<RegionHead, LeaseStoreFailure>, 2> results{
std::unexpected{LeaseStoreFailure{}}, std::unexpected{LeaseStoreFailure{}}};
std::latch start{2};
std::vector<std::thread> threads;
threads.reserve(2);
for (std::size_t i = 0; i < 2; ++i)
threads.emplace_back([&, i] {
const auto seed = static_cast<std::uint8_t>(100 + (round * 2) + i);
start.arrive_and_wait();
results[i] = advance_region_head(*workers[i], key, *lease, *head,
{id(seed), digest(seed), head->end_tick + 1}, 5s);
});
for (auto& t : threads)
t.join();
const std::size_t w = results[0] ? 0 : 1;
CHECK(results[w] && failed(results[1 - w], LeaseStoreError::head_mismatch));
CHECK(results[w]->sequence == head->sequence + 1);
head = *results[w];
}
auto committed = load_region_head(*workers[1], key, 5s);
CHECK(committed && *committed == *head && committed->sequence == 32);
}
void exhaustion_fails_closed() {
auto admin = owner();
const auto key = provision(*admin, 70);
CHECK(admin->query("UPDATE evolution.region_leases SET epoch = 18446744073709551615 "
"WHERE world_id = 1 AND region_id = 70",
{}, 5s));
auto db = runtime();
CHECK(failed(acquire_region_lease(*db, key, 20000ms, 5s), LeaseStoreError::exhausted));
const auto other = provision(*admin, 71);
auto lease = acquire_region_lease(*db, other, 20000ms, 5s);
CHECK(lease);
CHECK(admin->query("UPDATE evolution.region_heads SET commit_sequence = 18446744073709551615, "
"end_tick = 5, published_epoch = 1 WHERE world_id = 1 AND region_id = 71",
{}, 5s));
auto head = load_region_head(*db, other, 5s);
CHECK(head && head->sequence == 18446744073709551615ULL);
CHECK(failed(advance_region_head(*db, other, *lease, *head, {id(72), digest(72), 6}, 5s),
LeaseStoreError::exhausted));
}
struct Summary {
std::vector<long long> samples;
void print(const char* name) {
std::sort(samples.begin(), samples.end());
const auto pick = [&](std::size_t percent) {
return samples[std::min(samples.size() - 1, samples.size() * percent / 100)];
};
std::printf("lease_latency_us %s n=%zu p50=%lld p99=%lld max=%lld\n", name, samples.size(),
pick(50), pick(99), samples.back());
}
};
template <typename F> long long timed(F&& f) {
const auto begin = std::chrono::steady_clock::now();
f();
return std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() -
begin)
.count();
}
// Local Unix-socket round trips only; remote TLS latency must be measured in place.
void measure_round_trips() {
auto admin = owner();
const auto key = provision(*admin, 80);
auto db = runtime();
constexpr int n = 200;
Summary acquire, renew, advance, release;
auto head = load_region_head(*db, key, 5s);
CHECK(head);
for (int i = 0; i < n; ++i) {
std::expected<LeaseGrant, LeaseStoreFailure> lease = std::unexpected{LeaseStoreFailure{}};
acquire.samples.push_back(
timed([&] { lease = acquire_region_lease(*db, key, 20000ms, 5s); }));
CHECK(lease);
renew.samples.push_back(
timed([&] { CHECK(renew_region_lease(*db, key, *lease, 20000ms, 5s)); }));
const auto seed = static_cast<std::uint8_t>(i);
Uuid commit = id(seed);
commit[0] = 0xAA; // Distinct from genesis 0x50 bytes.
commit[1] = static_cast<std::uint8_t>(i >> 8);
advance.samples.push_back(timed([&] {
auto next = advance_region_head(*db, key, *lease, *head,
{commit, digest(seed), head->end_tick + 64}, 5s);
CHECK(next);
head = *next;
}));
release.samples.push_back(
timed([&] { CHECK(release_region_lease(*db, key, *lease, 5s)); }));
}
acquire.print("acquire");
renew.print("renew");
advance.print("advance_head");
release.print("release");
}
} // namespace
void run_region_lease_store_tests() {
runtime_cannot_bypass_the_guard();
acquire_renew_release_and_advance();
database_time_expiry_and_paused_writer();
queued_writer_uses_post_lock_database_time();
partitioned_renewal_fences_locally();
contenders_serialize_on_the_database();
exhaustion_fails_closed();
measure_round_trips();
}