Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (checks) (push) Failing after 43s
Foundation / checks (ecs) (push) Successful in 1m52s
Foundation / checks (postgres) (push) Failing after 52s
Foundation / checks (tsan) (push) Failing after 32s
Foundation / checks (network) (push) Failing after 3m4s
Foundation / checks (client) (push) Failing after 4m24s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
552 lines
26 KiB
C++
552 lines
26 KiB
C++
// Real PostgreSQL M7.3 fencing checks (ADR-0060), publishing M7.4 manifests
|
|
// (ADR-0061) with synthetic payload digests. Run by tools/ci/postgres-check.sh.
|
|
#include "evo/persistence/region_checkpoint.hpp"
|
|
#include "evo/persistence/region_lease_store.hpp"
|
|
#include <algorithm>
|
|
#include <array>
|
|
#include <chrono>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <latch>
|
|
#include <memory>
|
|
#include <source_location>
|
|
#include <span>
|
|
#include <stdexcept>
|
|
#include <string>
|
|
#include <thread>
|
|
#include <vector>
|
|
|
|
void run_region_lease_store_tests();
|
|
|
|
namespace {
|
|
using namespace evo::persistence;
|
|
using namespace std::chrono_literals;
|
|
using std::chrono::milliseconds;
|
|
void check(bool value, std::source_location at = std::source_location::current()) {
|
|
if (!value)
|
|
throw std::runtime_error("region lease store test line " + std::to_string(at.line()));
|
|
}
|
|
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
|
std::string env(const char* key) {
|
|
const auto* value = std::getenv(key);
|
|
if (!value)
|
|
throw std::runtime_error("missing test environment");
|
|
return value;
|
|
}
|
|
std::unique_ptr<Postgres> connect_as(const std::string& user) {
|
|
PgOptions options{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}};
|
|
auto made = Postgres::connect(options, 5s);
|
|
CHECK(made);
|
|
return std::move(*made);
|
|
}
|
|
std::unique_ptr<Postgres> owner() {
|
|
return connect_as(env("PGUSER"));
|
|
}
|
|
std::unique_ptr<Postgres> runtime() {
|
|
return connect_as(env("EVO_PG_RUNTIME_USER"));
|
|
}
|
|
Uuid id(std::uint8_t seed) {
|
|
Uuid out{};
|
|
out.fill(seed);
|
|
return out;
|
|
}
|
|
Sha256Digest digest(std::uint8_t seed) {
|
|
Sha256Digest out{};
|
|
out.fill(seed);
|
|
return out;
|
|
}
|
|
const Sha256Digest build = digest(0xB1);
|
|
void retain_build(Postgres& db) {
|
|
CHECK(db.query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
|
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
|
|
{}, 5s));
|
|
}
|
|
std::string bytea(std::span<const std::uint8_t> bytes) {
|
|
constexpr char digits[] = "0123456789abcdef";
|
|
std::string out = "\\x";
|
|
for (const auto b : bytes) {
|
|
out.push_back(digits[b >> 4U]);
|
|
out.push_back(digits[b & 0x0FU]);
|
|
}
|
|
return out;
|
|
}
|
|
// Genesis commit ID bytes are the region number; frame is 7. Payload digests are
|
|
// synthetic: the database records them and never reads the object store.
|
|
RegionKey provision(Postgres& db, std::uint64_t region) {
|
|
retain_build(db);
|
|
const RegionKey key{1, region};
|
|
auto g = genesis_manifest(key, 7, id(static_cast<std::uint8_t>(region)), 50000, build, 100,
|
|
digest(0xA0));
|
|
CHECK(g);
|
|
auto bytes = encode_manifest(*g);
|
|
CHECK(bytes);
|
|
const std::array<std::string, 1> p{bytea(*bytes)};
|
|
CHECK(db.query("SELECT evolution.provision_region_checkpoint($1::bytea)", p, 5s));
|
|
return key;
|
|
}
|
|
CommittedCheckpoint committed(Postgres& db, RegionKey key) {
|
|
auto c = load_region_checkpoint(db, key, 5s);
|
|
CHECK(c);
|
|
return std::move(*c);
|
|
}
|
|
// Successor fields set directly, so tests can send links the local builder refuses.
|
|
CheckpointManifest link(const CommittedCheckpoint& from, const LeaseGrant& lease,
|
|
std::uint8_t commit, std::uint64_t end_tick) {
|
|
auto m = decode_manifest(from.manifest);
|
|
CHECK(m);
|
|
m->commit = id(commit);
|
|
m->sequence = from.head.sequence + 1;
|
|
m->predecessor_commit = from.head.commit;
|
|
m->predecessor_digest = from.head.digest;
|
|
m->begin_tick = from.head.end_tick;
|
|
m->end_tick = end_tick;
|
|
m->holder = lease.holder;
|
|
m->epoch = lease.epoch;
|
|
m->payload_sha256 = digest(0xA1);
|
|
// Synthetic trace evidence of the right size; no decisions.
|
|
m->batch = {trace_bytes_for(end_tick - from.head.end_tick), digest(0x7C), 0, {}};
|
|
return *m;
|
|
}
|
|
std::expected<CommittedCheckpoint, LeaseStoreFailure> publish(Postgres& db,
|
|
std::vector<std::uint8_t> bytes) {
|
|
auto head = publish_region_checkpoint(db, bytes, {}, 5s);
|
|
if (!head)
|
|
return std::unexpected{head.error()};
|
|
return CommittedCheckpoint{*head, std::move(bytes)};
|
|
}
|
|
std::expected<CommittedCheckpoint, LeaseStoreFailure> publish(Postgres& db,
|
|
const CheckpointManifest& m) {
|
|
auto bytes = encode_manifest(m);
|
|
CHECK(bytes);
|
|
return publish(db, std::move(*bytes));
|
|
}
|
|
// Encodes a valid link, then overwrites raw bytes the encoder would refuse.
|
|
std::vector<std::uint8_t> patched(const CheckpointManifest& m, std::size_t offset,
|
|
std::span<const std::uint8_t> value) {
|
|
auto bytes = encode_manifest(m);
|
|
CHECK(bytes);
|
|
std::ranges::copy(value, bytes->begin() + static_cast<std::ptrdiff_t>(offset));
|
|
return std::move(*bytes);
|
|
}
|
|
// Expiry is decided by database time, which is the server's wall clock. A local
|
|
// sleep does not prove it: a VM can step that clock back (WSL2 was measured
|
|
// stepping ~0.5 s every ~32 s). Wait on the database's own judgement instead.
|
|
// Owner-only read; works inside a transaction that holds the row lock.
|
|
void wait_until_expired(Postgres& owner_db, RegionKey key) {
|
|
const std::array<std::string, 1> region{std::to_string(key.region)};
|
|
for (int attempt = 0; attempt < 400; ++attempt) {
|
|
auto rows = owner_db.query("SELECT (clock_timestamp() >= expires_at)::text FROM "
|
|
"evolution.region_leases WHERE world_id = 1 AND region_id = "
|
|
"$1::evolution.u64",
|
|
region, 5s);
|
|
CHECK(rows && rows->size() == 1);
|
|
if ((*rows)[0][0] == "true")
|
|
return;
|
|
std::this_thread::sleep_for(50ms);
|
|
}
|
|
CHECK(false); // Not expired after 20 s of database time polling.
|
|
}
|
|
bool failed(const auto& result, LeaseStoreError kind) {
|
|
return !result && result.error().kind == kind;
|
|
}
|
|
std::string sqlstate(const std::expected<PgRows, PgFailure>& result) {
|
|
return result ? std::string{} : std::string(result.error().sqlstate.data());
|
|
}
|
|
|
|
void runtime_cannot_bypass_the_guard() {
|
|
auto admin = owner();
|
|
const auto key = provision(*admin, 10);
|
|
auto db = runtime();
|
|
for (const auto* sql : {
|
|
"UPDATE evolution.region_leases SET holder = NULL",
|
|
"UPDATE evolution.region_heads SET commit_sequence = 5",
|
|
"DELETE FROM evolution.region_heads",
|
|
"INSERT INTO evolution.region_leases(world_id,region_id,frame_id) VALUES (9,9,9)",
|
|
"SELECT * FROM evolution.region_leases",
|
|
"SELECT * FROM evolution.region_heads",
|
|
"SELECT evolution.provision_region_checkpoint('\\x00')",
|
|
"SELECT * FROM evolution.region_commits",
|
|
"INSERT INTO evolution.retained_builds VALUES ('\\x00')",
|
|
"SELECT evolution.parse_checkpoint_manifest('\\x00')",
|
|
"SELECT evolution.check_region_lease(1, 10, gen_random_uuid(), 1)",
|
|
"CREATE FUNCTION evolution.bypass() RETURNS int LANGUAGE sql AS 'SELECT 1'",
|
|
"CREATE TABLE evolution.bypass(x int)",
|
|
})
|
|
CHECK(sqlstate(db->query(sql, {}, 5s)) == "42501");
|
|
const std::array<std::string, 1> owner_name{env("PGUSER")};
|
|
CHECK(!db->query("SELECT pg_catalog.set_config('role', $1, false)", owner_name, 5s));
|
|
// A SECURITY DEFINER function ignores a hostile caller search_path.
|
|
CHECK(db->query("SET search_path = pg_temp, public", {}, 5s));
|
|
CHECK(db->query("CREATE TEMP TABLE region_leases(x int)", {}, 5s));
|
|
auto lease = acquire_region_lease(*db, key, build, 20000ms, 5s);
|
|
CHECK(lease && lease->epoch == 1 && lease->frame == 7);
|
|
CHECK(release_region_lease(*db, key, *lease, 5s));
|
|
}
|
|
|
|
void cellular_publisher_adapter_rejects_malformed_requests() {
|
|
auto db = runtime();
|
|
const std::vector<std::uint8_t> empty;
|
|
auto published = publish_cellular_checkpoint(*db, empty, empty, empty, empty, 5s);
|
|
CHECK(failed(published, LeaseStoreError::invalid));
|
|
auto resolved = resolve_cellular_region_commit(*db, {1, 999}, id(0xC1), empty, empty, empty,
|
|
empty, 5s);
|
|
CHECK(failed(resolved, LeaseStoreError::invalid));
|
|
}
|
|
|
|
void cellular_action_block_load_is_lease_fenced_and_optional() {
|
|
auto admin = owner();
|
|
const auto key = provision(*admin, 11);
|
|
auto db = runtime();
|
|
auto lease = acquire_region_lease(*db, key, build, 5s, 5s);
|
|
CHECK(lease);
|
|
auto absent = load_cellular_publication_actions(*db, key, *lease, id(0xF9), 5s);
|
|
CHECK(absent && !*absent);
|
|
CHECK(release_region_lease(*db, key, *lease, 5s));
|
|
}
|
|
|
|
void acquire_renew_release_and_publish() {
|
|
auto admin = owner();
|
|
const auto key = provision(*admin, 20);
|
|
auto db = runtime();
|
|
CHECK(failed(acquire_region_lease(*db, {1, 999}, build, 20000ms, 5s),
|
|
LeaseStoreError::not_provisioned));
|
|
CHECK(failed(acquire_region_lease(*db, key, build, 999ms, 5s), LeaseStoreError::invalid));
|
|
CHECK(failed(acquire_region_lease(*db, key, build, 60001ms, 5s), LeaseStoreError::invalid));
|
|
auto lease = acquire_region_lease(*db, key, build, 20000ms, 5s);
|
|
CHECK(lease && lease->epoch == 1 && lease->duration == 20000ms && lease->holder != Uuid{});
|
|
CHECK(failed(acquire_region_lease(*db, key, build, 20000ms, 5s), LeaseStoreError::busy));
|
|
auto renewed = renew_region_lease(*db, key, *lease, 20000ms, 5s);
|
|
CHECK(renewed && *renewed == 20000ms);
|
|
auto forged = *lease;
|
|
forged.holder[0] ^= 1U;
|
|
CHECK(failed(renew_region_lease(*db, key, forged, 20000ms, 5s), LeaseStoreError::fenced));
|
|
auto stale_epoch = *lease;
|
|
stale_epoch.epoch = 2;
|
|
CHECK(failed(renew_region_lease(*db, key, stale_epoch, 20000ms, 5s), LeaseStoreError::fenced));
|
|
|
|
const auto genesis = committed(*db, key);
|
|
CHECK(genesis.head.sequence == 0 && genesis.head.end_tick == 0 && genesis.head.frame == 7);
|
|
CHECK(genesis.head.commit == id(20) && genesis.head.digest != Sha256Digest{});
|
|
const std::array<std::uint8_t, 1> tick65{65};
|
|
CHECK(failed(publish(*db, patched(link(genesis, *lease, 21, 64), 120, tick65)),
|
|
LeaseStoreError::invalid));
|
|
const auto reused = id(20);
|
|
CHECK(failed(publish(*db, patched(link(genesis, *lease, 21, 1), 40, reused)),
|
|
LeaseStoreError::invalid));
|
|
const auto first = link(genesis, *lease, 21, 64);
|
|
auto head = publish(*db, first);
|
|
CHECK(head && head->head.sequence == 1 && head->head.end_tick == 64 &&
|
|
head->head.commit == id(21));
|
|
CHECK(failed(publish(*db, link(*head, forged, 22, 65)), LeaseStoreError::fenced));
|
|
// Exact retry after an applied publication returns the original receipt (M7.5).
|
|
auto retried = publish(*db, first);
|
|
CHECK(retried && retried->head == head->head);
|
|
auto current = committed(*db, key);
|
|
CHECK(current.head == head->head && current.manifest == head->manifest);
|
|
auto changed = link(*head, *lease, 22, 65);
|
|
changed.predecessor_digest[31] ^= 1U;
|
|
CHECK(failed(publish(*db, changed), LeaseStoreError::head_mismatch));
|
|
const std::array<std::uint8_t, 1> tick64{64};
|
|
CHECK(failed(publish(*db, patched(link(*head, *lease, 22, 65), 120, tick64)),
|
|
LeaseStoreError::invalid));
|
|
auto other_quantum = link(*head, *lease, 22, 65);
|
|
other_quantum.tick_microseconds = 25000; // Fixed per region; changing it is a migration.
|
|
CHECK(failed(publish(*db, other_quantum), LeaseStoreError::unsupported));
|
|
auto unretained = link(*head, *lease, 22, 65);
|
|
unretained.build = digest(0xB2);
|
|
CHECK(failed(publish(*db, unretained), LeaseStoreError::unsupported));
|
|
auto future = link(*head, *lease, 22, 65);
|
|
future.versions.rules = 3;
|
|
CHECK(failed(publish(*db, future), LeaseStoreError::unsupported));
|
|
// Commit IDs are globally unique: region 10's genesis ID cannot be reused here.
|
|
CHECK(failed(publish(*db, link(*head, *lease, 10, 65)), LeaseStoreError::commit_conflict));
|
|
|
|
CHECK(release_region_lease(*db, key, *lease, 5s));
|
|
CHECK(failed(release_region_lease(*db, key, *lease, 5s), LeaseStoreError::fenced));
|
|
CHECK(failed(renew_region_lease(*db, key, *lease, 20000ms, 5s), LeaseStoreError::fenced));
|
|
CHECK(failed(publish(*db, link(*head, *lease, 22, 65)), LeaseStoreError::fenced));
|
|
auto next = acquire_region_lease(*db, key, build, 20000ms, 5s);
|
|
CHECK(next && next->epoch == 2 && next->holder != lease->holder);
|
|
auto final_head = publish(*db, link(*head, *next, 22, 65));
|
|
CHECK(final_head && final_head->head.sequence == 2);
|
|
}
|
|
|
|
void database_time_expiry_and_paused_writer() {
|
|
auto admin = owner();
|
|
const auto key = provision(*admin, 30);
|
|
auto worker_a = runtime();
|
|
auto worker_b = runtime();
|
|
const LeasePolicy fast{1000ms, 200ms, 100ms, 100ms, 300ms};
|
|
LeaseAuthority a{fast};
|
|
auto sent = LeaseClock::now();
|
|
auto lease_a = acquire_region_lease(*worker_a, key, build, fast.duration, 5s);
|
|
CHECK(lease_a && a.acquired(*lease_a, sent, LeaseClock::now()));
|
|
CHECK(failed(acquire_region_lease(*worker_b, key, build, 20000ms, 5s), LeaseStoreError::busy));
|
|
const auto genesis = committed(*worker_a, key);
|
|
|
|
std::this_thread::sleep_for(1200ms); // Worker A is paused past its lease.
|
|
CHECK(a.state(LeaseClock::now()) == LeaseState::fenced);
|
|
// Expiry is decided by database time; the successor acquires a newer epoch.
|
|
wait_until_expired(*admin, key);
|
|
auto lease_b = acquire_region_lease(*worker_b, key, build, 20000ms, 5s);
|
|
CHECK(lease_b && lease_b->epoch == lease_a->epoch + 1);
|
|
auto head_b = publish(*worker_b, link(genesis, *lease_b, 31, 3));
|
|
CHECK(head_b);
|
|
// A resumes and ignores its local fence: every database path rejects it.
|
|
CHECK(failed(renew_region_lease(*worker_a, key, *lease_a, fast.duration, 5s),
|
|
LeaseStoreError::fenced));
|
|
CHECK(failed(publish(*worker_a, link(genesis, *lease_a, 32, 1)), LeaseStoreError::fenced));
|
|
CHECK(failed(publish(*worker_a, link(*head_b, *lease_a, 32, 4)), LeaseStoreError::fenced));
|
|
CHECK(failed(release_region_lease(*worker_a, key, *lease_a, 5s), LeaseStoreError::fenced));
|
|
auto head = load_region_head(*worker_a, key, 5s);
|
|
CHECK(head && *head == head_b->head);
|
|
|
|
// Expired renewal cannot revive a lease even without a successor.
|
|
CHECK(release_region_lease(*worker_b, key, *lease_b, 5s));
|
|
auto lease_c = acquire_region_lease(*worker_a, key, build, 1000ms, 5s);
|
|
CHECK(lease_c && lease_c->epoch == lease_b->epoch + 1);
|
|
wait_until_expired(*admin, key);
|
|
CHECK(
|
|
failed(renew_region_lease(*worker_a, key, *lease_c, 1000ms, 5s), LeaseStoreError::fenced));
|
|
}
|
|
|
|
// A writer queued on the lease row lock is judged by the clock after the wait,
|
|
// not its statement start time, so a lease that expired while queued is fenced.
|
|
void queued_writer_uses_post_lock_database_time() {
|
|
auto admin = owner();
|
|
const auto key = provision(*admin, 40);
|
|
auto worker = runtime();
|
|
auto lease = acquire_region_lease(*worker, key, build, 1000ms, 5s);
|
|
CHECK(lease);
|
|
const auto genesis = committed(*worker, key);
|
|
auto blocker = owner();
|
|
const std::array<std::string, 1> region{"40"};
|
|
CHECK(blocker->query("BEGIN", {}, 5s));
|
|
CHECK(blocker->query("SELECT 1 FROM evolution.region_leases WHERE world_id = 1 AND "
|
|
"region_id = $1::evolution.u64 FOR UPDATE",
|
|
region, 5s));
|
|
std::expected<CommittedCheckpoint, LeaseStoreFailure> queued =
|
|
std::unexpected{LeaseStoreFailure{}};
|
|
std::thread writer([&] { queued = publish(*worker, link(genesis, *lease, 41, 1)); });
|
|
std::this_thread::sleep_for(300ms); // Let the writer queue behind the lock.
|
|
wait_until_expired(*blocker, key); // Still holding the row lock.
|
|
CHECK(blocker->query("ROLLBACK", {}, 5s));
|
|
writer.join();
|
|
CHECK(failed(queued, LeaseStoreError::fenced));
|
|
auto head = load_region_head(*worker, key, 5s);
|
|
CHECK(head && *head == genesis.head);
|
|
}
|
|
|
|
// Partition: the renewal's outcome is unknown and the connection is gone. The
|
|
// holder keeps its earlier deadline and fences itself before a successor starts.
|
|
void partitioned_renewal_fences_locally() {
|
|
auto admin = owner();
|
|
const auto key = provision(*admin, 50);
|
|
auto worker = runtime();
|
|
const LeasePolicy fast{1000ms, 200ms, 100ms, 100ms, 300ms};
|
|
LeaseAuthority authority{fast};
|
|
const auto sent = LeaseClock::now();
|
|
auto lease = acquire_region_lease(*worker, key, build, fast.duration, 5s);
|
|
CHECK(lease && authority.acquired(*lease, sent, LeaseClock::now()));
|
|
auto blocker = owner();
|
|
const std::array<std::string, 1> region{"50"};
|
|
CHECK(blocker->query("BEGIN", {}, 5s));
|
|
CHECK(blocker->query("SELECT 1 FROM evolution.region_leases WHERE world_id = 1 AND "
|
|
"region_id = $1::evolution.u64 FOR UPDATE",
|
|
region, 5s));
|
|
const auto renew_sent = LeaseClock::now();
|
|
auto renewal = renew_region_lease(*worker, key, *lease, fast.duration, fast.renew_timeout);
|
|
CHECK(failed(renewal, LeaseStoreError::unknown));
|
|
authority.renewal_unknown(fast.duration, renew_sent);
|
|
const auto local_deadline = authority.deadline();
|
|
CHECK(local_deadline <= sent + fast.duration - fast.safety_margin);
|
|
std::this_thread::sleep_for(1300ms);
|
|
CHECK(!authority.may_originate(LeaseClock::now()));
|
|
// Release the orphaned renewal only after database expiry, so it must be refused.
|
|
wait_until_expired(*blocker, key);
|
|
CHECK(blocker->query("ROLLBACK", {}, 5s));
|
|
// The orphaned renewal reached the row only after database expiry: rejected.
|
|
auto successor = runtime();
|
|
auto next = acquire_region_lease(*successor, key, build, 20000ms, 5s);
|
|
CHECK(next && next->epoch == lease->epoch + 1);
|
|
}
|
|
|
|
void contenders_serialize_on_the_database() {
|
|
auto admin = owner();
|
|
const auto key = provision(*admin, 60);
|
|
std::array<std::unique_ptr<Postgres>, 2> workers{runtime(), runtime()};
|
|
std::uint64_t previous_epoch = 0;
|
|
constexpr int rounds = 64;
|
|
for (int round = 0; round < rounds; ++round) {
|
|
std::array<std::expected<LeaseGrant, LeaseStoreFailure>, 2> results{
|
|
std::unexpected{LeaseStoreFailure{}}, std::unexpected{LeaseStoreFailure{}}};
|
|
std::latch start{2};
|
|
std::vector<std::thread> threads;
|
|
threads.reserve(2);
|
|
for (std::size_t i = 0; i < 2; ++i)
|
|
threads.emplace_back([&, i] {
|
|
start.arrive_and_wait();
|
|
results[i] = acquire_region_lease(*workers[i], key, build, 20000ms, 5s);
|
|
});
|
|
for (auto& t : threads)
|
|
t.join();
|
|
const auto winners = std::count_if(results.begin(), results.end(),
|
|
[](const auto& r) { return r.has_value(); });
|
|
CHECK(winners == 1);
|
|
const std::size_t w = results[0] ? 0 : 1;
|
|
CHECK(failed(results[1 - w], LeaseStoreError::busy));
|
|
CHECK(results[w]->epoch == previous_epoch + 1);
|
|
previous_epoch = results[w]->epoch;
|
|
CHECK(release_region_lease(*workers[w], key, *results[w], 5s));
|
|
}
|
|
|
|
// The same holder racing two commits from the same predecessor: exactly one wins.
|
|
auto lease = acquire_region_lease(*workers[0], key, build, 20000ms, 5s);
|
|
CHECK(lease);
|
|
auto head = committed(*workers[0], key);
|
|
for (std::uint8_t round = 0; round < 32; ++round) {
|
|
std::array<std::expected<CommittedCheckpoint, LeaseStoreFailure>, 2> results{
|
|
std::unexpected{LeaseStoreFailure{}}, std::unexpected{LeaseStoreFailure{}}};
|
|
std::latch start{2};
|
|
std::vector<std::thread> threads;
|
|
threads.reserve(2);
|
|
for (std::size_t i = 0; i < 2; ++i)
|
|
threads.emplace_back([&, i] {
|
|
const auto seed = static_cast<std::uint8_t>(100 + (round * 2) + i);
|
|
start.arrive_and_wait();
|
|
results[i] = publish(*workers[i], link(head, *lease, seed, head.head.end_tick + 1));
|
|
});
|
|
for (auto& t : threads)
|
|
t.join();
|
|
const std::size_t w = results[0] ? 0 : 1;
|
|
CHECK(results[w] && failed(results[1 - w], LeaseStoreError::head_mismatch));
|
|
CHECK(results[w]->head.sequence == head.head.sequence + 1);
|
|
head = *results[w];
|
|
}
|
|
auto last = load_region_head(*workers[1], key, 5s);
|
|
CHECK(last && *last == head.head && last->sequence == 32);
|
|
}
|
|
|
|
void exhaustion_fails_closed() {
|
|
auto admin = owner();
|
|
const auto key = provision(*admin, 70);
|
|
CHECK(admin->query("UPDATE evolution.region_leases SET epoch = 18446744073709551615 "
|
|
"WHERE world_id = 1 AND region_id = 70",
|
|
{}, 5s));
|
|
auto db = runtime();
|
|
CHECK(failed(acquire_region_lease(*db, key, build, 20000ms, 5s), LeaseStoreError::exhausted));
|
|
const auto other = provision(*admin, 71);
|
|
auto lease = acquire_region_lease(*db, other, build, 20000ms, 5s);
|
|
CHECK(lease);
|
|
// Owner-only fixture: a chain link at the last sequence (the head FK requires it).
|
|
CHECK(admin->query(
|
|
"INSERT INTO evolution.region_commits(world_id, region_id, commit_sequence, commit_id, "
|
|
"commit_digest, manifest, begin_tick, end_tick, published_epoch, tick_microseconds, "
|
|
"versions, build_identity, payload_bytes, payload_sha256) SELECT world_id, region_id, "
|
|
"18446744073709551615, '000000ff-0000-0000-0000-000000000000', commit_digest, manifest, 0, "
|
|
"5, 1, tick_microseconds, versions, build_identity, payload_bytes, payload_sha256 FROM "
|
|
"evolution.region_commits WHERE world_id = 1 AND region_id = 71",
|
|
{}, 5s));
|
|
CHECK(admin->query("UPDATE evolution.region_heads SET commit_sequence = 18446744073709551615, "
|
|
"commit_id = '000000ff-0000-0000-0000-000000000000', end_tick = 5, "
|
|
"published_epoch = 1 WHERE world_id = 1 AND region_id = 71",
|
|
{}, 5s));
|
|
const auto last = committed(*db, other);
|
|
CHECK(last.head.sequence == 18446744073709551615ULL);
|
|
auto beyond = link(last, *lease, 72, 6);
|
|
beyond.sequence = 18446744073709551615ULL; // Any value: exhaustion is checked first.
|
|
CHECK(failed(publish(*db, beyond), LeaseStoreError::exhausted));
|
|
}
|
|
|
|
// The SQL parser's per-profile payload caps match the C++ constants exactly.
|
|
void payload_caps_follow_the_profile() {
|
|
auto admin = owner();
|
|
retain_build(*admin);
|
|
const auto provision_raw = [&](std::vector<std::uint8_t> bytes) {
|
|
const std::array<std::string, 1> p{bytea(bytes)};
|
|
return admin->query("SELECT evolution.provision_region_checkpoint($1::bytea)", p, 5s);
|
|
};
|
|
auto lineage = genesis_manifest({1, 90}, 7, id(90), 50000, build, max_lineage_payload_bytes,
|
|
digest(0xA0), CheckpointProfile::environmental_lineage_v1);
|
|
CHECK(lineage);
|
|
CHECK(provision_raw(*encode_manifest(*lineage)));
|
|
lineage->key.region = 91;
|
|
lineage->commit = id(91);
|
|
const std::array<std::uint8_t, 8> over_lineage{0x71, 0x20, 0x41, 0, 0, 0, 0, 0}; // 4268145
|
|
CHECK(sqlstate(provision_raw(patched(*lineage, 240, over_lineage))) == "22023");
|
|
auto plain = genesis_manifest({1, 92}, 7, id(92), 50000, build, max_environmental_payload_bytes,
|
|
digest(0xA0));
|
|
CHECK(plain);
|
|
const std::array<std::uint8_t, 8> over_plain{0x01, 0x00, 0x10, 0, 0, 0, 0, 0}; // 1 MiB + 1
|
|
CHECK(sqlstate(provision_raw(patched(*plain, 240, over_plain))) == "22023");
|
|
// Profile and history profile must match: a lineage profile claiming no history.
|
|
const std::array<std::uint8_t, 1> no_history{0};
|
|
CHECK(sqlstate(provision_raw(patched(*lineage, 232, no_history))) == "EVL06");
|
|
}
|
|
|
|
struct Summary {
|
|
std::vector<long long> samples;
|
|
void print(const char* name) {
|
|
std::sort(samples.begin(), samples.end());
|
|
const auto pick = [&](std::size_t percent) {
|
|
return samples[std::min(samples.size() - 1, samples.size() * percent / 100)];
|
|
};
|
|
std::printf("lease_latency_us %s n=%zu p50=%lld p99=%lld max=%lld\n", name, samples.size(),
|
|
pick(50), pick(99), samples.back());
|
|
}
|
|
};
|
|
template <typename F> long long timed(F&& f) {
|
|
const auto begin = std::chrono::steady_clock::now();
|
|
f();
|
|
return std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() -
|
|
begin)
|
|
.count();
|
|
}
|
|
|
|
// Local Unix-socket round trips only; remote TLS latency must be measured in place.
|
|
void measure_round_trips() {
|
|
auto admin = owner();
|
|
const auto key = provision(*admin, 80);
|
|
auto db = runtime();
|
|
constexpr int n = 200;
|
|
Summary acquire, renew, advance, release;
|
|
auto head = committed(*db, key);
|
|
for (int i = 0; i < n; ++i) {
|
|
std::expected<LeaseGrant, LeaseStoreFailure> lease = std::unexpected{LeaseStoreFailure{}};
|
|
acquire.samples.push_back(
|
|
timed([&] { lease = acquire_region_lease(*db, key, build, 20000ms, 5s); }));
|
|
CHECK(lease);
|
|
renew.samples.push_back(
|
|
timed([&] { CHECK(renew_region_lease(*db, key, *lease, 20000ms, 5s)); }));
|
|
auto m = link(head, *lease, 0, head.head.end_tick + 64);
|
|
m.commit[0] = 0xAA; // Unique across all tests: 0xAA, index, zeros.
|
|
m.commit[1] = static_cast<std::uint8_t>(i);
|
|
advance.samples.push_back(timed([&] {
|
|
auto next = publish(*db, m);
|
|
CHECK(next);
|
|
head = std::move(*next);
|
|
}));
|
|
release.samples.push_back(
|
|
timed([&] { CHECK(release_region_lease(*db, key, *lease, 5s)); }));
|
|
}
|
|
acquire.print("acquire");
|
|
renew.print("renew");
|
|
advance.print("publish_manifest");
|
|
release.print("release");
|
|
}
|
|
} // namespace
|
|
|
|
void run_region_lease_store_tests() {
|
|
runtime_cannot_bypass_the_guard();
|
|
cellular_publisher_adapter_rejects_malformed_requests();
|
|
cellular_action_block_load_is_lease_fenced_and_optional();
|
|
acquire_renew_release_and_publish();
|
|
database_time_expiry_and_paused_writer();
|
|
queued_writer_uses_post_lock_database_time();
|
|
partitioned_renewal_fences_locally();
|
|
contenders_serialize_on_the_database();
|
|
exhaustion_fails_closed();
|
|
payload_caps_follow_the_profile();
|
|
measure_round_trips();
|
|
}
|