Complete M7.3 fenced region ownership

ADR-0060 and schema v2 add database-time region leases and a fenced head
advance. Acquire/renew/release/advance lock the lease row, check holder,
epoch and expiry with clock_timestamp() after the lock wait, and the
advance compares the exact predecessor. Holders are DB-generated tokens;
epochs never wrap or repeat. The runtime group role gets EXECUTE on five
SECURITY DEFINER functions only - no table, provisioning or CREATE rights.
Migration history admits clean->v2, v1->v2 and a v2 no-op, and refuses
privileged runtime roles.

Host side: portable LeaseAuthority fences from send-time deadlines on
CLOCK_BOOTTIME, keeps the earlier deadline on unknown renewal and stays
fenced until a newer epoch. Selected timings: 20s lease, 5s renew, 2s
margin, 2s renew and 5s commit deadlines. Linux evo::postgres wraps the
functions with typed LeaseStoreError results.

Validated: Windows CTest 55/55; WSL Release, clang-tidy and ASan/UBSan
private-PostgreSQL suites (contenders, expiry, paused/queued stale
writers, unknown renewal, exhaustion, bypass denial, upgrade path).
Three SQL mutations were each caught. Docker/Gitea CI, remote TLS and
cluster runs were not executed. No director, host or deployment added.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-28 11:14:32 -04:00
co-authored by Claude Opus 5.5
parent 598b3938e6
commit 2496649a9b
22 changed files with 1501 additions and 29 deletions
+96
View File
@@ -0,0 +1,96 @@
#include "evo/persistence/region_lease.hpp"
#include <algorithm>
#if defined(__linux__)
#include <time.h>
#endif
namespace evo::persistence {
namespace {
using std::chrono::milliseconds;
constexpr milliseconds minimum_duration{1000};
constexpr milliseconds maximum_duration{60000};
constexpr milliseconds maximum_io{30000}; // Postgres adapter deadline bound.
} // namespace
bool valid(const LeasePolicy& p) noexcept {
const milliseconds zero{};
if (p.duration < minimum_duration || p.duration > maximum_duration ||
p.renew_interval <= zero || p.safety_margin <= zero || p.renew_timeout <= zero ||
p.commit_timeout <= zero || p.renew_timeout > maximum_io || p.commit_timeout > maximum_io ||
p.safety_margin >= p.duration)
return false;
const auto usable = p.duration - p.safety_margin;
return p.safety_margin * 10 >= p.duration &&
(p.renew_interval + p.renew_timeout) * 2 <= usable && p.commit_timeout < usable;
}
LeaseClock::time_point LeaseClock::now() noexcept {
#if defined(__linux__)
timespec ts{};
// CLOCK_BOOTTIME cannot fail with a valid clock id and pointer.
static_cast<void>(clock_gettime(CLOCK_BOOTTIME, &ts));
return time_point{std::chrono::seconds{ts.tv_sec} + std::chrono::nanoseconds{ts.tv_nsec}};
#else
// Storage hosts are Linux-only; elsewhere suspend behavior is unqualified.
return time_point{
std::chrono::duration_cast<duration>(std::chrono::steady_clock::now().time_since_epoch())};
#endif
}
bool LeaseAuthority::acquired(const LeaseGrant& grant, LeaseClock::time_point sent,
LeaseClock::time_point received) noexcept {
const auto deadline = sent + grant.duration - policy_.safety_margin;
if (!valid(policy_) || grant.epoch == 0 || grant.epoch <= highest_epoch_ || grant.frame == 0 ||
grant.duration != policy_.duration || received < sent || received >= deadline) {
fence();
return false;
}
grant_ = grant;
highest_epoch_ = grant.epoch;
last_sent_ = sent;
deadline_ = deadline;
fenced_ = false;
return true;
}
bool LeaseAuthority::renewed(std::uint64_t epoch, milliseconds granted, LeaseClock::time_point sent,
LeaseClock::time_point received) noexcept {
if (!grant_ || fenced_ || epoch != grant_->epoch || granted != policy_.duration ||
sent < last_sent_ || received < sent || received >= deadline_) {
fence();
return false;
}
// Database expiry is its execution time plus granted, which is after sent.
deadline_ = sent + granted - policy_.safety_margin;
last_sent_ = sent;
return true;
}
void LeaseAuthority::renewal_unknown(milliseconds requested, LeaseClock::time_point sent) noexcept {
if (!grant_ || fenced_)
return;
deadline_ = std::min(deadline_, sent + requested - policy_.safety_margin);
last_sent_ = std::max(last_sent_, sent);
}
void LeaseAuthority::fence() noexcept {
fenced_ = true;
}
LeaseState LeaseAuthority::state(LeaseClock::time_point now) const noexcept {
if (!grant_)
return LeaseState::absent;
if (fenced_ || now >= deadline_)
return LeaseState::fenced;
return now >= last_sent_ + policy_.renew_interval ? LeaseState::renew_due : LeaseState::active;
}
bool LeaseAuthority::may_originate(LeaseClock::time_point now) const noexcept {
const auto s = state(now);
return s == LeaseState::active || s == LeaseState::renew_due;
}
bool LeaseAuthority::may_begin_commit(LeaseClock::time_point now) const noexcept {
return may_originate(now) && now + policy_.commit_timeout <= deadline_;
}
} // namespace evo::persistence
+204
View File
@@ -0,0 +1,204 @@
#include "evo/persistence/region_lease_store.hpp"
#include <array>
#include <charconv>
#include <optional>
#include <string>
#include <string_view>
namespace evo::persistence {
namespace {
using std::chrono::milliseconds;
constexpr std::string_view digits = "0123456789abcdef";
LeaseStoreFailure classify(const PgFailure& f) {
const std::string_view state(f.sqlstate.data(), 5);
auto kind = LeaseStoreError::store;
if (f.kind == PgError::unknown)
kind = LeaseStoreError::unknown;
else if (f.kind == PgError::config || f.kind == PgError::bounds || state == "22023")
kind = LeaseStoreError::invalid;
else if (state == "EVL01")
kind = LeaseStoreError::busy;
else if (state == "EVL02")
kind = LeaseStoreError::fenced;
else if (state == "EVL03")
kind = LeaseStoreError::head_mismatch;
else if (state == "EVL04")
kind = LeaseStoreError::not_provisioned;
else if (state == "EVL05")
kind = LeaseStoreError::exhausted;
return {kind, f};
}
LeaseStoreFailure malformed(bool mutation) {
// A mutation whose reply cannot be parsed may still have committed.
return {mutation ? LeaseStoreError::unknown : LeaseStoreError::store, {}};
}
template <std::size_t N> std::string hex(const std::array<std::uint8_t, N>& bytes) {
std::string out;
out.reserve(N * 2);
for (const auto b : bytes) {
out.push_back(digits[static_cast<std::size_t>(b >> 4U)]);
out.push_back(digits[static_cast<std::size_t>(b & 0x0FU)]);
}
return out;
}
std::optional<std::uint8_t> nibble(char c) {
const auto at = digits.find(c);
if (at == std::string_view::npos)
return std::nullopt;
return static_cast<std::uint8_t>(at);
}
template <std::size_t N> std::optional<std::array<std::uint8_t, N>> unhex(std::string_view s) {
if (s.size() != N * 2)
return std::nullopt;
std::array<std::uint8_t, N> out{};
for (std::size_t i = 0; i < N; ++i) {
const auto high = nibble(s[i * 2]);
const auto low = nibble(s[(i * 2) + 1]);
if (!high || !low)
return std::nullopt;
out[i] = static_cast<std::uint8_t>((*high << 4U) | *low);
}
return out;
}
std::string uuid_text(const Uuid& id) {
const auto h = hex(id);
return h.substr(0, 8) + '-' + h.substr(8, 4) + '-' + h.substr(12, 4) + '-' + h.substr(16, 4) +
'-' + h.substr(20);
}
std::optional<Uuid> parse_uuid(const std::optional<std::string>& s) {
if (!s || s->size() != 36)
return std::nullopt;
std::string compact;
for (std::size_t i = 0; i < s->size(); ++i) {
const bool dash = i == 8 || i == 13 || i == 18 || i == 23;
if (dash != ((*s)[i] == '-'))
return std::nullopt;
if (!dash)
compact.push_back((*s)[i]);
}
return unhex<16>(compact);
}
template <typename T> std::optional<T> parse_integer(const std::optional<std::string>& s) {
if (!s || s->empty() || s->size() > 20 || s->front() == '-' || s->front() == '+')
return std::nullopt;
T value{};
const auto* end = s->data() + s->size();
const auto [at, error] = std::from_chars(s->data(), end, value);
if (error != std::errc{} || at != end)
return std::nullopt;
return value;
}
std::string bytea(const Sha256Digest& d) {
return "\\x" + hex(d);
}
std::optional<RegionHead> parse_head(const PgRows& rows) {
if (rows.size() != 1 || rows[0].size() != 5)
return std::nullopt;
const auto& r = rows[0];
const auto frame = parse_integer<std::uint64_t>(r[0]);
const auto sequence = parse_integer<std::uint64_t>(r[1]);
const auto commit = parse_uuid(r[2]);
const auto& digest_text = r[3];
const auto digest = digest_text ? unhex<32>(*digest_text) : std::nullopt;
const auto tick = parse_integer<std::uint64_t>(r[4]);
if (!frame || !sequence || !commit || !digest || !tick)
return std::nullopt;
return RegionHead{*frame, *sequence, *commit, *digest, *tick};
}
} // namespace
std::expected<LeaseGrant, LeaseStoreFailure>
acquire_region_lease(Postgres& db, RegionKey key, milliseconds duration, milliseconds timeout) {
const std::array<std::string, 3> p{std::to_string(key.world), std::to_string(key.region),
std::to_string(duration.count())};
auto rows = db.query("SELECT * FROM evolution.acquire_region_lease($1::evolution.u64, "
"$2::evolution.u64, $3::integer)",
p, timeout);
if (!rows)
return std::unexpected{classify(rows.error())};
if (rows->size() != 1 || (*rows)[0].size() != 4)
return std::unexpected{malformed(true)};
const auto& r = (*rows)[0];
const auto holder = parse_uuid(r[0]);
const auto epoch = parse_integer<std::uint64_t>(r[1]);
const auto frame = parse_integer<std::uint64_t>(r[2]);
const auto granted = parse_integer<std::int64_t>(r[3]);
if (!holder || !epoch || !frame || !granted || *epoch == 0)
return std::unexpected{malformed(true)};
return LeaseGrant{*holder, *epoch, *frame, milliseconds{*granted}};
}
std::expected<milliseconds, LeaseStoreFailure> renew_region_lease(Postgres& db, RegionKey key,
const LeaseGrant& lease,
milliseconds duration,
milliseconds timeout) {
const std::array<std::string, 5> p{std::to_string(key.world), std::to_string(key.region),
uuid_text(lease.holder), std::to_string(lease.epoch),
std::to_string(duration.count())};
auto rows = db.query("SELECT evolution.renew_region_lease($1::evolution.u64, "
"$2::evolution.u64, $3::uuid, $4::evolution.u64, $5::integer)",
p, timeout);
if (!rows)
return std::unexpected{classify(rows.error())};
const auto granted = rows->size() == 1 && (*rows)[0].size() == 1
? parse_integer<std::int64_t>((*rows)[0][0])
: std::nullopt;
if (!granted)
return std::unexpected{malformed(true)};
return milliseconds{*granted};
}
std::expected<void, LeaseStoreFailure>
release_region_lease(Postgres& db, RegionKey key, const LeaseGrant& lease, milliseconds timeout) {
const std::array<std::string, 4> p{std::to_string(key.world), std::to_string(key.region),
uuid_text(lease.holder), std::to_string(lease.epoch)};
auto rows = db.query("SELECT evolution.release_region_lease($1::evolution.u64, "
"$2::evolution.u64, $3::uuid, $4::evolution.u64)",
p, timeout);
if (!rows)
return std::unexpected{classify(rows.error())};
if (rows->size() != 1)
return std::unexpected{malformed(true)};
return {};
}
std::expected<RegionHead, LeaseStoreFailure> load_region_head(Postgres& db, RegionKey key,
milliseconds timeout) {
const std::array<std::string, 2> p{std::to_string(key.world), std::to_string(key.region)};
auto rows =
db.query("SELECT * FROM evolution.load_region_head($1::evolution.u64, $2::evolution.u64)",
p, timeout);
if (!rows)
return std::unexpected{classify(rows.error())};
auto head = parse_head(*rows);
if (!head)
return std::unexpected{malformed(false)};
return *head;
}
std::expected<RegionHead, LeaseStoreFailure>
advance_region_head(Postgres& db, RegionKey key, const LeaseGrant& lease,
const RegionHead& expected, const HeadAdvance& next, milliseconds timeout) {
const std::array<std::string, 10> p{std::to_string(key.world),
std::to_string(key.region),
uuid_text(lease.holder),
std::to_string(lease.epoch),
std::to_string(expected.sequence),
uuid_text(expected.commit),
bytea(expected.digest),
uuid_text(next.commit),
bytea(next.digest),
std::to_string(next.end_tick)};
auto rows = db.query("SELECT * FROM evolution.advance_region_head($1::evolution.u64, "
"$2::evolution.u64, $3::uuid, $4::evolution.u64, $5::evolution.u64, "
"$6::uuid, $7::bytea, $8::uuid, $9::bytea, $10::evolution.u64)",
p, timeout);
if (!rows)
return std::unexpected{classify(rows.error())};
auto head = parse_head(*rows);
if (!head)
return std::unexpected{malformed(true)};
return *head;
}
} // namespace evo::persistence