Complete M7.3 fenced region ownership
ADR-0060 and schema v2 add database-time region leases and a fenced head advance. Acquire/renew/release/advance lock the lease row, check holder, epoch and expiry with clock_timestamp() after the lock wait, and the advance compares the exact predecessor. Holders are DB-generated tokens; epochs never wrap or repeat. The runtime group role gets EXECUTE on five SECURITY DEFINER functions only - no table, provisioning or CREATE rights. Migration history admits clean->v2, v1->v2 and a v2 no-op, and refuses privileged runtime roles. Host side: portable LeaseAuthority fences from send-time deadlines on CLOCK_BOOTTIME, keeps the earlier deadline on unknown renewal and stays fenced until a newer epoch. Selected timings: 20s lease, 5s renew, 2s margin, 2s renew and 5s commit deadlines. Linux evo::postgres wraps the functions with typed LeaseStoreError results. Validated: Windows CTest 55/55; WSL Release, clang-tidy and ASan/UBSan private-PostgreSQL suites (contenders, expiry, paused/queued stale writers, unknown renewal, exhaustion, bypass denial, upgrade path). Three SQL mutations were each caught. Docker/Gitea CI, remote TLS and cluster runs were not executed. No director, host or deployment added. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,96 @@
|
||||
#include "evo/persistence/region_lease.hpp"
|
||||
#include <algorithm>
|
||||
#if defined(__linux__)
|
||||
#include <time.h>
|
||||
#endif
|
||||
|
||||
namespace evo::persistence {
|
||||
namespace {
|
||||
using std::chrono::milliseconds;
|
||||
constexpr milliseconds minimum_duration{1000};
|
||||
constexpr milliseconds maximum_duration{60000};
|
||||
constexpr milliseconds maximum_io{30000}; // Postgres adapter deadline bound.
|
||||
} // namespace
|
||||
|
||||
bool valid(const LeasePolicy& p) noexcept {
|
||||
const milliseconds zero{};
|
||||
if (p.duration < minimum_duration || p.duration > maximum_duration ||
|
||||
p.renew_interval <= zero || p.safety_margin <= zero || p.renew_timeout <= zero ||
|
||||
p.commit_timeout <= zero || p.renew_timeout > maximum_io || p.commit_timeout > maximum_io ||
|
||||
p.safety_margin >= p.duration)
|
||||
return false;
|
||||
const auto usable = p.duration - p.safety_margin;
|
||||
return p.safety_margin * 10 >= p.duration &&
|
||||
(p.renew_interval + p.renew_timeout) * 2 <= usable && p.commit_timeout < usable;
|
||||
}
|
||||
|
||||
LeaseClock::time_point LeaseClock::now() noexcept {
|
||||
#if defined(__linux__)
|
||||
timespec ts{};
|
||||
// CLOCK_BOOTTIME cannot fail with a valid clock id and pointer.
|
||||
static_cast<void>(clock_gettime(CLOCK_BOOTTIME, &ts));
|
||||
return time_point{std::chrono::seconds{ts.tv_sec} + std::chrono::nanoseconds{ts.tv_nsec}};
|
||||
#else
|
||||
// Storage hosts are Linux-only; elsewhere suspend behavior is unqualified.
|
||||
return time_point{
|
||||
std::chrono::duration_cast<duration>(std::chrono::steady_clock::now().time_since_epoch())};
|
||||
#endif
|
||||
}
|
||||
|
||||
bool LeaseAuthority::acquired(const LeaseGrant& grant, LeaseClock::time_point sent,
|
||||
LeaseClock::time_point received) noexcept {
|
||||
const auto deadline = sent + grant.duration - policy_.safety_margin;
|
||||
if (!valid(policy_) || grant.epoch == 0 || grant.epoch <= highest_epoch_ || grant.frame == 0 ||
|
||||
grant.duration != policy_.duration || received < sent || received >= deadline) {
|
||||
fence();
|
||||
return false;
|
||||
}
|
||||
grant_ = grant;
|
||||
highest_epoch_ = grant.epoch;
|
||||
last_sent_ = sent;
|
||||
deadline_ = deadline;
|
||||
fenced_ = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool LeaseAuthority::renewed(std::uint64_t epoch, milliseconds granted, LeaseClock::time_point sent,
|
||||
LeaseClock::time_point received) noexcept {
|
||||
if (!grant_ || fenced_ || epoch != grant_->epoch || granted != policy_.duration ||
|
||||
sent < last_sent_ || received < sent || received >= deadline_) {
|
||||
fence();
|
||||
return false;
|
||||
}
|
||||
// Database expiry is its execution time plus granted, which is after sent.
|
||||
deadline_ = sent + granted - policy_.safety_margin;
|
||||
last_sent_ = sent;
|
||||
return true;
|
||||
}
|
||||
|
||||
void LeaseAuthority::renewal_unknown(milliseconds requested, LeaseClock::time_point sent) noexcept {
|
||||
if (!grant_ || fenced_)
|
||||
return;
|
||||
deadline_ = std::min(deadline_, sent + requested - policy_.safety_margin);
|
||||
last_sent_ = std::max(last_sent_, sent);
|
||||
}
|
||||
|
||||
void LeaseAuthority::fence() noexcept {
|
||||
fenced_ = true;
|
||||
}
|
||||
|
||||
LeaseState LeaseAuthority::state(LeaseClock::time_point now) const noexcept {
|
||||
if (!grant_)
|
||||
return LeaseState::absent;
|
||||
if (fenced_ || now >= deadline_)
|
||||
return LeaseState::fenced;
|
||||
return now >= last_sent_ + policy_.renew_interval ? LeaseState::renew_due : LeaseState::active;
|
||||
}
|
||||
|
||||
bool LeaseAuthority::may_originate(LeaseClock::time_point now) const noexcept {
|
||||
const auto s = state(now);
|
||||
return s == LeaseState::active || s == LeaseState::renew_due;
|
||||
}
|
||||
|
||||
bool LeaseAuthority::may_begin_commit(LeaseClock::time_point now) const noexcept {
|
||||
return may_originate(now) && now + policy_.commit_timeout <= deadline_;
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
@@ -0,0 +1,204 @@
|
||||
#include "evo/persistence/region_lease_store.hpp"
|
||||
#include <array>
|
||||
#include <charconv>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
namespace evo::persistence {
|
||||
namespace {
|
||||
using std::chrono::milliseconds;
|
||||
constexpr std::string_view digits = "0123456789abcdef";
|
||||
|
||||
LeaseStoreFailure classify(const PgFailure& f) {
|
||||
const std::string_view state(f.sqlstate.data(), 5);
|
||||
auto kind = LeaseStoreError::store;
|
||||
if (f.kind == PgError::unknown)
|
||||
kind = LeaseStoreError::unknown;
|
||||
else if (f.kind == PgError::config || f.kind == PgError::bounds || state == "22023")
|
||||
kind = LeaseStoreError::invalid;
|
||||
else if (state == "EVL01")
|
||||
kind = LeaseStoreError::busy;
|
||||
else if (state == "EVL02")
|
||||
kind = LeaseStoreError::fenced;
|
||||
else if (state == "EVL03")
|
||||
kind = LeaseStoreError::head_mismatch;
|
||||
else if (state == "EVL04")
|
||||
kind = LeaseStoreError::not_provisioned;
|
||||
else if (state == "EVL05")
|
||||
kind = LeaseStoreError::exhausted;
|
||||
return {kind, f};
|
||||
}
|
||||
LeaseStoreFailure malformed(bool mutation) {
|
||||
// A mutation whose reply cannot be parsed may still have committed.
|
||||
return {mutation ? LeaseStoreError::unknown : LeaseStoreError::store, {}};
|
||||
}
|
||||
template <std::size_t N> std::string hex(const std::array<std::uint8_t, N>& bytes) {
|
||||
std::string out;
|
||||
out.reserve(N * 2);
|
||||
for (const auto b : bytes) {
|
||||
out.push_back(digits[static_cast<std::size_t>(b >> 4U)]);
|
||||
out.push_back(digits[static_cast<std::size_t>(b & 0x0FU)]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
std::optional<std::uint8_t> nibble(char c) {
|
||||
const auto at = digits.find(c);
|
||||
if (at == std::string_view::npos)
|
||||
return std::nullopt;
|
||||
return static_cast<std::uint8_t>(at);
|
||||
}
|
||||
template <std::size_t N> std::optional<std::array<std::uint8_t, N>> unhex(std::string_view s) {
|
||||
if (s.size() != N * 2)
|
||||
return std::nullopt;
|
||||
std::array<std::uint8_t, N> out{};
|
||||
for (std::size_t i = 0; i < N; ++i) {
|
||||
const auto high = nibble(s[i * 2]);
|
||||
const auto low = nibble(s[(i * 2) + 1]);
|
||||
if (!high || !low)
|
||||
return std::nullopt;
|
||||
out[i] = static_cast<std::uint8_t>((*high << 4U) | *low);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
std::string uuid_text(const Uuid& id) {
|
||||
const auto h = hex(id);
|
||||
return h.substr(0, 8) + '-' + h.substr(8, 4) + '-' + h.substr(12, 4) + '-' + h.substr(16, 4) +
|
||||
'-' + h.substr(20);
|
||||
}
|
||||
std::optional<Uuid> parse_uuid(const std::optional<std::string>& s) {
|
||||
if (!s || s->size() != 36)
|
||||
return std::nullopt;
|
||||
std::string compact;
|
||||
for (std::size_t i = 0; i < s->size(); ++i) {
|
||||
const bool dash = i == 8 || i == 13 || i == 18 || i == 23;
|
||||
if (dash != ((*s)[i] == '-'))
|
||||
return std::nullopt;
|
||||
if (!dash)
|
||||
compact.push_back((*s)[i]);
|
||||
}
|
||||
return unhex<16>(compact);
|
||||
}
|
||||
template <typename T> std::optional<T> parse_integer(const std::optional<std::string>& s) {
|
||||
if (!s || s->empty() || s->size() > 20 || s->front() == '-' || s->front() == '+')
|
||||
return std::nullopt;
|
||||
T value{};
|
||||
const auto* end = s->data() + s->size();
|
||||
const auto [at, error] = std::from_chars(s->data(), end, value);
|
||||
if (error != std::errc{} || at != end)
|
||||
return std::nullopt;
|
||||
return value;
|
||||
}
|
||||
std::string bytea(const Sha256Digest& d) {
|
||||
return "\\x" + hex(d);
|
||||
}
|
||||
std::optional<RegionHead> parse_head(const PgRows& rows) {
|
||||
if (rows.size() != 1 || rows[0].size() != 5)
|
||||
return std::nullopt;
|
||||
const auto& r = rows[0];
|
||||
const auto frame = parse_integer<std::uint64_t>(r[0]);
|
||||
const auto sequence = parse_integer<std::uint64_t>(r[1]);
|
||||
const auto commit = parse_uuid(r[2]);
|
||||
const auto& digest_text = r[3];
|
||||
const auto digest = digest_text ? unhex<32>(*digest_text) : std::nullopt;
|
||||
const auto tick = parse_integer<std::uint64_t>(r[4]);
|
||||
if (!frame || !sequence || !commit || !digest || !tick)
|
||||
return std::nullopt;
|
||||
return RegionHead{*frame, *sequence, *commit, *digest, *tick};
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::expected<LeaseGrant, LeaseStoreFailure>
|
||||
acquire_region_lease(Postgres& db, RegionKey key, milliseconds duration, milliseconds timeout) {
|
||||
const std::array<std::string, 3> p{std::to_string(key.world), std::to_string(key.region),
|
||||
std::to_string(duration.count())};
|
||||
auto rows = db.query("SELECT * FROM evolution.acquire_region_lease($1::evolution.u64, "
|
||||
"$2::evolution.u64, $3::integer)",
|
||||
p, timeout);
|
||||
if (!rows)
|
||||
return std::unexpected{classify(rows.error())};
|
||||
if (rows->size() != 1 || (*rows)[0].size() != 4)
|
||||
return std::unexpected{malformed(true)};
|
||||
const auto& r = (*rows)[0];
|
||||
const auto holder = parse_uuid(r[0]);
|
||||
const auto epoch = parse_integer<std::uint64_t>(r[1]);
|
||||
const auto frame = parse_integer<std::uint64_t>(r[2]);
|
||||
const auto granted = parse_integer<std::int64_t>(r[3]);
|
||||
if (!holder || !epoch || !frame || !granted || *epoch == 0)
|
||||
return std::unexpected{malformed(true)};
|
||||
return LeaseGrant{*holder, *epoch, *frame, milliseconds{*granted}};
|
||||
}
|
||||
|
||||
std::expected<milliseconds, LeaseStoreFailure> renew_region_lease(Postgres& db, RegionKey key,
|
||||
const LeaseGrant& lease,
|
||||
milliseconds duration,
|
||||
milliseconds timeout) {
|
||||
const std::array<std::string, 5> p{std::to_string(key.world), std::to_string(key.region),
|
||||
uuid_text(lease.holder), std::to_string(lease.epoch),
|
||||
std::to_string(duration.count())};
|
||||
auto rows = db.query("SELECT evolution.renew_region_lease($1::evolution.u64, "
|
||||
"$2::evolution.u64, $3::uuid, $4::evolution.u64, $5::integer)",
|
||||
p, timeout);
|
||||
if (!rows)
|
||||
return std::unexpected{classify(rows.error())};
|
||||
const auto granted = rows->size() == 1 && (*rows)[0].size() == 1
|
||||
? parse_integer<std::int64_t>((*rows)[0][0])
|
||||
: std::nullopt;
|
||||
if (!granted)
|
||||
return std::unexpected{malformed(true)};
|
||||
return milliseconds{*granted};
|
||||
}
|
||||
|
||||
std::expected<void, LeaseStoreFailure>
|
||||
release_region_lease(Postgres& db, RegionKey key, const LeaseGrant& lease, milliseconds timeout) {
|
||||
const std::array<std::string, 4> p{std::to_string(key.world), std::to_string(key.region),
|
||||
uuid_text(lease.holder), std::to_string(lease.epoch)};
|
||||
auto rows = db.query("SELECT evolution.release_region_lease($1::evolution.u64, "
|
||||
"$2::evolution.u64, $3::uuid, $4::evolution.u64)",
|
||||
p, timeout);
|
||||
if (!rows)
|
||||
return std::unexpected{classify(rows.error())};
|
||||
if (rows->size() != 1)
|
||||
return std::unexpected{malformed(true)};
|
||||
return {};
|
||||
}
|
||||
|
||||
std::expected<RegionHead, LeaseStoreFailure> load_region_head(Postgres& db, RegionKey key,
|
||||
milliseconds timeout) {
|
||||
const std::array<std::string, 2> p{std::to_string(key.world), std::to_string(key.region)};
|
||||
auto rows =
|
||||
db.query("SELECT * FROM evolution.load_region_head($1::evolution.u64, $2::evolution.u64)",
|
||||
p, timeout);
|
||||
if (!rows)
|
||||
return std::unexpected{classify(rows.error())};
|
||||
auto head = parse_head(*rows);
|
||||
if (!head)
|
||||
return std::unexpected{malformed(false)};
|
||||
return *head;
|
||||
}
|
||||
|
||||
std::expected<RegionHead, LeaseStoreFailure>
|
||||
advance_region_head(Postgres& db, RegionKey key, const LeaseGrant& lease,
|
||||
const RegionHead& expected, const HeadAdvance& next, milliseconds timeout) {
|
||||
const std::array<std::string, 10> p{std::to_string(key.world),
|
||||
std::to_string(key.region),
|
||||
uuid_text(lease.holder),
|
||||
std::to_string(lease.epoch),
|
||||
std::to_string(expected.sequence),
|
||||
uuid_text(expected.commit),
|
||||
bytea(expected.digest),
|
||||
uuid_text(next.commit),
|
||||
bytea(next.digest),
|
||||
std::to_string(next.end_tick)};
|
||||
auto rows = db.query("SELECT * FROM evolution.advance_region_head($1::evolution.u64, "
|
||||
"$2::evolution.u64, $3::uuid, $4::evolution.u64, $5::evolution.u64, "
|
||||
"$6::uuid, $7::bytea, $8::uuid, $9::bytea, $10::evolution.u64)",
|
||||
p, timeout);
|
||||
if (!rows)
|
||||
return std::unexpected{classify(rows.error())};
|
||||
auto head = parse_head(*rows);
|
||||
if (!head)
|
||||
return std::unexpected{malformed(true)};
|
||||
return *head;
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
Reference in New Issue
Block a user