Complete M7.3 fenced region ownership

ADR-0060 and schema v2 add database-time region leases and a fenced head
advance. Acquire/renew/release/advance lock the lease row, check holder,
epoch and expiry with clock_timestamp() after the lock wait, and the
advance compares the exact predecessor. Holders are DB-generated tokens;
epochs never wrap or repeat. The runtime group role gets EXECUTE on five
SECURITY DEFINER functions only - no table, provisioning or CREATE rights.
Migration history admits clean->v2, v1->v2 and a v2 no-op, and refuses
privileged runtime roles.

Host side: portable LeaseAuthority fences from send-time deadlines on
CLOCK_BOOTTIME, keeps the earlier deadline on unknown renewal and stays
fenced until a newer epoch. Selected timings: 20s lease, 5s renew, 2s
margin, 2s renew and 5s commit deadlines. Linux evo::postgres wraps the
functions with typed LeaseStoreError results.

Validated: Windows CTest 55/55; WSL Release, clang-tidy and ASan/UBSan
private-PostgreSQL suites (contenders, expiry, paused/queued stale
writers, unknown renewal, exhaustion, bypass denial, upgrade path).
Three SQL mutations were each caught. Docker/Gitea CI, remote TLS and
cluster runs were not executed. No director, host or deployment added.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-28 11:14:32 -04:00
co-authored by Claude Opus 5.5
parent 598b3938e6
commit 2496649a9b
22 changed files with 1501 additions and 29 deletions
+18 -4
View File
@@ -71,25 +71,39 @@ add_executable(evo-sim apps/evo-sim/main.cpp)
target_link_libraries(evo-sim PRIVATE evo::simulation evo::processes)
evo_owned_target(evo-sim)
# Portable host-side lease timing contract; no database or simulation dependency.
add_library(evo_region_lease STATIC src/persistence/region_lease.cpp)
add_library(evo::region_lease ALIAS evo_region_lease)
target_include_directories(evo_region_lease PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
target_compile_features(evo_region_lease PUBLIC cxx_std_23)
evo_owned_target(evo_region_lease)
include(CTest)
if(EVO_BUILD_POSTGRES)
if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux")
message(FATAL_ERROR "The initial PostgreSQL adapter is qualified on Linux only")
endif()
find_package(PostgreSQL 18.6 EXACT REQUIRED)
add_library(evo_postgres STATIC src/persistence/postgres.cpp)
find_package(Threads REQUIRED)
add_library(evo_postgres STATIC src/persistence/postgres.cpp src/persistence/region_lease_store.cpp)
add_library(evo::postgres ALIAS evo_postgres)
target_include_directories(evo_postgres PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
target_link_libraries(evo_postgres PRIVATE PostgreSQL::PostgreSQL)
target_link_libraries(evo_postgres PUBLIC evo::region_lease PRIVATE PostgreSQL::PostgreSQL)
evo_owned_target(evo_postgres)
if(BUILD_TESTING)
add_executable(evo-postgres-tests tests/postgres_tests.cpp)
target_link_libraries(evo-postgres-tests PRIVATE evo::postgres)
# Contention tests use one thread and one connection per contender.
add_executable(evo-postgres-tests tests/postgres_tests.cpp tests/region_lease_store_tests.cpp)
target_link_libraries(evo-postgres-tests PRIVATE evo::postgres Threads::Threads)
evo_owned_target(evo-postgres-tests)
# Intentionally not an ordinary CTest: tools/ci/postgres-check.sh owns its private DB.
endif()
endif()
if(BUILD_TESTING)
add_executable(evo-region-lease-tests tests/region_lease_tests.cpp)
target_link_libraries(evo-region-lease-tests PRIVATE evo::region_lease)
evo_owned_target(evo-region-lease-tests)
add_test(NAME region-lease-contracts COMMAND evo-region-lease-tests)
set_tests_properties(region-lease-contracts PROPERTIES TIMEOUT 60)
add_executable(evo-environmental-qualification tests/environmental_qualification.cpp)
target_link_libraries(evo-environmental-qualification PRIVATE evo::processes)
evo_owned_target(evo-environmental-qualification)
+2 -1
View File
@@ -132,4 +132,5 @@ Do not reuse the simulation image repository for this entrypoint.
[Secret and role setup](../docs/infrastructure/POSTGRESQL.md) describes the separate
migrator/runtime credentials. The example is outside active Argo resources until
an operator supplies the database endpoint, role, Secret and egress rules. No region
server or world-state migration is deployed by M7.2.
server or world-state migration is deployed by M7.2. Schema v2 (M7.3) additionally
requires `EVO_PG_RUNTIME_ROLE`, naming an operator-created NOLOGIN runtime group role.
@@ -3,7 +3,7 @@
apiVersion: batch/v1
kind: Job
metadata:
name: evolution-db-migrate-v1
name: evolution-db-migrate-v2
namespace: evolution-dev
spec:
backoffLimit: 0
@@ -34,6 +34,8 @@ spec:
- {name: PGSERVICE, value: evolution-migrator}
- {name: PGSERVICEFILE, value: /run/postgres/pg_service.conf}
- {name: EVO_PGPASS_SOURCE, value: /run/postgres/pgpass}
# Operator-created NOLOGIN group role; the runtime login is its member.
- {name: EVO_PG_RUNTIME_ROLE, value: evolution_runtime}
volumeMounts:
- {name: connection, mountPath: /run/postgres, readOnly: true}
- {name: scratch, mountPath: /tmp}
+14
View File
@@ -501,3 +501,17 @@ server/libpq pair, optional persistence adapter and transactional schema v1.
Private PostgreSQL integration checks passed in Release, analysis and ASan/UBSan;
container/cluster verification remains separate. Secret provisioning instructions
are in infrastructure/POSTGRESQL.md. Next: M7.3 fenced region ownership.
M7.3 implemented and locally qualified (2026-09-28). ADR-0060 adds schema v2:
database-time leases, DB-generated holders, never-reused epochs and a fenced head
advance that checks lease then predecessor under one row lock. The runtime role
gets EXECUTE only; table writes, provisioning and object creation are denied.
Real-PostgreSQL tests cover the scenarios the roadmap row requires:
- 64 two-contender acquisition rounds and 32 racing advances;
- expired renewal and a paused stale writer;
- a writer queued past expiry, which is judged by post-lock database time;
- an unknown renewal that fences locally.
Timings were selected and measured locally. The director is intentionally deferred
because it has no authority and would be a placeholder. Next: M7.4 checkpoint
codecs/publication. It must also revoke runtime EXECUTE on the bare head kernel.
+14
View File
@@ -755,3 +755,17 @@ headless suite passed 54/54. Docker/remote CI and cluster deployment are not
claimed. See infrastructure/POSTGRESQL.md and engineering/VALIDATION.md.
Next: M7.3 fenced ownership and narrowly granted database functions. The bootstrap
lease table alone does not establish authority or durable region publication.
## M7.3 fenced region ownership (2026-09-28)
ADR-0060 and migration v2 add guarded lease and head functions for a runtime role
that has no table rights. The portable `LeaseAuthority` fences locally from
send-time deadlines on CLOCK_BOOTTIME. Selected timings are 20 s lease, 5 s renew,
2 s margin, 2 s renew deadline and 5 s commit deadline. Local round trips took
about 0.6-0.7 ms p50 and about 1 ms p99. Real PostgreSQL covered contenders,
expiry, paused and queued stale writers, unknown renewal, exhaustion and bypass
denial. Three SQL mutations (`now()` instead of `clock_timestamp()`, a missing
lease check, a disabled busy check) were each detected by a failing test.
Windows ran 55/55; Linux Release, clang-tidy and ASan/UBSan PostgreSQL suites
passed in WSL. No director, host, gateway, payload publication or deployment
exists yet. The v2 head must not carry real world state before M7.4. Next: M7.4.
+1
View File
@@ -2,6 +2,7 @@
## Status
Accepted for M7.2, 2026-09-28. No region authority or world deployment yet.
Extended by ADR-0060 (schema v2 fencing functions and runtime grants).
Supersedes ADR-0013's libpqxx recommendation for this scoped adapter.
## Context
+99
View File
@@ -0,0 +1,99 @@
# ADR-0060: Fenced region ownership
## Status
Accepted for M7.3, 2026-09-28. Implements the lease/fence portion of ADR-0058 on
the ADR-0059 adapter. No region host, director, gateway or deployment exists yet.
## Context
ADR-0058 requires that PostgreSQL arbitrate acquire/renew/commit/release under one
lease-row lock using database time, that commits compare the expected predecessor,
and that workers cannot bypass the guard. Schema v1 has only a lease table and no
runtime grants. REGION_DURABILITY.md deferred selecting lease duration, renewal
cadence, local safety margin and store deadlines to this gate.
## Decision
Schema v2 (`migrations/0002_region_fencing.sql`) adds `region_heads` and owner-owned
functions. Every mutating function is SECURITY DEFINER with `search_path` pinned to
`pg_catalog, pg_temp`. The runtime group role gets schema USAGE and EXECUTE on
acquire, renew, release, load_region_head and advance_region_head only. It has no
table privileges. Provisioning (genesis) and the internal lease check stay owner-only.
- Acquire locks the row, reads `clock_timestamp()` after the lock and succeeds only if
the row is released or expired. It then increments the u64 epoch and stores a
database-generated holder UUID. Epoch overflow fails closed. The holder is a
per-acquisition token, not a Pod identity.
- Renew, release and advance all go through the same row lock and require the exact
holder and epoch, with expiry in database time after the lock wait. Renewal never
revives an expired lease. Release keeps the epoch, so it is never reused.
- The advance function checks the lease, then locks the head. It requires the
expected sequence, commit ID and digest, a 1..64 tick delta and no sequence wrap.
M7.4/M7.5 publication wraps this kernel in the same transaction as the manifest,
decisions and outbox. Runtime EXECUTE on the bare kernel must then be revoked.
- Rejections use custom SQLSTATEs EVL01..EVL05 (busy, fenced, head mismatch, not
provisioned, exhausted) and 22023 for invalid arguments. A rejection aborts any
enclosing transaction. Clients never send timestamps.
- The migration requires an operator-created runtime role that is unprivileged and
does not inherit the owner. History admission is an exact prefix: clean→v1→v2,
v1→v2, or a v2 no-op. Anything else fails closed.
Host side: the portable `LeaseAuthority` sets its local deadline to request-send time
plus the granted duration minus the safety margin. Database expiry is set no earlier
than that send, so the holder stops before a successor can acquire. An ambiguous
renewal keeps the earlier of the old deadline and the one the attempt could have
set. A reply received after the local deadline fences. Fencing is sticky until a
strictly newer epoch is acquired, which is followed by recovery. `LeaseClock` uses
Linux CLOCK_BOOTTIME so that suspend time still counts.
Selected timings (`LeasePolicy` defaults; the database admits 1..60 s leases):
| Setting | Value | Reason |
|---|---|---|
| Lease duration | 20 s | Bounds orphaned-lease unavailability and leaves 40 s of the proposed 60 s recovery target for load and verification |
| Renewal interval | 5 s | Two full attempts (interval + timeout) fit in the 18 s usable window |
| Safety margin | 2 s | ≥10% of duration, covering clock-rate disagreement and check-to-effect delay |
| Renewal deadline | 2 s | About three orders of magnitude above the measured local p99 (about 1 ms); drop the connection and treat the outcome as unknown |
| Commit deadline | 5 s | Below the 10 s server statement timeout; begin only if it ends before local fencing |
`valid(LeasePolicy)` enforces these relations, not the specific numbers. Remote
TLS latency and clock-rate behavior must be measured in the target environment
before World0. These values are not a player SLA.
The world director is deliberately not implemented. Its output would be an
assignment hint with no authority, and the database decides ownership anyway.
Adding one now would be a placeholder service. It belongs with the host (M7.6)
or later operations work.
## Alternatives Considered
Kubernetes Leases or broker locks are not in the commit transaction. Client-supplied
holder IDs allow lost-acquire adoption but need uniqueness guarantees and let any
runtime guess tokens; DB tokens cost at most one lease period after an unknown
acquire. Transaction-start `now()` lets a writer queued on the lock commit after
expiry. Row-level security on direct table writes widens the attack surface compared
with no table grants. Status-row returns instead of SQLSTATEs would not abort a
future enclosing publication transaction.
## Benefits
One lock serializes every ownership and head transition. The runtime login cannot
write tables, create objects or call provisioning. Stale, paused, queued and
partitioned writers are rejected by the database. The local fence has an explicit
bound instead of trusting liveness.
## Costs
Each renewal or commit takes a row lock. After a worker crash or an unknown acquire,
the region stays unavailable for up to one lease period. The runtime role must be
pre-created by an operator. This adds a second migration and a chain of prefix checksums.
## Risks
A frozen VM/hypervisor can stop all guest clocks. Database commits stay fenced, but
provisional output could then be late. Gateways still need bounded-validity
registration (ADR-0058). The bare head kernel accepts any 32-byte digest until M7.4
binds it to verified payloads, so v2 must not host real world state. Custom
SQLSTATEs are project conventions; they are not PostgreSQL-reserved.
## Consequences
The M7.4 migration must add the manifest/payload publication function that calls
the kernel, and revoke runtime EXECUTE on `advance_region_head`. M7.5 adds
commit-ID resolution. Until then, "head equals my commit ID" is only the primitive
reconciliation used in tests. M7.6 hosts must drive `LeaseAuthority`, stop admission,
simulation and output when fenced, and recover after any new acquisition. Timing
changes require updated measurements and this ADR's table.
+1
View File
@@ -70,3 +70,4 @@ design. No unadopted candidate is a link dependency of the proof.
- [ADR-0055: Environmental process contract](0055-environmental-process-contract.md) - accepted M6.5b synthetic conversions, source interval policy and process checkpoint boundary; implementation follows in M6.5c.
- [ADR-0059: PostgreSQL bootstrap](0059-postgresql-bootstrap.md) - accepted for M7.2 optional adapter, pinned server/client, migrations and Secret boundary.
- [ADR-0060: Fenced region ownership](0060-fenced-region-ownership.md) - accepted for M7.3 database-time leases, guarded head advance, runtime grants and selected timings.
+11 -2
View File
@@ -48,8 +48,10 @@ process may retain memory but cannot publish a head or originate current output.
The gateway must have bounded-validity registration, not just an epoch number.
M7 has no gateway implementation; readiness alone does not prove remote fencing.
Lease duration, renewal cadence, local safety margin and store deadlines must be
selected/measured at M7.3 together; no arbitrary timing values are locked here.
M7.3 selected lease duration, renewal cadence, local safety margin and store
deadlines together in ADR-0060 (20 s / 5 s / 2 s / 2 s renew / 5 s commit). The
latency behind them was measured only on a local socket. The database functions
and runtime grants in migration v2 implement the guard described here.
Any uncertain lease stops admission, simulation and final output. A commit that
already linearized can remain durable after its writer expires; query can return
its receipt, but does not grant that writer authority to resume.
@@ -159,3 +161,10 @@ resurrect acknowledged deaths. World0 stays gated.
M7.1 verification is a source/state inventory and contract review. Real database,
object-store, process-kill and network partition tests are still required.
M7.3 executed its rows against real PostgreSQL. After lease expiry, a new owner
acquires a newer epoch. The old writer's renew, release and head advance are then
rejected, and the head is unchanged. A paused stale worker resuming past its lease
is fenced both locally and by the database. A writer queued on the lease lock past
expiry is also rejected. These are in-process pauses, not killed pods or network
partitions. The 7.6 parts of those rows remain open.
+40
View File
@@ -2041,3 +2041,43 @@ claim lineage recovery. No retained-history or World0 requirement was relaxed.
The adapter is separate from the headless kernel. Region fencing, publication,
ambiguous durable commit reconciliation and crash recovery are later M7 gates.
Only clean-to-v1 and exact-v1 restart are supported migration paths at this gate.
## M7.3 fenced region ownership - 2026-09-28
- Windows MSVC Release configure/build/CTest passed 55/55, including the new
portable `region-lease-contracts` test. Repository clang-format 19 check passed.
- WSL Ubuntu (GCC 14.2.0; clang 19.1.1; PostgreSQL/libpq 18.6 source build):
GCC14 Release, clang19 analysis (clang-tidy warnings-as-errors) and clang19
ASan/UBSan builds of `evo-postgres-tests` each passed `tools/ci/postgres-check.sh`
against a fresh private cluster. The first analysis build flagged two
unchecked optional accesses and two unreserved vectors. These were fixed and
all three configurations were rerun after formatting.
- Real-database coverage, as the runtime login rather than SET ROLE:
- Denied (42501): table read/write, provisioning, the internal check function,
and object creation. A hostile search_path is ignored.
- Busy, forged-holder, stale-epoch and released-lease rejections; epoch
non-reuse.
- 1..64 tick delta; predecessor mismatch; exact-retry detection via the head.
- 1 s database-time expiry and successor acquisition; paused stale writer fenced
locally and by the database.
- A writer queued 1.3 s on the lease row lock is rejected after expiry.
- A 100 ms renewal deadline under lock (unknown) fences locally before a
successor acquires.
- 64 two-thread acquisition rounds with exactly one winner and consecutive
epochs; 32 racing head advances with exactly one winner.
- Epoch and sequence exhaustion.
- Migrations tested: clean->v2, repeated v2 no-op, v1->v2 upgrade, and failed
0001 rollback. Superuser and missing runtime roles are refused with no schema
left. Unknown v3, changed v2 checksum and changed v1 checksum are refused.
- Mutation sensitivity (scratch copies only): replacing post-lock
`clock_timestamp()` with `now()`, removing the lease check from head advance,
and disabling the busy check each made the suite fail at the expected test.
- Local Unix-socket latency, 200 samples per operation in each final run:
- p50 about 0.61-0.74 ms and p99 about 0.8-1.15 ms for acquire, renew,
advance and release.
- Max up to 2.8 ms under analysis/ASan builds.
These are local numbers, not remote TLS or cluster measurements.
- Not executed: Docker/Gitea CI, remote TLS endpoints, Kubernetes Secrets, real pod
kill, real network partition (a lock wait plus client deadline stands in for
it), VM freeze and TSan. The contention threads each own a separate
connection; libpq is uninstrumented.
+42 -13
View File
@@ -1,4 +1,4 @@
# PostgreSQL M7.2 bootstrap
# PostgreSQL bootstrap (M7.2) and region fencing (M7.3)
ADR-0059 pins server/libpq 18.6. `evo::postgres` is optional, Linux-only and separate
from all simulation targets. Preparation explicitly downloads a verified source
@@ -21,7 +21,9 @@ bash tools/ci/postgres-check.sh "$PWD/build/postgres/evo-postgres-tests"
```
The test script overrides all connection coordinates and creates a new private
0700 /tmp directory, private Unix socket and disposable local cluster. TCP is off;
0700 /tmp directory, private Unix socket and disposable local cluster. It creates
the `evo_runtime_test` group and the `evo_runtime_login` member, then runs fencing
tests as that real runtime login. TCP is off;
local trust applies only inside that private directory. Cluster stops on exit;
logs/data are retained there for diagnosis. It never uses production PG credentials
or deletes a pre-existing database. Do not point this script at a shared server.
@@ -49,18 +51,45 @@ incomplete session. No PostgreSQL diagnostic strings or connection values are lo
## Migrations and privilege boundary
`tools/migrate-postgres.sh` uses psql environment/service files, no URI arguments.
It requires exactly server18.6, serializes migrations with a transaction advisory
lock, hashes the checked-in SQL, and admits clean->v1 or exact v1->v1 only.
Unknown versions/checksum changes/partial foreign schemas fail closed. No down
migration or destructive repair exists. The u64 domain rejects fractional,
negative and above-u64 values without signed truncation.
It requires exactly server 18.6 and serializes migrations with a transaction
advisory lock. It hashes each checked-in SQL file and applies everything in one
transaction. The recorded history must be an exact prefix: clean->v1->v2, v1->v2,
or a v2 no-op. Unknown versions, checksum changes and partial foreign schemas fail
closed. There is no down migration or destructive repair. The u64 domain rejects
fractional, negative and above-u64 values without signed truncation.
Use a dedicated owner/migrator login with CREATE on the database. Runtime login
must be separate, not a member of the owner and without database CREATE or public
schema CREATE. Bootstrap revokes PUBLIC access to its schema/tables and default
function execution. Runtime has no direct mutation rights; M7.3 adds fenced
functions and narrowly scoped EXECUTE. Operators own database/role provisioning;
the migration never creates login passwords or grants superuser.
Use a dedicated owner/migrator login with CREATE on the database. Before
migrating, operators create a NOLOGIN runtime group role and pass its name as
`EVO_PG_RUNTIME_ROLE`. The migration refuses a role that is missing, is the owner,
inherits the owner, or has superuser/CREATEROLE/CREATEDB/replication/BYPASSRLS. The
runtime login is a separate member of that group only, with no database CREATE or
public schema CREATE. Bootstrap revokes PUBLIC access to the schema, tables and
default function execution. Operators own database/role provisioning; the migration
never creates logins or passwords, and never grants superuser.
## Fenced region ownership (schema v2, ADR-0060)
The runtime role has schema USAGE and EXECUTE on five SECURITY DEFINER functions:
`acquire_region_lease`, `renew_region_lease`, `release_region_lease`,
`load_region_head` and `advance_region_head`. It cannot read or write tables, create
objects, call `check_region_lease` or provision regions. Each mutation locks the lease row
and checks holder, epoch and expiry with `clock_timestamp()` after the lock wait.
Clients never send times. Rejections: EVL01 busy, EVL02 fenced, EVL03 head mismatch,
EVL04 not provisioned, EVL05 epoch/sequence exhausted, 22023 invalid arguments.
`evo::postgres` maps them to `LeaseStoreError`; a timeout or broken connection is
`unknown`.
Genesis is operator-only (owner login), never done by recovery or runtime:
`SELECT evolution.provision_region(world, region, frame, genesis_commit_uuid,
'\x<64 hex digest>');`. In schema v2 the head is an identity/digest pointer with no
payload binding. Do not provision real world state before M7.4 publication.
Hosts use `LeaseAuthority` with `LeasePolicy` defaults: 20 s lease, 5 s renewal
interval, 2 s safety margin, 2 s renewal deadline and 5 s commit deadline. The
local deadline runs from request send time on CLOCK_BOOTTIME. If a renewal outcome
is unknown, keep the earlier deadline and reconnect. Once fenced, stop admission,
simulation and output, then acquire a new epoch and recover. Never resume the old
epoch.
## Kubernetes Secrets
+99
View File
@@ -0,0 +1,99 @@
#pragma once
#include <array>
#include <chrono>
#include <cstdint>
#include <optional>
// Portable host-side region authority contract (ADR-0060). No database types:
// PostgreSQL arbitrates ownership; this only decides when a holder must stop.
namespace evo::persistence {
using Uuid = std::array<std::uint8_t, 16>;
using Sha256Digest = std::array<std::uint8_t, 32>;
struct RegionKey {
std::uint64_t world{};
std::uint64_t region{};
};
struct RegionHead {
std::uint64_t frame{};
std::uint64_t sequence{};
Uuid commit{};
Sha256Digest digest{};
std::uint64_t end_tick{};
friend bool operator==(const RegionHead&, const RegionHead&) = default;
};
// Holder is a database-generated per-acquisition token, not a Pod identity.
struct LeaseGrant {
Uuid holder{};
std::uint64_t epoch{};
std::uint64_t frame{};
std::chrono::milliseconds duration{};
};
// Selected M7.3 timings. Validity: duration 1..60s (database bound); margin at
// least 10% of duration (clock-rate and check-to-effect allowance); two complete
// renewal attempts fit before local fencing; I/O deadlines within adapter limits.
struct LeasePolicy {
std::chrono::milliseconds duration{20000};
std::chrono::milliseconds renew_interval{5000};
std::chrono::milliseconds safety_margin{2000};
std::chrono::milliseconds renew_timeout{2000};
std::chrono::milliseconds commit_timeout{5000};
};
[[nodiscard]] bool valid(const LeasePolicy& policy) noexcept;
// Monotonic clock that keeps advancing while the process or machine is suspended
// (Linux CLOCK_BOOTTIME). A paused worker must observe that time passed.
struct LeaseClock {
using duration = std::chrono::nanoseconds;
using rep = duration::rep;
using period = duration::period;
using time_point = std::chrono::time_point<LeaseClock>;
static constexpr bool is_steady = true;
[[nodiscard]] static time_point now() noexcept;
};
enum class LeaseState : std::uint8_t { absent, active, renew_due, fenced };
// Single-owner local view of one region lease. Every time point is a LeaseClock
// reading supplied by the host. The local deadline is request-send time plus the
// granted duration minus the safety margin: the database set its expiry no earlier
// than that send, so the holder stops before any successor can acquire. Fencing is
// sticky until a strictly newer epoch is acquired; renewal never revives it.
class LeaseAuthority {
public:
explicit LeaseAuthority(const LeasePolicy& policy) noexcept : policy_(policy) {}
[[nodiscard]] const LeasePolicy& policy() const noexcept {
return policy_;
}
[[nodiscard]] const std::optional<LeaseGrant>& grant() const noexcept {
return grant_;
}
// Returns false (and fences) for an invalid policy, reused/older epoch or late reply.
[[nodiscard]] bool acquired(const LeaseGrant& grant, LeaseClock::time_point sent,
LeaseClock::time_point received) noexcept;
[[nodiscard]] bool renewed(std::uint64_t epoch, std::chrono::milliseconds granted,
LeaseClock::time_point sent,
LeaseClock::time_point received) noexcept;
// Ambiguous renewal: the database may have applied a shorter expiry, so keep
// the earlier of the old deadline and the one the attempt could have set.
void renewal_unknown(std::chrono::milliseconds requested, LeaseClock::time_point sent) noexcept;
void fence() noexcept;
[[nodiscard]] LeaseState state(LeaseClock::time_point now) const noexcept;
// Admission, simulation and final output require an unfenced lease.
[[nodiscard]] bool may_originate(LeaseClock::time_point now) const noexcept;
// Liveness guard: begin a commit only if its deadline ends before local fencing.
[[nodiscard]] bool may_begin_commit(LeaseClock::time_point now) const noexcept;
[[nodiscard]] LeaseClock::time_point deadline() const noexcept {
return deadline_;
}
private:
LeasePolicy policy_;
std::optional<LeaseGrant> grant_;
std::uint64_t highest_epoch_{};
LeaseClock::time_point last_sent_{};
LeaseClock::time_point deadline_{};
bool fenced_{};
};
} // namespace evo::persistence
@@ -0,0 +1,50 @@
#pragma once
#include "evo/persistence/postgres.hpp"
#include "evo/persistence/region_lease.hpp"
#include <chrono>
#include <cstdint>
#include <expected>
// Linux-only calls to the fenced database functions of migration v2 (ADR-0060).
// Each call is one statement on a caller-owned connection; outside an explicit
// transaction it commits by itself. No retries. Rejections are ordinary results.
namespace evo::persistence {
enum class LeaseStoreError : std::uint8_t {
busy, // Another holder's lease is unexpired in database time.
fenced, // Holder/epoch superseded, released or expired: stop originating.
head_mismatch, // Expected predecessor is not the committed head.
not_provisioned, // No genesis; runtime can never create one.
exhausted, // Epoch or commit sequence would wrap: fail closed.
invalid, // Rejected arguments; nothing changed.
store, // Other SQL/configuration failure; nothing committed.
unknown, // May have executed; reconnect and reconcile before reuse.
};
struct LeaseStoreFailure {
LeaseStoreError kind{};
PgFailure cause{};
};
struct HeadAdvance {
Uuid commit{};
Sha256Digest digest{};
std::uint64_t end_tick{};
};
[[nodiscard]] std::expected<LeaseGrant, LeaseStoreFailure>
acquire_region_lease(Postgres& db, RegionKey key, std::chrono::milliseconds duration,
std::chrono::milliseconds timeout);
// Returns the granted duration; the caller measures it from its request send time.
[[nodiscard]] std::expected<std::chrono::milliseconds, LeaseStoreFailure>
renew_region_lease(Postgres& db, RegionKey key, const LeaseGrant& lease,
std::chrono::milliseconds duration, std::chrono::milliseconds timeout);
[[nodiscard]] std::expected<void, LeaseStoreFailure>
release_region_lease(Postgres& db, RegionKey key, const LeaseGrant& lease,
std::chrono::milliseconds timeout);
[[nodiscard]] std::expected<RegionHead, LeaseStoreFailure>
load_region_head(Postgres& db, RegionKey key, std::chrono::milliseconds timeout);
// Locks the lease then the head; requires unexpired holder/epoch and the exact
// expected predecessor. head_mismatch after unknown: load the head and compare.
[[nodiscard]] std::expected<RegionHead, LeaseStoreFailure>
advance_region_head(Postgres& db, RegionKey key, const LeaseGrant& lease,
const RegionHead& expected, const HeadAdvance& next,
std::chrono::milliseconds timeout);
} // namespace evo::persistence
+214
View File
@@ -0,0 +1,214 @@
-- M7.3 fenced region ownership (ADR-0060). Owner-only definitions; the runtime
-- role receives schema USAGE and EXECUTE on the guarded functions, never tables.
-- Every mutation locks the region lease row first and decides expiry with the
-- database clock read after that lock. Clients never supply timestamps.
SELECT pg_catalog.set_config('evolution.runtime_role', :'runtime_role', true);
DO $$
DECLARE
runtime name := pg_catalog.current_setting('evolution.runtime_role');
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles r WHERE r.rolname = runtime
AND NOT r.rolsuper AND NOT r.rolcreaterole AND NOT r.rolcreatedb
AND NOT r.rolreplication AND NOT r.rolbypassrls)
OR runtime = current_user
OR pg_catalog.pg_has_role(runtime, current_user, 'MEMBER') THEN
RAISE EXCEPTION 'runtime role must exist, be unprivileged and not inherit the owner';
END IF;
END $$;
CREATE TABLE evolution.region_heads (
world_id evolution.u64 NOT NULL,
region_id evolution.u64 NOT NULL,
commit_sequence evolution.u64 NOT NULL,
commit_id uuid NOT NULL,
digest bytea NOT NULL CHECK (octet_length(digest) = 32),
end_tick evolution.u64 NOT NULL,
published_epoch evolution.u64 NOT NULL,
PRIMARY KEY (world_id, region_id),
FOREIGN KEY (world_id, region_id) REFERENCES evolution.region_leases,
-- Genesis is sequence 0, tick 0, provisioned outside any lease epoch.
CHECK ((commit_sequence = 0) = (end_tick = 0)),
CHECK ((commit_sequence = 0) = (published_epoch = 0))
);
REVOKE ALL ON evolution.region_heads FROM PUBLIC;
-- Owner-only provisioning; recovery and runtime can never create genesis.
CREATE FUNCTION evolution.provision_region(p_world evolution.u64, p_region evolution.u64,
p_frame evolution.u64, p_genesis_commit uuid, p_genesis_digest bytea)
RETURNS void LANGUAGE sql SECURITY INVOKER SET search_path = pg_catalog, pg_temp AS $$
INSERT INTO evolution.region_leases(world_id, region_id, frame_id)
VALUES (p_world, p_region, p_frame);
INSERT INTO evolution.region_heads(world_id, region_id, commit_sequence, commit_id,
digest, end_tick, published_epoch)
VALUES (p_world, p_region, 0, p_genesis_commit, p_genesis_digest, 0, 0);
$$;
CREATE FUNCTION evolution.acquire_region_lease(p_world evolution.u64,
p_region evolution.u64, p_duration_ms integer)
RETURNS TABLE (lease_holder uuid, lease_epoch evolution.u64, region_frame evolution.u64,
granted_ms integer)
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$
DECLARE
l evolution.region_leases%ROWTYPE;
t timestamptz;
BEGIN
IF p_world IS NULL OR p_region IS NULL OR p_duration_ms IS NULL
OR p_duration_ms NOT BETWEEN 1000 AND 60000 THEN
RAISE EXCEPTION 'invalid lease request' USING ERRCODE = '22023';
END IF;
SELECT * INTO l FROM evolution.region_leases r
WHERE r.world_id = p_world AND r.region_id = p_region FOR UPDATE;
IF NOT FOUND THEN
RAISE EXCEPTION 'region not provisioned' USING ERRCODE = 'EVL04';
END IF;
t := clock_timestamp();
IF l.holder IS NOT NULL AND l.expires_at > t THEN
RAISE EXCEPTION 'region lease held' USING ERRCODE = 'EVL01';
END IF;
IF l.epoch = 18446744073709551615 THEN
RAISE EXCEPTION 'lease epochs exhausted' USING ERRCODE = 'EVL05';
END IF;
UPDATE evolution.region_leases r
SET epoch = l.epoch + 1, holder = gen_random_uuid(),
expires_at = t + p_duration_ms * interval '1 millisecond'
WHERE r.world_id = p_world AND r.region_id = p_region
RETURNING r.holder, r.epoch, r.frame_id INTO lease_holder, lease_epoch, region_frame;
granted_ms := p_duration_ms;
RETURN NEXT;
END $$;
-- Locks the lease row and requires the exact unexpired holder/epoch. Internal:
-- callable only by the owner and the SECURITY DEFINER functions below.
CREATE FUNCTION evolution.check_region_lease(p_world evolution.u64, p_region evolution.u64,
p_holder uuid, p_epoch evolution.u64)
RETURNS timestamptz LANGUAGE plpgsql SECURITY INVOKER
SET search_path = pg_catalog, pg_temp AS $$
DECLARE
l evolution.region_leases%ROWTYPE;
t timestamptz;
BEGIN
IF p_world IS NULL OR p_region IS NULL OR p_holder IS NULL OR p_epoch IS NULL THEN
RAISE EXCEPTION 'invalid lease request' USING ERRCODE = '22023';
END IF;
SELECT * INTO l FROM evolution.region_leases r
WHERE r.world_id = p_world AND r.region_id = p_region FOR UPDATE;
IF NOT FOUND THEN
RAISE EXCEPTION 'region not provisioned' USING ERRCODE = 'EVL04';
END IF;
-- Read after the lock wait: a queued stale writer sees current database time.
t := clock_timestamp();
IF l.holder IS DISTINCT FROM p_holder OR l.epoch <> p_epoch OR l.expires_at <= t THEN
RAISE EXCEPTION 'region lease fenced' USING ERRCODE = 'EVL02';
END IF;
RETURN t;
END $$;
CREATE FUNCTION evolution.renew_region_lease(p_world evolution.u64, p_region evolution.u64,
p_holder uuid, p_epoch evolution.u64, p_duration_ms integer)
RETURNS integer LANGUAGE plpgsql SECURITY DEFINER
SET search_path = pg_catalog, pg_temp AS $$
DECLARE
t timestamptz;
BEGIN
IF p_duration_ms IS NULL OR p_duration_ms NOT BETWEEN 1000 AND 60000 THEN
RAISE EXCEPTION 'invalid lease request' USING ERRCODE = '22023';
END IF;
-- Renewal never revives an expired or superseded lease.
t := evolution.check_region_lease(p_world, p_region, p_holder, p_epoch);
UPDATE evolution.region_leases r
SET expires_at = t + p_duration_ms * interval '1 millisecond'
WHERE r.world_id = p_world AND r.region_id = p_region;
RETURN p_duration_ms;
END $$;
CREATE FUNCTION evolution.release_region_lease(p_world evolution.u64, p_region evolution.u64,
p_holder uuid, p_epoch evolution.u64)
RETURNS void LANGUAGE plpgsql SECURITY DEFINER
SET search_path = pg_catalog, pg_temp AS $$
BEGIN
PERFORM evolution.check_region_lease(p_world, p_region, p_holder, p_epoch);
-- The epoch is retained so the next acquisition can never reuse it.
UPDATE evolution.region_leases r SET holder = NULL, expires_at = NULL
WHERE r.world_id = p_world AND r.region_id = p_region;
END $$;
CREATE FUNCTION evolution.load_region_head(p_world evolution.u64, p_region evolution.u64)
RETURNS TABLE (region_frame evolution.u64, head_sequence evolution.u64, head_commit uuid,
head_digest text, head_tick evolution.u64)
LANGUAGE plpgsql STABLE SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$
BEGIN
RETURN QUERY SELECT l.frame_id, h.commit_sequence, h.commit_id,
encode(h.digest, 'hex'), h.end_tick
FROM evolution.region_heads h
JOIN evolution.region_leases l USING (world_id, region_id)
WHERE h.world_id = p_world AND h.region_id = p_region;
IF NOT FOUND THEN
RAISE EXCEPTION 'region not provisioned' USING ERRCODE = 'EVL04';
END IF;
END $$;
-- The fenced transition kernel. M7.4/M7.5 publication wraps it with manifest,
-- decision and outbox rows in the same transaction; the lease lock is taken first.
CREATE FUNCTION evolution.advance_region_head(p_world evolution.u64, p_region evolution.u64,
p_holder uuid, p_epoch evolution.u64, p_expected_sequence evolution.u64,
p_expected_commit uuid, p_expected_digest bytea, p_commit uuid, p_digest bytea,
p_end_tick evolution.u64)
RETURNS TABLE (region_frame evolution.u64, head_sequence evolution.u64, head_commit uuid,
head_digest text, head_tick evolution.u64)
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$
DECLARE
h evolution.region_heads%ROWTYPE;
BEGIN
IF p_expected_sequence IS NULL OR p_expected_commit IS NULL OR p_commit IS NULL
OR p_end_tick IS NULL OR p_commit = p_expected_commit
OR octet_length(p_expected_digest) IS DISTINCT FROM 32
OR octet_length(p_digest) IS DISTINCT FROM 32 THEN
RAISE EXCEPTION 'invalid head advance' USING ERRCODE = '22023';
END IF;
PERFORM evolution.check_region_lease(p_world, p_region, p_holder, p_epoch);
SELECT * INTO h FROM evolution.region_heads r
WHERE r.world_id = p_world AND r.region_id = p_region FOR UPDATE;
IF NOT FOUND THEN
RAISE EXCEPTION 'region not provisioned' USING ERRCODE = 'EVL04';
END IF;
IF h.commit_sequence <> p_expected_sequence OR h.commit_id <> p_expected_commit
OR h.digest <> p_expected_digest THEN
RAISE EXCEPTION 'region head moved' USING ERRCODE = 'EVL03';
END IF;
IF h.commit_sequence = 18446744073709551615 THEN
RAISE EXCEPTION 'commit sequences exhausted' USING ERRCODE = 'EVL05';
END IF;
IF p_end_tick <= h.end_tick OR p_end_tick - h.end_tick > 64 THEN
RAISE EXCEPTION 'invalid head advance' USING ERRCODE = '22023';
END IF;
UPDATE evolution.region_heads r
SET commit_sequence = h.commit_sequence + 1, commit_id = p_commit, digest = p_digest,
end_tick = p_end_tick, published_epoch = p_epoch
WHERE r.world_id = p_world AND r.region_id = p_region;
RETURN QUERY SELECT l.frame_id, n.commit_sequence, n.commit_id, encode(n.digest, 'hex'),
n.end_tick
FROM evolution.region_heads n
JOIN evolution.region_leases l USING (world_id, region_id)
WHERE n.world_id = p_world AND n.region_id = p_region;
END $$;
REVOKE ALL ON FUNCTION
evolution.provision_region(evolution.u64, evolution.u64, evolution.u64, uuid, bytea),
evolution.acquire_region_lease(evolution.u64, evolution.u64, integer),
evolution.check_region_lease(evolution.u64, evolution.u64, uuid, evolution.u64),
evolution.renew_region_lease(evolution.u64, evolution.u64, uuid, evolution.u64, integer),
evolution.release_region_lease(evolution.u64, evolution.u64, uuid, evolution.u64),
evolution.load_region_head(evolution.u64, evolution.u64),
evolution.advance_region_head(evolution.u64, evolution.u64, uuid, evolution.u64,
evolution.u64, uuid, bytea, uuid, bytea, evolution.u64)
FROM PUBLIC;
GRANT USAGE ON SCHEMA evolution TO :"runtime_role";
GRANT EXECUTE ON FUNCTION
evolution.acquire_region_lease(evolution.u64, evolution.u64, integer),
evolution.renew_region_lease(evolution.u64, evolution.u64, uuid, evolution.u64, integer),
evolution.release_region_lease(evolution.u64, evolution.u64, uuid, evolution.u64),
evolution.load_region_head(evolution.u64, evolution.u64),
evolution.advance_region_head(evolution.u64, evolution.u64, uuid, evolution.u64,
evolution.u64, uuid, bytea, uuid, bytea, evolution.u64)
TO :"runtime_role";
INSERT INTO evolution.schema_migrations(version, sha256) VALUES (2, :'migration_sha_2');
+10 -4
View File
@@ -12,15 +12,21 @@ SELECT current_setting('server_version_num')::int = 180006 AS supported \gset
\endif
SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') AS exists \gset
\if :exists
-- Unknown/partial schema fails closed; no IF NOT EXISTS repair of foreign data.
SELECT count(*) = 1 AND min(version) = 1 AND min(sha256) = :'migration_sha'
AS compatible FROM evolution.schema_migrations \gset
\if :compatible
-- Only an exact prefix of the checked-in history is admitted. Unknown or
-- partial/foreign schemas fail closed; no IF NOT EXISTS repair of foreign data.
SELECT coalesce(string_agg(version || ':' || sha256, ',' ORDER BY version), '')
AS history FROM evolution.schema_migrations \gset
SELECT :'history' = '1:' || :'migration_sha' AS at_v1,
:'history' = '1:' || :'migration_sha' || ',2:' || :'migration_sha_2' AS at_v2 \gset
\if :at_v2
\elif :at_v1
\ir 0002_region_fencing.sql
\else
\echo 'Unknown or changed migration history'
DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$;
\endif
\else
\ir 0001_region_bootstrap.sql
\ir 0002_region_fencing.sql
\endif
COMMIT;
+96
View File
@@ -0,0 +1,96 @@
#include "evo/persistence/region_lease.hpp"
#include <algorithm>
#if defined(__linux__)
#include <time.h>
#endif
namespace evo::persistence {
namespace {
using std::chrono::milliseconds;
constexpr milliseconds minimum_duration{1000};
constexpr milliseconds maximum_duration{60000};
constexpr milliseconds maximum_io{30000}; // Postgres adapter deadline bound.
} // namespace
bool valid(const LeasePolicy& p) noexcept {
const milliseconds zero{};
if (p.duration < minimum_duration || p.duration > maximum_duration ||
p.renew_interval <= zero || p.safety_margin <= zero || p.renew_timeout <= zero ||
p.commit_timeout <= zero || p.renew_timeout > maximum_io || p.commit_timeout > maximum_io ||
p.safety_margin >= p.duration)
return false;
const auto usable = p.duration - p.safety_margin;
return p.safety_margin * 10 >= p.duration &&
(p.renew_interval + p.renew_timeout) * 2 <= usable && p.commit_timeout < usable;
}
LeaseClock::time_point LeaseClock::now() noexcept {
#if defined(__linux__)
timespec ts{};
// CLOCK_BOOTTIME cannot fail with a valid clock id and pointer.
static_cast<void>(clock_gettime(CLOCK_BOOTTIME, &ts));
return time_point{std::chrono::seconds{ts.tv_sec} + std::chrono::nanoseconds{ts.tv_nsec}};
#else
// Storage hosts are Linux-only; elsewhere suspend behavior is unqualified.
return time_point{
std::chrono::duration_cast<duration>(std::chrono::steady_clock::now().time_since_epoch())};
#endif
}
bool LeaseAuthority::acquired(const LeaseGrant& grant, LeaseClock::time_point sent,
LeaseClock::time_point received) noexcept {
const auto deadline = sent + grant.duration - policy_.safety_margin;
if (!valid(policy_) || grant.epoch == 0 || grant.epoch <= highest_epoch_ || grant.frame == 0 ||
grant.duration != policy_.duration || received < sent || received >= deadline) {
fence();
return false;
}
grant_ = grant;
highest_epoch_ = grant.epoch;
last_sent_ = sent;
deadline_ = deadline;
fenced_ = false;
return true;
}
bool LeaseAuthority::renewed(std::uint64_t epoch, milliseconds granted, LeaseClock::time_point sent,
LeaseClock::time_point received) noexcept {
if (!grant_ || fenced_ || epoch != grant_->epoch || granted != policy_.duration ||
sent < last_sent_ || received < sent || received >= deadline_) {
fence();
return false;
}
// Database expiry is its execution time plus granted, which is after sent.
deadline_ = sent + granted - policy_.safety_margin;
last_sent_ = sent;
return true;
}
void LeaseAuthority::renewal_unknown(milliseconds requested, LeaseClock::time_point sent) noexcept {
if (!grant_ || fenced_)
return;
deadline_ = std::min(deadline_, sent + requested - policy_.safety_margin);
last_sent_ = std::max(last_sent_, sent);
}
void LeaseAuthority::fence() noexcept {
fenced_ = true;
}
LeaseState LeaseAuthority::state(LeaseClock::time_point now) const noexcept {
if (!grant_)
return LeaseState::absent;
if (fenced_ || now >= deadline_)
return LeaseState::fenced;
return now >= last_sent_ + policy_.renew_interval ? LeaseState::renew_due : LeaseState::active;
}
bool LeaseAuthority::may_originate(LeaseClock::time_point now) const noexcept {
const auto s = state(now);
return s == LeaseState::active || s == LeaseState::renew_due;
}
bool LeaseAuthority::may_begin_commit(LeaseClock::time_point now) const noexcept {
return may_originate(now) && now + policy_.commit_timeout <= deadline_;
}
} // namespace evo::persistence
+204
View File
@@ -0,0 +1,204 @@
#include "evo/persistence/region_lease_store.hpp"
#include <array>
#include <charconv>
#include <optional>
#include <string>
#include <string_view>
namespace evo::persistence {
namespace {
using std::chrono::milliseconds;
constexpr std::string_view digits = "0123456789abcdef";
LeaseStoreFailure classify(const PgFailure& f) {
const std::string_view state(f.sqlstate.data(), 5);
auto kind = LeaseStoreError::store;
if (f.kind == PgError::unknown)
kind = LeaseStoreError::unknown;
else if (f.kind == PgError::config || f.kind == PgError::bounds || state == "22023")
kind = LeaseStoreError::invalid;
else if (state == "EVL01")
kind = LeaseStoreError::busy;
else if (state == "EVL02")
kind = LeaseStoreError::fenced;
else if (state == "EVL03")
kind = LeaseStoreError::head_mismatch;
else if (state == "EVL04")
kind = LeaseStoreError::not_provisioned;
else if (state == "EVL05")
kind = LeaseStoreError::exhausted;
return {kind, f};
}
LeaseStoreFailure malformed(bool mutation) {
// A mutation whose reply cannot be parsed may still have committed.
return {mutation ? LeaseStoreError::unknown : LeaseStoreError::store, {}};
}
template <std::size_t N> std::string hex(const std::array<std::uint8_t, N>& bytes) {
std::string out;
out.reserve(N * 2);
for (const auto b : bytes) {
out.push_back(digits[static_cast<std::size_t>(b >> 4U)]);
out.push_back(digits[static_cast<std::size_t>(b & 0x0FU)]);
}
return out;
}
std::optional<std::uint8_t> nibble(char c) {
const auto at = digits.find(c);
if (at == std::string_view::npos)
return std::nullopt;
return static_cast<std::uint8_t>(at);
}
template <std::size_t N> std::optional<std::array<std::uint8_t, N>> unhex(std::string_view s) {
if (s.size() != N * 2)
return std::nullopt;
std::array<std::uint8_t, N> out{};
for (std::size_t i = 0; i < N; ++i) {
const auto high = nibble(s[i * 2]);
const auto low = nibble(s[(i * 2) + 1]);
if (!high || !low)
return std::nullopt;
out[i] = static_cast<std::uint8_t>((*high << 4U) | *low);
}
return out;
}
std::string uuid_text(const Uuid& id) {
const auto h = hex(id);
return h.substr(0, 8) + '-' + h.substr(8, 4) + '-' + h.substr(12, 4) + '-' + h.substr(16, 4) +
'-' + h.substr(20);
}
std::optional<Uuid> parse_uuid(const std::optional<std::string>& s) {
if (!s || s->size() != 36)
return std::nullopt;
std::string compact;
for (std::size_t i = 0; i < s->size(); ++i) {
const bool dash = i == 8 || i == 13 || i == 18 || i == 23;
if (dash != ((*s)[i] == '-'))
return std::nullopt;
if (!dash)
compact.push_back((*s)[i]);
}
return unhex<16>(compact);
}
template <typename T> std::optional<T> parse_integer(const std::optional<std::string>& s) {
if (!s || s->empty() || s->size() > 20 || s->front() == '-' || s->front() == '+')
return std::nullopt;
T value{};
const auto* end = s->data() + s->size();
const auto [at, error] = std::from_chars(s->data(), end, value);
if (error != std::errc{} || at != end)
return std::nullopt;
return value;
}
std::string bytea(const Sha256Digest& d) {
return "\\x" + hex(d);
}
std::optional<RegionHead> parse_head(const PgRows& rows) {
if (rows.size() != 1 || rows[0].size() != 5)
return std::nullopt;
const auto& r = rows[0];
const auto frame = parse_integer<std::uint64_t>(r[0]);
const auto sequence = parse_integer<std::uint64_t>(r[1]);
const auto commit = parse_uuid(r[2]);
const auto& digest_text = r[3];
const auto digest = digest_text ? unhex<32>(*digest_text) : std::nullopt;
const auto tick = parse_integer<std::uint64_t>(r[4]);
if (!frame || !sequence || !commit || !digest || !tick)
return std::nullopt;
return RegionHead{*frame, *sequence, *commit, *digest, *tick};
}
} // namespace
std::expected<LeaseGrant, LeaseStoreFailure>
acquire_region_lease(Postgres& db, RegionKey key, milliseconds duration, milliseconds timeout) {
const std::array<std::string, 3> p{std::to_string(key.world), std::to_string(key.region),
std::to_string(duration.count())};
auto rows = db.query("SELECT * FROM evolution.acquire_region_lease($1::evolution.u64, "
"$2::evolution.u64, $3::integer)",
p, timeout);
if (!rows)
return std::unexpected{classify(rows.error())};
if (rows->size() != 1 || (*rows)[0].size() != 4)
return std::unexpected{malformed(true)};
const auto& r = (*rows)[0];
const auto holder = parse_uuid(r[0]);
const auto epoch = parse_integer<std::uint64_t>(r[1]);
const auto frame = parse_integer<std::uint64_t>(r[2]);
const auto granted = parse_integer<std::int64_t>(r[3]);
if (!holder || !epoch || !frame || !granted || *epoch == 0)
return std::unexpected{malformed(true)};
return LeaseGrant{*holder, *epoch, *frame, milliseconds{*granted}};
}
std::expected<milliseconds, LeaseStoreFailure> renew_region_lease(Postgres& db, RegionKey key,
const LeaseGrant& lease,
milliseconds duration,
milliseconds timeout) {
const std::array<std::string, 5> p{std::to_string(key.world), std::to_string(key.region),
uuid_text(lease.holder), std::to_string(lease.epoch),
std::to_string(duration.count())};
auto rows = db.query("SELECT evolution.renew_region_lease($1::evolution.u64, "
"$2::evolution.u64, $3::uuid, $4::evolution.u64, $5::integer)",
p, timeout);
if (!rows)
return std::unexpected{classify(rows.error())};
const auto granted = rows->size() == 1 && (*rows)[0].size() == 1
? parse_integer<std::int64_t>((*rows)[0][0])
: std::nullopt;
if (!granted)
return std::unexpected{malformed(true)};
return milliseconds{*granted};
}
std::expected<void, LeaseStoreFailure>
release_region_lease(Postgres& db, RegionKey key, const LeaseGrant& lease, milliseconds timeout) {
const std::array<std::string, 4> p{std::to_string(key.world), std::to_string(key.region),
uuid_text(lease.holder), std::to_string(lease.epoch)};
auto rows = db.query("SELECT evolution.release_region_lease($1::evolution.u64, "
"$2::evolution.u64, $3::uuid, $4::evolution.u64)",
p, timeout);
if (!rows)
return std::unexpected{classify(rows.error())};
if (rows->size() != 1)
return std::unexpected{malformed(true)};
return {};
}
std::expected<RegionHead, LeaseStoreFailure> load_region_head(Postgres& db, RegionKey key,
milliseconds timeout) {
const std::array<std::string, 2> p{std::to_string(key.world), std::to_string(key.region)};
auto rows =
db.query("SELECT * FROM evolution.load_region_head($1::evolution.u64, $2::evolution.u64)",
p, timeout);
if (!rows)
return std::unexpected{classify(rows.error())};
auto head = parse_head(*rows);
if (!head)
return std::unexpected{malformed(false)};
return *head;
}
std::expected<RegionHead, LeaseStoreFailure>
advance_region_head(Postgres& db, RegionKey key, const LeaseGrant& lease,
const RegionHead& expected, const HeadAdvance& next, milliseconds timeout) {
const std::array<std::string, 10> p{std::to_string(key.world),
std::to_string(key.region),
uuid_text(lease.holder),
std::to_string(lease.epoch),
std::to_string(expected.sequence),
uuid_text(expected.commit),
bytea(expected.digest),
uuid_text(next.commit),
bytea(next.digest),
std::to_string(next.end_tick)};
auto rows = db.query("SELECT * FROM evolution.advance_region_head($1::evolution.u64, "
"$2::evolution.u64, $3::uuid, $4::evolution.u64, $5::evolution.u64, "
"$6::uuid, $7::bytea, $8::uuid, $9::bytea, $10::evolution.u64)",
p, timeout);
if (!rows)
return std::unexpected{classify(rows.error())};
auto head = parse_head(*rows);
if (!head)
return std::unexpected{malformed(true)};
return *head;
}
} // namespace evo::persistence
+3
View File
@@ -3,6 +3,7 @@
#include <cstdlib>
#include <source_location>
#include <stdexcept>
void run_region_lease_store_tests();
namespace {
using namespace evo::persistence;
using namespace std::chrono_literals;
@@ -69,6 +70,8 @@ int main() {
try {
run();
std::puts("postgres adapter integration passed");
run_region_lease_store_tests();
std::puts("region lease fencing integration passed");
return 0;
} catch (const std::exception& e) {
std::fprintf(stderr, "%s\n", e.what());
+413
View File
@@ -0,0 +1,413 @@
// Real PostgreSQL M7.3 fencing checks (ADR-0060). Run by tools/ci/postgres-check.sh.
#include "evo/persistence/region_lease_store.hpp"
#include <algorithm>
#include <array>
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <latch>
#include <memory>
#include <source_location>
#include <stdexcept>
#include <string>
#include <thread>
#include <vector>
void run_region_lease_store_tests();
namespace {
using namespace evo::persistence;
using namespace std::chrono_literals;
using std::chrono::milliseconds;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region lease store test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
std::string env(const char* key) {
const auto* value = std::getenv(key);
if (!value)
throw std::runtime_error("missing test environment");
return value;
}
std::unique_ptr<Postgres> connect_as(const std::string& user) {
PgOptions options{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}};
auto made = Postgres::connect(options, 5s);
CHECK(made);
return std::move(*made);
}
std::unique_ptr<Postgres> owner() {
return connect_as(env("PGUSER"));
}
std::unique_ptr<Postgres> runtime() {
return connect_as(env("EVO_PG_RUNTIME_USER"));
}
Uuid id(std::uint8_t seed) {
Uuid out{};
out.fill(seed);
return out;
}
Sha256Digest digest(std::uint8_t seed) {
Sha256Digest out{};
out.fill(seed);
return out;
}
std::string hex_text(std::uint8_t seed, std::size_t bytes) {
constexpr char digits[] = "0123456789abcdef";
std::string out;
for (std::size_t i = 0; i < bytes; ++i) {
out.push_back(digits[seed >> 4U]);
out.push_back(digits[seed & 0x0FU]);
}
return out;
}
std::string uuid_literal(std::uint8_t seed) {
const auto h = hex_text(seed, 16);
return h.substr(0, 8) + '-' + h.substr(8, 4) + '-' + h.substr(12, 4) + '-' + h.substr(16, 4) +
'-' + h.substr(20);
}
// Genesis commit/digest bytes are the region number; frame is 7.
RegionKey provision(Postgres& db, std::uint64_t region) {
const auto seed = static_cast<std::uint8_t>(region);
const std::array<std::string, 4> p{std::to_string(region), uuid_literal(seed),
"\\x" + hex_text(seed, 32), "7"};
CHECK(db.query("SELECT evolution.provision_region(1, $1::evolution.u64, $4::evolution.u64, "
"$2::uuid, $3::bytea)",
p, 5s));
return {1, region};
}
bool failed(const auto& result, LeaseStoreError kind) {
return !result && result.error().kind == kind;
}
std::string sqlstate(const std::expected<PgRows, PgFailure>& result) {
return result ? std::string{} : std::string(result.error().sqlstate.data());
}
void runtime_cannot_bypass_the_guard() {
auto admin = owner();
const auto key = provision(*admin, 10);
auto db = runtime();
for (const auto* sql : {
"UPDATE evolution.region_leases SET holder = NULL",
"UPDATE evolution.region_heads SET commit_sequence = 5",
"DELETE FROM evolution.region_heads",
"INSERT INTO evolution.region_leases(world_id,region_id,frame_id) VALUES (9,9,9)",
"SELECT * FROM evolution.region_leases",
"SELECT * FROM evolution.region_heads",
"SELECT evolution.provision_region(1, 11, 7, gen_random_uuid(), '\\x00')",
"SELECT evolution.check_region_lease(1, 10, gen_random_uuid(), 1)",
"CREATE FUNCTION evolution.bypass() RETURNS int LANGUAGE sql AS 'SELECT 1'",
"CREATE TABLE evolution.bypass(x int)",
})
CHECK(sqlstate(db->query(sql, {}, 5s)) == "42501");
const std::array<std::string, 1> owner_name{env("PGUSER")};
CHECK(!db->query("SELECT pg_catalog.set_config('role', $1, false)", owner_name, 5s));
// A SECURITY DEFINER function ignores a hostile caller search_path.
CHECK(db->query("SET search_path = pg_temp, public", {}, 5s));
CHECK(db->query("CREATE TEMP TABLE region_leases(x int)", {}, 5s));
auto lease = acquire_region_lease(*db, key, 20000ms, 5s);
CHECK(lease && lease->epoch == 1 && lease->frame == 7);
CHECK(release_region_lease(*db, key, *lease, 5s));
}
void acquire_renew_release_and_advance() {
auto admin = owner();
const auto key = provision(*admin, 20);
auto db = runtime();
CHECK(
failed(acquire_region_lease(*db, {1, 999}, 20000ms, 5s), LeaseStoreError::not_provisioned));
CHECK(failed(acquire_region_lease(*db, key, 999ms, 5s), LeaseStoreError::invalid));
CHECK(failed(acquire_region_lease(*db, key, 60001ms, 5s), LeaseStoreError::invalid));
auto lease = acquire_region_lease(*db, key, 20000ms, 5s);
CHECK(lease && lease->epoch == 1 && lease->duration == 20000ms && lease->holder != Uuid{});
CHECK(failed(acquire_region_lease(*db, key, 20000ms, 5s), LeaseStoreError::busy));
auto renewed = renew_region_lease(*db, key, *lease, 20000ms, 5s);
CHECK(renewed && *renewed == 20000ms);
auto forged = *lease;
forged.holder[0] ^= 1U;
CHECK(failed(renew_region_lease(*db, key, forged, 20000ms, 5s), LeaseStoreError::fenced));
auto stale_epoch = *lease;
stale_epoch.epoch = 2;
CHECK(failed(renew_region_lease(*db, key, stale_epoch, 20000ms, 5s), LeaseStoreError::fenced));
auto genesis = load_region_head(*db, key, 5s);
CHECK(genesis && genesis->sequence == 0 && genesis->end_tick == 0 && genesis->frame == 7);
CHECK(genesis->commit == id(20) && genesis->digest == digest(20));
CHECK(failed(advance_region_head(*db, key, *lease, *genesis, {id(21), digest(21), 65}, 5s),
LeaseStoreError::invalid));
CHECK(failed(advance_region_head(*db, key, *lease, *genesis, {id(20), digest(21), 1}, 5s),
LeaseStoreError::invalid));
const HeadAdvance first{id(21), digest(21), 64};
auto head = advance_region_head(*db, key, *lease, *genesis, first, 5s);
CHECK(head && head->sequence == 1 && head->end_tick == 64 && head->commit == id(21));
CHECK(failed(advance_region_head(*db, key, forged, *head, {id(22), digest(22), 65}, 5s),
LeaseStoreError::fenced));
// Exact retry after an applied advance: mismatch, then the head proves it committed.
CHECK(failed(advance_region_head(*db, key, *lease, *genesis, first, 5s),
LeaseStoreError::head_mismatch));
auto current = load_region_head(*db, key, 5s);
CHECK(current && *current == *head);
auto changed = *head;
changed.digest[31] ^= 1U;
CHECK(failed(advance_region_head(*db, key, *lease, changed, {id(22), digest(22), 65}, 5s),
LeaseStoreError::head_mismatch));
CHECK(failed(advance_region_head(*db, key, *lease, *head, {id(22), digest(22), 64}, 5s),
LeaseStoreError::invalid));
CHECK(release_region_lease(*db, key, *lease, 5s));
CHECK(failed(release_region_lease(*db, key, *lease, 5s), LeaseStoreError::fenced));
CHECK(failed(renew_region_lease(*db, key, *lease, 20000ms, 5s), LeaseStoreError::fenced));
CHECK(failed(advance_region_head(*db, key, *lease, *head, {id(22), digest(22), 65}, 5s),
LeaseStoreError::fenced));
auto next = acquire_region_lease(*db, key, 20000ms, 5s);
CHECK(next && next->epoch == 2 && next->holder != lease->holder);
auto final_head = advance_region_head(*db, key, *next, *head, {id(22), digest(22), 65}, 5s);
CHECK(final_head && final_head->sequence == 2);
}
void database_time_expiry_and_paused_writer() {
auto admin = owner();
const auto key = provision(*admin, 30);
auto worker_a = runtime();
auto worker_b = runtime();
const LeasePolicy fast{1000ms, 200ms, 100ms, 100ms, 300ms};
LeaseAuthority a{fast};
auto sent = LeaseClock::now();
auto lease_a = acquire_region_lease(*worker_a, key, fast.duration, 5s);
CHECK(lease_a && a.acquired(*lease_a, sent, LeaseClock::now()));
CHECK(failed(acquire_region_lease(*worker_b, key, 20000ms, 5s), LeaseStoreError::busy));
auto genesis = load_region_head(*worker_a, key, 5s);
CHECK(genesis);
std::this_thread::sleep_for(1200ms); // Worker A is paused past its lease.
CHECK(a.state(LeaseClock::now()) == LeaseState::fenced);
// Expiry is decided by database time; the successor acquires a newer epoch.
auto lease_b = acquire_region_lease(*worker_b, key, 20000ms, 5s);
CHECK(lease_b && lease_b->epoch == lease_a->epoch + 1);
auto head_b =
advance_region_head(*worker_b, key, *lease_b, *genesis, {id(31), digest(31), 3}, 5s);
CHECK(head_b);
// A resumes and ignores its local fence: every database path rejects it.
CHECK(failed(renew_region_lease(*worker_a, key, *lease_a, fast.duration, 5s),
LeaseStoreError::fenced));
CHECK(
failed(advance_region_head(*worker_a, key, *lease_a, *genesis, {id(32), digest(32), 1}, 5s),
LeaseStoreError::fenced));
CHECK(
failed(advance_region_head(*worker_a, key, *lease_a, *head_b, {id(32), digest(32), 4}, 5s),
LeaseStoreError::fenced));
CHECK(failed(release_region_lease(*worker_a, key, *lease_a, 5s), LeaseStoreError::fenced));
auto head = load_region_head(*worker_a, key, 5s);
CHECK(head && *head == *head_b);
// Expired renewal cannot revive a lease even without a successor.
CHECK(release_region_lease(*worker_b, key, *lease_b, 5s));
auto lease_c = acquire_region_lease(*worker_a, key, 1000ms, 5s);
CHECK(lease_c && lease_c->epoch == lease_b->epoch + 1);
std::this_thread::sleep_for(1200ms);
CHECK(
failed(renew_region_lease(*worker_a, key, *lease_c, 1000ms, 5s), LeaseStoreError::fenced));
}
// A writer queued on the lease row lock is judged by the clock after the wait,
// not its statement start time, so a lease that expired while queued is fenced.
void queued_writer_uses_post_lock_database_time() {
auto admin = owner();
const auto key = provision(*admin, 40);
auto worker = runtime();
auto lease = acquire_region_lease(*worker, key, 1000ms, 5s);
auto genesis = load_region_head(*worker, key, 5s);
CHECK(lease && genesis);
auto blocker = owner();
const std::array<std::string, 1> region{"40"};
CHECK(blocker->query("BEGIN", {}, 5s));
CHECK(blocker->query("SELECT 1 FROM evolution.region_leases WHERE world_id = 1 AND "
"region_id = $1::evolution.u64 FOR UPDATE",
region, 5s));
std::expected<RegionHead, LeaseStoreFailure> queued = std::unexpected{LeaseStoreFailure{}};
std::thread writer([&] {
queued = advance_region_head(*worker, key, *lease, *genesis, {id(41), digest(41), 1}, 5s);
});
std::this_thread::sleep_for(1300ms);
CHECK(blocker->query("ROLLBACK", {}, 5s));
writer.join();
CHECK(failed(queued, LeaseStoreError::fenced));
auto head = load_region_head(*worker, key, 5s);
CHECK(head && *head == *genesis);
}
// Partition: the renewal's outcome is unknown and the connection is gone. The
// holder keeps its earlier deadline and fences itself before a successor starts.
void partitioned_renewal_fences_locally() {
auto admin = owner();
const auto key = provision(*admin, 50);
auto worker = runtime();
const LeasePolicy fast{1000ms, 200ms, 100ms, 100ms, 300ms};
LeaseAuthority authority{fast};
const auto sent = LeaseClock::now();
auto lease = acquire_region_lease(*worker, key, fast.duration, 5s);
CHECK(lease && authority.acquired(*lease, sent, LeaseClock::now()));
auto blocker = owner();
const std::array<std::string, 1> region{"50"};
CHECK(blocker->query("BEGIN", {}, 5s));
CHECK(blocker->query("SELECT 1 FROM evolution.region_leases WHERE world_id = 1 AND "
"region_id = $1::evolution.u64 FOR UPDATE",
region, 5s));
const auto renew_sent = LeaseClock::now();
auto renewal = renew_region_lease(*worker, key, *lease, fast.duration, fast.renew_timeout);
CHECK(failed(renewal, LeaseStoreError::unknown));
authority.renewal_unknown(fast.duration, renew_sent);
const auto local_deadline = authority.deadline();
CHECK(local_deadline <= sent + fast.duration - fast.safety_margin);
std::this_thread::sleep_for(1300ms);
CHECK(!authority.may_originate(LeaseClock::now()));
CHECK(blocker->query("ROLLBACK", {}, 5s));
// The orphaned renewal reached the row only after database expiry: rejected.
auto successor = runtime();
auto next = acquire_region_lease(*successor, key, 20000ms, 5s);
CHECK(next && next->epoch == lease->epoch + 1);
}
void contenders_serialize_on_the_database() {
auto admin = owner();
const auto key = provision(*admin, 60);
std::array<std::unique_ptr<Postgres>, 2> workers{runtime(), runtime()};
std::uint64_t previous_epoch = 0;
constexpr int rounds = 64;
for (int round = 0; round < rounds; ++round) {
std::array<std::expected<LeaseGrant, LeaseStoreFailure>, 2> results{
std::unexpected{LeaseStoreFailure{}}, std::unexpected{LeaseStoreFailure{}}};
std::latch start{2};
std::vector<std::thread> threads;
threads.reserve(2);
for (std::size_t i = 0; i < 2; ++i)
threads.emplace_back([&, i] {
start.arrive_and_wait();
results[i] = acquire_region_lease(*workers[i], key, 20000ms, 5s);
});
for (auto& t : threads)
t.join();
const auto winners = std::count_if(results.begin(), results.end(),
[](const auto& r) { return r.has_value(); });
CHECK(winners == 1);
const std::size_t w = results[0] ? 0 : 1;
CHECK(failed(results[1 - w], LeaseStoreError::busy));
CHECK(results[w]->epoch == previous_epoch + 1);
previous_epoch = results[w]->epoch;
CHECK(release_region_lease(*workers[w], key, *results[w], 5s));
}
// The same holder racing two commits from the same predecessor: exactly one wins.
auto lease = acquire_region_lease(*workers[0], key, 20000ms, 5s);
CHECK(lease);
auto head = load_region_head(*workers[0], key, 5s);
CHECK(head);
for (std::uint8_t round = 0; round < 32; ++round) {
std::array<std::expected<RegionHead, LeaseStoreFailure>, 2> results{
std::unexpected{LeaseStoreFailure{}}, std::unexpected{LeaseStoreFailure{}}};
std::latch start{2};
std::vector<std::thread> threads;
threads.reserve(2);
for (std::size_t i = 0; i < 2; ++i)
threads.emplace_back([&, i] {
const auto seed = static_cast<std::uint8_t>(100 + (round * 2) + i);
start.arrive_and_wait();
results[i] = advance_region_head(*workers[i], key, *lease, *head,
{id(seed), digest(seed), head->end_tick + 1}, 5s);
});
for (auto& t : threads)
t.join();
const std::size_t w = results[0] ? 0 : 1;
CHECK(results[w] && failed(results[1 - w], LeaseStoreError::head_mismatch));
CHECK(results[w]->sequence == head->sequence + 1);
head = *results[w];
}
auto committed = load_region_head(*workers[1], key, 5s);
CHECK(committed && *committed == *head && committed->sequence == 32);
}
void exhaustion_fails_closed() {
auto admin = owner();
const auto key = provision(*admin, 70);
CHECK(admin->query("UPDATE evolution.region_leases SET epoch = 18446744073709551615 "
"WHERE world_id = 1 AND region_id = 70",
{}, 5s));
auto db = runtime();
CHECK(failed(acquire_region_lease(*db, key, 20000ms, 5s), LeaseStoreError::exhausted));
const auto other = provision(*admin, 71);
auto lease = acquire_region_lease(*db, other, 20000ms, 5s);
CHECK(lease);
CHECK(admin->query("UPDATE evolution.region_heads SET commit_sequence = 18446744073709551615, "
"end_tick = 5, published_epoch = 1 WHERE world_id = 1 AND region_id = 71",
{}, 5s));
auto head = load_region_head(*db, other, 5s);
CHECK(head && head->sequence == 18446744073709551615ULL);
CHECK(failed(advance_region_head(*db, other, *lease, *head, {id(72), digest(72), 6}, 5s),
LeaseStoreError::exhausted));
}
struct Summary {
std::vector<long long> samples;
void print(const char* name) {
std::sort(samples.begin(), samples.end());
const auto pick = [&](std::size_t percent) {
return samples[std::min(samples.size() - 1, samples.size() * percent / 100)];
};
std::printf("lease_latency_us %s n=%zu p50=%lld p99=%lld max=%lld\n", name, samples.size(),
pick(50), pick(99), samples.back());
}
};
template <typename F> long long timed(F&& f) {
const auto begin = std::chrono::steady_clock::now();
f();
return std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() -
begin)
.count();
}
// Local Unix-socket round trips only; remote TLS latency must be measured in place.
void measure_round_trips() {
auto admin = owner();
const auto key = provision(*admin, 80);
auto db = runtime();
constexpr int n = 200;
Summary acquire, renew, advance, release;
auto head = load_region_head(*db, key, 5s);
CHECK(head);
for (int i = 0; i < n; ++i) {
std::expected<LeaseGrant, LeaseStoreFailure> lease = std::unexpected{LeaseStoreFailure{}};
acquire.samples.push_back(
timed([&] { lease = acquire_region_lease(*db, key, 20000ms, 5s); }));
CHECK(lease);
renew.samples.push_back(
timed([&] { CHECK(renew_region_lease(*db, key, *lease, 20000ms, 5s)); }));
const auto seed = static_cast<std::uint8_t>(i);
Uuid commit = id(seed);
commit[0] = 0xAA; // Distinct from genesis 0x50 bytes.
commit[1] = static_cast<std::uint8_t>(i >> 8);
advance.samples.push_back(timed([&] {
auto next = advance_region_head(*db, key, *lease, *head,
{commit, digest(seed), head->end_tick + 64}, 5s);
CHECK(next);
head = *next;
}));
release.samples.push_back(
timed([&] { CHECK(release_region_lease(*db, key, *lease, 5s)); }));
}
acquire.print("acquire");
renew.print("renew");
advance.print("advance_head");
release.print("release");
}
} // namespace
void run_region_lease_store_tests() {
runtime_cannot_bypass_the_guard();
acquire_renew_release_and_advance();
database_time_expiry_and_paused_writer();
queued_writer_uses_post_lock_database_time();
partitioned_renewal_fences_locally();
contenders_serialize_on_the_database();
exhaustion_fails_closed();
measure_round_trips();
}
+133
View File
@@ -0,0 +1,133 @@
#include "evo/persistence/region_lease.hpp"
#include <cstdio>
#include <source_location>
#include <stdexcept>
#include <string>
namespace {
using namespace evo::persistence;
using namespace std::chrono_literals;
using Time = LeaseClock::time_point;
void check(bool ok, std::source_location at = std::source_location::current()) {
if (!ok)
throw std::runtime_error("region lease line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
constexpr Time at(std::chrono::milliseconds t) {
return Time{t};
}
LeaseGrant grant(std::uint64_t epoch) {
return {{1}, epoch, 7, LeasePolicy{}.duration};
}
void policies() {
CHECK(valid(LeasePolicy{}));
CHECK(valid(LeasePolicy{1000ms, 200ms, 100ms, 100ms, 300ms}));
const auto invalid = [](auto change) {
LeasePolicy p;
change(p);
return !valid(p);
};
CHECK(invalid([](LeasePolicy& p) { p.duration = 999ms; }));
CHECK(invalid([](LeasePolicy& p) { p.duration = 60001ms; }));
CHECK(invalid([](LeasePolicy& p) { p.safety_margin = 1999ms; })); // under 10%
CHECK(invalid([](LeasePolicy& p) { p.safety_margin = 20000ms; }));
CHECK(invalid([](LeasePolicy& p) { p.renew_interval = 0ms; }));
CHECK(invalid([](LeasePolicy& p) { p.renew_timeout = -1ms; }));
CHECK(invalid([](LeasePolicy& p) { p.renew_timeout = 30001ms; }));
CHECK(invalid([](LeasePolicy& p) { p.commit_timeout = 18000ms; }));
// Two complete renewal attempts must fit before local fencing: 2*(7+2.5) > 18.
CHECK(invalid([](LeasePolicy& p) {
p.renew_interval = 7000ms;
p.renew_timeout = 2500ms;
}));
CHECK(!valid(LeasePolicy{2000ms, 400ms, 200ms, 200ms, 1800ms}));
LeaseAuthority broken(LeasePolicy{999ms, 200ms, 100ms, 100ms, 300ms});
CHECK(!broken.acquired({{1}, 1, 7, 999ms}, at(0ms), at(1ms)));
CHECK(broken.state(at(1ms)) == LeaseState::absent);
}
void deadlines_follow_send_time() {
LeaseAuthority a{LeasePolicy{}};
CHECK(a.state(at(0ms)) == LeaseState::absent && !a.may_originate(at(0ms)));
// Reply arrival does not move the deadline: the DB clock started after sending.
CHECK(a.acquired(grant(1), at(1000ms), at(4000ms)));
CHECK(a.deadline() == at(19000ms));
CHECK(a.state(at(5999ms)) == LeaseState::active);
CHECK(a.state(at(6000ms)) == LeaseState::renew_due && a.may_originate(at(6000ms)));
CHECK(a.may_begin_commit(at(14000ms)) && !a.may_begin_commit(at(14001ms)));
CHECK(a.may_originate(at(18999ms)));
CHECK(a.state(at(19000ms)) == LeaseState::fenced && !a.may_originate(at(19000ms)));
// A renewal answered after the local deadline cannot revive authority.
CHECK(!a.renewed(1, 20000ms, at(18000ms), at(19000ms)));
CHECK(a.state(at(18500ms)) == LeaseState::fenced);
}
void renewals() {
LeaseAuthority a{LeasePolicy{}};
CHECK(a.acquired(grant(4), at(0ms), at(10ms)));
CHECK(a.renewed(4, 20000ms, at(5000ms), at(5100ms)));
CHECK(a.deadline() == at(23000ms) && a.state(at(9999ms)) == LeaseState::active);
CHECK(a.state(at(10000ms)) == LeaseState::renew_due);
// Ambiguous renewal keeps the earlier bound; retries still fit before fencing.
a.renewal_unknown(20000ms, at(10000ms));
CHECK(a.deadline() == at(23000ms) && a.state(at(14999ms)) == LeaseState::active);
a.renewal_unknown(5000ms, at(15000ms)); // Could have shortened: take the earlier.
CHECK(a.deadline() == at(18000ms));
CHECK(a.renewed(4, 20000ms, at(16000ms), at(16500ms)) && a.deadline() == at(34000ms));
CHECK(!a.renewed(4, 20000ms, at(15000ms), at(16600ms))); // Out of order: fence.
CHECK(a.state(at(16600ms)) == LeaseState::fenced);
CHECK(!a.renewed(4, 20000ms, at(17000ms), at(17001ms))); // Sticky.
LeaseAuthority b{LeasePolicy{}};
CHECK(b.acquired(grant(2), at(0ms), at(1ms)));
CHECK(!b.renewed(3, 20000ms, at(1000ms), at(1001ms))); // Foreign epoch.
CHECK(b.state(at(1001ms)) == LeaseState::fenced);
LeaseAuthority c{LeasePolicy{}};
CHECK(c.acquired(grant(2), at(0ms), at(1ms)));
CHECK(!c.renewed(2, 19000ms, at(1000ms), at(1001ms))); // Unexpected duration.
LeaseAuthority d{LeasePolicy{}};
CHECK(d.acquired(grant(2), at(0ms), at(1ms)));
d.fence();
CHECK(d.state(at(2ms)) == LeaseState::fenced && !d.may_begin_commit(at(2ms)));
}
void epochs_are_never_reused() {
LeaseAuthority a{LeasePolicy{}};
CHECK(!a.acquired(grant(0), at(0ms), at(1ms)));
CHECK(!a.acquired({{1}, 1, 0, 20000ms}, at(0ms), at(1ms))); // Zero frame.
CHECK(!a.acquired({{1}, 1, 7, 10000ms}, at(0ms), at(1ms))); // Not the policy.
CHECK(!a.acquired(grant(1), at(10ms), at(9ms))); // Reply before send.
CHECK(!a.acquired(grant(1), at(0ms), at(18000ms))); // Too late to use.
CHECK(a.acquired(grant(3), at(0ms), at(1ms)));
CHECK(!a.acquired(grant(3), at(2ms), at(3ms)));
CHECK(a.state(at(3ms)) == LeaseState::fenced);
CHECK(!a.acquired(grant(2), at(4ms), at(5ms)));
// Only a strictly newer database epoch clears fencing (followed by recovery).
CHECK(a.acquired(grant(4), at(6ms), at(7ms)) && a.state(at(7ms)) == LeaseState::active);
const auto& current = a.grant();
CHECK(current && current->epoch == 4);
}
void clock_is_monotonic() {
const auto first = LeaseClock::now();
const auto second = LeaseClock::now();
CHECK(second >= first && first.time_since_epoch().count() > 0);
}
} // namespace
int main() {
try {
policies();
deadlines_follow_send_time();
renewals();
epochs_are_never_reused();
clock_is_monotonic();
std::puts("region lease contracts passed");
return 0;
} catch (const std::exception& e) {
std::fprintf(stderr, "%s\n", e.what());
return 1;
}
}
+29 -4
View File
@@ -11,10 +11,15 @@ unset PGHOSTADDR PGPASSWORD PGSERVICE PGSERVICEFILE
initdb -D "$test_dir/data" --no-locale --encoding=UTF8 --auth-local=trust --auth-host=reject > "$test_dir/init.log"
trap 'pg_ctl -D "$test_dir/data" -m immediate stop > /dev/null || true' EXIT
pg_ctl -D "$test_dir/data" -l "$test_dir/server.log" -o "-k $test_dir -p 55432 -c listen_addresses=''" -w start
psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_runtime_test NOLOGIN'
# Runtime group role receives grants; a separate login inherits only that role.
psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_runtime_test NOLOGIN' \
-c 'CREATE ROLE evo_runtime_login LOGIN IN ROLE evo_runtime_test'
export EVO_PG_RUNTIME_ROLE=evo_runtime_test EVO_PG_RUNTIME_USER=evo_runtime_login
bash "$root/tools/migrate-postgres.sh"
bash "$root/tools/migrate-postgres.sh" # exact supported upgrade/restart is a no-op
"${1:?path to evo-postgres-tests}"
v1_sha="$(sha256sum "$root/migrations/0001_region_bootstrap.sql")"
v1_sha="${v1_sha%% *}"
# Migration failure must not publish a partial schema.
createdb migration_rollback
if (export PGDATABASE=migration_rollback; psql -X -v ON_ERROR_STOP=1 -1 \
@@ -22,9 +27,29 @@ if (export PGDATABASE=migration_rollback; psql -X -v ON_ERROR_STOP=1 -1 \
echo 'Expected failed migration' >&2; exit 1
fi
test "$(PGDATABASE=migration_rollback psql -XAtc "SELECT count(*) FROM pg_namespace WHERE nspname='evolution'")" = 0
# A privileged or owner runtime role is refused and leaves no schema behind.
if (export PGDATABASE=migration_rollback EVO_PG_RUNTIME_ROLE="$PGUSER"; \
bash "$root/tools/migrate-postgres.sh"); then
echo 'Expected privileged runtime role rejection' >&2; exit 1
fi
if (export PGDATABASE=migration_rollback EVO_PG_RUNTIME_ROLE=no_such_evo_role; \
bash "$root/tools/migrate-postgres.sh"); then
echo 'Expected missing runtime role rejection' >&2; exit 1
fi
test "$(PGDATABASE=migration_rollback psql -XAtc "SELECT count(*) FROM pg_namespace WHERE nspname='evolution'")" = 0
# Supported upgrade path: an existing exact v1 database migrates to v2.
createdb upgrade_path
PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" \
-f "$root/migrations/0001_region_bootstrap.sql" > /dev/null
test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1
PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh"
test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2
# Unknown future versions and checksum drift must not be repaired silently.
psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=2'
psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=3 WHERE version=2'
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=1,sha256=repeat('0',64)"
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=2 WHERE version=3"
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=2"
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
echo 'postgres migrations, rollback, constraints and permissions passed'
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64)"
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
echo 'postgres migrations, rollback, constraints, permissions and fencing passed'
+5
View File
@@ -1,6 +1,8 @@
#!/usr/bin/env bash
set -euo pipefail
root="$(cd "$(dirname "$0")/.." && pwd)"
# Operator-created NOLOGIN group role granted to the region runtime login (ADR-0060).
runtime_role="${EVO_PG_RUNTIME_ROLE:?set EVO_PG_RUNTIME_ROLE to the runtime group role}"
if [ -n "${EVO_PGPASS_SOURCE:-}" ]; then
umask 077
export PGPASSFILE
@@ -12,5 +14,8 @@ fi
# Never print the connection or enable shell tracing here.
digest="$(sha256sum "$root/migrations/0001_region_bootstrap.sql")"
digest="${digest%% *}"
digest_2="$(sha256sum "$root/migrations/0002_region_fencing.sql")"
digest_2="${digest_2%% *}"
psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \
--set="migration_sha_2=$digest_2" --set="runtime_role=$runtime_role" \
--file="$root/migrations/apply.sql"