Complete M7.3 fenced region ownership
ADR-0060 and schema v2 add database-time region leases and a fenced head advance. Acquire/renew/release/advance lock the lease row, check holder, epoch and expiry with clock_timestamp() after the lock wait, and the advance compares the exact predecessor. Holders are DB-generated tokens; epochs never wrap or repeat. The runtime group role gets EXECUTE on five SECURITY DEFINER functions only - no table, provisioning or CREATE rights. Migration history admits clean->v2, v1->v2 and a v2 no-op, and refuses privileged runtime roles. Host side: portable LeaseAuthority fences from send-time deadlines on CLOCK_BOOTTIME, keeps the earlier deadline on unknown renewal and stays fenced until a newer epoch. Selected timings: 20s lease, 5s renew, 2s margin, 2s renew and 5s commit deadlines. Linux evo::postgres wraps the functions with typed LeaseStoreError results. Validated: Windows CTest 55/55; WSL Release, clang-tidy and ASan/UBSan private-PostgreSQL suites (contenders, expiry, paused/queued stale writers, unknown renewal, exhaustion, bypass denial, upgrade path). Three SQL mutations were each caught. Docker/Gitea CI, remote TLS and cluster runs were not executed. No director, host or deployment added. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+18
-4
@@ -71,25 +71,39 @@ add_executable(evo-sim apps/evo-sim/main.cpp)
|
||||
target_link_libraries(evo-sim PRIVATE evo::simulation evo::processes)
|
||||
evo_owned_target(evo-sim)
|
||||
|
||||
# Portable host-side lease timing contract; no database or simulation dependency.
|
||||
add_library(evo_region_lease STATIC src/persistence/region_lease.cpp)
|
||||
add_library(evo::region_lease ALIAS evo_region_lease)
|
||||
target_include_directories(evo_region_lease PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
|
||||
target_compile_features(evo_region_lease PUBLIC cxx_std_23)
|
||||
evo_owned_target(evo_region_lease)
|
||||
|
||||
include(CTest)
|
||||
if(EVO_BUILD_POSTGRES)
|
||||
if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux")
|
||||
message(FATAL_ERROR "The initial PostgreSQL adapter is qualified on Linux only")
|
||||
endif()
|
||||
find_package(PostgreSQL 18.6 EXACT REQUIRED)
|
||||
add_library(evo_postgres STATIC src/persistence/postgres.cpp)
|
||||
find_package(Threads REQUIRED)
|
||||
add_library(evo_postgres STATIC src/persistence/postgres.cpp src/persistence/region_lease_store.cpp)
|
||||
add_library(evo::postgres ALIAS evo_postgres)
|
||||
target_include_directories(evo_postgres PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
|
||||
target_link_libraries(evo_postgres PRIVATE PostgreSQL::PostgreSQL)
|
||||
target_link_libraries(evo_postgres PUBLIC evo::region_lease PRIVATE PostgreSQL::PostgreSQL)
|
||||
evo_owned_target(evo_postgres)
|
||||
if(BUILD_TESTING)
|
||||
add_executable(evo-postgres-tests tests/postgres_tests.cpp)
|
||||
target_link_libraries(evo-postgres-tests PRIVATE evo::postgres)
|
||||
# Contention tests use one thread and one connection per contender.
|
||||
add_executable(evo-postgres-tests tests/postgres_tests.cpp tests/region_lease_store_tests.cpp)
|
||||
target_link_libraries(evo-postgres-tests PRIVATE evo::postgres Threads::Threads)
|
||||
evo_owned_target(evo-postgres-tests)
|
||||
# Intentionally not an ordinary CTest: tools/ci/postgres-check.sh owns its private DB.
|
||||
endif()
|
||||
endif()
|
||||
if(BUILD_TESTING)
|
||||
add_executable(evo-region-lease-tests tests/region_lease_tests.cpp)
|
||||
target_link_libraries(evo-region-lease-tests PRIVATE evo::region_lease)
|
||||
evo_owned_target(evo-region-lease-tests)
|
||||
add_test(NAME region-lease-contracts COMMAND evo-region-lease-tests)
|
||||
set_tests_properties(region-lease-contracts PROPERTIES TIMEOUT 60)
|
||||
add_executable(evo-environmental-qualification tests/environmental_qualification.cpp)
|
||||
target_link_libraries(evo-environmental-qualification PRIVATE evo::processes)
|
||||
evo_owned_target(evo-environmental-qualification)
|
||||
|
||||
+2
-1
@@ -132,4 +132,5 @@ Do not reuse the simulation image repository for this entrypoint.
|
||||
[Secret and role setup](../docs/infrastructure/POSTGRESQL.md) describes the separate
|
||||
migrator/runtime credentials. The example is outside active Argo resources until
|
||||
an operator supplies the database endpoint, role, Secret and egress rules. No region
|
||||
server or world-state migration is deployed by M7.2.
|
||||
server or world-state migration is deployed by M7.2. Schema v2 (M7.3) additionally
|
||||
requires `EVO_PG_RUNTIME_ROLE`, naming an operator-created NOLOGIN runtime group role.
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: evolution-db-migrate-v1
|
||||
name: evolution-db-migrate-v2
|
||||
namespace: evolution-dev
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
@@ -34,6 +34,8 @@ spec:
|
||||
- {name: PGSERVICE, value: evolution-migrator}
|
||||
- {name: PGSERVICEFILE, value: /run/postgres/pg_service.conf}
|
||||
- {name: EVO_PGPASS_SOURCE, value: /run/postgres/pgpass}
|
||||
# Operator-created NOLOGIN group role; the runtime login is its member.
|
||||
- {name: EVO_PG_RUNTIME_ROLE, value: evolution_runtime}
|
||||
volumeMounts:
|
||||
- {name: connection, mountPath: /run/postgres, readOnly: true}
|
||||
- {name: scratch, mountPath: /tmp}
|
||||
|
||||
@@ -501,3 +501,17 @@ server/libpq pair, optional persistence adapter and transactional schema v1.
|
||||
Private PostgreSQL integration checks passed in Release, analysis and ASan/UBSan;
|
||||
container/cluster verification remains separate. Secret provisioning instructions
|
||||
are in infrastructure/POSTGRESQL.md. Next: M7.3 fenced region ownership.
|
||||
|
||||
M7.3 implemented and locally qualified (2026-09-28). ADR-0060 adds schema v2:
|
||||
database-time leases, DB-generated holders, never-reused epochs and a fenced head
|
||||
advance that checks lease then predecessor under one row lock. The runtime role
|
||||
gets EXECUTE only; table writes, provisioning and object creation are denied.
|
||||
Real-PostgreSQL tests cover the scenarios the roadmap row requires:
|
||||
- 64 two-contender acquisition rounds and 32 racing advances;
|
||||
- expired renewal and a paused stale writer;
|
||||
- a writer queued past expiry, which is judged by post-lock database time;
|
||||
- an unknown renewal that fences locally.
|
||||
|
||||
Timings were selected and measured locally. The director is intentionally deferred
|
||||
because it has no authority and would be a placeholder. Next: M7.4 checkpoint
|
||||
codecs/publication. It must also revoke runtime EXECUTE on the bare head kernel.
|
||||
|
||||
@@ -755,3 +755,17 @@ headless suite passed 54/54. Docker/remote CI and cluster deployment are not
|
||||
claimed. See infrastructure/POSTGRESQL.md and engineering/VALIDATION.md.
|
||||
Next: M7.3 fenced ownership and narrowly granted database functions. The bootstrap
|
||||
lease table alone does not establish authority or durable region publication.
|
||||
|
||||
## M7.3 fenced region ownership (2026-09-28)
|
||||
|
||||
ADR-0060 and migration v2 add guarded lease and head functions for a runtime role
|
||||
that has no table rights. The portable `LeaseAuthority` fences locally from
|
||||
send-time deadlines on CLOCK_BOOTTIME. Selected timings are 20 s lease, 5 s renew,
|
||||
2 s margin, 2 s renew deadline and 5 s commit deadline. Local round trips took
|
||||
about 0.6-0.7 ms p50 and about 1 ms p99. Real PostgreSQL covered contenders,
|
||||
expiry, paused and queued stale writers, unknown renewal, exhaustion and bypass
|
||||
denial. Three SQL mutations (`now()` instead of `clock_timestamp()`, a missing
|
||||
lease check, a disabled busy check) were each detected by a failing test.
|
||||
Windows ran 55/55; Linux Release, clang-tidy and ASan/UBSan PostgreSQL suites
|
||||
passed in WSL. No director, host, gateway, payload publication or deployment
|
||||
exists yet. The v2 head must not carry real world state before M7.4. Next: M7.4.
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
## Status
|
||||
Accepted for M7.2, 2026-09-28. No region authority or world deployment yet.
|
||||
Extended by ADR-0060 (schema v2 fencing functions and runtime grants).
|
||||
Supersedes ADR-0013's libpqxx recommendation for this scoped adapter.
|
||||
|
||||
## Context
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
# ADR-0060: Fenced region ownership
|
||||
|
||||
## Status
|
||||
Accepted for M7.3, 2026-09-28. Implements the lease/fence portion of ADR-0058 on
|
||||
the ADR-0059 adapter. No region host, director, gateway or deployment exists yet.
|
||||
|
||||
## Context
|
||||
ADR-0058 requires that PostgreSQL arbitrate acquire/renew/commit/release under one
|
||||
lease-row lock using database time, that commits compare the expected predecessor,
|
||||
and that workers cannot bypass the guard. Schema v1 has only a lease table and no
|
||||
runtime grants. REGION_DURABILITY.md deferred selecting lease duration, renewal
|
||||
cadence, local safety margin and store deadlines to this gate.
|
||||
|
||||
## Decision
|
||||
Schema v2 (`migrations/0002_region_fencing.sql`) adds `region_heads` and owner-owned
|
||||
functions. Every mutating function is SECURITY DEFINER with `search_path` pinned to
|
||||
`pg_catalog, pg_temp`. The runtime group role gets schema USAGE and EXECUTE on
|
||||
acquire, renew, release, load_region_head and advance_region_head only. It has no
|
||||
table privileges. Provisioning (genesis) and the internal lease check stay owner-only.
|
||||
|
||||
- Acquire locks the row, reads `clock_timestamp()` after the lock and succeeds only if
|
||||
the row is released or expired. It then increments the u64 epoch and stores a
|
||||
database-generated holder UUID. Epoch overflow fails closed. The holder is a
|
||||
per-acquisition token, not a Pod identity.
|
||||
- Renew, release and advance all go through the same row lock and require the exact
|
||||
holder and epoch, with expiry in database time after the lock wait. Renewal never
|
||||
revives an expired lease. Release keeps the epoch, so it is never reused.
|
||||
- The advance function checks the lease, then locks the head. It requires the
|
||||
expected sequence, commit ID and digest, a 1..64 tick delta and no sequence wrap.
|
||||
M7.4/M7.5 publication wraps this kernel in the same transaction as the manifest,
|
||||
decisions and outbox. Runtime EXECUTE on the bare kernel must then be revoked.
|
||||
- Rejections use custom SQLSTATEs EVL01..EVL05 (busy, fenced, head mismatch, not
|
||||
provisioned, exhausted) and 22023 for invalid arguments. A rejection aborts any
|
||||
enclosing transaction. Clients never send timestamps.
|
||||
- The migration requires an operator-created runtime role that is unprivileged and
|
||||
does not inherit the owner. History admission is an exact prefix: clean→v1→v2,
|
||||
v1→v2, or a v2 no-op. Anything else fails closed.
|
||||
|
||||
Host side: the portable `LeaseAuthority` sets its local deadline to request-send time
|
||||
plus the granted duration minus the safety margin. Database expiry is set no earlier
|
||||
than that send, so the holder stops before a successor can acquire. An ambiguous
|
||||
renewal keeps the earlier of the old deadline and the one the attempt could have
|
||||
set. A reply received after the local deadline fences. Fencing is sticky until a
|
||||
strictly newer epoch is acquired, which is followed by recovery. `LeaseClock` uses
|
||||
Linux CLOCK_BOOTTIME so that suspend time still counts.
|
||||
|
||||
Selected timings (`LeasePolicy` defaults; the database admits 1..60 s leases):
|
||||
|
||||
| Setting | Value | Reason |
|
||||
|---|---|---|
|
||||
| Lease duration | 20 s | Bounds orphaned-lease unavailability and leaves 40 s of the proposed 60 s recovery target for load and verification |
|
||||
| Renewal interval | 5 s | Two full attempts (interval + timeout) fit in the 18 s usable window |
|
||||
| Safety margin | 2 s | ≥10% of duration, covering clock-rate disagreement and check-to-effect delay |
|
||||
| Renewal deadline | 2 s | About three orders of magnitude above the measured local p99 (about 1 ms); drop the connection and treat the outcome as unknown |
|
||||
| Commit deadline | 5 s | Below the 10 s server statement timeout; begin only if it ends before local fencing |
|
||||
|
||||
`valid(LeasePolicy)` enforces these relations, not the specific numbers. Remote
|
||||
TLS latency and clock-rate behavior must be measured in the target environment
|
||||
before World0. These values are not a player SLA.
|
||||
|
||||
The world director is deliberately not implemented. Its output would be an
|
||||
assignment hint with no authority, and the database decides ownership anyway.
|
||||
Adding one now would be a placeholder service. It belongs with the host (M7.6)
|
||||
or later operations work.
|
||||
|
||||
## Alternatives Considered
|
||||
Kubernetes Leases or broker locks are not in the commit transaction. Client-supplied
|
||||
holder IDs allow lost-acquire adoption but need uniqueness guarantees and let any
|
||||
runtime guess tokens; DB tokens cost at most one lease period after an unknown
|
||||
acquire. Transaction-start `now()` lets a writer queued on the lock commit after
|
||||
expiry. Row-level security on direct table writes widens the attack surface compared
|
||||
with no table grants. Status-row returns instead of SQLSTATEs would not abort a
|
||||
future enclosing publication transaction.
|
||||
|
||||
## Benefits
|
||||
One lock serializes every ownership and head transition. The runtime login cannot
|
||||
write tables, create objects or call provisioning. Stale, paused, queued and
|
||||
partitioned writers are rejected by the database. The local fence has an explicit
|
||||
bound instead of trusting liveness.
|
||||
|
||||
## Costs
|
||||
Each renewal or commit takes a row lock. After a worker crash or an unknown acquire,
|
||||
the region stays unavailable for up to one lease period. The runtime role must be
|
||||
pre-created by an operator. This adds a second migration and a chain of prefix checksums.
|
||||
|
||||
## Risks
|
||||
A frozen VM/hypervisor can stop all guest clocks. Database commits stay fenced, but
|
||||
provisional output could then be late. Gateways still need bounded-validity
|
||||
registration (ADR-0058). The bare head kernel accepts any 32-byte digest until M7.4
|
||||
binds it to verified payloads, so v2 must not host real world state. Custom
|
||||
SQLSTATEs are project conventions; they are not PostgreSQL-reserved.
|
||||
|
||||
## Consequences
|
||||
The M7.4 migration must add the manifest/payload publication function that calls
|
||||
the kernel, and revoke runtime EXECUTE on `advance_region_head`. M7.5 adds
|
||||
commit-ID resolution. Until then, "head equals my commit ID" is only the primitive
|
||||
reconciliation used in tests. M7.6 hosts must drive `LeaseAuthority`, stop admission,
|
||||
simulation and output when fenced, and recover after any new acquisition. Timing
|
||||
changes require updated measurements and this ADR's table.
|
||||
@@ -70,3 +70,4 @@ design. No unadopted candidate is a link dependency of the proof.
|
||||
- [ADR-0055: Environmental process contract](0055-environmental-process-contract.md) - accepted M6.5b synthetic conversions, source interval policy and process checkpoint boundary; implementation follows in M6.5c.
|
||||
|
||||
- [ADR-0059: PostgreSQL bootstrap](0059-postgresql-bootstrap.md) - accepted for M7.2 optional adapter, pinned server/client, migrations and Secret boundary.
|
||||
- [ADR-0060: Fenced region ownership](0060-fenced-region-ownership.md) - accepted for M7.3 database-time leases, guarded head advance, runtime grants and selected timings.
|
||||
|
||||
@@ -48,8 +48,10 @@ process may retain memory but cannot publish a head or originate current output.
|
||||
The gateway must have bounded-validity registration, not just an epoch number.
|
||||
M7 has no gateway implementation; readiness alone does not prove remote fencing.
|
||||
|
||||
Lease duration, renewal cadence, local safety margin and store deadlines must be
|
||||
selected/measured at M7.3 together; no arbitrary timing values are locked here.
|
||||
M7.3 selected lease duration, renewal cadence, local safety margin and store
|
||||
deadlines together in ADR-0060 (20 s / 5 s / 2 s / 2 s renew / 5 s commit). The
|
||||
latency behind them was measured only on a local socket. The database functions
|
||||
and runtime grants in migration v2 implement the guard described here.
|
||||
Any uncertain lease stops admission, simulation and final output. A commit that
|
||||
already linearized can remain durable after its writer expires; query can return
|
||||
its receipt, but does not grant that writer authority to resume.
|
||||
@@ -159,3 +161,10 @@ resurrect acknowledged deaths. World0 stays gated.
|
||||
|
||||
M7.1 verification is a source/state inventory and contract review. Real database,
|
||||
object-store, process-kill and network partition tests are still required.
|
||||
|
||||
M7.3 executed its rows against real PostgreSQL. After lease expiry, a new owner
|
||||
acquires a newer epoch. The old writer's renew, release and head advance are then
|
||||
rejected, and the head is unchanged. A paused stale worker resuming past its lease
|
||||
is fenced both locally and by the database. A writer queued on the lease lock past
|
||||
expiry is also rejected. These are in-process pauses, not killed pods or network
|
||||
partitions. The 7.6 parts of those rows remain open.
|
||||
|
||||
@@ -2041,3 +2041,43 @@ claim lineage recovery. No retained-history or World0 requirement was relaxed.
|
||||
The adapter is separate from the headless kernel. Region fencing, publication,
|
||||
ambiguous durable commit reconciliation and crash recovery are later M7 gates.
|
||||
Only clean-to-v1 and exact-v1 restart are supported migration paths at this gate.
|
||||
|
||||
## M7.3 fenced region ownership - 2026-09-28
|
||||
|
||||
- Windows MSVC Release configure/build/CTest passed 55/55, including the new
|
||||
portable `region-lease-contracts` test. Repository clang-format 19 check passed.
|
||||
- WSL Ubuntu (GCC 14.2.0; clang 19.1.1; PostgreSQL/libpq 18.6 source build):
|
||||
GCC14 Release, clang19 analysis (clang-tidy warnings-as-errors) and clang19
|
||||
ASan/UBSan builds of `evo-postgres-tests` each passed `tools/ci/postgres-check.sh`
|
||||
against a fresh private cluster. The first analysis build flagged two
|
||||
unchecked optional accesses and two unreserved vectors. These were fixed and
|
||||
all three configurations were rerun after formatting.
|
||||
- Real-database coverage, as the runtime login rather than SET ROLE:
|
||||
- Denied (42501): table read/write, provisioning, the internal check function,
|
||||
and object creation. A hostile search_path is ignored.
|
||||
- Busy, forged-holder, stale-epoch and released-lease rejections; epoch
|
||||
non-reuse.
|
||||
- 1..64 tick delta; predecessor mismatch; exact-retry detection via the head.
|
||||
- 1 s database-time expiry and successor acquisition; paused stale writer fenced
|
||||
locally and by the database.
|
||||
- A writer queued 1.3 s on the lease row lock is rejected after expiry.
|
||||
- A 100 ms renewal deadline under lock (unknown) fences locally before a
|
||||
successor acquires.
|
||||
- 64 two-thread acquisition rounds with exactly one winner and consecutive
|
||||
epochs; 32 racing head advances with exactly one winner.
|
||||
- Epoch and sequence exhaustion.
|
||||
- Migrations tested: clean->v2, repeated v2 no-op, v1->v2 upgrade, and failed
|
||||
0001 rollback. Superuser and missing runtime roles are refused with no schema
|
||||
left. Unknown v3, changed v2 checksum and changed v1 checksum are refused.
|
||||
- Mutation sensitivity (scratch copies only): replacing post-lock
|
||||
`clock_timestamp()` with `now()`, removing the lease check from head advance,
|
||||
and disabling the busy check each made the suite fail at the expected test.
|
||||
- Local Unix-socket latency, 200 samples per operation in each final run:
|
||||
- p50 about 0.61-0.74 ms and p99 about 0.8-1.15 ms for acquire, renew,
|
||||
advance and release.
|
||||
- Max up to 2.8 ms under analysis/ASan builds.
|
||||
These are local numbers, not remote TLS or cluster measurements.
|
||||
- Not executed: Docker/Gitea CI, remote TLS endpoints, Kubernetes Secrets, real pod
|
||||
kill, real network partition (a lock wait plus client deadline stands in for
|
||||
it), VM freeze and TSan. The contention threads each own a separate
|
||||
connection; libpq is uninstrumented.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# PostgreSQL M7.2 bootstrap
|
||||
# PostgreSQL bootstrap (M7.2) and region fencing (M7.3)
|
||||
|
||||
ADR-0059 pins server/libpq 18.6. `evo::postgres` is optional, Linux-only and separate
|
||||
from all simulation targets. Preparation explicitly downloads a verified source
|
||||
@@ -21,7 +21,9 @@ bash tools/ci/postgres-check.sh "$PWD/build/postgres/evo-postgres-tests"
|
||||
```
|
||||
|
||||
The test script overrides all connection coordinates and creates a new private
|
||||
0700 /tmp directory, private Unix socket and disposable local cluster. TCP is off;
|
||||
0700 /tmp directory, private Unix socket and disposable local cluster. It creates
|
||||
the `evo_runtime_test` group and the `evo_runtime_login` member, then runs fencing
|
||||
tests as that real runtime login. TCP is off;
|
||||
local trust applies only inside that private directory. Cluster stops on exit;
|
||||
logs/data are retained there for diagnosis. It never uses production PG credentials
|
||||
or deletes a pre-existing database. Do not point this script at a shared server.
|
||||
@@ -49,18 +51,45 @@ incomplete session. No PostgreSQL diagnostic strings or connection values are lo
|
||||
## Migrations and privilege boundary
|
||||
|
||||
`tools/migrate-postgres.sh` uses psql environment/service files, no URI arguments.
|
||||
It requires exactly server18.6, serializes migrations with a transaction advisory
|
||||
lock, hashes the checked-in SQL, and admits clean->v1 or exact v1->v1 only.
|
||||
Unknown versions/checksum changes/partial foreign schemas fail closed. No down
|
||||
migration or destructive repair exists. The u64 domain rejects fractional,
|
||||
negative and above-u64 values without signed truncation.
|
||||
It requires exactly server 18.6 and serializes migrations with a transaction
|
||||
advisory lock. It hashes each checked-in SQL file and applies everything in one
|
||||
transaction. The recorded history must be an exact prefix: clean->v1->v2, v1->v2,
|
||||
or a v2 no-op. Unknown versions, checksum changes and partial foreign schemas fail
|
||||
closed. There is no down migration or destructive repair. The u64 domain rejects
|
||||
fractional, negative and above-u64 values without signed truncation.
|
||||
|
||||
Use a dedicated owner/migrator login with CREATE on the database. Runtime login
|
||||
must be separate, not a member of the owner and without database CREATE or public
|
||||
schema CREATE. Bootstrap revokes PUBLIC access to its schema/tables and default
|
||||
function execution. Runtime has no direct mutation rights; M7.3 adds fenced
|
||||
functions and narrowly scoped EXECUTE. Operators own database/role provisioning;
|
||||
the migration never creates login passwords or grants superuser.
|
||||
Use a dedicated owner/migrator login with CREATE on the database. Before
|
||||
migrating, operators create a NOLOGIN runtime group role and pass its name as
|
||||
`EVO_PG_RUNTIME_ROLE`. The migration refuses a role that is missing, is the owner,
|
||||
inherits the owner, or has superuser/CREATEROLE/CREATEDB/replication/BYPASSRLS. The
|
||||
runtime login is a separate member of that group only, with no database CREATE or
|
||||
public schema CREATE. Bootstrap revokes PUBLIC access to the schema, tables and
|
||||
default function execution. Operators own database/role provisioning; the migration
|
||||
never creates logins or passwords, and never grants superuser.
|
||||
|
||||
## Fenced region ownership (schema v2, ADR-0060)
|
||||
|
||||
The runtime role has schema USAGE and EXECUTE on five SECURITY DEFINER functions:
|
||||
`acquire_region_lease`, `renew_region_lease`, `release_region_lease`,
|
||||
`load_region_head` and `advance_region_head`. It cannot read or write tables, create
|
||||
objects, call `check_region_lease` or provision regions. Each mutation locks the lease row
|
||||
and checks holder, epoch and expiry with `clock_timestamp()` after the lock wait.
|
||||
Clients never send times. Rejections: EVL01 busy, EVL02 fenced, EVL03 head mismatch,
|
||||
EVL04 not provisioned, EVL05 epoch/sequence exhausted, 22023 invalid arguments.
|
||||
`evo::postgres` maps them to `LeaseStoreError`; a timeout or broken connection is
|
||||
`unknown`.
|
||||
|
||||
Genesis is operator-only (owner login), never done by recovery or runtime:
|
||||
`SELECT evolution.provision_region(world, region, frame, genesis_commit_uuid,
|
||||
'\x<64 hex digest>');`. In schema v2 the head is an identity/digest pointer with no
|
||||
payload binding. Do not provision real world state before M7.4 publication.
|
||||
|
||||
Hosts use `LeaseAuthority` with `LeasePolicy` defaults: 20 s lease, 5 s renewal
|
||||
interval, 2 s safety margin, 2 s renewal deadline and 5 s commit deadline. The
|
||||
local deadline runs from request send time on CLOCK_BOOTTIME. If a renewal outcome
|
||||
is unknown, keep the earlier deadline and reconnect. Once fenced, stop admission,
|
||||
simulation and output, then acquire a new epoch and recover. Never resume the old
|
||||
epoch.
|
||||
|
||||
## Kubernetes Secrets
|
||||
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
#pragma once
|
||||
#include <array>
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
|
||||
// Portable host-side region authority contract (ADR-0060). No database types:
|
||||
// PostgreSQL arbitrates ownership; this only decides when a holder must stop.
|
||||
namespace evo::persistence {
|
||||
using Uuid = std::array<std::uint8_t, 16>;
|
||||
using Sha256Digest = std::array<std::uint8_t, 32>;
|
||||
|
||||
struct RegionKey {
|
||||
std::uint64_t world{};
|
||||
std::uint64_t region{};
|
||||
};
|
||||
struct RegionHead {
|
||||
std::uint64_t frame{};
|
||||
std::uint64_t sequence{};
|
||||
Uuid commit{};
|
||||
Sha256Digest digest{};
|
||||
std::uint64_t end_tick{};
|
||||
friend bool operator==(const RegionHead&, const RegionHead&) = default;
|
||||
};
|
||||
// Holder is a database-generated per-acquisition token, not a Pod identity.
|
||||
struct LeaseGrant {
|
||||
Uuid holder{};
|
||||
std::uint64_t epoch{};
|
||||
std::uint64_t frame{};
|
||||
std::chrono::milliseconds duration{};
|
||||
};
|
||||
|
||||
// Selected M7.3 timings. Validity: duration 1..60s (database bound); margin at
|
||||
// least 10% of duration (clock-rate and check-to-effect allowance); two complete
|
||||
// renewal attempts fit before local fencing; I/O deadlines within adapter limits.
|
||||
struct LeasePolicy {
|
||||
std::chrono::milliseconds duration{20000};
|
||||
std::chrono::milliseconds renew_interval{5000};
|
||||
std::chrono::milliseconds safety_margin{2000};
|
||||
std::chrono::milliseconds renew_timeout{2000};
|
||||
std::chrono::milliseconds commit_timeout{5000};
|
||||
};
|
||||
[[nodiscard]] bool valid(const LeasePolicy& policy) noexcept;
|
||||
|
||||
// Monotonic clock that keeps advancing while the process or machine is suspended
|
||||
// (Linux CLOCK_BOOTTIME). A paused worker must observe that time passed.
|
||||
struct LeaseClock {
|
||||
using duration = std::chrono::nanoseconds;
|
||||
using rep = duration::rep;
|
||||
using period = duration::period;
|
||||
using time_point = std::chrono::time_point<LeaseClock>;
|
||||
static constexpr bool is_steady = true;
|
||||
[[nodiscard]] static time_point now() noexcept;
|
||||
};
|
||||
|
||||
enum class LeaseState : std::uint8_t { absent, active, renew_due, fenced };
|
||||
|
||||
// Single-owner local view of one region lease. Every time point is a LeaseClock
|
||||
// reading supplied by the host. The local deadline is request-send time plus the
|
||||
// granted duration minus the safety margin: the database set its expiry no earlier
|
||||
// than that send, so the holder stops before any successor can acquire. Fencing is
|
||||
// sticky until a strictly newer epoch is acquired; renewal never revives it.
|
||||
class LeaseAuthority {
|
||||
public:
|
||||
explicit LeaseAuthority(const LeasePolicy& policy) noexcept : policy_(policy) {}
|
||||
[[nodiscard]] const LeasePolicy& policy() const noexcept {
|
||||
return policy_;
|
||||
}
|
||||
[[nodiscard]] const std::optional<LeaseGrant>& grant() const noexcept {
|
||||
return grant_;
|
||||
}
|
||||
// Returns false (and fences) for an invalid policy, reused/older epoch or late reply.
|
||||
[[nodiscard]] bool acquired(const LeaseGrant& grant, LeaseClock::time_point sent,
|
||||
LeaseClock::time_point received) noexcept;
|
||||
[[nodiscard]] bool renewed(std::uint64_t epoch, std::chrono::milliseconds granted,
|
||||
LeaseClock::time_point sent,
|
||||
LeaseClock::time_point received) noexcept;
|
||||
// Ambiguous renewal: the database may have applied a shorter expiry, so keep
|
||||
// the earlier of the old deadline and the one the attempt could have set.
|
||||
void renewal_unknown(std::chrono::milliseconds requested, LeaseClock::time_point sent) noexcept;
|
||||
void fence() noexcept;
|
||||
[[nodiscard]] LeaseState state(LeaseClock::time_point now) const noexcept;
|
||||
// Admission, simulation and final output require an unfenced lease.
|
||||
[[nodiscard]] bool may_originate(LeaseClock::time_point now) const noexcept;
|
||||
// Liveness guard: begin a commit only if its deadline ends before local fencing.
|
||||
[[nodiscard]] bool may_begin_commit(LeaseClock::time_point now) const noexcept;
|
||||
[[nodiscard]] LeaseClock::time_point deadline() const noexcept {
|
||||
return deadline_;
|
||||
}
|
||||
|
||||
private:
|
||||
LeasePolicy policy_;
|
||||
std::optional<LeaseGrant> grant_;
|
||||
std::uint64_t highest_epoch_{};
|
||||
LeaseClock::time_point last_sent_{};
|
||||
LeaseClock::time_point deadline_{};
|
||||
bool fenced_{};
|
||||
};
|
||||
} // namespace evo::persistence
|
||||
@@ -0,0 +1,50 @@
|
||||
#pragma once
|
||||
#include "evo/persistence/postgres.hpp"
|
||||
#include "evo/persistence/region_lease.hpp"
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <expected>
|
||||
|
||||
// Linux-only calls to the fenced database functions of migration v2 (ADR-0060).
|
||||
// Each call is one statement on a caller-owned connection; outside an explicit
|
||||
// transaction it commits by itself. No retries. Rejections are ordinary results.
|
||||
namespace evo::persistence {
|
||||
enum class LeaseStoreError : std::uint8_t {
|
||||
busy, // Another holder's lease is unexpired in database time.
|
||||
fenced, // Holder/epoch superseded, released or expired: stop originating.
|
||||
head_mismatch, // Expected predecessor is not the committed head.
|
||||
not_provisioned, // No genesis; runtime can never create one.
|
||||
exhausted, // Epoch or commit sequence would wrap: fail closed.
|
||||
invalid, // Rejected arguments; nothing changed.
|
||||
store, // Other SQL/configuration failure; nothing committed.
|
||||
unknown, // May have executed; reconnect and reconcile before reuse.
|
||||
};
|
||||
struct LeaseStoreFailure {
|
||||
LeaseStoreError kind{};
|
||||
PgFailure cause{};
|
||||
};
|
||||
struct HeadAdvance {
|
||||
Uuid commit{};
|
||||
Sha256Digest digest{};
|
||||
std::uint64_t end_tick{};
|
||||
};
|
||||
|
||||
[[nodiscard]] std::expected<LeaseGrant, LeaseStoreFailure>
|
||||
acquire_region_lease(Postgres& db, RegionKey key, std::chrono::milliseconds duration,
|
||||
std::chrono::milliseconds timeout);
|
||||
// Returns the granted duration; the caller measures it from its request send time.
|
||||
[[nodiscard]] std::expected<std::chrono::milliseconds, LeaseStoreFailure>
|
||||
renew_region_lease(Postgres& db, RegionKey key, const LeaseGrant& lease,
|
||||
std::chrono::milliseconds duration, std::chrono::milliseconds timeout);
|
||||
[[nodiscard]] std::expected<void, LeaseStoreFailure>
|
||||
release_region_lease(Postgres& db, RegionKey key, const LeaseGrant& lease,
|
||||
std::chrono::milliseconds timeout);
|
||||
[[nodiscard]] std::expected<RegionHead, LeaseStoreFailure>
|
||||
load_region_head(Postgres& db, RegionKey key, std::chrono::milliseconds timeout);
|
||||
// Locks the lease then the head; requires unexpired holder/epoch and the exact
|
||||
// expected predecessor. head_mismatch after unknown: load the head and compare.
|
||||
[[nodiscard]] std::expected<RegionHead, LeaseStoreFailure>
|
||||
advance_region_head(Postgres& db, RegionKey key, const LeaseGrant& lease,
|
||||
const RegionHead& expected, const HeadAdvance& next,
|
||||
std::chrono::milliseconds timeout);
|
||||
} // namespace evo::persistence
|
||||
@@ -0,0 +1,214 @@
|
||||
-- M7.3 fenced region ownership (ADR-0060). Owner-only definitions; the runtime
|
||||
-- role receives schema USAGE and EXECUTE on the guarded functions, never tables.
|
||||
-- Every mutation locks the region lease row first and decides expiry with the
|
||||
-- database clock read after that lock. Clients never supply timestamps.
|
||||
SELECT pg_catalog.set_config('evolution.runtime_role', :'runtime_role', true);
|
||||
DO $$
|
||||
DECLARE
|
||||
runtime name := pg_catalog.current_setting('evolution.runtime_role');
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles r WHERE r.rolname = runtime
|
||||
AND NOT r.rolsuper AND NOT r.rolcreaterole AND NOT r.rolcreatedb
|
||||
AND NOT r.rolreplication AND NOT r.rolbypassrls)
|
||||
OR runtime = current_user
|
||||
OR pg_catalog.pg_has_role(runtime, current_user, 'MEMBER') THEN
|
||||
RAISE EXCEPTION 'runtime role must exist, be unprivileged and not inherit the owner';
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
CREATE TABLE evolution.region_heads (
|
||||
world_id evolution.u64 NOT NULL,
|
||||
region_id evolution.u64 NOT NULL,
|
||||
commit_sequence evolution.u64 NOT NULL,
|
||||
commit_id uuid NOT NULL,
|
||||
digest bytea NOT NULL CHECK (octet_length(digest) = 32),
|
||||
end_tick evolution.u64 NOT NULL,
|
||||
published_epoch evolution.u64 NOT NULL,
|
||||
PRIMARY KEY (world_id, region_id),
|
||||
FOREIGN KEY (world_id, region_id) REFERENCES evolution.region_leases,
|
||||
-- Genesis is sequence 0, tick 0, provisioned outside any lease epoch.
|
||||
CHECK ((commit_sequence = 0) = (end_tick = 0)),
|
||||
CHECK ((commit_sequence = 0) = (published_epoch = 0))
|
||||
);
|
||||
REVOKE ALL ON evolution.region_heads FROM PUBLIC;
|
||||
|
||||
-- Owner-only provisioning; recovery and runtime can never create genesis.
|
||||
CREATE FUNCTION evolution.provision_region(p_world evolution.u64, p_region evolution.u64,
|
||||
p_frame evolution.u64, p_genesis_commit uuid, p_genesis_digest bytea)
|
||||
RETURNS void LANGUAGE sql SECURITY INVOKER SET search_path = pg_catalog, pg_temp AS $$
|
||||
INSERT INTO evolution.region_leases(world_id, region_id, frame_id)
|
||||
VALUES (p_world, p_region, p_frame);
|
||||
INSERT INTO evolution.region_heads(world_id, region_id, commit_sequence, commit_id,
|
||||
digest, end_tick, published_epoch)
|
||||
VALUES (p_world, p_region, 0, p_genesis_commit, p_genesis_digest, 0, 0);
|
||||
$$;
|
||||
|
||||
CREATE FUNCTION evolution.acquire_region_lease(p_world evolution.u64,
|
||||
p_region evolution.u64, p_duration_ms integer)
|
||||
RETURNS TABLE (lease_holder uuid, lease_epoch evolution.u64, region_frame evolution.u64,
|
||||
granted_ms integer)
|
||||
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$
|
||||
DECLARE
|
||||
l evolution.region_leases%ROWTYPE;
|
||||
t timestamptz;
|
||||
BEGIN
|
||||
IF p_world IS NULL OR p_region IS NULL OR p_duration_ms IS NULL
|
||||
OR p_duration_ms NOT BETWEEN 1000 AND 60000 THEN
|
||||
RAISE EXCEPTION 'invalid lease request' USING ERRCODE = '22023';
|
||||
END IF;
|
||||
SELECT * INTO l FROM evolution.region_leases r
|
||||
WHERE r.world_id = p_world AND r.region_id = p_region FOR UPDATE;
|
||||
IF NOT FOUND THEN
|
||||
RAISE EXCEPTION 'region not provisioned' USING ERRCODE = 'EVL04';
|
||||
END IF;
|
||||
t := clock_timestamp();
|
||||
IF l.holder IS NOT NULL AND l.expires_at > t THEN
|
||||
RAISE EXCEPTION 'region lease held' USING ERRCODE = 'EVL01';
|
||||
END IF;
|
||||
IF l.epoch = 18446744073709551615 THEN
|
||||
RAISE EXCEPTION 'lease epochs exhausted' USING ERRCODE = 'EVL05';
|
||||
END IF;
|
||||
UPDATE evolution.region_leases r
|
||||
SET epoch = l.epoch + 1, holder = gen_random_uuid(),
|
||||
expires_at = t + p_duration_ms * interval '1 millisecond'
|
||||
WHERE r.world_id = p_world AND r.region_id = p_region
|
||||
RETURNING r.holder, r.epoch, r.frame_id INTO lease_holder, lease_epoch, region_frame;
|
||||
granted_ms := p_duration_ms;
|
||||
RETURN NEXT;
|
||||
END $$;
|
||||
|
||||
-- Locks the lease row and requires the exact unexpired holder/epoch. Internal:
|
||||
-- callable only by the owner and the SECURITY DEFINER functions below.
|
||||
CREATE FUNCTION evolution.check_region_lease(p_world evolution.u64, p_region evolution.u64,
|
||||
p_holder uuid, p_epoch evolution.u64)
|
||||
RETURNS timestamptz LANGUAGE plpgsql SECURITY INVOKER
|
||||
SET search_path = pg_catalog, pg_temp AS $$
|
||||
DECLARE
|
||||
l evolution.region_leases%ROWTYPE;
|
||||
t timestamptz;
|
||||
BEGIN
|
||||
IF p_world IS NULL OR p_region IS NULL OR p_holder IS NULL OR p_epoch IS NULL THEN
|
||||
RAISE EXCEPTION 'invalid lease request' USING ERRCODE = '22023';
|
||||
END IF;
|
||||
SELECT * INTO l FROM evolution.region_leases r
|
||||
WHERE r.world_id = p_world AND r.region_id = p_region FOR UPDATE;
|
||||
IF NOT FOUND THEN
|
||||
RAISE EXCEPTION 'region not provisioned' USING ERRCODE = 'EVL04';
|
||||
END IF;
|
||||
-- Read after the lock wait: a queued stale writer sees current database time.
|
||||
t := clock_timestamp();
|
||||
IF l.holder IS DISTINCT FROM p_holder OR l.epoch <> p_epoch OR l.expires_at <= t THEN
|
||||
RAISE EXCEPTION 'region lease fenced' USING ERRCODE = 'EVL02';
|
||||
END IF;
|
||||
RETURN t;
|
||||
END $$;
|
||||
|
||||
CREATE FUNCTION evolution.renew_region_lease(p_world evolution.u64, p_region evolution.u64,
|
||||
p_holder uuid, p_epoch evolution.u64, p_duration_ms integer)
|
||||
RETURNS integer LANGUAGE plpgsql SECURITY DEFINER
|
||||
SET search_path = pg_catalog, pg_temp AS $$
|
||||
DECLARE
|
||||
t timestamptz;
|
||||
BEGIN
|
||||
IF p_duration_ms IS NULL OR p_duration_ms NOT BETWEEN 1000 AND 60000 THEN
|
||||
RAISE EXCEPTION 'invalid lease request' USING ERRCODE = '22023';
|
||||
END IF;
|
||||
-- Renewal never revives an expired or superseded lease.
|
||||
t := evolution.check_region_lease(p_world, p_region, p_holder, p_epoch);
|
||||
UPDATE evolution.region_leases r
|
||||
SET expires_at = t + p_duration_ms * interval '1 millisecond'
|
||||
WHERE r.world_id = p_world AND r.region_id = p_region;
|
||||
RETURN p_duration_ms;
|
||||
END $$;
|
||||
|
||||
CREATE FUNCTION evolution.release_region_lease(p_world evolution.u64, p_region evolution.u64,
|
||||
p_holder uuid, p_epoch evolution.u64)
|
||||
RETURNS void LANGUAGE plpgsql SECURITY DEFINER
|
||||
SET search_path = pg_catalog, pg_temp AS $$
|
||||
BEGIN
|
||||
PERFORM evolution.check_region_lease(p_world, p_region, p_holder, p_epoch);
|
||||
-- The epoch is retained so the next acquisition can never reuse it.
|
||||
UPDATE evolution.region_leases r SET holder = NULL, expires_at = NULL
|
||||
WHERE r.world_id = p_world AND r.region_id = p_region;
|
||||
END $$;
|
||||
|
||||
CREATE FUNCTION evolution.load_region_head(p_world evolution.u64, p_region evolution.u64)
|
||||
RETURNS TABLE (region_frame evolution.u64, head_sequence evolution.u64, head_commit uuid,
|
||||
head_digest text, head_tick evolution.u64)
|
||||
LANGUAGE plpgsql STABLE SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$
|
||||
BEGIN
|
||||
RETURN QUERY SELECT l.frame_id, h.commit_sequence, h.commit_id,
|
||||
encode(h.digest, 'hex'), h.end_tick
|
||||
FROM evolution.region_heads h
|
||||
JOIN evolution.region_leases l USING (world_id, region_id)
|
||||
WHERE h.world_id = p_world AND h.region_id = p_region;
|
||||
IF NOT FOUND THEN
|
||||
RAISE EXCEPTION 'region not provisioned' USING ERRCODE = 'EVL04';
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
-- The fenced transition kernel. M7.4/M7.5 publication wraps it with manifest,
|
||||
-- decision and outbox rows in the same transaction; the lease lock is taken first.
|
||||
CREATE FUNCTION evolution.advance_region_head(p_world evolution.u64, p_region evolution.u64,
|
||||
p_holder uuid, p_epoch evolution.u64, p_expected_sequence evolution.u64,
|
||||
p_expected_commit uuid, p_expected_digest bytea, p_commit uuid, p_digest bytea,
|
||||
p_end_tick evolution.u64)
|
||||
RETURNS TABLE (region_frame evolution.u64, head_sequence evolution.u64, head_commit uuid,
|
||||
head_digest text, head_tick evolution.u64)
|
||||
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$
|
||||
DECLARE
|
||||
h evolution.region_heads%ROWTYPE;
|
||||
BEGIN
|
||||
IF p_expected_sequence IS NULL OR p_expected_commit IS NULL OR p_commit IS NULL
|
||||
OR p_end_tick IS NULL OR p_commit = p_expected_commit
|
||||
OR octet_length(p_expected_digest) IS DISTINCT FROM 32
|
||||
OR octet_length(p_digest) IS DISTINCT FROM 32 THEN
|
||||
RAISE EXCEPTION 'invalid head advance' USING ERRCODE = '22023';
|
||||
END IF;
|
||||
PERFORM evolution.check_region_lease(p_world, p_region, p_holder, p_epoch);
|
||||
SELECT * INTO h FROM evolution.region_heads r
|
||||
WHERE r.world_id = p_world AND r.region_id = p_region FOR UPDATE;
|
||||
IF NOT FOUND THEN
|
||||
RAISE EXCEPTION 'region not provisioned' USING ERRCODE = 'EVL04';
|
||||
END IF;
|
||||
IF h.commit_sequence <> p_expected_sequence OR h.commit_id <> p_expected_commit
|
||||
OR h.digest <> p_expected_digest THEN
|
||||
RAISE EXCEPTION 'region head moved' USING ERRCODE = 'EVL03';
|
||||
END IF;
|
||||
IF h.commit_sequence = 18446744073709551615 THEN
|
||||
RAISE EXCEPTION 'commit sequences exhausted' USING ERRCODE = 'EVL05';
|
||||
END IF;
|
||||
IF p_end_tick <= h.end_tick OR p_end_tick - h.end_tick > 64 THEN
|
||||
RAISE EXCEPTION 'invalid head advance' USING ERRCODE = '22023';
|
||||
END IF;
|
||||
UPDATE evolution.region_heads r
|
||||
SET commit_sequence = h.commit_sequence + 1, commit_id = p_commit, digest = p_digest,
|
||||
end_tick = p_end_tick, published_epoch = p_epoch
|
||||
WHERE r.world_id = p_world AND r.region_id = p_region;
|
||||
RETURN QUERY SELECT l.frame_id, n.commit_sequence, n.commit_id, encode(n.digest, 'hex'),
|
||||
n.end_tick
|
||||
FROM evolution.region_heads n
|
||||
JOIN evolution.region_leases l USING (world_id, region_id)
|
||||
WHERE n.world_id = p_world AND n.region_id = p_region;
|
||||
END $$;
|
||||
|
||||
REVOKE ALL ON FUNCTION
|
||||
evolution.provision_region(evolution.u64, evolution.u64, evolution.u64, uuid, bytea),
|
||||
evolution.acquire_region_lease(evolution.u64, evolution.u64, integer),
|
||||
evolution.check_region_lease(evolution.u64, evolution.u64, uuid, evolution.u64),
|
||||
evolution.renew_region_lease(evolution.u64, evolution.u64, uuid, evolution.u64, integer),
|
||||
evolution.release_region_lease(evolution.u64, evolution.u64, uuid, evolution.u64),
|
||||
evolution.load_region_head(evolution.u64, evolution.u64),
|
||||
evolution.advance_region_head(evolution.u64, evolution.u64, uuid, evolution.u64,
|
||||
evolution.u64, uuid, bytea, uuid, bytea, evolution.u64)
|
||||
FROM PUBLIC;
|
||||
GRANT USAGE ON SCHEMA evolution TO :"runtime_role";
|
||||
GRANT EXECUTE ON FUNCTION
|
||||
evolution.acquire_region_lease(evolution.u64, evolution.u64, integer),
|
||||
evolution.renew_region_lease(evolution.u64, evolution.u64, uuid, evolution.u64, integer),
|
||||
evolution.release_region_lease(evolution.u64, evolution.u64, uuid, evolution.u64),
|
||||
evolution.load_region_head(evolution.u64, evolution.u64),
|
||||
evolution.advance_region_head(evolution.u64, evolution.u64, uuid, evolution.u64,
|
||||
evolution.u64, uuid, bytea, uuid, bytea, evolution.u64)
|
||||
TO :"runtime_role";
|
||||
INSERT INTO evolution.schema_migrations(version, sha256) VALUES (2, :'migration_sha_2');
|
||||
+10
-4
@@ -12,15 +12,21 @@ SELECT current_setting('server_version_num')::int = 180006 AS supported \gset
|
||||
\endif
|
||||
SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') AS exists \gset
|
||||
\if :exists
|
||||
-- Unknown/partial schema fails closed; no IF NOT EXISTS repair of foreign data.
|
||||
SELECT count(*) = 1 AND min(version) = 1 AND min(sha256) = :'migration_sha'
|
||||
AS compatible FROM evolution.schema_migrations \gset
|
||||
\if :compatible
|
||||
-- Only an exact prefix of the checked-in history is admitted. Unknown or
|
||||
-- partial/foreign schemas fail closed; no IF NOT EXISTS repair of foreign data.
|
||||
SELECT coalesce(string_agg(version || ':' || sha256, ',' ORDER BY version), '')
|
||||
AS history FROM evolution.schema_migrations \gset
|
||||
SELECT :'history' = '1:' || :'migration_sha' AS at_v1,
|
||||
:'history' = '1:' || :'migration_sha' || ',2:' || :'migration_sha_2' AS at_v2 \gset
|
||||
\if :at_v2
|
||||
\elif :at_v1
|
||||
\ir 0002_region_fencing.sql
|
||||
\else
|
||||
\echo 'Unknown or changed migration history'
|
||||
DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$;
|
||||
\endif
|
||||
\else
|
||||
\ir 0001_region_bootstrap.sql
|
||||
\ir 0002_region_fencing.sql
|
||||
\endif
|
||||
COMMIT;
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
#include "evo/persistence/region_lease.hpp"
|
||||
#include <algorithm>
|
||||
#if defined(__linux__)
|
||||
#include <time.h>
|
||||
#endif
|
||||
|
||||
namespace evo::persistence {
|
||||
namespace {
|
||||
using std::chrono::milliseconds;
|
||||
constexpr milliseconds minimum_duration{1000};
|
||||
constexpr milliseconds maximum_duration{60000};
|
||||
constexpr milliseconds maximum_io{30000}; // Postgres adapter deadline bound.
|
||||
} // namespace
|
||||
|
||||
bool valid(const LeasePolicy& p) noexcept {
|
||||
const milliseconds zero{};
|
||||
if (p.duration < minimum_duration || p.duration > maximum_duration ||
|
||||
p.renew_interval <= zero || p.safety_margin <= zero || p.renew_timeout <= zero ||
|
||||
p.commit_timeout <= zero || p.renew_timeout > maximum_io || p.commit_timeout > maximum_io ||
|
||||
p.safety_margin >= p.duration)
|
||||
return false;
|
||||
const auto usable = p.duration - p.safety_margin;
|
||||
return p.safety_margin * 10 >= p.duration &&
|
||||
(p.renew_interval + p.renew_timeout) * 2 <= usable && p.commit_timeout < usable;
|
||||
}
|
||||
|
||||
LeaseClock::time_point LeaseClock::now() noexcept {
|
||||
#if defined(__linux__)
|
||||
timespec ts{};
|
||||
// CLOCK_BOOTTIME cannot fail with a valid clock id and pointer.
|
||||
static_cast<void>(clock_gettime(CLOCK_BOOTTIME, &ts));
|
||||
return time_point{std::chrono::seconds{ts.tv_sec} + std::chrono::nanoseconds{ts.tv_nsec}};
|
||||
#else
|
||||
// Storage hosts are Linux-only; elsewhere suspend behavior is unqualified.
|
||||
return time_point{
|
||||
std::chrono::duration_cast<duration>(std::chrono::steady_clock::now().time_since_epoch())};
|
||||
#endif
|
||||
}
|
||||
|
||||
bool LeaseAuthority::acquired(const LeaseGrant& grant, LeaseClock::time_point sent,
|
||||
LeaseClock::time_point received) noexcept {
|
||||
const auto deadline = sent + grant.duration - policy_.safety_margin;
|
||||
if (!valid(policy_) || grant.epoch == 0 || grant.epoch <= highest_epoch_ || grant.frame == 0 ||
|
||||
grant.duration != policy_.duration || received < sent || received >= deadline) {
|
||||
fence();
|
||||
return false;
|
||||
}
|
||||
grant_ = grant;
|
||||
highest_epoch_ = grant.epoch;
|
||||
last_sent_ = sent;
|
||||
deadline_ = deadline;
|
||||
fenced_ = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool LeaseAuthority::renewed(std::uint64_t epoch, milliseconds granted, LeaseClock::time_point sent,
|
||||
LeaseClock::time_point received) noexcept {
|
||||
if (!grant_ || fenced_ || epoch != grant_->epoch || granted != policy_.duration ||
|
||||
sent < last_sent_ || received < sent || received >= deadline_) {
|
||||
fence();
|
||||
return false;
|
||||
}
|
||||
// Database expiry is its execution time plus granted, which is after sent.
|
||||
deadline_ = sent + granted - policy_.safety_margin;
|
||||
last_sent_ = sent;
|
||||
return true;
|
||||
}
|
||||
|
||||
void LeaseAuthority::renewal_unknown(milliseconds requested, LeaseClock::time_point sent) noexcept {
|
||||
if (!grant_ || fenced_)
|
||||
return;
|
||||
deadline_ = std::min(deadline_, sent + requested - policy_.safety_margin);
|
||||
last_sent_ = std::max(last_sent_, sent);
|
||||
}
|
||||
|
||||
void LeaseAuthority::fence() noexcept {
|
||||
fenced_ = true;
|
||||
}
|
||||
|
||||
LeaseState LeaseAuthority::state(LeaseClock::time_point now) const noexcept {
|
||||
if (!grant_)
|
||||
return LeaseState::absent;
|
||||
if (fenced_ || now >= deadline_)
|
||||
return LeaseState::fenced;
|
||||
return now >= last_sent_ + policy_.renew_interval ? LeaseState::renew_due : LeaseState::active;
|
||||
}
|
||||
|
||||
bool LeaseAuthority::may_originate(LeaseClock::time_point now) const noexcept {
|
||||
const auto s = state(now);
|
||||
return s == LeaseState::active || s == LeaseState::renew_due;
|
||||
}
|
||||
|
||||
bool LeaseAuthority::may_begin_commit(LeaseClock::time_point now) const noexcept {
|
||||
return may_originate(now) && now + policy_.commit_timeout <= deadline_;
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
@@ -0,0 +1,204 @@
|
||||
#include "evo/persistence/region_lease_store.hpp"
|
||||
#include <array>
|
||||
#include <charconv>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
namespace evo::persistence {
|
||||
namespace {
|
||||
using std::chrono::milliseconds;
|
||||
constexpr std::string_view digits = "0123456789abcdef";
|
||||
|
||||
LeaseStoreFailure classify(const PgFailure& f) {
|
||||
const std::string_view state(f.sqlstate.data(), 5);
|
||||
auto kind = LeaseStoreError::store;
|
||||
if (f.kind == PgError::unknown)
|
||||
kind = LeaseStoreError::unknown;
|
||||
else if (f.kind == PgError::config || f.kind == PgError::bounds || state == "22023")
|
||||
kind = LeaseStoreError::invalid;
|
||||
else if (state == "EVL01")
|
||||
kind = LeaseStoreError::busy;
|
||||
else if (state == "EVL02")
|
||||
kind = LeaseStoreError::fenced;
|
||||
else if (state == "EVL03")
|
||||
kind = LeaseStoreError::head_mismatch;
|
||||
else if (state == "EVL04")
|
||||
kind = LeaseStoreError::not_provisioned;
|
||||
else if (state == "EVL05")
|
||||
kind = LeaseStoreError::exhausted;
|
||||
return {kind, f};
|
||||
}
|
||||
LeaseStoreFailure malformed(bool mutation) {
|
||||
// A mutation whose reply cannot be parsed may still have committed.
|
||||
return {mutation ? LeaseStoreError::unknown : LeaseStoreError::store, {}};
|
||||
}
|
||||
template <std::size_t N> std::string hex(const std::array<std::uint8_t, N>& bytes) {
|
||||
std::string out;
|
||||
out.reserve(N * 2);
|
||||
for (const auto b : bytes) {
|
||||
out.push_back(digits[static_cast<std::size_t>(b >> 4U)]);
|
||||
out.push_back(digits[static_cast<std::size_t>(b & 0x0FU)]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
std::optional<std::uint8_t> nibble(char c) {
|
||||
const auto at = digits.find(c);
|
||||
if (at == std::string_view::npos)
|
||||
return std::nullopt;
|
||||
return static_cast<std::uint8_t>(at);
|
||||
}
|
||||
template <std::size_t N> std::optional<std::array<std::uint8_t, N>> unhex(std::string_view s) {
|
||||
if (s.size() != N * 2)
|
||||
return std::nullopt;
|
||||
std::array<std::uint8_t, N> out{};
|
||||
for (std::size_t i = 0; i < N; ++i) {
|
||||
const auto high = nibble(s[i * 2]);
|
||||
const auto low = nibble(s[(i * 2) + 1]);
|
||||
if (!high || !low)
|
||||
return std::nullopt;
|
||||
out[i] = static_cast<std::uint8_t>((*high << 4U) | *low);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
std::string uuid_text(const Uuid& id) {
|
||||
const auto h = hex(id);
|
||||
return h.substr(0, 8) + '-' + h.substr(8, 4) + '-' + h.substr(12, 4) + '-' + h.substr(16, 4) +
|
||||
'-' + h.substr(20);
|
||||
}
|
||||
std::optional<Uuid> parse_uuid(const std::optional<std::string>& s) {
|
||||
if (!s || s->size() != 36)
|
||||
return std::nullopt;
|
||||
std::string compact;
|
||||
for (std::size_t i = 0; i < s->size(); ++i) {
|
||||
const bool dash = i == 8 || i == 13 || i == 18 || i == 23;
|
||||
if (dash != ((*s)[i] == '-'))
|
||||
return std::nullopt;
|
||||
if (!dash)
|
||||
compact.push_back((*s)[i]);
|
||||
}
|
||||
return unhex<16>(compact);
|
||||
}
|
||||
template <typename T> std::optional<T> parse_integer(const std::optional<std::string>& s) {
|
||||
if (!s || s->empty() || s->size() > 20 || s->front() == '-' || s->front() == '+')
|
||||
return std::nullopt;
|
||||
T value{};
|
||||
const auto* end = s->data() + s->size();
|
||||
const auto [at, error] = std::from_chars(s->data(), end, value);
|
||||
if (error != std::errc{} || at != end)
|
||||
return std::nullopt;
|
||||
return value;
|
||||
}
|
||||
std::string bytea(const Sha256Digest& d) {
|
||||
return "\\x" + hex(d);
|
||||
}
|
||||
std::optional<RegionHead> parse_head(const PgRows& rows) {
|
||||
if (rows.size() != 1 || rows[0].size() != 5)
|
||||
return std::nullopt;
|
||||
const auto& r = rows[0];
|
||||
const auto frame = parse_integer<std::uint64_t>(r[0]);
|
||||
const auto sequence = parse_integer<std::uint64_t>(r[1]);
|
||||
const auto commit = parse_uuid(r[2]);
|
||||
const auto& digest_text = r[3];
|
||||
const auto digest = digest_text ? unhex<32>(*digest_text) : std::nullopt;
|
||||
const auto tick = parse_integer<std::uint64_t>(r[4]);
|
||||
if (!frame || !sequence || !commit || !digest || !tick)
|
||||
return std::nullopt;
|
||||
return RegionHead{*frame, *sequence, *commit, *digest, *tick};
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::expected<LeaseGrant, LeaseStoreFailure>
|
||||
acquire_region_lease(Postgres& db, RegionKey key, milliseconds duration, milliseconds timeout) {
|
||||
const std::array<std::string, 3> p{std::to_string(key.world), std::to_string(key.region),
|
||||
std::to_string(duration.count())};
|
||||
auto rows = db.query("SELECT * FROM evolution.acquire_region_lease($1::evolution.u64, "
|
||||
"$2::evolution.u64, $3::integer)",
|
||||
p, timeout);
|
||||
if (!rows)
|
||||
return std::unexpected{classify(rows.error())};
|
||||
if (rows->size() != 1 || (*rows)[0].size() != 4)
|
||||
return std::unexpected{malformed(true)};
|
||||
const auto& r = (*rows)[0];
|
||||
const auto holder = parse_uuid(r[0]);
|
||||
const auto epoch = parse_integer<std::uint64_t>(r[1]);
|
||||
const auto frame = parse_integer<std::uint64_t>(r[2]);
|
||||
const auto granted = parse_integer<std::int64_t>(r[3]);
|
||||
if (!holder || !epoch || !frame || !granted || *epoch == 0)
|
||||
return std::unexpected{malformed(true)};
|
||||
return LeaseGrant{*holder, *epoch, *frame, milliseconds{*granted}};
|
||||
}
|
||||
|
||||
std::expected<milliseconds, LeaseStoreFailure> renew_region_lease(Postgres& db, RegionKey key,
|
||||
const LeaseGrant& lease,
|
||||
milliseconds duration,
|
||||
milliseconds timeout) {
|
||||
const std::array<std::string, 5> p{std::to_string(key.world), std::to_string(key.region),
|
||||
uuid_text(lease.holder), std::to_string(lease.epoch),
|
||||
std::to_string(duration.count())};
|
||||
auto rows = db.query("SELECT evolution.renew_region_lease($1::evolution.u64, "
|
||||
"$2::evolution.u64, $3::uuid, $4::evolution.u64, $5::integer)",
|
||||
p, timeout);
|
||||
if (!rows)
|
||||
return std::unexpected{classify(rows.error())};
|
||||
const auto granted = rows->size() == 1 && (*rows)[0].size() == 1
|
||||
? parse_integer<std::int64_t>((*rows)[0][0])
|
||||
: std::nullopt;
|
||||
if (!granted)
|
||||
return std::unexpected{malformed(true)};
|
||||
return milliseconds{*granted};
|
||||
}
|
||||
|
||||
std::expected<void, LeaseStoreFailure>
|
||||
release_region_lease(Postgres& db, RegionKey key, const LeaseGrant& lease, milliseconds timeout) {
|
||||
const std::array<std::string, 4> p{std::to_string(key.world), std::to_string(key.region),
|
||||
uuid_text(lease.holder), std::to_string(lease.epoch)};
|
||||
auto rows = db.query("SELECT evolution.release_region_lease($1::evolution.u64, "
|
||||
"$2::evolution.u64, $3::uuid, $4::evolution.u64)",
|
||||
p, timeout);
|
||||
if (!rows)
|
||||
return std::unexpected{classify(rows.error())};
|
||||
if (rows->size() != 1)
|
||||
return std::unexpected{malformed(true)};
|
||||
return {};
|
||||
}
|
||||
|
||||
std::expected<RegionHead, LeaseStoreFailure> load_region_head(Postgres& db, RegionKey key,
|
||||
milliseconds timeout) {
|
||||
const std::array<std::string, 2> p{std::to_string(key.world), std::to_string(key.region)};
|
||||
auto rows =
|
||||
db.query("SELECT * FROM evolution.load_region_head($1::evolution.u64, $2::evolution.u64)",
|
||||
p, timeout);
|
||||
if (!rows)
|
||||
return std::unexpected{classify(rows.error())};
|
||||
auto head = parse_head(*rows);
|
||||
if (!head)
|
||||
return std::unexpected{malformed(false)};
|
||||
return *head;
|
||||
}
|
||||
|
||||
std::expected<RegionHead, LeaseStoreFailure>
|
||||
advance_region_head(Postgres& db, RegionKey key, const LeaseGrant& lease,
|
||||
const RegionHead& expected, const HeadAdvance& next, milliseconds timeout) {
|
||||
const std::array<std::string, 10> p{std::to_string(key.world),
|
||||
std::to_string(key.region),
|
||||
uuid_text(lease.holder),
|
||||
std::to_string(lease.epoch),
|
||||
std::to_string(expected.sequence),
|
||||
uuid_text(expected.commit),
|
||||
bytea(expected.digest),
|
||||
uuid_text(next.commit),
|
||||
bytea(next.digest),
|
||||
std::to_string(next.end_tick)};
|
||||
auto rows = db.query("SELECT * FROM evolution.advance_region_head($1::evolution.u64, "
|
||||
"$2::evolution.u64, $3::uuid, $4::evolution.u64, $5::evolution.u64, "
|
||||
"$6::uuid, $7::bytea, $8::uuid, $9::bytea, $10::evolution.u64)",
|
||||
p, timeout);
|
||||
if (!rows)
|
||||
return std::unexpected{classify(rows.error())};
|
||||
auto head = parse_head(*rows);
|
||||
if (!head)
|
||||
return std::unexpected{malformed(true)};
|
||||
return *head;
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
@@ -3,6 +3,7 @@
|
||||
#include <cstdlib>
|
||||
#include <source_location>
|
||||
#include <stdexcept>
|
||||
void run_region_lease_store_tests();
|
||||
namespace {
|
||||
using namespace evo::persistence;
|
||||
using namespace std::chrono_literals;
|
||||
@@ -69,6 +70,8 @@ int main() {
|
||||
try {
|
||||
run();
|
||||
std::puts("postgres adapter integration passed");
|
||||
run_region_lease_store_tests();
|
||||
std::puts("region lease fencing integration passed");
|
||||
return 0;
|
||||
} catch (const std::exception& e) {
|
||||
std::fprintf(stderr, "%s\n", e.what());
|
||||
|
||||
@@ -0,0 +1,413 @@
|
||||
// Real PostgreSQL M7.3 fencing checks (ADR-0060). Run by tools/ci/postgres-check.sh.
|
||||
#include "evo/persistence/region_lease_store.hpp"
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <chrono>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <latch>
|
||||
#include <memory>
|
||||
#include <source_location>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
void run_region_lease_store_tests();
|
||||
|
||||
namespace {
|
||||
using namespace evo::persistence;
|
||||
using namespace std::chrono_literals;
|
||||
using std::chrono::milliseconds;
|
||||
void check(bool value, std::source_location at = std::source_location::current()) {
|
||||
if (!value)
|
||||
throw std::runtime_error("region lease store test line " + std::to_string(at.line()));
|
||||
}
|
||||
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
||||
std::string env(const char* key) {
|
||||
const auto* value = std::getenv(key);
|
||||
if (!value)
|
||||
throw std::runtime_error("missing test environment");
|
||||
return value;
|
||||
}
|
||||
std::unique_ptr<Postgres> connect_as(const std::string& user) {
|
||||
PgOptions options{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}};
|
||||
auto made = Postgres::connect(options, 5s);
|
||||
CHECK(made);
|
||||
return std::move(*made);
|
||||
}
|
||||
std::unique_ptr<Postgres> owner() {
|
||||
return connect_as(env("PGUSER"));
|
||||
}
|
||||
std::unique_ptr<Postgres> runtime() {
|
||||
return connect_as(env("EVO_PG_RUNTIME_USER"));
|
||||
}
|
||||
Uuid id(std::uint8_t seed) {
|
||||
Uuid out{};
|
||||
out.fill(seed);
|
||||
return out;
|
||||
}
|
||||
Sha256Digest digest(std::uint8_t seed) {
|
||||
Sha256Digest out{};
|
||||
out.fill(seed);
|
||||
return out;
|
||||
}
|
||||
std::string hex_text(std::uint8_t seed, std::size_t bytes) {
|
||||
constexpr char digits[] = "0123456789abcdef";
|
||||
std::string out;
|
||||
for (std::size_t i = 0; i < bytes; ++i) {
|
||||
out.push_back(digits[seed >> 4U]);
|
||||
out.push_back(digits[seed & 0x0FU]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
std::string uuid_literal(std::uint8_t seed) {
|
||||
const auto h = hex_text(seed, 16);
|
||||
return h.substr(0, 8) + '-' + h.substr(8, 4) + '-' + h.substr(12, 4) + '-' + h.substr(16, 4) +
|
||||
'-' + h.substr(20);
|
||||
}
|
||||
// Genesis commit/digest bytes are the region number; frame is 7.
|
||||
RegionKey provision(Postgres& db, std::uint64_t region) {
|
||||
const auto seed = static_cast<std::uint8_t>(region);
|
||||
const std::array<std::string, 4> p{std::to_string(region), uuid_literal(seed),
|
||||
"\\x" + hex_text(seed, 32), "7"};
|
||||
CHECK(db.query("SELECT evolution.provision_region(1, $1::evolution.u64, $4::evolution.u64, "
|
||||
"$2::uuid, $3::bytea)",
|
||||
p, 5s));
|
||||
return {1, region};
|
||||
}
|
||||
bool failed(const auto& result, LeaseStoreError kind) {
|
||||
return !result && result.error().kind == kind;
|
||||
}
|
||||
std::string sqlstate(const std::expected<PgRows, PgFailure>& result) {
|
||||
return result ? std::string{} : std::string(result.error().sqlstate.data());
|
||||
}
|
||||
|
||||
void runtime_cannot_bypass_the_guard() {
|
||||
auto admin = owner();
|
||||
const auto key = provision(*admin, 10);
|
||||
auto db = runtime();
|
||||
for (const auto* sql : {
|
||||
"UPDATE evolution.region_leases SET holder = NULL",
|
||||
"UPDATE evolution.region_heads SET commit_sequence = 5",
|
||||
"DELETE FROM evolution.region_heads",
|
||||
"INSERT INTO evolution.region_leases(world_id,region_id,frame_id) VALUES (9,9,9)",
|
||||
"SELECT * FROM evolution.region_leases",
|
||||
"SELECT * FROM evolution.region_heads",
|
||||
"SELECT evolution.provision_region(1, 11, 7, gen_random_uuid(), '\\x00')",
|
||||
"SELECT evolution.check_region_lease(1, 10, gen_random_uuid(), 1)",
|
||||
"CREATE FUNCTION evolution.bypass() RETURNS int LANGUAGE sql AS 'SELECT 1'",
|
||||
"CREATE TABLE evolution.bypass(x int)",
|
||||
})
|
||||
CHECK(sqlstate(db->query(sql, {}, 5s)) == "42501");
|
||||
const std::array<std::string, 1> owner_name{env("PGUSER")};
|
||||
CHECK(!db->query("SELECT pg_catalog.set_config('role', $1, false)", owner_name, 5s));
|
||||
// A SECURITY DEFINER function ignores a hostile caller search_path.
|
||||
CHECK(db->query("SET search_path = pg_temp, public", {}, 5s));
|
||||
CHECK(db->query("CREATE TEMP TABLE region_leases(x int)", {}, 5s));
|
||||
auto lease = acquire_region_lease(*db, key, 20000ms, 5s);
|
||||
CHECK(lease && lease->epoch == 1 && lease->frame == 7);
|
||||
CHECK(release_region_lease(*db, key, *lease, 5s));
|
||||
}
|
||||
|
||||
void acquire_renew_release_and_advance() {
|
||||
auto admin = owner();
|
||||
const auto key = provision(*admin, 20);
|
||||
auto db = runtime();
|
||||
CHECK(
|
||||
failed(acquire_region_lease(*db, {1, 999}, 20000ms, 5s), LeaseStoreError::not_provisioned));
|
||||
CHECK(failed(acquire_region_lease(*db, key, 999ms, 5s), LeaseStoreError::invalid));
|
||||
CHECK(failed(acquire_region_lease(*db, key, 60001ms, 5s), LeaseStoreError::invalid));
|
||||
auto lease = acquire_region_lease(*db, key, 20000ms, 5s);
|
||||
CHECK(lease && lease->epoch == 1 && lease->duration == 20000ms && lease->holder != Uuid{});
|
||||
CHECK(failed(acquire_region_lease(*db, key, 20000ms, 5s), LeaseStoreError::busy));
|
||||
auto renewed = renew_region_lease(*db, key, *lease, 20000ms, 5s);
|
||||
CHECK(renewed && *renewed == 20000ms);
|
||||
auto forged = *lease;
|
||||
forged.holder[0] ^= 1U;
|
||||
CHECK(failed(renew_region_lease(*db, key, forged, 20000ms, 5s), LeaseStoreError::fenced));
|
||||
auto stale_epoch = *lease;
|
||||
stale_epoch.epoch = 2;
|
||||
CHECK(failed(renew_region_lease(*db, key, stale_epoch, 20000ms, 5s), LeaseStoreError::fenced));
|
||||
|
||||
auto genesis = load_region_head(*db, key, 5s);
|
||||
CHECK(genesis && genesis->sequence == 0 && genesis->end_tick == 0 && genesis->frame == 7);
|
||||
CHECK(genesis->commit == id(20) && genesis->digest == digest(20));
|
||||
CHECK(failed(advance_region_head(*db, key, *lease, *genesis, {id(21), digest(21), 65}, 5s),
|
||||
LeaseStoreError::invalid));
|
||||
CHECK(failed(advance_region_head(*db, key, *lease, *genesis, {id(20), digest(21), 1}, 5s),
|
||||
LeaseStoreError::invalid));
|
||||
const HeadAdvance first{id(21), digest(21), 64};
|
||||
auto head = advance_region_head(*db, key, *lease, *genesis, first, 5s);
|
||||
CHECK(head && head->sequence == 1 && head->end_tick == 64 && head->commit == id(21));
|
||||
CHECK(failed(advance_region_head(*db, key, forged, *head, {id(22), digest(22), 65}, 5s),
|
||||
LeaseStoreError::fenced));
|
||||
// Exact retry after an applied advance: mismatch, then the head proves it committed.
|
||||
CHECK(failed(advance_region_head(*db, key, *lease, *genesis, first, 5s),
|
||||
LeaseStoreError::head_mismatch));
|
||||
auto current = load_region_head(*db, key, 5s);
|
||||
CHECK(current && *current == *head);
|
||||
auto changed = *head;
|
||||
changed.digest[31] ^= 1U;
|
||||
CHECK(failed(advance_region_head(*db, key, *lease, changed, {id(22), digest(22), 65}, 5s),
|
||||
LeaseStoreError::head_mismatch));
|
||||
CHECK(failed(advance_region_head(*db, key, *lease, *head, {id(22), digest(22), 64}, 5s),
|
||||
LeaseStoreError::invalid));
|
||||
|
||||
CHECK(release_region_lease(*db, key, *lease, 5s));
|
||||
CHECK(failed(release_region_lease(*db, key, *lease, 5s), LeaseStoreError::fenced));
|
||||
CHECK(failed(renew_region_lease(*db, key, *lease, 20000ms, 5s), LeaseStoreError::fenced));
|
||||
CHECK(failed(advance_region_head(*db, key, *lease, *head, {id(22), digest(22), 65}, 5s),
|
||||
LeaseStoreError::fenced));
|
||||
auto next = acquire_region_lease(*db, key, 20000ms, 5s);
|
||||
CHECK(next && next->epoch == 2 && next->holder != lease->holder);
|
||||
auto final_head = advance_region_head(*db, key, *next, *head, {id(22), digest(22), 65}, 5s);
|
||||
CHECK(final_head && final_head->sequence == 2);
|
||||
}
|
||||
|
||||
void database_time_expiry_and_paused_writer() {
|
||||
auto admin = owner();
|
||||
const auto key = provision(*admin, 30);
|
||||
auto worker_a = runtime();
|
||||
auto worker_b = runtime();
|
||||
const LeasePolicy fast{1000ms, 200ms, 100ms, 100ms, 300ms};
|
||||
LeaseAuthority a{fast};
|
||||
auto sent = LeaseClock::now();
|
||||
auto lease_a = acquire_region_lease(*worker_a, key, fast.duration, 5s);
|
||||
CHECK(lease_a && a.acquired(*lease_a, sent, LeaseClock::now()));
|
||||
CHECK(failed(acquire_region_lease(*worker_b, key, 20000ms, 5s), LeaseStoreError::busy));
|
||||
auto genesis = load_region_head(*worker_a, key, 5s);
|
||||
CHECK(genesis);
|
||||
|
||||
std::this_thread::sleep_for(1200ms); // Worker A is paused past its lease.
|
||||
CHECK(a.state(LeaseClock::now()) == LeaseState::fenced);
|
||||
// Expiry is decided by database time; the successor acquires a newer epoch.
|
||||
auto lease_b = acquire_region_lease(*worker_b, key, 20000ms, 5s);
|
||||
CHECK(lease_b && lease_b->epoch == lease_a->epoch + 1);
|
||||
auto head_b =
|
||||
advance_region_head(*worker_b, key, *lease_b, *genesis, {id(31), digest(31), 3}, 5s);
|
||||
CHECK(head_b);
|
||||
// A resumes and ignores its local fence: every database path rejects it.
|
||||
CHECK(failed(renew_region_lease(*worker_a, key, *lease_a, fast.duration, 5s),
|
||||
LeaseStoreError::fenced));
|
||||
CHECK(
|
||||
failed(advance_region_head(*worker_a, key, *lease_a, *genesis, {id(32), digest(32), 1}, 5s),
|
||||
LeaseStoreError::fenced));
|
||||
CHECK(
|
||||
failed(advance_region_head(*worker_a, key, *lease_a, *head_b, {id(32), digest(32), 4}, 5s),
|
||||
LeaseStoreError::fenced));
|
||||
CHECK(failed(release_region_lease(*worker_a, key, *lease_a, 5s), LeaseStoreError::fenced));
|
||||
auto head = load_region_head(*worker_a, key, 5s);
|
||||
CHECK(head && *head == *head_b);
|
||||
|
||||
// Expired renewal cannot revive a lease even without a successor.
|
||||
CHECK(release_region_lease(*worker_b, key, *lease_b, 5s));
|
||||
auto lease_c = acquire_region_lease(*worker_a, key, 1000ms, 5s);
|
||||
CHECK(lease_c && lease_c->epoch == lease_b->epoch + 1);
|
||||
std::this_thread::sleep_for(1200ms);
|
||||
CHECK(
|
||||
failed(renew_region_lease(*worker_a, key, *lease_c, 1000ms, 5s), LeaseStoreError::fenced));
|
||||
}
|
||||
|
||||
// A writer queued on the lease row lock is judged by the clock after the wait,
|
||||
// not its statement start time, so a lease that expired while queued is fenced.
|
||||
void queued_writer_uses_post_lock_database_time() {
|
||||
auto admin = owner();
|
||||
const auto key = provision(*admin, 40);
|
||||
auto worker = runtime();
|
||||
auto lease = acquire_region_lease(*worker, key, 1000ms, 5s);
|
||||
auto genesis = load_region_head(*worker, key, 5s);
|
||||
CHECK(lease && genesis);
|
||||
auto blocker = owner();
|
||||
const std::array<std::string, 1> region{"40"};
|
||||
CHECK(blocker->query("BEGIN", {}, 5s));
|
||||
CHECK(blocker->query("SELECT 1 FROM evolution.region_leases WHERE world_id = 1 AND "
|
||||
"region_id = $1::evolution.u64 FOR UPDATE",
|
||||
region, 5s));
|
||||
std::expected<RegionHead, LeaseStoreFailure> queued = std::unexpected{LeaseStoreFailure{}};
|
||||
std::thread writer([&] {
|
||||
queued = advance_region_head(*worker, key, *lease, *genesis, {id(41), digest(41), 1}, 5s);
|
||||
});
|
||||
std::this_thread::sleep_for(1300ms);
|
||||
CHECK(blocker->query("ROLLBACK", {}, 5s));
|
||||
writer.join();
|
||||
CHECK(failed(queued, LeaseStoreError::fenced));
|
||||
auto head = load_region_head(*worker, key, 5s);
|
||||
CHECK(head && *head == *genesis);
|
||||
}
|
||||
|
||||
// Partition: the renewal's outcome is unknown and the connection is gone. The
|
||||
// holder keeps its earlier deadline and fences itself before a successor starts.
|
||||
void partitioned_renewal_fences_locally() {
|
||||
auto admin = owner();
|
||||
const auto key = provision(*admin, 50);
|
||||
auto worker = runtime();
|
||||
const LeasePolicy fast{1000ms, 200ms, 100ms, 100ms, 300ms};
|
||||
LeaseAuthority authority{fast};
|
||||
const auto sent = LeaseClock::now();
|
||||
auto lease = acquire_region_lease(*worker, key, fast.duration, 5s);
|
||||
CHECK(lease && authority.acquired(*lease, sent, LeaseClock::now()));
|
||||
auto blocker = owner();
|
||||
const std::array<std::string, 1> region{"50"};
|
||||
CHECK(blocker->query("BEGIN", {}, 5s));
|
||||
CHECK(blocker->query("SELECT 1 FROM evolution.region_leases WHERE world_id = 1 AND "
|
||||
"region_id = $1::evolution.u64 FOR UPDATE",
|
||||
region, 5s));
|
||||
const auto renew_sent = LeaseClock::now();
|
||||
auto renewal = renew_region_lease(*worker, key, *lease, fast.duration, fast.renew_timeout);
|
||||
CHECK(failed(renewal, LeaseStoreError::unknown));
|
||||
authority.renewal_unknown(fast.duration, renew_sent);
|
||||
const auto local_deadline = authority.deadline();
|
||||
CHECK(local_deadline <= sent + fast.duration - fast.safety_margin);
|
||||
std::this_thread::sleep_for(1300ms);
|
||||
CHECK(!authority.may_originate(LeaseClock::now()));
|
||||
CHECK(blocker->query("ROLLBACK", {}, 5s));
|
||||
// The orphaned renewal reached the row only after database expiry: rejected.
|
||||
auto successor = runtime();
|
||||
auto next = acquire_region_lease(*successor, key, 20000ms, 5s);
|
||||
CHECK(next && next->epoch == lease->epoch + 1);
|
||||
}
|
||||
|
||||
void contenders_serialize_on_the_database() {
|
||||
auto admin = owner();
|
||||
const auto key = provision(*admin, 60);
|
||||
std::array<std::unique_ptr<Postgres>, 2> workers{runtime(), runtime()};
|
||||
std::uint64_t previous_epoch = 0;
|
||||
constexpr int rounds = 64;
|
||||
for (int round = 0; round < rounds; ++round) {
|
||||
std::array<std::expected<LeaseGrant, LeaseStoreFailure>, 2> results{
|
||||
std::unexpected{LeaseStoreFailure{}}, std::unexpected{LeaseStoreFailure{}}};
|
||||
std::latch start{2};
|
||||
std::vector<std::thread> threads;
|
||||
threads.reserve(2);
|
||||
for (std::size_t i = 0; i < 2; ++i)
|
||||
threads.emplace_back([&, i] {
|
||||
start.arrive_and_wait();
|
||||
results[i] = acquire_region_lease(*workers[i], key, 20000ms, 5s);
|
||||
});
|
||||
for (auto& t : threads)
|
||||
t.join();
|
||||
const auto winners = std::count_if(results.begin(), results.end(),
|
||||
[](const auto& r) { return r.has_value(); });
|
||||
CHECK(winners == 1);
|
||||
const std::size_t w = results[0] ? 0 : 1;
|
||||
CHECK(failed(results[1 - w], LeaseStoreError::busy));
|
||||
CHECK(results[w]->epoch == previous_epoch + 1);
|
||||
previous_epoch = results[w]->epoch;
|
||||
CHECK(release_region_lease(*workers[w], key, *results[w], 5s));
|
||||
}
|
||||
|
||||
// The same holder racing two commits from the same predecessor: exactly one wins.
|
||||
auto lease = acquire_region_lease(*workers[0], key, 20000ms, 5s);
|
||||
CHECK(lease);
|
||||
auto head = load_region_head(*workers[0], key, 5s);
|
||||
CHECK(head);
|
||||
for (std::uint8_t round = 0; round < 32; ++round) {
|
||||
std::array<std::expected<RegionHead, LeaseStoreFailure>, 2> results{
|
||||
std::unexpected{LeaseStoreFailure{}}, std::unexpected{LeaseStoreFailure{}}};
|
||||
std::latch start{2};
|
||||
std::vector<std::thread> threads;
|
||||
threads.reserve(2);
|
||||
for (std::size_t i = 0; i < 2; ++i)
|
||||
threads.emplace_back([&, i] {
|
||||
const auto seed = static_cast<std::uint8_t>(100 + (round * 2) + i);
|
||||
start.arrive_and_wait();
|
||||
results[i] = advance_region_head(*workers[i], key, *lease, *head,
|
||||
{id(seed), digest(seed), head->end_tick + 1}, 5s);
|
||||
});
|
||||
for (auto& t : threads)
|
||||
t.join();
|
||||
const std::size_t w = results[0] ? 0 : 1;
|
||||
CHECK(results[w] && failed(results[1 - w], LeaseStoreError::head_mismatch));
|
||||
CHECK(results[w]->sequence == head->sequence + 1);
|
||||
head = *results[w];
|
||||
}
|
||||
auto committed = load_region_head(*workers[1], key, 5s);
|
||||
CHECK(committed && *committed == *head && committed->sequence == 32);
|
||||
}
|
||||
|
||||
void exhaustion_fails_closed() {
|
||||
auto admin = owner();
|
||||
const auto key = provision(*admin, 70);
|
||||
CHECK(admin->query("UPDATE evolution.region_leases SET epoch = 18446744073709551615 "
|
||||
"WHERE world_id = 1 AND region_id = 70",
|
||||
{}, 5s));
|
||||
auto db = runtime();
|
||||
CHECK(failed(acquire_region_lease(*db, key, 20000ms, 5s), LeaseStoreError::exhausted));
|
||||
const auto other = provision(*admin, 71);
|
||||
auto lease = acquire_region_lease(*db, other, 20000ms, 5s);
|
||||
CHECK(lease);
|
||||
CHECK(admin->query("UPDATE evolution.region_heads SET commit_sequence = 18446744073709551615, "
|
||||
"end_tick = 5, published_epoch = 1 WHERE world_id = 1 AND region_id = 71",
|
||||
{}, 5s));
|
||||
auto head = load_region_head(*db, other, 5s);
|
||||
CHECK(head && head->sequence == 18446744073709551615ULL);
|
||||
CHECK(failed(advance_region_head(*db, other, *lease, *head, {id(72), digest(72), 6}, 5s),
|
||||
LeaseStoreError::exhausted));
|
||||
}
|
||||
|
||||
struct Summary {
|
||||
std::vector<long long> samples;
|
||||
void print(const char* name) {
|
||||
std::sort(samples.begin(), samples.end());
|
||||
const auto pick = [&](std::size_t percent) {
|
||||
return samples[std::min(samples.size() - 1, samples.size() * percent / 100)];
|
||||
};
|
||||
std::printf("lease_latency_us %s n=%zu p50=%lld p99=%lld max=%lld\n", name, samples.size(),
|
||||
pick(50), pick(99), samples.back());
|
||||
}
|
||||
};
|
||||
template <typename F> long long timed(F&& f) {
|
||||
const auto begin = std::chrono::steady_clock::now();
|
||||
f();
|
||||
return std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() -
|
||||
begin)
|
||||
.count();
|
||||
}
|
||||
|
||||
// Local Unix-socket round trips only; remote TLS latency must be measured in place.
|
||||
void measure_round_trips() {
|
||||
auto admin = owner();
|
||||
const auto key = provision(*admin, 80);
|
||||
auto db = runtime();
|
||||
constexpr int n = 200;
|
||||
Summary acquire, renew, advance, release;
|
||||
auto head = load_region_head(*db, key, 5s);
|
||||
CHECK(head);
|
||||
for (int i = 0; i < n; ++i) {
|
||||
std::expected<LeaseGrant, LeaseStoreFailure> lease = std::unexpected{LeaseStoreFailure{}};
|
||||
acquire.samples.push_back(
|
||||
timed([&] { lease = acquire_region_lease(*db, key, 20000ms, 5s); }));
|
||||
CHECK(lease);
|
||||
renew.samples.push_back(
|
||||
timed([&] { CHECK(renew_region_lease(*db, key, *lease, 20000ms, 5s)); }));
|
||||
const auto seed = static_cast<std::uint8_t>(i);
|
||||
Uuid commit = id(seed);
|
||||
commit[0] = 0xAA; // Distinct from genesis 0x50 bytes.
|
||||
commit[1] = static_cast<std::uint8_t>(i >> 8);
|
||||
advance.samples.push_back(timed([&] {
|
||||
auto next = advance_region_head(*db, key, *lease, *head,
|
||||
{commit, digest(seed), head->end_tick + 64}, 5s);
|
||||
CHECK(next);
|
||||
head = *next;
|
||||
}));
|
||||
release.samples.push_back(
|
||||
timed([&] { CHECK(release_region_lease(*db, key, *lease, 5s)); }));
|
||||
}
|
||||
acquire.print("acquire");
|
||||
renew.print("renew");
|
||||
advance.print("advance_head");
|
||||
release.print("release");
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void run_region_lease_store_tests() {
|
||||
runtime_cannot_bypass_the_guard();
|
||||
acquire_renew_release_and_advance();
|
||||
database_time_expiry_and_paused_writer();
|
||||
queued_writer_uses_post_lock_database_time();
|
||||
partitioned_renewal_fences_locally();
|
||||
contenders_serialize_on_the_database();
|
||||
exhaustion_fails_closed();
|
||||
measure_round_trips();
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
#include "evo/persistence/region_lease.hpp"
|
||||
|
||||
#include <cstdio>
|
||||
#include <source_location>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
|
||||
namespace {
|
||||
using namespace evo::persistence;
|
||||
using namespace std::chrono_literals;
|
||||
using Time = LeaseClock::time_point;
|
||||
void check(bool ok, std::source_location at = std::source_location::current()) {
|
||||
if (!ok)
|
||||
throw std::runtime_error("region lease line " + std::to_string(at.line()));
|
||||
}
|
||||
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
||||
constexpr Time at(std::chrono::milliseconds t) {
|
||||
return Time{t};
|
||||
}
|
||||
LeaseGrant grant(std::uint64_t epoch) {
|
||||
return {{1}, epoch, 7, LeasePolicy{}.duration};
|
||||
}
|
||||
|
||||
void policies() {
|
||||
CHECK(valid(LeasePolicy{}));
|
||||
CHECK(valid(LeasePolicy{1000ms, 200ms, 100ms, 100ms, 300ms}));
|
||||
const auto invalid = [](auto change) {
|
||||
LeasePolicy p;
|
||||
change(p);
|
||||
return !valid(p);
|
||||
};
|
||||
CHECK(invalid([](LeasePolicy& p) { p.duration = 999ms; }));
|
||||
CHECK(invalid([](LeasePolicy& p) { p.duration = 60001ms; }));
|
||||
CHECK(invalid([](LeasePolicy& p) { p.safety_margin = 1999ms; })); // under 10%
|
||||
CHECK(invalid([](LeasePolicy& p) { p.safety_margin = 20000ms; }));
|
||||
CHECK(invalid([](LeasePolicy& p) { p.renew_interval = 0ms; }));
|
||||
CHECK(invalid([](LeasePolicy& p) { p.renew_timeout = -1ms; }));
|
||||
CHECK(invalid([](LeasePolicy& p) { p.renew_timeout = 30001ms; }));
|
||||
CHECK(invalid([](LeasePolicy& p) { p.commit_timeout = 18000ms; }));
|
||||
// Two complete renewal attempts must fit before local fencing: 2*(7+2.5) > 18.
|
||||
CHECK(invalid([](LeasePolicy& p) {
|
||||
p.renew_interval = 7000ms;
|
||||
p.renew_timeout = 2500ms;
|
||||
}));
|
||||
CHECK(!valid(LeasePolicy{2000ms, 400ms, 200ms, 200ms, 1800ms}));
|
||||
LeaseAuthority broken(LeasePolicy{999ms, 200ms, 100ms, 100ms, 300ms});
|
||||
CHECK(!broken.acquired({{1}, 1, 7, 999ms}, at(0ms), at(1ms)));
|
||||
CHECK(broken.state(at(1ms)) == LeaseState::absent);
|
||||
}
|
||||
|
||||
void deadlines_follow_send_time() {
|
||||
LeaseAuthority a{LeasePolicy{}};
|
||||
CHECK(a.state(at(0ms)) == LeaseState::absent && !a.may_originate(at(0ms)));
|
||||
// Reply arrival does not move the deadline: the DB clock started after sending.
|
||||
CHECK(a.acquired(grant(1), at(1000ms), at(4000ms)));
|
||||
CHECK(a.deadline() == at(19000ms));
|
||||
CHECK(a.state(at(5999ms)) == LeaseState::active);
|
||||
CHECK(a.state(at(6000ms)) == LeaseState::renew_due && a.may_originate(at(6000ms)));
|
||||
CHECK(a.may_begin_commit(at(14000ms)) && !a.may_begin_commit(at(14001ms)));
|
||||
CHECK(a.may_originate(at(18999ms)));
|
||||
CHECK(a.state(at(19000ms)) == LeaseState::fenced && !a.may_originate(at(19000ms)));
|
||||
// A renewal answered after the local deadline cannot revive authority.
|
||||
CHECK(!a.renewed(1, 20000ms, at(18000ms), at(19000ms)));
|
||||
CHECK(a.state(at(18500ms)) == LeaseState::fenced);
|
||||
}
|
||||
|
||||
void renewals() {
|
||||
LeaseAuthority a{LeasePolicy{}};
|
||||
CHECK(a.acquired(grant(4), at(0ms), at(10ms)));
|
||||
CHECK(a.renewed(4, 20000ms, at(5000ms), at(5100ms)));
|
||||
CHECK(a.deadline() == at(23000ms) && a.state(at(9999ms)) == LeaseState::active);
|
||||
CHECK(a.state(at(10000ms)) == LeaseState::renew_due);
|
||||
// Ambiguous renewal keeps the earlier bound; retries still fit before fencing.
|
||||
a.renewal_unknown(20000ms, at(10000ms));
|
||||
CHECK(a.deadline() == at(23000ms) && a.state(at(14999ms)) == LeaseState::active);
|
||||
a.renewal_unknown(5000ms, at(15000ms)); // Could have shortened: take the earlier.
|
||||
CHECK(a.deadline() == at(18000ms));
|
||||
CHECK(a.renewed(4, 20000ms, at(16000ms), at(16500ms)) && a.deadline() == at(34000ms));
|
||||
CHECK(!a.renewed(4, 20000ms, at(15000ms), at(16600ms))); // Out of order: fence.
|
||||
CHECK(a.state(at(16600ms)) == LeaseState::fenced);
|
||||
CHECK(!a.renewed(4, 20000ms, at(17000ms), at(17001ms))); // Sticky.
|
||||
|
||||
LeaseAuthority b{LeasePolicy{}};
|
||||
CHECK(b.acquired(grant(2), at(0ms), at(1ms)));
|
||||
CHECK(!b.renewed(3, 20000ms, at(1000ms), at(1001ms))); // Foreign epoch.
|
||||
CHECK(b.state(at(1001ms)) == LeaseState::fenced);
|
||||
LeaseAuthority c{LeasePolicy{}};
|
||||
CHECK(c.acquired(grant(2), at(0ms), at(1ms)));
|
||||
CHECK(!c.renewed(2, 19000ms, at(1000ms), at(1001ms))); // Unexpected duration.
|
||||
LeaseAuthority d{LeasePolicy{}};
|
||||
CHECK(d.acquired(grant(2), at(0ms), at(1ms)));
|
||||
d.fence();
|
||||
CHECK(d.state(at(2ms)) == LeaseState::fenced && !d.may_begin_commit(at(2ms)));
|
||||
}
|
||||
|
||||
void epochs_are_never_reused() {
|
||||
LeaseAuthority a{LeasePolicy{}};
|
||||
CHECK(!a.acquired(grant(0), at(0ms), at(1ms)));
|
||||
CHECK(!a.acquired({{1}, 1, 0, 20000ms}, at(0ms), at(1ms))); // Zero frame.
|
||||
CHECK(!a.acquired({{1}, 1, 7, 10000ms}, at(0ms), at(1ms))); // Not the policy.
|
||||
CHECK(!a.acquired(grant(1), at(10ms), at(9ms))); // Reply before send.
|
||||
CHECK(!a.acquired(grant(1), at(0ms), at(18000ms))); // Too late to use.
|
||||
CHECK(a.acquired(grant(3), at(0ms), at(1ms)));
|
||||
CHECK(!a.acquired(grant(3), at(2ms), at(3ms)));
|
||||
CHECK(a.state(at(3ms)) == LeaseState::fenced);
|
||||
CHECK(!a.acquired(grant(2), at(4ms), at(5ms)));
|
||||
// Only a strictly newer database epoch clears fencing (followed by recovery).
|
||||
CHECK(a.acquired(grant(4), at(6ms), at(7ms)) && a.state(at(7ms)) == LeaseState::active);
|
||||
const auto& current = a.grant();
|
||||
CHECK(current && current->epoch == 4);
|
||||
}
|
||||
|
||||
void clock_is_monotonic() {
|
||||
const auto first = LeaseClock::now();
|
||||
const auto second = LeaseClock::now();
|
||||
CHECK(second >= first && first.time_since_epoch().count() > 0);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
try {
|
||||
policies();
|
||||
deadlines_follow_send_time();
|
||||
renewals();
|
||||
epochs_are_never_reused();
|
||||
clock_is_monotonic();
|
||||
std::puts("region lease contracts passed");
|
||||
return 0;
|
||||
} catch (const std::exception& e) {
|
||||
std::fprintf(stderr, "%s\n", e.what());
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
@@ -11,10 +11,15 @@ unset PGHOSTADDR PGPASSWORD PGSERVICE PGSERVICEFILE
|
||||
initdb -D "$test_dir/data" --no-locale --encoding=UTF8 --auth-local=trust --auth-host=reject > "$test_dir/init.log"
|
||||
trap 'pg_ctl -D "$test_dir/data" -m immediate stop > /dev/null || true' EXIT
|
||||
pg_ctl -D "$test_dir/data" -l "$test_dir/server.log" -o "-k $test_dir -p 55432 -c listen_addresses=''" -w start
|
||||
psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_runtime_test NOLOGIN'
|
||||
# Runtime group role receives grants; a separate login inherits only that role.
|
||||
psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_runtime_test NOLOGIN' \
|
||||
-c 'CREATE ROLE evo_runtime_login LOGIN IN ROLE evo_runtime_test'
|
||||
export EVO_PG_RUNTIME_ROLE=evo_runtime_test EVO_PG_RUNTIME_USER=evo_runtime_login
|
||||
bash "$root/tools/migrate-postgres.sh"
|
||||
bash "$root/tools/migrate-postgres.sh" # exact supported upgrade/restart is a no-op
|
||||
"${1:?path to evo-postgres-tests}"
|
||||
v1_sha="$(sha256sum "$root/migrations/0001_region_bootstrap.sql")"
|
||||
v1_sha="${v1_sha%% *}"
|
||||
# Migration failure must not publish a partial schema.
|
||||
createdb migration_rollback
|
||||
if (export PGDATABASE=migration_rollback; psql -X -v ON_ERROR_STOP=1 -1 \
|
||||
@@ -22,9 +27,29 @@ if (export PGDATABASE=migration_rollback; psql -X -v ON_ERROR_STOP=1 -1 \
|
||||
echo 'Expected failed migration' >&2; exit 1
|
||||
fi
|
||||
test "$(PGDATABASE=migration_rollback psql -XAtc "SELECT count(*) FROM pg_namespace WHERE nspname='evolution'")" = 0
|
||||
# A privileged or owner runtime role is refused and leaves no schema behind.
|
||||
if (export PGDATABASE=migration_rollback EVO_PG_RUNTIME_ROLE="$PGUSER"; \
|
||||
bash "$root/tools/migrate-postgres.sh"); then
|
||||
echo 'Expected privileged runtime role rejection' >&2; exit 1
|
||||
fi
|
||||
if (export PGDATABASE=migration_rollback EVO_PG_RUNTIME_ROLE=no_such_evo_role; \
|
||||
bash "$root/tools/migrate-postgres.sh"); then
|
||||
echo 'Expected missing runtime role rejection' >&2; exit 1
|
||||
fi
|
||||
test "$(PGDATABASE=migration_rollback psql -XAtc "SELECT count(*) FROM pg_namespace WHERE nspname='evolution'")" = 0
|
||||
# Supported upgrade path: an existing exact v1 database migrates to v2.
|
||||
createdb upgrade_path
|
||||
PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" \
|
||||
-f "$root/migrations/0001_region_bootstrap.sql" > /dev/null
|
||||
test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1
|
||||
PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh"
|
||||
test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2
|
||||
# Unknown future versions and checksum drift must not be repaired silently.
|
||||
psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=2'
|
||||
psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=3 WHERE version=2'
|
||||
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
|
||||
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=1,sha256=repeat('0',64)"
|
||||
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=2 WHERE version=3"
|
||||
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=2"
|
||||
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
|
||||
echo 'postgres migrations, rollback, constraints and permissions passed'
|
||||
psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64)"
|
||||
if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi
|
||||
echo 'postgres migrations, rollback, constraints, permissions and fencing passed'
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
root="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
# Operator-created NOLOGIN group role granted to the region runtime login (ADR-0060).
|
||||
runtime_role="${EVO_PG_RUNTIME_ROLE:?set EVO_PG_RUNTIME_ROLE to the runtime group role}"
|
||||
if [ -n "${EVO_PGPASS_SOURCE:-}" ]; then
|
||||
umask 077
|
||||
export PGPASSFILE
|
||||
@@ -12,5 +14,8 @@ fi
|
||||
# Never print the connection or enable shell tracing here.
|
||||
digest="$(sha256sum "$root/migrations/0001_region_bootstrap.sql")"
|
||||
digest="${digest%% *}"
|
||||
digest_2="$(sha256sum "$root/migrations/0002_region_fencing.sql")"
|
||||
digest_2="${digest_2%% *}"
|
||||
psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \
|
||||
--set="migration_sha_2=$digest_2" --set="runtime_role=$runtime_role" \
|
||||
--file="$root/migrations/apply.sql"
|
||||
|
||||
Reference in New Issue
Block a user