diff --git a/CMakeLists.txt b/CMakeLists.txt index 9376d0d..91e09d0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,25 +71,39 @@ add_executable(evo-sim apps/evo-sim/main.cpp) target_link_libraries(evo-sim PRIVATE evo::simulation evo::processes) evo_owned_target(evo-sim) +# Portable host-side lease timing contract; no database or simulation dependency. +add_library(evo_region_lease STATIC src/persistence/region_lease.cpp) +add_library(evo::region_lease ALIAS evo_region_lease) +target_include_directories(evo_region_lease PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") +target_compile_features(evo_region_lease PUBLIC cxx_std_23) +evo_owned_target(evo_region_lease) + include(CTest) if(EVO_BUILD_POSTGRES) if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") message(FATAL_ERROR "The initial PostgreSQL adapter is qualified on Linux only") endif() find_package(PostgreSQL 18.6 EXACT REQUIRED) - add_library(evo_postgres STATIC src/persistence/postgres.cpp) + find_package(Threads REQUIRED) + add_library(evo_postgres STATIC src/persistence/postgres.cpp src/persistence/region_lease_store.cpp) add_library(evo::postgres ALIAS evo_postgres) target_include_directories(evo_postgres PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") - target_link_libraries(evo_postgres PRIVATE PostgreSQL::PostgreSQL) + target_link_libraries(evo_postgres PUBLIC evo::region_lease PRIVATE PostgreSQL::PostgreSQL) evo_owned_target(evo_postgres) if(BUILD_TESTING) - add_executable(evo-postgres-tests tests/postgres_tests.cpp) - target_link_libraries(evo-postgres-tests PRIVATE evo::postgres) + # Contention tests use one thread and one connection per contender. + add_executable(evo-postgres-tests tests/postgres_tests.cpp tests/region_lease_store_tests.cpp) + target_link_libraries(evo-postgres-tests PRIVATE evo::postgres Threads::Threads) evo_owned_target(evo-postgres-tests) # Intentionally not an ordinary CTest: tools/ci/postgres-check.sh owns its private DB. endif() endif() if(BUILD_TESTING) + add_executable(evo-region-lease-tests tests/region_lease_tests.cpp) + target_link_libraries(evo-region-lease-tests PRIVATE evo::region_lease) + evo_owned_target(evo-region-lease-tests) + add_test(NAME region-lease-contracts COMMAND evo-region-lease-tests) + set_tests_properties(region-lease-contracts PROPERTIES TIMEOUT 60) add_executable(evo-environmental-qualification tests/environmental_qualification.cpp) target_link_libraries(evo-environmental-qualification PRIVATE evo::processes) evo_owned_target(evo-environmental-qualification) diff --git a/deploy/README.md b/deploy/README.md index 2f5e237..e5978ad 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -132,4 +132,5 @@ Do not reuse the simulation image repository for this entrypoint. [Secret and role setup](../docs/infrastructure/POSTGRESQL.md) describes the separate migrator/runtime credentials. The example is outside active Argo resources until an operator supplies the database endpoint, role, Secret and egress rules. No region -server or world-state migration is deployed by M7.2. +server or world-state migration is deployed by M7.2. Schema v2 (M7.3) additionally +requires `EVO_PG_RUNTIME_ROLE`, naming an operator-created NOLOGIN runtime group role. diff --git a/deploy/kubernetes/postgres-migration/job.yaml.example b/deploy/kubernetes/postgres-migration/job.yaml.example index f09ce81..b643314 100644 --- a/deploy/kubernetes/postgres-migration/job.yaml.example +++ b/deploy/kubernetes/postgres-migration/job.yaml.example @@ -3,7 +3,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: evolution-db-migrate-v1 + name: evolution-db-migrate-v2 namespace: evolution-dev spec: backoffLimit: 0 @@ -34,6 +34,8 @@ spec: - {name: PGSERVICE, value: evolution-migrator} - {name: PGSERVICEFILE, value: /run/postgres/pg_service.conf} - {name: EVO_PGPASS_SOURCE, value: /run/postgres/pgpass} + # Operator-created NOLOGIN group role; the runtime login is its member. + - {name: EVO_PG_RUNTIME_ROLE, value: evolution_runtime} volumeMounts: - {name: connection, mountPath: /run/postgres, readOnly: true} - {name: scratch, mountPath: /tmp} diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 7056c56..64cbca3 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -501,3 +501,17 @@ server/libpq pair, optional persistence adapter and transactional schema v1. Private PostgreSQL integration checks passed in Release, analysis and ASan/UBSan; container/cluster verification remains separate. Secret provisioning instructions are in infrastructure/POSTGRESQL.md. Next: M7.3 fenced region ownership. + +M7.3 implemented and locally qualified (2026-09-28). ADR-0060 adds schema v2: +database-time leases, DB-generated holders, never-reused epochs and a fenced head +advance that checks lease then predecessor under one row lock. The runtime role +gets EXECUTE only; table writes, provisioning and object creation are denied. +Real-PostgreSQL tests cover the scenarios the roadmap row requires: +- 64 two-contender acquisition rounds and 32 racing advances; +- expired renewal and a paused stale writer; +- a writer queued past expiry, which is judged by post-lock database time; +- an unknown renewal that fences locally. + +Timings were selected and measured locally. The director is intentionally deferred +because it has no authority and would be a placeholder. Next: M7.4 checkpoint +codecs/publication. It must also revoke runtime EXECUTE on the bare head kernel. diff --git a/docs/STATUS.md b/docs/STATUS.md index 422f9b0..eaf69d0 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -755,3 +755,17 @@ headless suite passed 54/54. Docker/remote CI and cluster deployment are not claimed. See infrastructure/POSTGRESQL.md and engineering/VALIDATION.md. Next: M7.3 fenced ownership and narrowly granted database functions. The bootstrap lease table alone does not establish authority or durable region publication. + +## M7.3 fenced region ownership (2026-09-28) + +ADR-0060 and migration v2 add guarded lease and head functions for a runtime role +that has no table rights. The portable `LeaseAuthority` fences locally from +send-time deadlines on CLOCK_BOOTTIME. Selected timings are 20 s lease, 5 s renew, +2 s margin, 2 s renew deadline and 5 s commit deadline. Local round trips took +about 0.6-0.7 ms p50 and about 1 ms p99. Real PostgreSQL covered contenders, +expiry, paused and queued stale writers, unknown renewal, exhaustion and bypass +denial. Three SQL mutations (`now()` instead of `clock_timestamp()`, a missing +lease check, a disabled busy check) were each detected by a failing test. +Windows ran 55/55; Linux Release, clang-tidy and ASan/UBSan PostgreSQL suites +passed in WSL. No director, host, gateway, payload publication or deployment +exists yet. The v2 head must not carry real world state before M7.4. Next: M7.4. diff --git a/docs/adr/0059-postgresql-bootstrap.md b/docs/adr/0059-postgresql-bootstrap.md index 7133904..687343a 100644 --- a/docs/adr/0059-postgresql-bootstrap.md +++ b/docs/adr/0059-postgresql-bootstrap.md @@ -2,6 +2,7 @@ ## Status Accepted for M7.2, 2026-09-28. No region authority or world deployment yet. +Extended by ADR-0060 (schema v2 fencing functions and runtime grants). Supersedes ADR-0013's libpqxx recommendation for this scoped adapter. ## Context diff --git a/docs/adr/0060-fenced-region-ownership.md b/docs/adr/0060-fenced-region-ownership.md new file mode 100644 index 0000000..51a3c56 --- /dev/null +++ b/docs/adr/0060-fenced-region-ownership.md @@ -0,0 +1,99 @@ +# ADR-0060: Fenced region ownership + +## Status +Accepted for M7.3, 2026-09-28. Implements the lease/fence portion of ADR-0058 on +the ADR-0059 adapter. No region host, director, gateway or deployment exists yet. + +## Context +ADR-0058 requires that PostgreSQL arbitrate acquire/renew/commit/release under one +lease-row lock using database time, that commits compare the expected predecessor, +and that workers cannot bypass the guard. Schema v1 has only a lease table and no +runtime grants. REGION_DURABILITY.md deferred selecting lease duration, renewal +cadence, local safety margin and store deadlines to this gate. + +## Decision +Schema v2 (`migrations/0002_region_fencing.sql`) adds `region_heads` and owner-owned +functions. Every mutating function is SECURITY DEFINER with `search_path` pinned to +`pg_catalog, pg_temp`. The runtime group role gets schema USAGE and EXECUTE on +acquire, renew, release, load_region_head and advance_region_head only. It has no +table privileges. Provisioning (genesis) and the internal lease check stay owner-only. + +- Acquire locks the row, reads `clock_timestamp()` after the lock and succeeds only if + the row is released or expired. It then increments the u64 epoch and stores a + database-generated holder UUID. Epoch overflow fails closed. The holder is a + per-acquisition token, not a Pod identity. +- Renew, release and advance all go through the same row lock and require the exact + holder and epoch, with expiry in database time after the lock wait. Renewal never + revives an expired lease. Release keeps the epoch, so it is never reused. +- The advance function checks the lease, then locks the head. It requires the + expected sequence, commit ID and digest, a 1..64 tick delta and no sequence wrap. + M7.4/M7.5 publication wraps this kernel in the same transaction as the manifest, + decisions and outbox. Runtime EXECUTE on the bare kernel must then be revoked. +- Rejections use custom SQLSTATEs EVL01..EVL05 (busy, fenced, head mismatch, not + provisioned, exhausted) and 22023 for invalid arguments. A rejection aborts any + enclosing transaction. Clients never send timestamps. +- The migration requires an operator-created runtime role that is unprivileged and + does not inherit the owner. History admission is an exact prefix: clean→v1→v2, + v1→v2, or a v2 no-op. Anything else fails closed. + +Host side: the portable `LeaseAuthority` sets its local deadline to request-send time +plus the granted duration minus the safety margin. Database expiry is set no earlier +than that send, so the holder stops before a successor can acquire. An ambiguous +renewal keeps the earlier of the old deadline and the one the attempt could have +set. A reply received after the local deadline fences. Fencing is sticky until a +strictly newer epoch is acquired, which is followed by recovery. `LeaseClock` uses +Linux CLOCK_BOOTTIME so that suspend time still counts. + +Selected timings (`LeasePolicy` defaults; the database admits 1..60 s leases): + +| Setting | Value | Reason | +|---|---|---| +| Lease duration | 20 s | Bounds orphaned-lease unavailability and leaves 40 s of the proposed 60 s recovery target for load and verification | +| Renewal interval | 5 s | Two full attempts (interval + timeout) fit in the 18 s usable window | +| Safety margin | 2 s | ≥10% of duration, covering clock-rate disagreement and check-to-effect delay | +| Renewal deadline | 2 s | About three orders of magnitude above the measured local p99 (about 1 ms); drop the connection and treat the outcome as unknown | +| Commit deadline | 5 s | Below the 10 s server statement timeout; begin only if it ends before local fencing | + +`valid(LeasePolicy)` enforces these relations, not the specific numbers. Remote +TLS latency and clock-rate behavior must be measured in the target environment +before World0. These values are not a player SLA. + +The world director is deliberately not implemented. Its output would be an +assignment hint with no authority, and the database decides ownership anyway. +Adding one now would be a placeholder service. It belongs with the host (M7.6) +or later operations work. + +## Alternatives Considered +Kubernetes Leases or broker locks are not in the commit transaction. Client-supplied +holder IDs allow lost-acquire adoption but need uniqueness guarantees and let any +runtime guess tokens; DB tokens cost at most one lease period after an unknown +acquire. Transaction-start `now()` lets a writer queued on the lock commit after +expiry. Row-level security on direct table writes widens the attack surface compared +with no table grants. Status-row returns instead of SQLSTATEs would not abort a +future enclosing publication transaction. + +## Benefits +One lock serializes every ownership and head transition. The runtime login cannot +write tables, create objects or call provisioning. Stale, paused, queued and +partitioned writers are rejected by the database. The local fence has an explicit +bound instead of trusting liveness. + +## Costs +Each renewal or commit takes a row lock. After a worker crash or an unknown acquire, +the region stays unavailable for up to one lease period. The runtime role must be +pre-created by an operator. This adds a second migration and a chain of prefix checksums. + +## Risks +A frozen VM/hypervisor can stop all guest clocks. Database commits stay fenced, but +provisional output could then be late. Gateways still need bounded-validity +registration (ADR-0058). The bare head kernel accepts any 32-byte digest until M7.4 +binds it to verified payloads, so v2 must not host real world state. Custom +SQLSTATEs are project conventions; they are not PostgreSQL-reserved. + +## Consequences +The M7.4 migration must add the manifest/payload publication function that calls +the kernel, and revoke runtime EXECUTE on `advance_region_head`. M7.5 adds +commit-ID resolution. Until then, "head equals my commit ID" is only the primitive +reconciliation used in tests. M7.6 hosts must drive `LeaseAuthority`, stop admission, +simulation and output when fenced, and recover after any new acquisition. Timing +changes require updated measurements and this ADR's table. diff --git a/docs/adr/README.md b/docs/adr/README.md index db9d260..e5865f7 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -70,3 +70,4 @@ design. No unadopted candidate is a link dependency of the proof. - [ADR-0055: Environmental process contract](0055-environmental-process-contract.md) - accepted M6.5b synthetic conversions, source interval policy and process checkpoint boundary; implementation follows in M6.5c. - [ADR-0059: PostgreSQL bootstrap](0059-postgresql-bootstrap.md) - accepted for M7.2 optional adapter, pinned server/client, migrations and Secret boundary. +- [ADR-0060: Fenced region ownership](0060-fenced-region-ownership.md) - accepted for M7.3 database-time leases, guarded head advance, runtime grants and selected timings. diff --git a/docs/architecture/REGION_DURABILITY.md b/docs/architecture/REGION_DURABILITY.md index 2f25a97..9d8b14b 100644 --- a/docs/architecture/REGION_DURABILITY.md +++ b/docs/architecture/REGION_DURABILITY.md @@ -48,8 +48,10 @@ process may retain memory but cannot publish a head or originate current output. The gateway must have bounded-validity registration, not just an epoch number. M7 has no gateway implementation; readiness alone does not prove remote fencing. -Lease duration, renewal cadence, local safety margin and store deadlines must be -selected/measured at M7.3 together; no arbitrary timing values are locked here. +M7.3 selected lease duration, renewal cadence, local safety margin and store +deadlines together in ADR-0060 (20 s / 5 s / 2 s / 2 s renew / 5 s commit). The +latency behind them was measured only on a local socket. The database functions +and runtime grants in migration v2 implement the guard described here. Any uncertain lease stops admission, simulation and final output. A commit that already linearized can remain durable after its writer expires; query can return its receipt, but does not grant that writer authority to resume. @@ -159,3 +161,10 @@ resurrect acknowledged deaths. World0 stays gated. M7.1 verification is a source/state inventory and contract review. Real database, object-store, process-kill and network partition tests are still required. + +M7.3 executed its rows against real PostgreSQL. After lease expiry, a new owner +acquires a newer epoch. The old writer's renew, release and head advance are then +rejected, and the head is unchanged. A paused stale worker resuming past its lease +is fenced both locally and by the database. A writer queued on the lease lock past +expiry is also rejected. These are in-process pauses, not killed pods or network +partitions. The 7.6 parts of those rows remain open. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 99120e7..0d19009 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -2041,3 +2041,43 @@ claim lineage recovery. No retained-history or World0 requirement was relaxed. The adapter is separate from the headless kernel. Region fencing, publication, ambiguous durable commit reconciliation and crash recovery are later M7 gates. Only clean-to-v1 and exact-v1 restart are supported migration paths at this gate. + +## M7.3 fenced region ownership - 2026-09-28 + +- Windows MSVC Release configure/build/CTest passed 55/55, including the new + portable `region-lease-contracts` test. Repository clang-format 19 check passed. +- WSL Ubuntu (GCC 14.2.0; clang 19.1.1; PostgreSQL/libpq 18.6 source build): + GCC14 Release, clang19 analysis (clang-tidy warnings-as-errors) and clang19 + ASan/UBSan builds of `evo-postgres-tests` each passed `tools/ci/postgres-check.sh` + against a fresh private cluster. The first analysis build flagged two + unchecked optional accesses and two unreserved vectors. These were fixed and + all three configurations were rerun after formatting. +- Real-database coverage, as the runtime login rather than SET ROLE: + - Denied (42501): table read/write, provisioning, the internal check function, + and object creation. A hostile search_path is ignored. + - Busy, forged-holder, stale-epoch and released-lease rejections; epoch + non-reuse. + - 1..64 tick delta; predecessor mismatch; exact-retry detection via the head. + - 1 s database-time expiry and successor acquisition; paused stale writer fenced + locally and by the database. + - A writer queued 1.3 s on the lease row lock is rejected after expiry. + - A 100 ms renewal deadline under lock (unknown) fences locally before a + successor acquires. + - 64 two-thread acquisition rounds with exactly one winner and consecutive + epochs; 32 racing head advances with exactly one winner. + - Epoch and sequence exhaustion. +- Migrations tested: clean->v2, repeated v2 no-op, v1->v2 upgrade, and failed + 0001 rollback. Superuser and missing runtime roles are refused with no schema + left. Unknown v3, changed v2 checksum and changed v1 checksum are refused. +- Mutation sensitivity (scratch copies only): replacing post-lock + `clock_timestamp()` with `now()`, removing the lease check from head advance, + and disabling the busy check each made the suite fail at the expected test. +- Local Unix-socket latency, 200 samples per operation in each final run: + - p50 about 0.61-0.74 ms and p99 about 0.8-1.15 ms for acquire, renew, + advance and release. + - Max up to 2.8 ms under analysis/ASan builds. + These are local numbers, not remote TLS or cluster measurements. +- Not executed: Docker/Gitea CI, remote TLS endpoints, Kubernetes Secrets, real pod + kill, real network partition (a lock wait plus client deadline stands in for + it), VM freeze and TSan. The contention threads each own a separate + connection; libpq is uninstrumented. diff --git a/docs/infrastructure/POSTGRESQL.md b/docs/infrastructure/POSTGRESQL.md index 4aa56a0..7aaeca8 100644 --- a/docs/infrastructure/POSTGRESQL.md +++ b/docs/infrastructure/POSTGRESQL.md @@ -1,4 +1,4 @@ -# PostgreSQL M7.2 bootstrap +# PostgreSQL bootstrap (M7.2) and region fencing (M7.3) ADR-0059 pins server/libpq 18.6. `evo::postgres` is optional, Linux-only and separate from all simulation targets. Preparation explicitly downloads a verified source @@ -21,7 +21,9 @@ bash tools/ci/postgres-check.sh "$PWD/build/postgres/evo-postgres-tests" ``` The test script overrides all connection coordinates and creates a new private -0700 /tmp directory, private Unix socket and disposable local cluster. TCP is off; +0700 /tmp directory, private Unix socket and disposable local cluster. It creates +the `evo_runtime_test` group and the `evo_runtime_login` member, then runs fencing +tests as that real runtime login. TCP is off; local trust applies only inside that private directory. Cluster stops on exit; logs/data are retained there for diagnosis. It never uses production PG credentials or deletes a pre-existing database. Do not point this script at a shared server. @@ -49,18 +51,45 @@ incomplete session. No PostgreSQL diagnostic strings or connection values are lo ## Migrations and privilege boundary `tools/migrate-postgres.sh` uses psql environment/service files, no URI arguments. -It requires exactly server18.6, serializes migrations with a transaction advisory -lock, hashes the checked-in SQL, and admits clean->v1 or exact v1->v1 only. -Unknown versions/checksum changes/partial foreign schemas fail closed. No down -migration or destructive repair exists. The u64 domain rejects fractional, -negative and above-u64 values without signed truncation. +It requires exactly server 18.6 and serializes migrations with a transaction +advisory lock. It hashes each checked-in SQL file and applies everything in one +transaction. The recorded history must be an exact prefix: clean->v1->v2, v1->v2, +or a v2 no-op. Unknown versions, checksum changes and partial foreign schemas fail +closed. There is no down migration or destructive repair. The u64 domain rejects +fractional, negative and above-u64 values without signed truncation. -Use a dedicated owner/migrator login with CREATE on the database. Runtime login -must be separate, not a member of the owner and without database CREATE or public -schema CREATE. Bootstrap revokes PUBLIC access to its schema/tables and default -function execution. Runtime has no direct mutation rights; M7.3 adds fenced -functions and narrowly scoped EXECUTE. Operators own database/role provisioning; -the migration never creates login passwords or grants superuser. +Use a dedicated owner/migrator login with CREATE on the database. Before +migrating, operators create a NOLOGIN runtime group role and pass its name as +`EVO_PG_RUNTIME_ROLE`. The migration refuses a role that is missing, is the owner, +inherits the owner, or has superuser/CREATEROLE/CREATEDB/replication/BYPASSRLS. The +runtime login is a separate member of that group only, with no database CREATE or +public schema CREATE. Bootstrap revokes PUBLIC access to the schema, tables and +default function execution. Operators own database/role provisioning; the migration +never creates logins or passwords, and never grants superuser. + +## Fenced region ownership (schema v2, ADR-0060) + +The runtime role has schema USAGE and EXECUTE on five SECURITY DEFINER functions: +`acquire_region_lease`, `renew_region_lease`, `release_region_lease`, +`load_region_head` and `advance_region_head`. It cannot read or write tables, create +objects, call `check_region_lease` or provision regions. Each mutation locks the lease row +and checks holder, epoch and expiry with `clock_timestamp()` after the lock wait. +Clients never send times. Rejections: EVL01 busy, EVL02 fenced, EVL03 head mismatch, +EVL04 not provisioned, EVL05 epoch/sequence exhausted, 22023 invalid arguments. +`evo::postgres` maps them to `LeaseStoreError`; a timeout or broken connection is +`unknown`. + +Genesis is operator-only (owner login), never done by recovery or runtime: +`SELECT evolution.provision_region(world, region, frame, genesis_commit_uuid, +'\x<64 hex digest>');`. In schema v2 the head is an identity/digest pointer with no +payload binding. Do not provision real world state before M7.4 publication. + +Hosts use `LeaseAuthority` with `LeasePolicy` defaults: 20 s lease, 5 s renewal +interval, 2 s safety margin, 2 s renewal deadline and 5 s commit deadline. The +local deadline runs from request send time on CLOCK_BOOTTIME. If a renewal outcome +is unknown, keep the earlier deadline and reconnect. Once fenced, stop admission, +simulation and output, then acquire a new epoch and recover. Never resume the old +epoch. ## Kubernetes Secrets diff --git a/include/evo/persistence/region_lease.hpp b/include/evo/persistence/region_lease.hpp new file mode 100644 index 0000000..53c44a3 --- /dev/null +++ b/include/evo/persistence/region_lease.hpp @@ -0,0 +1,99 @@ +#pragma once +#include +#include +#include +#include + +// Portable host-side region authority contract (ADR-0060). No database types: +// PostgreSQL arbitrates ownership; this only decides when a holder must stop. +namespace evo::persistence { +using Uuid = std::array; +using Sha256Digest = std::array; + +struct RegionKey { + std::uint64_t world{}; + std::uint64_t region{}; +}; +struct RegionHead { + std::uint64_t frame{}; + std::uint64_t sequence{}; + Uuid commit{}; + Sha256Digest digest{}; + std::uint64_t end_tick{}; + friend bool operator==(const RegionHead&, const RegionHead&) = default; +}; +// Holder is a database-generated per-acquisition token, not a Pod identity. +struct LeaseGrant { + Uuid holder{}; + std::uint64_t epoch{}; + std::uint64_t frame{}; + std::chrono::milliseconds duration{}; +}; + +// Selected M7.3 timings. Validity: duration 1..60s (database bound); margin at +// least 10% of duration (clock-rate and check-to-effect allowance); two complete +// renewal attempts fit before local fencing; I/O deadlines within adapter limits. +struct LeasePolicy { + std::chrono::milliseconds duration{20000}; + std::chrono::milliseconds renew_interval{5000}; + std::chrono::milliseconds safety_margin{2000}; + std::chrono::milliseconds renew_timeout{2000}; + std::chrono::milliseconds commit_timeout{5000}; +}; +[[nodiscard]] bool valid(const LeasePolicy& policy) noexcept; + +// Monotonic clock that keeps advancing while the process or machine is suspended +// (Linux CLOCK_BOOTTIME). A paused worker must observe that time passed. +struct LeaseClock { + using duration = std::chrono::nanoseconds; + using rep = duration::rep; + using period = duration::period; + using time_point = std::chrono::time_point; + static constexpr bool is_steady = true; + [[nodiscard]] static time_point now() noexcept; +}; + +enum class LeaseState : std::uint8_t { absent, active, renew_due, fenced }; + +// Single-owner local view of one region lease. Every time point is a LeaseClock +// reading supplied by the host. The local deadline is request-send time plus the +// granted duration minus the safety margin: the database set its expiry no earlier +// than that send, so the holder stops before any successor can acquire. Fencing is +// sticky until a strictly newer epoch is acquired; renewal never revives it. +class LeaseAuthority { + public: + explicit LeaseAuthority(const LeasePolicy& policy) noexcept : policy_(policy) {} + [[nodiscard]] const LeasePolicy& policy() const noexcept { + return policy_; + } + [[nodiscard]] const std::optional& grant() const noexcept { + return grant_; + } + // Returns false (and fences) for an invalid policy, reused/older epoch or late reply. + [[nodiscard]] bool acquired(const LeaseGrant& grant, LeaseClock::time_point sent, + LeaseClock::time_point received) noexcept; + [[nodiscard]] bool renewed(std::uint64_t epoch, std::chrono::milliseconds granted, + LeaseClock::time_point sent, + LeaseClock::time_point received) noexcept; + // Ambiguous renewal: the database may have applied a shorter expiry, so keep + // the earlier of the old deadline and the one the attempt could have set. + void renewal_unknown(std::chrono::milliseconds requested, LeaseClock::time_point sent) noexcept; + void fence() noexcept; + [[nodiscard]] LeaseState state(LeaseClock::time_point now) const noexcept; + // Admission, simulation and final output require an unfenced lease. + [[nodiscard]] bool may_originate(LeaseClock::time_point now) const noexcept; + // Liveness guard: begin a commit only if its deadline ends before local fencing. + [[nodiscard]] bool may_begin_commit(LeaseClock::time_point now) const noexcept; + [[nodiscard]] LeaseClock::time_point deadline() const noexcept { + return deadline_; + } + + private: + LeasePolicy policy_; + std::optional grant_; + std::uint64_t highest_epoch_{}; + LeaseClock::time_point last_sent_{}; + LeaseClock::time_point deadline_{}; + bool fenced_{}; +}; +} // namespace evo::persistence diff --git a/include/evo/persistence/region_lease_store.hpp b/include/evo/persistence/region_lease_store.hpp new file mode 100644 index 0000000..5be84dc --- /dev/null +++ b/include/evo/persistence/region_lease_store.hpp @@ -0,0 +1,50 @@ +#pragma once +#include "evo/persistence/postgres.hpp" +#include "evo/persistence/region_lease.hpp" +#include +#include +#include + +// Linux-only calls to the fenced database functions of migration v2 (ADR-0060). +// Each call is one statement on a caller-owned connection; outside an explicit +// transaction it commits by itself. No retries. Rejections are ordinary results. +namespace evo::persistence { +enum class LeaseStoreError : std::uint8_t { + busy, // Another holder's lease is unexpired in database time. + fenced, // Holder/epoch superseded, released or expired: stop originating. + head_mismatch, // Expected predecessor is not the committed head. + not_provisioned, // No genesis; runtime can never create one. + exhausted, // Epoch or commit sequence would wrap: fail closed. + invalid, // Rejected arguments; nothing changed. + store, // Other SQL/configuration failure; nothing committed. + unknown, // May have executed; reconnect and reconcile before reuse. +}; +struct LeaseStoreFailure { + LeaseStoreError kind{}; + PgFailure cause{}; +}; +struct HeadAdvance { + Uuid commit{}; + Sha256Digest digest{}; + std::uint64_t end_tick{}; +}; + +[[nodiscard]] std::expected +acquire_region_lease(Postgres& db, RegionKey key, std::chrono::milliseconds duration, + std::chrono::milliseconds timeout); +// Returns the granted duration; the caller measures it from its request send time. +[[nodiscard]] std::expected +renew_region_lease(Postgres& db, RegionKey key, const LeaseGrant& lease, + std::chrono::milliseconds duration, std::chrono::milliseconds timeout); +[[nodiscard]] std::expected +release_region_lease(Postgres& db, RegionKey key, const LeaseGrant& lease, + std::chrono::milliseconds timeout); +[[nodiscard]] std::expected +load_region_head(Postgres& db, RegionKey key, std::chrono::milliseconds timeout); +// Locks the lease then the head; requires unexpired holder/epoch and the exact +// expected predecessor. head_mismatch after unknown: load the head and compare. +[[nodiscard]] std::expected +advance_region_head(Postgres& db, RegionKey key, const LeaseGrant& lease, + const RegionHead& expected, const HeadAdvance& next, + std::chrono::milliseconds timeout); +} // namespace evo::persistence diff --git a/migrations/0002_region_fencing.sql b/migrations/0002_region_fencing.sql new file mode 100644 index 0000000..676fad8 --- /dev/null +++ b/migrations/0002_region_fencing.sql @@ -0,0 +1,214 @@ +-- M7.3 fenced region ownership (ADR-0060). Owner-only definitions; the runtime +-- role receives schema USAGE and EXECUTE on the guarded functions, never tables. +-- Every mutation locks the region lease row first and decides expiry with the +-- database clock read after that lock. Clients never supply timestamps. +SELECT pg_catalog.set_config('evolution.runtime_role', :'runtime_role', true); +DO $$ +DECLARE + runtime name := pg_catalog.current_setting('evolution.runtime_role'); +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles r WHERE r.rolname = runtime + AND NOT r.rolsuper AND NOT r.rolcreaterole AND NOT r.rolcreatedb + AND NOT r.rolreplication AND NOT r.rolbypassrls) + OR runtime = current_user + OR pg_catalog.pg_has_role(runtime, current_user, 'MEMBER') THEN + RAISE EXCEPTION 'runtime role must exist, be unprivileged and not inherit the owner'; + END IF; +END $$; + +CREATE TABLE evolution.region_heads ( + world_id evolution.u64 NOT NULL, + region_id evolution.u64 NOT NULL, + commit_sequence evolution.u64 NOT NULL, + commit_id uuid NOT NULL, + digest bytea NOT NULL CHECK (octet_length(digest) = 32), + end_tick evolution.u64 NOT NULL, + published_epoch evolution.u64 NOT NULL, + PRIMARY KEY (world_id, region_id), + FOREIGN KEY (world_id, region_id) REFERENCES evolution.region_leases, + -- Genesis is sequence 0, tick 0, provisioned outside any lease epoch. + CHECK ((commit_sequence = 0) = (end_tick = 0)), + CHECK ((commit_sequence = 0) = (published_epoch = 0)) +); +REVOKE ALL ON evolution.region_heads FROM PUBLIC; + +-- Owner-only provisioning; recovery and runtime can never create genesis. +CREATE FUNCTION evolution.provision_region(p_world evolution.u64, p_region evolution.u64, + p_frame evolution.u64, p_genesis_commit uuid, p_genesis_digest bytea) +RETURNS void LANGUAGE sql SECURITY INVOKER SET search_path = pg_catalog, pg_temp AS $$ + INSERT INTO evolution.region_leases(world_id, region_id, frame_id) + VALUES (p_world, p_region, p_frame); + INSERT INTO evolution.region_heads(world_id, region_id, commit_sequence, commit_id, + digest, end_tick, published_epoch) + VALUES (p_world, p_region, 0, p_genesis_commit, p_genesis_digest, 0, 0); +$$; + +CREATE FUNCTION evolution.acquire_region_lease(p_world evolution.u64, + p_region evolution.u64, p_duration_ms integer) +RETURNS TABLE (lease_holder uuid, lease_epoch evolution.u64, region_frame evolution.u64, + granted_ms integer) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + l evolution.region_leases%ROWTYPE; + t timestamptz; +BEGIN + IF p_world IS NULL OR p_region IS NULL OR p_duration_ms IS NULL + OR p_duration_ms NOT BETWEEN 1000 AND 60000 THEN + RAISE EXCEPTION 'invalid lease request' USING ERRCODE = '22023'; + END IF; + SELECT * INTO l FROM evolution.region_leases r + WHERE r.world_id = p_world AND r.region_id = p_region FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'region not provisioned' USING ERRCODE = 'EVL04'; + END IF; + t := clock_timestamp(); + IF l.holder IS NOT NULL AND l.expires_at > t THEN + RAISE EXCEPTION 'region lease held' USING ERRCODE = 'EVL01'; + END IF; + IF l.epoch = 18446744073709551615 THEN + RAISE EXCEPTION 'lease epochs exhausted' USING ERRCODE = 'EVL05'; + END IF; + UPDATE evolution.region_leases r + SET epoch = l.epoch + 1, holder = gen_random_uuid(), + expires_at = t + p_duration_ms * interval '1 millisecond' + WHERE r.world_id = p_world AND r.region_id = p_region + RETURNING r.holder, r.epoch, r.frame_id INTO lease_holder, lease_epoch, region_frame; + granted_ms := p_duration_ms; + RETURN NEXT; +END $$; + +-- Locks the lease row and requires the exact unexpired holder/epoch. Internal: +-- callable only by the owner and the SECURITY DEFINER functions below. +CREATE FUNCTION evolution.check_region_lease(p_world evolution.u64, p_region evolution.u64, + p_holder uuid, p_epoch evolution.u64) +RETURNS timestamptz LANGUAGE plpgsql SECURITY INVOKER +SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + l evolution.region_leases%ROWTYPE; + t timestamptz; +BEGIN + IF p_world IS NULL OR p_region IS NULL OR p_holder IS NULL OR p_epoch IS NULL THEN + RAISE EXCEPTION 'invalid lease request' USING ERRCODE = '22023'; + END IF; + SELECT * INTO l FROM evolution.region_leases r + WHERE r.world_id = p_world AND r.region_id = p_region FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'region not provisioned' USING ERRCODE = 'EVL04'; + END IF; + -- Read after the lock wait: a queued stale writer sees current database time. + t := clock_timestamp(); + IF l.holder IS DISTINCT FROM p_holder OR l.epoch <> p_epoch OR l.expires_at <= t THEN + RAISE EXCEPTION 'region lease fenced' USING ERRCODE = 'EVL02'; + END IF; + RETURN t; +END $$; + +CREATE FUNCTION evolution.renew_region_lease(p_world evolution.u64, p_region evolution.u64, + p_holder uuid, p_epoch evolution.u64, p_duration_ms integer) +RETURNS integer LANGUAGE plpgsql SECURITY DEFINER +SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + t timestamptz; +BEGIN + IF p_duration_ms IS NULL OR p_duration_ms NOT BETWEEN 1000 AND 60000 THEN + RAISE EXCEPTION 'invalid lease request' USING ERRCODE = '22023'; + END IF; + -- Renewal never revives an expired or superseded lease. + t := evolution.check_region_lease(p_world, p_region, p_holder, p_epoch); + UPDATE evolution.region_leases r + SET expires_at = t + p_duration_ms * interval '1 millisecond' + WHERE r.world_id = p_world AND r.region_id = p_region; + RETURN p_duration_ms; +END $$; + +CREATE FUNCTION evolution.release_region_lease(p_world evolution.u64, p_region evolution.u64, + p_holder uuid, p_epoch evolution.u64) +RETURNS void LANGUAGE plpgsql SECURITY DEFINER +SET search_path = pg_catalog, pg_temp AS $$ +BEGIN + PERFORM evolution.check_region_lease(p_world, p_region, p_holder, p_epoch); + -- The epoch is retained so the next acquisition can never reuse it. + UPDATE evolution.region_leases r SET holder = NULL, expires_at = NULL + WHERE r.world_id = p_world AND r.region_id = p_region; +END $$; + +CREATE FUNCTION evolution.load_region_head(p_world evolution.u64, p_region evolution.u64) +RETURNS TABLE (region_frame evolution.u64, head_sequence evolution.u64, head_commit uuid, + head_digest text, head_tick evolution.u64) +LANGUAGE plpgsql STABLE SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +BEGIN + RETURN QUERY SELECT l.frame_id, h.commit_sequence, h.commit_id, + encode(h.digest, 'hex'), h.end_tick + FROM evolution.region_heads h + JOIN evolution.region_leases l USING (world_id, region_id) + WHERE h.world_id = p_world AND h.region_id = p_region; + IF NOT FOUND THEN + RAISE EXCEPTION 'region not provisioned' USING ERRCODE = 'EVL04'; + END IF; +END $$; + +-- The fenced transition kernel. M7.4/M7.5 publication wraps it with manifest, +-- decision and outbox rows in the same transaction; the lease lock is taken first. +CREATE FUNCTION evolution.advance_region_head(p_world evolution.u64, p_region evolution.u64, + p_holder uuid, p_epoch evolution.u64, p_expected_sequence evolution.u64, + p_expected_commit uuid, p_expected_digest bytea, p_commit uuid, p_digest bytea, + p_end_tick evolution.u64) +RETURNS TABLE (region_frame evolution.u64, head_sequence evolution.u64, head_commit uuid, + head_digest text, head_tick evolution.u64) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + h evolution.region_heads%ROWTYPE; +BEGIN + IF p_expected_sequence IS NULL OR p_expected_commit IS NULL OR p_commit IS NULL + OR p_end_tick IS NULL OR p_commit = p_expected_commit + OR octet_length(p_expected_digest) IS DISTINCT FROM 32 + OR octet_length(p_digest) IS DISTINCT FROM 32 THEN + RAISE EXCEPTION 'invalid head advance' USING ERRCODE = '22023'; + END IF; + PERFORM evolution.check_region_lease(p_world, p_region, p_holder, p_epoch); + SELECT * INTO h FROM evolution.region_heads r + WHERE r.world_id = p_world AND r.region_id = p_region FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'region not provisioned' USING ERRCODE = 'EVL04'; + END IF; + IF h.commit_sequence <> p_expected_sequence OR h.commit_id <> p_expected_commit + OR h.digest <> p_expected_digest THEN + RAISE EXCEPTION 'region head moved' USING ERRCODE = 'EVL03'; + END IF; + IF h.commit_sequence = 18446744073709551615 THEN + RAISE EXCEPTION 'commit sequences exhausted' USING ERRCODE = 'EVL05'; + END IF; + IF p_end_tick <= h.end_tick OR p_end_tick - h.end_tick > 64 THEN + RAISE EXCEPTION 'invalid head advance' USING ERRCODE = '22023'; + END IF; + UPDATE evolution.region_heads r + SET commit_sequence = h.commit_sequence + 1, commit_id = p_commit, digest = p_digest, + end_tick = p_end_tick, published_epoch = p_epoch + WHERE r.world_id = p_world AND r.region_id = p_region; + RETURN QUERY SELECT l.frame_id, n.commit_sequence, n.commit_id, encode(n.digest, 'hex'), + n.end_tick + FROM evolution.region_heads n + JOIN evolution.region_leases l USING (world_id, region_id) + WHERE n.world_id = p_world AND n.region_id = p_region; +END $$; + +REVOKE ALL ON FUNCTION + evolution.provision_region(evolution.u64, evolution.u64, evolution.u64, uuid, bytea), + evolution.acquire_region_lease(evolution.u64, evolution.u64, integer), + evolution.check_region_lease(evolution.u64, evolution.u64, uuid, evolution.u64), + evolution.renew_region_lease(evolution.u64, evolution.u64, uuid, evolution.u64, integer), + evolution.release_region_lease(evolution.u64, evolution.u64, uuid, evolution.u64), + evolution.load_region_head(evolution.u64, evolution.u64), + evolution.advance_region_head(evolution.u64, evolution.u64, uuid, evolution.u64, + evolution.u64, uuid, bytea, uuid, bytea, evolution.u64) + FROM PUBLIC; +GRANT USAGE ON SCHEMA evolution TO :"runtime_role"; +GRANT EXECUTE ON FUNCTION + evolution.acquire_region_lease(evolution.u64, evolution.u64, integer), + evolution.renew_region_lease(evolution.u64, evolution.u64, uuid, evolution.u64, integer), + evolution.release_region_lease(evolution.u64, evolution.u64, uuid, evolution.u64), + evolution.load_region_head(evolution.u64, evolution.u64), + evolution.advance_region_head(evolution.u64, evolution.u64, uuid, evolution.u64, + evolution.u64, uuid, bytea, uuid, bytea, evolution.u64) + TO :"runtime_role"; +INSERT INTO evolution.schema_migrations(version, sha256) VALUES (2, :'migration_sha_2'); diff --git a/migrations/apply.sql b/migrations/apply.sql index e29b5e6..f09e17f 100644 --- a/migrations/apply.sql +++ b/migrations/apply.sql @@ -12,15 +12,21 @@ SELECT current_setting('server_version_num')::int = 180006 AS supported \gset \endif SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') AS exists \gset \if :exists - -- Unknown/partial schema fails closed; no IF NOT EXISTS repair of foreign data. - SELECT count(*) = 1 AND min(version) = 1 AND min(sha256) = :'migration_sha' - AS compatible FROM evolution.schema_migrations \gset - \if :compatible + -- Only an exact prefix of the checked-in history is admitted. Unknown or + -- partial/foreign schemas fail closed; no IF NOT EXISTS repair of foreign data. + SELECT coalesce(string_agg(version || ':' || sha256, ',' ORDER BY version), '') + AS history FROM evolution.schema_migrations \gset + SELECT :'history' = '1:' || :'migration_sha' AS at_v1, + :'history' = '1:' || :'migration_sha' || ',2:' || :'migration_sha_2' AS at_v2 \gset + \if :at_v2 + \elif :at_v1 + \ir 0002_region_fencing.sql \else \echo 'Unknown or changed migration history' DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$; \endif \else \ir 0001_region_bootstrap.sql + \ir 0002_region_fencing.sql \endif COMMIT; diff --git a/src/persistence/region_lease.cpp b/src/persistence/region_lease.cpp new file mode 100644 index 0000000..66ec740 --- /dev/null +++ b/src/persistence/region_lease.cpp @@ -0,0 +1,96 @@ +#include "evo/persistence/region_lease.hpp" +#include +#if defined(__linux__) +#include +#endif + +namespace evo::persistence { +namespace { +using std::chrono::milliseconds; +constexpr milliseconds minimum_duration{1000}; +constexpr milliseconds maximum_duration{60000}; +constexpr milliseconds maximum_io{30000}; // Postgres adapter deadline bound. +} // namespace + +bool valid(const LeasePolicy& p) noexcept { + const milliseconds zero{}; + if (p.duration < minimum_duration || p.duration > maximum_duration || + p.renew_interval <= zero || p.safety_margin <= zero || p.renew_timeout <= zero || + p.commit_timeout <= zero || p.renew_timeout > maximum_io || p.commit_timeout > maximum_io || + p.safety_margin >= p.duration) + return false; + const auto usable = p.duration - p.safety_margin; + return p.safety_margin * 10 >= p.duration && + (p.renew_interval + p.renew_timeout) * 2 <= usable && p.commit_timeout < usable; +} + +LeaseClock::time_point LeaseClock::now() noexcept { +#if defined(__linux__) + timespec ts{}; + // CLOCK_BOOTTIME cannot fail with a valid clock id and pointer. + static_cast(clock_gettime(CLOCK_BOOTTIME, &ts)); + return time_point{std::chrono::seconds{ts.tv_sec} + std::chrono::nanoseconds{ts.tv_nsec}}; +#else + // Storage hosts are Linux-only; elsewhere suspend behavior is unqualified. + return time_point{ + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch())}; +#endif +} + +bool LeaseAuthority::acquired(const LeaseGrant& grant, LeaseClock::time_point sent, + LeaseClock::time_point received) noexcept { + const auto deadline = sent + grant.duration - policy_.safety_margin; + if (!valid(policy_) || grant.epoch == 0 || grant.epoch <= highest_epoch_ || grant.frame == 0 || + grant.duration != policy_.duration || received < sent || received >= deadline) { + fence(); + return false; + } + grant_ = grant; + highest_epoch_ = grant.epoch; + last_sent_ = sent; + deadline_ = deadline; + fenced_ = false; + return true; +} + +bool LeaseAuthority::renewed(std::uint64_t epoch, milliseconds granted, LeaseClock::time_point sent, + LeaseClock::time_point received) noexcept { + if (!grant_ || fenced_ || epoch != grant_->epoch || granted != policy_.duration || + sent < last_sent_ || received < sent || received >= deadline_) { + fence(); + return false; + } + // Database expiry is its execution time plus granted, which is after sent. + deadline_ = sent + granted - policy_.safety_margin; + last_sent_ = sent; + return true; +} + +void LeaseAuthority::renewal_unknown(milliseconds requested, LeaseClock::time_point sent) noexcept { + if (!grant_ || fenced_) + return; + deadline_ = std::min(deadline_, sent + requested - policy_.safety_margin); + last_sent_ = std::max(last_sent_, sent); +} + +void LeaseAuthority::fence() noexcept { + fenced_ = true; +} + +LeaseState LeaseAuthority::state(LeaseClock::time_point now) const noexcept { + if (!grant_) + return LeaseState::absent; + if (fenced_ || now >= deadline_) + return LeaseState::fenced; + return now >= last_sent_ + policy_.renew_interval ? LeaseState::renew_due : LeaseState::active; +} + +bool LeaseAuthority::may_originate(LeaseClock::time_point now) const noexcept { + const auto s = state(now); + return s == LeaseState::active || s == LeaseState::renew_due; +} + +bool LeaseAuthority::may_begin_commit(LeaseClock::time_point now) const noexcept { + return may_originate(now) && now + policy_.commit_timeout <= deadline_; +} +} // namespace evo::persistence diff --git a/src/persistence/region_lease_store.cpp b/src/persistence/region_lease_store.cpp new file mode 100644 index 0000000..369b2e1 --- /dev/null +++ b/src/persistence/region_lease_store.cpp @@ -0,0 +1,204 @@ +#include "evo/persistence/region_lease_store.hpp" +#include +#include +#include +#include +#include + +namespace evo::persistence { +namespace { +using std::chrono::milliseconds; +constexpr std::string_view digits = "0123456789abcdef"; + +LeaseStoreFailure classify(const PgFailure& f) { + const std::string_view state(f.sqlstate.data(), 5); + auto kind = LeaseStoreError::store; + if (f.kind == PgError::unknown) + kind = LeaseStoreError::unknown; + else if (f.kind == PgError::config || f.kind == PgError::bounds || state == "22023") + kind = LeaseStoreError::invalid; + else if (state == "EVL01") + kind = LeaseStoreError::busy; + else if (state == "EVL02") + kind = LeaseStoreError::fenced; + else if (state == "EVL03") + kind = LeaseStoreError::head_mismatch; + else if (state == "EVL04") + kind = LeaseStoreError::not_provisioned; + else if (state == "EVL05") + kind = LeaseStoreError::exhausted; + return {kind, f}; +} +LeaseStoreFailure malformed(bool mutation) { + // A mutation whose reply cannot be parsed may still have committed. + return {mutation ? LeaseStoreError::unknown : LeaseStoreError::store, {}}; +} +template std::string hex(const std::array& bytes) { + std::string out; + out.reserve(N * 2); + for (const auto b : bytes) { + out.push_back(digits[static_cast(b >> 4U)]); + out.push_back(digits[static_cast(b & 0x0FU)]); + } + return out; +} +std::optional nibble(char c) { + const auto at = digits.find(c); + if (at == std::string_view::npos) + return std::nullopt; + return static_cast(at); +} +template std::optional> unhex(std::string_view s) { + if (s.size() != N * 2) + return std::nullopt; + std::array out{}; + for (std::size_t i = 0; i < N; ++i) { + const auto high = nibble(s[i * 2]); + const auto low = nibble(s[(i * 2) + 1]); + if (!high || !low) + return std::nullopt; + out[i] = static_cast((*high << 4U) | *low); + } + return out; +} +std::string uuid_text(const Uuid& id) { + const auto h = hex(id); + return h.substr(0, 8) + '-' + h.substr(8, 4) + '-' + h.substr(12, 4) + '-' + h.substr(16, 4) + + '-' + h.substr(20); +} +std::optional parse_uuid(const std::optional& s) { + if (!s || s->size() != 36) + return std::nullopt; + std::string compact; + for (std::size_t i = 0; i < s->size(); ++i) { + const bool dash = i == 8 || i == 13 || i == 18 || i == 23; + if (dash != ((*s)[i] == '-')) + return std::nullopt; + if (!dash) + compact.push_back((*s)[i]); + } + return unhex<16>(compact); +} +template std::optional parse_integer(const std::optional& s) { + if (!s || s->empty() || s->size() > 20 || s->front() == '-' || s->front() == '+') + return std::nullopt; + T value{}; + const auto* end = s->data() + s->size(); + const auto [at, error] = std::from_chars(s->data(), end, value); + if (error != std::errc{} || at != end) + return std::nullopt; + return value; +} +std::string bytea(const Sha256Digest& d) { + return "\\x" + hex(d); +} +std::optional parse_head(const PgRows& rows) { + if (rows.size() != 1 || rows[0].size() != 5) + return std::nullopt; + const auto& r = rows[0]; + const auto frame = parse_integer(r[0]); + const auto sequence = parse_integer(r[1]); + const auto commit = parse_uuid(r[2]); + const auto& digest_text = r[3]; + const auto digest = digest_text ? unhex<32>(*digest_text) : std::nullopt; + const auto tick = parse_integer(r[4]); + if (!frame || !sequence || !commit || !digest || !tick) + return std::nullopt; + return RegionHead{*frame, *sequence, *commit, *digest, *tick}; +} +} // namespace + +std::expected +acquire_region_lease(Postgres& db, RegionKey key, milliseconds duration, milliseconds timeout) { + const std::array p{std::to_string(key.world), std::to_string(key.region), + std::to_string(duration.count())}; + auto rows = db.query("SELECT * FROM evolution.acquire_region_lease($1::evolution.u64, " + "$2::evolution.u64, $3::integer)", + p, timeout); + if (!rows) + return std::unexpected{classify(rows.error())}; + if (rows->size() != 1 || (*rows)[0].size() != 4) + return std::unexpected{malformed(true)}; + const auto& r = (*rows)[0]; + const auto holder = parse_uuid(r[0]); + const auto epoch = parse_integer(r[1]); + const auto frame = parse_integer(r[2]); + const auto granted = parse_integer(r[3]); + if (!holder || !epoch || !frame || !granted || *epoch == 0) + return std::unexpected{malformed(true)}; + return LeaseGrant{*holder, *epoch, *frame, milliseconds{*granted}}; +} + +std::expected renew_region_lease(Postgres& db, RegionKey key, + const LeaseGrant& lease, + milliseconds duration, + milliseconds timeout) { + const std::array p{std::to_string(key.world), std::to_string(key.region), + uuid_text(lease.holder), std::to_string(lease.epoch), + std::to_string(duration.count())}; + auto rows = db.query("SELECT evolution.renew_region_lease($1::evolution.u64, " + "$2::evolution.u64, $3::uuid, $4::evolution.u64, $5::integer)", + p, timeout); + if (!rows) + return std::unexpected{classify(rows.error())}; + const auto granted = rows->size() == 1 && (*rows)[0].size() == 1 + ? parse_integer((*rows)[0][0]) + : std::nullopt; + if (!granted) + return std::unexpected{malformed(true)}; + return milliseconds{*granted}; +} + +std::expected +release_region_lease(Postgres& db, RegionKey key, const LeaseGrant& lease, milliseconds timeout) { + const std::array p{std::to_string(key.world), std::to_string(key.region), + uuid_text(lease.holder), std::to_string(lease.epoch)}; + auto rows = db.query("SELECT evolution.release_region_lease($1::evolution.u64, " + "$2::evolution.u64, $3::uuid, $4::evolution.u64)", + p, timeout); + if (!rows) + return std::unexpected{classify(rows.error())}; + if (rows->size() != 1) + return std::unexpected{malformed(true)}; + return {}; +} + +std::expected load_region_head(Postgres& db, RegionKey key, + milliseconds timeout) { + const std::array p{std::to_string(key.world), std::to_string(key.region)}; + auto rows = + db.query("SELECT * FROM evolution.load_region_head($1::evolution.u64, $2::evolution.u64)", + p, timeout); + if (!rows) + return std::unexpected{classify(rows.error())}; + auto head = parse_head(*rows); + if (!head) + return std::unexpected{malformed(false)}; + return *head; +} + +std::expected +advance_region_head(Postgres& db, RegionKey key, const LeaseGrant& lease, + const RegionHead& expected, const HeadAdvance& next, milliseconds timeout) { + const std::array p{std::to_string(key.world), + std::to_string(key.region), + uuid_text(lease.holder), + std::to_string(lease.epoch), + std::to_string(expected.sequence), + uuid_text(expected.commit), + bytea(expected.digest), + uuid_text(next.commit), + bytea(next.digest), + std::to_string(next.end_tick)}; + auto rows = db.query("SELECT * FROM evolution.advance_region_head($1::evolution.u64, " + "$2::evolution.u64, $3::uuid, $4::evolution.u64, $5::evolution.u64, " + "$6::uuid, $7::bytea, $8::uuid, $9::bytea, $10::evolution.u64)", + p, timeout); + if (!rows) + return std::unexpected{classify(rows.error())}; + auto head = parse_head(*rows); + if (!head) + return std::unexpected{malformed(true)}; + return *head; +} +} // namespace evo::persistence diff --git a/tests/postgres_tests.cpp b/tests/postgres_tests.cpp index 50f6195..b63f7fb 100644 --- a/tests/postgres_tests.cpp +++ b/tests/postgres_tests.cpp @@ -3,6 +3,7 @@ #include #include #include +void run_region_lease_store_tests(); namespace { using namespace evo::persistence; using namespace std::chrono_literals; @@ -69,6 +70,8 @@ int main() { try { run(); std::puts("postgres adapter integration passed"); + run_region_lease_store_tests(); + std::puts("region lease fencing integration passed"); return 0; } catch (const std::exception& e) { std::fprintf(stderr, "%s\n", e.what()); diff --git a/tests/region_lease_store_tests.cpp b/tests/region_lease_store_tests.cpp new file mode 100644 index 0000000..17a1be2 --- /dev/null +++ b/tests/region_lease_store_tests.cpp @@ -0,0 +1,413 @@ +// Real PostgreSQL M7.3 fencing checks (ADR-0060). Run by tools/ci/postgres-check.sh. +#include "evo/persistence/region_lease_store.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +void run_region_lease_store_tests(); + +namespace { +using namespace evo::persistence; +using namespace std::chrono_literals; +using std::chrono::milliseconds; +void check(bool value, std::source_location at = std::source_location::current()) { + if (!value) + throw std::runtime_error("region lease store test line " + std::to_string(at.line())); +} +#define CHECK(...) check(static_cast(__VA_ARGS__)) +std::string env(const char* key) { + const auto* value = std::getenv(key); + if (!value) + throw std::runtime_error("missing test environment"); + return value; +} +std::unique_ptr connect_as(const std::string& user) { + PgOptions options{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}}; + auto made = Postgres::connect(options, 5s); + CHECK(made); + return std::move(*made); +} +std::unique_ptr owner() { + return connect_as(env("PGUSER")); +} +std::unique_ptr runtime() { + return connect_as(env("EVO_PG_RUNTIME_USER")); +} +Uuid id(std::uint8_t seed) { + Uuid out{}; + out.fill(seed); + return out; +} +Sha256Digest digest(std::uint8_t seed) { + Sha256Digest out{}; + out.fill(seed); + return out; +} +std::string hex_text(std::uint8_t seed, std::size_t bytes) { + constexpr char digits[] = "0123456789abcdef"; + std::string out; + for (std::size_t i = 0; i < bytes; ++i) { + out.push_back(digits[seed >> 4U]); + out.push_back(digits[seed & 0x0FU]); + } + return out; +} +std::string uuid_literal(std::uint8_t seed) { + const auto h = hex_text(seed, 16); + return h.substr(0, 8) + '-' + h.substr(8, 4) + '-' + h.substr(12, 4) + '-' + h.substr(16, 4) + + '-' + h.substr(20); +} +// Genesis commit/digest bytes are the region number; frame is 7. +RegionKey provision(Postgres& db, std::uint64_t region) { + const auto seed = static_cast(region); + const std::array p{std::to_string(region), uuid_literal(seed), + "\\x" + hex_text(seed, 32), "7"}; + CHECK(db.query("SELECT evolution.provision_region(1, $1::evolution.u64, $4::evolution.u64, " + "$2::uuid, $3::bytea)", + p, 5s)); + return {1, region}; +} +bool failed(const auto& result, LeaseStoreError kind) { + return !result && result.error().kind == kind; +} +std::string sqlstate(const std::expected& result) { + return result ? std::string{} : std::string(result.error().sqlstate.data()); +} + +void runtime_cannot_bypass_the_guard() { + auto admin = owner(); + const auto key = provision(*admin, 10); + auto db = runtime(); + for (const auto* sql : { + "UPDATE evolution.region_leases SET holder = NULL", + "UPDATE evolution.region_heads SET commit_sequence = 5", + "DELETE FROM evolution.region_heads", + "INSERT INTO evolution.region_leases(world_id,region_id,frame_id) VALUES (9,9,9)", + "SELECT * FROM evolution.region_leases", + "SELECT * FROM evolution.region_heads", + "SELECT evolution.provision_region(1, 11, 7, gen_random_uuid(), '\\x00')", + "SELECT evolution.check_region_lease(1, 10, gen_random_uuid(), 1)", + "CREATE FUNCTION evolution.bypass() RETURNS int LANGUAGE sql AS 'SELECT 1'", + "CREATE TABLE evolution.bypass(x int)", + }) + CHECK(sqlstate(db->query(sql, {}, 5s)) == "42501"); + const std::array owner_name{env("PGUSER")}; + CHECK(!db->query("SELECT pg_catalog.set_config('role', $1, false)", owner_name, 5s)); + // A SECURITY DEFINER function ignores a hostile caller search_path. + CHECK(db->query("SET search_path = pg_temp, public", {}, 5s)); + CHECK(db->query("CREATE TEMP TABLE region_leases(x int)", {}, 5s)); + auto lease = acquire_region_lease(*db, key, 20000ms, 5s); + CHECK(lease && lease->epoch == 1 && lease->frame == 7); + CHECK(release_region_lease(*db, key, *lease, 5s)); +} + +void acquire_renew_release_and_advance() { + auto admin = owner(); + const auto key = provision(*admin, 20); + auto db = runtime(); + CHECK( + failed(acquire_region_lease(*db, {1, 999}, 20000ms, 5s), LeaseStoreError::not_provisioned)); + CHECK(failed(acquire_region_lease(*db, key, 999ms, 5s), LeaseStoreError::invalid)); + CHECK(failed(acquire_region_lease(*db, key, 60001ms, 5s), LeaseStoreError::invalid)); + auto lease = acquire_region_lease(*db, key, 20000ms, 5s); + CHECK(lease && lease->epoch == 1 && lease->duration == 20000ms && lease->holder != Uuid{}); + CHECK(failed(acquire_region_lease(*db, key, 20000ms, 5s), LeaseStoreError::busy)); + auto renewed = renew_region_lease(*db, key, *lease, 20000ms, 5s); + CHECK(renewed && *renewed == 20000ms); + auto forged = *lease; + forged.holder[0] ^= 1U; + CHECK(failed(renew_region_lease(*db, key, forged, 20000ms, 5s), LeaseStoreError::fenced)); + auto stale_epoch = *lease; + stale_epoch.epoch = 2; + CHECK(failed(renew_region_lease(*db, key, stale_epoch, 20000ms, 5s), LeaseStoreError::fenced)); + + auto genesis = load_region_head(*db, key, 5s); + CHECK(genesis && genesis->sequence == 0 && genesis->end_tick == 0 && genesis->frame == 7); + CHECK(genesis->commit == id(20) && genesis->digest == digest(20)); + CHECK(failed(advance_region_head(*db, key, *lease, *genesis, {id(21), digest(21), 65}, 5s), + LeaseStoreError::invalid)); + CHECK(failed(advance_region_head(*db, key, *lease, *genesis, {id(20), digest(21), 1}, 5s), + LeaseStoreError::invalid)); + const HeadAdvance first{id(21), digest(21), 64}; + auto head = advance_region_head(*db, key, *lease, *genesis, first, 5s); + CHECK(head && head->sequence == 1 && head->end_tick == 64 && head->commit == id(21)); + CHECK(failed(advance_region_head(*db, key, forged, *head, {id(22), digest(22), 65}, 5s), + LeaseStoreError::fenced)); + // Exact retry after an applied advance: mismatch, then the head proves it committed. + CHECK(failed(advance_region_head(*db, key, *lease, *genesis, first, 5s), + LeaseStoreError::head_mismatch)); + auto current = load_region_head(*db, key, 5s); + CHECK(current && *current == *head); + auto changed = *head; + changed.digest[31] ^= 1U; + CHECK(failed(advance_region_head(*db, key, *lease, changed, {id(22), digest(22), 65}, 5s), + LeaseStoreError::head_mismatch)); + CHECK(failed(advance_region_head(*db, key, *lease, *head, {id(22), digest(22), 64}, 5s), + LeaseStoreError::invalid)); + + CHECK(release_region_lease(*db, key, *lease, 5s)); + CHECK(failed(release_region_lease(*db, key, *lease, 5s), LeaseStoreError::fenced)); + CHECK(failed(renew_region_lease(*db, key, *lease, 20000ms, 5s), LeaseStoreError::fenced)); + CHECK(failed(advance_region_head(*db, key, *lease, *head, {id(22), digest(22), 65}, 5s), + LeaseStoreError::fenced)); + auto next = acquire_region_lease(*db, key, 20000ms, 5s); + CHECK(next && next->epoch == 2 && next->holder != lease->holder); + auto final_head = advance_region_head(*db, key, *next, *head, {id(22), digest(22), 65}, 5s); + CHECK(final_head && final_head->sequence == 2); +} + +void database_time_expiry_and_paused_writer() { + auto admin = owner(); + const auto key = provision(*admin, 30); + auto worker_a = runtime(); + auto worker_b = runtime(); + const LeasePolicy fast{1000ms, 200ms, 100ms, 100ms, 300ms}; + LeaseAuthority a{fast}; + auto sent = LeaseClock::now(); + auto lease_a = acquire_region_lease(*worker_a, key, fast.duration, 5s); + CHECK(lease_a && a.acquired(*lease_a, sent, LeaseClock::now())); + CHECK(failed(acquire_region_lease(*worker_b, key, 20000ms, 5s), LeaseStoreError::busy)); + auto genesis = load_region_head(*worker_a, key, 5s); + CHECK(genesis); + + std::this_thread::sleep_for(1200ms); // Worker A is paused past its lease. + CHECK(a.state(LeaseClock::now()) == LeaseState::fenced); + // Expiry is decided by database time; the successor acquires a newer epoch. + auto lease_b = acquire_region_lease(*worker_b, key, 20000ms, 5s); + CHECK(lease_b && lease_b->epoch == lease_a->epoch + 1); + auto head_b = + advance_region_head(*worker_b, key, *lease_b, *genesis, {id(31), digest(31), 3}, 5s); + CHECK(head_b); + // A resumes and ignores its local fence: every database path rejects it. + CHECK(failed(renew_region_lease(*worker_a, key, *lease_a, fast.duration, 5s), + LeaseStoreError::fenced)); + CHECK( + failed(advance_region_head(*worker_a, key, *lease_a, *genesis, {id(32), digest(32), 1}, 5s), + LeaseStoreError::fenced)); + CHECK( + failed(advance_region_head(*worker_a, key, *lease_a, *head_b, {id(32), digest(32), 4}, 5s), + LeaseStoreError::fenced)); + CHECK(failed(release_region_lease(*worker_a, key, *lease_a, 5s), LeaseStoreError::fenced)); + auto head = load_region_head(*worker_a, key, 5s); + CHECK(head && *head == *head_b); + + // Expired renewal cannot revive a lease even without a successor. + CHECK(release_region_lease(*worker_b, key, *lease_b, 5s)); + auto lease_c = acquire_region_lease(*worker_a, key, 1000ms, 5s); + CHECK(lease_c && lease_c->epoch == lease_b->epoch + 1); + std::this_thread::sleep_for(1200ms); + CHECK( + failed(renew_region_lease(*worker_a, key, *lease_c, 1000ms, 5s), LeaseStoreError::fenced)); +} + +// A writer queued on the lease row lock is judged by the clock after the wait, +// not its statement start time, so a lease that expired while queued is fenced. +void queued_writer_uses_post_lock_database_time() { + auto admin = owner(); + const auto key = provision(*admin, 40); + auto worker = runtime(); + auto lease = acquire_region_lease(*worker, key, 1000ms, 5s); + auto genesis = load_region_head(*worker, key, 5s); + CHECK(lease && genesis); + auto blocker = owner(); + const std::array region{"40"}; + CHECK(blocker->query("BEGIN", {}, 5s)); + CHECK(blocker->query("SELECT 1 FROM evolution.region_leases WHERE world_id = 1 AND " + "region_id = $1::evolution.u64 FOR UPDATE", + region, 5s)); + std::expected queued = std::unexpected{LeaseStoreFailure{}}; + std::thread writer([&] { + queued = advance_region_head(*worker, key, *lease, *genesis, {id(41), digest(41), 1}, 5s); + }); + std::this_thread::sleep_for(1300ms); + CHECK(blocker->query("ROLLBACK", {}, 5s)); + writer.join(); + CHECK(failed(queued, LeaseStoreError::fenced)); + auto head = load_region_head(*worker, key, 5s); + CHECK(head && *head == *genesis); +} + +// Partition: the renewal's outcome is unknown and the connection is gone. The +// holder keeps its earlier deadline and fences itself before a successor starts. +void partitioned_renewal_fences_locally() { + auto admin = owner(); + const auto key = provision(*admin, 50); + auto worker = runtime(); + const LeasePolicy fast{1000ms, 200ms, 100ms, 100ms, 300ms}; + LeaseAuthority authority{fast}; + const auto sent = LeaseClock::now(); + auto lease = acquire_region_lease(*worker, key, fast.duration, 5s); + CHECK(lease && authority.acquired(*lease, sent, LeaseClock::now())); + auto blocker = owner(); + const std::array region{"50"}; + CHECK(blocker->query("BEGIN", {}, 5s)); + CHECK(blocker->query("SELECT 1 FROM evolution.region_leases WHERE world_id = 1 AND " + "region_id = $1::evolution.u64 FOR UPDATE", + region, 5s)); + const auto renew_sent = LeaseClock::now(); + auto renewal = renew_region_lease(*worker, key, *lease, fast.duration, fast.renew_timeout); + CHECK(failed(renewal, LeaseStoreError::unknown)); + authority.renewal_unknown(fast.duration, renew_sent); + const auto local_deadline = authority.deadline(); + CHECK(local_deadline <= sent + fast.duration - fast.safety_margin); + std::this_thread::sleep_for(1300ms); + CHECK(!authority.may_originate(LeaseClock::now())); + CHECK(blocker->query("ROLLBACK", {}, 5s)); + // The orphaned renewal reached the row only after database expiry: rejected. + auto successor = runtime(); + auto next = acquire_region_lease(*successor, key, 20000ms, 5s); + CHECK(next && next->epoch == lease->epoch + 1); +} + +void contenders_serialize_on_the_database() { + auto admin = owner(); + const auto key = provision(*admin, 60); + std::array, 2> workers{runtime(), runtime()}; + std::uint64_t previous_epoch = 0; + constexpr int rounds = 64; + for (int round = 0; round < rounds; ++round) { + std::array, 2> results{ + std::unexpected{LeaseStoreFailure{}}, std::unexpected{LeaseStoreFailure{}}}; + std::latch start{2}; + std::vector threads; + threads.reserve(2); + for (std::size_t i = 0; i < 2; ++i) + threads.emplace_back([&, i] { + start.arrive_and_wait(); + results[i] = acquire_region_lease(*workers[i], key, 20000ms, 5s); + }); + for (auto& t : threads) + t.join(); + const auto winners = std::count_if(results.begin(), results.end(), + [](const auto& r) { return r.has_value(); }); + CHECK(winners == 1); + const std::size_t w = results[0] ? 0 : 1; + CHECK(failed(results[1 - w], LeaseStoreError::busy)); + CHECK(results[w]->epoch == previous_epoch + 1); + previous_epoch = results[w]->epoch; + CHECK(release_region_lease(*workers[w], key, *results[w], 5s)); + } + + // The same holder racing two commits from the same predecessor: exactly one wins. + auto lease = acquire_region_lease(*workers[0], key, 20000ms, 5s); + CHECK(lease); + auto head = load_region_head(*workers[0], key, 5s); + CHECK(head); + for (std::uint8_t round = 0; round < 32; ++round) { + std::array, 2> results{ + std::unexpected{LeaseStoreFailure{}}, std::unexpected{LeaseStoreFailure{}}}; + std::latch start{2}; + std::vector threads; + threads.reserve(2); + for (std::size_t i = 0; i < 2; ++i) + threads.emplace_back([&, i] { + const auto seed = static_cast(100 + (round * 2) + i); + start.arrive_and_wait(); + results[i] = advance_region_head(*workers[i], key, *lease, *head, + {id(seed), digest(seed), head->end_tick + 1}, 5s); + }); + for (auto& t : threads) + t.join(); + const std::size_t w = results[0] ? 0 : 1; + CHECK(results[w] && failed(results[1 - w], LeaseStoreError::head_mismatch)); + CHECK(results[w]->sequence == head->sequence + 1); + head = *results[w]; + } + auto committed = load_region_head(*workers[1], key, 5s); + CHECK(committed && *committed == *head && committed->sequence == 32); +} + +void exhaustion_fails_closed() { + auto admin = owner(); + const auto key = provision(*admin, 70); + CHECK(admin->query("UPDATE evolution.region_leases SET epoch = 18446744073709551615 " + "WHERE world_id = 1 AND region_id = 70", + {}, 5s)); + auto db = runtime(); + CHECK(failed(acquire_region_lease(*db, key, 20000ms, 5s), LeaseStoreError::exhausted)); + const auto other = provision(*admin, 71); + auto lease = acquire_region_lease(*db, other, 20000ms, 5s); + CHECK(lease); + CHECK(admin->query("UPDATE evolution.region_heads SET commit_sequence = 18446744073709551615, " + "end_tick = 5, published_epoch = 1 WHERE world_id = 1 AND region_id = 71", + {}, 5s)); + auto head = load_region_head(*db, other, 5s); + CHECK(head && head->sequence == 18446744073709551615ULL); + CHECK(failed(advance_region_head(*db, other, *lease, *head, {id(72), digest(72), 6}, 5s), + LeaseStoreError::exhausted)); +} + +struct Summary { + std::vector samples; + void print(const char* name) { + std::sort(samples.begin(), samples.end()); + const auto pick = [&](std::size_t percent) { + return samples[std::min(samples.size() - 1, samples.size() * percent / 100)]; + }; + std::printf("lease_latency_us %s n=%zu p50=%lld p99=%lld max=%lld\n", name, samples.size(), + pick(50), pick(99), samples.back()); + } +}; +template long long timed(F&& f) { + const auto begin = std::chrono::steady_clock::now(); + f(); + return std::chrono::duration_cast(std::chrono::steady_clock::now() - + begin) + .count(); +} + +// Local Unix-socket round trips only; remote TLS latency must be measured in place. +void measure_round_trips() { + auto admin = owner(); + const auto key = provision(*admin, 80); + auto db = runtime(); + constexpr int n = 200; + Summary acquire, renew, advance, release; + auto head = load_region_head(*db, key, 5s); + CHECK(head); + for (int i = 0; i < n; ++i) { + std::expected lease = std::unexpected{LeaseStoreFailure{}}; + acquire.samples.push_back( + timed([&] { lease = acquire_region_lease(*db, key, 20000ms, 5s); })); + CHECK(lease); + renew.samples.push_back( + timed([&] { CHECK(renew_region_lease(*db, key, *lease, 20000ms, 5s)); })); + const auto seed = static_cast(i); + Uuid commit = id(seed); + commit[0] = 0xAA; // Distinct from genesis 0x50 bytes. + commit[1] = static_cast(i >> 8); + advance.samples.push_back(timed([&] { + auto next = advance_region_head(*db, key, *lease, *head, + {commit, digest(seed), head->end_tick + 64}, 5s); + CHECK(next); + head = *next; + })); + release.samples.push_back( + timed([&] { CHECK(release_region_lease(*db, key, *lease, 5s)); })); + } + acquire.print("acquire"); + renew.print("renew"); + advance.print("advance_head"); + release.print("release"); +} +} // namespace + +void run_region_lease_store_tests() { + runtime_cannot_bypass_the_guard(); + acquire_renew_release_and_advance(); + database_time_expiry_and_paused_writer(); + queued_writer_uses_post_lock_database_time(); + partitioned_renewal_fences_locally(); + contenders_serialize_on_the_database(); + exhaustion_fails_closed(); + measure_round_trips(); +} diff --git a/tests/region_lease_tests.cpp b/tests/region_lease_tests.cpp new file mode 100644 index 0000000..ad231b2 --- /dev/null +++ b/tests/region_lease_tests.cpp @@ -0,0 +1,133 @@ +#include "evo/persistence/region_lease.hpp" + +#include +#include +#include +#include + +namespace { +using namespace evo::persistence; +using namespace std::chrono_literals; +using Time = LeaseClock::time_point; +void check(bool ok, std::source_location at = std::source_location::current()) { + if (!ok) + throw std::runtime_error("region lease line " + std::to_string(at.line())); +} +#define CHECK(...) check(static_cast(__VA_ARGS__)) +constexpr Time at(std::chrono::milliseconds t) { + return Time{t}; +} +LeaseGrant grant(std::uint64_t epoch) { + return {{1}, epoch, 7, LeasePolicy{}.duration}; +} + +void policies() { + CHECK(valid(LeasePolicy{})); + CHECK(valid(LeasePolicy{1000ms, 200ms, 100ms, 100ms, 300ms})); + const auto invalid = [](auto change) { + LeasePolicy p; + change(p); + return !valid(p); + }; + CHECK(invalid([](LeasePolicy& p) { p.duration = 999ms; })); + CHECK(invalid([](LeasePolicy& p) { p.duration = 60001ms; })); + CHECK(invalid([](LeasePolicy& p) { p.safety_margin = 1999ms; })); // under 10% + CHECK(invalid([](LeasePolicy& p) { p.safety_margin = 20000ms; })); + CHECK(invalid([](LeasePolicy& p) { p.renew_interval = 0ms; })); + CHECK(invalid([](LeasePolicy& p) { p.renew_timeout = -1ms; })); + CHECK(invalid([](LeasePolicy& p) { p.renew_timeout = 30001ms; })); + CHECK(invalid([](LeasePolicy& p) { p.commit_timeout = 18000ms; })); + // Two complete renewal attempts must fit before local fencing: 2*(7+2.5) > 18. + CHECK(invalid([](LeasePolicy& p) { + p.renew_interval = 7000ms; + p.renew_timeout = 2500ms; + })); + CHECK(!valid(LeasePolicy{2000ms, 400ms, 200ms, 200ms, 1800ms})); + LeaseAuthority broken(LeasePolicy{999ms, 200ms, 100ms, 100ms, 300ms}); + CHECK(!broken.acquired({{1}, 1, 7, 999ms}, at(0ms), at(1ms))); + CHECK(broken.state(at(1ms)) == LeaseState::absent); +} + +void deadlines_follow_send_time() { + LeaseAuthority a{LeasePolicy{}}; + CHECK(a.state(at(0ms)) == LeaseState::absent && !a.may_originate(at(0ms))); + // Reply arrival does not move the deadline: the DB clock started after sending. + CHECK(a.acquired(grant(1), at(1000ms), at(4000ms))); + CHECK(a.deadline() == at(19000ms)); + CHECK(a.state(at(5999ms)) == LeaseState::active); + CHECK(a.state(at(6000ms)) == LeaseState::renew_due && a.may_originate(at(6000ms))); + CHECK(a.may_begin_commit(at(14000ms)) && !a.may_begin_commit(at(14001ms))); + CHECK(a.may_originate(at(18999ms))); + CHECK(a.state(at(19000ms)) == LeaseState::fenced && !a.may_originate(at(19000ms))); + // A renewal answered after the local deadline cannot revive authority. + CHECK(!a.renewed(1, 20000ms, at(18000ms), at(19000ms))); + CHECK(a.state(at(18500ms)) == LeaseState::fenced); +} + +void renewals() { + LeaseAuthority a{LeasePolicy{}}; + CHECK(a.acquired(grant(4), at(0ms), at(10ms))); + CHECK(a.renewed(4, 20000ms, at(5000ms), at(5100ms))); + CHECK(a.deadline() == at(23000ms) && a.state(at(9999ms)) == LeaseState::active); + CHECK(a.state(at(10000ms)) == LeaseState::renew_due); + // Ambiguous renewal keeps the earlier bound; retries still fit before fencing. + a.renewal_unknown(20000ms, at(10000ms)); + CHECK(a.deadline() == at(23000ms) && a.state(at(14999ms)) == LeaseState::active); + a.renewal_unknown(5000ms, at(15000ms)); // Could have shortened: take the earlier. + CHECK(a.deadline() == at(18000ms)); + CHECK(a.renewed(4, 20000ms, at(16000ms), at(16500ms)) && a.deadline() == at(34000ms)); + CHECK(!a.renewed(4, 20000ms, at(15000ms), at(16600ms))); // Out of order: fence. + CHECK(a.state(at(16600ms)) == LeaseState::fenced); + CHECK(!a.renewed(4, 20000ms, at(17000ms), at(17001ms))); // Sticky. + + LeaseAuthority b{LeasePolicy{}}; + CHECK(b.acquired(grant(2), at(0ms), at(1ms))); + CHECK(!b.renewed(3, 20000ms, at(1000ms), at(1001ms))); // Foreign epoch. + CHECK(b.state(at(1001ms)) == LeaseState::fenced); + LeaseAuthority c{LeasePolicy{}}; + CHECK(c.acquired(grant(2), at(0ms), at(1ms))); + CHECK(!c.renewed(2, 19000ms, at(1000ms), at(1001ms))); // Unexpected duration. + LeaseAuthority d{LeasePolicy{}}; + CHECK(d.acquired(grant(2), at(0ms), at(1ms))); + d.fence(); + CHECK(d.state(at(2ms)) == LeaseState::fenced && !d.may_begin_commit(at(2ms))); +} + +void epochs_are_never_reused() { + LeaseAuthority a{LeasePolicy{}}; + CHECK(!a.acquired(grant(0), at(0ms), at(1ms))); + CHECK(!a.acquired({{1}, 1, 0, 20000ms}, at(0ms), at(1ms))); // Zero frame. + CHECK(!a.acquired({{1}, 1, 7, 10000ms}, at(0ms), at(1ms))); // Not the policy. + CHECK(!a.acquired(grant(1), at(10ms), at(9ms))); // Reply before send. + CHECK(!a.acquired(grant(1), at(0ms), at(18000ms))); // Too late to use. + CHECK(a.acquired(grant(3), at(0ms), at(1ms))); + CHECK(!a.acquired(grant(3), at(2ms), at(3ms))); + CHECK(a.state(at(3ms)) == LeaseState::fenced); + CHECK(!a.acquired(grant(2), at(4ms), at(5ms))); + // Only a strictly newer database epoch clears fencing (followed by recovery). + CHECK(a.acquired(grant(4), at(6ms), at(7ms)) && a.state(at(7ms)) == LeaseState::active); + const auto& current = a.grant(); + CHECK(current && current->epoch == 4); +} + +void clock_is_monotonic() { + const auto first = LeaseClock::now(); + const auto second = LeaseClock::now(); + CHECK(second >= first && first.time_since_epoch().count() > 0); +} +} // namespace + +int main() { + try { + policies(); + deadlines_follow_send_time(); + renewals(); + epochs_are_never_reused(); + clock_is_monotonic(); + std::puts("region lease contracts passed"); + return 0; + } catch (const std::exception& e) { + std::fprintf(stderr, "%s\n", e.what()); + return 1; + } +} diff --git a/tools/ci/postgres-check.sh b/tools/ci/postgres-check.sh index 0e0e12a..5d5f764 100644 --- a/tools/ci/postgres-check.sh +++ b/tools/ci/postgres-check.sh @@ -11,10 +11,15 @@ unset PGHOSTADDR PGPASSWORD PGSERVICE PGSERVICEFILE initdb -D "$test_dir/data" --no-locale --encoding=UTF8 --auth-local=trust --auth-host=reject > "$test_dir/init.log" trap 'pg_ctl -D "$test_dir/data" -m immediate stop > /dev/null || true' EXIT pg_ctl -D "$test_dir/data" -l "$test_dir/server.log" -o "-k $test_dir -p 55432 -c listen_addresses=''" -w start -psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_runtime_test NOLOGIN' +# Runtime group role receives grants; a separate login inherits only that role. +psql -X -v ON_ERROR_STOP=1 -c 'CREATE ROLE evo_runtime_test NOLOGIN' \ + -c 'CREATE ROLE evo_runtime_login LOGIN IN ROLE evo_runtime_test' +export EVO_PG_RUNTIME_ROLE=evo_runtime_test EVO_PG_RUNTIME_USER=evo_runtime_login bash "$root/tools/migrate-postgres.sh" bash "$root/tools/migrate-postgres.sh" # exact supported upgrade/restart is a no-op "${1:?path to evo-postgres-tests}" +v1_sha="$(sha256sum "$root/migrations/0001_region_bootstrap.sql")" +v1_sha="${v1_sha%% *}" # Migration failure must not publish a partial schema. createdb migration_rollback if (export PGDATABASE=migration_rollback; psql -X -v ON_ERROR_STOP=1 -1 \ @@ -22,9 +27,29 @@ if (export PGDATABASE=migration_rollback; psql -X -v ON_ERROR_STOP=1 -1 \ echo 'Expected failed migration' >&2; exit 1 fi test "$(PGDATABASE=migration_rollback psql -XAtc "SELECT count(*) FROM pg_namespace WHERE nspname='evolution'")" = 0 +# A privileged or owner runtime role is refused and leaves no schema behind. +if (export PGDATABASE=migration_rollback EVO_PG_RUNTIME_ROLE="$PGUSER"; \ + bash "$root/tools/migrate-postgres.sh"); then + echo 'Expected privileged runtime role rejection' >&2; exit 1 +fi +if (export PGDATABASE=migration_rollback EVO_PG_RUNTIME_ROLE=no_such_evo_role; \ + bash "$root/tools/migrate-postgres.sh"); then + echo 'Expected missing runtime role rejection' >&2; exit 1 +fi +test "$(PGDATABASE=migration_rollback psql -XAtc "SELECT count(*) FROM pg_namespace WHERE nspname='evolution'")" = 0 +# Supported upgrade path: an existing exact v1 database migrates to v2. +createdb upgrade_path +PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" \ + -f "$root/migrations/0001_region_bootstrap.sql" > /dev/null +test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1 +PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh" +test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2 # Unknown future versions and checksum drift must not be repaired silently. -psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=2' +psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=3 WHERE version=2' if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi -psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=1,sha256=repeat('0',64)" +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=2 WHERE version=3" +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=2" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi -echo 'postgres migrations, rollback, constraints and permissions passed' +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64)" +if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi +echo 'postgres migrations, rollback, constraints, permissions and fencing passed' diff --git a/tools/migrate-postgres.sh b/tools/migrate-postgres.sh index 1d69b97..964e101 100644 --- a/tools/migrate-postgres.sh +++ b/tools/migrate-postgres.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -euo pipefail root="$(cd "$(dirname "$0")/.." && pwd)" +# Operator-created NOLOGIN group role granted to the region runtime login (ADR-0060). +runtime_role="${EVO_PG_RUNTIME_ROLE:?set EVO_PG_RUNTIME_ROLE to the runtime group role}" if [ -n "${EVO_PGPASS_SOURCE:-}" ]; then umask 077 export PGPASSFILE @@ -12,5 +14,8 @@ fi # Never print the connection or enable shell tracing here. digest="$(sha256sum "$root/migrations/0001_region_bootstrap.sql")" digest="${digest%% *}" +digest_2="$(sha256sum "$root/migrations/0002_region_fencing.sql")" +digest_2="${digest_2%% *}" psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ + --set="migration_sha_2=$digest_2" --set="runtime_role=$runtime_role" \ --file="$root/migrations/apply.sql"