diff --git a/CMakeLists.txt b/CMakeLists.txt index 03d9878..2dae047 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -219,6 +219,12 @@ if(EVO_BUILD_POSTGRES) target_compile_definitions(evo-server PRIVATE EVO_SERVER_SESSIONS=1) endif() evo_owned_target(evo-server) + if(EVO_BUILD_NETWORK) + # M10.3c durable account control for region sessions (ADR-0091, schema v10). + add_library(evo_control_ledger_postgres STATIC src/server/session/postgres_control_ledger.cpp) + target_link_libraries(evo_control_ledger_postgres PUBLIC evo::postgres evo_network_protocol) + evo_owned_target(evo_control_ledger_postgres) + endif() # M7.9 clone and history audit tool (ADR-0067). add_executable(evo-world-clone apps/evo-world-clone/main.cpp) target_link_libraries(evo-world-clone PRIVATE evo::region_store) @@ -226,6 +232,7 @@ if(EVO_BUILD_POSTGRES) if(BUILD_TESTING) # Contention tests use one thread and one connection per contender. add_executable(evo-postgres-tests tests/postgres_tests.cpp tests/region_lease_store_tests.cpp + tests/account_control_store_tests.cpp tests/region_checkpoint_store_tests.cpp tests/region_host_tests.cpp tests/region_server_process_tests.cpp tests/region_recovery_tests.cpp tests/region_ops_tests.cpp tests/region_clone_tests.cpp @@ -235,7 +242,7 @@ if(EVO_BUILD_POSTGRES) if(EVO_BUILD_NETWORK) # M9.7: real evo-server sessions over TLS (ADR-0083). target_sources(evo-postgres-tests PRIVATE tests/region_session_tests.cpp) - target_link_libraries(evo-postgres-tests PRIVATE evo_network_session) + target_link_libraries(evo-postgres-tests PRIVATE evo_network_session evo_control_ledger_postgres) target_compile_definitions(evo-postgres-tests PRIVATE EVO_POSTGRES_SESSIONS=1) endif() add_dependencies(evo-postgres-tests evo-server evo-world-clone) # Spawned by process tests. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index ecd875a..15ca9f8 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -521,7 +521,8 @@ hardware (10.9). 10.2 split: 10.2a selection, pins and build of msquic/OpenSSL d 10.2c loss/head-of-line measurements done (networking/M10_TRANSPORT_MEASUREMENTS.md). 10.3 split (ADR-0091): 10.3a account ids, ECDSA grants, identity adapter and region session control done; 10.3b QUIC session login (schemas/network/session-v1.md) done; -10.3c durable control lease follows. 10.4-10.9 planned. +10.3c durable per-world account control (schema v10) done. Process wiring into +evo-server and the client comes with 10.4. 10.4-10.9 planned. **Entry:** M9 client/session evidence and M7/M8 dev/stage durability. Read NETWORK_ARCHITECTURE, PERSISTENCE_ARCHITECTURE and ADR-0017. The M9 transport is diff --git a/docs/STATUS.md b/docs/STATUS.md index 56f7086..9476b05 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -21,8 +21,11 @@ session grants verified with public keys only, the identity adapter and grant is library, and per-region session control (newest grant wins and replaces the old session, replay and stale grants refused, in-session refresh, attempt rate limits). M10.3b adds the session login over QUIC (session-v1: Hello, Welcome, Refresh, Close with reasons) on both -the region and client side, tested end to end on loopback. Next: the durable control -lease (10.3c). +the region and client side, tested end to end on loopback. M10.3c adds durable per-world +account control (schema v10): grant serials survive restarts and apply across regions, +cross-region takeover waits out the previous holder, and sessions close when their +control is fenced or cannot be renewed. Next: M10.4 intentions and durable outcomes, +including wiring M10 sessions into evo-server and the client. ## Previous gate - 2026-10-02 diff --git a/docs/adr/0091-m10-session-identity-and-control.md b/docs/adr/0091-m10-session-identity-and-control.md index 54b7736..26524c7 100644 --- a/docs/adr/0091-m10-session-identity-and-control.md +++ b/docs/adr/0091-m10-session-identity-and-control.md @@ -103,3 +103,33 @@ message with an `EVSN` header on the QUIC reliable stream. place; any Close, malformed message, deadline or transport loss ends the session. - Not yet wired into evo-server or the engineering client: they keep the M9 dev path until the durable control lease (10.3c) exists, so nothing deploys M10 sessions. + +## M10.3c durable account control (2026-10-03) +Schema v10 (`migrations/0010_account_control.sql`) keeps one row per world and account: +the highest grant serial any region of the world accepted, and the region session holding +control, with a fencing generation and a database-time expiry. The runtime reaches it +only through `claim_account_control`, `renew_account_control` and +`release_account_control`; each first requires the caller's exact unexpired region lease +(lock order: upgrade lock, region lease row, account control row). +- **Serials are durable and world-wide.** A claim with a serial not above the stored one + is `stale_grant`, so a replayed or older grant fails after a region restart and in every + other region of the world. This closes the 10.3a limits. +- **Newest grant wins across regions without overlap.** A claim may take control from an + unexpired holder in another region; it then returns that holder's remaining database + time as a wait, and the region delays Welcome until the wait has passed (the old holder + stops before its own expiry, as with region leases, and its next renewal is fenced). + Control held by the claimant's own region needs no wait: under the current epoch the + region closes its own session, and an older epoch's holder stopped before the current + region lease could be acquired. +- **Session layer:** `ControlLedger` (transport-neutral interface) with a PostgreSQL + implementation (`PostgresControlLedger`, Linux). `RegionSessions` with a ledger runs + `check()` (no side effects), then the durable claim, then local admission; renews every + 5 s; closes with `superseded` when its control is fenced (another region accepted a newer + grant) without releasing it, and with `shutdown` when renewals cannot be confirmed before + its local deadline (send time plus granted duration minus a 2 s margin). Refresh records + the newer serial for the same session. Close and transport loss release control. + Durations: 20 s control, 5 s renewal, 2 s margin (as for region leases). +- Organism binding is not part of v10: no controllable organism exists yet. It is added + by a later migration when organisms arrive, under the same row and fencing. +- Not yet wired into evo-server or the engineering client (both keep the M9 dev path); + that integration comes with 10.4, so nothing deploys M10 sessions yet. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 62c9ed0..6a30f9e 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -4165,3 +4165,41 @@ directory is refused and its marker file preserved. `git diff --check` passed. Not done: no end-to-end trace against a server deliberately configured to reject an intention; no Docker/Gitea run. The return path is compile-checked but rejection is not covered by a dedicated application integration test. + +## 2026-10-03 — M10.3c durable account control (ADR-0091, schema v10) + +Implemented migration v10 (`account_controls`, three fenced functions, apply chain, +privilege audit), the store adapter (`account_control_store.hpp` in evo_postgres), the +`ControlLedger` interface and `SessionControl::check`, `PostgresControlLedger` +(evo_control_ledger_postgres) and the ledger path of `RegionSessions`. +Database tests (`tests/account_control_store_tests.cpp`, world 9100, real runtime login): +runtime cannot read the table; first claim; replay and older serial refused; same-region +takeover without wait and fencing of the old generation; cross-region claim waiting +15-20 s of the previous 20 s control, fencing the old region's renewal and release; renewal +never shortening the waited expiry; release and renewal after release; durable serials +after a region restart under a new epoch with no wait for the older epoch and the old +lease fenced; database-time expiry ending control and not being waited for; invalid +durations, serials, sessions and the zero account refused; two regions presenting the +same serial concurrently with exactly one winner; the PostgreSQL ledger's claim, stale, +renew, release and missing-lease mapping. `postgres-check.sh`: history probes moved to +v11/v10 checksum drift; audit refuses SELECT on `account_controls` and a missing EXECUTE +on `claim_account_control`; migration tests expect history 1..10. +Session layer with a scripted ledger (`evo-quic-session-tests`): durable stale, fenced and +undecided claims refused without admission; claim names account, serial and the Welcome +session; periodic renewals; refresh records the newer serial; fenced renewal closes as +`superseded` without release; Welcome delayed by the claim's wait; undecided renewals +close as `shutdown` before the control lapses and release it; a refresh finding a newer +grant elsewhere closes as `superseded` with no refresh answer; client close releases. + +Found and fixed during development: a refresh that found a newer grant elsewhere first +sent an "accepted" RefreshResult with expiry 0; a `postgres-check.sh` probe that used +version 10 as an unknown future version; a commit-id collision between test files. + +Executed: Linux GCC 14 `tools/ci/postgres-check.sh` complete (all integration suites, +backup, host-loss and upgrade rehearsals, migration and audit probes); clang-tidy 19 on +the store, ledger and database tests (Clang build with network and PostgreSQL) and on the +session sources and tests; format check; GCC preset `quic` CTest 4/4, session tests 10/10 +repeated; Clang 19 ASan/UBSan session and network tests; Windows MSVC 19.44 preset `quic` +build and CTest 4/4 (binary timestamp checked). +Not done: evo-server and client integration (10.4), ASan of the PostgreSQL suite, +a real two-process cross-region takeover with QUIC clients, cluster migration (no deploy). diff --git a/docs/infrastructure/POSTGRESQL.md b/docs/infrastructure/POSTGRESQL.md index ac0eb58..d1811ee 100644 --- a/docs/infrastructure/POSTGRESQL.md +++ b/docs/infrastructure/POSTGRESQL.md @@ -178,6 +178,18 @@ There is no operator CLI for provisioning original regions yet, and no real worl has been provisioned. Migrating v2→v3 fails closed if any v2 head exists, because v2 heads have no payload. +## Account control (schema v10, ADR-0091) + +`evolution.account_controls` holds, per world and account, the highest accepted grant +serial and the region session in control (generation, region epoch, database-time +expiry). Runtime access is only through `claim_account_control`, +`renew_account_control` and `release_account_control`, each under the caller's exact +unexpired region lease. A claim refuses serials not above the stored one (`EVL11`), +returns a wait when another region still holds control, and bumps the generation; +renewal requires the same generation, region and epoch and an unexpired control +(`EVL12` otherwise). The privilege audit includes all three functions. Lock order: upgrade +lock (shared), region lease row, account control row. + ## Host lease discipline Hosts use `LeaseAuthority` with `LeasePolicy` defaults: 20 s lease, 5 s renewal diff --git a/include/evo/network/session_control.hpp b/include/evo/network/session_control.hpp index c2b3ef8..ddff2fa 100644 --- a/include/evo/network/session_control.hpp +++ b/include/evo/network/session_control.hpp @@ -2,6 +2,7 @@ #include "evo/network/admission.hpp" #include "evo/world/chunks.hpp" #include +#include #include #include #include @@ -94,6 +95,14 @@ class SessionControl { [[nodiscard]] static std::expected create(ControlConfig config); // Region-wide attempt limit, checked before any signature work. Counts the attempt. [[nodiscard]] std::expected attempt(std::uint64_t now_seconds); + // Whether open() would accept the grant now, without changing anything (no attempt is + // counted). Lets the host take durable control before admitting the session locally. + [[nodiscard]] std::expected check(const VerifiedSessionGrant& grant, + std::uint64_t now_seconds) const; + // The id the next successful open() assigns. + [[nodiscard]] std::uint64_t next_session() const noexcept { + return next_session_; + } // A newer grant for an account that already holds a session takes control: the old // session is closed first and reported as superseded. Control is never shared. [[nodiscard]] std::expected open(const VerifiedSessionGrant& grant, @@ -125,6 +134,9 @@ class SessionControl { std::uint32_t attempts{}; }; explicit SessionControl(ControlConfig config) : config_{config} {} + [[nodiscard]] std::optional claims_error(const VerifiedSessionGrant& grant, + std::uint64_t now_seconds) const; + [[nodiscard]] bool forgettable(const Account& account, std::uint64_t now_seconds) const; [[nodiscard]] std::expected admit(const VerifiedSessionGrant& grant, std::uint64_t now_seconds); void accept(Account& account, const GrantClaims& claims); @@ -135,4 +147,34 @@ class SessionControl { std::uint32_t region_attempts_{}; std::vector accounts_; }; + +// Durable per-world account control (10.3c, schema v10). The region host implements it +// over its database lease; tests use fakes. Calls block for one statement each. +enum class ControlLedgerError : std::uint8_t { + stale_grant, // The world already accepted this or a newer serial for the account. + fenced, // This region's lease or this control generation is no longer current. + unavailable // Not decided (database unreachable, timeout, or an unknown outcome). +}; +struct ControlLease { + std::uint64_t generation{}; + // Another region still held control: do not act for the account before this passes. + std::chrono::milliseconds wait{}; + std::chrono::milliseconds granted{}; // From the request send, after the wait. +}; +class ControlLedger { + public: + ControlLedger() = default; + ControlLedger(const ControlLedger&) = delete; + ControlLedger& operator=(const ControlLedger&) = delete; + virtual ~ControlLedger() = default; + [[nodiscard]] virtual std::expected + claim(const AccountId& account, std::uint64_t serial, std::uint64_t session) = 0; + [[nodiscard]] virtual std::expected + renew(const AccountId& account, std::uint64_t generation) = 0; + virtual void release(const AccountId& account, std::uint64_t generation) = 0; + + protected: + ControlLedger(ControlLedger&&) = default; + ControlLedger& operator=(ControlLedger&&) = default; +}; } // namespace evo::network diff --git a/include/evo/persistence/account_control_store.hpp b/include/evo/persistence/account_control_store.hpp new file mode 100644 index 0000000..5bebc39 --- /dev/null +++ b/include/evo/persistence/account_control_store.hpp @@ -0,0 +1,46 @@ +#pragma once +#include "evo/persistence/postgres.hpp" +#include "evo/persistence/region_lease.hpp" +#include +#include +#include + +// Linux-only calls to the durable account control functions of schema v10 (ADR-0091, +// migrations/0010_account_control.sql). Each is one statement under the caller's region +// lease: a fenced or expired lease refuses everything. No retries. +namespace evo::persistence { +enum class ControlStoreError : std::uint8_t { + stale_grant, // A grant with this or a newer serial was already accepted in the world. + fenced, // Region lease or this control generation is no longer current. + exhausted, // Generations would wrap: fail closed. + invalid, // Rejected arguments; nothing changed. + store, // Other SQL/configuration failure; nothing committed. + unknown, // May have executed; reconnect and reconcile before reuse. +}; +struct ControlStoreFailure { + ControlStoreError kind{}; + PgFailure cause{}; +}; +struct AccountControlClaim { + std::uint64_t generation{}; + // Remaining database time of another region's (or epoch's) unexpired control. The + // claimant must not act for the account before it passes (measured from its send). + std::chrono::milliseconds wait{}; + std::chrono::milliseconds granted{}; // Valid after the wait, from the request send. +}; +// Takes control of `account` in the lease's world for `session`, recording `serial` as the +// world's newest accepted grant serial for the account. Duration 1..60 s. +[[nodiscard]] std::expected +claim_account_control(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& account, + std::uint64_t serial, std::uint64_t session, + std::chrono::milliseconds duration, std::chrono::milliseconds timeout); +// Returns the granted duration; the caller measures it from its request send time. +[[nodiscard]] std::expected +renew_account_control(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& account, + std::uint64_t generation, std::chrono::milliseconds duration, + std::chrono::milliseconds timeout); +// True when released, false when control had already moved on. +[[nodiscard]] std::expected +release_account_control(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& account, + std::uint64_t generation, std::chrono::milliseconds timeout); +} // namespace evo::persistence diff --git a/include/evo/server/postgres_control_ledger.hpp b/include/evo/server/postgres_control_ledger.hpp new file mode 100644 index 0000000..02e4d67 --- /dev/null +++ b/include/evo/server/postgres_control_ledger.hpp @@ -0,0 +1,34 @@ +#pragma once +#include "evo/network/session_control.hpp" +#include "evo/persistence/account_control_store.hpp" +#include +#include +#include +#include + +// M10.3c durable account control over the region's PostgreSQL lease (ADR-0091, schema v10). +// Linux only. Owned by the region host's thread, on the host's own connection. `lease` +// returns the host's current region lease, or nothing while it holds none (every call then +// reports `fenced`). Store failures that may have executed or are undecided report +// `unavailable`; the session layer then keeps or closes sessions by its own deadlines. +namespace evo::server { +class PostgresControlLedger final : public network::ControlLedger { + public: + using LeaseSource = std::function()>; + PostgresControlLedger(persistence::Postgres& db, persistence::RegionKey key, LeaseSource lease, + std::chrono::milliseconds duration = std::chrono::milliseconds{20000}, + std::chrono::milliseconds timeout = std::chrono::milliseconds{2000}) + : db_{&db}, key_{key}, lease_{std::move(lease)}, duration_{duration}, timeout_{timeout} {} + [[nodiscard]] std::expected + claim(const network::AccountId& account, std::uint64_t serial, std::uint64_t session) override; + [[nodiscard]] std::expected + renew(const network::AccountId& account, std::uint64_t generation) override; + void release(const network::AccountId& account, std::uint64_t generation) override; + + private: + persistence::Postgres* db_; + persistence::RegionKey key_; + LeaseSource lease_; + std::chrono::milliseconds duration_, timeout_; +}; +} // namespace evo::server diff --git a/include/evo/server/region_sessions.hpp b/include/evo/server/region_sessions.hpp index 9739722..db77322 100644 --- a/include/evo/server/region_sessions.hpp +++ b/include/evo/server/region_sessions.hpp @@ -19,6 +19,9 @@ // account's previous connection with `superseded`. Refusals and closes send Close with a // reason, then the connection is closed after a short flush. Intentions and views are // later segments (10.4, 10.5). Events carry account log labels only, never grants. +// With a ControlLedger, control is durable and unique per world: a session that loses its +// durable control (another region accepted a newer grant) closes with `superseded`, and +// one whose control cannot be renewed in time closes with `shutdown`. namespace evo::server { struct IssuerKey { std::uint32_t key_id{}; @@ -30,6 +33,12 @@ struct RegionSessionsConfig { std::vector issuer_keys; std::chrono::milliseconds hello_deadline{5000}; // Connected to Hello (ADR-0089). std::chrono::milliseconds close_flush{1000}; // Close sent to connection closed. + // Durable per-world control (10.3c), owned by the host; null keeps control in memory + // only (tests, tools). With a ledger, a session is admitted only after its durable + // claim, waits out another region's control before Welcome, renews every + // `renew_interval` and closes before its control can lapse (`safety_margin`). + network::ControlLedger* ledger{}; + std::chrono::milliseconds renew_interval{5000}, safety_margin{2000}; }; enum class RegionSessionsError : std::uint8_t { configuration, keys, transport }; struct RegionSessionEvent { diff --git a/migrations/0010_account_control.sql b/migrations/0010_account_control.sql new file mode 100644 index 0000000..a6617df --- /dev/null +++ b/migrations/0010_account_control.sql @@ -0,0 +1,156 @@ +-- M10.3c durable account control (ADR-0091). One row per (world, account): the highest +-- grant serial any region of the world accepted for the account, and the region session +-- that holds control, with a fencing generation and a database-time expiry. The runtime +-- reaches it only through the three functions below, each of which first requires the +-- caller's exact unexpired region lease (check_region_lease), so only a region's current +-- authority can grant, keep or release control. +-- +-- Lock order (extends schema v9): upgrade lock (shared) -> region lease row -> account +-- control row. A claim never locks another region's lease row. +-- +-- Newest grant wins across regions: a claim with a higher serial takes control even from +-- an unexpired holder in another region. It returns the remaining time of that holder's +-- expiry; the claimant must not act for the account until that wait has passed, by which +-- time the old holder has stopped (it stops before its own expiry, as with region leases) +-- and its next renewal is fenced. Control held by this region needs no wait: under the +-- current epoch the claimant closes its own session, and an older epoch's holder already +-- stopped before this epoch's region lease could be acquired. +CREATE TABLE evolution.account_controls ( + world_id evolution.u64 NOT NULL, + account_id uuid NOT NULL CHECK (account_id <> '00000000-0000-0000-0000-000000000000'), + serial evolution.u64 NOT NULL CHECK (serial > 0), + generation evolution.u64 NOT NULL CHECK (generation > 0), + region_id evolution.u64, + region_epoch evolution.u64, + session_id evolution.u64, + expires_at timestamptz, + PRIMARY KEY (world_id, account_id), + CHECK ((region_id IS NULL) = (region_epoch IS NULL)), + CHECK ((region_id IS NULL) = (session_id IS NULL)), + CHECK ((region_id IS NULL) = (expires_at IS NULL)) +); +REVOKE ALL ON evolution.account_controls FROM PUBLIC; + +CREATE FUNCTION evolution.claim_account_control(p_world evolution.u64, p_region evolution.u64, + p_holder uuid, p_epoch evolution.u64, p_account uuid, p_serial evolution.u64, + p_session evolution.u64, p_duration_ms integer) +RETURNS TABLE (control_generation evolution.u64, wait_ms integer, granted_ms integer) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + c evolution.account_controls%ROWTYPE; + t timestamptz; + pending interval; +BEGIN + IF p_account IS NULL OR p_account = '00000000-0000-0000-0000-000000000000' + OR p_serial IS NULL OR p_serial = 0 OR p_session IS NULL OR p_session = 0 + OR p_duration_ms IS NULL OR p_duration_ms NOT BETWEEN 1000 AND 60000 THEN + RAISE EXCEPTION 'invalid control request' USING ERRCODE = '22023'; + END IF; + PERFORM evolution.check_region_lease(p_world, p_region, p_holder, p_epoch); + SELECT * INTO c FROM evolution.account_controls a + WHERE a.world_id = p_world AND a.account_id = p_account FOR UPDATE; + IF NOT FOUND THEN + t := clock_timestamp(); + INSERT INTO evolution.account_controls(world_id, account_id, serial, generation, + region_id, region_epoch, session_id, expires_at) + VALUES (p_world, p_account, p_serial, 1, p_region, p_epoch, p_session, + t + p_duration_ms * interval '1 millisecond') + ON CONFLICT DO NOTHING; + IF FOUND THEN + control_generation := 1; + wait_ms := 0; + granted_ms := p_duration_ms; + RETURN NEXT; + RETURN; + END IF; + -- A concurrent first claim committed meanwhile: decide against it. + SELECT * INTO c FROM evolution.account_controls a + WHERE a.world_id = p_world AND a.account_id = p_account FOR UPDATE; + END IF; + -- Read after the row lock wait, as with region leases. + t := clock_timestamp(); + IF p_serial <= c.serial THEN + RAISE EXCEPTION 'stale grant' USING ERRCODE = 'EVL11'; + END IF; + IF c.generation = 18446744073709551615 THEN + RAISE EXCEPTION 'control generations exhausted' USING ERRCODE = 'EVL05'; + END IF; + -- Only another region's unexpired holder must be outlived (see above). + pending := CASE + WHEN c.region_id IS NULL OR c.expires_at <= t THEN interval '0' + WHEN c.region_id = p_region AND c.region_epoch <= p_epoch THEN interval '0' + ELSE c.expires_at - t END; + UPDATE evolution.account_controls a + SET serial = p_serial, generation = c.generation + 1, region_id = p_region, + region_epoch = p_epoch, session_id = p_session, + expires_at = t + pending + p_duration_ms * interval '1 millisecond' + WHERE a.world_id = p_world AND a.account_id = p_account + RETURNING a.generation INTO control_generation; + wait_ms := ceil(extract(epoch FROM pending) * 1000)::integer; + granted_ms := p_duration_ms; + RETURN NEXT; +END $$; + +-- Extends the caller's control from database time; the generation, region and epoch +-- must still match and the control must be unexpired. Expiry is final, as for leases. +CREATE FUNCTION evolution.renew_account_control(p_world evolution.u64, p_region evolution.u64, + p_holder uuid, p_epoch evolution.u64, p_account uuid, p_generation evolution.u64, + p_duration_ms integer) +RETURNS integer LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + c evolution.account_controls%ROWTYPE; + t timestamptz; +BEGIN + IF p_account IS NULL OR p_generation IS NULL OR p_duration_ms IS NULL + OR p_duration_ms NOT BETWEEN 1000 AND 60000 THEN + RAISE EXCEPTION 'invalid control request' USING ERRCODE = '22023'; + END IF; + PERFORM evolution.check_region_lease(p_world, p_region, p_holder, p_epoch); + SELECT * INTO c FROM evolution.account_controls a + WHERE a.world_id = p_world AND a.account_id = p_account FOR UPDATE; + t := clock_timestamp(); + IF NOT FOUND OR c.generation <> p_generation OR c.region_id IS DISTINCT FROM p_region + OR c.region_epoch IS DISTINCT FROM p_epoch OR c.expires_at <= t THEN + RAISE EXCEPTION 'account control fenced' USING ERRCODE = 'EVL12'; + END IF; + -- Never shortens: a claim that waited out another holder keeps its later expiry. + UPDATE evolution.account_controls a + SET expires_at = greatest(c.expires_at, t + p_duration_ms * interval '1 millisecond') + WHERE a.world_id = p_world AND a.account_id = p_account; + RETURN p_duration_ms; +END $$; + +-- Gives up the caller's control (the serial stays). True when released, false when the +-- control had already moved on (nothing changed). +CREATE FUNCTION evolution.release_account_control(p_world evolution.u64, p_region evolution.u64, + p_holder uuid, p_epoch evolution.u64, p_account uuid, p_generation evolution.u64) +RETURNS boolean LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +BEGIN + IF p_account IS NULL OR p_generation IS NULL THEN + RAISE EXCEPTION 'invalid control request' USING ERRCODE = '22023'; + END IF; + PERFORM evolution.check_region_lease(p_world, p_region, p_holder, p_epoch); + UPDATE evolution.account_controls a + SET region_id = NULL, region_epoch = NULL, session_id = NULL, expires_at = NULL + WHERE a.world_id = p_world AND a.account_id = p_account AND a.generation = p_generation + AND a.region_id = p_region AND a.region_epoch = p_epoch; + RETURN FOUND; +END $$; + +REVOKE ALL ON FUNCTION + evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, + evolution.u64, evolution.u64, integer), + evolution.renew_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, + evolution.u64, integer), + evolution.release_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, + evolution.u64) + FROM PUBLIC; +GRANT EXECUTE ON FUNCTION + evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, + evolution.u64, evolution.u64, integer), + evolution.renew_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, + evolution.u64, integer), + evolution.release_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, + evolution.u64) + TO :"runtime_role"; +INSERT INTO evolution.schema_migrations(version, sha256) VALUES (10, :'migration_sha_10'); diff --git a/migrations/apply.sql b/migrations/apply.sql index 3b9c1d7..a3352ad 100644 --- a/migrations/apply.sql +++ b/migrations/apply.sql @@ -40,22 +40,33 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') || ',3:' || :'migration_sha_3' || ',4:' || :'migration_sha_4' || ',5:' || :'migration_sha_5' || ',6:' || :'migration_sha_6' || ',7:' || :'migration_sha_7' || ',8:' || :'migration_sha_8' - || ',9:' || :'migration_sha_9' AS at_v9 \gset - \if :at_v9 + || ',9:' || :'migration_sha_9' AS at_v9, + :'history' = '1:' || :'migration_sha' || ',2:' || :'migration_sha_2' + || ',3:' || :'migration_sha_3' || ',4:' || :'migration_sha_4' + || ',5:' || :'migration_sha_5' || ',6:' || :'migration_sha_6' + || ',7:' || :'migration_sha_7' || ',8:' || :'migration_sha_8' + || ',9:' || :'migration_sha_9' || ',10:' || :'migration_sha_10' AS at_v10 \gset + \if :at_v10 + \elif :at_v9 + \ir 0010_account_control.sql \elif :at_v8 \ir 0009_serialized_authority.sql + \ir 0010_account_control.sql \elif :at_v7 \ir 0008_upgrade_barrier.sql \ir 0009_serialized_authority.sql + \ir 0010_account_control.sql \elif :at_v6 \ir 0007_region_clones.sql \ir 0008_upgrade_barrier.sql \ir 0009_serialized_authority.sql + \ir 0010_account_control.sql \elif :at_v5 \ir 0006_commit_lookup.sql \ir 0007_region_clones.sql \ir 0008_upgrade_barrier.sql \ir 0009_serialized_authority.sql + \ir 0010_account_control.sql \elif :at_v4 \ir pre_v5_guard.sql \ir 0005_batch_decisions.sql @@ -63,6 +74,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0007_region_clones.sql \ir 0008_upgrade_barrier.sql \ir 0009_serialized_authority.sql + \ir 0010_account_control.sql \elif :at_v3 \ir 0004_lineage_profile.sql \ir pre_v5_guard.sql @@ -71,6 +83,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0007_region_clones.sql \ir 0008_upgrade_barrier.sql \ir 0009_serialized_authority.sql + \ir 0010_account_control.sql \elif :at_v2 \ir 0003_region_checkpoints.sql \ir 0004_lineage_profile.sql @@ -80,6 +93,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0007_region_clones.sql \ir 0008_upgrade_barrier.sql \ir 0009_serialized_authority.sql + \ir 0010_account_control.sql \elif :at_v1 \ir 0002_region_fencing.sql \ir 0003_region_checkpoints.sql @@ -90,6 +104,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0007_region_clones.sql \ir 0008_upgrade_barrier.sql \ir 0009_serialized_authority.sql + \ir 0010_account_control.sql \else \echo 'Unknown or changed migration history' DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$; @@ -105,6 +120,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0007_region_clones.sql \ir 0008_upgrade_barrier.sql \ir 0009_serialized_authority.sql + \ir 0010_account_control.sql \endif -- Effective-privilege audit on every run, including no-ops (ADR-0060). Attributes -- alone are insufficient: predefined roles such as pg_write_all_data, PUBLIC @@ -136,7 +152,10 @@ BEGIN ('evolution.load_region_decisions(evolution.u64,evolution.u64)'::regprocedure::oid), ('evolution.pending_region_outbox(evolution.u64,evolution.u64,integer)'::regprocedure::oid), ('evolution.acknowledge_region_outbox(evolution.u64,evolution.u64,integer,integer,evolution.u64)'::regprocedure::oid), - ('evolution.load_region_commit(evolution.u64,evolution.u64,evolution.u64)'::regprocedure::oid) + ('evolution.load_region_commit(evolution.u64,evolution.u64,evolution.u64)'::regprocedure::oid), + ('evolution.claim_account_control(evolution.u64,evolution.u64,uuid,evolution.u64,uuid,evolution.u64,evolution.u64,integer)'::regprocedure::oid), + ('evolution.renew_account_control(evolution.u64,evolution.u64,uuid,evolution.u64,uuid,evolution.u64,integer)'::regprocedure::oid), + ('evolution.release_account_control(evolution.u64,evolution.u64,uuid,evolution.u64,uuid,evolution.u64)'::regprocedure::oid) ), violations(role_oid, reason) AS ( SELECT r.oid, 'privileged attribute or owner identity' FROM pg_catalog.pg_roles r JOIN closure c ON c.oid = r.oid @@ -160,7 +179,7 @@ BEGIN OR EXISTS (SELECT 1 FROM pg_catalog.pg_namespace n WHERE n.nspname = 'public' AND pg_catalog.has_schema_privilege(c.oid, n.oid, 'CREATE')) UNION ALL - -- Only the twelve guarded functions are executable; provisioning/internal are not. + -- Only the fifteen guarded functions are executable; provisioning/internal are not. SELECT c.oid, 'EXECUTE outside the fenced function set' FROM closure c CROSS JOIN pg_catalog.pg_proc p WHERE p.pronamespace = 'evolution'::regnamespace diff --git a/src/network/protocol/session_control.cpp b/src/network/protocol/session_control.cpp index ea881d5..f942f8d 100644 --- a/src/network/protocol/session_control.cpp +++ b/src/network/protocol/session_control.cpp @@ -48,28 +48,62 @@ std::expected SessionControl::attempt(std::uint64_t now_seco return {}; } +bool SessionControl::forgettable(const Account& a, std::uint64_t now_seconds) const { + return a.session == 0 && a.remember_until <= now_seconds && + window_over(a.window_start, now_seconds, config_.attempt_window); +} + void SessionControl::forget_idle(std::uint64_t now_seconds) { - std::erase_if(accounts_, [&](const Account& a) { - return a.session == 0 && a.remember_until <= now_seconds && - window_over(a.window_start, now_seconds, config_.attempt_window); - }); + std::erase_if(accounts_, [&](const Account& a) { return forgettable(a, now_seconds); }); +} + +std::optional SessionControl::claims_error(const VerifiedSessionGrant& grant, + std::uint64_t now_seconds) const { + const auto& c = grant.claims(); + if (grant.environment() != config_.environment) + return ControlError::wrong_environment; + if (c.world != config_.world) + return ControlError::wrong_world; + if (c.region != config_.region) + return ControlError::wrong_region; + if (c.expires_at <= c.issued_at || c.expires_at - c.issued_at > config_.max_grant_lifetime) + return ControlError::invalid_lifetime; + if (c.issued_at > now_seconds && c.issued_at - now_seconds > config_.clock_skew) + return ControlError::not_yet_valid; + if (now_seconds >= c.expires_at) + return ControlError::expired; + return std::nullopt; +} + +std::expected SessionControl::check(const VerifiedSessionGrant& grant, + std::uint64_t now_seconds) const { + if (const auto e = claims_error(grant, now_seconds)) + return std::unexpected(*e); + const auto& c = grant.claims(); + const auto found = + std::ranges::find_if(accounts_, [&](const Account& a) { return a.id == c.account; }); + if (found == accounts_.end()) { + const auto kept = std::ranges::count_if( + accounts_, [&](const Account& a) { return !forgettable(a, now_seconds); }); + if (static_cast(kept) >= config_.max_accounts) + return std::unexpected(ControlError::accounts_full); + } else { + if (!window_over(found->window_start, now_seconds, config_.attempt_window) && + found->attempts >= config_.max_account_attempts) + return std::unexpected(ControlError::rate_limited); + if (c.serial <= found->serial) + return std::unexpected(ControlError::stale_grant); + } + if ((found == accounts_.end() || found->session == 0) && sessions() >= config_.max_sessions) + return std::unexpected(ControlError::sessions_full); + return {}; } std::expected SessionControl::admit(const VerifiedSessionGrant& grant, std::uint64_t now_seconds) { + if (const auto e = claims_error(grant, now_seconds)) + return std::unexpected(*e); const auto& c = grant.claims(); - if (grant.environment() != config_.environment) - return std::unexpected(ControlError::wrong_environment); - if (c.world != config_.world) - return std::unexpected(ControlError::wrong_world); - if (c.region != config_.region) - return std::unexpected(ControlError::wrong_region); - if (c.expires_at <= c.issued_at || c.expires_at - c.issued_at > config_.max_grant_lifetime) - return std::unexpected(ControlError::invalid_lifetime); - if (c.issued_at > now_seconds && c.issued_at - now_seconds > config_.clock_skew) - return std::unexpected(ControlError::not_yet_valid); - if (now_seconds >= c.expires_at) - return std::unexpected(ControlError::expired); forget_idle(now_seconds); auto found = std::ranges::find_if(accounts_, [&](const Account& a) { return a.id == c.account; }); diff --git a/src/persistence/region_lease_store.cpp b/src/persistence/region_lease_store.cpp index 76e164c..6fafca9 100644 --- a/src/persistence/region_lease_store.cpp +++ b/src/persistence/region_lease_store.cpp @@ -1,4 +1,5 @@ #include "evo/persistence/region_lease_store.hpp" +#include "evo/persistence/account_control_store.hpp" #include #include #include @@ -343,4 +344,95 @@ std::expected acknowledge_region_outbox(Postgres& db, R return std::unexpected{malformed(true)}; return *marked == "true"; } + +// Schema v10 account control (ADR-0091). Shares this file's parsing and SQLSTATE rules. +namespace { +ControlStoreFailure control_failure(const PgFailure& f) { + const std::string_view state(f.sqlstate.data(), 5); + auto kind = ControlStoreError::store; + if (f.kind == PgError::unknown) + kind = ControlStoreError::unknown; + else if (f.kind == PgError::config || f.kind == PgError::bounds || state == "22023" || + state == "EVL04") + kind = ControlStoreError::invalid; + else if (state == "EVL11") + kind = ControlStoreError::stale_grant; + else if (state == "EVL02" || state == "EVL06" || state == "EVL12") + kind = ControlStoreError::fenced; + else if (state == "EVL05") + kind = ControlStoreError::exhausted; + return {kind, f}; +} +std::array control_parameters(RegionKey key, const LeaseGrant& lease, + const Uuid& account, std::uint64_t value) { + return {std::to_string(key.world), std::to_string(key.region), uuid_text(lease.holder), + std::to_string(lease.epoch), uuid_text(account), std::to_string(value)}; +} +} // namespace + +std::expected +claim_account_control(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& account, + std::uint64_t serial, std::uint64_t session, milliseconds duration, + milliseconds timeout) { + const auto base = control_parameters(key, lease, account, serial); + const std::array p{base[0], + base[1], + base[2], + base[3], + base[4], + base[5], + std::to_string(session), + std::to_string(duration.count())}; + auto rows = db.query("SELECT * FROM evolution.claim_account_control($1::evolution.u64, " + "$2::evolution.u64, $3::uuid, $4::evolution.u64, $5::uuid, " + "$6::evolution.u64, $7::evolution.u64, $8::integer)", + p, timeout); + if (!rows) + return std::unexpected{control_failure(rows.error())}; + if (rows->size() != 1 || (*rows)[0].size() != 3) + return std::unexpected{ControlStoreFailure{ControlStoreError::unknown, {}}}; + const auto& r = (*rows)[0]; + const auto generation = parse_integer(r[0]); + const auto wait = parse_integer(r[1]); + const auto granted = parse_integer(r[2]); + if (!generation || !wait || !granted || *generation == 0) + return std::unexpected{ControlStoreFailure{ControlStoreError::unknown, {}}}; + return AccountControlClaim{*generation, milliseconds{*wait}, milliseconds{*granted}}; +} + +std::expected +renew_account_control(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& account, + std::uint64_t generation, milliseconds duration, milliseconds timeout) { + const auto base = control_parameters(key, lease, account, generation); + const std::array p{ + base[0], base[1], base[2], base[3], base[4], base[5], std::to_string(duration.count())}; + auto rows = db.query("SELECT evolution.renew_account_control($1::evolution.u64, " + "$2::evolution.u64, $3::uuid, $4::evolution.u64, $5::uuid, " + "$6::evolution.u64, $7::integer)", + p, timeout); + if (!rows) + return std::unexpected{control_failure(rows.error())}; + const auto granted = rows->size() == 1 && (*rows)[0].size() == 1 + ? parse_integer((*rows)[0][0]) + : std::nullopt; + if (!granted) + return std::unexpected{ControlStoreFailure{ControlStoreError::unknown, {}}}; + return milliseconds{*granted}; +} + +std::expected +release_account_control(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& account, + std::uint64_t generation, milliseconds timeout) { + const auto p = control_parameters(key, lease, account, generation); + auto rows = db.query("SELECT evolution.release_account_control($1::evolution.u64, " + "$2::evolution.u64, $3::uuid, $4::evolution.u64, $5::uuid, " + "$6::evolution.u64)::text", + p, timeout); + if (!rows) + return std::unexpected{control_failure(rows.error())}; + const auto text = rows->size() == 1 && (*rows)[0].size() == 1 ? (*rows)[0][0] : std::nullopt; + if (text != "true" && text != "false") + return std::unexpected{ControlStoreFailure{ControlStoreError::unknown, {}}}; + return text == "true"; +} } // namespace evo::persistence diff --git a/src/server/session/postgres_control_ledger.cpp b/src/server/session/postgres_control_ledger.cpp new file mode 100644 index 0000000..75e589a --- /dev/null +++ b/src/server/session/postgres_control_ledger.cpp @@ -0,0 +1,57 @@ +#include "evo/server/postgres_control_ledger.hpp" + +namespace evo::server { +namespace { +using network::ControlLedgerError; +using persistence::ControlStoreError; + +ControlLedgerError ledger_error(ControlStoreError e) { + switch (e) { + case ControlStoreError::stale_grant: + return ControlLedgerError::stale_grant; + case ControlStoreError::fenced: + case ControlStoreError::exhausted: + case ControlStoreError::invalid: // Fail closed: never grant on a rejected request. + return ControlLedgerError::fenced; + default: + return ControlLedgerError::unavailable; + } +} +persistence::Uuid uuid(const network::AccountId& account) { + return account.bytes; +} +} // namespace + +std::expected +PostgresControlLedger::claim(const network::AccountId& account, std::uint64_t serial, + std::uint64_t session) { + const auto lease = lease_(); + if (!lease) + return std::unexpected(ControlLedgerError::fenced); + const auto claimed = persistence::claim_account_control(*db_, key_, *lease, uuid(account), + serial, session, duration_, timeout_); + if (!claimed) + return std::unexpected(ledger_error(claimed.error().kind)); + return network::ControlLease{claimed->generation, claimed->wait, claimed->granted}; +} + +std::expected +PostgresControlLedger::renew(const network::AccountId& account, std::uint64_t generation) { + const auto lease = lease_(); + if (!lease) + return std::unexpected(ControlLedgerError::fenced); + const auto renewed = persistence::renew_account_control(*db_, key_, *lease, uuid(account), + generation, duration_, timeout_); + if (!renewed) + return std::unexpected(ledger_error(renewed.error().kind)); + return *renewed; +} + +void PostgresControlLedger::release(const network::AccountId& account, std::uint64_t generation) { + // Best effort: an unreleased control simply expires; a fenced one already moved on. + if (const auto lease = lease_()) + (void)persistence::release_account_control(*db_, key_, *lease, uuid(account), generation, + timeout_) + .has_value(); +} +} // namespace evo::server diff --git a/src/server/session/region_sessions.cpp b/src/server/session/region_sessions.cpp index c7131f4..3113db9 100644 --- a/src/server/session/region_sessions.cpp +++ b/src/server/session/region_sessions.cpp @@ -28,6 +28,16 @@ SessionCloseReason reason_for(ControlError e) { return SessionCloseReason::unauthenticated; } } +SessionCloseReason reason_for(network::ControlLedgerError e) { + switch (e) { + case network::ControlLedgerError::stale_grant: + return SessionCloseReason::stale_grant; + case network::ControlLedgerError::fenced: + return SessionCloseReason::shutdown; // This region cannot grant control now. + default: + return SessionCloseReason::capacity; // Not decided: the client may retry later. + } +} SessionCloseReason reason_for(SessionGrantError e) { return e == SessionGrantError::wrong_environment ? SessionCloseReason::wrong_scope : SessionCloseReason::unauthenticated; @@ -35,14 +45,20 @@ SessionCloseReason reason_for(SessionGrantError e) { } // namespace struct RegionSessions::Impl { - enum class State : std::uint8_t { hello, established, closing }; + // pending: admitted and durably claimed, waiting out another region's control. + enum class State : std::uint8_t { hello, pending, established, closing }; struct Connection { network::quic::ConnectionId id{}; State state{State::hello}; std::uint64_t session{}; std::string account; - Clock::time_point deadline; + Clock::time_point deadline; // Hello, pending (Welcome due) or close flush. bool close_requested{}; + // Durable control (ledger only). + network::AccountId account_id; + std::uint64_t generation{}; + Clock::time_point control_deadline, next_renew; + network::SessionWelcome welcome; }; Impl(RegionSessionsConfig c, std::unique_ptr e, @@ -64,7 +80,8 @@ struct RegionSessions::Impl { } Connection* by_session(std::uint64_t session) { const auto it = std::ranges::find_if(connections, [&](const Connection& c) { - return c.session == session && c.state == State::established; + return c.session == session && + (c.state == State::established || c.state == State::pending); }); return it == connections.end() ? nullptr : &*it; } @@ -86,6 +103,10 @@ struct RegionSessions::Impl { events.push_back(std::move(event)); if (c.session != 0) // May already be released (superseded, expired, epoch). (void)control.close(c.session).has_value(); + // A superseded session's durable control already moved to its successor. + if (config.ledger && c.generation != 0 && reason != SessionCloseReason::superseded) + config.ledger->release(c.account_id, c.generation); + c.generation = 0; c.session = 0; c.state = State::closing; c.deadline = now + config.close_flush; @@ -100,28 +121,80 @@ struct RegionSessions::Impl { const auto grant = verifier.verify(m.grant); if (!grant) return close(c, reason_for(grant.error()), now); - c.account = network::log_label(grant->claims().account); + const auto& claims = grant->claims(); + c.account = network::log_label(claims.account); + if (const auto checked = control.check(*grant, now_seconds); !checked) + return close(c, reason_for(checked.error()), now); + // Durable control first, so the world never has two controlling regions; then the + // local admission, which check() has just confirmed. + std::optional lease; + const auto promised = control.next_session(); // The session the ledger records. + if (config.ledger) { + auto claimed = config.ledger->claim(claims.account, claims.serial, promised); + if (!claimed) + return close(c, reason_for(claimed.error()), now); + lease = *claimed; + } const auto opened = control.open(*grant, now_seconds); - if (!opened) - return close(c, reason_for(opened.error()), now); + if (!opened || opened->session != promised) { + if (opened) + (void)control.close(opened->session).has_value(); + if (lease) + config.ledger->release(claims.account, lease->generation); + return close(c, opened ? SessionCloseReason::capacity : reason_for(opened.error()), + now); + } if (opened->superseded != 0) if (auto* old = by_session(opened->superseded)) close(*old, SessionCloseReason::superseded, now); - c.state = State::established; c.session = opened->session; - network::SessionWelcome welcome; - welcome.session = opened->session; - welcome.epoch = opened->epoch; - welcome.world = config.control.world.world; - welcome.frame = config.control.world.frame; - welcome.region = config.control.region; - welcome.expires_at = opened->expires_at; - welcome.server_seconds = now_seconds; - send(c, welcome); + c.account_id = claims.account; + c.welcome.session = opened->session; + c.welcome.epoch = opened->epoch; + c.welcome.world = config.control.world.world; + c.welcome.frame = config.control.world.frame; + c.welcome.region = config.control.region; + c.welcome.expires_at = opened->expires_at; + c.welcome.server_seconds = now_seconds; + if (lease) { + c.generation = lease->generation; + c.control_deadline = now + lease->wait + lease->granted - config.safety_margin; + c.next_renew = now + config.renew_interval; + if (lease->wait.count() > 0) { + c.state = State::pending; + c.deadline = now + lease->wait; + return; + } + } + welcome(c); + } + + void welcome(Connection& c) { + c.state = State::established; + send(c, c.welcome); events.push_back({RegionSessionEvent::Kind::opened, c.session, c.account, {}}); } - void refresh(Connection& c, const network::SessionRefresh& m, std::uint64_t now_seconds) { + // Keeps durable control alive; closes sessions that lost it or cannot keep it. + void renew(Connection& c, Clock::time_point now) { + if (!config.ledger || c.generation == 0 || + (c.state != State::established && c.state != State::pending)) + return; + if (now >= c.next_renew) { + c.next_renew = now + config.renew_interval; + const auto renewed = config.ledger->renew(c.account_id, c.generation); + if (renewed) + c.control_deadline = + std::max(c.control_deadline, now + *renewed - config.safety_margin); + else if (renewed.error() != network::ControlLedgerError::unavailable) + return close(c, SessionCloseReason::superseded, now); + } + if (now >= c.control_deadline) + close(c, SessionCloseReason::shutdown, now); + } + + void refresh(Connection& c, const network::SessionRefresh& m, std::uint64_t now_seconds, + Clock::time_point now) { network::SessionRefreshResult result; if (!control.attempt(now_seconds)) { result.status = static_cast(SessionCloseReason::rate_limited); @@ -131,6 +204,21 @@ struct RegionSessions::Impl { result.status = static_cast(renewed.error() == ControlError::wrong_account ? SessionCloseReason::unauthenticated : reason_for(renewed.error())); + } else if (config.ledger) { + // Record the newer serial durably; the session keeps its id. + const auto claimed = + config.ledger->claim(grant->claims().account, grant->claims().serial, c.session); + if (claimed) { + c.generation = claimed->generation; + c.control_deadline = + std::max(c.control_deadline, + now + claimed->wait + claimed->granted - config.safety_margin); + result.expires_at = *renewed; + } else if (claimed.error() == network::ControlLedgerError::unavailable) { + result.status = static_cast(SessionCloseReason::capacity); + } else { // Another region accepted a newer grant: this session is superseded. + return close(c, SessionCloseReason::superseded, now); + } } else { result.expires_at = *renewed; } @@ -141,6 +229,8 @@ struct RegionSessions::Impl { Clock::time_point now) { if (c.state == State::closing) return; + if (c.state == State::pending) // Nothing is expected before Welcome. + return close(c, SessionCloseReason::protocol, now); const auto decoded = network::decode_session(bytes); if (!decoded) return close(c, @@ -154,7 +244,7 @@ struct RegionSessions::Impl { return close(c, SessionCloseReason::protocol, now); } if (const auto* m = std::get_if(&*decoded)) - return refresh(c, *m, now_seconds); + return refresh(c, *m, now_seconds, now); if (const auto* m = std::get_if(&*decoded); m && m->reason == SessionCloseReason::client_closed) { close(c, SessionCloseReason::client_closed, now); @@ -173,7 +263,8 @@ RegionSessions::create(RegionSessionsConfig config) { // Connections beyond sessions are headroom for logins and closes still flushing. if (config.issuer_keys.empty() || config.transport.limits.max_connections < config.control.max_sessions || - config.hello_deadline.count() <= 0 || config.close_flush.count() <= 0) + config.hello_deadline.count() <= 0 || config.close_flush.count() <= 0 || + config.renew_interval.count() <= 0 || config.safety_margin.count() <= 0) return std::unexpected(RegionSessionsError::configuration); auto control = network::SessionControl::create(config.control); if (!control) @@ -204,8 +295,10 @@ void RegionSessions::poll(std::uint64_t epoch, std::uint64_t now_seconds, Clock: for (auto& e : m.endpoint->poll()) { using network::quic::EventKind; if (e.kind == EventKind::connected) { - m.connections.push_back( - {e.connection, Impl::State::hello, 0, {}, now + m.config.hello_deadline, false}); + Impl::Connection c; + c.id = e.connection; + c.deadline = now + m.config.hello_deadline; + m.connections.push_back(std::move(c)); } else if (e.kind == EventKind::message) { if (auto* c = m.find(e.connection)) m.message(*c, e.bytes, now_seconds, now); @@ -214,8 +307,14 @@ void RegionSessions::poll(std::uint64_t epoch, std::uint64_t now_seconds, Clock: m.connections, [&](const Impl::Connection& c) { return c.id == e.connection; }); if (it == m.connections.end()) continue; - if (it->state == Impl::State::established) { + if (it->state == Impl::State::pending) { (void)m.control.close(it->session).has_value(); + if (m.config.ledger && it->generation != 0) + m.config.ledger->release(it->account_id, it->generation); + } else if (it->state == Impl::State::established) { + (void)m.control.close(it->session).has_value(); + if (m.config.ledger && it->generation != 0) + m.config.ledger->release(it->account_id, it->generation); // The client closes its connection right after sending Close, which may be // discarded with it: a peer close is the client leaving, anything else is loss. m.events.push_back({RegionSessionEvent::Kind::closed, it->session, it->account, @@ -230,6 +329,9 @@ void RegionSessions::poll(std::uint64_t epoch, std::uint64_t now_seconds, Clock: if (auto* c = m.by_session(session)) m.close(*c, SessionCloseReason::expired, now); for (auto& c : m.connections) { + m.renew(c, now); + if (c.state == Impl::State::pending && now >= c.deadline) + m.welcome(c); if (c.state == Impl::State::hello && now >= c.deadline) m.close(c, SessionCloseReason::timeout, now); if (c.state == Impl::State::closing && now >= c.deadline && !c.close_requested) { diff --git a/tests/account_control_store_tests.cpp b/tests/account_control_store_tests.cpp new file mode 100644 index 0000000..2fdb069 --- /dev/null +++ b/tests/account_control_store_tests.cpp @@ -0,0 +1,235 @@ +// Real PostgreSQL M10.3c account control checks (ADR-0091, schema v10): durable grant +// serials across regions and restarts, cross-region takeover with a wait, fencing by +// generation, region lease and expiry, and contention. World 9100 is used only here. +// Run by tools/ci/postgres-check.sh. +#include "evo/persistence/account_control_store.hpp" +#include "evo/persistence/region_checkpoint.hpp" +#include "evo/persistence/region_lease_store.hpp" +#ifdef EVO_POSTGRES_SESSIONS +#include "evo/server/postgres_control_ledger.hpp" +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +void run_account_control_store_tests(); + +namespace { +using namespace evo::persistence; +using namespace std::chrono_literals; +using std::chrono::milliseconds; +void check(bool value, std::source_location at = std::source_location::current()) { + if (!value) + throw std::runtime_error("account control store test line " + std::to_string(at.line())); +} +#define CHECK(...) check(static_cast(__VA_ARGS__)) +constexpr std::uint64_t world = 9100; + +std::string env(const char* key) { + const auto* value = std::getenv(key); + if (!value) + throw std::runtime_error("missing test environment"); + return value; +} +std::unique_ptr connect_as(const std::string& user) { + PgOptions options{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}}; + auto made = Postgres::connect(options, 5s); + CHECK(made); + return std::move(*made); +} +Uuid id(std::uint8_t seed) { + Uuid out{}; + out.fill(seed); + return out; +} +Sha256Digest digest(std::uint8_t seed) { + Sha256Digest out{}; + out.fill(seed); + return out; +} +std::string bytea(std::span bytes) { + constexpr char digits[] = "0123456789abcdef"; + std::string out = "\\x"; + for (const auto b : bytes) { + out.push_back(digits[b >> 4U]); + out.push_back(digits[b & 0x0FU]); + } + return out; +} +const Sha256Digest build = digest(0xB1); +RegionKey provision(Postgres& db, std::uint64_t region) { + CHECK(db.query("INSERT INTO evolution.retained_builds(build_identity) VALUES " + "(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING", + {}, 5s)); + const RegionKey key{world, region}; + auto commit = id(0xAC); // Commit ids are unique across the database: tag this test's. + commit.back() = static_cast(region); + auto g = genesis_manifest(key, 7, commit, 50000, build, 100, digest(0xA0)); + CHECK(g); + auto bytes = encode_manifest(*g); + CHECK(bytes); + const std::array p{bytea(*bytes)}; + const auto provisioned = + db.query("SELECT evolution.provision_region_checkpoint($1::bytea)", p, 5s); + if (!provisioned) + std::fprintf(stderr, "provision sqlstate %s\n", provisioned.error().sqlstate.data()); + CHECK(provisioned); + return key; +} +LeaseGrant acquire(Postgres& db, RegionKey key) { + auto lease = acquire_region_lease(db, key, build, 20000ms, 5s); + CHECK(lease); + return *lease; +} +bool failed(const auto& result, ControlStoreError kind) { + return !result && result.error().kind == kind; +} +// Database-time expiry of one account's control (owner read), as for region leases. +void wait_until_expired(Postgres& owner_db, const Uuid& account) { + const std::array p{bytea(account)}; + for (int attempt = 0; attempt < 400; ++attempt) { + auto rows = owner_db.query( + "SELECT (clock_timestamp() >= expires_at)::text FROM evolution.account_controls " + "WHERE world_id = 9100 AND account_id = encode($1::bytea, 'hex')::uuid", + p, 5s); + CHECK(rows && rows->size() == 1); + if ((*rows)[0][0] == "true") + return; + std::this_thread::sleep_for(50ms); + } + CHECK(false); +} + +void serials_takeover_and_fencing() { + auto admin = connect_as(env("PGUSER")); + const auto a = provision(*admin, 1); + const auto b = provision(*admin, 2); + auto db = connect_as(env("EVO_PG_RUNTIME_USER")); + const auto lease_a = acquire(*db, a); + const auto lease_b = acquire(*db, b); + const auto alice = id(0x31); + + // The runtime reaches the table only through the functions. + CHECK(!db->query("SELECT count(*) FROM evolution.account_controls", {}, 5s)); + + auto first = claim_account_control(*db, a, lease_a, alice, 5, 1, 20000ms, 5s); + CHECK(first && first->generation == 1 && first->wait == 0ms && first->granted == 20000ms); + CHECK(failed(claim_account_control(*db, a, lease_a, alice, 5, 2, 20000ms, 5s), + ControlStoreError::stale_grant)); // Replay. + CHECK(failed(claim_account_control(*db, a, lease_a, alice, 4, 2, 20000ms, 5s), + ControlStoreError::stale_grant)); // Older grant. + // Same region takeover: no wait; the old generation is fenced. + auto second = claim_account_control(*db, a, lease_a, alice, 6, 2, 20000ms, 5s); + CHECK(second && second->generation == 2 && second->wait == 0ms); + CHECK(failed(renew_account_control(*db, a, lease_a, alice, 1, 20000ms, 5s), + ControlStoreError::fenced)); + CHECK(renew_account_control(*db, a, lease_a, alice, 2, 20000ms, 5s) == 20000ms); + // Another region with a newer grant takes control but must wait out region A's. + auto moved = claim_account_control(*db, b, lease_b, alice, 7, 1, 20000ms, 5s); + CHECK(moved && moved->generation == 3 && moved->wait > 15000ms && moved->wait <= 20000ms); + CHECK(failed(renew_account_control(*db, a, lease_a, alice, 2, 20000ms, 5s), + ControlStoreError::fenced)); + CHECK(release_account_control(*db, a, lease_a, alice, 2, 5s) == false); + // Region B's renewal never shortens the waited-out expiry. + CHECK(renew_account_control(*db, b, lease_b, alice, 3, 1000ms, 5s) == 1000ms); + CHECK(release_account_control(*db, b, lease_b, alice, 3, 5s) == true); + CHECK(failed(renew_account_control(*db, b, lease_b, alice, 3, 20000ms, 5s), + ControlStoreError::fenced)); + // Released control is free: the next claim waits for nobody; the serial stays durable. + CHECK(failed(claim_account_control(*db, a, lease_a, alice, 7, 3, 20000ms, 5s), + ControlStoreError::stale_grant)); + auto back = claim_account_control(*db, a, lease_a, alice, 8, 3, 20000ms, 5s); + CHECK(back && back->wait == 0ms); + + // Restart: a new process takes region A's lease under a new epoch. Serials survive, + // the old epoch's control needs no wait, and the old lease can do nothing. + CHECK(release_region_lease(*db, a, lease_a, 5s)); + const auto restarted = acquire(*db, a); + CHECK(restarted.epoch > lease_a.epoch); + CHECK(failed(claim_account_control(*db, a, restarted, alice, 8, 1, 20000ms, 5s), + ControlStoreError::stale_grant)); + auto resumed = claim_account_control(*db, a, restarted, alice, 9, 1, 20000ms, 5s); + CHECK(resumed && resumed->wait == 0ms); + CHECK(failed(claim_account_control(*db, a, lease_a, alice, 10, 4, 20000ms, 5s), + ControlStoreError::fenced)); + CHECK(failed(renew_account_control(*db, a, lease_a, alice, resumed->generation, 20000ms, 5s), + ControlStoreError::fenced)); + + // Expiry is final, and an expired holder is not waited for. + const auto bob = id(0x32); + auto brief = claim_account_control(*db, a, restarted, bob, 1, 2, 1000ms, 5s); + CHECK(brief && brief->wait == 0ms); + wait_until_expired(*admin, bob); + CHECK(failed(renew_account_control(*db, a, restarted, bob, brief->generation, 20000ms, 5s), + ControlStoreError::fenced)); + auto elsewhere = claim_account_control(*db, b, lease_b, bob, 2, 2, 20000ms, 5s); + CHECK(elsewhere && elsewhere->wait == 0ms); + + // Invalid requests change nothing. + CHECK(failed(claim_account_control(*db, a, restarted, id(0x33), 1, 1, 500ms, 5s), + ControlStoreError::invalid)); + CHECK(failed(claim_account_control(*db, a, restarted, id(0x33), 0, 1, 20000ms, 5s), + ControlStoreError::invalid)); + CHECK(failed(claim_account_control(*db, a, restarted, id(0x33), 1, 0, 20000ms, 5s), + ControlStoreError::invalid)); + CHECK(failed(claim_account_control(*db, a, restarted, Uuid{}, 1, 1, 20000ms, 5s), + ControlStoreError::invalid)); + CHECK(claim_account_control(*db, a, restarted, id(0x33), 1, 1, 20000ms, 5s)); + + // Contention: two regions present the same newest serial at once; exactly one wins. + const auto carol = id(0x34); + auto first_db = connect_as(env("EVO_PG_RUNTIME_USER")); + auto second_db = connect_as(env("EVO_PG_RUNTIME_USER")); + std::latch start{2}; + std::expected ra, rb; + std::thread ta{[&] { + start.arrive_and_wait(); + ra = claim_account_control(*first_db, a, restarted, carol, 20, 7, 20000ms, 5s); + }}; + std::thread tb{[&] { + start.arrive_and_wait(); + rb = claim_account_control(*second_db, b, lease_b, carol, 20, 7, 20000ms, 5s); + }}; + ta.join(); + tb.join(); + CHECK(ra.has_value() != rb.has_value()); + CHECK(failed(ra.has_value() ? rb : ra, ControlStoreError::stale_grant)); + +#ifdef EVO_POSTGRES_SESSIONS + // The session layer's ledger over these functions. + std::optional current = restarted; + evo::server::PostgresControlLedger ledger{*db, a, [&] { return current; }}; + evo::network::AccountId dave; + dave.bytes = id(0x35); + auto claimed = ledger.claim(dave, 1, 9); + CHECK(claimed && claimed->wait == 0ms && claimed->granted == 20000ms); + CHECK(!ledger.claim(dave, 1, 9) && + ledger.claim(dave, 1, 9).error() == evo::network::ControlLedgerError::stale_grant); + CHECK(ledger.renew(dave, claimed->generation) == 20000ms); + ledger.release(dave, claimed->generation); + CHECK(!ledger.renew(dave, claimed->generation) && + ledger.renew(dave, claimed->generation).error() == + evo::network::ControlLedgerError::fenced); + current.reset(); // The host lost its lease: nothing is granted or renewed. + CHECK(!ledger.claim(dave, 2, 9) && + ledger.claim(dave, 2, 9).error() == evo::network::ControlLedgerError::fenced); +#endif + CHECK(release_region_lease(*db, a, restarted, 5s)); + CHECK(release_region_lease(*db, b, lease_b, 5s)); +} +} // namespace + +void run_account_control_store_tests() { + serials_takeover_and_fencing(); + std::puts("account control integration passed"); +} diff --git a/tests/postgres_tests.cpp b/tests/postgres_tests.cpp index 0aae5db..baf6329 100644 --- a/tests/postgres_tests.cpp +++ b/tests/postgres_tests.cpp @@ -4,6 +4,7 @@ #include #include void run_region_lease_store_tests(); +void run_account_control_store_tests(); void run_region_checkpoint_store_tests(); void run_region_host_tests(); void run_region_server_process_tests(); @@ -116,6 +117,7 @@ int main() { run(); std::puts("postgres adapter integration passed"); run_region_lease_store_tests(); + run_account_control_store_tests(); std::puts("region lease fencing integration passed"); run_region_checkpoint_store_tests(); std::puts("region checkpoint publication integration passed"); diff --git a/tests/quic_sessions.cpp b/tests/quic_sessions.cpp index 3653c55..07271d3 100644 --- a/tests/quic_sessions.cpp +++ b/tests/quic_sessions.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -104,7 +105,7 @@ struct Issuer { struct Harness { explicit Harness(const Files& f, std::size_t max_sessions = 4, - std::uint32_t region_attempts = 64) + std::uint32_t region_attempts = 64, ControlLedger* ledger = nullptr) : files{f} { server::RegionSessionsConfig c; c.transport.certificate_pem_file = f.certificate.string(); @@ -119,6 +120,9 @@ struct Harness { c.issuer_keys.push_back({1, *issuer.key.public_key()}); c.hello_deadline = std::chrono::milliseconds{500}; c.close_flush = std::chrono::milliseconds{300}; + c.ledger = ledger; + c.renew_interval = std::chrono::milliseconds{100}; + c.safety_margin = std::chrono::milliseconds{50}; auto made = RegionSessions::create(std::move(c)); CHECK(made.has_value()); server = std::move(*made); @@ -336,12 +340,110 @@ void limits(const Files& f) { CHECK(refused_with(h, h.issuer.grant(account(1), 4), Reason::rate_limited)); } } +// Scripted durable control (10.3c): records calls, answers as configured. +struct FakeLedger final : ControlLedger { + struct Claim { + AccountId account; + std::uint64_t serial{}, session{}; + }; + std::expected + claim(const AccountId& account, std::uint64_t serial, std::uint64_t session) override { + if (claim_error) + return std::unexpected(*claim_error); + claims.push_back({account, serial, session}); + return ControlLease{next_generation++, wait, granted}; + } + std::expected + renew(const AccountId& /*account*/, std::uint64_t generation) override { + renewed.push_back(generation); + if (renew_error) + return std::unexpected(*renew_error); + return granted; + } + void release(const AccountId& /*account*/, std::uint64_t generation) override { + released.push_back(generation); + } + std::optional claim_error, renew_error; + std::chrono::milliseconds wait{}, granted{std::chrono::milliseconds{20000}}; + std::uint64_t next_generation{1}; + std::vector claims; + std::vector renewed, released; +}; + +void durable_control(const Files& f) { + FakeLedger ledger; + Harness h{f, 4, 64, &ledger}; + const auto alice = account(1), bob = account(2), carol = account(3); + // Durable refusals: nothing is admitted and the grant stays usable locally. + ledger.claim_error = ControlLedgerError::stale_grant; + CHECK(refused_with(h, h.issuer.grant(alice, 1), Reason::stale_grant)); + ledger.claim_error = ControlLedgerError::fenced; + CHECK(refused_with(h, h.issuer.grant(alice, 1), Reason::shutdown)); + ledger.claim_error = ControlLedgerError::unavailable; + CHECK(refused_with(h, h.issuer.grant(alice, 1), Reason::capacity)); + CHECK(!h.server->holder(alice) && ledger.claims.empty()); + ledger.claim_error.reset(); + // The claim names the account, serial and the session that Welcome then reports. + auto a = h.connect(h.issuer.grant(alice, 1)); + CHECK(h.settled(*a) && a->state() == RegionSessionState::established); + CHECK(ledger.claims.size() == 1 && ledger.claims[0].account == alice && + ledger.claims[0].serial == 1 && ledger.claims[0].session == must(a->welcome()).session); + CHECK(h.until([&] { return ledger.renewed.size() >= 2; })); // Renewed every 100 ms. + // Refresh records the newer serial for the same session. + CHECK(a->refresh(h.issuer.grant(alice, 2))); + std::optional r; + CHECK(h.until([&] { return (r = a->take_refresh_result()).has_value(); })); + CHECK(must(r).status == 0 && ledger.claims.size() == 2 && ledger.claims[1].serial == 2 && + ledger.claims[1].session == must(a->welcome()).session); + // Another region took control: the renewal is fenced and the session is superseded, + // without releasing control it no longer holds. + ledger.renew_error = ControlLedgerError::fenced; + CHECK(h.until([&] { return a->state() == RegionSessionState::closed; })); + CHECK(a->close_reason() == Reason::superseded && ledger.released.empty()); + CHECK(!h.server->holder(alice)); + // Control held by another region: Welcome waits it out. + ledger.renew_error.reset(); + ledger.wait = std::chrono::milliseconds{400}; + const auto sent = Clock::now(); + auto b = h.connect(h.issuer.grant(bob, 1)); + CHECK(h.until([&] { return h.server->holder(bob).has_value(); })); + CHECK(b->state() != RegionSessionState::established); + CHECK(h.settled(*b) && b->state() == RegionSessionState::established); + CHECK(Clock::now() - sent >= std::chrono::milliseconds{400}); + // Undecided renewals: the session closes before its control could lapse, and releases. + ledger.wait = {}; + ledger.granted = std::chrono::milliseconds{300}; + auto c = h.connect(h.issuer.grant(carol, 1)); + CHECK(h.settled(*c) && c->state() == RegionSessionState::established); + const auto carol_generation = ledger.next_generation - 1; + ledger.renew_error = ControlLedgerError::unavailable; + CHECK(h.until([&] { return c->state() == RegionSessionState::closed; })); + CHECK(c->close_reason() == Reason::shutdown); + CHECK(std::ranges::find(ledger.released, carol_generation) != ledger.released.end()); + // A refresh that finds a newer grant elsewhere closes the session as superseded. + ledger.renew_error.reset(); + ledger.granted = std::chrono::milliseconds{20000}; + ledger.claim_error = ControlLedgerError::stale_grant; + CHECK(b->refresh(h.issuer.grant(bob, 2))); + CHECK(h.until([&] { return b->state() == RegionSessionState::closed; })); + CHECK(b->close_reason() == Reason::superseded && !b->take_refresh_result()); + // Client close releases durable control. + ledger.claim_error.reset(); + auto d = h.connect(h.issuer.grant(alice, 3)); + CHECK(h.settled(*d) && d->state() == RegionSessionState::established); + const auto alice_generation = ledger.next_generation - 1; + d->close(); + CHECK(h.until([&] { + return std::ranges::find(ledger.released, alice_generation) != ledger.released.end(); + })); +} } // namespace int main() try { const auto files = credentials(); login_rules(files); limits(files); + durable_control(files); std::filesystem::remove_all(files.dir); std::puts("quic session tests passed"); return 0; diff --git a/tests/region_migration_tests.cpp b/tests/region_migration_tests.cpp index f248576..f26c19e 100644 --- a/tests/region_migration_tests.cpp +++ b/tests/region_migration_tests.cpp @@ -175,6 +175,7 @@ void upgrade_across_migration_and_barrier() { " -v migration_sha_6=$(s 0006_commit_lookup.sql)" " -v migration_sha_7=$(s 0007_region_clones.sql) -v migration_sha_8=invalid" " -v migration_sha_9=$(s 0009_serialized_authority.sql)" + " -v migration_sha_10=$(s 0010_account_control.sql)" " -v runtime_role=evo_runtime_test -f migrations/apply.sql" " > /dev/null 2>&1") != 0); CHECK(history(*owner_db) == "1,2,3,4,5,6,7"); @@ -190,7 +191,7 @@ void upgrade_across_migration_and_barrier() { CHECK(shell("PGDATABASE=" + database + " EVO_PG_RUNTIME_ROLE=evo_runtime_test bash \"$EVO_SOURCE_ROOT/" "tools/migrate-postgres.sh\" > /dev/null") == 0); - CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8,9"); + CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8,9,10"); CHECK(renew_region_lease(*runtime, r.key, *lease, 20000ms, 5s)); confirm(publish_next(20)); // 41..60, now v8. CHECK(head.head.end_tick == 60); diff --git a/tools/ci/postgres-check.sh b/tools/ci/postgres-check.sh index c0defba..2e65ffc 100644 --- a/tools/ci/postgres-check.sh +++ b/tools/ci/postgres-check.sh @@ -61,7 +61,7 @@ PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" -f "$root/migrations/0001_region_bootstrap.sql" > /dev/null test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1 PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9 +test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10 # v2 heads have no payload binding: v2->v3 fails closed and leaves history at v2. v2_sha="$(sha256sum "$root/migrations/0002_region_fencing.sql")" v2_sha="${v2_sha%% *}" @@ -82,7 +82,7 @@ PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '01000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32), 'hex'))" > /dev/null PGDATABASE=v3_commits bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 9:1,2,3,4,5,6,7,8,9 +test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 10:1,2,3,4,5,6,7,8,9,10 test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) FROM evolution.region_commits")" = 1 # Review P1: a v4 database holding a published (possibly terminal) checkpoint has # no durable decisions for it. The v5 upgrade must refuse before changing anything. @@ -136,7 +136,7 @@ PGDATABASE=v6_populated psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '02000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32) || repeat('00', 80), 'hex'))" > /dev/null PGDATABASE=v6_populated bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9 +test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT count(*) FROM evolution.region_commits c, LATERAL evolution.parse_checkpoint_manifest(c.manifest) m WHERE m.commit_sequence = 0 AND m.predecessor_commit IS NULL")" = 1 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(conname, ',') FROM pg_constraint WHERE conrelid = 'evolution.region_heads'::regclass AND contype = 'c' AND pg_get_constraintdef(oid) LIKE '%end_tick%'")" = region_heads_published_tick # Inherited privileges bypass fencing, so every migrate run audits effective rights. @@ -169,11 +169,16 @@ expect_audit_failure 'GRANT EXECUTE ON FUNCTION evolution.check_region_lease(evo expect_audit_failure 'ALTER ROLE evo_runtime_login CREATEDB' 'ALTER ROLE evo_runtime_login NOCREATEDB' expect_audit_failure 'GRANT EXECUTE ON FUNCTION evolution.provision_region_checkpoint(bytea) TO evo_runtime_test' 'REVOKE EXECUTE ON FUNCTION evolution.provision_region_checkpoint(bytea) FROM evo_runtime_test' expect_audit_failure 'GRANT SELECT ON evolution.region_commits TO evo_runtime_test' 'REVOKE SELECT ON evolution.region_commits FROM evo_runtime_test' +# Schema v10: account controls are reachable only through the three fenced functions. +expect_audit_failure 'GRANT SELECT ON evolution.account_controls TO evo_runtime_test' 'REVOKE SELECT ON evolution.account_controls FROM evo_runtime_test' +expect_audit_failure 'REVOKE EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) FROM evo_runtime_test' 'GRANT EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) TO evo_runtime_test' bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again. # Unknown future versions and checksum drift must not be repaired silently. -psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=10 WHERE version=9' +psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=11 WHERE version=10' +if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=10 WHERE version=11" +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=10" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi -psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=9 WHERE version=10" psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=9" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=8" diff --git a/tools/ci/upgrade-check.sh b/tools/ci/upgrade-check.sh index 5ad514a..5bb6967 100644 --- a/tools/ci/upgrade-check.sh +++ b/tools/ci/upgrade-check.sh @@ -4,7 +4,7 @@ # exported). Real evo-server processes on world 500: # 1. Failed migration: a migration that takes the migration lock, alters a history # table, holds its locks for 3 s and then fails runs while build b1 serves. -# Nothing it did remains (schema still v9, no new column); the host keeps its +# Nothing it did remains (schema still v10, no new column); the host keeps its # authority, and every acknowledgement stays durable. # 2. Barrier (ADR-0068 operator order): build b2 is retained, and a b2 host is started # while b1 still holds the lease (it waits, never ready). upgrade_barrier(b1) fences @@ -61,7 +61,7 @@ SQL [ "$code" != 0 ] && grep -q 'division by zero' "$work/migration.log" || fail "the rehearsal migration did not fail" [ "$(q -c "SELECT count(*) FROM information_schema.columns WHERE table_name = 'region_commits' AND column_name = 'rehearsal_column'")" = 0 ] || fail "the failed migration left a column" -[ "$(q -c 'SELECT max(version) FROM evolution.schema_migrations')" = 9 ] || fail "schema version moved" +[ "$(q -c 'SELECT max(version) FROM evolution.schema_migrations')" = 10 ] || fail "schema version moved" wait_ack "$work/b1.log" $((before + 40)) || fail "b1 did not resume after the failed migration" kill -0 "$b1_pid" 2> /dev/null || fail "b1 exited during the failed migration" bash "$root/tools/migrate-postgres.sh" > /dev/null || fail "migrate after the failure" diff --git a/tools/migrate-postgres.sh b/tools/migrate-postgres.sh index 6ef9e2a..bb752f1 100644 --- a/tools/migrate-postgres.sh +++ b/tools/migrate-postgres.sh @@ -30,10 +30,13 @@ digest_8="$(sha256sum "$root/migrations/0008_upgrade_barrier.sql")" digest_8="${digest_8%% *}" digest_9="$(sha256sum "$root/migrations/0009_serialized_authority.sql")" digest_9="${digest_9%% *}" +digest_10="$(sha256sum "$root/migrations/0010_account_control.sql")" +digest_10="${digest_10%% *}" psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_2=$digest_2" --set="migration_sha_3=$digest_3" \ --set="migration_sha_4=$digest_4" --set="migration_sha_5=$digest_5" \ --set="migration_sha_6=$digest_6" --set="migration_sha_7=$digest_7" \ --set="migration_sha_8=$digest_8" --set="migration_sha_9=$digest_9" \ + --set="migration_sha_10=$digest_10" \ --set="runtime_role=$runtime_role" \ --file="$root/migrations/apply.sql"