diff --git a/CMakeLists.txt b/CMakeLists.txt index d301e20..470b551 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -248,7 +248,8 @@ if(EVO_BUILD_POSTGRES) if(BUILD_TESTING) # Contention tests use one thread and one connection per contender. add_executable(evo-postgres-tests tests/postgres_tests.cpp tests/region_lease_store_tests.cpp - tests/account_control_store_tests.cpp tests/region_action_tests.cpp + tests/account_control_store_tests.cpp tests/probe_handoff_store_tests.cpp + tests/region_action_tests.cpp tests/region_checkpoint_store_tests.cpp tests/region_host_tests.cpp tests/region_server_process_tests.cpp tests/region_recovery_tests.cpp tests/region_ops_tests.cpp tests/region_clone_tests.cpp diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 8e41262..aed0d8e 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -533,7 +533,10 @@ probe-motion-v1 rule, server markers, session protocol v3 marker records and the predictor done; 10.6b wiring (sessions, evo-server, probe, GUI marker) done; 10.6c impaired traces done (networking/M10_PREDICTION_TRACES.md). 10.7 split (ADR-0095): 10.7a tick clock, remote-marker interpolation and fractional prediction done; 10.7b client wiring, stale loss -view and two-client interpolation traces done. 10.8-10.9 planned. +view and two-client interpolation traces done. 10.8 split (ADR-0096, the persistence +handoff ADR; EXPERIMENT durable probe records): 10.8a schema v12 probe records and the +transfer state machine with a PostgreSQL fault matrix done; 10.8b handoff tickets, 10.8c +region wiring and 10.8d two-process kill tests follow. 10.9 planned. **Entry:** M9 client/session evidence and M7/M8 dev/stage durability. Read NETWORK_ARCHITECTURE, PERSISTENCE_ARCHITECTURE and ADR-0017. The M9 transport is diff --git a/docs/STATUS.md b/docs/STATUS.md index a2d0047..d7ad962 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -64,8 +64,15 @@ interpolation of remote markers (2-6 ticks behind, at most 2 ticks extrapolated, stale; organisms never) and sub-tick own-marker presentation. M10.7b wires it into the engineering client. A lost session's view stays frozen and labelled stale until the reconnect's fresh baseline. In two-client traces, remote markers shown as current stay -within about 0.1 m of the true path, and outages show stale. M10.7 is complete. Next: -M10.8 authority transfer between two regions (needs a ratified handoff ADR). +within about 0.1 m of the true path, and outages show stale. M10.7 is complete. M10.8a (ADR-0096, the ratified handoff ADR) adds schema v12: +- durable probe records, one owning region per account; +- a fenced transfer state machine (prepared, staged, committed, aborted) whose every retry + returns the recorded outcome; +- an account-control handoff for committed transfers. + +A PostgreSQL fault matrix covers restarts at each transition, stale epochs, aborts, +partitions and commit/abort races, checking a unique owner and the conserved test inventory. +Deploying v12 needs a rebuilt migrations image. Next: 10.8b handoff tickets. ## Previous gate - 2026-10-02 diff --git a/docs/adr/0096-m10-probe-handoff.md b/docs/adr/0096-m10-probe-handoff.md new file mode 100644 index 0000000..c6703c3 --- /dev/null +++ b/docs/adr/0096-m10-probe-handoff.md @@ -0,0 +1,116 @@ +# ADR-0096: M10 cross-region handoff of probe records + +## Status +Accepted for M10.8 (2026-10-06). This is the persistence handoff ADR the roadmap requires +before 10.8. Decisions confirmed with the maintainer on 2026-10-06: +- the transferred entity is a durable probe record with a conserved test inventory + (EXPERIMENT, amends ADR-0094); +- regions own adjacent disjoint areas, and crossing the edge starts the handoff, the + destination holding a read-only ghost first; +- the client is admitted at the destination with a short-lived transfer ticket signed by + the source region; +- evidence is a library-level fault matrix plus two real region processes on PostgreSQL, + with a kill at each durable step. + +Delivery: +1. **10.8a:** schema v12 and the store, with the fault matrix (this text). +2. **10.8b:** handoff keys and tickets, and ticket admission. +3. **10.8c:** region-host wiring (edge crossing, ghost, freeze, activation, client + rerouting without a loading screen). +4. **10.8d:** two-process kill tests and conservation evidence. + +Each segment amends this ADR. + +**EXPERIMENT.** A probe is a non-biological engineering entity. Nothing here decides how +organisms move between regions; M11 reuses the state machine for organisms only through +its own ADR. + +## Context +PERSISTENCE_ARCHITECTURE "Entity handoff across regions" outlines a handoff: +1. the destination holds a lease and a read-only ghost; +2. the source freezes and durably prepares the final state; +3. the destination durably stages it; +4. one transaction switches ownership; +5. the destination activates and the source tombstones. + +Abort is durable and only before commit. Retries return the recorded outcome, and a crash +at any step yields one owner, never duplication or loss. OPEN_QUESTIONS 24 proposes fenced +leases plus an explicit handoff state machine. Until M10.8 no entity was durable and +controllable: probe markers lived in region memory (ADR-0094), and organisms are not +controllable before M11. + +## Decision (10.8a, schema v12) +- **Probe record** (`evolution.probe_records`). One row per (world, account): the owning + region, an ownership generation, position (micrometres, cell slice) and an EXPERIMENT + test inventory, plus the open transfer while frozen. Ownership is a single column of a + single row, so the owner is unique and the inventory is conserved by construction: a + transfer moves the row's ownership, never copies it. A region acts for a probe only if + it owns it and it is not frozen. Positions are saved durably at session end, at + preparation and on commit, never per tick (PROJECT_VISION). +- **Transfer** (`evolution.probe_transfers`), state machine: + - **prepared:** the source froze the probe at its final state for a named destination. + - **staged:** the destination validated and durably accepted it under its current + epoch; again after its own restart. + - **committed:** the destination switched ownership in one transaction, with generation + + 1 and the final state. + - **aborted:** by the source or the destination, before commit only; the source resumes. + - At most one open transfer per probe. +- **Retries.** A retry of any call with the same transfer id returns the recorded outcome: + prepare by its source, stage and commit by its destination, abort by either. After a + commit, an abort returns the commit; after an abort, stage and commit return the abort. +- **Fencing.** Every function first requires the caller's exact unexpired region lease. + - A commit requires the transfer staged by the caller's current epoch: a destination + restarted after staging must stage again. + - No function locks another region's lease row (lock order: upgrade lock, own lease, + record, transfer). The frozen record is what stops the source, under any of its + epochs, from acting while a transfer is open. + - Timeout alone never authorises a second copy: the source resumes only through a + durable abort. +- **Account control.** `handoff_account_control` gives the destination the account's + durable control (schema v10) for a transfer committed to it, instead of a newer grant + serial. Another region's unexpired control is still outlived; the source releases its + control when it prepares, so normally there is no wait. +- **SQLSTATE:** + - EVL13 probe fenced; + - EVL14 unknown transfer; + - EVL15 not staged by this epoch; + - EVL12 for the control handoff. +- The runtime reaches both tables only through the eight functions, enforced by the + effective-privilege audit. + +## Planned (10.8b-d, amended when delivered) +- **Tickets.** + - The source signs a short-lived ticket with its region handoff key (ECDSA P-256, as + grants, domain-separated): world, account, transfer id, source, destination, expiry. + - The destination verifies it against the world's configured region keys and admits + the client only for a transfer it can stage and commit. + - Reconnects after a handoff use grants for the owning region; a dev bundle may hold + grants for several regions. +- **Crossing.** + - Regions own adjacent disjoint areas. A marker crossing its region's edge is frozen at + the crossing tick and prepared; the source releases its account control and offers + the client the ticket. + - The client keeps its last view (no loading screen), connects to the destination, and + the destination stages, commits, takes control and opens the session. + - Until the commit the destination holds the staged state as a read-only ghost. + - If the client never arrives, the source aborts after the ticket expires and resumes. +- **Evidence.** Two evo-server processes on one PostgreSQL: + - a crossing and a crossing back; + - SIGKILL of the source or the destination around each durable step, duplicate and + stale retries, and a failed destination; + - unique-owner and conservation audits throughout. + +## Alternatives considered +- Copying the state into the transfer and deleting it from the source: conservation then + needs a cross-row invariant instead of holding by construction. +- Locking the source's lease row at commit: breaks the lock order (deadlock between + crossing regions) and adds nothing, since the frozen record already fences the source. +- World-scoped grants or per-region bundles for admission: a grant would open any region + (or bundles would not scale); the ticket names the one destination. +- Durable markers every tick: forbidden (no per-tick database writes). + +## Consequences +- Schema v12 must be deployed with the tools image before a region uses probe records. + The dev migrations image must be rebuilt (the v10 lesson). +- A probe's durable position can lag its in-memory position by up to a session: + presentation-only state, never biological. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 1cce469..104cff4 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -4679,3 +4679,50 @@ The Postgres suite was not rerun: no server code changed in M10.7. Not done: - interpolating organisms (none move); - a GUI run against evolution-dev (needs a v3 server). + +## 2026-10-06 — M10.8a probe handoff schema v12 (ADR-0096) + +Implemented: +- `migrations/0012_probe_handoff.sql`: + - `probe_records` and `probe_transfers`; + - `open_probe`, `save_probe`, `prepare`, `stage`, `commit`, `abort`, `find`; + - `handoff_account_control`; + - SQLSTATEs EVL13-15; +- the apply chain, migration runner, privilege audit allowlist, CI version checks and + tamper tests; +- the C++ store (`include/evo/persistence/probe_store.hpp`, and `handoff_account_control` + in `account_control_store.hpp`); +- `schemas/probe-handoff-v1.md` and the POSTGRESQL.md section. + +`tests/probe_handoff_store_tests.cpp` (world 9200, real PostgreSQL) covers: +- ownership and fencing; argument refusals; +- the happy path with retries of every call before and after commit; +- one open transfer per probe; transfer ids bound to one account; +- crossing back; +- source restart after prepare; +- destination restart after staging (re-stage required; the old epoch fenced) and after + commit (the retry answers); +- a failed destination (source abort); a destination abort; +- a partition (the source lease lapses while the destination commits; the recovered source + is fenced); +- 24 commit/abort races on separate connections (7 commits, 17 aborts; both sides always + observed the same outcome); +- the account-control handoff (refused before commit, for another region or account; a wait + while the source holds control; none after release). + +After every scenario: one row per probe, the test inventory conserved (n x 1000), and frozen +exactly when an open transfer names the owner as source. + +Executed: +- **Linux GCC 14:** `postgres-check.sh` with QUIC: 21 suites, including probe handoff, + migrations (v1-v11 upgrade paths to v12), tamper and audit checks; +- preset `quic`: 4/4. +- **Linux Clang 19:** clang-tidy 19 on `region_lease_store.cpp` and the new test (two + findings fixed); clang-format 19. + +The suite was rerun on the final code after the tidy fixes: 21 suites passed again; that +run's races ended 11 commits, 13 aborts. + +Not done: +- tickets (10.8b), region wiring (10.8c) and two-process kills (10.8d); +- deploying v12 (needs a rebuilt migrations image; a maintainer step). diff --git a/docs/infrastructure/POSTGRESQL.md b/docs/infrastructure/POSTGRESQL.md index c783319..baaeed5 100644 --- a/docs/infrastructure/POSTGRESQL.md +++ b/docs/infrastructure/POSTGRESQL.md @@ -200,6 +200,25 @@ updated). Exact retries and `resolve_region_commit(..., actions)` compare the bl `find_region_action` reads one record under the caller's exact region lease. The privilege audit covers all three functions; the tables and `check_action_block` are owner-only. +## Probe records and handoff (schema v12, ADR-0096) + +EXPERIMENT (non-biological probes, ADR-0094). `evolution.probe_records` holds one row per +(world, account): owning region, generation, position, test inventory and the open +transfer while frozen. `evolution.probe_transfers` holds the transfer state machine: +prepared, staged, committed, aborted. Runtime access is only through `open_probe`, +`save_probe`, `prepare_probe_transfer`, `stage_probe_transfer`, `commit_probe_transfer`, +`abort_probe_transfer`, `find_probe_transfer` and `handoff_account_control`, each under the +caller's exact unexpired region lease. Every transfer call can be retried with its id and +returns the recorded outcome. SQLSTATEs: +- `EVL13`: probe fenced; +- `EVL14`: unknown transfer; +- `EVL15`: not staged by this epoch. + +Lock order: upgrade lock (shared), own region lease row, probe record, transfer; +`handoff_account_control` reads the transfer, then locks the account control row. The +privilege audit covers all eight functions, and the tables are owner-only. Deploying v12 +needs a rebuilt migrations (tools) image. + ## Host lease discipline Hosts use `LeaseAuthority` with `LeasePolicy` defaults: 20 s lease, 5 s renewal diff --git a/include/evo/persistence/account_control_store.hpp b/include/evo/persistence/account_control_store.hpp index 5bebc39..9d8b010 100644 --- a/include/evo/persistence/account_control_store.hpp +++ b/include/evo/persistence/account_control_store.hpp @@ -39,6 +39,13 @@ claim_account_control(Postgres& db, RegionKey key, const LeaseGrant& lease, cons renew_account_control(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& account, std::uint64_t generation, std::chrono::milliseconds duration, std::chrono::milliseconds timeout); +// M10.8 (ADR-0096, schema v12): takes control of `account` for `session` on the strength of +// probe transfer `transfer` committed to the caller's region (no serial: a handoff ticket +// has none). Fenced unless that transfer is committed to this region for this account. +[[nodiscard]] std::expected +handoff_account_control(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& account, + const Uuid& transfer, std::uint64_t session, + std::chrono::milliseconds duration, std::chrono::milliseconds timeout); // True when released, false when control had already moved on. [[nodiscard]] std::expected release_account_control(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& account, diff --git a/include/evo/persistence/probe_store.hpp b/include/evo/persistence/probe_store.hpp new file mode 100644 index 0000000..e3bc1c7 --- /dev/null +++ b/include/evo/persistence/probe_store.hpp @@ -0,0 +1,78 @@ +#pragma once +#include "evo/persistence/postgres.hpp" +#include "evo/persistence/region_lease.hpp" +#include +#include +#include +#include + +// Linux-only calls to the probe record and handoff functions of schema v12 (ADR-0096, +// migrations/0012_probe_handoff.sql). EXPERIMENT: probes are non-biological engineering +// entities (ADR-0094). Each call is one statement under the caller's region lease: a fenced +// or expired lease refuses everything. No retries; every transfer call can be retried with +// the same transfer id and returns the recorded outcome. +namespace evo::persistence { +enum class ProbeStoreError : std::uint8_t { + fenced, // Lease, ownership, generation or frozen state no longer allows it. + unknown_transfer, // No transfer with this id in the world. + not_staged, // Commit before the caller's current epoch staged the transfer. + exhausted, // Generations would wrap: fail closed. + invalid, // Rejected arguments (or an unprovisioned region); nothing changed. + store, // Other SQL/configuration failure; nothing committed. + unknown, // May have executed; reconnect and reconcile (retry the same id). +}; +struct ProbeStoreFailure { + ProbeStoreError kind{}; + PgFailure cause{}; +}; +struct ProbeRecord { + std::uint64_t region{}, generation{}; // Owning region; generation of this ownership. + std::int64_t x{}, y{}, z{}; // Micrometres; z is the cell slice. + std::uint64_t inventory{}; // EXPERIMENT conserved test inventory. + std::optional transfer; // Frozen under this open transfer. +}; +enum class TransferState : std::uint8_t { prepared, staged, committed, aborted }; +struct ProbeTransfer { + TransferState state{}; + Uuid account{}; + std::uint64_t generation{}, source{}, destination{}; + std::int64_t x{}, y{}, z{}; + std::uint64_t inventory{}; + friend bool operator==(const ProbeTransfer&, const ProbeTransfer&) = default; +}; + +// The account's probe, created owned by the caller's region (at the position, with the +// inventory) when the world has none. The caller acts for it only if it owns it and it is +// not frozen. +[[nodiscard]] std::expected +open_probe(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& account, + std::int64_t x, std::int64_t y, std::int64_t z, std::uint64_t inventory, + std::chrono::milliseconds timeout); +// Records the active, owned probe's position (fenced otherwise). +[[nodiscard]] std::expected +save_probe(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& account, + std::uint64_t generation, std::int64_t x, std::int64_t y, std::int64_t z, + std::chrono::milliseconds timeout); +// Source: freezes the probe at its final position under `transfer` for `destination`. +[[nodiscard]] std::expected +prepare_probe_transfer(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& account, + std::uint64_t generation, const Uuid& transfer, std::uint64_t destination, + std::int64_t x, std::int64_t y, std::int64_t z, + std::chrono::milliseconds timeout); +// Destination: durably accepts it under its current epoch. +[[nodiscard]] std::expected +stage_probe_transfer(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& transfer, + std::chrono::milliseconds timeout); +// Destination: the authority switch (staged by this epoch only). +[[nodiscard]] std::expected +commit_probe_transfer(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& transfer, + std::chrono::milliseconds timeout); +// Source or destination: abandons it before its commit; the source resumes. +[[nodiscard]] std::expected +abort_probe_transfer(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& transfer, + std::chrono::milliseconds timeout); +// Any region of the world: the transfer as recorded, or nothing. +[[nodiscard]] std::expected, ProbeStoreFailure> +find_probe_transfer(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& transfer, + std::chrono::milliseconds timeout); +} // namespace evo::persistence diff --git a/migrations/0012_probe_handoff.sql b/migrations/0012_probe_handoff.sql new file mode 100644 index 0000000..dca207a --- /dev/null +++ b/migrations/0012_probe_handoff.sql @@ -0,0 +1,383 @@ +-- M10.8a probe records and their cross-region handoff (ADR-0096). EXPERIMENT: a probe is a +-- non-biological engineering entity (ADR-0094) with a durable home, so that authority +-- transfer between regions can be proven before organisms are controllable. Nothing here +-- is a game rule. +-- +-- One row per (world, account) holds the probe and its single owning region, so ownership +-- is unique and the test inventory is conserved by construction: a transfer never copies +-- it, it moves the row's ownership in one transaction. A transfer is a durable state +-- machine: prepared (source froze the probe at its final state) -> staged (destination +-- validated and durably accepted it) -> committed (ownership switched) or aborted (before +-- commit only; the source resumes). Every retry returns the recorded outcome. +-- +-- The runtime reaches both tables only through the functions below, each of which first +-- requires the caller's exact unexpired region lease. Lock order (extends schema v11): +-- upgrade lock (shared) -> the caller's region lease row -> probe record row -> transfer +-- row. No function locks another region's lease row: a frozen record is what stops the +-- source (under any epoch) from acting while a transfer is open. +-- +-- SQLSTATE: EVL13 probe fenced (not owned by the caller, wrong generation, frozen, or a +-- transfer for another region); EVL14 unknown transfer; EVL15 transfer not staged by the +-- caller's current epoch. +CREATE TABLE evolution.probe_records ( + world_id evolution.u64 NOT NULL, + account_id uuid NOT NULL CHECK (account_id <> '00000000-0000-0000-0000-000000000000'), + region_id evolution.u64 NOT NULL, + generation evolution.u64 NOT NULL CHECK (generation > 0), + x bigint NOT NULL, + y bigint NOT NULL, + z bigint NOT NULL, + inventory evolution.u64 NOT NULL, + transfer_id uuid, -- The open transfer while frozen; NULL while active. + PRIMARY KEY (world_id, account_id) +); +CREATE TABLE evolution.probe_transfers ( + world_id evolution.u64 NOT NULL, + transfer_id uuid NOT NULL CHECK (transfer_id <> '00000000-0000-0000-0000-000000000000'), + account_id uuid NOT NULL, + state text NOT NULL CHECK (state IN ('prepared', 'staged', 'committed', 'aborted')), + generation evolution.u64 NOT NULL, -- The record's generation when prepared. + source_region evolution.u64 NOT NULL, + source_epoch evolution.u64 NOT NULL, + destination_region evolution.u64 NOT NULL, + destination_epoch evolution.u64, + x bigint NOT NULL, + y bigint NOT NULL, + z bigint NOT NULL, + inventory evolution.u64 NOT NULL, + prepared_at timestamptz NOT NULL, + decided_at timestamptz, + PRIMARY KEY (world_id, transfer_id), + CHECK (source_region <> destination_region), + CHECK (state NOT IN ('staged', 'committed') OR destination_epoch IS NOT NULL), + CHECK ((state IN ('committed', 'aborted')) = (decided_at IS NOT NULL)) +); +-- At most one open transfer per probe. +CREATE UNIQUE INDEX probe_transfers_open ON evolution.probe_transfers (world_id, account_id) + WHERE state IN ('prepared', 'staged'); +REVOKE ALL ON evolution.probe_records, evolution.probe_transfers FROM PUBLIC; + +-- The probe of `p_account`, created owned by the caller's region at the given position and +-- inventory when the world has none yet (otherwise unchanged). The caller acts for it only +-- when it owns it (region_id) and it is not frozen (transfer_id NULL). +CREATE FUNCTION evolution.open_probe(p_world evolution.u64, p_region evolution.u64, + p_holder uuid, p_epoch evolution.u64, p_account uuid, p_x bigint, p_y bigint, p_z bigint, + p_inventory evolution.u64) +RETURNS TABLE (owner_region evolution.u64, probe_generation evolution.u64, probe_x bigint, + probe_y bigint, probe_z bigint, probe_inventory evolution.u64, + open_transfer uuid) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + r evolution.probe_records%ROWTYPE; +BEGIN + IF p_account IS NULL OR p_account = '00000000-0000-0000-0000-000000000000' OR p_x IS NULL + OR p_y IS NULL OR p_z IS NULL OR p_inventory IS NULL THEN + RAISE EXCEPTION 'invalid probe request' USING ERRCODE = '22023'; + END IF; + PERFORM evolution.check_region_lease(p_world, p_region, p_holder, p_epoch); + INSERT INTO evolution.probe_records(world_id, account_id, region_id, generation, x, y, z, + inventory) + VALUES (p_world, p_account, p_region, 1, p_x, p_y, p_z, p_inventory) + ON CONFLICT DO NOTHING; + SELECT * INTO r FROM evolution.probe_records p + WHERE p.world_id = p_world AND p.account_id = p_account; + RETURN QUERY SELECT r.region_id, r.generation, r.x, r.y, r.z, r.inventory, r.transfer_id; +END $$; + +-- Records the active probe's position (session end, periodic); never per tick. +CREATE FUNCTION evolution.save_probe(p_world evolution.u64, p_region evolution.u64, + p_holder uuid, p_epoch evolution.u64, p_account uuid, p_generation evolution.u64, + p_x bigint, p_y bigint, p_z bigint) +RETURNS void LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +BEGIN + IF p_account IS NULL OR p_generation IS NULL OR p_x IS NULL OR p_y IS NULL + OR p_z IS NULL THEN + RAISE EXCEPTION 'invalid probe request' USING ERRCODE = '22023'; + END IF; + PERFORM evolution.check_region_lease(p_world, p_region, p_holder, p_epoch); + UPDATE evolution.probe_records p SET x = p_x, y = p_y, z = p_z + WHERE p.world_id = p_world AND p.account_id = p_account AND p.region_id = p_region + AND p.generation = p_generation AND p.transfer_id IS NULL; + IF NOT FOUND THEN + RAISE EXCEPTION 'probe fenced' USING ERRCODE = 'EVL13'; + END IF; +END $$; + +-- Columns every transfer function returns: the recorded transfer as it now stands. +CREATE TYPE evolution.probe_transfer_row AS ( + transfer_state text, transfer_account uuid, transfer_generation evolution.u64, + transfer_source evolution.u64, transfer_destination evolution.u64, transfer_x bigint, + transfer_y bigint, transfer_z bigint, transfer_inventory evolution.u64); + +CREATE FUNCTION evolution.probe_transfer_of(t evolution.probe_transfers) +RETURNS evolution.probe_transfer_row LANGUAGE sql IMMUTABLE +SET search_path = pg_catalog, pg_temp AS $$ + SELECT ROW(t.state, t.account_id, t.generation, t.source_region, t.destination_region, + t.x, t.y, t.z, t.inventory)::evolution.probe_transfer_row +$$; + +-- Source: freezes its active probe at its final state under `p_transfer` for +-- `p_destination`. A retry with the same transfer id returns it whatever its state. +CREATE FUNCTION evolution.prepare_probe_transfer(p_world evolution.u64, p_region evolution.u64, + p_holder uuid, p_epoch evolution.u64, p_account uuid, p_generation evolution.u64, + p_transfer uuid, p_destination evolution.u64, p_x bigint, p_y bigint, p_z bigint) +RETURNS SETOF evolution.probe_transfer_row +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + r evolution.probe_records%ROWTYPE; + t evolution.probe_transfers%ROWTYPE; + now_t timestamptz; +BEGIN + IF p_account IS NULL OR p_generation IS NULL OR p_transfer IS NULL + OR p_transfer = '00000000-0000-0000-0000-000000000000' OR p_destination IS NULL + OR p_x IS NULL OR p_y IS NULL OR p_z IS NULL OR p_destination = p_region THEN + RAISE EXCEPTION 'invalid probe request' USING ERRCODE = '22023'; + END IF; + now_t := evolution.check_region_lease(p_world, p_region, p_holder, p_epoch); + SELECT * INTO r FROM evolution.probe_records p + WHERE p.world_id = p_world AND p.account_id = p_account FOR UPDATE; + SELECT * INTO t FROM evolution.probe_transfers x + WHERE x.world_id = p_world AND x.transfer_id = p_transfer FOR UPDATE; + IF FOUND THEN + IF t.account_id <> p_account OR t.source_region <> p_region THEN + RAISE EXCEPTION 'probe fenced' USING ERRCODE = 'EVL13'; + END IF; + RETURN NEXT evolution.probe_transfer_of(t); -- Retry: the recorded outcome. + RETURN; + END IF; + IF r.account_id IS NULL OR r.region_id <> p_region OR r.generation <> p_generation + OR r.transfer_id IS NOT NULL THEN + RAISE EXCEPTION 'probe fenced' USING ERRCODE = 'EVL13'; + END IF; + -- The destination must be a provisioned region of the world. + IF NOT EXISTS (SELECT 1 FROM evolution.region_leases l + WHERE l.world_id = p_world AND l.region_id = p_destination) THEN + RAISE EXCEPTION 'region not provisioned' USING ERRCODE = 'EVL04'; + END IF; + INSERT INTO evolution.probe_transfers(world_id, transfer_id, account_id, state, generation, + source_region, source_epoch, destination_region, x, y, z, inventory, prepared_at) + VALUES (p_world, p_transfer, p_account, 'prepared', r.generation, p_region, p_epoch, + p_destination, p_x, p_y, p_z, r.inventory, now_t) + RETURNING * INTO t; + UPDATE evolution.probe_records p SET x = p_x, y = p_y, z = p_z, transfer_id = p_transfer + WHERE p.world_id = p_world AND p.account_id = p_account; + RETURN NEXT evolution.probe_transfer_of(t); +END $$; + +-- Destination: durably accepts a prepared transfer under its current epoch (again after +-- its own restart). Committed or aborted transfers are returned unchanged. +CREATE FUNCTION evolution.stage_probe_transfer(p_world evolution.u64, p_region evolution.u64, + p_holder uuid, p_epoch evolution.u64, p_transfer uuid) +RETURNS SETOF evolution.probe_transfer_row +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + t evolution.probe_transfers%ROWTYPE; +BEGIN + IF p_transfer IS NULL THEN + RAISE EXCEPTION 'invalid probe request' USING ERRCODE = '22023'; + END IF; + PERFORM evolution.check_region_lease(p_world, p_region, p_holder, p_epoch); + SELECT * INTO t FROM evolution.probe_transfers x + WHERE x.world_id = p_world AND x.transfer_id = p_transfer FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'unknown transfer' USING ERRCODE = 'EVL14'; + END IF; + IF t.destination_region <> p_region THEN + RAISE EXCEPTION 'probe fenced' USING ERRCODE = 'EVL13'; + END IF; + IF t.state IN ('prepared', 'staged') THEN + UPDATE evolution.probe_transfers x SET state = 'staged', destination_epoch = p_epoch + WHERE x.world_id = p_world AND x.transfer_id = p_transfer + RETURNING * INTO t; + END IF; + RETURN NEXT evolution.probe_transfer_of(t); +END $$; + +-- Destination: the one authority switch. Requires the transfer staged by the caller's +-- current epoch and the record still frozen under it; moves ownership (generation + 1) and +-- the final state in one transaction. A retry returns the recorded outcome. +CREATE FUNCTION evolution.commit_probe_transfer(p_world evolution.u64, p_region evolution.u64, + p_holder uuid, p_epoch evolution.u64, p_transfer uuid) +RETURNS SETOF evolution.probe_transfer_row +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + r evolution.probe_records%ROWTYPE; + t evolution.probe_transfers%ROWTYPE; + now_t timestamptz; + account uuid; +BEGIN + IF p_transfer IS NULL THEN + RAISE EXCEPTION 'invalid probe request' USING ERRCODE = '22023'; + END IF; + now_t := evolution.check_region_lease(p_world, p_region, p_holder, p_epoch); + SELECT x.account_id INTO account FROM evolution.probe_transfers x + WHERE x.world_id = p_world AND x.transfer_id = p_transfer; + IF NOT FOUND THEN + RAISE EXCEPTION 'unknown transfer' USING ERRCODE = 'EVL14'; + END IF; + SELECT * INTO r FROM evolution.probe_records p + WHERE p.world_id = p_world AND p.account_id = account FOR UPDATE; + SELECT * INTO t FROM evolution.probe_transfers x + WHERE x.world_id = p_world AND x.transfer_id = p_transfer FOR UPDATE; + IF t.destination_region <> p_region THEN + RAISE EXCEPTION 'probe fenced' USING ERRCODE = 'EVL13'; + END IF; + IF t.state IN ('committed', 'aborted') THEN + RETURN NEXT evolution.probe_transfer_of(t); + RETURN; + END IF; + IF t.state <> 'staged' OR t.destination_epoch <> p_epoch THEN + RAISE EXCEPTION 'transfer not staged' USING ERRCODE = 'EVL15'; + END IF; + IF r.transfer_id IS DISTINCT FROM p_transfer OR r.region_id <> t.source_region + OR r.generation <> t.generation THEN + RAISE EXCEPTION 'probe fenced' USING ERRCODE = 'EVL13'; -- Unreachable by construction. + END IF; + IF r.generation = 18446744073709551615 THEN + RAISE EXCEPTION 'probe generations exhausted' USING ERRCODE = 'EVL05'; + END IF; + UPDATE evolution.probe_records p + SET region_id = p_region, generation = r.generation + 1, x = t.x, y = t.y, z = t.z, + transfer_id = NULL + WHERE p.world_id = p_world AND p.account_id = account; + UPDATE evolution.probe_transfers x SET state = 'committed', decided_at = now_t + WHERE x.world_id = p_world AND x.transfer_id = p_transfer + RETURNING * INTO t; + RETURN NEXT evolution.probe_transfer_of(t); +END $$; + +-- Source or destination: abandons a transfer before its commit (durable); the probe is +-- active again at its source. After a commit, the committed transfer is returned. +CREATE FUNCTION evolution.abort_probe_transfer(p_world evolution.u64, p_region evolution.u64, + p_holder uuid, p_epoch evolution.u64, p_transfer uuid) +RETURNS SETOF evolution.probe_transfer_row +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + t evolution.probe_transfers%ROWTYPE; + now_t timestamptz; + account uuid; +BEGIN + IF p_transfer IS NULL THEN + RAISE EXCEPTION 'invalid probe request' USING ERRCODE = '22023'; + END IF; + now_t := evolution.check_region_lease(p_world, p_region, p_holder, p_epoch); + SELECT x.account_id INTO account FROM evolution.probe_transfers x + WHERE x.world_id = p_world AND x.transfer_id = p_transfer; + IF NOT FOUND THEN + RAISE EXCEPTION 'unknown transfer' USING ERRCODE = 'EVL14'; + END IF; + PERFORM 1 FROM evolution.probe_records p + WHERE p.world_id = p_world AND p.account_id = account FOR UPDATE; + SELECT * INTO t FROM evolution.probe_transfers x + WHERE x.world_id = p_world AND x.transfer_id = p_transfer FOR UPDATE; + IF p_region NOT IN (t.source_region, t.destination_region) THEN + RAISE EXCEPTION 'probe fenced' USING ERRCODE = 'EVL13'; + END IF; + IF t.state IN ('prepared', 'staged') THEN + UPDATE evolution.probe_records p SET transfer_id = NULL + WHERE p.world_id = p_world AND p.account_id = account AND p.transfer_id = p_transfer; + UPDATE evolution.probe_transfers x SET state = 'aborted', decided_at = now_t + WHERE x.world_id = p_world AND x.transfer_id = p_transfer + RETURNING * INTO t; + END IF; + RETURN NEXT evolution.probe_transfer_of(t); +END $$; + +-- Any region of the world may read a transfer (recovery, routing); no row when unknown. +CREATE FUNCTION evolution.find_probe_transfer(p_world evolution.u64, p_region evolution.u64, + p_holder uuid, p_epoch evolution.u64, p_transfer uuid) +RETURNS SETOF evolution.probe_transfer_row +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + t evolution.probe_transfers%ROWTYPE; +BEGIN + IF p_transfer IS NULL THEN + RAISE EXCEPTION 'invalid probe request' USING ERRCODE = '22023'; + END IF; + PERFORM evolution.check_region_lease(p_world, p_region, p_holder, p_epoch); + SELECT * INTO t FROM evolution.probe_transfers x + WHERE x.world_id = p_world AND x.transfer_id = p_transfer; + IF FOUND THEN + RETURN NEXT evolution.probe_transfer_of(t); + END IF; +END $$; + +-- Destination: takes the account's durable control (schema v10) for its session on the +-- strength of a transfer committed to it, instead of a newer grant serial (a handoff ticket +-- carries none). As for claims, another region's unexpired control must be outlived (the +-- source releases its control when it prepares, so normally there is nothing to wait for). +CREATE FUNCTION evolution.handoff_account_control(p_world evolution.u64, p_region evolution.u64, + p_holder uuid, p_epoch evolution.u64, p_account uuid, p_transfer uuid, + p_session evolution.u64, p_duration_ms integer) +RETURNS TABLE (control_generation evolution.u64, wait_ms integer, granted_ms integer) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + c evolution.account_controls%ROWTYPE; + t timestamptz; + pending interval; +BEGIN + IF p_account IS NULL OR p_transfer IS NULL OR p_session IS NULL OR p_session = 0 + OR p_duration_ms IS NULL OR p_duration_ms NOT BETWEEN 1000 AND 60000 THEN + RAISE EXCEPTION 'invalid control request' USING ERRCODE = '22023'; + END IF; + PERFORM evolution.check_region_lease(p_world, p_region, p_holder, p_epoch); + IF NOT EXISTS (SELECT 1 FROM evolution.probe_transfers x + WHERE x.world_id = p_world AND x.transfer_id = p_transfer + AND x.account_id = p_account AND x.destination_region = p_region + AND x.state = 'committed') THEN + RAISE EXCEPTION 'account control fenced' USING ERRCODE = 'EVL12'; + END IF; + SELECT * INTO c FROM evolution.account_controls a + WHERE a.world_id = p_world AND a.account_id = p_account FOR UPDATE; + IF NOT FOUND THEN -- The account never logged in with a grant: nothing to hand off. + RAISE EXCEPTION 'account control fenced' USING ERRCODE = 'EVL12'; + END IF; + t := clock_timestamp(); + IF c.generation = 18446744073709551615 THEN + RAISE EXCEPTION 'control generations exhausted' USING ERRCODE = 'EVL05'; + END IF; + pending := CASE + WHEN c.region_id IS NULL OR c.expires_at <= t THEN interval '0' + WHEN c.region_id = p_region AND c.region_epoch <= p_epoch THEN interval '0' + ELSE c.expires_at - t END; + UPDATE evolution.account_controls a + SET generation = c.generation + 1, region_id = p_region, region_epoch = p_epoch, + session_id = p_session, + expires_at = t + pending + p_duration_ms * interval '1 millisecond' + WHERE a.world_id = p_world AND a.account_id = p_account + RETURNING a.generation INTO control_generation; + wait_ms := ceil(extract(epoch FROM pending) * 1000)::integer; + granted_ms := p_duration_ms; + RETURN NEXT; +END $$; + +REVOKE ALL ON FUNCTION + evolution.open_probe(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, bigint, bigint, + bigint, evolution.u64), + evolution.save_probe(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, + bigint, bigint, bigint), + evolution.probe_transfer_of(evolution.probe_transfers), + evolution.prepare_probe_transfer(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, + evolution.u64, uuid, evolution.u64, bigint, bigint, bigint), + evolution.stage_probe_transfer(evolution.u64, evolution.u64, uuid, evolution.u64, uuid), + evolution.commit_probe_transfer(evolution.u64, evolution.u64, uuid, evolution.u64, uuid), + evolution.abort_probe_transfer(evolution.u64, evolution.u64, uuid, evolution.u64, uuid), + evolution.find_probe_transfer(evolution.u64, evolution.u64, uuid, evolution.u64, uuid), + evolution.handoff_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, + uuid, evolution.u64, integer) + FROM PUBLIC; +GRANT EXECUTE ON FUNCTION + evolution.open_probe(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, bigint, bigint, + bigint, evolution.u64), + evolution.save_probe(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, + bigint, bigint, bigint), + evolution.prepare_probe_transfer(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, + evolution.u64, uuid, evolution.u64, bigint, bigint, bigint), + evolution.stage_probe_transfer(evolution.u64, evolution.u64, uuid, evolution.u64, uuid), + evolution.commit_probe_transfer(evolution.u64, evolution.u64, uuid, evolution.u64, uuid), + evolution.abort_probe_transfer(evolution.u64, evolution.u64, uuid, evolution.u64, uuid), + evolution.find_probe_transfer(evolution.u64, evolution.u64, uuid, evolution.u64, uuid), + evolution.handoff_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, + uuid, evolution.u64, integer) + TO :"runtime_role"; +INSERT INTO evolution.schema_migrations(version, sha256) VALUES (12, :'migration_sha_12'); diff --git a/migrations/apply.sql b/migrations/apply.sql index e77dc58..cd97db0 100644 --- a/migrations/apply.sql +++ b/migrations/apply.sql @@ -51,28 +51,42 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') || ',5:' || :'migration_sha_5' || ',6:' || :'migration_sha_6' || ',7:' || :'migration_sha_7' || ',8:' || :'migration_sha_8' || ',9:' || :'migration_sha_9' || ',10:' || :'migration_sha_10' - || ',11:' || :'migration_sha_11' AS at_v11 \gset - \if :at_v11 + || ',11:' || :'migration_sha_11' AS at_v11, + :'history' = '1:' || :'migration_sha' || ',2:' || :'migration_sha_2' + || ',3:' || :'migration_sha_3' || ',4:' || :'migration_sha_4' + || ',5:' || :'migration_sha_5' || ',6:' || :'migration_sha_6' + || ',7:' || :'migration_sha_7' || ',8:' || :'migration_sha_8' + || ',9:' || :'migration_sha_9' || ',10:' || :'migration_sha_10' + || ',11:' || :'migration_sha_11' || ',12:' || :'migration_sha_12' + AS at_v12 \gset + \if :at_v12 + \elif :at_v11 + \ir 0012_probe_handoff.sql \elif :at_v10 \ir 0011_region_actions.sql + \ir 0012_probe_handoff.sql \elif :at_v9 \ir 0010_account_control.sql \ir 0011_region_actions.sql + \ir 0012_probe_handoff.sql \elif :at_v8 \ir 0009_serialized_authority.sql \ir 0010_account_control.sql \ir 0011_region_actions.sql + \ir 0012_probe_handoff.sql \elif :at_v7 \ir 0008_upgrade_barrier.sql \ir 0009_serialized_authority.sql \ir 0010_account_control.sql \ir 0011_region_actions.sql + \ir 0012_probe_handoff.sql \elif :at_v6 \ir 0007_region_clones.sql \ir 0008_upgrade_barrier.sql \ir 0009_serialized_authority.sql \ir 0010_account_control.sql \ir 0011_region_actions.sql + \ir 0012_probe_handoff.sql \elif :at_v5 \ir 0006_commit_lookup.sql \ir 0007_region_clones.sql @@ -80,6 +94,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0009_serialized_authority.sql \ir 0010_account_control.sql \ir 0011_region_actions.sql + \ir 0012_probe_handoff.sql \elif :at_v4 \ir pre_v5_guard.sql \ir 0005_batch_decisions.sql @@ -89,6 +104,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0009_serialized_authority.sql \ir 0010_account_control.sql \ir 0011_region_actions.sql + \ir 0012_probe_handoff.sql \elif :at_v3 \ir 0004_lineage_profile.sql \ir pre_v5_guard.sql @@ -99,6 +115,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0009_serialized_authority.sql \ir 0010_account_control.sql \ir 0011_region_actions.sql + \ir 0012_probe_handoff.sql \elif :at_v2 \ir 0003_region_checkpoints.sql \ir 0004_lineage_profile.sql @@ -110,6 +127,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0009_serialized_authority.sql \ir 0010_account_control.sql \ir 0011_region_actions.sql + \ir 0012_probe_handoff.sql \elif :at_v1 \ir 0002_region_fencing.sql \ir 0003_region_checkpoints.sql @@ -122,6 +140,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0009_serialized_authority.sql \ir 0010_account_control.sql \ir 0011_region_actions.sql + \ir 0012_probe_handoff.sql \else \echo 'Unknown or changed migration history' DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$; @@ -139,6 +158,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0009_serialized_authority.sql \ir 0010_account_control.sql \ir 0011_region_actions.sql + \ir 0012_probe_handoff.sql \endif -- Effective-privilege audit on every run, including no-ops (ADR-0060). Attributes -- alone are insufficient: predefined roles such as pg_write_all_data, PUBLIC @@ -176,7 +196,15 @@ BEGIN ('evolution.release_account_control(evolution.u64,evolution.u64,uuid,evolution.u64,uuid,evolution.u64)'::regprocedure::oid), ('evolution.publish_region_checkpoint(bytea,bytea,bytea)'::regprocedure::oid), ('evolution.resolve_region_commit(evolution.u64,evolution.u64,uuid,bytea,bytea,bytea)'::regprocedure::oid), - ('evolution.find_region_action(evolution.u64,evolution.u64,uuid,evolution.u64,uuid,evolution.u64)'::regprocedure::oid) + ('evolution.find_region_action(evolution.u64,evolution.u64,uuid,evolution.u64,uuid,evolution.u64)'::regprocedure::oid), + ('evolution.open_probe(evolution.u64,evolution.u64,uuid,evolution.u64,uuid,bigint,bigint,bigint,evolution.u64)'::regprocedure::oid), + ('evolution.save_probe(evolution.u64,evolution.u64,uuid,evolution.u64,uuid,evolution.u64,bigint,bigint,bigint)'::regprocedure::oid), + ('evolution.prepare_probe_transfer(evolution.u64,evolution.u64,uuid,evolution.u64,uuid,evolution.u64,uuid,evolution.u64,bigint,bigint,bigint)'::regprocedure::oid), + ('evolution.stage_probe_transfer(evolution.u64,evolution.u64,uuid,evolution.u64,uuid)'::regprocedure::oid), + ('evolution.commit_probe_transfer(evolution.u64,evolution.u64,uuid,evolution.u64,uuid)'::regprocedure::oid), + ('evolution.abort_probe_transfer(evolution.u64,evolution.u64,uuid,evolution.u64,uuid)'::regprocedure::oid), + ('evolution.find_probe_transfer(evolution.u64,evolution.u64,uuid,evolution.u64,uuid)'::regprocedure::oid), + ('evolution.handoff_account_control(evolution.u64,evolution.u64,uuid,evolution.u64,uuid,uuid,evolution.u64,integer)'::regprocedure::oid) ), violations(role_oid, reason) AS ( SELECT r.oid, 'privileged attribute or owner identity' FROM pg_catalog.pg_roles r JOIN closure c ON c.oid = r.oid diff --git a/schemas/probe-handoff-v1.md b/schemas/probe-handoff-v1.md new file mode 100644 index 0000000..5f6def6 --- /dev/null +++ b/schemas/probe-handoff-v1.md @@ -0,0 +1,46 @@ +# Probe records and handoff, version 1 + +EXPERIMENT (ADR-0094, ADR-0096; schema v12, migrations/0012_probe_handoff.sql). Durable +state of the non-biological probes and their cross-region transfers. It is reached only +through the fenced functions, and C++ callers use include/evo/persistence/probe_store.hpp. + +## Probe record (`evolution.probe_records`, one per world and account) + +| Field | Type | Rule | +|---|---|---| +| region_id | u64 | The single owning region | +| generation | u64 | > 0; + 1 at each committed transfer | +| x, y | bigint | Micrometres | +| z | bigint | Cell slice | +| inventory | u64 | EXPERIMENT test inventory; never copied, only moved with ownership | +| transfer_id | uuid or NULL | The open transfer while frozen | + +## Transfer (`evolution.probe_transfers`) + +States: +- **prepared:** source froze the record. +- **staged:** destination accepted it under `destination_epoch`. +- **committed:** ownership switched. +- **aborted:** before commit only; the source resumes. + +Each transfer keeps the record's generation at preparation, the source and destination +regions, the source epoch and the final state (x, y, z, inventory). At most one transfer is +open (prepared or staged) per record. + +## Calls + +Each call is made under the caller's exact unexpired region lease. + +| Call | By | Effect | +|---|---|---| +| open_probe | any region | Creates the record owned by the caller if none exists; returns it | +| save_probe | owner | Position of the active (not frozen) record at its generation | +| prepare_probe_transfer | owner | Freezes the record and records the transfer (prepared) | +| stage_probe_transfer | destination | prepared or staged becomes staged under the caller's epoch | +| commit_probe_transfer | destination | staged by the caller's current epoch becomes committed: owner = destination, generation + 1, final state | +| abort_probe_transfer | source or destination | prepared or staged becomes aborted; the record is active at the source | +| find_probe_transfer | any region | The transfer as recorded | +| handoff_account_control | destination | Account control (v10) for a transfer committed to the caller | + +A retry with the same transfer id returns the recorded outcome. SQLSTATE: EVL13 fenced, +EVL14 unknown transfer, EVL15 not staged by this epoch, EVL12 control handoff refused. diff --git a/src/persistence/region_lease_store.cpp b/src/persistence/region_lease_store.cpp index 0903524..d250254 100644 --- a/src/persistence/region_lease_store.cpp +++ b/src/persistence/region_lease_store.cpp @@ -1,5 +1,6 @@ #include "evo/persistence/region_lease_store.hpp" #include "evo/persistence/account_control_store.hpp" +#include "evo/persistence/probe_store.hpp" #include #include #include @@ -471,6 +472,31 @@ claim_account_control(Postgres& db, RegionKey key, const LeaseGrant& lease, cons return AccountControlClaim{*generation, milliseconds{*wait}, milliseconds{*granted}}; } +std::expected +handoff_account_control(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& account, + const Uuid& transfer, std::uint64_t session, milliseconds duration, + milliseconds timeout) { + const std::array p{std::to_string(key.world), std::to_string(key.region), + uuid_text(lease.holder), std::to_string(lease.epoch), + uuid_text(account), uuid_text(transfer), + std::to_string(session), std::to_string(duration.count())}; + auto rows = db.query("SELECT * FROM evolution.handoff_account_control($1::evolution.u64, " + "$2::evolution.u64, $3::uuid, $4::evolution.u64, $5::uuid, $6::uuid, " + "$7::evolution.u64, $8::integer)", + p, timeout); + if (!rows) + return std::unexpected{control_failure(rows.error())}; + if (rows->size() != 1 || (*rows)[0].size() != 3) + return std::unexpected{ControlStoreFailure{ControlStoreError::unknown, {}}}; + const auto& r = (*rows)[0]; + const auto generation = parse_integer(r[0]); + const auto wait = parse_integer(r[1]); + const auto granted = parse_integer(r[2]); + if (!generation || !wait || !granted || *generation == 0) + return std::unexpected{ControlStoreFailure{ControlStoreError::unknown, {}}}; + return AccountControlClaim{*generation, milliseconds{*wait}, milliseconds{*granted}}; +} + std::expected renew_account_control(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& account, std::uint64_t generation, milliseconds duration, milliseconds timeout) { @@ -506,4 +532,193 @@ release_account_control(Postgres& db, RegionKey key, const LeaseGrant& lease, co return std::unexpected{ControlStoreFailure{ControlStoreError::unknown, {}}}; return text == "true"; } + +// Schema v12 probe records and handoff (ADR-0096). Shares this file's parsing rules. +namespace { +ProbeStoreFailure probe_failure(const PgFailure& f) { + const std::string_view state(f.sqlstate.data(), 5); + auto kind = ProbeStoreError::store; + if (f.kind == PgError::unknown) + kind = ProbeStoreError::unknown; + else if (f.kind == PgError::config || f.kind == PgError::bounds || state == "22023" || + state == "EVL04") + kind = ProbeStoreError::invalid; + else if (state == "EVL02" || state == "EVL06" || state == "EVL13") + kind = ProbeStoreError::fenced; + else if (state == "EVL14") + kind = ProbeStoreError::unknown_transfer; + else if (state == "EVL15") + kind = ProbeStoreError::not_staged; + else if (state == "EVL05") + kind = ProbeStoreError::exhausted; + return {kind, f}; +} +std::optional parse_signed(const std::optional& s) { + if (!s || s->empty() || s->size() > 20 || s->front() == '+') + return std::nullopt; + std::int64_t value{}; + const auto* end = s->data() + s->size(); + const auto [at, error] = std::from_chars(s->data(), end, value); + if (error != std::errc{} || at != end) + return std::nullopt; + return value; +} +std::optional parse_state(const std::optional& s) { + if (s == "prepared") + return TransferState::prepared; + if (s == "staged") + return TransferState::staged; + if (s == "committed") + return TransferState::committed; + if (s == "aborted") + return TransferState::aborted; + return std::nullopt; +} +// One row of evolution.probe_transfer_row (9 columns). +template std::optional parse_transfer(const Row& r) { + if (r.size() != 9) + return std::nullopt; + const auto state = parse_state(r[0]); + const auto account = parse_uuid(r[1]); + const auto generation = parse_integer(r[2]); + const auto source = parse_integer(r[3]); + const auto destination = parse_integer(r[4]); + const auto x = parse_signed(r[5]); + const auto y = parse_signed(r[6]); + const auto z = parse_signed(r[7]); + const auto inventory = parse_integer(r[8]); + if (!state || !account || !generation || !source || !destination || !x || !y || !z || + !inventory) + return std::nullopt; + return ProbeTransfer{*state, *account, *generation, *source, *destination, + *x, *y, *z, *inventory}; +} +std::array transfer_parameters(RegionKey key, const LeaseGrant& lease, + const Uuid& transfer) { + return {std::to_string(key.world), std::to_string(key.region), uuid_text(lease.holder), + std::to_string(lease.epoch), uuid_text(transfer)}; +} +// Calls a transfer function whose parameters are the lease and one transfer id. +std::expected +transfer_call(Postgres& db, std::string_view function, RegionKey key, const LeaseGrant& lease, + const Uuid& transfer, milliseconds timeout) { + auto rows = db.query("SELECT * FROM evolution." + std::string{function} + + "($1::evolution.u64, $2::evolution.u64, $3::uuid, " + "$4::evolution.u64, $5::uuid)", + transfer_parameters(key, lease, transfer), timeout); + if (!rows) + return std::unexpected{probe_failure(rows.error())}; + const auto t = rows->size() == 1 ? parse_transfer((*rows)[0]) : std::nullopt; + if (!t) + return std::unexpected{ProbeStoreFailure{ProbeStoreError::unknown, {}}}; + return *t; +} +} // namespace + +std::expected +open_probe(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& account, + std::int64_t x, std::int64_t y, std::int64_t z, std::uint64_t inventory, + milliseconds timeout) { + const std::array p{ + std::to_string(key.world), std::to_string(key.region), uuid_text(lease.holder), + std::to_string(lease.epoch), uuid_text(account), std::to_string(x), + std::to_string(y), std::to_string(z), std::to_string(inventory)}; + auto rows = db.query("SELECT * FROM evolution.open_probe($1::evolution.u64, " + "$2::evolution.u64, $3::uuid, $4::evolution.u64, $5::uuid, $6::bigint, " + "$7::bigint, $8::bigint, $9::evolution.u64)", + p, timeout); + if (!rows) + return std::unexpected{probe_failure(rows.error())}; + if (rows->size() != 1 || (*rows)[0].size() != 7) + return std::unexpected{ProbeStoreFailure{ProbeStoreError::unknown, {}}}; + const auto& r = (*rows)[0]; + const auto region = parse_integer(r[0]); + const auto generation = parse_integer(r[1]); + const auto px = parse_signed(r[2]); + const auto py = parse_signed(r[3]); + const auto pz = parse_signed(r[4]); + const auto held = parse_integer(r[5]); + const auto transfer = r[6] ? parse_uuid(r[6]) : std::nullopt; + if (!region || !generation || !px || !py || !pz || !held || (r[6] && !transfer)) + return std::unexpected{ProbeStoreFailure{ProbeStoreError::unknown, {}}}; + return ProbeRecord{*region, *generation, *px, *py, *pz, *held, transfer}; +} + +std::expected save_probe(Postgres& db, RegionKey key, + const LeaseGrant& lease, const Uuid& account, + std::uint64_t generation, std::int64_t x, + std::int64_t y, std::int64_t z, + milliseconds timeout) { + const std::array p{ + std::to_string(key.world), std::to_string(key.region), uuid_text(lease.holder), + std::to_string(lease.epoch), uuid_text(account), std::to_string(generation), + std::to_string(x), std::to_string(y), std::to_string(z)}; + auto rows = db.query("SELECT evolution.save_probe($1::evolution.u64, $2::evolution.u64, " + "$3::uuid, $4::evolution.u64, $5::uuid, $6::evolution.u64, $7::bigint, " + "$8::bigint, $9::bigint)::text", + p, timeout); + if (!rows) + return std::unexpected{probe_failure(rows.error())}; + return {}; +} + +std::expected +prepare_probe_transfer(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& account, + std::uint64_t generation, const Uuid& transfer, std::uint64_t destination, + std::int64_t x, std::int64_t y, std::int64_t z, milliseconds timeout) { + const std::array p{std::to_string(key.world), std::to_string(key.region), + uuid_text(lease.holder), std::to_string(lease.epoch), + uuid_text(account), std::to_string(generation), + uuid_text(transfer), std::to_string(destination), + std::to_string(x), std::to_string(y), + std::to_string(z)}; + auto rows = db.query("SELECT * FROM evolution.prepare_probe_transfer($1::evolution.u64, " + "$2::evolution.u64, $3::uuid, $4::evolution.u64, $5::uuid, " + "$6::evolution.u64, $7::uuid, $8::evolution.u64, $9::bigint, " + "$10::bigint, $11::bigint)", + p, timeout); + if (!rows) + return std::unexpected{probe_failure(rows.error())}; + const auto t = rows->size() == 1 ? parse_transfer((*rows)[0]) : std::nullopt; + if (!t) + return std::unexpected{ProbeStoreFailure{ProbeStoreError::unknown, {}}}; + return *t; +} + +std::expected stage_probe_transfer(Postgres& db, RegionKey key, + const LeaseGrant& lease, + const Uuid& transfer, + milliseconds timeout) { + return transfer_call(db, "stage_probe_transfer", key, lease, transfer, timeout); +} + +std::expected commit_probe_transfer(Postgres& db, RegionKey key, + const LeaseGrant& lease, + const Uuid& transfer, + milliseconds timeout) { + return transfer_call(db, "commit_probe_transfer", key, lease, transfer, timeout); +} + +std::expected abort_probe_transfer(Postgres& db, RegionKey key, + const LeaseGrant& lease, + const Uuid& transfer, + milliseconds timeout) { + return transfer_call(db, "abort_probe_transfer", key, lease, transfer, timeout); +} + +std::expected, ProbeStoreFailure> +find_probe_transfer(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& transfer, + milliseconds timeout) { + auto rows = db.query("SELECT * FROM evolution.find_probe_transfer($1::evolution.u64, " + "$2::evolution.u64, $3::uuid, $4::evolution.u64, $5::uuid)", + transfer_parameters(key, lease, transfer), timeout); + if (!rows) + return std::unexpected{probe_failure(rows.error())}; + if (rows->empty()) + return std::optional{}; + const auto t = rows->size() == 1 ? parse_transfer((*rows)[0]) : std::nullopt; + if (!t) + return std::unexpected{ProbeStoreFailure{ProbeStoreError::unknown, {}}}; + return t; +} } // namespace evo::persistence diff --git a/tests/postgres_tests.cpp b/tests/postgres_tests.cpp index 175025b..4423cdc 100644 --- a/tests/postgres_tests.cpp +++ b/tests/postgres_tests.cpp @@ -5,6 +5,7 @@ #include void run_region_lease_store_tests(); void run_account_control_store_tests(); +void run_probe_handoff_store_tests(); void run_region_action_tests(); void run_region_checkpoint_store_tests(); void run_region_host_tests(); @@ -122,6 +123,7 @@ int main() { std::puts("postgres adapter integration passed"); run_region_lease_store_tests(); run_account_control_store_tests(); + run_probe_handoff_store_tests(); run_region_action_tests(); std::puts("region lease fencing integration passed"); run_region_checkpoint_store_tests(); diff --git a/tests/probe_handoff_store_tests.cpp b/tests/probe_handoff_store_tests.cpp new file mode 100644 index 0000000..66f8b40 --- /dev/null +++ b/tests/probe_handoff_store_tests.cpp @@ -0,0 +1,379 @@ +// Real PostgreSQL M10.8a probe handoff checks (ADR-0096, schema v12): ownership and +// fencing, the transfer state machine with retries at every step, source and destination +// restarts (new epochs) at every transition, stale epochs, commit/abort races, crossing back, +// failed destinations and partitions, and the destination's account control handoff, with +// unique-owner and conservation checks after each scenario. World 9200 is used only here. Run by +// tools/ci/postgres-check.sh. +#include "evo/persistence/account_control_store.hpp" +#include "evo/persistence/probe_store.hpp" +#include "evo/persistence/region_checkpoint.hpp" +#include "evo/persistence/region_lease_store.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +void run_probe_handoff_store_tests(); + +namespace { +using namespace evo::persistence; +using namespace std::chrono_literals; +void check(bool value, std::source_location at = std::source_location::current()) { + if (!value) + throw std::runtime_error("probe handoff store test line " + std::to_string(at.line())); +} +#define CHECK(...) check(static_cast(__VA_ARGS__)) +constexpr std::uint64_t world = 9200; +constexpr std::uint64_t inventory = 1000; // EXPERIMENT test inventory per probe. + +std::string env(const char* key) { + const auto* value = std::getenv(key); + if (!value) + throw std::runtime_error("missing test environment"); + return value; +} +std::unique_ptr connect_as(const std::string& user) { + PgOptions options{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}}; + auto made = Postgres::connect(options, 5s); + CHECK(made); + return std::move(*made); +} +Uuid id(std::uint8_t seed, std::uint8_t tag = 0) { + Uuid out{}; + out.fill(seed); + out.back() = tag; + return out; +} +std::string bytea(std::span bytes) { + constexpr char digits[] = "0123456789abcdef"; + std::string out = "\\x"; + for (const auto b : bytes) { + out.push_back(digits[b >> 4U]); + out.push_back(digits[b & 0x0FU]); + } + return out; +} +Sha256Digest digest(std::uint8_t seed) { + Sha256Digest out{}; + out.fill(seed); + return out; +} +const Sha256Digest build = digest(0xB1); +RegionKey provision(Postgres& db, std::uint64_t region) { + CHECK(db.query("INSERT INTO evolution.retained_builds(build_identity) VALUES " + "(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING", + {}, 5s)); + const RegionKey key{world, region}; + auto commit = id(0xAD, static_cast(region)); // Unique commit ids per test. + auto g = genesis_manifest(key, 7, commit, 50000, build, 100, digest(0xA0)); + CHECK(g); + auto bytes = encode_manifest(*g); + CHECK(bytes); + const std::array p{bytea(*bytes)}; + CHECK(db.query("SELECT evolution.provision_region_checkpoint($1::bytea)", p, 5s)); + return key; +} +LeaseGrant acquire(Postgres& db, RegionKey key) { + auto lease = acquire_region_lease(db, key, build, 20000ms, 5s); + CHECK(lease); + return *lease; +} +// A region process "restart": its lease is released and a new epoch acquired. +LeaseGrant restart(Postgres& db, RegionKey key, const LeaseGrant& lease) { + CHECK(release_region_lease(db, key, lease, 5s)); + const auto next = acquire(db, key); + CHECK(next.epoch > lease.epoch); + return next; +} +bool failed(const auto& result, ProbeStoreError kind) { + return !result && result.error().kind == kind; +} +TransferState state_of(const auto& result) { + if (!result) + throw std::runtime_error("probe handoff store: missing transfer"); + return result->state; +} + +// World-wide invariants, read as the owner: one row per probe (primary key), the test +// inventory conserved, and a probe frozen exactly when it has one open transfer, which +// names its owner as the source. +void invariants(Postgres& admin, std::uint64_t probes) { + auto rows = admin.query( + "SELECT count(*)::text, coalesce(sum(inventory), 0)::text FROM evolution.probe_records " + "WHERE world_id = 9200", + {}, 5s); + CHECK(rows && rows->size() == 1); + CHECK((*rows)[0][0] == std::to_string(probes)); + CHECK((*rows)[0][1] == std::to_string(probes * inventory)); + rows = admin.query( + "SELECT count(*)::text FROM evolution.probe_records r WHERE r.world_id = 9200 AND " + "(r.transfer_id IS NOT NULL) <> EXISTS (SELECT 1 FROM evolution.probe_transfers t " + "WHERE t.world_id = r.world_id AND t.transfer_id = r.transfer_id AND t.state IN " + "('prepared', 'staged') AND t.source_region = r.region_id AND t.account_id = " + "r.account_id)", + {}, 5s); + CHECK(rows && rows->size() == 1 && (*rows)[0][0] == "0"); + rows = admin.query("SELECT count(*)::text FROM evolution.probe_transfers t WHERE " + "t.world_id = 9200 AND t.state IN ('prepared', 'staged') AND NOT EXISTS " + "(SELECT 1 FROM evolution.probe_records r WHERE r.world_id = t.world_id " + "AND r.account_id = t.account_id AND r.transfer_id = t.transfer_id)", + {}, 5s); + CHECK(rows && rows->size() == 1 && (*rows)[0][0] == "0"); +} + +struct World { + std::unique_ptr admin = connect_as(env("PGUSER")); + std::unique_ptr db = connect_as(env("EVO_PG_RUNTIME_USER")); + RegionKey a = provision(*admin, 1), b = provision(*admin, 2), c = provision(*admin, 3); + LeaseGrant la = acquire(*db, a), lb = acquire(*db, b), lc = acquire(*db, c); + std::uint64_t probes{}; + ProbeRecord open(const RegionKey& key, const LeaseGrant& lease, const Uuid& account, + std::int64_t x = 0) { + const auto r = open_probe(*db, key, lease, account, x, 0, -7, inventory, 5s); + CHECK(r); + return *r; + } +}; + +void ownership_and_happy_path(World& w) { + auto& db = *w.db; + // The runtime reaches the tables only through the functions. + CHECK(!db.query("SELECT count(*) FROM evolution.probe_records", {}, 5s)); + CHECK(!db.query("SELECT count(*) FROM evolution.probe_transfers", {}, 5s)); + const auto alice = id(0x41); + // Created owned by A; B sees A as owner and cannot act; reopening changes nothing. + auto r = w.open(w.a, w.la, alice, 500'000); + ++w.probes; + CHECK(r.region == 1 && r.generation == 1 && r.x == 500'000 && r.inventory == inventory && + !r.transfer); + const auto seen = w.open(w.b, w.lb, alice, 9); + CHECK(seen.region == 1 && seen.x == 500'000); + CHECK(save_probe(db, w.a, w.la, alice, 1, 600'000, 0, -7, 5s)); + CHECK(failed(save_probe(db, w.b, w.lb, alice, 1, 0, 0, -7, 5s), ProbeStoreError::fenced)); + CHECK(failed(save_probe(db, w.a, w.la, alice, 2, 0, 0, -7, 5s), ProbeStoreError::fenced)); + invariants(*w.admin, w.probes); + // Arguments: a transfer to itself or to an unprovisioned region is refused. + const auto t1 = id(0x71, 1); + CHECK(failed(prepare_probe_transfer(db, w.a, w.la, alice, 1, t1, 1, 0, 0, -7, 5s), + ProbeStoreError::invalid)); + CHECK(failed(prepare_probe_transfer(db, w.a, w.la, alice, 1, t1, 99, 0, 0, -7, 5s), + ProbeStoreError::invalid)); + CHECK(failed(stage_probe_transfer(db, w.b, w.lb, t1, 5s), ProbeStoreError::unknown_transfer)); + // Prepare: frozen at its final state; nothing else may change it; retries are answered. + auto p = prepare_probe_transfer(db, w.a, w.la, alice, 1, t1, 2, 700'000, 10, -7, 5s); + CHECK(p && p->state == TransferState::prepared && p->source == 1 && p->destination == 2 && + p->x == 700'000 && p->inventory == inventory && p->account == alice); + CHECK(prepare_probe_transfer(db, w.a, w.la, alice, 1, t1, 2, 1, 1, 1, 5s) == *p); + CHECK(failed(save_probe(db, w.a, w.la, alice, 1, 0, 0, -7, 5s), ProbeStoreError::fenced)); + CHECK(failed(prepare_probe_transfer(db, w.a, w.la, alice, 1, id(0x71, 2), 3, 0, 0, -7, 5s), + ProbeStoreError::fenced)); // One open transfer per probe. + CHECK(w.open(w.a, w.la, alice).transfer == t1); + invariants(*w.admin, w.probes); + // Only the named destination stages and commits; a commit before staging is refused. + CHECK(failed(stage_probe_transfer(db, w.c, w.lc, t1, 5s), ProbeStoreError::fenced)); + CHECK(failed(commit_probe_transfer(db, w.b, w.lb, t1, 5s), ProbeStoreError::not_staged)); + CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t1, 5s)) == TransferState::staged); + CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t1, 5s)) == TransferState::staged); + CHECK(failed(commit_probe_transfer(db, w.c, w.lc, t1, 5s), ProbeStoreError::fenced)); + // The one switch: owner B, next generation, the frozen final state, same inventory. + const auto committed = commit_probe_transfer(db, w.b, w.lb, t1, 5s); + CHECK(state_of(committed) == TransferState::committed); + r = w.open(w.b, w.lb, alice); + CHECK(r.region == 2 && r.generation == 2 && r.x == 700'000 && r.y == 10 && + r.inventory == inventory && !r.transfer); + // Every retry and late call returns the committed outcome and changes nothing. + CHECK(commit_probe_transfer(db, w.b, w.lb, t1, 5s) == *committed); + CHECK(stage_probe_transfer(db, w.b, w.lb, t1, 5s) == *committed); + CHECK(prepare_probe_transfer(db, w.a, w.la, alice, 1, t1, 2, 0, 0, 0, 5s) == *committed); + CHECK(abort_probe_transfer(db, w.a, w.la, t1, 5s) == *committed); + const auto found = find_probe_transfer(db, w.c, w.lc, t1, 5s); + CHECK(found && *found == std::optional{*committed}); + const auto missing = find_probe_transfer(db, w.c, w.lc, id(0x7F), 5s); + CHECK(missing && !missing->has_value()); + // The old owner is fenced; the new one acts. + CHECK(failed(save_probe(db, w.a, w.la, alice, 1, 0, 0, -7, 5s), ProbeStoreError::fenced)); + CHECK(save_probe(db, w.b, w.lb, alice, 2, 800'000, 10, -7, 5s)); + // A transfer id belongs to one account and source. + const auto bob = id(0x42); + w.open(w.a, w.la, bob); + ++w.probes; + CHECK(failed(prepare_probe_transfer(db, w.a, w.la, bob, 1, t1, 2, 0, 0, -7, 5s), + ProbeStoreError::fenced)); + // Crossing back: B -> A with a new transfer, next generation. + const auto t3 = id(0x71, 3); + CHECK(state_of(prepare_probe_transfer(db, w.b, w.lb, alice, 2, t3, 1, 800'000, 10, -7, 5s)) == + TransferState::prepared); + CHECK(state_of(stage_probe_transfer(db, w.a, w.la, t3, 5s)) == TransferState::staged); + CHECK(state_of(commit_probe_transfer(db, w.a, w.la, t3, 5s)) == TransferState::committed); + r = w.open(w.a, w.la, alice); + CHECK(r.region == 1 && r.generation == 3 && r.inventory == inventory); + invariants(*w.admin, w.probes); +} + +// A crash (restart: new epoch) at each transition, on either side. +void restarts(World& w) { + auto& db = *w.db; + const auto carol = id(0x43); + w.open(w.a, w.la, carol); + ++w.probes; + // 1. Source crashes after prepare. Its new epoch sees the probe frozen (it must not act) + // and the transfer proceeds without it. + const auto t = id(0x72, 1); + CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, carol, 1, t, 2, 1, 2, -7, 5s)) == + TransferState::prepared); + w.la = restart(db, w.a, w.la); + const auto frozen = w.open(w.a, w.la, carol); + CHECK(frozen.region == 1 && frozen.transfer == t); + CHECK(failed(save_probe(db, w.a, w.la, carol, 1, 0, 0, -7, 5s), ProbeStoreError::fenced)); + // The new source epoch may still answer retries of its own prepare. + CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, carol, 1, t, 2, 1, 2, -7, 5s)) == + TransferState::prepared); + CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::staged); + // 2. Destination crashes after staging: its new epoch must stage again before committing; + // the old epoch is fenced everywhere. + const auto old_b = w.lb; + w.lb = restart(db, w.b, w.lb); + CHECK(failed(commit_probe_transfer(db, w.b, w.lb, t, 5s), ProbeStoreError::not_staged)); + CHECK(failed(commit_probe_transfer(db, w.b, old_b, t, 5s), ProbeStoreError::fenced)); + CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::staged); + // 3. Destination crashes after its commit, before observing it: the retry answers. + CHECK(state_of(commit_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::committed); + w.lb = restart(db, w.b, w.lb); + CHECK(state_of(commit_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::committed); + const auto r = w.open(w.b, w.lb, carol); + CHECK(r.region == 2 && r.generation == 2 && r.x == 1 && r.y == 2 && !r.transfer); + invariants(*w.admin, w.probes); +} + +// Failed destination, abort rules, partition, and a commit/abort race. +void aborts_and_races(World& w) { + auto& db = *w.db; + const auto dave = id(0x44); + w.open(w.a, w.la, dave); + ++w.probes; + // The destination never stages: the source aborts and resumes, durably. + const auto t1 = id(0x73, 1); + CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, dave, 1, t1, 2, 5, 5, -7, 5s)) == + TransferState::prepared); + CHECK(failed(abort_probe_transfer(db, w.c, w.lc, t1, 5s), ProbeStoreError::fenced)); + CHECK(state_of(abort_probe_transfer(db, w.a, w.la, t1, 5s)) == TransferState::aborted); + CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t1, 5s)) == TransferState::aborted); + CHECK(state_of(commit_probe_transfer(db, w.b, w.lb, t1, 5s)) == TransferState::aborted); + auto r = w.open(w.a, w.la, dave); + CHECK(r.region == 1 && r.generation == 1 && !r.transfer && r.x == 5); + CHECK(save_probe(db, w.a, w.la, dave, 1, 6, 6, -7, 5s)); // Active again at its source. + // The destination aborts a staged transfer it cannot accept. + const auto t2 = id(0x73, 2); + CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, dave, 1, t2, 2, 6, 6, -7, 5s)) == + TransferState::prepared); + CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t2, 5s)) == TransferState::staged); + CHECK(state_of(abort_probe_transfer(db, w.b, w.lb, t2, 5s)) == TransferState::aborted); + CHECK(!w.open(w.a, w.la, dave).transfer); + // Partition: the source's lease lapses (it stops) while the destination commits; on + // recovery the source finds the probe owned elsewhere and the transfer committed. + const auto t3 = id(0x73, 3); + CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, dave, 1, t3, 2, 7, 7, -7, 5s)) == + TransferState::prepared); + CHECK(release_region_lease(db, w.a, w.la, 5s)); // Cut off: no source authority at all. + CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t3, 5s)) == TransferState::staged); + CHECK(state_of(commit_probe_transfer(db, w.b, w.lb, t3, 5s)) == TransferState::committed); + w.la = acquire(db, w.a); + CHECK(state_of(find_probe_transfer(db, w.a, w.la, t3, 5s).value_or(std::nullopt)) == + TransferState::committed); + r = w.open(w.a, w.la, dave); + CHECK(r.region == 2 && r.generation == 2); + CHECK(failed(save_probe(db, w.a, w.la, dave, 2, 0, 0, -7, 5s), ProbeStoreError::fenced)); + invariants(*w.admin, w.probes); + // Race: the destination commits while the source aborts, on separate connections. + // Exactly one wins, both observe the same outcome, and the owner agrees with it. + auto other = connect_as(env("EVO_PG_RUNTIME_USER")); + int commits = 0, aborts = 0; + for (std::uint8_t round = 0; round < 24; ++round) { + const auto account = id(0x50, round); + w.open(w.a, w.la, account); + ++w.probes; + const auto t = id(0x74, round); + CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, account, 1, t, 2, 1, 1, -7, 5s)) == + TransferState::prepared); + CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::staged); + std::latch go{2}; + std::optional committed, aborted; + std::thread racer{[&] { + go.arrive_and_wait(); + committed = state_of(commit_probe_transfer(*other, w.b, w.lb, t, 5s)); + }}; + go.arrive_and_wait(); + aborted = state_of(abort_probe_transfer(db, w.a, w.la, t, 5s)); + racer.join(); + CHECK(committed == aborted); + const auto owner = w.open(w.a, w.la, account); + if (committed == TransferState::committed) { + ++commits; + CHECK(owner.region == 2 && owner.generation == 2 && !owner.transfer); + } else { + ++aborts; + CHECK(committed == TransferState::aborted && owner.region == 1 && + owner.generation == 1 && !owner.transfer); + } + } + CHECK(commits + aborts == 24); + std::printf("probe handoff race commits=%d aborts=%d\n", commits, aborts); + invariants(*w.admin, w.probes); +} +// The destination takes the account's durable control on the strength of a transfer +// committed to it; nothing else (another region, an open or aborted transfer) qualifies. +void control_handoff(World& w) { + auto& db = *w.db; + const auto eve = id(0x45); + const auto none = id(0x46); + w.open(w.a, w.la, eve); + w.open(w.a, w.la, none); + w.probes += 2; + const auto claim = claim_account_control(db, w.a, w.la, eve, 1, 11, 20000ms, 5s); + CHECK(claim && claim->generation == 1); + const auto t = id(0x75, 1); + CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, eve, 1, t, 2, 0, 0, -7, 5s)) == + TransferState::prepared); + const auto refused = [&](const RegionKey& key, const LeaseGrant& lease, const Uuid& account) { + const auto r = handoff_account_control(db, key, lease, account, t, 21, 20000ms, 5s); + return !r && r.error().kind == ControlStoreError::fenced; + }; + CHECK(refused(w.b, w.lb, eve)); // Not committed yet. + CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::staged); + CHECK(state_of(commit_probe_transfer(db, w.b, w.lb, t, 5s)) == TransferState::committed); + CHECK(refused(w.c, w.lc, eve)); // Not the destination. + CHECK(refused(w.b, w.lb, none)); // Not this transfer's account. + // The source still holds unexpired control: the destination must outlive it. + const auto waited = handoff_account_control(db, w.b, w.lb, eve, t, 21, 20000ms, 5s); + CHECK(waited && waited->generation == 2 && waited->wait > 0ms); + // After the source released (the normal path), there is nothing to wait for. + CHECK(!release_account_control(db, w.a, w.la, eve, 1, 5s).value_or(true)); // Moved on. + CHECK(release_account_control(db, w.b, w.lb, eve, 2, 5s).value_or(false)); + const auto again = handoff_account_control(db, w.b, w.lb, eve, t, 22, 20000ms, 5s); + CHECK(again && again->generation == 3 && again->wait == 0ms); + CHECK(release_account_control(db, w.b, w.lb, eve, 3, 5s).value_or(false)); + invariants(*w.admin, w.probes); +} +} // namespace + +void run_probe_handoff_store_tests() { + World w; + ownership_and_happy_path(w); + restarts(w); + aborts_and_races(w); + control_handoff(w); + CHECK(release_region_lease(*w.db, w.a, w.la, 5s)); + CHECK(release_region_lease(*w.db, w.b, w.lb, 5s)); + CHECK(release_region_lease(*w.db, w.c, w.lc, 5s)); + std::puts("probe handoff integration passed"); +} diff --git a/tests/region_migration_tests.cpp b/tests/region_migration_tests.cpp index 4b51f53..2ef682a 100644 --- a/tests/region_migration_tests.cpp +++ b/tests/region_migration_tests.cpp @@ -177,6 +177,7 @@ void upgrade_across_migration_and_barrier() { " -v migration_sha_9=$(s 0009_serialized_authority.sql)" " -v migration_sha_10=$(s 0010_account_control.sql)" " -v migration_sha_11=$(s 0011_region_actions.sql)" + " -v migration_sha_12=$(s 0012_probe_handoff.sql)" " -v runtime_role=evo_runtime_test -f migrations/apply.sql" " > /dev/null 2>&1") != 0); CHECK(history(*owner_db) == "1,2,3,4,5,6,7"); @@ -192,7 +193,7 @@ void upgrade_across_migration_and_barrier() { CHECK(shell("PGDATABASE=" + database + " EVO_PG_RUNTIME_ROLE=evo_runtime_test bash \"$EVO_SOURCE_ROOT/" "tools/migrate-postgres.sh\" > /dev/null") == 0); - CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8,9,10,11"); + CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8,9,10,11,12"); CHECK(renew_region_lease(*runtime, r.key, *lease, 20000ms, 5s)); confirm(publish_next(20)); // 41..60, now v8. CHECK(head.head.end_tick == 60); diff --git a/tools/ci/postgres-check.sh b/tools/ci/postgres-check.sh index beeed1d..eb63029 100644 --- a/tools/ci/postgres-check.sh +++ b/tools/ci/postgres-check.sh @@ -61,7 +61,7 @@ PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" -f "$root/migrations/0001_region_bootstrap.sql" > /dev/null test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1 PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11 +test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12 # v2 heads have no payload binding: v2->v3 fails closed and leaves history at v2. v2_sha="$(sha256sum "$root/migrations/0002_region_fencing.sql")" v2_sha="${v2_sha%% *}" @@ -82,7 +82,7 @@ PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '01000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32), 'hex'))" > /dev/null PGDATABASE=v3_commits bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 11:1,2,3,4,5,6,7,8,9,10,11 +test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 12:1,2,3,4,5,6,7,8,9,10,11,12 test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) FROM evolution.region_commits")" = 1 # Review P1: a v4 database holding a published (possibly terminal) checkpoint has # no durable decisions for it. The v5 upgrade must refuse before changing anything. @@ -136,7 +136,7 @@ PGDATABASE=v6_populated psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '02000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32) || repeat('00', 80), 'hex'))" > /dev/null PGDATABASE=v6_populated bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11 +test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT count(*) FROM evolution.region_commits c, LATERAL evolution.parse_checkpoint_manifest(c.manifest) m WHERE m.commit_sequence = 0 AND m.predecessor_commit IS NULL")" = 1 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(conname, ',') FROM pg_constraint WHERE conrelid = 'evolution.region_heads'::regclass AND contype = 'c' AND pg_get_constraintdef(oid) LIKE '%end_tick%'")" = region_heads_published_tick # Inherited privileges bypass fencing, so every migrate run audits effective rights. @@ -169,6 +169,9 @@ expect_audit_failure 'GRANT EXECUTE ON FUNCTION evolution.check_region_lease(evo expect_audit_failure 'ALTER ROLE evo_runtime_login CREATEDB' 'ALTER ROLE evo_runtime_login NOCREATEDB' expect_audit_failure 'GRANT EXECUTE ON FUNCTION evolution.provision_region_checkpoint(bytea) TO evo_runtime_test' 'REVOKE EXECUTE ON FUNCTION evolution.provision_region_checkpoint(bytea) FROM evo_runtime_test' expect_audit_failure 'GRANT SELECT ON evolution.region_commits TO evo_runtime_test' 'REVOKE SELECT ON evolution.region_commits FROM evo_runtime_test' +# Schema v12: probe records and transfers are reachable only through the fenced functions. +expect_audit_failure 'GRANT SELECT ON evolution.probe_records TO evo_runtime_test' 'REVOKE SELECT ON evolution.probe_records FROM evo_runtime_test' +expect_audit_failure 'GRANT UPDATE ON evolution.probe_transfers TO evo_runtime_test' 'REVOKE UPDATE ON evolution.probe_transfers FROM evo_runtime_test' # Schema v11: action records are reachable only through the fenced functions. expect_audit_failure 'GRANT SELECT ON evolution.region_actions TO evo_runtime_test' 'REVOKE SELECT ON evolution.region_actions FROM evo_runtime_test' expect_audit_failure 'GRANT SELECT ON evolution.region_commit_actions TO evo_runtime_test' 'REVOKE SELECT ON evolution.region_commit_actions FROM evo_runtime_test' @@ -178,9 +181,11 @@ expect_audit_failure 'GRANT SELECT ON evolution.account_controls TO evo_runtime_ expect_audit_failure 'REVOKE EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) FROM evo_runtime_test' 'GRANT EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) TO evo_runtime_test' bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again. # Unknown future versions and checksum drift must not be repaired silently. -psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=12 WHERE version=11' +psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=13 WHERE version=12' +if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=12 WHERE version=13" +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=12" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi -psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=11 WHERE version=12" psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=11" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=10" diff --git a/tools/ci/upgrade-check.sh b/tools/ci/upgrade-check.sh index 22cda3f..766ee10 100644 --- a/tools/ci/upgrade-check.sh +++ b/tools/ci/upgrade-check.sh @@ -4,7 +4,7 @@ # exported). Real evo-server processes on world 500: # 1. Failed migration: a migration that takes the migration lock, alters a history # table, holds its locks for 3 s and then fails runs while build b1 serves. -# Nothing it did remains (schema still v11, no new column); the host keeps its +# Nothing it did remains (schema still v12, no new column); the host keeps its # authority, and every acknowledgement stays durable. # 2. Barrier (ADR-0068 operator order): build b2 is retained, and a b2 host is started # while b1 still holds the lease (it waits, never ready). upgrade_barrier(b1) fences @@ -61,7 +61,7 @@ SQL [ "$code" != 0 ] && grep -q 'division by zero' "$work/migration.log" || fail "the rehearsal migration did not fail" [ "$(q -c "SELECT count(*) FROM information_schema.columns WHERE table_name = 'region_commits' AND column_name = 'rehearsal_column'")" = 0 ] || fail "the failed migration left a column" -[ "$(q -c 'SELECT max(version) FROM evolution.schema_migrations')" = 11 ] || fail "schema version moved" +[ "$(q -c 'SELECT max(version) FROM evolution.schema_migrations')" = 12 ] || fail "schema version moved" wait_ack "$work/b1.log" $((before + 40)) || fail "b1 did not resume after the failed migration" kill -0 "$b1_pid" 2> /dev/null || fail "b1 exited during the failed migration" bash "$root/tools/migrate-postgres.sh" > /dev/null || fail "migrate after the failure" diff --git a/tools/migrate-postgres.sh b/tools/migrate-postgres.sh index 3347c2d..a647e3e 100644 --- a/tools/migrate-postgres.sh +++ b/tools/migrate-postgres.sh @@ -34,11 +34,13 @@ digest_10="$(sha256sum "$root/migrations/0010_account_control.sql")" digest_10="${digest_10%% *}" digest_11="$(sha256sum "$root/migrations/0011_region_actions.sql")" digest_11="${digest_11%% *}" +digest_12="$(sha256sum "$root/migrations/0012_probe_handoff.sql")" +digest_12="${digest_12%% *}" psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_2=$digest_2" --set="migration_sha_3=$digest_3" \ --set="migration_sha_4=$digest_4" --set="migration_sha_5=$digest_5" \ --set="migration_sha_6=$digest_6" --set="migration_sha_7=$digest_7" \ --set="migration_sha_8=$digest_8" --set="migration_sha_9=$digest_9" \ - --set="migration_sha_10=$digest_10" --set="migration_sha_11=$digest_11" \ + --set="migration_sha_10=$digest_10" --set="migration_sha_11=$digest_11" --set="migration_sha_12=$digest_12" \ --set="runtime_role=$runtime_role" \ --file="$root/migrations/apply.sql"