From 265163a9a6ccef2e99249788c549c8ef0d7ed635 Mon Sep 17 00:00:00 2001 From: dingo Date: Tue, 6 Oct 2026 23:17:49 -0400 Subject: [PATCH] Add M10.8d two-process handoff fault matrix and recovery fixes ADR-0096. Two real evo-server processes on PostgreSQL (regions 7/161 and 7/162, adjacent areas) run crossings and crossings back, SIGKILL of the source after prepare and of the destination after stage and after commit, duplicate and stale ticket retries, a client that never arrives and a destination that is down. After every scenario SQL checks one owner per probe, the conserved inventory and frozen-iff-open-transfer. The matrix found two recovery gaps, now fixed: - a source restarted after a prepare left the probe frozen for good (its abort deadline was in memory): schema v14 overdue_probe_transfers and a session-layer sweep abort open outgoing transfers past the ticket lifetime plus 10 s by the database clock; - a ticket replayed at a restarted destination after the probe moved on took control back: admission of a committed transfer now requires the record as that commit left it. M10.8 is complete. Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 4 +- .../region/dev-session-route/listener.yaml | 2 +- docs/ROADMAP.md | 4 +- docs/STATUS.md | 13 +- docs/adr/0096-m10-probe-handoff.md | 47 ++ docs/engineering/VALIDATION.md | 46 ++ docs/infrastructure/POSTGRESQL.md | 7 +- include/evo/persistence/probe_store.hpp | 5 + .../evo/server/postgres_handoff_ledger.hpp | 4 +- include/evo/server/probe_ledger.hpp | 5 + include/evo/server/region_sessions.hpp | 3 + migrations/0014_probe_overdue.sql | 35 ++ migrations/apply.sql | 30 +- schemas/probe-handoff-v1.md | 2 + src/persistence/region_lease_store.cpp | 23 + .../session/postgres_handoff_ledger.cpp | 28 + src/server/session/region_sessions.cpp | 20 +- tests/fixtures/fake_handoff_world.hpp | 11 + tests/postgres_tests.cpp | 2 + tests/probe_handoff_store_tests.cpp | 28 + tests/quic_sessions.cpp | 16 + tests/region_handoff_process_tests.cpp | 519 ++++++++++++++++++ tests/region_migration_tests.cpp | 3 +- tools/ci/postgres-check.sh | 12 +- tools/ci/upgrade-check.sh | 4 +- tools/migrate-postgres.sh | 3 + 26 files changed, 856 insertions(+), 20 deletions(-) create mode 100644 migrations/0014_probe_overdue.sql create mode 100644 tests/region_handoff_process_tests.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 6bb5138..d762323 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -264,7 +264,9 @@ if(EVO_BUILD_POSTGRES) endif() if(EVO_BUILD_QUIC) # M10.4c: evo-server with the QUIC listener, real clients and the probe (ADR-0092). - target_sources(evo-postgres-tests PRIVATE tests/region_m10_session_tests.cpp) + # M10.8d: two evo-server processes handing probes over (ADR-0096). + target_sources(evo-postgres-tests PRIVATE tests/region_m10_session_tests.cpp + tests/region_handoff_process_tests.cpp) target_link_libraries(evo-postgres-tests PRIVATE evo_session_client) target_compile_definitions(evo-postgres-tests PRIVATE EVO_POSTGRES_M10=1) add_dependencies(evo-postgres-tests evo-session-probe) diff --git a/deploy/kubernetes/region/dev-session-route/listener.yaml b/deploy/kubernetes/region/dev-session-route/listener.yaml index 73ed370..99fcde7 100644 --- a/deploy/kubernetes/region/dev-session-route/listener.yaml +++ b/deploy/kubernetes/region/dev-session-route/listener.yaml @@ -12,7 +12,7 @@ spec: env: - {name: REGION_SESSION, value: "dev"} - {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"} - - {name: REGION_SESSION_WINDOW, value: "dev-20261007T015045-2140:1791339045"} + - {name: REGION_SESSION_WINDOW, value: "dev-20261007T031149-a452:1791343909"} ports: # QUIC (ADR-0090): UDP only; nothing listens on TCP 7400. - {name: session, containerPort: 7400, protocol: UDP} diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 23fdbc9..5f3587a 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -539,7 +539,9 @@ transfer state machine with a PostgreSQL fault matrix done; 10.8b signed handoff ticket admission (durable stage, commit and control move) done; 10.8c-1 session protocol v4, durable probes in sessions, edge crossing, ghosts (schema v13) and a two-region loopback test done; 10.8c-2 client switch (offers, redirects, multi-region grants) and evo-server/probe -configuration done; 10.8d two-process kill tests follow. 10.9 planned. +configuration done; 10.8d two evo-server processes on PostgreSQL with SIGKILL around each +durable step, duplicate/stale retries and a failed destination done (schema v14 recovery +sweep, replay guard). M10.8 complete. 10.9 planned. **Entry:** M9 client/session evidence and M7/M8 dev/stage durability. Read NETWORK_ARCHITECTURE, PERSISTENCE_ARCHITECTURE and ADR-0017. The M9 transport is diff --git a/docs/STATUS.md b/docs/STATUS.md index 977e50c..6dc4cea 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -87,7 +87,18 @@ the session. M10.8c-1 wires the regions: M10.8c-2 has the engineering client follow crossings (ticket, no loading screen) and redirects, with grants picked by region in rising serial order. evo-server gains `--m10-area`, `--m10-handoff-key` and `--m10-neighbour`; the probe tool writes multi-region -bundles. Next: 10.8d, two evo-server processes on PostgreSQL with kills at each step. +bundles. M10.8d runs two evo-server processes on PostgreSQL through crossings and +crossings back, with SIGKILL of the source after prepare and of the destination after stage +and after commit, duplicate and stale ticket retries, a client that never arrives and a +destination that is down. After every scenario each probe has one owner, the inventory is +conserved and a record is frozen only under its open transfer. It found and fixed two +recovery gaps: +- a source restarted after a prepare now aborts overdue transfers from the database + (schema v14); +- a ticket replayed at a restarted destination after the probe moved on is refused. + +M10.8 is complete. Deploying needs protocol v4 and schemas v12-v14 (a rebuilt migrations +image). Next: M10.9, load and abuse qualification. ## Previous gate - 2026-10-02 diff --git a/docs/adr/0096-m10-probe-handoff.md b/docs/adr/0096-m10-probe-handoff.md index d0f7bd2..3fe95ea 100644 --- a/docs/adr/0096-m10-probe-handoff.md +++ b/docs/adr/0096-m10-probe-handoff.md @@ -220,3 +220,50 @@ controllable before M11. It passed 3 times in a row on Linux and also passes on Windows. - The test fixture `tests/fixtures/fake_handoff_world.hpp` (the v12 state machine) is shared by both handoff tests. + +## M10.8d two-process fault matrix (2026-10-06) +Real evo-server processes on PostgreSQL found two recovery gaps. Both are fixed within this +ADR's decisions, with no change to the protocol or the ticket format: +- **A source restarted after a prepare left the probe frozen for good.** The abort deadline + for a client that never arrives was kept only in the source process's memory. Schema v14 + adds `overdue_probe_transfers`: read-only, under the lease, it lists the caller's open + outgoing transfers prepared more than a given age ago by the database clock. Every + `overdue_poll` (5 s), sessions abort those older than the ticket lifetime plus 10 s, + whichever process prepared them. The in-memory deadline (lifetime plus 5 s) stays as the + fast path. Abort is serialized with the destination's stage and commit, so a transfer + committed meanwhile stays committed: clock skew can only delay an abort. +- **A ticket replayed at a restarted destination could take control back.** The + destination's used-ticket list is in memory. After its restart, a ticket for a transfer + committed earlier (still within its lifetime) was admitted durably even after the probe + had moved on: it moved the account's control away from the live owner and opened a + phantom probe. Ownership in the database stayed unique. `PostgresHandoffLedger::admit` + now admits a committed transfer only while the record is as that commit left it: owned + here, at the transfer's generation plus 1, and not frozen. Otherwise it refuses + (`unauthenticated`). An undecided retry right after the commit still converges. + +Evidence: `tests/region_handoff_process_tests.cpp` (in `evo-postgres-tests`, regions 7/161 +and 7/162 with adjacent areas, handoff keys and each other as neighbour), with these +scenarios: +1. A crossing and a crossing back (generation 1, 2, 3). A stale grant login at the old + region is redirected (`moved`). +2. Destination SIGKILL after stage (staged under its lease, then killed). The restarted + destination stages again under its new epoch and commits on the ticket. After the probe + crosses back, a replay of that ticket at a destination process that never saw it is + refused, and control stays with the owner. +3. Destination SIGKILL after commit (client admitted). A retry of the same ticket at the + restarted destination converges: same commit, and control moves again. A duplicate in + the same process is refused locally (`stale_grant`). +4. Source SIGKILL after prepare. One client's ticket still commits at the destination. The + other client never arrives, and its probe is refused as frozen (`capacity`) at the + restarted source until the recovery sweep aborts it. Then it logs in where it stopped. +5. Destination down. The ticket cannot be used, and the live source aborts after the + ticket's lifetime. The late ticket at the restarted destination is refused (`expired`). + +After every scenario, SQL checks that: +- each probe has one owning region (A or B); +- the inventory total is conserved, in records and in transfers; +- each record is frozen exactly while its single open transfer exists. + +Library tests add the v14 call (age bounds, source only, decided transfers excluded), the +refused replay in the ledger test, and the sweep over the fake world (never before the +store reports a transfer overdue). diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 5d9fcb0..5073864 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -4858,3 +4858,49 @@ Not done: - real two-process handoffs on PostgreSQL with kills (10.8d); - a GUI run across regions on evolution-dev (one dev region only; needs v4 and v13 deployed). + +## 2026-10-06 — M10.8d two-process handoff fault matrix (ADR-0096) + +Implemented: +- `tests/region_handoff_process_tests.cpp`: two evo-server processes (regions 7/161 and + 7/162, adjacent `--m10-area`s, handoff keys and each other as `--m10-neighbour`, fixed + loopback ports) with in-process session clients that follow offers and tickets. It covers: + - a crossing and a crossing back, plus a redirected stale login; + - destination SIGKILL after stage, then a re-stage and commit under the new epoch; + - destination SIGKILL after commit, then a converging ticket retry; + - a refused ticket replay after the probe moved on; + - a refused same-process duplicate; + - source SIGKILL after prepare: one client still arrives, the other never does and is + aborted by the restarted source's sweep; + - a destination that is down, with the live source aborting and the late ticket expired. + + After every scenario, SQL checks one owner per probe, the conserved inventory, and + frozen-iff-open-transfer. +- Schema v14 `overdue_probe_transfers` (apply chain, audit allowlist, upgrade and drift + checks), `ProbeLedger::overdue`, and the session layer's recovery sweep + (`overdue_poll`, ticket lifetime plus 10 s). +- The replay guard in `PostgresHandoffLedger::admit`: a committed transfer admits only + while the record is still owned here at generation plus 1 and not frozen. +- Library tests: + - the v14 store call (age bounds, source only, aborted transfers excluded); + - the refused retry in the ledger test; + - the sweep over the fake world in `two_region_handoff`. + +Findings: the two recovery gaps above were found by this matrix, before any deployment of +v4/v12/v13. + +Executed: +- **Linux GCC 14:** `postgres-check.sh` with QUIC, all suites including the new one (restart + waits 0.03-4 s; the replay ran at 8 s of the 30 s ticket lifetime). It passed 3 times in a + row, plus a final run after a tidy-only change. Presets `quic` 4/4 and `client-network` + 16/16. +- **Linux Clang 19:** clang-tidy 19 on `region_sessions.cpp`, `postgres_handoff_ledger.cpp`, + `region_lease_store.cpp`, the QUIC session, remote client, probe store, process, + migration and Postgres runner tests (one finding fixed); clang-format 19. +- **Windows MSVC 19.44:** presets `quic` 4/4 and `client-network` 14/14. The PostgreSQL + process tests are Linux-only. + +Not done: +- deployment of protocol v4 and schemas v12-v14 (needs a rebuilt migrations image; not done + from routine development); +- a GUI run across two regions on evolution-dev (only one dev region exists). diff --git a/docs/infrastructure/POSTGRESQL.md b/docs/infrastructure/POSTGRESQL.md index fe593dc..a8f5c0d 100644 --- a/docs/infrastructure/POSTGRESQL.md +++ b/docs/infrastructure/POSTGRESQL.md @@ -218,8 +218,11 @@ Lock order: upgrade lock (shared), own region lease row, probe record, transfer; `handoff_account_control` reads the transfer, then locks the account control row. The privilege audit covers all eight functions, and the tables are owner-only. Schema v13 adds `incoming_probe_transfers`, a read-only function under the lease that lists the open -transfers addressed to the caller's region (for ghosts). Deploying v12 and v13 needs a -rebuilt migrations (tools) image. +transfers addressed to the caller's region (for ghosts). Schema v14 adds +`overdue_probe_transfers`: read-only, under the lease, it lists the caller's open outgoing +transfers prepared more than the given age (1 s to 1 day) ago by the database clock. A +restarted source uses it to abort transfers whose client never arrived (the dead process's +deadlines were in memory). Deploying v12-v14 needs a rebuilt migrations (tools) image. ## Host lease discipline diff --git a/include/evo/persistence/probe_store.hpp b/include/evo/persistence/probe_store.hpp index 2036044..d3b0de7 100644 --- a/include/evo/persistence/probe_store.hpp +++ b/include/evo/persistence/probe_store.hpp @@ -87,4 +87,9 @@ struct IncomingTransfer { [[nodiscard]] std::expected, ProbeStoreFailure> incoming_probe_transfers(Postgres& db, RegionKey key, const LeaseGrant& lease, std::chrono::milliseconds timeout); +// Schema v14: the caller's open outgoing transfers prepared more than `age` (1 s .. 1 day) +// ago by the database clock, for recovery aborts after a restart (at most 256, oldest first). +[[nodiscard]] std::expected, ProbeStoreFailure> +overdue_probe_transfers(Postgres& db, RegionKey key, const LeaseGrant& lease, + std::chrono::seconds age, std::chrono::milliseconds timeout); } // namespace evo::persistence diff --git a/include/evo/server/postgres_handoff_ledger.hpp b/include/evo/server/postgres_handoff_ledger.hpp index 318492e..439dddd 100644 --- a/include/evo/server/postgres_handoff_ledger.hpp +++ b/include/evo/server/postgres_handoff_ledger.hpp @@ -33,7 +33,7 @@ class PostgresHandoffLedger final : public network::HandoffLedger { std::chrono::milliseconds duration_, timeout_; }; -// M10.8c the region's durable probe records over PostgreSQL (ADR-0096, schema v12-13), on the +// M10.8c the region's durable probe records over PostgreSQL (ADR-0096, schema v12-14), on the // host's lease and its own connection slot. Undecided or failed calls report `unavailable` // (the session layer retries or refuses); a fenced lease reports `fenced`. class PostgresProbeLedger final : public ProbeLedger { @@ -55,6 +55,8 @@ class PostgresProbeLedger final : public ProbeLedger { [[nodiscard]] std::expected abort(const network::TransferId& transfer) override; [[nodiscard]] std::expected, ProbeLedgerError> incoming() override; + [[nodiscard]] std::expected, ProbeLedgerError> + overdue(std::chrono::seconds age) override; private: PgSlot* db_; diff --git a/include/evo/server/probe_ledger.hpp b/include/evo/server/probe_ledger.hpp index a9e23dd..5af75c7 100644 --- a/include/evo/server/probe_ledger.hpp +++ b/include/evo/server/probe_ledger.hpp @@ -1,6 +1,7 @@ #pragma once #include "evo/network/handoff_ticket.hpp" #include "evo/network/session_control.hpp" +#include #include #include #include @@ -43,6 +44,10 @@ class ProbeLedger { // Open transfers addressed to this region (shown as ghosts). [[nodiscard]] virtual std::expected, ProbeLedgerError> incoming() = 0; + // M10.8d: this region's open outgoing transfers prepared more than `age` ago by the + // store's clock, whoever prepared them (an earlier process too): recovery aborts them. + [[nodiscard]] virtual std::expected, ProbeLedgerError> + overdue(std::chrono::seconds age) = 0; protected: ProbeLedger(ProbeLedger&&) = default; diff --git a/include/evo/server/region_sessions.hpp b/include/evo/server/region_sessions.hpp index 43b21f1..8e48ef8 100644 --- a/include/evo/server/region_sessions.hpp +++ b/include/evo/server/region_sessions.hpp @@ -88,6 +88,9 @@ struct RegionSessionsConfig { std::vector neighbours; std::uint64_t ticket_lifetime{30}; // Seconds a handoff ticket is valid. std::chrono::milliseconds ghost_poll{1000}; // Incoming transfers read this often. + // M10.8d recovery: this often, open outgoing transfers prepared more than ticket_lifetime + // + 10 s ago (by the store's clock, e.g. by a process that died) are aborted. + std::chrono::milliseconds overdue_poll{5000}; }; // The region host's authority, read once per poll. struct HostFacts { diff --git a/migrations/0014_probe_overdue.sql b/migrations/0014_probe_overdue.sql new file mode 100644 index 0000000..52e2e3c --- /dev/null +++ b/migrations/0014_probe_overdue.sql @@ -0,0 +1,35 @@ +-- M10.8d overdue probe transfers (ADR-0096). A source region aborts the transfers whose +-- client never arrived once their ticket has expired; it kept that deadline only in memory, +-- so a source restarted after a prepare (SIGKILL, lease loss) left the probe frozen for good. +-- This lists the caller's open outgoing transfers prepared more than `p_age_seconds` ago by +-- the database clock, so any later process holding the region's lease can abort them (abort +-- is serialized with stage and commit: a transfer committed meanwhile stays committed). +-- Read-only, under the caller's exact unexpired region lease. At most 256 rows, oldest first. +CREATE FUNCTION evolution.overdue_probe_transfers(p_world evolution.u64, p_region evolution.u64, + p_holder uuid, p_epoch evolution.u64, p_age_seconds integer) +RETURNS TABLE (transfer_id uuid) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + now_t timestamptz; +BEGIN + IF p_age_seconds IS NULL OR p_age_seconds NOT BETWEEN 1 AND 86400 THEN + RAISE EXCEPTION 'invalid probe request' USING ERRCODE = '22023'; + END IF; + now_t := evolution.check_region_lease(p_world, p_region, p_holder, p_epoch); + RETURN QUERY + SELECT t.transfer_id + FROM evolution.probe_transfers t + WHERE t.world_id = p_world AND t.source_region = p_region + AND t.state IN ('prepared', 'staged') + AND t.prepared_at < now_t - p_age_seconds * interval '1 second' + ORDER BY t.prepared_at, t.transfer_id + LIMIT 256; +END $$; + +REVOKE ALL ON FUNCTION + evolution.overdue_probe_transfers(evolution.u64, evolution.u64, uuid, evolution.u64, integer) + FROM PUBLIC; +GRANT EXECUTE ON FUNCTION + evolution.overdue_probe_transfers(evolution.u64, evolution.u64, uuid, evolution.u64, integer) + TO :"runtime_role"; +INSERT INTO evolution.schema_migrations(version, sha256) VALUES (14, :'migration_sha_14'); diff --git a/migrations/apply.sql b/migrations/apply.sql index 24eb572..25f832c 100644 --- a/migrations/apply.sql +++ b/migrations/apply.sql @@ -65,28 +65,43 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') || ',7:' || :'migration_sha_7' || ',8:' || :'migration_sha_8' || ',9:' || :'migration_sha_9' || ',10:' || :'migration_sha_10' || ',11:' || :'migration_sha_11' || ',12:' || :'migration_sha_12' - || ',13:' || :'migration_sha_13' AS at_v13 \gset - \if :at_v13 + || ',13:' || :'migration_sha_13' AS at_v13, + :'history' = '1:' || :'migration_sha' || ',2:' || :'migration_sha_2' + || ',3:' || :'migration_sha_3' || ',4:' || :'migration_sha_4' + || ',5:' || :'migration_sha_5' || ',6:' || :'migration_sha_6' + || ',7:' || :'migration_sha_7' || ',8:' || :'migration_sha_8' + || ',9:' || :'migration_sha_9' || ',10:' || :'migration_sha_10' + || ',11:' || :'migration_sha_11' || ',12:' || :'migration_sha_12' + || ',13:' || :'migration_sha_13' || ',14:' || :'migration_sha_14' + AS at_v14 \gset + \if :at_v14 + \elif :at_v13 + \ir 0014_probe_overdue.sql \elif :at_v12 \ir 0013_probe_incoming.sql + \ir 0014_probe_overdue.sql \elif :at_v11 \ir 0012_probe_handoff.sql \ir 0013_probe_incoming.sql + \ir 0014_probe_overdue.sql \elif :at_v10 \ir 0011_region_actions.sql \ir 0012_probe_handoff.sql \ir 0013_probe_incoming.sql + \ir 0014_probe_overdue.sql \elif :at_v9 \ir 0010_account_control.sql \ir 0011_region_actions.sql \ir 0012_probe_handoff.sql \ir 0013_probe_incoming.sql + \ir 0014_probe_overdue.sql \elif :at_v8 \ir 0009_serialized_authority.sql \ir 0010_account_control.sql \ir 0011_region_actions.sql \ir 0012_probe_handoff.sql \ir 0013_probe_incoming.sql + \ir 0014_probe_overdue.sql \elif :at_v7 \ir 0008_upgrade_barrier.sql \ir 0009_serialized_authority.sql @@ -94,6 +109,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0011_region_actions.sql \ir 0012_probe_handoff.sql \ir 0013_probe_incoming.sql + \ir 0014_probe_overdue.sql \elif :at_v6 \ir 0007_region_clones.sql \ir 0008_upgrade_barrier.sql @@ -102,6 +118,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0011_region_actions.sql \ir 0012_probe_handoff.sql \ir 0013_probe_incoming.sql + \ir 0014_probe_overdue.sql \elif :at_v5 \ir 0006_commit_lookup.sql \ir 0007_region_clones.sql @@ -111,6 +128,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0011_region_actions.sql \ir 0012_probe_handoff.sql \ir 0013_probe_incoming.sql + \ir 0014_probe_overdue.sql \elif :at_v4 \ir pre_v5_guard.sql \ir 0005_batch_decisions.sql @@ -122,6 +140,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0011_region_actions.sql \ir 0012_probe_handoff.sql \ir 0013_probe_incoming.sql + \ir 0014_probe_overdue.sql \elif :at_v3 \ir 0004_lineage_profile.sql \ir pre_v5_guard.sql @@ -134,6 +153,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0011_region_actions.sql \ir 0012_probe_handoff.sql \ir 0013_probe_incoming.sql + \ir 0014_probe_overdue.sql \elif :at_v2 \ir 0003_region_checkpoints.sql \ir 0004_lineage_profile.sql @@ -147,6 +167,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0011_region_actions.sql \ir 0012_probe_handoff.sql \ir 0013_probe_incoming.sql + \ir 0014_probe_overdue.sql \elif :at_v1 \ir 0002_region_fencing.sql \ir 0003_region_checkpoints.sql @@ -161,6 +182,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0011_region_actions.sql \ir 0012_probe_handoff.sql \ir 0013_probe_incoming.sql + \ir 0014_probe_overdue.sql \else \echo 'Unknown or changed migration history' DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$; @@ -180,6 +202,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0011_region_actions.sql \ir 0012_probe_handoff.sql \ir 0013_probe_incoming.sql + \ir 0014_probe_overdue.sql \endif -- Effective-privilege audit on every run, including no-ops (ADR-0060). Attributes -- alone are insufficient: predefined roles such as pg_write_all_data, PUBLIC @@ -226,7 +249,8 @@ BEGIN ('evolution.abort_probe_transfer(evolution.u64,evolution.u64,uuid,evolution.u64,uuid)'::regprocedure::oid), ('evolution.find_probe_transfer(evolution.u64,evolution.u64,uuid,evolution.u64,uuid)'::regprocedure::oid), ('evolution.handoff_account_control(evolution.u64,evolution.u64,uuid,evolution.u64,uuid,uuid,evolution.u64,integer)'::regprocedure::oid), - ('evolution.incoming_probe_transfers(evolution.u64,evolution.u64,uuid,evolution.u64)'::regprocedure::oid) + ('evolution.incoming_probe_transfers(evolution.u64,evolution.u64,uuid,evolution.u64)'::regprocedure::oid), + ('evolution.overdue_probe_transfers(evolution.u64,evolution.u64,uuid,evolution.u64,integer)'::regprocedure::oid) ), violations(role_oid, reason) AS ( SELECT r.oid, 'privileged attribute or owner identity' FROM pg_catalog.pg_roles r JOIN closure c ON c.oid = r.oid diff --git a/schemas/probe-handoff-v1.md b/schemas/probe-handoff-v1.md index 5f6def6..8704694 100644 --- a/schemas/probe-handoff-v1.md +++ b/schemas/probe-handoff-v1.md @@ -41,6 +41,8 @@ Each call is made under the caller's exact unexpired region lease. | abort_probe_transfer | source or destination | prepared or staged becomes aborted; the record is active at the source | | find_probe_transfer | any region | The transfer as recorded | | handoff_account_control | destination | Account control (v10) for a transfer committed to the caller | +| incoming_probe_transfers (v13) | any region | Open transfers addressed to the caller (ghosts), at most 256 | +| overdue_probe_transfers (v14) | source | Caller's open outgoing transfers prepared more than the given age (1 s to 1 day) ago by the database clock, at most 256 (recovery aborts) | A retry with the same transfer id returns the recorded outcome. SQLSTATE: EVL13 fenced, EVL14 unknown transfer, EVL15 not staged by this epoch, EVL12 control handoff refused. diff --git a/src/persistence/region_lease_store.cpp b/src/persistence/region_lease_store.cpp index bda5d68..3abdcda 100644 --- a/src/persistence/region_lease_store.cpp +++ b/src/persistence/region_lease_store.cpp @@ -748,4 +748,27 @@ incoming_probe_transfers(Postgres& db, RegionKey key, const LeaseGrant& lease, } return out; } + +std::expected, ProbeStoreFailure> +overdue_probe_transfers(Postgres& db, RegionKey key, const LeaseGrant& lease, + std::chrono::seconds age, milliseconds timeout) { + if (age.count() < 1 || age.count() > 86400) + return std::unexpected{ProbeStoreFailure{ProbeStoreError::invalid, {}}}; + const std::array p{std::to_string(key.world), std::to_string(key.region), + uuid_text(lease.holder), std::to_string(lease.epoch), + std::to_string(age.count())}; + auto rows = db.query("SELECT * FROM evolution.overdue_probe_transfers($1::evolution.u64, " + "$2::evolution.u64, $3::uuid, $4::evolution.u64, $5::integer)", + p, timeout); + if (!rows) + return std::unexpected{probe_failure(rows.error())}; + std::vector out; + for (const auto& r : *rows) { + const auto transfer = r.size() == 1 ? parse_uuid(r[0]) : std::nullopt; + if (!transfer) + return std::unexpected{ProbeStoreFailure{ProbeStoreError::unknown, {}}}; + out.push_back(*transfer); + } + return out; +} } // namespace evo::persistence diff --git a/src/server/session/postgres_handoff_ledger.cpp b/src/server/session/postgres_handoff_ledger.cpp index 644376b..9421bdb 100644 --- a/src/server/session/postgres_handoff_ledger.cpp +++ b/src/server/session/postgres_handoff_ledger.cpp @@ -53,6 +53,17 @@ PostgresHandoffLedger::admit(const network::TicketClaims& ticket, std::uint64_t return fail(committed.error()); if (committed->state != persistence::TransferState::committed) return std::unexpected(HandoffLedgerError::refused); // Aborted meanwhile. + // M10.8d: a retried ticket (e.g. after this region restarted and forgot it was used) is + // honoured only while the probe is still as that commit left it: owned here at the next + // generation and not frozen. Once it moved on, the ticket must not take control back. + const auto record = + persistence::open_probe(*db, key_, *lease, ticket.account.bytes, committed->x, committed->y, + committed->z, PostgresProbeLedger::inventory, timeout_); + if (!record) + return fail(record.error()); + if (record->region != key_.region || record->generation != committed->generation + 1 || + record->transfer) + return std::unexpected(HandoffLedgerError::refused); const auto control = persistence::handoff_account_control( *db, key_, *lease, ticket.account.bytes, ticket.transfer, session, duration_, timeout_); if (!control) { @@ -169,4 +180,21 @@ std::expected, ProbeLedgerError> PostgresProbeLedger: out.push_back({t.transfer, t.x, t.y}); return out; } + +std::expected, ProbeLedgerError> +PostgresProbeLedger::overdue(std::chrono::seconds age) { + const auto lease = lease_(); + if (!lease) + return std::unexpected(ProbeLedgerError::fenced); + auto* db = db_->get(); + if (!db) + return std::unexpected(ProbeLedgerError::unavailable); + auto r = persistence::overdue_probe_transfers(*db, key_, *lease, age, timeout_); + if (!r) { + if (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown) + db_->broken(); + return std::unexpected(probe_error(r.error())); + } + return std::vector(r->begin(), r->end()); +} } // namespace evo::server diff --git a/src/server/session/region_sessions.cpp b/src/server/session/region_sessions.cpp index e469ff1..7cc805e 100644 --- a/src/server/session/region_sessions.cpp +++ b/src/server/session/region_sessions.cpp @@ -106,7 +106,7 @@ struct RegionSessions::Impl { std::optional own_area; // This region's part of the marker area. std::int64_t spawn_x{}, spawn_y{}; // Where new probes start. std::vector outgoing; // Handed off, not yet known committed. - Clock::time_point next_ghosts{}; + Clock::time_point next_ghosts{}, next_overdue{}; std::optional default_interest; HostFacts facts; std::vector connections; @@ -471,6 +471,20 @@ struct RegionSessions::Impl { return config.probes->abort(o.transfer).has_value(); }); } + // Recovery (M10.8d): the in-memory deadlines die with the process, so outgoing transfers + // still open well after any ticket for them expired are aborted from the store's list. + // Abort is serialized with the destination's commit: a committed one stays committed. + void abort_overdue(Clock::time_point now) { + if (now < next_overdue) + return; + next_overdue = now + config.overdue_poll; + const auto overdue = + config.probes->overdue(std::chrono::seconds{config.ticket_lifetime + 10}); + if (!overdue) + return; // Retried next poll. + for (const auto& t : *overdue) + (void)config.probes->abort(t).has_value(); + } void refresh_ghosts(Clock::time_point now) { if (now < next_ghosts) return; @@ -549,7 +563,8 @@ RegionSessions::create(RegionSessionsConfig config) { config.ticket_lifetime == 0 || config.ticket_lifetime > network::max_ticket_lifetime || config.transport.limits.max_connections < config.control.max_sessions || config.hello_deadline.count() <= 0 || config.close_flush.count() <= 0 || - config.renew_interval.count() <= 0 || config.safety_margin.count() <= 0) + config.renew_interval.count() <= 0 || config.safety_margin.count() <= 0 || + config.overdue_poll.count() <= 0) return std::unexpected(RegionSessionsError::configuration); auto control = network::SessionControl::create(config.control); if (!control) @@ -609,6 +624,7 @@ void RegionSessions::poll(const HostFacts& facts, std::uint64_t now_seconds, m.crossings(now_seconds, now); if (m.config.probes) { m.expire_outgoing(now_seconds); + m.abort_overdue(now); m.refresh_ghosts(now); } } diff --git a/tests/fixtures/fake_handoff_world.hpp b/tests/fixtures/fake_handoff_world.hpp index 1889a55..52db609 100644 --- a/tests/fixtures/fake_handoff_world.hpp +++ b/tests/fixtures/fake_handoff_world.hpp @@ -37,6 +37,8 @@ struct WorldDb { }; std::map probes; std::map transfers; + bool all_overdue{}; // overdue() reports every open transfer (a restarted source's view). + std::uint64_t overdue_reads{}; }; struct RegionDb final : server::ProbeLedger, HandoffLedger, ControlLedger { RegionDb(WorldDb& w, std::uint64_t r) : world{&w}, region{r} {} @@ -85,6 +87,15 @@ struct RegionDb final : server::ProbeLedger, HandoffLedger, ControlLedger { out.push_back({id, t.x, t.y}); return out; } + std::expected, server::ProbeLedgerError> + overdue(std::chrono::seconds) override { + ++world->overdue_reads; + std::vector out; + for (const auto& [id, t] : world->transfers) + if (world->all_overdue && t.source == region && t.state == WorldDb::State::prepared) + out.push_back(id); + return out; + } std::expected admit(const TicketClaims& ticket, std::uint64_t) override { const auto found = world->transfers.find(ticket.transfer); diff --git a/tests/postgres_tests.cpp b/tests/postgres_tests.cpp index 4423cdc..3b6e4d2 100644 --- a/tests/postgres_tests.cpp +++ b/tests/postgres_tests.cpp @@ -15,6 +15,7 @@ void run_region_session_tests(); #endif #ifdef EVO_POSTGRES_M10 void run_region_m10_session_tests(); +void run_region_handoff_process_tests(); #endif void run_region_recovery_tests(); void run_region_ops_tests(); @@ -138,6 +139,7 @@ int main() { #endif #ifdef EVO_POSTGRES_M10 run_region_m10_session_tests(); + run_region_handoff_process_tests(); #endif run_region_recovery_tests(); std::puts("region recovery integration passed"); diff --git a/tests/probe_handoff_store_tests.cpp b/tests/probe_handoff_store_tests.cpp index 4ebcbd0..f4e55f6 100644 --- a/tests/probe_handoff_store_tests.cpp +++ b/tests/probe_handoff_store_tests.cpp @@ -11,6 +11,7 @@ #ifdef EVO_POSTGRES_SESSIONS #include "evo/server/postgres_handoff_ledger.hpp" #endif +#include #include #include #include @@ -425,6 +426,9 @@ void handoff_ledger(World& w) { const auto t2 = id(0x76, 2); CHECK(state_of(prepare_probe_transfer(db, w.b, w.lb, frank, 2, t2, 1, 3, 4, -7, 5s)) == TransferState::prepared); + // M10.8d: once the probe moved on (here frozen for the next transfer), a retry of the + // committed transfer's ticket no longer admits (nor takes control). + CHECK(ledger.admit(ticket, 47).error() == evo::network::HandoffLedgerError::refused); CHECK(state_of(abort_probe_transfer(db, w.b, w.lb, t2, 5s)) == TransferState::aborted); auto aborted = ticket; aborted.transfer = t2; @@ -437,6 +441,29 @@ void handoff_ledger(World& w) { invariants(*w.admin, w.probes); } #endif +// Schema v14: a source's open outgoing transfers older than the given age, by the database +// clock, for recovery aborts; never another region's, never decided ones. +void overdue(World& w) { + auto& db = *w.db; + const auto gina = id(0x49); + w.open(w.a, w.la, gina); + ++w.probes; + const auto t = id(0x77, 1); + CHECK(state_of(prepare_probe_transfer(db, w.a, w.la, gina, 1, t, 2, 5, 6, -7, 5s)) == + TransferState::prepared); + const auto has = [&](RegionKey key, const LeaseGrant& lease, std::chrono::seconds age) { + const auto found = overdue_probe_transfers(db, key, lease, age, 5s); + CHECK(found.has_value()); + return std::ranges::find(*found, t) != found->end(); + }; + CHECK(!has(w.a, w.la, 60s)); + std::this_thread::sleep_for(1100ms); + CHECK(has(w.a, w.la, 1s) && !has(w.b, w.lb, 1s)); + CHECK(failed(overdue_probe_transfers(db, w.a, w.la, 0s, 5s), ProbeStoreError::invalid)); + CHECK(state_of(abort_probe_transfer(db, w.a, w.la, t, 5s)) == TransferState::aborted); + CHECK(!has(w.a, w.la, 1s)); + invariants(*w.admin, w.probes); +} } // namespace void run_probe_handoff_store_tests() { @@ -445,6 +472,7 @@ void run_probe_handoff_store_tests() { restarts(w); aborts_and_races(w); control_handoff(w); + overdue(w); #ifdef EVO_POSTGRES_SESSIONS handoff_ledger(w); #endif diff --git a/tests/quic_sessions.cpp b/tests/quic_sessions.cpp index e9de284..4b6906d 100644 --- a/tests/quic_sessions.cpp +++ b/tests/quic_sessions.cpp @@ -667,6 +667,7 @@ struct Region { area_of(neighbour_x0)}); c.ticket_lifetime = 20; c.ghost_poll = std::chrono::milliseconds{20}; + c.overdue_poll = std::chrono::milliseconds{20}; auto made = RegionSessions::create(std::move(c)); CHECK(made.has_value()); sessions = std::move(*made); @@ -846,6 +847,21 @@ void two_region_handoff(const Files& f) { CHECK(pump([&] { return late->state() == RegionSessionState::closed; })); CHECK(late->close_reason() == SessionCloseReason::expired); CHECK(b->state() == RegionSessionState::established && r1.db.releases > 0); + // M10.8d recovery: a transfer prepared by an earlier process (unknown to this one's + // in-memory deadlines) is aborted once the store reports it overdue; never before. + const auto dave = account(24); + TransferId orphan{}; + orphan.fill(0x6D); + world.probes[dave] = {1, 1, 9 * m_um, 5 * m_um, orphan}; + world.transfers[orphan] = {dave, 1, 2, 1, 9 * m_um, 5 * m_um, WorldDb::State::prepared}; + const auto reads = world.overdue_reads; + CHECK(pump([&] { return world.overdue_reads > reads + 2; })); + CHECK(world.transfers.at(orphan).state == WorldDb::State::prepared); + world.all_overdue = true; + CHECK(pump([&] { return !world.probes.at(dave).transfer.has_value(); })); + CHECK(world.transfers.at(orphan).state == WorldDb::State::aborted && + world.probes.at(dave).owner == 1); + world.all_overdue = false; } void durable_control(const Files& f) { diff --git a/tests/region_handoff_process_tests.cpp b/tests/region_handoff_process_tests.cpp new file mode 100644 index 0000000..849e98c --- /dev/null +++ b/tests/region_handoff_process_tests.cpp @@ -0,0 +1,519 @@ +// Real process tests for M10.8d (ADR-0096): two evo-server processes on PostgreSQL with +// adjacent marker areas, handoff keys and each other as neighbour; in-process +// RegionSessionClients follow offers and tickets. Regions 7/161 (A, cells 0..1) and 7/162 +// (B, cells 2..3). Covers a crossing and a crossing back, SIGKILL of the source after +// prepare and of the destination after stage and after commit, duplicate and stale ticket +// retries, a client that never arrives (aborted by the restarted source's recovery sweep) +// and a destination that is down (aborted by the live source). After every scenario: one +// owning region per probe, the test inventory conserved, and a record frozen exactly while +// its one open transfer exists. Run by tools/ci/postgres-check.sh when the build has +// EVO_BUILD_QUIC. +#include "evo/client/region_session.hpp" +#include "evo/network/dev_credentials.hpp" +#include "evo/network/handoff_ticket.hpp" +#include "evo/network/session_grant.hpp" +#include "fixtures/server_process.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +void run_region_handoff_process_tests(); + +namespace { +using namespace evo::test::process; +using namespace evo; +using namespace evo::network; +using client::RegionSessionClient; +using client::RegionSessionState; +using evo::test::region_identity; +void check(bool value, std::source_location at = std::source_location::current()) { + if (!value) + throw std::runtime_error("region handoff process test line " + std::to_string(at.line())); +} +#define CHECK(...) check(static_cast(__VA_ARGS__)) +template +const T& must(const std::optional& o, + std::source_location at = std::source_location::current()) { + if (!o) + throw std::runtime_error("region handoff process missing value line " + + std::to_string(at.line())); + return *o; +} +constexpr const char* name = "dev-region.evolution.test"; +constexpr std::uint64_t region_a = 161, region_b = 162; + +void write_file(const fs::path& path, std::span bytes) { + const int fd = ::open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0600); + expect(fd >= 0 && + ::write(fd, bytes.data(), bytes.size()) == static_cast(bytes.size()) && + ::close(fd) == 0, + "write file"); +} +void write_text(const fs::path& path, const std::string& text) { + write_file(path, {reinterpret_cast(text.data()), text.size()}); +} +AccountId account(std::uint8_t tag) { + AccountId a; + a.bytes.fill(tag); + return a; +} +template std::string hex(const std::array& bytes) { + constexpr char digits[] = "0123456789abcdef"; + std::string out; + for (const auto b : bytes) { + out.push_back(digits[b >> 4U]); + out.push_back(digits[b & 0xFU]); + } + return out; +} +std::uint64_t wall_seconds() { + return static_cast(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count()); +} +// A UDP port that was free a moment ago (each server must know its neighbour's port). +std::uint16_t free_port() { + const int fd = ::socket(AF_INET, SOCK_DGRAM | SOCK_CLOEXEC, 0); + expect(fd >= 0, "socket"); + sockaddr_in a{}; + a.sin_family = AF_INET; + a.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + socklen_t size = sizeof a; + const bool ok = ::bind(fd, reinterpret_cast(&a), sizeof a) == 0 && + ::getsockname(fd, reinterpret_cast(&a), &size) == 0; + ::close(fd); + expect(ok, "free port"); + return ntohs(a.sin_port); +} +// A 32-byte scalar that imports as a key, written with its public key. +template +void make_key(const fs::path& scalar_file, const fs::path& public_file, + const std::function(std::span)>& import) { + for (int attempt = 0; attempt < 8; ++attempt) { + std::array scalar{}; + expect(random_bytes(scalar).has_value(), "random"); + if (auto key = import(scalar)) { + write_file(scalar_file, scalar); + write_file(public_file, *key->public_key()); + return; + } + } + expect(false, "key"); +} + +// A genesis commit byte for an all-equal commit id (ids are unique across the suite): one +// no other test names (later tests provision theirs by literal) and none earlier produced. +std::uint8_t unused_genesis() { + for (const unsigned b : {0x4BU, 0x4CU, 0x4DU, 0x8BU, 0x8CU, 0x8DU, 0x8EU}) { + std::string hex_id; + for (int i = 0; i < 16; ++i) { + hex_id.push_back("0123456789abcdef"[b >> 4U]); + hex_id.push_back("0123456789abcdef"[b & 0xFU]); + } + const std::array p{hex_id}; + const auto r = owner()->query( + "SELECT count(*) FROM evolution.region_commits WHERE commit_id = $1::uuid", p, 5s); + expect(r.has_value() && r->size() == 1, "genesis lookup"); + if ((*r)[0][0] == "0") + return static_cast(b); + } + expect(false, "genesis byte"); + return 0; +} + +// Both regions: one CA and server certificate, one grant issuer, a handoff key each. +struct Pair { + ServerRegion a{provision(region_a, unused_genesis())}, b{provision(region_b, unused_genesis())}; + fs::path files{a.root / "pair"}; + std::string ca_pem, window; + std::optional issuer; + std::uint16_t port_a{free_port()}, port_b{free_port()}; + Pair() { + fs::create_directory(files); + const auto made = generate_dev_credentials(DeploymentEnvironment::dev, name, 1, + std::chrono::system_clock::now()); + expect(made.has_value(), "credentials"); + ca_pem = made->authority_pem; + write_text(files / "ca.pem", made->authority_pem); + write_text(files / "server.pem", made->server_certificate_pem); + write_text(files / "server.key", made->server_key_pem); + make_key( + files / "issuer.key", files / "issuer.pub", [](std::span s) { + auto k = GrantSigningKey::import(DeploymentEnvironment::dev, 1, s); + return k ? std::optional{std::move(*k)} : std::nullopt; + }); + std::array scalar{}; + const int fd = ::open((files / "issuer.key").c_str(), O_RDONLY | O_CLOEXEC); + expect(fd >= 0 && + ::read(fd, scalar.data(), scalar.size()) == + static_cast(scalar.size()) && + ::close(fd) == 0, + "issuer key"); + auto key = GrantSigningKey::import(DeploymentEnvironment::dev, 1, scalar); + expect(key.has_value(), "issuer import"); + issuer.emplace(std::move(*key)); + for (const auto r : {region_a, region_b}) + make_key( + files / ("handoff-" + std::to_string(r) + ".key"), + files / ("handoff-" + std::to_string(r) + ".pub"), + [r](std::span s) { + auto k = HandoffSigningKey::import(DeploymentEnvironment::dev, + static_cast(r), s); + return k ? std::optional{std::move(*k)} : std::nullopt; + }); + const auto now = wall_seconds(); + window = "test-" + std::to_string(now) + ":" + std::to_string(now + 1200); + } + [[nodiscard]] std::vector args(bool at_a) const { + const auto& own = at_a ? a : b; + const auto other = at_a ? region_b : region_a; + const std::string other_cells = at_a ? "2,0,3,1" : "0,0,1,1"; + return server_args( + own, {"--synthetic-inputs", + "development-v1", + "--m10-listen", + "127.0.0.1:" + std::to_string(at_a ? port_a : port_b), + "--m10-cert", + (files / "server.pem").string(), + "--m10-key", + (files / "server.key").string(), + "--m10-issuer", + (files / "issuer.pub").string(), + "--m10-environment", + "dev", + "--session-window", + window, + "--m10-area", + at_a ? "0,0,1,1" : "2,0,3,1", + "--m10-handoff-key", + (files / ("handoff-" + std::to_string(own.key.region) + ".key")).string(), + "--m10-neighbour", + std::to_string(other) + "=" + other_cells + + "=127.0.0.1:" + std::to_string(at_a ? port_b : port_a) + "=" + name + "=" + + (files / ("handoff-" + std::to_string(other) + ".pub")).string()}); + } + [[nodiscard]] WireGrant grant(const AccountId& who, std::uint64_t serial, + std::uint64_t region) const { + GrantClaims c; + c.grant_id = (region * 100'000) + serial; + c.account = who; + c.serial = serial; + c.world = {7, region_identity.frame}; + c.region = region; + c.issued_at = wall_seconds(); + c.expires_at = c.issued_at + 120; + const auto g = must(issuer).sign(c); + expect(g.has_value(), "grant"); + return *g; + } +}; + +// A running evo-server, its output drained while waiting. +struct Server { + std::unique_ptr child; + explicit Server(const Pair& p, bool at_a) : child{std::make_unique(p.args(at_a))} { + const bool started = + child->wait_line("m10 listening ", 30s) && child->wait_line("ready 1", 30s) && + child->wait_line("m10 motion area ", 30s) && child->printed("m10 handoff neighbours=1"); + if (!started) + for (const auto& line : child->lines()) + if (starts(line, "m10") || starts(line, "error") || starts(line, "usage")) + std::printf("server: %s\n", line.c_str()); + CHECK(started); + } + void kill() { + child->signal(SIGKILL); + CHECK(child->wait_exit(30s) == 128 + SIGKILL); + } +}; + +struct Bench { + Pair pair; + std::unique_ptr a, b; + std::vector accounts; // Every probe this test created. + std::array serials{}; + + void pump() { + for (auto* s : {a.get(), b.get()}) + if (s) + s->child->pump(1ms); + } + // Polls clients and live servers until `done`, at most `limit`. + bool until(const std::vector& clients, const std::function& done, + std::chrono::milliseconds limit = 20s) { + const auto end = Clock::now() + limit; + while (Clock::now() < end) { + for (auto* c : clients) + c->poll(Clock::now()); + pump(); + if (done()) + return true; + } + return done(); + } + std::unique_ptr dial(std::uint16_t port, const WireGrant& grant) { + client::RegionSessionClientConfig c; + c.transport.port = port; + c.transport.server_name = name; + c.transport.authority_pem = pair.ca_pem; + c.transport.authority_pem_file = (pair.files / "ca.pem").string(); + c.grant = grant; + auto made = RegionSessionClient::connect(std::move(c), Clock::now()); + CHECK(made.has_value()); + return std::move(*made); + } + // A grant login (next serial for the account) at a region; waits for the outcome. + std::unique_ptr login(const AccountId& who, std::uint64_t region) { + if (std::ranges::find(accounts, who) == accounts.end()) + accounts.push_back(who); + auto c = dial(region == region_a ? pair.port_a : pair.port_b, + pair.grant(who, ++serials.at(who.bytes[0]), region)); + settle(*c); + return c; + } + // A ticket login at the offer's destination; waits for the outcome. + std::unique_ptr follow(const HandoffOffer& offer) { + const auto& to = offer.destination; + CHECK(to.server_name == name); + const auto port = to.region == region_a ? pair.port_a : pair.port_b; + CHECK(to.endpoint == "127.0.0.1:" + std::to_string(port)); + auto c = dial(port, offer.ticket); + settle(*c); + return c; + } + void settle(RegionSessionClient& c) { + CHECK(until({&c}, [&] { + return c.state() == RegionSessionState::established || + c.state() == RegionSessionState::closed; + })); + } + // Walks the client's probe east (+1) or west (-1) until it is handed off; the offer. + HandoffOffer cross(RegionSessionClient& c, int direction) { + CHECK(c.state() == RegionSessionState::established); + bool moving = false; + std::optional offer; + CHECK(until({&c}, [&] { + for (const auto& r : c.take_intention_receipts()) + moving = moving || r.status == AdmitStatus::admitted; + if (!moving && c.state() == RegionSessionState::established) + static_cast(c.send_intention(c.server_tick() + 4, + static_cast(direction * 1000), 0)); + if (!offer) + offer = c.take_handoff(); + return offer.has_value() && c.state() == RegionSessionState::closed; + })); + CHECK(c.close_reason() == SessionCloseReason::handed_off); + return must(offer); + } + + // SQL views of world 7. + static persistence::PgRows rows(const std::string& sql, + const std::vector& params) { + auto r = owner()->query(sql, params, 5s); + CHECK(r.has_value()); + return *r; + } + static std::string one(const std::string& sql, const std::vector& params) { + const auto r = rows(sql, params); + CHECK(r.size() == 1 && r[0].size() == 1); + return r[0][0].value_or(""); + } + // "owner:generation:frozen" of the account's probe. + static std::string probe(const AccountId& who) { + return one("SELECT region_id || ':' || generation || ':' || (transfer_id IS NOT NULL) " + "FROM evolution.probe_records WHERE world_id = 7 AND account_id = $1::uuid", + {hex(who.bytes)}); + } + static std::string transfer(const TransferId& t) { + return one("SELECT state FROM evolution.probe_transfers WHERE world_id = 7 AND " + "transfer_id = $1::uuid", + {hex(t)}); + } + static std::uint64_t control_generation(const AccountId& who) { + return std::stoull(one("SELECT generation FROM evolution.account_controls WHERE " + "world_id = 7 AND account_id = $1::uuid", + {hex(who.bytes)})); + } + // After every scenario: a single owning region (A or B) per probe, the inventory + // conserved, at most one open transfer per probe and the record frozen exactly under it. + void invariants() const { + std::string list; + for (const auto& who : accounts) + list += (list.empty() ? "'" : ",'") + hex(who.bytes) + "'::uuid"; + const auto where = "r.world_id = 7 AND r.account_id IN (" + list + ")"; + CHECK(one("SELECT count(*) || ':' || sum(r.inventory) FROM evolution.probe_records r " + "WHERE " + + where + " AND r.region_id IN (161, 162)", + {}) == + std::to_string(accounts.size()) + ":" + std::to_string(accounts.size() * 1000)); + CHECK(one("SELECT count(*) FROM evolution.probe_records r WHERE " + where + + " AND r.transfer_id IS DISTINCT FROM (SELECT t.transfer_id FROM " + "evolution.probe_transfers t WHERE t.world_id = 7 AND t.account_id = " + "r.account_id AND t.state IN ('prepared', 'staged'))", + {}) == "0"); + CHECK(one("SELECT count(*) FROM evolution.probe_transfers t WHERE t.world_id = 7 AND " + "t.account_id IN (" + + list + ") AND t.inventory <> 1000", + {}) == "0"); + } + // The region holding the account's durable control. + static std::string control_holder(const AccountId& who) { + return one("SELECT coalesce(region_id::text, '') FROM evolution.account_controls " + "WHERE world_id = 7 AND account_id = $1::uuid", + {hex(who.bytes)}); + } + void kill(bool at_a) { + auto& s = at_a ? a : b; + s->kill(); + s.reset(); + } + void restart(bool at_a) { + const auto started = Clock::now(); + (at_a ? a : b) = std::make_unique(pair, at_a); // Waits out the old lease. + std::printf("handoff process restart %c %lld ms\n", at_a ? 'A' : 'B', + static_cast(std::chrono::duration_cast( + Clock::now() - started) + .count())); + } +}; + +void handoffs() { + Bench w; + w.restart(true); + w.restart(false); + + // 1. A crossing and a crossing back (generation 1 -> 2 -> 3), then a stale grant login + // at the old region is redirected. + const auto alice = account(0xA1); + auto a1 = w.login(alice, region_a); + CHECK(a1->state() == RegionSessionState::established && w.probe(alice) == "161:1:false"); + const auto east = w.cross(*a1, 1); + CHECK(east.destination.region == region_b); + auto a2 = w.follow(east); + CHECK(a2->state() == RegionSessionState::established && w.probe(alice) == "162:2:false" && + w.transfer(east.transfer) == "committed"); + const auto west = w.cross(*a2, -1); + CHECK(west.destination.region == region_a); + auto a3 = w.follow(west); + CHECK(a3->state() == RegionSessionState::established && w.probe(alice) == "161:3:false"); + auto stale = w.login(alice, region_b); + CHECK(stale->close_reason() == SessionCloseReason::moved && stale->redirect() && + must(stale->redirect()).region == region_a); + w.invariants(); + + // 2. Destination SIGKILL after stage: the restarted destination stages again under its + // new epoch and commits when the ticket arrives. Then the probe moves on, and a replay of + // that ticket at a destination process that never saw it is refused durably. + const auto bob = account(0xB2); + auto b1 = w.login(bob, region_a); + const auto b_east = w.cross(*b1, 1); + const auto ticket_issued = wall_seconds(); + CHECK(w.transfer(b_east.transfer) == "prepared" && w.probe(bob) == "161:1:true"); + CHECK(w.one("SELECT transfer_state FROM evolution.stage_probe_transfer(7, 162, " + "(SELECT holder FROM evolution.region_leases WHERE world_id = 7 AND " + "region_id = 162), (SELECT epoch FROM evolution.region_leases WHERE " + "world_id = 7 AND region_id = 162), $1::uuid)", + {hex(b_east.transfer)}) == "staged"); // As B's process would, then B dies. + w.kill(false); + w.invariants(); + w.restart(false); + auto b2 = w.follow(b_east); + CHECK(b2->state() == RegionSessionState::established && w.probe(bob) == "162:2:false" && + w.transfer(b_east.transfer) == "committed"); + const auto b_west = w.cross(*b2, -1); + auto b3 = w.follow(b_west); + CHECK(b3->state() == RegionSessionState::established && w.probe(bob) == "161:3:false"); + + // 3. Destination SIGKILL after commit (the client was admitted): a retry of the same + // ticket at the restarted destination converges (same commit, control moves again). + const auto carol = account(0xC3); + auto c1 = w.login(carol, region_a); + const auto c_east = w.cross(*c1, 1); + auto c2 = w.follow(c_east); + CHECK(c2->state() == RegionSessionState::established && w.probe(carol) == "162:2:false"); + const auto control_before = w.control_generation(carol); + w.kill(false); + w.invariants(); + w.restart(false); + // Bob's first ticket, replayed after his probe moved on (and before it expires). + auto replay = w.follow(b_east); + std::printf("handoff process replay at %llu s of the ticket's lifetime\n", + static_cast(wall_seconds() - ticket_issued)); + CHECK(replay->state() == RegionSessionState::closed && + replay->close_reason() == SessionCloseReason::unauthenticated && + w.probe(bob) == "161:3:false" && w.control_holder(bob) == "161"); + auto c3 = w.follow(c_east); + CHECK(c3->state() == RegionSessionState::established && w.probe(carol) == "162:2:false" && + w.transfer(c_east.transfer) == "committed" && + w.control_generation(carol) > control_before); + // A duplicate in the same process is refused locally (the ticket is used). + auto again = w.follow(c_east); + CHECK(again->close_reason() == SessionCloseReason::stale_grant && + w.control_holder(carol) == "162"); + w.invariants(); + + // 4. Source SIGKILL after prepare: Dave's ticket still commits at the destination; Erin + // never arrives, and her frozen probe is aborted by the restarted source's recovery + // sweep (the dead process's in-memory deadline is gone). + const auto dave = account(0xD4), erin = account(0xE5); + auto d1 = w.login(dave, region_a); + auto e1 = w.login(erin, region_a); + const auto d_east = w.cross(*d1, 1); + const auto e_east = w.cross(*e1, 1); + w.kill(true); + auto d2 = w.follow(d_east); + CHECK(d2->state() == RegionSessionState::established && w.probe(dave) == "162:2:false"); + w.invariants(); + + // 5. A destination that is down: Frank's ticket cannot be used; the live source aborts + // after the ticket's lifetime. Erin's wait (above) overlaps. + w.kill(false); + w.restart(true); + auto e2 = w.login(erin, region_a); // Frozen under the open transfer: retry later. + CHECK(e2->close_reason() == SessionCloseReason::capacity && w.probe(erin) == "161:1:true"); + const auto frank = account(0xF6); + auto f1 = w.login(frank, region_a); + const auto f_east = w.cross(*f1, 1); + auto f2 = w.dial(w.pair.port_b, f_east.ticket); + w.settle(*f2); + CHECK(f2->state() == RegionSessionState::closed && !f2->welcome()); + w.invariants(); + CHECK(w.until( + {}, + [&] { + return w.transfer(e_east.transfer) == "aborted" && + w.transfer(f_east.transfer) == "aborted"; + }, + 90s)); + CHECK(w.probe(erin) == "161:1:false" && w.probe(frank) == "161:1:false"); + auto e3 = w.login(erin, region_a); + auto f3 = w.login(frank, region_a); + CHECK(e3->state() == RegionSessionState::established && + f3->state() == RegionSessionState::established); + w.restart(false); + auto late = w.follow(f_east); // Expired by now, and its transfer aborted. + CHECK(late->close_reason() == SessionCloseReason::expired && w.probe(frank) == "161:1:false"); + w.invariants(); + for (auto* s : {w.a.get(), w.b.get()}) { + s->child->signal(SIGTERM); + CHECK(s->child->wait_exit(30s) == 0); + } + // Output is flushed at exit. This B process only refused (expired). + CHECK(w.a->child->printed("m10 opened ") && w.b->child->printed("m10 refused ")); +} +} // namespace + +void run_region_handoff_process_tests() { + handoffs(); + std::puts("region handoff process integration passed"); +} diff --git a/tests/region_migration_tests.cpp b/tests/region_migration_tests.cpp index 6fdb9bd..7b82785 100644 --- a/tests/region_migration_tests.cpp +++ b/tests/region_migration_tests.cpp @@ -179,6 +179,7 @@ void upgrade_across_migration_and_barrier() { " -v migration_sha_11=$(s 0011_region_actions.sql)" " -v migration_sha_12=$(s 0012_probe_handoff.sql)" " -v migration_sha_13=$(s 0013_probe_incoming.sql)" + " -v migration_sha_14=$(s 0014_probe_overdue.sql)" " -v runtime_role=evo_runtime_test -f migrations/apply.sql" " > /dev/null 2>&1") != 0); CHECK(history(*owner_db) == "1,2,3,4,5,6,7"); @@ -194,7 +195,7 @@ void upgrade_across_migration_and_barrier() { CHECK(shell("PGDATABASE=" + database + " EVO_PG_RUNTIME_ROLE=evo_runtime_test bash \"$EVO_SOURCE_ROOT/" "tools/migrate-postgres.sh\" > /dev/null") == 0); - CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8,9,10,11,12,13"); + CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8,9,10,11,12,13,14"); CHECK(renew_region_lease(*runtime, r.key, *lease, 20000ms, 5s)); confirm(publish_next(20)); // 41..60, now v8. CHECK(head.head.end_tick == 60); diff --git a/tools/ci/postgres-check.sh b/tools/ci/postgres-check.sh index 444c93a..b10c45a 100644 --- a/tools/ci/postgres-check.sh +++ b/tools/ci/postgres-check.sh @@ -61,7 +61,7 @@ PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" -f "$root/migrations/0001_region_bootstrap.sql" > /dev/null test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1 PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13 +test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14 # v2 heads have no payload binding: v2->v3 fails closed and leaves history at v2. v2_sha="$(sha256sum "$root/migrations/0002_region_fencing.sql")" v2_sha="${v2_sha%% *}" @@ -82,7 +82,7 @@ PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '01000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32), 'hex'))" > /dev/null PGDATABASE=v3_commits bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 13:1,2,3,4,5,6,7,8,9,10,11,12,13 +test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 14:1,2,3,4,5,6,7,8,9,10,11,12,13,14 test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) FROM evolution.region_commits")" = 1 # Review P1: a v4 database holding a published (possibly terminal) checkpoint has # no durable decisions for it. The v5 upgrade must refuse before changing anything. @@ -136,7 +136,7 @@ PGDATABASE=v6_populated psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '02000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32) || repeat('00', 80), 'hex'))" > /dev/null PGDATABASE=v6_populated bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13 +test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13,14 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT count(*) FROM evolution.region_commits c, LATERAL evolution.parse_checkpoint_manifest(c.manifest) m WHERE m.commit_sequence = 0 AND m.predecessor_commit IS NULL")" = 1 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(conname, ',') FROM pg_constraint WHERE conrelid = 'evolution.region_heads'::regclass AND contype = 'c' AND pg_get_constraintdef(oid) LIKE '%end_tick%'")" = region_heads_published_tick # Inherited privileges bypass fencing, so every migrate run audits effective rights. @@ -181,9 +181,11 @@ expect_audit_failure 'GRANT SELECT ON evolution.account_controls TO evo_runtime_ expect_audit_failure 'REVOKE EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) FROM evo_runtime_test' 'GRANT EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) TO evo_runtime_test' bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again. # Unknown future versions and checksum drift must not be repaired silently. -psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=14 WHERE version=13' +psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=15 WHERE version=14' +if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=14 WHERE version=15" +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=14" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi -psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=13 WHERE version=14" psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=13" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=12" diff --git a/tools/ci/upgrade-check.sh b/tools/ci/upgrade-check.sh index be1e822..05ea01a 100644 --- a/tools/ci/upgrade-check.sh +++ b/tools/ci/upgrade-check.sh @@ -4,7 +4,7 @@ # exported). Real evo-server processes on world 500: # 1. Failed migration: a migration that takes the migration lock, alters a history # table, holds its locks for 3 s and then fails runs while build b1 serves. -# Nothing it did remains (schema still v13, no new column); the host keeps its +# Nothing it did remains (schema still v14, no new column); the host keeps its # authority, and every acknowledgement stays durable. # 2. Barrier (ADR-0068 operator order): build b2 is retained, and a b2 host is started # while b1 still holds the lease (it waits, never ready). upgrade_barrier(b1) fences @@ -61,7 +61,7 @@ SQL [ "$code" != 0 ] && grep -q 'division by zero' "$work/migration.log" || fail "the rehearsal migration did not fail" [ "$(q -c "SELECT count(*) FROM information_schema.columns WHERE table_name = 'region_commits' AND column_name = 'rehearsal_column'")" = 0 ] || fail "the failed migration left a column" -[ "$(q -c 'SELECT max(version) FROM evolution.schema_migrations')" = 13 ] || fail "schema version moved" +[ "$(q -c 'SELECT max(version) FROM evolution.schema_migrations')" = 14 ] || fail "schema version moved" wait_ack "$work/b1.log" $((before + 40)) || fail "b1 did not resume after the failed migration" kill -0 "$b1_pid" 2> /dev/null || fail "b1 exited during the failed migration" bash "$root/tools/migrate-postgres.sh" > /dev/null || fail "migrate after the failure" diff --git a/tools/migrate-postgres.sh b/tools/migrate-postgres.sh index cbabfca..14cf898 100644 --- a/tools/migrate-postgres.sh +++ b/tools/migrate-postgres.sh @@ -38,11 +38,14 @@ digest_12="$(sha256sum "$root/migrations/0012_probe_handoff.sql")" digest_12="${digest_12%% *}" digest_13="$(sha256sum "$root/migrations/0013_probe_incoming.sql")" digest_13="${digest_13%% *}" +digest_14="$(sha256sum "$root/migrations/0014_probe_overdue.sql")" +digest_14="${digest_14%% *}" psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_2=$digest_2" --set="migration_sha_3=$digest_3" \ --set="migration_sha_4=$digest_4" --set="migration_sha_5=$digest_5" \ --set="migration_sha_6=$digest_6" --set="migration_sha_7=$digest_7" \ --set="migration_sha_8=$digest_8" --set="migration_sha_9=$digest_9" \ --set="migration_sha_10=$digest_10" --set="migration_sha_11=$digest_11" --set="migration_sha_12=$digest_12" --set="migration_sha_13=$digest_13" \ + --set="migration_sha_14=$digest_14" \ --set="runtime_role=$runtime_role" \ --file="$root/migrations/apply.sql"