diff --git a/apps/evo-client/main.cpp b/apps/evo-client/main.cpp index 96cde64..ea9850d 100644 --- a/apps/evo-client/main.cpp +++ b/apps/evo-client/main.cpp @@ -185,18 +185,20 @@ std::optional parse_point(std::string_view value) { return evo::client::ScreenPoint{static_cast(x), static_cast(y)}; } // A probe marker (ADR-0094): the own one larger and amber, others smaller and blue; stale -// ones (no current authority, ADR-0095) grey. Clipped to the viewport. +// ones (no current authority, ADR-0095) grey; ghosts (arriving, ADR-0096) violet. Clipped +// to the viewport. void marker_box(std::vector& out, double x, double y, bool own, bool stale, - double width, double height) { + bool ghost, double width, double height) { const double half = own ? 7 : 4; const double x0 = std::max(x - half, 0.0), y0 = std::max(y - half, 0.0); const double x1 = std::min(x + half, width), y1 = std::min(y + half, height); if (x1 > x0 && y1 > y0) out.push_back({static_cast(x0), static_cast(y0), static_cast(x1 - x0), static_cast(y1 - y0), - stale ? evo::render::Color{120, 120, 120, 255} - : own ? evo::render::Color{250, 190, 40, 255} - : evo::render::Color{90, 170, 250, 255}}); + stale ? evo::render::Color{120, 120, 120, 255} + : ghost ? evo::render::Color{190, 120, 230, 255} + : own ? evo::render::Color{250, 190, 40, 255} + : evo::render::Color{90, 170, 250, 255}}); } void outline(std::vector& out, double l, double t, double r, double b, double width, double height) { @@ -546,7 +548,7 @@ int main(int argc, char** argv) try { if (const auto* remote = host.session(); remote && mode() == Mode::remote) for (const auto& m : remote->markers()) if (const auto at = camera->world_to_screen({m.x_mm, m.y_mm})) - marker_box(boxes, at->x, at->y, m.own, m.stale, width, height); + marker_box(boxes, at->x, at->y, m.own, m.stale, m.ghost, width, height); if (const auto* o = selection.organism(orgs)) { const auto b = evo::render::organism_bounds(*o, *camera); outline(boxes, b.left - 3, b.top - 3, b.right + 3, b.bottom + 3, width, height); diff --git a/apps/evo-client/remote_session.hpp b/apps/evo-client/remote_session.hpp index 9e1dd2e..4ebc622 100644 --- a/apps/evo-client/remote_session.hpp +++ b/apps/evo-client/remote_session.hpp @@ -83,6 +83,7 @@ struct MarkerView { bool own{}; bool stale{}; // No current authority behind it (draw dimmed). bool extrapolated{}; // Continued beyond the newest snapshot (at most two ticks). + bool ghost{}; // A probe being handed to this region, read-only (ADR-0096). }; #ifdef EVO_CLIENT_NETWORK @@ -401,7 +402,7 @@ class RemoteSession { if (m.own) predictor_.authoritative({v->tick, m.applied_sequence, m.state, m.area}); else - remote.push_back({m.id, m.state}); + remote.push_back({m.id, m.state, m.ghost}); } interpolator_.push(v->tick, remote); if (!following_ || v->environment.empty() || predictor_.authority()) @@ -456,7 +457,8 @@ class RemoteSession { // what makes the shown markers stale. const bool silent = clock_.silence(now).value_or(0) > silent_ticks; for (const auto& m : interpolator_.at(render_tick_)) - markers_.push_back({m.x / 1000, m.y / 1000, false, m.stale || silent, m.extrapolated}); + markers_.push_back( + {m.x / 1000, m.y / 1000, false, m.stale || silent, m.extrapolated, m.ghost}); } static std::int64_t cell(std::int64_t um) { constexpr auto size = network::motion::micrometres_per_cell; diff --git a/deploy/kubernetes/region/dev-session-route/listener.yaml b/deploy/kubernetes/region/dev-session-route/listener.yaml index 0260ee2..f0e686b 100644 --- a/deploy/kubernetes/region/dev-session-route/listener.yaml +++ b/deploy/kubernetes/region/dev-session-route/listener.yaml @@ -12,7 +12,7 @@ spec: env: - {name: REGION_SESSION, value: "dev"} - {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"} - - {name: REGION_SESSION_WINDOW, value: "dev-20261006T231342-4948:1791329622"} + - {name: REGION_SESSION_WINDOW, value: "dev-20261007T011838-10712:1791337118"} ports: # QUIC (ADR-0090): UDP only; nothing listens on TCP 7400. - {name: session, containerPort: 7400, protocol: UDP} diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 4ea6cd5..b0e6f75 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -536,8 +536,9 @@ remote-marker interpolation and fractional prediction done; 10.7b client wiring, view and two-client interpolation traces done. 10.8 split (ADR-0096, the persistence handoff ADR; EXPERIMENT durable probe records): 10.8a schema v12 probe records and the transfer state machine with a PostgreSQL fault matrix done; 10.8b signed handoff tickets and -ticket admission (durable stage, commit and control move) done; 10.8c region wiring and 10.8d -two-process kill tests follow. 10.9 planned. +ticket admission (durable stage, commit and control move) done; 10.8c-1 session protocol v4, +durable probes in sessions, edge crossing, ghosts (schema v13) and a two-region loopback test +done; 10.8c-2 client switch and server configuration, 10.8d two-process kill tests follow. 10.9 planned. **Entry:** M9 client/session evidence and M7/M8 dev/stage durability. Read NETWORK_ARCHITECTURE, PERSISTENCE_ARCHITECTURE and ADR-0017. The M9 transport is diff --git a/docs/STATUS.md b/docs/STATUS.md index 8283a6c..adf3972 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -76,7 +76,15 @@ Deploying v12 needs a rebuilt migrations image. M10.8b adds signed handoff ticke region's key signs one transfer for one account and destination; the ticket rides the Hello grant slot and is verified against the neighbour keys registered here. The destination then durably stages and commits the transfer and takes the account's control before admitting -the session. Next: 10.8c region wiring (edge crossing, ghost, client switch). +the session. M10.8c-1 wires the regions: +- session protocol v4 (handoff offer, redirect, ghosts; v3 is deployed and frozen); +- sessions use the durable probe: opened where it was saved, redirected when it lives + elsewhere; +- a marker crossing into a neighbour's area is frozen, prepared and offered with a ticket; +- the destination shows arriving probes as ghosts (schema v13) and opens the marker where it + crossed. + +Next: 10.8c-2, the client switch and server configuration. ## Previous gate - 2026-10-02 diff --git a/docs/adr/0096-m10-probe-handoff.md b/docs/adr/0096-m10-probe-handoff.md index 1d86a6e..8ac4434 100644 --- a/docs/adr/0096-m10-probe-handoff.md +++ b/docs/adr/0096-m10-probe-handoff.md @@ -143,3 +143,47 @@ controllable before M11. `capacity`. - Not yet: the source side (freeze at the edge, prepare, release control, offer the ticket), the client switch, keys in evo-server and the dev tool (10.8c). + +## M10.8c-1 region wiring (2026-10-06) +- **Session protocol v4** ([session-v4](../../schemas/network/session-v4.md); v3 was deployed + and is frozen) adds: + - `HandoffOffer` (transfer id, destination address, ticket); + - `SessionRedirect` (where the probe is, before Welcome); + - close reasons `handed_off` and `moved`; + - marker kind 2 (ghost). +- **Durable probes in sessions** (`ProbeLedger`, `include/evo/server/probe_ledger.hpp`). + - A grant login opens the account's probe record after the durable control claim. + - Owned here and active: the marker opens at its saved position, clamped into this + region's area, so a probe left beyond the edge by an aborted handoff does not cross + again at once. + - Owned elsewhere: Redirect (when that region is a known neighbour), then `moved`. + - Frozen: `capacity` (retry once the transfer is decided). + - The position is saved at session end, but not after a handoff. +- **Neighbours.** A region owns its area; markers move within the bounding box of its area + and its neighbours'. Every poll, a marker that left the area into a neighbour's area is + frozen and its transfer prepared. Then the session's control is released, a ticket is + signed (lifetime 30 s; its session expiry is the session's), the HandoffOffer sent, and + the session closed with `handed_off`. Anywhere else, the marker is put back at the edge. + An outgoing transfer not committed within its ticket's lifetime plus 5 s is aborted (the + probe resumes here); undecided aborts retry. +- **Destination.** The ticket admission now returns the probe's state after the switch + (`HandoffAdmission`), and the marker opens there. Every second (configurable) the region + reads the open transfers addressed to it (schema v13 `incoming_probe_transfers`) and shows + them as ghosts: stable ids per transfer, read-only, replicated to sessions whose square + holds them, and gone once decided. +- **Configuration.** Handoffs need probe records, this region's handoff key and neighbours. A + handoff key without probe records is a configuration error. +- **Evidence.** `two_region_handoff` (`tests/quic_sessions.cpp`): two regions with adjacent + areas in one process, over a fake world database with the v12 state machine. It covers: + - a crossing at the border; + - a ghost seen by another client at the destination before the switch; + - the ticket login (owner and generation moved, marker where it crossed); + - the ghost giving way; + - a redirect at the old region; + - a crossing back; + - an abandoned handoff aborted after its deadline, with the probe resumed at the edge and + the late ticket refused as expired. + + It passed 3 times in a row on Linux and also passes on Windows. +- Not yet: the engineering client's switch (offer, redirect, multi-region grants), + evo-server and probe configuration (10.8c-2), and real processes on PostgreSQL (10.8d). diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index ab5923c..634fffd 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -4767,3 +4767,59 @@ Executed: Not done: - the source side and the client switch (10.8c); - two-process kill tests (10.8d). + +## 2026-10-06 — M10.8c-1 region handoff wiring (ADR-0096) + +Implemented: +- session protocol v4 (HandoffOffer, SessionRedirect, `handed_off` and `moved`, ghost marker + kind; v1-v3 refused); +- the client library keeps offers and redirects; +- `ProbeLedger` (durable open, save, prepare, abort, incoming) and the PostgreSQL version; +- `HandoffAdmission` (state after the switch); +- `RegionSessions`: + - neighbours and the movement area; + - crossing detection, freeze, prepare, control release, a signed ticket, the offer and the + close; + - timed aborts and ghost polling; + - logins placing the durable probe (clamped into the area), with a redirect or `moved` + when it is owned elsewhere; + - saves at session end; +- `ProbeMarkers` `open_at`, `place`, `freeze` and ghosts; +- schema v13 `incoming_probe_transfers` (apply chain, audit, CI checks); +- client ghost drawing. + +Found and fixed during development: +- a dangling marker pointer after clamping (two `place` calls); +- the spawn computed from the area instead of the configured one; +- a probe left beyond the edge by an aborted handoff crossing again at login (now clamped). + +Tests: +- codec tests for the offer and redirect (round trip, text and padding rules, zero region or + transfer, lengths), with the v4 version, type and reason boundaries; +- ghosts and freezing through replication; +- `two_region_handoff` (two regions in one process over a fake world database): + - a crossing; a ghost at the destination before the switch; + - a ticket login at the crossing position (owner and generation moved); the ghost gone; + - a redirect; a crossing back; + - an abandoned handoff aborted after its deadline, the probe resumed at the edge, and the + late ticket refused; +- PostgreSQL: incoming transfers appear for the destination only while open. + +Executed: +- **Linux GCC 14:** + - `postgres-check.sh` with QUIC: 21 suites (migrations to v13; races 8 commits, 16 + aborts); + - presets `quic` 4/4 and `client-network` 16/16; + - `evo-quic-session-tests` 3 times in a row. +- **Linux Clang 19:** ASan/UBSan `evo-network-tests`; clang-tidy 19 on the protocol, motion, + replication, interpolation, client, region-sessions, handoff-ledger, store and test sources + (findings fixed); clang-format 19. +- **Windows MSVC 19.44:** presets `quic` 4/4 and `client-network` 14/14. + +Not done: +- the engineering client's handling of offers, redirects and multi-region grants, and + evo-server and probe configuration (10.8c-2); +- real processes (10.8d). + +Deployed v3 servers refuse v4 clients until redeployed; v12 and v13 need a rebuilt +migrations image. diff --git a/docs/infrastructure/POSTGRESQL.md b/docs/infrastructure/POSTGRESQL.md index baaeed5..fe593dc 100644 --- a/docs/infrastructure/POSTGRESQL.md +++ b/docs/infrastructure/POSTGRESQL.md @@ -216,8 +216,10 @@ returns the recorded outcome. SQLSTATEs: Lock order: upgrade lock (shared), own region lease row, probe record, transfer; `handoff_account_control` reads the transfer, then locks the account control row. The -privilege audit covers all eight functions, and the tables are owner-only. Deploying v12 -needs a rebuilt migrations (tools) image. +privilege audit covers all eight functions, and the tables are owner-only. Schema v13 adds +`incoming_probe_transfers`, a read-only function under the lease that lists the open +transfers addressed to the caller's region (for ghosts). Deploying v12 and v13 needs a +rebuilt migrations (tools) image. ## Host lease discipline diff --git a/include/evo/client/interpolation.hpp b/include/evo/client/interpolation.hpp index 0b735c4..1beb2d6 100644 --- a/include/evo/client/interpolation.hpp +++ b/include/evo/client/interpolation.hpp @@ -68,11 +68,12 @@ struct InterpolationLimits { struct RemoteMarker { std::uint64_t id{}; network::motion::State state; + bool ghost{}; // A probe being handed to this region (ADR-0096): read-only. }; struct PresentedMarker { std::uint64_t id{}; double x{}, y{}; // Micrometres. - bool extrapolated{}, stale{}; + bool extrapolated{}, stale{}, ghost{}; }; struct InterpolationStats { std::uint64_t snapshots{}, ignored{}, extrapolated_frames{}, stale_frames{}, gap_snaps{}; diff --git a/include/evo/client/region_session.hpp b/include/evo/client/region_session.hpp index 59369f7..5d79617 100644 --- a/include/evo/client/region_session.hpp +++ b/include/evo/client/region_session.hpp @@ -74,6 +74,11 @@ class RegionSessionClient { [[nodiscard]] RegionSessionState state() const noexcept; [[nodiscard]] const std::optional& welcome() const noexcept; [[nodiscard]] std::optional close_reason() const noexcept; + // v4 (ADR-0096): the offer to continue at another region (kept until taken; the session + // then closes with handed_off), and where the account's probe is when a login was closed + // with moved. + [[nodiscard]] std::optional take_handoff(); + [[nodiscard]] const std::optional& redirect() const noexcept; // Why the QUIC connection itself ended (e.g. `untrusted`: the server answered but its // certificate failed the pinned check), when it ended without a session Close. [[nodiscard]] std::optional transport_cause() const noexcept; diff --git a/include/evo/network/handoff_ticket.hpp b/include/evo/network/handoff_ticket.hpp index 5d6dbf2..b500f56 100644 --- a/include/evo/network/handoff_ticket.hpp +++ b/include/evo/network/handoff_ticket.hpp @@ -133,13 +133,20 @@ enum class HandoffLedgerError : std::uint8_t { fenced, // This region's lease is not current. unavailable // Not decided (database unreachable, timeout, unknown outcome): retry later. }; +// What the destination now owns: the account's control lease and the probe's state after +// the committed switch (its generation and position, in micrometres). +struct HandoffAdmission { + ControlLease lease; + std::uint64_t generation{}; + std::int64_t x{}, y{}; +}; class HandoffLedger { public: HandoffLedger() = default; HandoffLedger(const HandoffLedger&) = delete; HandoffLedger& operator=(const HandoffLedger&) = delete; virtual ~HandoffLedger() = default; - [[nodiscard]] virtual std::expected + [[nodiscard]] virtual std::expected admit(const TicketClaims& ticket, std::uint64_t session) = 0; protected: diff --git a/include/evo/network/probe_motion.hpp b/include/evo/network/probe_motion.hpp index 146006a..2c71715 100644 --- a/include/evo/network/probe_motion.hpp +++ b/include/evo/network/probe_motion.hpp @@ -1,9 +1,11 @@ #pragma once +#include #include #include #include #include #include +#include #include // M10.6 probe-motion experiment (ADR-0094). EXPERIMENT, not a game rule: one @@ -47,6 +49,16 @@ struct Marker { std::uint64_t id{}, session{}; State state; std::uint64_t applied_sequence{}; // Newest intention whose vector is in `state`. + // M10.8c (ADR-0096): frozen markers neither move nor take intentions (a handoff is being + // prepared); ghosts are read-only copies of probes being handed to this region (no + // session, never moving). + bool frozen{}, ghost{}; +}; +// A probe arriving here, shown read-only until its transfer commits (keyed by transfer id). +struct Ghost { + std::array transfer{}; + std::int64_t x{}, y{}; + friend bool operator==(const Ghost&, const Ghost&) = default; }; enum class MarkerError : std::uint8_t { no_area, unknown_session, full, invalid }; class ProbeMarkers { @@ -61,6 +73,15 @@ class ProbeMarkers { } // A marker for the session, at the spawn, holding no movement. [[nodiscard]] std::expected open(std::uint64_t session); + // The same at a given position (a durable probe's, clamped into the area). + [[nodiscard]] std::expected open_at(std::uint64_t session, + std::int64_t x, std::int64_t y); + // Moves the session's marker to a position (clamped into the area), keeping its vector. + void place(std::uint64_t session, std::int64_t x, std::int64_t y); + // Stops the session's marker where it is: no motion and no new intentions. + void freeze(std::uint64_t session); + // Replaces the ghosts (stable ids per transfer; at most max_markers). + void set_ghosts(std::span ghosts); void close(std::uint64_t session); // Removes every marker and forgets the tick (a new epoch may restart at an earlier one). void clear() noexcept; @@ -90,11 +111,16 @@ class ProbeMarkers { Marker marker; std::deque scheduled; // Ascending tick, then sequence. }; + struct GhostEntry { + Ghost ghost; + std::uint64_t id{}; + }; std::size_t max_markers_, max_scheduled_; std::optional area_; std::int64_t spawn_x_{}, spawn_y_{}; std::vector markers_; // Mirror of entries_ for readers, in id order. std::vector entries_; + std::vector ghosts_; std::uint64_t tick_{}, next_id_{1}; void publish(); }; diff --git a/include/evo/network/replication.hpp b/include/evo/network/replication.hpp index 71c2baa..b321852 100644 --- a/include/evo/network/replication.hpp +++ b/include/evo/network/replication.hpp @@ -120,6 +120,7 @@ struct ReplicatedMarker { motion::State state; std::int64_t z{}; bool own{}; + bool ghost{}; // v4: a probe being handed to this region, read-only (ADR-0096). std::uint64_t applied_sequence{}; motion::Area area; friend bool operator==(const ReplicatedMarker&, const ReplicatedMarker&) = default; diff --git a/include/evo/network/session_protocol.hpp b/include/evo/network/session_protocol.hpp index f1b01f2..867339d 100644 --- a/include/evo/network/session_protocol.hpp +++ b/include/evo/network/session_protocol.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -17,9 +18,12 @@ namespace evo::network { inline constexpr std::uint32_t session_magic = 0x4E535645; // Bytes "EVSN". // v2 (M10.5, ADR-0093): v1 plus interest, snapshot chunks and snapshot acks. -// v3 (M10.6, ADR-0094): v2 plus probe marker records inside snapshot chunks; no new message -// types. v1 and v2 were deployed to evolution-dev and are frozen; only v3 is accepted. -inline constexpr std::uint16_t session_protocol_version = 3; +// v3 (M10.6, ADR-0094): v2 plus probe marker records inside snapshot chunks. +// v4 (M10.8c, ADR-0096): v3 plus HandoffOffer and SessionRedirect (stream), close reasons +// handed_off and moved, and ghost marker records. v1-v3 were deployed to evolution-dev and +// are frozen; only v4 is accepted. +inline constexpr std::uint16_t session_protocol_version = 4; +inline constexpr std::size_t session_address_bytes = 64; // Endpoint or server name field. inline constexpr std::size_t session_datagram_bytes = 1200; // Whole message, ADR-0089. inline constexpr std::size_t session_header_bytes = 8; inline constexpr std::size_t session_grant_wire_bytes = 160; @@ -39,7 +43,9 @@ enum class SessionMessageType : std::uint8_t { action_query, // Client -> server, stream: resolve an action by id. interest, // Client -> server, stream: spatial focus (v2). snapshot_chunk, // Server -> client, datagram: part of one snapshot (v2). - snapshot_ack // Client -> server, datagram: newest complete snapshot (v2). + snapshot_ack, // Client -> server, datagram: newest complete snapshot (v2). + handoff_offer, // Server -> client, stream: continue at another region (v4). + redirect // Server -> client, stream: the account's probe is elsewhere (v4). }; // Admission answers for intentions and actions (wire 0..10). enum class AdmitStatus : std::uint8_t { @@ -80,6 +86,8 @@ enum class SessionCloseReason : std::uint8_t { timeout, // No Hello before the deadline. unsupported_version, shutdown, + handed_off, // v4: the probe crossed to another region; a HandoffOffer preceded this. + moved, // v4: the account's probe is owned by another region (a Redirect may precede). connection_lost = 255 }; enum class SessionCodecError : std::uint8_t { @@ -184,10 +192,31 @@ struct SnapshotAck { std::uint64_t epoch{}, tick{}; friend bool operator==(const SnapshotAck&, const SnapshotAck&) = default; }; +// A region's public address as a client dials it: "host:port" and the certificate name, +// printable ASCII (no spaces), 1-64 characters each. +struct RegionAddress { + std::uint64_t region{}; + std::string endpoint, server_name; + friend bool operator==(const RegionAddress&, const RegionAddress&) = default; +}; +// v4 (ADR-0096): the session's probe crossed into `destination`; log in there with `ticket` +// (a handoff ticket, used in Hello's grant slot). The source then closes with handed_off. +struct HandoffOffer { + std::array transfer{}; + RegionAddress destination; + WireGrant ticket{}; + friend bool operator==(const HandoffOffer&, const HandoffOffer&) = default; +}; +// v4: at login, the account's probe is owned by `owner`; log in there with a grant for it. +struct SessionRedirect { + RegionAddress owner; + friend bool operator==(const SessionRedirect&, const SessionRedirect&) = default; +}; using SessionMessage = std::variant; + ActionOutcome, ActionQuery, SessionInterest, SnapshotChunk, SnapshotAck, + HandoffOffer, SessionRedirect>; [[nodiscard]] std::vector encode_session(const SessionMessage& message); // Exactly one message; any other length for its type rejects. diff --git a/include/evo/persistence/probe_store.hpp b/include/evo/persistence/probe_store.hpp index e3bc1c7..2036044 100644 --- a/include/evo/persistence/probe_store.hpp +++ b/include/evo/persistence/probe_store.hpp @@ -5,6 +5,7 @@ #include #include #include +#include // Linux-only calls to the probe record and handoff functions of schema v12 (ADR-0096, // migrations/0012_probe_handoff.sql). EXPERIMENT: probes are non-biological engineering @@ -75,4 +76,15 @@ abort_probe_transfer(Postgres& db, RegionKey key, const LeaseGrant& lease, const [[nodiscard]] std::expected, ProbeStoreFailure> find_probe_transfer(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& transfer, std::chrono::milliseconds timeout); +// Schema v13: the open (prepared or staged) transfers addressed to the caller's region, for +// read-only ghosts (at most 256, oldest first). +struct IncomingTransfer { + Uuid transfer{}, account{}; + std::uint64_t source{}; + std::int64_t x{}, y{}, z{}; + friend bool operator==(const IncomingTransfer&, const IncomingTransfer&) = default; +}; +[[nodiscard]] std::expected, ProbeStoreFailure> +incoming_probe_transfers(Postgres& db, RegionKey key, const LeaseGrant& lease, + std::chrono::milliseconds timeout); } // namespace evo::persistence diff --git a/include/evo/server/postgres_handoff_ledger.hpp b/include/evo/server/postgres_handoff_ledger.hpp index 39ecad9..318492e 100644 --- a/include/evo/server/postgres_handoff_ledger.hpp +++ b/include/evo/server/postgres_handoff_ledger.hpp @@ -2,10 +2,12 @@ #include "evo/network/handoff_ticket.hpp" #include "evo/persistence/probe_store.hpp" #include "evo/server/pg_slot.hpp" +#include "evo/server/probe_ledger.hpp" #include #include #include #include +#include // M10.8b destination side of a ticket login over PostgreSQL (ADR-0096, schema v12). Linux // only; owned by the region host's thread, on its own connection slot. For a verified ticket: @@ -21,7 +23,7 @@ class PostgresHandoffLedger final : public network::HandoffLedger { std::chrono::milliseconds duration = std::chrono::milliseconds{20000}, std::chrono::milliseconds timeout = std::chrono::milliseconds{2000}) : db_{&db}, key_{key}, lease_{std::move(lease)}, duration_{duration}, timeout_{timeout} {} - [[nodiscard]] std::expected + [[nodiscard]] std::expected admit(const network::TicketClaims& ticket, std::uint64_t session) override; private: @@ -30,4 +32,34 @@ class PostgresHandoffLedger final : public network::HandoffLedger { LeaseSource lease_; std::chrono::milliseconds duration_, timeout_; }; + +// M10.8c the region's durable probe records over PostgreSQL (ADR-0096, schema v12-13), on the +// host's lease and its own connection slot. Undecided or failed calls report `unavailable` +// (the session layer retries or refuses); a fenced lease reports `fenced`. +class PostgresProbeLedger final : public ProbeLedger { + public: + using LeaseSource = std::function()>; + static constexpr std::uint64_t inventory = 1000; // EXPERIMENT test inventory per probe. + PostgresProbeLedger(PgSlot& db, persistence::RegionKey key, LeaseSource lease, + std::chrono::milliseconds timeout = std::chrono::milliseconds{2000}) + : db_{&db}, key_{key}, lease_{std::move(lease)}, timeout_{timeout} {} + [[nodiscard]] std::expected open(const network::AccountId& account, + std::int64_t x, std::int64_t y, + std::int64_t z) override; + void save(const network::AccountId& account, std::uint64_t generation, std::int64_t x, + std::int64_t y, std::int64_t z) override; + [[nodiscard]] std::expected + prepare(const network::AccountId& account, std::uint64_t generation, + const network::TransferId& transfer, std::uint64_t destination, std::int64_t x, + std::int64_t y, std::int64_t z) override; + [[nodiscard]] std::expected + abort(const network::TransferId& transfer) override; + [[nodiscard]] std::expected, ProbeLedgerError> incoming() override; + + private: + PgSlot* db_; + persistence::RegionKey key_; + LeaseSource lease_; + std::chrono::milliseconds timeout_; +}; } // namespace evo::server diff --git a/include/evo/server/probe_ledger.hpp b/include/evo/server/probe_ledger.hpp new file mode 100644 index 0000000..a9e23dd --- /dev/null +++ b/include/evo/server/probe_ledger.hpp @@ -0,0 +1,51 @@ +#pragma once +#include "evo/network/handoff_ticket.hpp" +#include "evo/network/session_control.hpp" +#include +#include +#include + +// M10.8c (ADR-0096): the region's durable probe records as the session layer uses them. +// Transport-neutral; the host implements it over PostgreSQL (PostgresProbeLedger), tests use +// fakes. EXPERIMENT: probes are non-biological engineering entities (ADR-0094). +namespace evo::server { +struct ProbeHome { + std::uint64_t owner{}, generation{}; + std::int64_t x{}, y{}; + bool frozen{}; // An open transfer: neither this region nor the owner may act yet. +}; +struct IncomingProbe { + network::TransferId transfer{}; + std::int64_t x{}, y{}; +}; +enum class ProbeLedgerError : std::uint8_t { fenced, unavailable }; +class ProbeLedger { + public: + ProbeLedger() = default; + ProbeLedger(const ProbeLedger&) = delete; + ProbeLedger& operator=(const ProbeLedger&) = delete; + virtual ~ProbeLedger() = default; + // The account's probe, created here at the position when the world has none. + [[nodiscard]] virtual std::expected + open(const network::AccountId& account, std::int64_t x, std::int64_t y, std::int64_t z) = 0; + // Best effort: the owned, active probe's position (session end). + virtual void save(const network::AccountId& account, std::uint64_t generation, std::int64_t x, + std::int64_t y, std::int64_t z) = 0; + // Freezes the probe at its final position for `destination`. + [[nodiscard]] virtual std::expected + prepare(const network::AccountId& account, std::uint64_t generation, + const network::TransferId& transfer, std::uint64_t destination, std::int64_t x, + std::int64_t y, std::int64_t z) = 0; + // Abandons a transfer whose client never arrived: true when aborted (the probe is active + // here again), false when it had been committed. + [[nodiscard]] virtual std::expected + abort(const network::TransferId& transfer) = 0; + // Open transfers addressed to this region (shown as ghosts). + [[nodiscard]] virtual std::expected, ProbeLedgerError> + incoming() = 0; + + protected: + ProbeLedger(ProbeLedger&&) = default; + ProbeLedger& operator=(ProbeLedger&&) = default; +}; +} // namespace evo::server diff --git a/include/evo/server/region_sessions.hpp b/include/evo/server/region_sessions.hpp index 63a09ac..43b21f1 100644 --- a/include/evo/server/region_sessions.hpp +++ b/include/evo/server/region_sessions.hpp @@ -7,6 +7,7 @@ #include "evo/network/session_control.hpp" #include "evo/network/session_grant.hpp" #include "evo/network/session_protocol.hpp" +#include "evo/server/probe_ledger.hpp" #include #include #include @@ -46,6 +47,11 @@ struct IssuerKey { std::uint32_t key_id{}; network::GrantPublicKey public_key{}; }; +// A region whose area adjoins this one: where its clients dial and the area it owns. +struct Neighbour { + network::RegionAddress address; + network::motion::Area area; // Micrometres, same slice as this region's. +}; // A neighbouring region's handoff public key (ADR-0096): tickets it signs are accepted here. struct HandoffKey { std::uint64_t source_region{}; @@ -73,6 +79,15 @@ struct RegionSessionsConfig { // ticket also needs a HandoffLedger (control must move durably with the transfer). std::vector handoff_keys; network::HandoffLedger* handoff{}; + // M10.8c: with probe records, a session's marker is the account's durable probe (it opens + // where it was saved, a login whose probe lives elsewhere is redirected). With this + // region's handoff key and neighbours as well, a marker crossing into a neighbour's area + // is frozen, prepared and handed over with a ticket. + ProbeLedger* probes{}; + std::shared_ptr handoff_key; + std::vector neighbours; + std::uint64_t ticket_lifetime{30}; // Seconds a handoff ticket is valid. + std::chrono::milliseconds ghost_poll{1000}; // Incoming transfers read this often. }; // The region host's authority, read once per poll. struct HostFacts { @@ -119,7 +134,8 @@ class RegionSessions { const network::Perceives& perceives = {}); [[nodiscard]] network::ReplicationStats replication_stats() const; // Where probe markers live and spawn (ADR-0094); sessions already established receive a - // marker. False for an invalid area. + // marker. False for an invalid area. With neighbours (ADR-0096), markers move within the + // bounding box of this area and the neighbours' areas, and leaving this area hands off. [[nodiscard]] bool set_motion_area(const network::motion::Area& area, std::int64_t spawn_x, std::int64_t spawn_y); [[nodiscard]] const network::motion::ProbeMarkers& markers() const noexcept; diff --git a/migrations/0013_probe_incoming.sql b/migrations/0013_probe_incoming.sql new file mode 100644 index 0000000..9831cce --- /dev/null +++ b/migrations/0013_probe_incoming.sql @@ -0,0 +1,27 @@ +-- M10.8c incoming probe transfers (ADR-0096). A destination region shows a probe that is +-- being handed to it as a read-only ghost before the switch; there is no region-to-region +-- channel, so it reads the open transfers addressed to it. Read-only, under the caller's +-- exact unexpired region lease like every runtime function. At most 256 rows, oldest first. +CREATE FUNCTION evolution.incoming_probe_transfers(p_world evolution.u64, p_region evolution.u64, + p_holder uuid, p_epoch evolution.u64) +RETURNS TABLE (transfer_id uuid, account_id uuid, source_region evolution.u64, x bigint, + y bigint, z bigint) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +BEGIN + PERFORM evolution.check_region_lease(p_world, p_region, p_holder, p_epoch); + RETURN QUERY + SELECT t.transfer_id, t.account_id, t.source_region, t.x, t.y, t.z + FROM evolution.probe_transfers t + WHERE t.world_id = p_world AND t.destination_region = p_region + AND t.state IN ('prepared', 'staged') + ORDER BY t.prepared_at, t.transfer_id + LIMIT 256; +END $$; + +REVOKE ALL ON FUNCTION + evolution.incoming_probe_transfers(evolution.u64, evolution.u64, uuid, evolution.u64) + FROM PUBLIC; +GRANT EXECUTE ON FUNCTION + evolution.incoming_probe_transfers(evolution.u64, evolution.u64, uuid, evolution.u64) + TO :"runtime_role"; +INSERT INTO evolution.schema_migrations(version, sha256) VALUES (13, :'migration_sha_13'); diff --git a/migrations/apply.sql b/migrations/apply.sql index cd97db0..24eb572 100644 --- a/migrations/apply.sql +++ b/migrations/apply.sql @@ -58,28 +58,42 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') || ',7:' || :'migration_sha_7' || ',8:' || :'migration_sha_8' || ',9:' || :'migration_sha_9' || ',10:' || :'migration_sha_10' || ',11:' || :'migration_sha_11' || ',12:' || :'migration_sha_12' - AS at_v12 \gset - \if :at_v12 + AS at_v12, + :'history' = '1:' || :'migration_sha' || ',2:' || :'migration_sha_2' + || ',3:' || :'migration_sha_3' || ',4:' || :'migration_sha_4' + || ',5:' || :'migration_sha_5' || ',6:' || :'migration_sha_6' + || ',7:' || :'migration_sha_7' || ',8:' || :'migration_sha_8' + || ',9:' || :'migration_sha_9' || ',10:' || :'migration_sha_10' + || ',11:' || :'migration_sha_11' || ',12:' || :'migration_sha_12' + || ',13:' || :'migration_sha_13' AS at_v13 \gset + \if :at_v13 + \elif :at_v12 + \ir 0013_probe_incoming.sql \elif :at_v11 \ir 0012_probe_handoff.sql + \ir 0013_probe_incoming.sql \elif :at_v10 \ir 0011_region_actions.sql \ir 0012_probe_handoff.sql + \ir 0013_probe_incoming.sql \elif :at_v9 \ir 0010_account_control.sql \ir 0011_region_actions.sql \ir 0012_probe_handoff.sql + \ir 0013_probe_incoming.sql \elif :at_v8 \ir 0009_serialized_authority.sql \ir 0010_account_control.sql \ir 0011_region_actions.sql \ir 0012_probe_handoff.sql + \ir 0013_probe_incoming.sql \elif :at_v7 \ir 0008_upgrade_barrier.sql \ir 0009_serialized_authority.sql \ir 0010_account_control.sql \ir 0011_region_actions.sql \ir 0012_probe_handoff.sql + \ir 0013_probe_incoming.sql \elif :at_v6 \ir 0007_region_clones.sql \ir 0008_upgrade_barrier.sql @@ -87,6 +101,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0010_account_control.sql \ir 0011_region_actions.sql \ir 0012_probe_handoff.sql + \ir 0013_probe_incoming.sql \elif :at_v5 \ir 0006_commit_lookup.sql \ir 0007_region_clones.sql @@ -95,6 +110,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0010_account_control.sql \ir 0011_region_actions.sql \ir 0012_probe_handoff.sql + \ir 0013_probe_incoming.sql \elif :at_v4 \ir pre_v5_guard.sql \ir 0005_batch_decisions.sql @@ -105,6 +121,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0010_account_control.sql \ir 0011_region_actions.sql \ir 0012_probe_handoff.sql + \ir 0013_probe_incoming.sql \elif :at_v3 \ir 0004_lineage_profile.sql \ir pre_v5_guard.sql @@ -116,6 +133,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0010_account_control.sql \ir 0011_region_actions.sql \ir 0012_probe_handoff.sql + \ir 0013_probe_incoming.sql \elif :at_v2 \ir 0003_region_checkpoints.sql \ir 0004_lineage_profile.sql @@ -128,6 +146,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0010_account_control.sql \ir 0011_region_actions.sql \ir 0012_probe_handoff.sql + \ir 0013_probe_incoming.sql \elif :at_v1 \ir 0002_region_fencing.sql \ir 0003_region_checkpoints.sql @@ -141,6 +160,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0010_account_control.sql \ir 0011_region_actions.sql \ir 0012_probe_handoff.sql + \ir 0013_probe_incoming.sql \else \echo 'Unknown or changed migration history' DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$; @@ -159,6 +179,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0010_account_control.sql \ir 0011_region_actions.sql \ir 0012_probe_handoff.sql + \ir 0013_probe_incoming.sql \endif -- Effective-privilege audit on every run, including no-ops (ADR-0060). Attributes -- alone are insufficient: predefined roles such as pg_write_all_data, PUBLIC @@ -204,7 +225,8 @@ BEGIN ('evolution.commit_probe_transfer(evolution.u64,evolution.u64,uuid,evolution.u64,uuid)'::regprocedure::oid), ('evolution.abort_probe_transfer(evolution.u64,evolution.u64,uuid,evolution.u64,uuid)'::regprocedure::oid), ('evolution.find_probe_transfer(evolution.u64,evolution.u64,uuid,evolution.u64,uuid)'::regprocedure::oid), - ('evolution.handoff_account_control(evolution.u64,evolution.u64,uuid,evolution.u64,uuid,uuid,evolution.u64,integer)'::regprocedure::oid) + ('evolution.handoff_account_control(evolution.u64,evolution.u64,uuid,evolution.u64,uuid,uuid,evolution.u64,integer)'::regprocedure::oid), + ('evolution.incoming_probe_transfers(evolution.u64,evolution.u64,uuid,evolution.u64)'::regprocedure::oid) ), violations(role_oid, reason) AS ( SELECT r.oid, 'privileged attribute or owner identity' FROM pg_catalog.pg_roles r JOIN closure c ON c.oid = r.oid diff --git a/schemas/network/session-v3.md b/schemas/network/session-v3.md index 1505f99..68560ba 100644 --- a/schemas/network/session-v3.md +++ b/schemas/network/session-v3.md @@ -1,5 +1,8 @@ # M10 session protocol, version 3 +Superseded by [version 4](session-v4.md) (ADR-0096): v3 was deployed to evolution-dev and is +frozen; v4 servers and clients refuse it. + Version 3 is [version 2](session-v2.md) with two more snapshot record kinds for the M10.6 probe-motion experiment (ADR-0094). Message types, lengths and rules are those of v2. The header version is 3. Versions 1 and 2 were deployed to evolution-dev and are frozen; both diff --git a/schemas/network/session-v4.md b/schemas/network/session-v4.md new file mode 100644 index 0000000..9d16b43 --- /dev/null +++ b/schemas/network/session-v4.md @@ -0,0 +1,37 @@ +# M10 session protocol, version 4 + +Version 4 is [version 3](session-v3.md) plus two server-to-client stream messages, two close +reasons and a ghost marker kind, for cross-region handoff (ADR-0096). Everything else is as +in v3. The header version is 4. Versions 1-3 were deployed to evolution-dev and are frozen; +all three are refused (`unsupported_version`). Once v4 is deployed it is frozen like them. + +## Messages (in addition to v3 types 1-15) + +| Type | Direction | Body | Bytes | +|---|---|---|---| +| 16 HandoffOffer | server -> client, stream | transfer id (16), destination address, ticket (160) | 312 | +| 17 SessionRedirect | server -> client, stream | owner address | 136 | + +An address is: region u64 (nonzero), endpoint (64 bytes), server name (64 bytes). Each text +field is 1-64 printable ASCII characters without spaces (0x21-0x7E), padded with zero bytes. +A nonzero byte after the first zero is invalid, and the transfer id must be nonzero. + +- **HandoffOffer**, while established: the session's probe crossed into the destination + region. The ticket ([handoff-ticket-v1](handoff-ticket-v1.md)) goes in `Hello`'s grant slot + there. The source then sends `Close(handed_off)`. +- **SessionRedirect**, before Welcome: the account's probe is owned by that region. A grant for + it is needed there. The region then sends `Close(moved)`. + +## Close reasons (in addition to v3's 1-13) + +- **14 handed_off:** the session ended because its probe crossed into another region. +- **15 moved:** the account's probe is owned by another region; a Redirect may precede it. + +## Snapshot records + +Marker upsert (kind 5): the `own` byte becomes a kind. +- **0:** another session's marker. +- **1:** the receiving session's own marker (sequence and area follow). +- **2:** a ghost, a probe being handed to this region, read-only, with no sequence or area. + +Any other value is malformed. diff --git a/src/client/prediction/interpolation.cpp b/src/client/prediction/interpolation.cpp index 5921a97..74e3097 100644 --- a/src/client/prediction/interpolation.cpp +++ b/src/client/prediction/interpolation.cpp @@ -105,7 +105,8 @@ std::vector MarkerInterpolator::at(double render_tick) { const auto& newest = snapshots_.back(); const auto show = [&](const Snapshot& s) { for (const auto& m : s.markers) - out.push_back({m.id, static_cast(m.state.x), static_cast(m.state.y)}); + out.push_back({m.id, static_cast(m.state.x), static_cast(m.state.y), + false, false, m.ghost}); }; if (render_tick >= static_cast(newest.tick)) { // Beyond the newest snapshot: continue the held vector briefly, then freeze (stale). @@ -116,7 +117,7 @@ std::vector MarkerInterpolator::at(double render_tick) { out.push_back({m.id, static_cast(m.state.x) + motion_um(m.state.vector.x, ticks), static_cast(m.state.y) + motion_um(m.state.vector.y, ticks), - ticks > 0, stale}); + ticks > 0, stale, m.ghost}); stats_.extrapolated_frames += ticks > 0 ? 1 : 0; stats_.stale_frames += stale ? 1 : 0; return out; @@ -142,14 +143,16 @@ std::vector MarkerInterpolator::at(double render_tick) { const auto next = std::ranges::lower_bound(b->markers, m.id, {}, &RemoteMarker::id); if (next == b->markers.end() || next->id != m.id) { // Removed by b: shown where it was until b's tick, never moved toward a removal. - out.push_back({m.id, static_cast(m.state.x), static_cast(m.state.y)}); + out.push_back({m.id, static_cast(m.state.x), static_cast(m.state.y), + false, false, m.ghost}); continue; } const auto lerp = [t](std::int64_t from, std::int64_t to) { return static_cast(from) + ((static_cast(to) - static_cast(from)) * t); }; - out.push_back({m.id, lerp(m.state.x, next->state.x), lerp(m.state.y, next->state.y)}); + out.push_back({m.id, lerp(m.state.x, next->state.x), lerp(m.state.y, next->state.y), false, + false, next->ghost}); } return out; // Markers new in b appear from b's tick. } diff --git a/src/client/session/region_session.cpp b/src/client/session/region_session.cpp index e6cadb1..a76d1e3 100644 --- a/src/client/session/region_session.cpp +++ b/src/client/session/region_session.cpp @@ -25,7 +25,9 @@ struct RegionSessionClient::Impl { std::vector action_outcomes; std::optional snapshots; std::uint64_t snapshot_chunks{}, snapshot_bytes{}; - static constexpr std::size_t max_kept = 1024; // Per kind, until taken; oldest dropped. + std::optional handoff; // v4: continue at another region. + std::optional redirect; // v4: the probe is owned elsewhere. + static constexpr std::size_t max_kept = 1024; // Per kind, until taken; oldest dropped. template static void keep(std::vector& list, const T& item) { if (list.size() >= max_kept) @@ -95,6 +97,10 @@ struct RegionSessionClient::Impl { if (const auto* m = std::get_if(&*decoded)) return end(m->reason, false); if (state == RegionSessionState::awaiting_welcome) { + if (const auto* m = std::get_if(&*decoded)) { + redirect = m->owner; // A Close (moved) follows. + return; + } if (const auto* m = std::get_if(&*decoded)) { welcome = *m; snapshots.emplace(world::WorldIdentity{m->world, m->frame}, m->epoch); @@ -115,6 +121,10 @@ struct RegionSessionClient::Impl { } if (const auto* m = std::get_if(&*decoded)) return keep(action_outcomes, *m); + if (const auto* m = std::get_if(&*decoded)) { + handoff = *m; // A Close (handed_off) follows. + return; + } } end(SessionCloseReason::protocol, true); } @@ -258,6 +268,14 @@ std::uint64_t RegionSessionClient::snapshot_bytes() const noexcept { return impl_->snapshot_bytes; } +std::optional RegionSessionClient::take_handoff() { + return std::exchange(impl_->handoff, std::nullopt); +} + +const std::optional& RegionSessionClient::redirect() const noexcept { + return impl_->redirect; +} + RegionSessionState RegionSessionClient::state() const noexcept { return impl_->state; } diff --git a/src/network/protocol/probe_motion.cpp b/src/network/protocol/probe_motion.cpp index a76d2d2..6284180 100644 --- a/src/network/protocol/probe_motion.cpp +++ b/src/network/protocol/probe_motion.cpp @@ -41,6 +41,11 @@ bool ProbeMarkers::set_area(const Area& area, std::int64_t spawn_x, std::int64_t } std::expected ProbeMarkers::open(std::uint64_t session) { + return open_at(session, spawn_x_, spawn_y_); +} + +std::expected ProbeMarkers::open_at(std::uint64_t session, + std::int64_t x, std::int64_t y) { if (!area_) return std::unexpected(MarkerError::no_area); if (session == 0 || @@ -51,12 +56,47 @@ std::expected ProbeMarkers::open(std::uint64_t sessi Entry e; e.marker.id = next_id_++; e.marker.session = session; - e.marker.state = {spawn_x_, spawn_y_, {}}; + e.marker.state = { + std::clamp(x, area_->min_x, area_->max_x), std::clamp(y, area_->min_y, area_->max_y), {}}; entries_.push_back(std::move(e)); publish(); return entries_.back().marker.id; } +void ProbeMarkers::place(std::uint64_t session, std::int64_t x, std::int64_t y) { + if (!area_) + return; + for (auto& e : entries_) + if (e.marker.session == session) { + e.marker.state.x = std::clamp(x, area_->min_x, area_->max_x); + e.marker.state.y = std::clamp(y, area_->min_y, area_->max_y); + } + publish(); +} + +void ProbeMarkers::freeze(std::uint64_t session) { + for (auto& e : entries_) + if (e.marker.session == session) { + e.marker.frozen = true; + e.marker.state.vector = {}; + e.scheduled.clear(); + } + publish(); +} + +void ProbeMarkers::set_ghosts(std::span ghosts) { + std::vector next; + for (const auto& g : ghosts) { + if (next.size() >= max_markers_) + break; + const auto known = std::ranges::find_if( + ghosts_, [&](const GhostEntry& e) { return e.ghost.transfer == g.transfer; }); + next.push_back({g, known == ghosts_.end() ? next_id_++ : known->id}); + } + ghosts_ = std::move(next); + publish(); +} + void ProbeMarkers::close(std::uint64_t session) { std::erase_if(entries_, [&](const Entry& e) { return e.marker.session == session; }); publish(); @@ -64,6 +104,7 @@ void ProbeMarkers::close(std::uint64_t session) { void ProbeMarkers::clear() noexcept { entries_.clear(); + ghosts_.clear(); markers_.clear(); tick_ = 0; } @@ -75,6 +116,8 @@ std::expected ProbeMarkers::schedule(std::uint64_t session, std::ranges::find_if(entries_, [&](const Entry& e) { return e.marker.session == session; }); if (it == entries_.end()) return std::unexpected(MarkerError::unknown_session); + if (it->marker.frozen) + return std::unexpected(MarkerError::invalid); if (!valid(vector) || tick <= tick_ || sequence <= it->marker.applied_sequence || (!it->scheduled.empty() && sequence <= it->scheduled.back().sequence)) return std::unexpected(MarkerError::invalid); @@ -97,6 +140,8 @@ void ProbeMarkers::advance(std::uint64_t tick) { const auto first = tick - tick_ > max_catch_up ? tick - max_catch_up + 1 : tick_ + 1; for (auto& e : entries_) { auto& m = e.marker; + if (m.frozen) + continue; // Intentions for ticks that can no longer be stepped still set the held vector. while (!e.scheduled.empty() && e.scheduled.front().tick < first) { m.state.vector = e.scheduled.front().vector; @@ -117,8 +162,8 @@ void ProbeMarkers::advance(std::uint64_t tick) { } const Marker* ProbeMarkers::of(std::uint64_t session) const noexcept { - const auto it = - std::ranges::find_if(markers_, [&](const Marker& m) { return m.session == session; }); + const auto it = std::ranges::find_if( + markers_, [&](const Marker& m) { return !m.ghost && m.session == session; }); return it == markers_.end() ? nullptr : &*it; } @@ -126,6 +171,13 @@ void ProbeMarkers::publish() { markers_.clear(); for (const auto& e : entries_) markers_.push_back(e.marker); + for (const auto& g : ghosts_) { + Marker m; + m.id = g.id; + m.state = {g.ghost.x, g.ghost.y, {}}; + m.ghost = true; + markers_.push_back(m); + } std::ranges::sort(markers_, {}, &Marker::id); } } // namespace evo::network::motion diff --git a/src/network/protocol/replication.cpp b/src/network/protocol/replication.cpp index 8a2ad99..412e3d3 100644 --- a/src/network/protocol/replication.cpp +++ b/src/network/protocol/replication.cpp @@ -150,7 +150,7 @@ std::vector encode_record(const motion::Marker& m, std::int64_t z, put_signed(out, z); put(out, static_cast(m.state.vector.x), 2); put(out, static_cast(m.state.vector.y), 2); - out.push_back(own_area != nullptr ? 1 : 0); + out.push_back(own_area != nullptr ? 1 : m.ghost ? 2 : 0); // v4: 2 is a ghost. if (own_area != nullptr) { put(out, m.applied_sequence, 8); for (const auto v : {own_area->min_x, own_area->min_y, own_area->max_x, own_area->max_y}) @@ -261,7 +261,7 @@ std::vector>> ReplicationSer // every marker first and makes it due every tick it changed. if (src.marker_area) for (const auto& m : src.markers) { - const bool own = m.session == s.id; + const bool own = !m.ghost && m.session == s.id; if (!own && (src.marker_area->z != in.z || distance(cell_of(m.state.x), cell_of(m.state.y), in) > in.radius)) continue; @@ -502,10 +502,11 @@ SnapshotAssembler::accept(const SnapshotChunk& chunk) { m.z = r.i64(); m.state.vector = {static_cast(r.u(2)), static_cast(r.u(2))}; - const auto own = r.u(1); - if (own > 1) + const auto kind_byte = r.u(1); + if (kind_byte > 2) return std::unexpected(AssemblyError::malformed); - m.own = own == 1; + m.own = kind_byte == 1; + m.ghost = kind_byte == 2; if (m.own) { m.applied_sequence = r.u(8); m.area = {r.i64(), r.i64(), r.i64(), r.i64(), m.z}; diff --git a/src/network/protocol/session_protocol.cpp b/src/network/protocol/session_protocol.cpp index cbe8302..c883b24 100644 --- a/src/network/protocol/session_protocol.cpp +++ b/src/network/protocol/session_protocol.cpp @@ -1,5 +1,7 @@ #include "evo/network/session_protocol.hpp" #include +#include +#include namespace evo::network { namespace { @@ -19,6 +21,9 @@ constexpr std::uint8_t max_admit_status = static_cast(AdmitStatus: constexpr std::size_t interest_bytes = (3 * std::size_t{8}) + 2; constexpr std::size_t snapshot_ack_bytes = 2 * std::size_t{8}; constexpr std::uint8_t max_snapshot_chunks = 16; +constexpr std::size_t address_bytes = 8 + (2 * session_address_bytes); +constexpr std::size_t handoff_offer_bytes = 16 + address_bytes + session_grant_wire_bytes; +constexpr std::size_t redirect_bytes = address_bytes; void put(std::vector& out, std::uint64_t v, std::size_t count) { for (std::size_t i = 0; i < count; ++i) @@ -41,7 +46,44 @@ std::vector header(SessionMessageType type, std::size_t payload) { } bool wire_reason(std::uint64_t v) { return v >= static_cast(SessionCloseReason::client_closed) && - v <= static_cast(SessionCloseReason::shutdown); + v <= static_cast(SessionCloseReason::moved); +} +// Printable ASCII without spaces, then zero padding to the field size. +bool valid_text(const std::string& s) { + return !s.empty() && s.size() <= session_address_bytes && + std::ranges::all_of(s, [](char ch) { return ch > 0x20 && ch < 0x7F; }); +} +void put_text(std::vector& out, const std::string& s) { + for (std::size_t i = 0; i < session_address_bytes; ++i) + out.push_back(i < s.size() ? static_cast(s[i]) : std::uint8_t{0}); +} +std::optional text_at(std::span in, std::size_t at) { + std::string s; + std::size_t i = 0; + for (; i < session_address_bytes && in[at + i] != 0; ++i) + s.push_back(static_cast(in[at + i])); + for (; i < session_address_bytes; ++i) + if (in[at + i] != 0) + return std::nullopt; // Bytes after the terminator must be padding. + if (!valid_text(s)) + return std::nullopt; + return s; +} +void put_address(std::vector& out, const RegionAddress& a) { + put(out, a.region, 8); + put_text(out, a.endpoint); + put_text(out, a.server_name); +} +std::optional address_at(std::span in, std::size_t at) { + RegionAddress a; + a.region = get(in, at, 8); + auto endpoint = text_at(in, at + 8); + auto name = text_at(in, at + 8 + session_address_bytes); + if (a.region == 0 || !endpoint || !name) + return std::nullopt; + a.endpoint = std::move(*endpoint); + a.server_name = std::move(*name); + return a; } WireGrant grant_at(std::span in, std::size_t at) { WireGrant g{}; @@ -157,6 +199,21 @@ std::vector encode_session(const SessionMessage& message) { put(out, m.tick, 8); return out; } + // Invalid text fields are encoded as given; the receiver refuses them. + std::vector operator()(const HandoffOffer& m) const { + auto out = header(SessionMessageType::handoff_offer, handoff_offer_bytes); + out.insert(out.end(), m.transfer.begin(), m.transfer.end()); + put_address(out, m.destination); + out.insert(out.end(), m.ticket.begin(), m.ticket.end()); + out.resize(session_header_bytes + handoff_offer_bytes); // Overlong text truncates. + return out; + } + std::vector operator()(const SessionRedirect& m) const { + auto out = header(SessionMessageType::redirect, redirect_bytes); + put_address(out, m.owner); + out.resize(session_header_bytes + redirect_bytes); + return out; + } }; return std::visit(Encoder{}, message); } @@ -306,6 +363,26 @@ decode_session(std::span bytes) { return std::unexpected(SessionCodecError::wrong_length); return SnapshotAck{get(payload, 0, 8), get(payload, 8, 8)}; } + case static_cast(SessionMessageType::handoff_offer): { + if (!expect(handoff_offer_bytes)) + return std::unexpected(SessionCodecError::wrong_length); + HandoffOffer m; + std::copy_n(payload.begin(), m.transfer.size(), m.transfer.begin()); + auto destination = address_at(payload, 16); + if (!destination || std::ranges::all_of(m.transfer, [](auto b) { return b == 0; })) + return std::unexpected(SessionCodecError::invalid_value); + m.destination = std::move(*destination); + m.ticket = grant_at(payload, 16 + address_bytes); + return m; + } + case static_cast(SessionMessageType::redirect): { + if (!expect(redirect_bytes)) + return std::unexpected(SessionCodecError::wrong_length); + auto owner = address_at(payload, 0); + if (!owner) + return std::unexpected(SessionCodecError::invalid_value); + return SessionRedirect{std::move(*owner)}; + } default: return std::unexpected(SessionCodecError::unknown_type); } diff --git a/src/persistence/region_lease_store.cpp b/src/persistence/region_lease_store.cpp index d250254..bda5d68 100644 --- a/src/persistence/region_lease_store.cpp +++ b/src/persistence/region_lease_store.cpp @@ -721,4 +721,31 @@ find_probe_transfer(Postgres& db, RegionKey key, const LeaseGrant& lease, const return std::unexpected{ProbeStoreFailure{ProbeStoreError::unknown, {}}}; return t; } + +std::expected, ProbeStoreFailure> +incoming_probe_transfers(Postgres& db, RegionKey key, const LeaseGrant& lease, + milliseconds timeout) { + const std::array p{std::to_string(key.world), std::to_string(key.region), + uuid_text(lease.holder), std::to_string(lease.epoch)}; + auto rows = db.query("SELECT * FROM evolution.incoming_probe_transfers($1::evolution.u64, " + "$2::evolution.u64, $3::uuid, $4::evolution.u64)", + p, timeout); + if (!rows) + return std::unexpected{probe_failure(rows.error())}; + std::vector out; + for (const auto& r : *rows) { + if (r.size() != 6) + return std::unexpected{ProbeStoreFailure{ProbeStoreError::unknown, {}}}; + const auto transfer = parse_uuid(r[0]); + const auto account = parse_uuid(r[1]); + const auto source = parse_integer(r[2]); + const auto x = parse_signed(r[3]); + const auto y = parse_signed(r[4]); + const auto z = parse_signed(r[5]); + if (!transfer || !account || !source || !x || !y || !z) + return std::unexpected{ProbeStoreFailure{ProbeStoreError::unknown, {}}}; + out.push_back({*transfer, *account, *source, *x, *y, *z}); + } + return out; +} } // namespace evo::persistence diff --git a/src/server/session/postgres_handoff_ledger.cpp b/src/server/session/postgres_handoff_ledger.cpp index d463cbf..644376b 100644 --- a/src/server/session/postgres_handoff_ledger.cpp +++ b/src/server/session/postgres_handoff_ledger.cpp @@ -27,7 +27,7 @@ HandoffLedgerError ledger_error(const persistence::ProbeStoreFailure& f) { } } // namespace -std::expected +std::expected PostgresHandoffLedger::admit(const network::TicketClaims& ticket, std::uint64_t session) { const auto lease = lease_(); if (!lease) @@ -65,6 +65,108 @@ PostgresHandoffLedger::admit(const network::TicketClaims& ticket, std::uint64_t return std::unexpected(lease_fenced(control.error().cause) ? HandoffLedgerError::fenced : HandoffLedgerError::refused); } - return network::ControlLease{control->generation, control->wait, control->granted}; + return network::HandoffAdmission{ + network::ControlLease{control->generation, control->wait, control->granted}, + committed->generation + 1, committed->x, committed->y}; +} + +namespace { +ProbeLedgerError probe_error(const persistence::ProbeStoreFailure& f) { + return f.kind == ProbeStoreError::fenced && lease_fenced(f.cause) + ? ProbeLedgerError::fenced + : ProbeLedgerError::unavailable; +} +} // namespace + +std::expected +PostgresProbeLedger::open(const network::AccountId& account, std::int64_t x, std::int64_t y, + std::int64_t z) { + const auto lease = lease_(); + if (!lease) + return std::unexpected(ProbeLedgerError::fenced); + auto* db = db_->get(); + if (!db) + return std::unexpected(ProbeLedgerError::unavailable); + const auto r = + persistence::open_probe(*db, key_, *lease, account.bytes, x, y, z, inventory, timeout_); + if (!r) { + if (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown) + db_->broken(); + return std::unexpected(probe_error(r.error())); + } + return ProbeHome{r->region, r->generation, r->x, r->y, r->transfer.has_value()}; +} + +void PostgresProbeLedger::save(const network::AccountId& account, std::uint64_t generation, + std::int64_t x, std::int64_t y, std::int64_t z) { + const auto lease = lease_(); + if (!lease) + return; + auto* db = db_->get(); + if (!db) + return; + if (const auto r = persistence::save_probe(*db, key_, *lease, account.bytes, generation, x, y, + z, timeout_); + !r && + (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown)) + db_->broken(); +} + +std::expected +PostgresProbeLedger::prepare(const network::AccountId& account, std::uint64_t generation, + const network::TransferId& transfer, std::uint64_t destination, + std::int64_t x, std::int64_t y, std::int64_t z) { + const auto lease = lease_(); + if (!lease) + return std::unexpected(ProbeLedgerError::fenced); + auto* db = db_->get(); + if (!db) + return std::unexpected(ProbeLedgerError::unavailable); + const auto r = persistence::prepare_probe_transfer(*db, key_, *lease, account.bytes, generation, + transfer, destination, x, y, z, timeout_); + if (!r) { + if (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown) + db_->broken(); + return std::unexpected(probe_error(r.error())); + } + if (r->state != persistence::TransferState::prepared) + return std::unexpected(ProbeLedgerError::unavailable); // A retry found it decided. + return {}; +} + +std::expected +PostgresProbeLedger::abort(const network::TransferId& transfer) { + const auto lease = lease_(); + if (!lease) + return std::unexpected(ProbeLedgerError::fenced); + auto* db = db_->get(); + if (!db) + return std::unexpected(ProbeLedgerError::unavailable); + const auto r = persistence::abort_probe_transfer(*db, key_, *lease, transfer, timeout_); + if (!r) { + if (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown) + db_->broken(); + return std::unexpected(probe_error(r.error())); + } + return r->state == persistence::TransferState::aborted; +} + +std::expected, ProbeLedgerError> PostgresProbeLedger::incoming() { + const auto lease = lease_(); + if (!lease) + return std::unexpected(ProbeLedgerError::fenced); + auto* db = db_->get(); + if (!db) + return std::unexpected(ProbeLedgerError::unavailable); + const auto r = persistence::incoming_probe_transfers(*db, key_, *lease, timeout_); + if (!r) { + if (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown) + db_->broken(); + return std::unexpected(probe_error(r.error())); + } + std::vector out; + for (const auto& t : *r) + out.push_back({t.transfer, t.x, t.y}); + return out; } } // namespace evo::server diff --git a/src/server/session/region_sessions.cpp b/src/server/session/region_sessions.cpp index e7ef3b7..e469ff1 100644 --- a/src/server/session/region_sessions.cpp +++ b/src/server/session/region_sessions.cpp @@ -80,6 +80,12 @@ struct RegionSessions::Impl { std::uint64_t generation{}; Clock::time_point control_deadline, next_renew; network::SessionWelcome welcome; + // Durable probe (probe records only): its ownership generation and position. + std::optional probe; + }; + struct Outgoing { + network::TransferId transfer{}; + std::uint64_t deadline{}; // Wall seconds: abort unless committed by then. }; Impl(RegionSessionsConfig c, std::unique_ptr e, @@ -96,7 +102,11 @@ struct RegionSessions::Impl { network::SessionControl control; network::IntentAdmission intents; network::ReplicationServer replication; - network::motion::ProbeMarkers markers; // Ephemeral probe markers (ADR-0094). + network::motion::ProbeMarkers markers; // Probe markers (ADR-0094, durable: ADR-0096). + std::optional own_area; // This region's part of the marker area. + std::int64_t spawn_x{}, spawn_y{}; // Where new probes start. + std::vector outgoing; // Handed off, not yet known committed. + Clock::time_point next_ghosts{}; std::optional default_interest; HostFacts facts; std::vector connections; @@ -140,8 +150,10 @@ struct RegionSessions::Impl { if (c.state == State::established) { intents.close(c.session); replication.close(c.session); + save_probe(c); markers.close(c.session); } + c.probe.reset(); if (c.session != 0) // May already be released (superseded, expired, epoch). (void)control.close(c.session).has_value(); // A superseded session's durable control already moved to its successor. @@ -153,6 +165,49 @@ struct RegionSessions::Impl { c.deadline = now + config.close_flush; } + // Best effort, at session end: the durable probe's position (a frozen or moved probe is + // fenced by the store and stays as it was). + void save_probe(const Connection& c) { + if (!config.probes || !c.probe) + return; + if (const auto* m = markers.of(c.session); m && !m->frozen && own_area) + config.probes->save(c.account_id, c.probe->generation, m->state.x, m->state.y, + own_area->z); + } + // After the durable control claim of a grant login: where the account's probe lives. + // False when the session must not open here (the connection is then closed). + bool place_probe(Connection& c, const network::AccountId& account, + const std::optional& lease, Clock::time_point now) { + if (!config.probes || !own_area) + return true; + const auto home = config.probes->open(account, spawn_x, spawn_y, own_area->z); + const auto refuse = [&](SessionCloseReason why) { + if (lease) + config.ledger->release(account, lease->generation); + close(c, why, now); + return false; + }; + if (!home) + return refuse(home.error() == ProbeLedgerError::fenced ? SessionCloseReason::shutdown + : SessionCloseReason::capacity); + if (home->owner != config.control.region) { + const auto n = std::ranges::find_if(config.neighbours, [&](const Neighbour& x) { + return x.address.region == home->owner; + }); + if (n != config.neighbours.end()) + send(c, network::SessionRedirect{n->address}); + return refuse(SessionCloseReason::moved); + } + if (home->frozen) // A transfer is open: decided by its commit or abort, retry later. + return refuse(SessionCloseReason::capacity); + c.probe = *home; + // After an aborted handoff the probe was saved where it crossed: back inside, so the + // new session does not cross again at once. + c.probe->x = std::clamp(c.probe->x, own_area->min_x, own_area->max_x); + c.probe->y = std::clamp(c.probe->y, own_area->min_y, own_area->max_y); + return true; + } + void hello(Connection& c, const network::SessionHello& m, std::uint64_t now_seconds, Clock::time_point now) { if (m.required_features != 0) @@ -178,6 +233,8 @@ struct RegionSessions::Impl { return close(c, reason_for(claimed.error()), now); lease = *claimed; } + if (!place_probe(c, claims.account, lease, now)) + return; admitted(c, control.open(*grant, now_seconds), lease, promised, claims.account, now_seconds, now); } @@ -201,7 +258,10 @@ struct RegionSessions::Impl { auto moved = config.handoff->admit(claims, promised); if (!moved) return close(c, reason_for(moved.error()), now); - lease = *moved; + lease = moved->lease; + if (config.probes) + c.probe = + ProbeHome{config.control.region, moved->generation, moved->x, moved->y, false}; } else if (config.ledger) { return close(c, SessionCloseReason::capacity, now); // Misconfigured: fail closed. } @@ -257,7 +317,9 @@ struct RegionSessions::Impl { if (default_interest) (void)replication.set_interest(c.session, *default_interest).has_value(); if (markers.area()) // Full or refused: the session simply has no marker. - (void)markers.open(c.session).has_value(); + (void)(c.probe ? markers.open_at(c.session, c.probe->x, c.probe->y) + : markers.open(c.session)) + .has_value(); c.state = State::established; c.welcome.server_tick = facts.simulated_tick; send(c, c.welcome); @@ -333,6 +395,95 @@ struct RegionSessions::Impl { send_datagram(c, receipt); } + [[nodiscard]] bool handoffs_enabled() const { + return config.probes != nullptr && config.handoff_key != nullptr && + !config.neighbours.empty() && own_area.has_value(); + } + // Markers that left this region's area: into a neighbour's area they are handed over, + // anywhere else they are put back at the edge. + void crossings(std::uint64_t now_seconds, Clock::time_point now) { + if (!own_area) + return; + const auto inside = [](const network::motion::Area& a, std::int64_t x, std::int64_t y) { + return x >= a.min_x && x <= a.max_x && y >= a.min_y && y <= a.max_y; + }; + for (auto& c : connections) { + if (c.state != State::established) + continue; + const auto* m = markers.of(c.session); + if (m == nullptr || m->frozen || inside(*own_area, m->state.x, m->state.y)) + continue; + const auto n = std::ranges::find_if(config.neighbours, [&](const Neighbour& x) { + return inside(x.area, m->state.x, m->state.y); + }); + if (n == config.neighbours.end() || !handoffs_enabled() || !c.probe) { + const auto x = std::clamp(m->state.x, own_area->min_x, own_area->max_x); + const auto y = std::clamp(m->state.y, own_area->min_y, own_area->max_y); + markers.place(c.session, x, y); // Back at the edge (`m` is stale after this). + continue; + } + hand_off(c, *n, now_seconds, now); + } + } + // The source side of a crossing: freeze, prepare durably, release control, offer the + // ticket, close. On any failure the probe stays here, at the edge. + void hand_off(Connection& c, const Neighbour& n, std::uint64_t now_seconds, + Clock::time_point now) { + if (!c.probe || !own_area) // Checked by the caller (handoffs_enabled, c.probe). + return; + const auto generation = c.probe->generation; + const auto z = own_area->z; + markers.freeze(c.session); + const auto* m = markers.of(c.session); + network::TransferId transfer{}; + if (m == nullptr || !network::random_bytes(transfer)) + return close(c, SessionCloseReason::shutdown, now); + const auto x = m->state.x, y = m->state.y; + if (const auto prepared = config.probes->prepare(c.account_id, generation, transfer, + n.address.region, x, y, z); + !prepared) + return close(c, SessionCloseReason::shutdown, now); + c.probe.reset(); // Frozen under the transfer: never saved here again. + network::TicketClaims claims; + claims.transfer = transfer; + claims.account = c.account_id; + claims.world = config.control.world; + claims.source = config.control.region; + claims.destination = n.address.region; + claims.expires_at = static_cast(now_seconds + config.ticket_lifetime); + claims.session_expires_at = static_cast(c.welcome.expires_at); + const auto ticket = config.handoff_key->sign(claims); + // Control goes first so the destination need not wait it out. + if (config.ledger && c.generation != 0) + config.ledger->release(c.account_id, c.generation); + c.generation = 0; + outgoing.push_back({transfer, now_seconds + config.ticket_lifetime + 5}); + if (ticket) + send(c, network::HandoffOffer{transfer, n.address, *ticket}); + close(c, SessionCloseReason::handed_off, now); + } + // Transfers whose client never arrived are aborted (the probe resumes here); committed + // ones are forgotten. Undecided calls retry next poll. + void expire_outgoing(std::uint64_t now_seconds) { + std::erase_if(outgoing, [&](const Outgoing& o) { + if (now_seconds < o.deadline) + return false; + return config.probes->abort(o.transfer).has_value(); + }); + } + void refresh_ghosts(Clock::time_point now) { + if (now < next_ghosts) + return; + next_ghosts = now + config.ghost_poll; + const auto incoming = config.probes->incoming(); + if (!incoming) + return; // Keep the current ghosts until the next answer. + std::vector ghosts; + for (const auto& p : *incoming) + ghosts.push_back({p.transfer, p.x, p.y}); + markers.set_ghosts(ghosts); + } + void deliver(const std::vector& outcomes) { for (const auto& o : outcomes) if (const auto session = intents.session_of(o.account)) @@ -394,6 +545,8 @@ std::expected, RegionSessionsError> RegionSessions::create(RegionSessionsConfig config) { // Connections beyond sessions are headroom for logins and closes still flushing. if (config.issuer_keys.empty() || (config.handoff != nullptr && config.ledger == nullptr) || + (config.handoff_key != nullptr && config.probes == nullptr) || + config.ticket_lifetime == 0 || config.ticket_lifetime > network::max_ticket_lifetime || config.transport.limits.max_connections < config.control.max_sessions || config.hello_deadline.count() <= 0 || config.close_flush.count() <= 0 || config.renew_interval.count() <= 0 || config.safety_margin.count() <= 0) @@ -451,8 +604,14 @@ void RegionSessions::poll(const HostFacts& facts, std::uint64_t now_seconds, m.markers.clear(); // The new epoch may restart at an earlier tick. } // Markers reach the simulated tick before this poll's intentions are admitted. - if (facts.ready) + if (facts.ready) { m.markers.advance(facts.simulated_tick); + m.crossings(now_seconds, now); + if (m.config.probes) { + m.expire_outgoing(now_seconds); + m.refresh_ghosts(now); + } + } for (auto& e : m.endpoint->poll()) { using network::quic::EventKind; if (e.kind == EventKind::connected) { @@ -471,6 +630,7 @@ void RegionSessions::poll(const HostFacts& facts, std::uint64_t now_seconds, if (it->state == Impl::State::established) { m.intents.close(it->session); m.replication.close(it->session); + m.save_probe(*it); m.markers.close(it->session); } if (it->state == Impl::State::pending) { @@ -563,8 +723,18 @@ network::ReplicationStats RegionSessions::replication_stats() const { bool RegionSessions::set_motion_area(const network::motion::Area& area, std::int64_t spawn_x, std::int64_t spawn_y) { auto& m = *impl_; - if (!m.markers.set_area(area, spawn_x, spawn_y)) + auto movement = area; + for (const auto& n : m.config.neighbours) { + movement.min_x = std::min(movement.min_x, n.area.min_x); + movement.min_y = std::min(movement.min_y, n.area.min_y); + movement.max_x = std::max(movement.max_x, n.area.max_x); + movement.max_y = std::max(movement.max_y, n.area.max_y); + } + if (!network::motion::valid(area) || !m.markers.set_area(movement, spawn_x, spawn_y)) return false; + m.own_area = area; + m.spawn_x = std::clamp(spawn_x, area.min_x, area.max_x); + m.spawn_y = std::clamp(spawn_y, area.min_y, area.max_y); for (const auto& c : m.connections) if (c.state == Impl::State::established && !m.markers.of(c.session)) (void)m.markers.open(c.session).has_value(); diff --git a/tests/network_control.cpp b/tests/network_control.cpp index b9c5fb4..fcf4807 100644 --- a/tests/network_control.cpp +++ b/tests/network_control.cpp @@ -363,12 +363,12 @@ void session_codec() { mutated.at(0) ^= 1; CHECK(refused(decode_session(mutated), SessionCodecError::bad_magic)); mutated = bytes; - mutated.at(4) = 4; // A future version. + mutated.at(4) = 5; // A future version. CHECK(refused(decode_session(mutated), SessionCodecError::unsupported_version)); mutated = bytes; mutated.at(7) = 1; CHECK(refused(decode_session(mutated), SessionCodecError::reserved_flags)); - for (const auto type : std::array{0, 16, 255}) { + for (const auto type : std::array{0, 18, 255}) { mutated = bytes; mutated.at(6) = type; CHECK(refused(decode_session(mutated), SessionCodecError::unknown_type)); @@ -377,7 +377,7 @@ void session_codec() { CHECK(std::get(*decode_session(encode_session(welcome))) == welcome); CHECK(std::get(*decode_session(encode_session(SessionHello{0, grant}))).grant == grant); - for (const auto reason : std::array{0, 14, 200, 255}) { + for (const auto reason : std::array{0, 16, 200, 255}) { auto close = encode_session(SessionClose{SessionCloseReason::protocol}); close.back() = reason; CHECK(refused(decode_session(close), SessionCodecError::invalid_value)); @@ -708,11 +708,45 @@ void handoff_tickets() { CHECK(control.open(dev.verified(claims(alice, 4)), now + 2)); } +// Session protocol v4 (ADR-0096): HandoffOffer and SessionRedirect. +void handoff_codec() { + HandoffOffer offer; + offer.transfer.fill(0x5C); + offer.destination = {12, "void.aeternalisgames.com:7402", "dev-region.evolution.test"}; + offer.ticket.fill(0xA1); + const auto bytes = encode_session(offer); + const auto decoded = decode_session(bytes); + CHECK(decoded && std::get(*decoded) == offer); + const SessionRedirect redirect{{12, "10.0.0.2:7400", "region-12.test"}}; + const auto r = decode_session(encode_session(redirect)); + CHECK(r && std::get(*r) == redirect); + CHECK(decode_session(encode_session(SessionClose{SessionCloseReason::handed_off}))); + CHECK(decode_session(encode_session(SessionClose{SessionCloseReason::moved}))); + auto beyond = encode_session(SessionClose{SessionCloseReason::moved}); + beyond.back() = static_cast(SessionCloseReason::moved) + 1; + CHECK(refused(decode_session(beyond), SessionCodecError::invalid_value)); + // Text fields: printable, no spaces, padded with zeros; region and transfer nonzero. + for (const auto& bad : {RegionAddress{12, "has space:1", "n"}, RegionAddress{12, "", "n"}, + RegionAddress{0, "h:1", "n"}, RegionAddress{12, "h:1", ""}}) + CHECK(refused(decode_session(encode_session(SessionRedirect{bad})), + SessionCodecError::invalid_value)); + auto padded = encode_session(redirect); + padded.at(session_header_bytes + 8 + 20) = 'x'; // A byte after the endpoint's end. + CHECK(refused(decode_session(padded), SessionCodecError::invalid_value)); + auto zero = offer; + zero.transfer.fill(0); + CHECK(refused(decode_session(encode_session(zero)), SessionCodecError::invalid_value)); + auto longer = bytes; + longer.push_back(0); + CHECK(refused(decode_session(longer), SessionCodecError::wrong_length)); +} + void control_tests() { grants(); identity_adapter(); control_rules(); handoff_tickets(); + handoff_codec(); session_codec(); intent_admission(); durable_actions(); diff --git a/tests/network_motion.cpp b/tests/network_motion.cpp index c551c15..f2be1b1 100644 --- a/tests/network_motion.cpp +++ b/tests/network_motion.cpp @@ -306,7 +306,7 @@ void replicated_markers() { const auto second = encode_record(m, -7, &area); two_own.insert(two_own.end(), second.begin(), second.end()); auto bad_flag = encode_record(m, -7, nullptr); - bad_flag.back() = 2; + bad_flag.back() = 3; // 0 other, 1 own, 2 ghost (v4). m.state.vector = {2000, 0}; const auto bad_vector = encode_record(m, -7, nullptr); m = {}; @@ -316,6 +316,52 @@ void replicated_markers() { CHECK(refused(c.accept(SnapshotChunk{5, ++t, 0, 0, 0, 0, -7, 0, 1, records}), AssemblyError::malformed)); } +// Ghosts (ADR-0096): read-only markers of probes being handed to this region, stable per +// transfer, replicated as kind 2 to every session whose square holds them; frozen markers +// neither move nor take intentions; place and open_at clamp into the area. +void ghosts_and_freezing() { + ProbeMarkers m; + CHECK(m.set_area(area, 0, 0)); + const auto own = m.open_at(1, 9'000'000, 100); + CHECK(own && m.of(1)->state.x == 5'000'000 && m.of(1)->state.y == 100); + m.place(1, -9'000'000, 7); + CHECK(m.of(1)->state == State{-5'000'000, 7, {}}); + CHECK(m.schedule(1, 1, 5, {1000, 0})); + m.freeze(1); + CHECK(m.of(1)->frozen && refused(m.schedule(1, 2, 6, {1000, 0}), motion::MarkerError::invalid)); + m.advance(10); + CHECK(m.of(1)->state.x == -5'000'000); // Frozen: the scheduled motion never happened. + motion::Ghost g{}; + g.transfer.fill(0x61); + g.x = 100'000; + m.set_ghosts(std::span{&g, 1}); + const auto ghost_id = std::ranges::find_if(m.markers(), &motion::Marker::ghost)->id; + m.set_ghosts(std::span{&g, 1}); // Same transfer, same id. + CHECK(std::ranges::count_if(m.markers(), &motion::Marker::ghost) == 1 && + std::ranges::find_if(m.markers(), &motion::Marker::ghost)->id == ghost_id && + !m.of(0)); // Ghosts belong to no session. + // Through replication and the codec: kind 2, not own. + constexpr world::WorldIdentity identity{11, 12}; + auto made = ReplicationServer::create({}); + CHECK(made); + auto server = std::move(*made); + CHECK(server.open(1, 5) && server.set_interest(1, {0, 0, -7, 7})); + const auto out = server.tick(1, + [&](const SessionInterest&) { + ReplicationSource s; + s.identity = identity; + s.markers = m.markers(); + s.marker_area = m.area(); + return s; + }, + {}); + SnapshotAssembler a{identity, 5}; + CHECK(!deliver(a, chunks_of(out, 1)).empty()); + const auto* seen = marker(a, ghost_id); + CHECK(seen && seen->ghost && !seen->own && seen->state.x == 100'000); + m.set_ghosts({}); + CHECK(std::ranges::none_of(m.markers(), &motion::Marker::ghost)); +} } // namespace void motion_tests() { @@ -325,5 +371,6 @@ void motion_tests() { predictor_rules(); smoothing_and_lead(); replicated_markers(); + ghosts_and_freezing(); } } // namespace evo::network_test diff --git a/tests/network_replication.cpp b/tests/network_replication.cpp index eeb6750..e61a21f 100644 --- a/tests/network_replication.cpp +++ b/tests/network_replication.cpp @@ -121,8 +121,8 @@ void codec_v2() { SnapshotChunk chunk{5, 9, 7, 3, 4, mask, -7, 1, 2, {1, 2, 3}}; auto bytes = encode_session(chunk); CHECK(std::get(*decode_session(bytes)) == chunk); - // Versions 1 and 2 (frozen, deployed) are no longer accepted (v3, ADR-0094). - for (const std::uint8_t old : {std::uint8_t{1}, std::uint8_t{2}}) { + // Versions 1-3 (frozen, deployed) are no longer accepted (v4, ADR-0096). + for (const std::uint8_t old : {std::uint8_t{1}, std::uint8_t{2}, std::uint8_t{3}}) { auto previous = encode_session(ack); previous.at(4) = old; CHECK(refused(decode_session(previous), SessionCodecError::unsupported_version)); diff --git a/tests/probe_handoff_store_tests.cpp b/tests/probe_handoff_store_tests.cpp index b642df9..4ebcbd0 100644 --- a/tests/probe_handoff_store_tests.cpp +++ b/tests/probe_handoff_store_tests.cpp @@ -188,6 +188,14 @@ void ownership_and_happy_path(World& w) { CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t1, 5s)) == TransferState::staged); CHECK(state_of(stage_probe_transfer(db, w.b, w.lb, t1, 5s)) == TransferState::staged); CHECK(failed(commit_probe_transfer(db, w.c, w.lc, t1, 5s), ProbeStoreError::fenced)); + // Schema v13: the destination sees the open transfer addressed to it (for a ghost). + const auto incoming = incoming_probe_transfers(db, w.b, w.lb, 5s); + CHECK(incoming && incoming->size() == 1 && incoming->front().transfer == t1 && + incoming->front().account == alice && incoming->front().source == 1 && + incoming->front().x == 700'000); + CHECK(incoming_probe_transfers(db, w.c, w.lc, 5s) + .value_or(std::vector{1}) + .empty()); // The one switch: owner B, next generation, the frozen final state, same inventory. const auto committed = commit_probe_transfer(db, w.b, w.lb, t1, 5s); CHECK(state_of(committed) == TransferState::committed); @@ -199,6 +207,9 @@ void ownership_and_happy_path(World& w) { CHECK(stage_probe_transfer(db, w.b, w.lb, t1, 5s) == *committed); CHECK(prepare_probe_transfer(db, w.a, w.la, alice, 1, t1, 2, 0, 0, 0, 5s) == *committed); CHECK(abort_probe_transfer(db, w.a, w.la, t1, 5s) == *committed); + CHECK(incoming_probe_transfers(db, w.b, w.lb, 5s) + .value_or(std::vector{1}) + .empty()); // Decided transfers are no longer incoming. const auto found = find_probe_transfer(db, w.c, w.lc, t1, 5s); CHECK(found && *found == std::optional{*committed}); const auto missing = find_probe_transfer(db, w.c, w.lc, id(0x7F), 5s); @@ -396,12 +407,13 @@ void handoff_ledger(World& w) { ticket.source = 1; ticket.destination = 2; const auto admitted = ledger.admit(ticket, 41); - CHECK(admitted && admitted->generation == 2 && admitted->wait == 0ms); + CHECK(admitted && admitted->lease.generation == 2 && admitted->lease.wait == 0ms && + admitted->generation == 2 && admitted->x == 3 && admitted->y == 4); const auto r = w.open(w.b, w.lb, frank); CHECK(r.region == 2 && r.generation == 2 && r.x == 3 && !r.transfer); // A retried login after an undecided outcome converges (control moves on again). const auto again = ledger.admit(ticket, 42); - CHECK(again && again->generation == 3); + CHECK(again && again->lease.generation == 3); // Refusals: another account, an unknown transfer, an aborted transfer; no lease. auto wrong = ticket; wrong.account.bytes = id(0x48); diff --git a/tests/quic_sessions.cpp b/tests/quic_sessions.cpp index 5dadd6e..6667e59 100644 --- a/tests/quic_sessions.cpp +++ b/tests/quic_sessions.cpp @@ -6,6 +6,7 @@ #include "evo/network/dev_credentials.hpp" #include "evo/network/handoff_ticket.hpp" #include "evo/network/session_grant.hpp" +#include "evo/server/probe_ledger.hpp" #include "evo/server/region_sessions.hpp" #include #include @@ -14,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -311,7 +313,7 @@ void login_rules(const Files& f) { CHECK(raw_peer(h, std::nullopt) == Reason::timeout); CHECK(raw_peer(h, std::vector{1, 2, 3}) == Reason::protocol); auto future = encode_session(SessionHello{0, h.issuer.grant(carol, 2, start + 60)}); - future.at(4) = 4; + future.at(4) = 5; CHECK(raw_peer(h, future) == Reason::unsupported_version); CHECK(raw_peer(h, encode_session(SessionClose{Reason::client_closed})) == Reason::protocol); CHECK(raw_peer(h, encode_session(SessionHello{1, h.issuer.grant(carol, 3, start + 60)})) == @@ -552,12 +554,13 @@ void snapshots_over_quic(const Files& f) { // M10.8b: a Hello carrying a handoff ticket (ADR-0096) instead of a grant. struct FakeHandoff final : HandoffLedger { - std::expected admit(const TicketClaims& ticket, - std::uint64_t session) override { + std::expected admit(const TicketClaims& ticket, + std::uint64_t session) override { if (error) return std::unexpected(*error); admitted.emplace_back(ticket, session); - return ControlLease{100 + admitted.size(), {}, std::chrono::milliseconds{20000}}; + return HandoffAdmission{ + ControlLease{100 + admitted.size(), {}, std::chrono::milliseconds{20000}}, 2, 0, 0}; } std::optional error; std::vector> admitted; @@ -625,6 +628,317 @@ void ticket_login(const Files& f) { CHECK(a->state() == RegionSessionState::established && handoff.admitted.size() == 1); } +// M10.8c (ADR-0096): two regions with adjacent areas in one process, over a fake world +// database with the same transfer state machine as schema v12. +struct WorldDb { + struct Probe { + std::uint64_t owner{}, generation{}; + std::int64_t x{}, y{}; + std::optional transfer; + }; + enum class State : std::uint8_t { prepared, committed, aborted }; + struct Transfer { + AccountId account; + std::uint64_t source{}, destination{}, generation{}; + std::int64_t x{}, y{}; + State state{}; + }; + std::map probes; + std::map transfers; +}; +struct RegionDb final : server::ProbeLedger, HandoffLedger, ControlLedger { + RegionDb(WorldDb& w, std::uint64_t r) : world{&w}, region{r} {} + std::expected + open(const AccountId& account, std::int64_t x, std::int64_t y, std::int64_t) override { + auto [it, created] = + world->probes.try_emplace(account, WorldDb::Probe{region, 1, x, y, std::nullopt}); + const auto& p = it->second; + return server::ProbeHome{p.owner, p.generation, p.x, p.y, p.transfer.has_value()}; + } + void save(const AccountId& account, std::uint64_t generation, std::int64_t x, std::int64_t y, + std::int64_t) override { + auto& p = world->probes.at(account); + if (p.owner == region && p.generation == generation && !p.transfer) { + p.x = x; + p.y = y; + ++saves; + } + } + std::expected + prepare(const AccountId& account, std::uint64_t generation, const TransferId& transfer, + std::uint64_t destination, std::int64_t x, std::int64_t y, std::int64_t) override { + auto& p = world->probes.at(account); + if (p.owner != region || p.generation != generation || p.transfer) + return std::unexpected(server::ProbeLedgerError::unavailable); + p.transfer = transfer; + p.x = x; + p.y = y; + world->transfers[transfer] = { + account, region, destination, generation, x, y, WorldDb::State::prepared}; + return {}; + } + std::expected abort(const TransferId& transfer) override { + auto& t = world->transfers.at(transfer); + if (t.state == WorldDb::State::prepared) { + t.state = WorldDb::State::aborted; + world->probes.at(t.account).transfer.reset(); + } + return t.state == WorldDb::State::aborted; + } + std::expected, server::ProbeLedgerError> + incoming() override { + std::vector out; + for (const auto& [id, t] : world->transfers) + if (t.destination == region && t.state == WorldDb::State::prepared) + out.push_back({id, t.x, t.y}); + return out; + } + std::expected admit(const TicketClaims& ticket, + std::uint64_t) override { + const auto found = world->transfers.find(ticket.transfer); + if (found == world->transfers.end() || found->second.destination != region || + found->second.account != ticket.account || + found->second.state == WorldDb::State::aborted) + return std::unexpected(HandoffLedgerError::refused); + auto& t = found->second; + auto& p = world->probes.at(t.account); + if (t.state == WorldDb::State::prepared) { // Stage and commit: the switch. + t.state = WorldDb::State::committed; + p = {region, t.generation + 1, t.x, t.y, std::nullopt}; + } + return HandoffAdmission{ControlLease{++control, {}, std::chrono::milliseconds{20000}}, + p.generation, p.x, p.y}; + } + std::expected claim(const AccountId&, std::uint64_t, + std::uint64_t) override { + return ControlLease{++control, {}, std::chrono::milliseconds{20000}}; + } + std::expected renew(const AccountId&, + std::uint64_t) override { + return std::chrono::milliseconds{20000}; + } + void release(const AccountId&, std::uint64_t) override { + ++releases; + } + WorldDb* world; + std::uint64_t region; + std::uint64_t control{}, saves{}, releases{}; +}; +constexpr std::int64_t m_um = 1'000'000; // One metre (one cell) in micrometres. +// Region `id` owns x in [x0, x0 + 10 m), y in [0, 10 m) of slice 0. +network::motion::Area area_of(std::int64_t x0) { + return {x0, 0, x0 + (10 * m_um) - 1, (10 * m_um) - 1, 0}; +} +struct Region { + Region(const Files& f, std::uint64_t id, std::int64_t x0, WorldDb& world, + std::shared_ptr key, const GrantPublicKey& neighbour_key, + std::uint64_t neighbour, std::int64_t neighbour_x0, std::uint16_t neighbour_port, + std::uint16_t own_port = 0) + : db{world, id} { + server::RegionSessionsConfig c; + c.transport.port = own_port; + c.transport.certificate_pem_file = f.certificate.string(); + c.transport.key_pem_file = f.key.string(); + c.control.environment = DeploymentEnvironment::dev; + c.control.world = world_id; + c.control.region = id; + c.control.epoch = 5; + c.control.max_account_attempts = 16; + c.issuer_keys.push_back({1, *Issuer{}.key.public_key()}); + c.hello_deadline = std::chrono::milliseconds{2000}; + c.close_flush = std::chrono::milliseconds{300}; + c.ledger = &db; + c.handoff = &db; + c.probes = &db; + c.handoff_key = std::move(key); + c.handoff_keys.push_back( + {neighbour, static_cast(neighbour + 40), neighbour_key}); + c.neighbours.push_back( + {{neighbour, "127.0.0.1:" + std::to_string(neighbour_port), std::string{name}}, + area_of(neighbour_x0)}); + c.ticket_lifetime = 20; + c.ghost_poll = std::chrono::milliseconds{20}; + auto made = RegionSessions::create(std::move(c)); + CHECK(made.has_value()); + sessions = std::move(*made); + CHECK(sessions->set_motion_area(area_of(x0), x0 + (5 * m_um), 5 * m_um)); + sessions->set_default_interest({(x0 / m_um) + 5, 5, 0, 7}); + } + void pump(std::uint64_t now_seconds) { + ++tick; + sessions->poll({true, 5, tick}, now_seconds, Clock::now()); + sessions->replicate(tick, [](const SessionInterest&) { + ReplicationSource s; + s.identity = world_id; + return s; + }); + for (auto& e : sessions->take_events()) + events.push_back(std::move(e)); + } + RegionDb db; + std::unique_ptr sessions; + std::uint64_t tick{100}; + std::vector events; +}; +void two_region_handoff(const Files& f) { + WorldDb world; + auto key1 = std::make_shared( + *HandoffSigningKey::generate(DeploymentEnvironment::dev, 41)); + auto key2 = std::make_shared( + *HandoffSigningKey::generate(DeploymentEnvironment::dev, 42)); + // Ports are only known once listening: region 1 dials region 2 at a port fixed below. + std::uint16_t port2 = 0; + { + // Find a free port for region 2 by binding a throwaway region first. + server::RegionSessionsConfig probe; + probe.transport.certificate_pem_file = f.certificate.string(); + probe.transport.key_pem_file = f.key.string(); + probe.control.environment = DeploymentEnvironment::dev; + probe.control.world = world_id; + probe.control.region = 2; + probe.control.epoch = 5; + probe.issuer_keys.push_back({1, *Issuer{}.key.public_key()}); + auto made = RegionSessions::create(std::move(probe)); + CHECK(made.has_value()); + port2 = (*made)->port(); + } + Region r1{f, 1, 0, world, key1, *key2->public_key(), 2, 10 * m_um, port2}; + Region r2{f, 2, 10 * m_um, world, key2, *key1->public_key(), 1, 0, r1.sessions->port(), port2}; + CHECK(r2.sessions->port() == port2); + std::uint64_t now_s = start; + Issuer issuer; + std::vector clients; + const auto dial = [&](std::uint16_t port, const WireGrant& credential) { + client::RegionSessionClientConfig c; + c.transport.port = port; + c.transport.server_name = std::string{name}; + c.transport.authority_pem = f.ca_pem; + c.transport.authority_pem_file = f.ca.string(); + c.grant = credential; + auto made = RegionSessionClient::connect(std::move(c), Clock::now()); + CHECK(made.has_value()); + clients.push_back(made->get()); + return std::move(*made); + }; + const auto pump = [&](const std::function& done, + std::chrono::milliseconds limit = std::chrono::milliseconds{15000}) { + const auto end = Clock::now() + limit; + while (Clock::now() < end) { + r1.pump(now_s); + r2.pump(now_s); + for (auto* c : clients) + c->poll(Clock::now()); + if (done()) + return true; + std::this_thread::sleep_for(std::chrono::milliseconds{2}); + } + return done(); + }; + // Sends a held vector until the region admits it (ticks here are fast: a refused, late + // intention is simply sent again). + const auto walk = [&](RegionSessionClient& c, std::int16_t vx) { + return pump([&] { + for (const auto& r : c.take_intention_receipts()) + if (r.status == AdmitStatus::admitted) + return true; + static_cast(c.send_intention(c.server_tick() + 4, vx, 0)); + return false; + }); + }; + const auto own = [](const RegionSessionClient& c) -> std::optional { + if (const auto* v = c.view()) + for (const auto& m : v->markers) + if (m.own) + return m; + return std::nullopt; + }; + const auto alice = account(21), bob = account(22); + const auto grant = [&](const AccountId& who, std::uint64_t serial, std::uint64_t in_region) { + return issuer.grant(who, serial, start, 300, in_region); + }; + // Bob watches the border from region 2. + auto b = dial(r2.sessions->port(), grant(bob, 1, 2)); + CHECK(pump([&] { return b->state() == RegionSessionState::established; })); + // Alice logs in to region 1: her durable probe is created at the spawn. + auto a = dial(r1.sessions->port(), grant(alice, 1, 1)); + CHECK(pump([&] { return own(*a).has_value(); })); + CHECK(must(own(*a)).state.x == 5 * m_um && world.probes.at(alice).owner == 1); + // She walks east (100 mm per tick) to the border: frozen, prepared, offered, closed. + CHECK(walk(*a, 1000)); + std::optional offer; + CHECK(pump([&] { + if (!offer) + offer = a->take_handoff(); + return offer.has_value() && a->state() == RegionSessionState::closed; + })); + CHECK(a->close_reason() == SessionCloseReason::handed_off && + must(offer).destination.region == 2 && + must(offer).destination.endpoint == "127.0.0.1:" + std::to_string(r2.sessions->port())); + const auto& t = world.transfers.at(must(offer).transfer); + CHECK(t.state == WorldDb::State::prepared && t.x >= 10 * m_um && t.x < 11 * m_um && + t.y == 5 * m_um && world.probes.at(alice).transfer == must(offer).transfer); + // Before the switch, Bob sees her as a read-only ghost at the transferred position. + const auto ghost = [&] { + if (const auto* v = b->view()) + for (const auto& m : v->markers) + if (m.ghost && m.state.x == t.x) + return true; + return false; + }; + CHECK(pump(ghost)); + // Alice continues at region 2 with the ticket: no grant, her probe where it crossed. + auto a2 = dial(r2.sessions->port(), must(offer).ticket); + CHECK(pump([&] { return own(*a2).has_value(); })); + CHECK(must(own(*a2)).state.x == t.x && world.probes.at(alice).owner == 2 && + world.probes.at(alice).generation == 2 && + world.transfers.at(must(offer).transfer).state == WorldDb::State::committed); + CHECK(pump([&] { return !ghost(); })); // The ghost gave way to her marker. + // A login at region 1 is now redirected to region 2. + auto stale = dial(r1.sessions->port(), grant(alice, 2, 1)); + CHECK(pump([&] { return stale->state() == RegionSessionState::closed; })); + CHECK(stale->close_reason() == SessionCloseReason::moved && stale->redirect() && + must(stale->redirect()).region == 2); + // Crossing back west: region 1 owns her again, generation 3. + CHECK(walk(*a2, -1000)); + std::optional back; + CHECK(pump([&] { + if (!back) + back = a2->take_handoff(); + return back.has_value() && a2->state() == RegionSessionState::closed; + })); + CHECK(must(back).destination.region == 1); + auto a3 = dial(r1.sessions->port(), must(back).ticket); + CHECK(pump([&] { return own(*a3).has_value(); })); + CHECK(world.probes.at(alice).owner == 1 && world.probes.at(alice).generation == 3 && + must(own(*a3)).state.x < 10 * m_um); + // An abandoned handoff: Carol crosses but never arrives; after the ticket's lifetime the + // source aborts and she logs in at region 1 again, where she stopped. + const auto carol = account(23); + auto c = dial(r1.sessions->port(), grant(carol, 1, 1)); + CHECK(pump([&] { return own(*c).has_value(); })); + CHECK(walk(*c, 1000)); + std::optional unused; + CHECK(pump([&] { + if (!unused) + unused = c->take_handoff(); + return unused.has_value() && c->state() == RegionSessionState::closed; + })); + const auto frozen_at = world.transfers.at(must(unused).transfer).x; + CHECK(pump([&] { + ++now_s; // A second per round: the ticket's lifetime and the abort deadline pass. + return !world.probes.at(carol).transfer.has_value(); + })); + CHECK(world.transfers.at(must(unused).transfer).state == WorldDb::State::aborted); + auto c2 = dial(r1.sessions->port(), grant(carol, 2, 1)); + CHECK(pump([&] { return own(*c2).has_value(); })); + CHECK(must(own(*c2)).state.x <= 10 * m_um - 1 && world.probes.at(carol).x == frozen_at); + // The late ticket is refused: expired by now (and its transfer aborted). + auto late = dial(r2.sessions->port(), must(unused).ticket); + CHECK(pump([&] { return late->state() == RegionSessionState::closed; })); + CHECK(late->close_reason() == SessionCloseReason::expired); + CHECK(b->state() == RegionSessionState::established && r1.db.releases > 0); +} + void durable_control(const Files& f) { FakeLedger ledger; Harness h{f, 4, 64, &ledger}; @@ -702,6 +1016,7 @@ int main() try { intentions_and_actions(files); snapshots_over_quic(files); ticket_login(files); + two_region_handoff(files); std::filesystem::remove_all(files.dir); std::puts("quic session tests passed"); return 0; diff --git a/tests/region_migration_tests.cpp b/tests/region_migration_tests.cpp index 2ef682a..6fdb9bd 100644 --- a/tests/region_migration_tests.cpp +++ b/tests/region_migration_tests.cpp @@ -178,6 +178,7 @@ void upgrade_across_migration_and_barrier() { " -v migration_sha_10=$(s 0010_account_control.sql)" " -v migration_sha_11=$(s 0011_region_actions.sql)" " -v migration_sha_12=$(s 0012_probe_handoff.sql)" + " -v migration_sha_13=$(s 0013_probe_incoming.sql)" " -v runtime_role=evo_runtime_test -f migrations/apply.sql" " > /dev/null 2>&1") != 0); CHECK(history(*owner_db) == "1,2,3,4,5,6,7"); @@ -193,7 +194,7 @@ void upgrade_across_migration_and_barrier() { CHECK(shell("PGDATABASE=" + database + " EVO_PG_RUNTIME_ROLE=evo_runtime_test bash \"$EVO_SOURCE_ROOT/" "tools/migrate-postgres.sh\" > /dev/null") == 0); - CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8,9,10,11,12"); + CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8,9,10,11,12,13"); CHECK(renew_region_lease(*runtime, r.key, *lease, 20000ms, 5s)); confirm(publish_next(20)); // 41..60, now v8. CHECK(head.head.end_tick == 60); diff --git a/tools/ci/postgres-check.sh b/tools/ci/postgres-check.sh index eb63029..444c93a 100644 --- a/tools/ci/postgres-check.sh +++ b/tools/ci/postgres-check.sh @@ -61,7 +61,7 @@ PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" -f "$root/migrations/0001_region_bootstrap.sql" > /dev/null test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1 PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12 +test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13 # v2 heads have no payload binding: v2->v3 fails closed and leaves history at v2. v2_sha="$(sha256sum "$root/migrations/0002_region_fencing.sql")" v2_sha="${v2_sha%% *}" @@ -82,7 +82,7 @@ PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '01000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32), 'hex'))" > /dev/null PGDATABASE=v3_commits bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 12:1,2,3,4,5,6,7,8,9,10,11,12 +test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 13:1,2,3,4,5,6,7,8,9,10,11,12,13 test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) FROM evolution.region_commits")" = 1 # Review P1: a v4 database holding a published (possibly terminal) checkpoint has # no durable decisions for it. The v5 upgrade must refuse before changing anything. @@ -136,7 +136,7 @@ PGDATABASE=v6_populated psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '02000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32) || repeat('00', 80), 'hex'))" > /dev/null PGDATABASE=v6_populated bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12 +test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11,12,13 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT count(*) FROM evolution.region_commits c, LATERAL evolution.parse_checkpoint_manifest(c.manifest) m WHERE m.commit_sequence = 0 AND m.predecessor_commit IS NULL")" = 1 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(conname, ',') FROM pg_constraint WHERE conrelid = 'evolution.region_heads'::regclass AND contype = 'c' AND pg_get_constraintdef(oid) LIKE '%end_tick%'")" = region_heads_published_tick # Inherited privileges bypass fencing, so every migrate run audits effective rights. @@ -181,9 +181,11 @@ expect_audit_failure 'GRANT SELECT ON evolution.account_controls TO evo_runtime_ expect_audit_failure 'REVOKE EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) FROM evo_runtime_test' 'GRANT EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) TO evo_runtime_test' bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again. # Unknown future versions and checksum drift must not be repaired silently. -psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=13 WHERE version=12' +psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=14 WHERE version=13' +if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=13 WHERE version=14" +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=13" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi -psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=12 WHERE version=13" psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=12" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=11" diff --git a/tools/ci/upgrade-check.sh b/tools/ci/upgrade-check.sh index 766ee10..be1e822 100644 --- a/tools/ci/upgrade-check.sh +++ b/tools/ci/upgrade-check.sh @@ -4,7 +4,7 @@ # exported). Real evo-server processes on world 500: # 1. Failed migration: a migration that takes the migration lock, alters a history # table, holds its locks for 3 s and then fails runs while build b1 serves. -# Nothing it did remains (schema still v12, no new column); the host keeps its +# Nothing it did remains (schema still v13, no new column); the host keeps its # authority, and every acknowledgement stays durable. # 2. Barrier (ADR-0068 operator order): build b2 is retained, and a b2 host is started # while b1 still holds the lease (it waits, never ready). upgrade_barrier(b1) fences @@ -61,7 +61,7 @@ SQL [ "$code" != 0 ] && grep -q 'division by zero' "$work/migration.log" || fail "the rehearsal migration did not fail" [ "$(q -c "SELECT count(*) FROM information_schema.columns WHERE table_name = 'region_commits' AND column_name = 'rehearsal_column'")" = 0 ] || fail "the failed migration left a column" -[ "$(q -c 'SELECT max(version) FROM evolution.schema_migrations')" = 12 ] || fail "schema version moved" +[ "$(q -c 'SELECT max(version) FROM evolution.schema_migrations')" = 13 ] || fail "schema version moved" wait_ack "$work/b1.log" $((before + 40)) || fail "b1 did not resume after the failed migration" kill -0 "$b1_pid" 2> /dev/null || fail "b1 exited during the failed migration" bash "$root/tools/migrate-postgres.sh" > /dev/null || fail "migrate after the failure" diff --git a/tools/migrate-postgres.sh b/tools/migrate-postgres.sh index a647e3e..cbabfca 100644 --- a/tools/migrate-postgres.sh +++ b/tools/migrate-postgres.sh @@ -36,11 +36,13 @@ digest_11="$(sha256sum "$root/migrations/0011_region_actions.sql")" digest_11="${digest_11%% *}" digest_12="$(sha256sum "$root/migrations/0012_probe_handoff.sql")" digest_12="${digest_12%% *}" +digest_13="$(sha256sum "$root/migrations/0013_probe_incoming.sql")" +digest_13="${digest_13%% *}" psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_2=$digest_2" --set="migration_sha_3=$digest_3" \ --set="migration_sha_4=$digest_4" --set="migration_sha_5=$digest_5" \ --set="migration_sha_6=$digest_6" --set="migration_sha_7=$digest_7" \ --set="migration_sha_8=$digest_8" --set="migration_sha_9=$digest_9" \ - --set="migration_sha_10=$digest_10" --set="migration_sha_11=$digest_11" --set="migration_sha_12=$digest_12" \ + --set="migration_sha_10=$digest_10" --set="migration_sha_11=$digest_11" --set="migration_sha_12=$digest_12" --set="migration_sha_13=$digest_13" \ --set="runtime_role=$runtime_role" \ --file="$root/migrations/apply.sql"