diff --git a/apps/evo-server/main.cpp b/apps/evo-server/main.cpp index 4e6a089..3aa7221 100644 --- a/apps/evo-server/main.cpp +++ b/apps/evo-server/main.cpp @@ -36,7 +36,9 @@ // // Dev sessions (M9.7, ADR-0083; builds with EVO_BUILD_NETWORK only): --session-listen // --session-cert F --session-key F --session-grant-key F -// --session-environment dev serve the TLS 1.3 dev session protocol. They require +// --session-grant-ledger F --session-environment dev serve the TLS 1.3 dev session +// protocol. The ledger durably records consumed grants (on a persistent volume), so a +// restarted process refuses them. They require // --synthetic-inputs: an admitted intention rides the next synthetic input (content // unchanged), so its final outcome is the input's durable acknowledgement, with no // biological effect. The driver then looks ahead 8 ticks instead of 64. Prints @@ -207,6 +209,8 @@ std::optional parse(int argc, char** argv) { o.session.key = value; else if (flag == "--session-grant-key") o.session.grant_key = value; + else if (flag == "--session-grant-ledger") + o.session.grant_ledger = value; else if (flag == "--session-environment" && (value == "dev" || value == "stage" || value == "world0")) o.session.environment = value; @@ -559,7 +563,7 @@ int main(int argc, char** argv) { "[--ops-listen ADDRESS:PORT] [--synthetic-inputs development-v1 " "[--synthetic-until TICK] [--accelerate N] [--session-listen ADDRESS:PORT " "--session-cert F --session-key F --session-grant-key F " - "--session-environment dev]]\n", + "--session-grant-ledger F --session-environment dev]]\n", stderr); return 2; } diff --git a/apps/evo-server/session_bridge.hpp b/apps/evo-server/session_bridge.hpp index 4be6660..d67c919 100644 --- a/apps/evo-server/session_bridge.hpp +++ b/apps/evo-server/session_bridge.hpp @@ -22,14 +22,14 @@ namespace evo::server_app { struct SessionOptions { - std::string listen, certificate, key, grant_key, environment; + std::string listen, certificate, key, grant_key, grant_ledger, environment; [[nodiscard]] bool any() const noexcept { return !listen.empty() || !certificate.empty() || !key.empty() || !grant_key.empty() || - !environment.empty(); + !grant_ledger.empty() || !environment.empty(); } [[nodiscard]] bool complete() const noexcept { return !listen.empty() && !certificate.empty() && !key.empty() && !grant_key.empty() && - !environment.empty(); + !grant_ledger.empty() && !environment.empty(); } }; // Intentions ride synthetic inputs, so sessions keep the driver's lookahead short. @@ -65,11 +65,15 @@ class SessionBridge { config.admission.intent_window = session_lookahead_ticks; config.admission.window_ticks = session_lookahead_ticks; config.admission.max_intents_per_window = session_lookahead_ticks; - auto server = network::SessionServer::create(std::move(config), std::move(*grant_key)); - if (!server) - return fail(server.error() == network::SessionServerError::listen - ? "error session-listen" - : "error session-credentials"); + config.grant_ledger = o.grant_ledger; + auto server = network::SessionServer::create(std::move(config), std::move(*grant_key), + wall_seconds()); + if (!server) { + const auto e = server.error(); + return fail(e == network::SessionServerError::listen ? "error session-listen" + : e == network::SessionServerError::ledger ? "error session-grant-ledger" + : "error session-credentials"); + } bridge.server_ = std::move(*server); std::printf("session listening %u\n", static_cast(bridge.server_->port())); return bridge; @@ -97,12 +101,7 @@ class SessionBridge { if (state != nullptr && facts.ready && (!published_ || *published_ != s.simulated_tick || epoch_ != s.epoch)) publish(community(*state), facts); - server_->poll( - facts, - static_cast(std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count()), - std::chrono::steady_clock::now()); + server_->poll(facts, wall_seconds(), std::chrono::steady_clock::now()); for (const auto& e : server_->take_events()) std::printf("session %s %llu subject=%llu reason=%u\n", e.kind == network::SessionEvent::Kind::opened ? "opened" @@ -114,6 +113,11 @@ class SessionBridge { } private: + static std::uint64_t wall_seconds() { + return static_cast(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count()); + } static std::optional fail(const char* message) { std::puts(message); return std::nullopt; diff --git a/cmake/Network.cmake b/cmake/Network.cmake index 6b82950..030eb81 100644 --- a/cmake/Network.cmake +++ b/cmake/Network.cmake @@ -41,7 +41,7 @@ target_link_libraries(evo_network_tls PUBLIC evo_network_protocol PRIVATE mbedtl evo_owned_target(evo_network_tls) # M9.7 sessions: non-blocking TCP, the host-agnostic session server and the client. add_library(evo_network_session STATIC src/network/socket/tcp.cpp src/network/session/server.cpp - src/network/session/client.cpp) + src/network/session/client.cpp src/network/session/grant_ledger.cpp) target_link_libraries(evo_network_session PUBLIC evo_network_tls) if(WIN32) target_link_libraries(evo_network_session PRIVATE ws2_32) diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 2dc892b..117a322 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -40,7 +40,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libx11-dev libxext-dev libxrandr-dev libxcursor-dev libxi-dev libxss-dev libxtst-dev xvfb xauth \ && rm -rf /var/lib/apt/lists/* RUN cmake -P tools/prepare-sdl3.cmake \ - && cmake --preset client-probe -DCMAKE_CXX_COMPILER=clang++-19 \ + && cmake --preset client-probe -DCMAKE_CXX_COMPILER=clang++-19 -DCMAKE_C_COMPILER=clang-19 \ -DEVO_BUILD_CLIENT=ON -DEVO_ENABLE_TIDY=ON -DSDL_X11=ON -DSDL_WAYLAND=OFF \ && cmake --build build/client-probe --target evo-client-probe evo-client evo-client-lifecycle-tests evo-client-environment-tests evo-client-organism-tests --parallel 2 \ && ctest --preset client-probe --no-tests=error \ diff --git a/deploy/kubernetes/region/base/deployment.yaml b/deploy/kubernetes/region/base/deployment.yaml index 4a7f9bd..da190b7 100644 --- a/deploy/kubernetes/region/base/deployment.yaml +++ b/deploy/kubernetes/region/base/deployment.yaml @@ -61,10 +61,13 @@ spec: # Dev sessions are a drill-time override too (tools/ops/session-drill.sh, # ADR-0083): loopback only, reached by kubectl port-forward, never a Service. # evo-server refuses any environment but dev and needs the synthetic carrier. + # Consumed grants persist on the objects volume (outside the object + # namespaces, ignored by backups) so a restarted pod refuses them. sessions=() if [ -n "$REGION_SESSION" ]; then sessions=(--session-listen 127.0.0.1:7400 --session-cert /run/session/server.pem --session-key /run/session/server.key --session-grant-key /run/session/grant.key + --session-grant-ledger /objects/dev-session-grants.v1 --session-environment "$REGION_SESSION") fi exec /usr/local/bin/evo-server --profile lineage "${retained[@]}" "${inputs[@]}" \ diff --git a/docs/adr/0083-dev-region-sessions.md b/docs/adr/0083-dev-region-sessions.md index e452a80..0a19537 100644 --- a/docs/adr/0083-dev-region-sessions.md +++ b/docs/adr/0083-dev-region-sessions.md @@ -28,9 +28,12 @@ Findings that constrain the design: **Session server in the region process.** `SessionServer` (src/network/session) is host-agnostic and non-blocking; `evo-server` calls it from its existing single-thread loop (as it does the ops server). It never mutates host state. Per connection: -bounded output (newest view replaces an unsent one; overflow drops the connection), -1 s Close flush deadline, 5 s accept-to-Welcome deadline, 64 KiB/s input limit, -session bound to its connection (a connection can only speak for its own session). +all output goes through the TLS channel's bounded queue (1 MiB; an interrupted TLS +record is retried with identical arguments, later frames wait behind it; a full queue +drops the slow consumer), the newest view replaces an unsent one, 1 s Close flush +deadline, 5 s accept-to-Welcome deadline, 64 KiB/s input limit, heartbeat after 1 s +without output and `timeout` close after 5 s without input, session bound to its +connection (a connection can only speak for its own session). At most 4 connections/sessions. New epochs close existing sessions (`stale_epoch`) and discard intentions not yet carried. @@ -42,21 +45,37 @@ committed tick as the durable watermark. Every session starts from the latest vi **Intentions.** Sessions require `--synthetic-inputs development-v1`. An admitted intention is queued and rides the next synthetic input whose tick is at or after its -intended tick: the input's request id becomes `INT1 | epoch | session | sequence` -(low 32 bits each); its **content stays the host-chosen development input**. The +intended tick under a fresh **server-issued carrier id** `'I','N' | epoch (48 bits) +| ordinal (64 bits)`, never derived from client values; the client sequence is kept +separately for replies. Epochs beyond 48 bits are refused at Hello (`capacity`) and +ordinal exhaustion stops carrying; nothing wraps or truncates. The input's **content +stays the host-chosen development input**. The host's FinalAck for that request yields `IntentOutcome{committed_no_effect, tick, commit_sequence}`; a dropped input yields `not_committed`. The driver looks ahead 8 ticks (instead of 64) when sessions are on, matching the admission window. -**Protocol v1 amendment.** Before any v1 peer was deployed, v1 gained message type 8 -`IntentOutcome` and close reason 9 `capacity` (schemas/view/dev-session-v1.md). Once -a dev deployment runs, further changes need a new version. +**Protocol v1 amendment.** Before any v1 peer was deployed, v1 gained message types 8 +`IntentOutcome` and 9 `Heartbeat` (empty) and close reasons 9 `capacity` and 10 +`timeout` (schemas/view/dev-session-v1.md). Once a dev deployment runs, further +changes need a new version. + +**Grant consumption survives restarts.** Consumed grant ids are appended to a durable +ledger (`--session-grant-ledger`; `/objects/dev-session-grants.v1` in dev, outside the +object namespaces and ignored by backups) and fsynced **before** Welcome; a failed +write refuses the session and keeps the grant consumed in memory. A replacement +process loads unexpired entries into admission at startup. Corrupt content, I/O +failure or more than 65536 entries fail closed. The ledger does not survive a +restore-to-bundle (a rewind) or a lost volume; grants expire within an hour. **Client.** `SessionClient` connects, verifies the pinned dev CA, consumes one -single-use grant per Hello, and on Welcome resets both layers (fresh baseline). On -loss it keeps the last presentation marked stale, reports outstanding admitted -intentions as `unknown`, and retries with capped exponential backoff until grants run -out (explicit `exhausted`). Intended tick = newest view tick + 4. The desktop app +single-use grant per Hello (except after `server_shutdown`/`capacity`, which the +server refuses before recording the grant), and on Welcome resets both layers (fresh +baseline). It heartbeats when idle and treats 5 s without any server frame as loss. +On loss it keeps the last presentation marked stale, reports every sent-but-unresolved +intention as `unknown` (sent without a result, or admitted without an outcome), and +retries with capped exponential backoff until grants run out (explicit `exhausted`). +`send` is bounded: at most 256 unresolved intentions and the channel queue's capacity; +beyond that it refuses rather than buffers. Intended tick = newest view tick + 4. The desktop app sends an intention only when its vector changes, at most once per view tick. `evo-client --connect ... --ca ... --grants ...` replaces the local fixture. @@ -103,6 +122,14 @@ tokens: anyone holding an unused grant (or the dev key) can open a dev session. Restart gaps: an intention admitted just before a crash has an unknown outcome until the client reconnects (reported as such, never assumed). +## Review corrections (2026-10-02) +The M9.4-M9.7 review (`.cache/reviews/M9.4-M9.7-review.md`, not in Git) reproduced +and this revision fixes: TLS retry with changed arguments losing queued frames; +carrier ids aliasing sequences 2^32 apart (the original low-32-bit packing above was +unsafe and is withdrawn); consumed grants becoming valid again after a restart; no +liveness after Welcome; and the `ci-client` configure missing its C compiler. Each has +a regression test (engineering/VALIDATION.md). + ## Consequences M9.8 packages the client and qualifies the presentation. M10 replaces the carrier with real intention semantics once organisms move, and reopens transport choice. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 11f85d4..059050a 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -3509,3 +3509,58 @@ were not executed remotely. Port-forward reachability of the pod-loopback listen is the documented design, not yet observed. M8-dev/stage remains not passed. The connected desktop run was hidden/software-rendered; no human visual check of remote mode. Intentions have no biological effect by design. + +## M9.4–M9.7 review corrections — 2026-10-02 + +Responds to the review of `bb0412c..8e31932` (`.cache/reviews/M9.4-M9.7-review.md`, +not in Git). All five findings were confirmed against the code and fixed: + +1. **[P1] TLS retry lost queued frames.** `TlsChannel::send` is replaced by a bounded + (1 MiB) `queue` + `flush`: an interrupted record is retried with identical + arguments, later frames wait behind it, `close()` flushes before close_notify, a + full queue returns `backpressure` without queuing. Server and client route every + frame through it. Regression (`interrupted_writes`): zero progress and partial + progress then zero, frames queued during the stall, both directions, exact ordered + sequences; close while stalled; backpressure. +2. **[P1] Carrier ids aliased.** Carriers are server-issued `'I','N' | 48-bit epoch | + 64-bit ordinal`; the client sequence is kept separately. Epochs beyond 48 bits are + refused at Hello; ordinal exhaustion stops carrying. ADR-0083's low-32-bit packing + is withdrawn. Regression (`carriers_never_alias`): sequences 1 and 2^32+1 get + distinct ids; `not_committed` and `committed` reach their own sequences. +3. **[P2] Grants reusable after restart.** New durable `GrantLedger` (fsync before + Welcome, atomic prune on open, fail closed on corruption/I-O/65536 entries), loaded + into admission by a replacement process; `--session-grant-ledger` (dev: + `/objects/dev-session-grants.v1`). Regressions: restart with the same ledger + refuses the consumed grant and admits a fresh one; corrupt ledger refuses server + creation; expired entries pruned; and in the end-to-end rehearsal the successor of + a SIGKILLed evo-server refused the first, consumed grant (`closed reason=3`). +4. **[P2] No liveness after Welcome.** Protocol v1 (still undeployed) gains Heartbeat + (type 9) and Close `timeout` (10). Both sides heartbeat after 1 s without output and + treat 5 s of silence as dead (server closes `timeout`, freeing the subject; client + marks stale, reports unknown outcomes, reconnects). Regressions (`liveness`): idle + but healthy session survives 1.5 s with no ticks; a silent server is detected and + reconnected to a fresh baseline; a silent client is closed and its subject reused. +5. **[P1, inherited] ci-client C compiler.** `-DCMAKE_C_COMPILER=clang-19` added; all + other C-enabling fresh configures already name it. + +Follow-ups taken: `SessionClient::send` is bounded (256 unresolved, channel queue +capacity) and refuses instead of buffering; unresolved tracking now starts at send, +so a loss before IntentResult is also reported as unknown. The abuse test now floods +through a raw TLS peer (the client can no longer flood) and checks the server's +`rate_limited` decision. Client event-vector bounds remain M9.8 work. + +Executed after the fixes: Windows MSVC `network` 1/1 and `client-network` 6/6; Linux +GCC 14 network suite and the **full** PostgreSQL integration suite (all suites, +including the session process test with the ledger and new carrier ids); Clang 19 +ASan/UBSan network suite; clang-tidy 19 (warnings as errors) on all changed network, +client and server units (one `string_view::data` finding fixed in the ledger parser); +libFuzzer on the amended codec, 523,479 runs in 61 s, no crash or canonicality abort; +repository format, CI selector fixtures and deploy policy (67 objects, 0 failures). +End-to-end rehearsal: durable ack, SIGKILL, successor Ready in 3472 ms at epoch 2, +grants kept across two `server_shutdown` refusals, second durable ack, consumed grant +refused by the successor, Windows evo-client welcomed (`session=2 epoch=2`), successor +drained with `exit 0`. + +Not done: the reviewer's own reproduction binaries target the removed `send` API and +were not rebuilt; equivalent regressions above replace them. The `ci-client` Docker +image and Gitea pipeline were not rerun here, and no cluster operation was performed. diff --git a/docs/networking/DEV_SESSION_OPERATIONS.md b/docs/networking/DEV_SESSION_OPERATIONS.md index 4fabc3c..5981ffe 100644 --- a/docs/networking/DEV_SESSION_OPERATIONS.md +++ b/docs/networking/DEV_SESSION_OPERATIONS.md @@ -37,9 +37,13 @@ kubectl -n evolution-dev create secret generic evolution-dev-session \ --from-file=grant.key=$HOME/aevolution-dev-session/grant.key ``` -The Deployment mounts it optionally (0440). Rotation: create new credentials, -replace the Secret, restart evo-region, redistribute `dev-ca.pem`. Old grants stop -verifying with the new key. +The Deployment mounts it optionally (0440). Consumed grants are recorded in +`/objects/dev-session-grants.v1` on the region's persistent volume (fsynced before +each Welcome), so a restarted pod refuses grants its predecessor used. Corrupt or +unwritable ledgers fail closed (`error session-grant-ledger`, or sessions refused). + +Rotation: create new credentials, replace the Secret, restart evo-region and +redistribute `dev-ca.pem`. Old grants stop verifying with the new key. ## 4. Session drill (enables sessions temporarily, then restores Git values) diff --git a/include/evo/network/admission.hpp b/include/evo/network/admission.hpp index 5b38d37..1d2404b 100644 --- a/include/evo/network/admission.hpp +++ b/include/evo/network/admission.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include // Server-side session admission for the M9.6 dev session (ADR-0082). Transport- @@ -17,6 +18,10 @@ struct DevGrantClaims { world::WorldIdentity world; std::uint64_t region{}, issued_at{}, expires_at{}; // Server clock, seconds. }; +// A grant already used to open a session, remembered until it expires. +struct ConsumedGrant { + std::uint64_t grant_id{}, expires_at{}; +}; class DevGrantKey; // Only DevGrantKey::verify constructs this, so admission cannot be handed // unauthenticated claims by accident. @@ -68,6 +73,10 @@ class SessionAdmission { [[nodiscard]] IntentResult admit(const IntentMessage& intent, std::uint64_t server_tick, std::uint64_t now_seconds); [[nodiscard]] std::expected close(std::uint64_t session); + // Restores grants consumed before a restart (from a durable ledger), so a replacement + // host refuses them too. Expired entries are skipped; too many fail closed. + [[nodiscard]] std::expected + remember(std::span consumed, std::uint64_t now_seconds); // New ownership epoch: existing sessions become stale and must reconnect. [[nodiscard]] std::expected advance_epoch(std::uint64_t epoch); [[nodiscard]] std::size_t sessions() const noexcept { diff --git a/include/evo/network/grant_ledger.hpp b/include/evo/network/grant_ledger.hpp new file mode 100644 index 0000000..277c85e --- /dev/null +++ b/include/evo/network/grant_ledger.hpp @@ -0,0 +1,42 @@ +#pragma once +#include "evo/network/admission.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Durable record of consumed dev grants (ADR-0083), so a replacement region process +// refuses grants its predecessor already used. Append-only text lines +// " ", flushed and fsynced before the session is welcomed. +// Opening drops expired entries by an atomic rewrite. Corrupt content, I/O failure or +// a full ledger fail closed. Single owner; not a general database. +namespace evo::network { +enum class LedgerError : std::uint8_t { io, corrupt, full }; +class GrantLedger { + public: + static constexpr std::size_t max_entries = 65536; + [[nodiscard]] static std::expected, LedgerError> + open(const std::filesystem::path& path, std::uint64_t now_seconds); + GrantLedger(const GrantLedger&) = delete; + GrantLedger& operator=(const GrantLedger&) = delete; + ~GrantLedger(); + // Unexpired entries loaded at open (for SessionAdmission::remember). + [[nodiscard]] std::span loaded() const noexcept { + return loaded_; + } + // Durable once this returns success. + [[nodiscard]] std::expected record(ConsumedGrant grant); + + private: + GrantLedger(std::FILE* file, std::vector loaded) + : file_{file}, loaded_{std::move(loaded)}, entries_{loaded_.size()} {} + std::FILE* file_; + std::vector loaded_; + std::size_t entries_; +}; +} // namespace evo::network diff --git a/include/evo/network/protocol.hpp b/include/evo/network/protocol.hpp index 5a11317..be14157 100644 --- a/include/evo/network/protocol.hpp +++ b/include/evo/network/protocol.hpp @@ -28,7 +28,8 @@ enum class MessageType : std::uint8_t { environment_view, organism_view, close, - intent_outcome + intent_outcome, + heartbeat // Empty payload; proves liveness when no other frame flows (ADR-0083). }; enum class ProtocolError : std::uint8_t { truncated, @@ -121,7 +122,8 @@ enum class CloseReason : std::uint8_t { stale_epoch, rate_limited, server_shutdown, - capacity + capacity, + timeout }; // Final, after the host's durable commit (or loss): the M9.7 dev region has no // movable organism, so a committed intention has no biological effect. @@ -138,6 +140,7 @@ struct IntentOutcome { [[nodiscard]] std::vector encode(const IntentResult& m); [[nodiscard]] std::vector encode(CloseReason reason); [[nodiscard]] std::vector encode(const IntentOutcome& m); +[[nodiscard]] std::vector encode_heartbeat(); [[nodiscard]] std::vector encode(const ViewHeader& header, const client::EnvironmentPresentation& view); [[nodiscard]] std::vector encode(const ViewHeader& header, @@ -151,6 +154,7 @@ struct IntentOutcome { [[nodiscard]] std::expected decode_intent_result(FrameView frame); [[nodiscard]] std::expected decode_close(FrameView frame); [[nodiscard]] std::expected decode_intent_outcome(FrameView frame); +[[nodiscard]] std::expected decode_heartbeat(FrameView frame); [[nodiscard]] std::expected, ProtocolError> decode_environment_view(FrameView frame); [[nodiscard]] std::expected, ProtocolError> diff --git a/include/evo/network/session_client.hpp b/include/evo/network/session_client.hpp index fb81ea5..077e85a 100644 --- a/include/evo/network/session_client.hpp +++ b/include/evo/network/session_client.hpp @@ -28,6 +28,9 @@ struct SessionClientConfig { std::uint32_t lead_ticks{4}; // Intended tick = newest view tick + lead. std::chrono::milliseconds attempt_deadline{5000}; // Connect to Welcome. std::chrono::milliseconds first_retry{500}, max_retry{8000}; + // Liveness after Welcome: heartbeat when idle; a silent server past the timeout is + // treated as lost (stale presentation, unknown outcomes, reconnect). + std::chrono::milliseconds heartbeat_interval{1000}, liveness_timeout{5000}; }; enum class SessionPhase : std::uint8_t { backoff, connecting, established, exhausted }; struct ClientEvent { @@ -56,7 +59,9 @@ class SessionClient { SessionClient& operator=(const SessionClient&) = delete; ~SessionClient(); void poll(std::chrono::steady_clock::time_point now); - // Sends when established and a view has arrived; never queued across connections. + // Queues one intention when established, a view has arrived, fewer than + // max_outstanding are unresolved and the channel queue has room; otherwise false. + // Never carried across connections. [[nodiscard]] bool send(const client::Intention& intention); [[nodiscard]] SessionPhase phase() const noexcept; [[nodiscard]] const client::EnvironmentLayer& environment() const noexcept; diff --git a/include/evo/network/session_server.hpp b/include/evo/network/session_server.hpp index 129e059..37de7c8 100644 --- a/include/evo/network/session_server.hpp +++ b/include/evo/network/session_server.hpp @@ -1,6 +1,7 @@ #pragma once #include "evo/network/admission.hpp" #include "evo/network/dev_grant.hpp" +#include "evo/network/grant_ledger.hpp" #include "evo/network/tcp.hpp" #include "evo/network/tls_channel.hpp" #include @@ -9,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -30,13 +32,18 @@ struct SessionServerConfig { std::size_t max_input_bytes_per_second{65536}; std::size_t max_pending_intents{64}; // Admitted, not yet carried. std::size_t max_carried_intents{1024}; // Carried, awaiting durable/lost. + // Liveness both ways: a Heartbeat is sent when nothing else was sent for an + // interval; a peer silent for the timeout is closed with `timeout`. + std::chrono::milliseconds heartbeat_interval{1000}, liveness_timeout{5000}; + // Durable consumed-grant ledger (required); survives process and pod restarts. + std::filesystem::path grant_ledger; }; // The host's current authority facts, read once per loop iteration. struct RegionFacts { bool ready{}; std::uint64_t epoch{}, simulated_tick{}, committed_tick{}, commit_sequence{}; }; -enum class SessionServerError : std::uint8_t { configuration, listen, credentials }; +enum class SessionServerError : std::uint8_t { configuration, listen, credentials, ledger }; struct SessionEvent { enum class Kind : std::uint8_t { opened, closed, refused } kind{}; std::uint64_t session{}, subject{}; @@ -45,7 +52,7 @@ struct SessionEvent { class SessionServer { public: [[nodiscard]] static std::expected, SessionServerError> - create(SessionServerConfig config, DevGrantKey key); + create(SessionServerConfig config, DevGrantKey key, std::uint64_t now_seconds); SessionServer(const SessionServer&) = delete; SessionServer& operator=(const SessionServer&) = delete; ~SessionServer(); @@ -59,7 +66,8 @@ class SessionServer { void publish(const client::EnvironmentPresentation& environment, const client::OrganismPresentation& organisms, const RegionFacts& facts); // The oldest admitted intention whose intended tick is <= `tick`, to ride the host - // input for `tick`. The host input's content stays host-chosen. + // input for `tick`, under a fresh server-issued carrier id ('I','N', 48-bit epoch, + // 64-bit ordinal). The host input's content stays host-chosen. [[nodiscard]] std::optional carry(std::uint64_t tick); // Host final acknowledgement / drop for a request id; ids not carried here are ignored. void committed(const RequestId& request, std::uint64_t tick, std::uint64_t commit_sequence); diff --git a/include/evo/network/tls_channel.hpp b/include/evo/network/tls_channel.hpp index e0b33ea..333e1db 100644 --- a/include/evo/network/tls_channel.hpp +++ b/include/evo/network/tls_channel.hpp @@ -37,7 +37,8 @@ enum class TlsError : std::uint8_t { peer_verification, transport, closed, - record + record, + backpressure }; enum class TlsProgress : std::uint8_t { done, want_io }; class TlsChannel { @@ -50,9 +51,17 @@ class TlsChannel { TlsChannel& operator=(const TlsChannel&) = delete; ~TlsChannel(); [[nodiscard]] std::expected handshake(); - // Zero means "would block". Errors latch until destruction. - [[nodiscard]] std::expected send(std::span bytes); + // Outbound plaintext is queued, never handed to the caller to retry: a record that + // TLS could not finish writing is retried with identical arguments (Mbed TLS + // requires this), and newly queued bytes wait behind it. Bounded; a full queue + // returns `backpressure` and queues nothing. Errors latch until destruction. + static constexpr std::size_t max_queued_bytes = std::size_t{1} << 20; + [[nodiscard]] std::expected queue(std::span bytes); + // Writes queued plaintext until done or the transport would block. + [[nodiscard]] std::expected flush(); + [[nodiscard]] std::size_t queued() const noexcept; [[nodiscard]] std::expected receive(std::span bytes); + // Flushes queued plaintext first, then sends close_notify (want_io until both done). [[nodiscard]] std::expected close(); // Negotiated "TLSv1.3" and suite name after a completed handshake. [[nodiscard]] std::string_view version() const noexcept; diff --git a/schemas/view/dev-session-v1.md b/schemas/view/dev-session-v1.md index 3657c7f..0f81584 100644 --- a/schemas/view/dev-session-v1.md +++ b/schemas/view/dev-session-v1.md @@ -12,7 +12,7 @@ message re-encodes to identical bytes (fuzz-tested). |---|---|---|---| | 0 | magic | u32 | `0x53445645` (bytes `E V D S`) | | 4 | version | u16 | exactly 1, otherwise `unsupported_version` | -| 6 | type | u8 | 1..8, otherwise `unknown_type` | +| 6 | type | u8 | 1..9, otherwise `unknown_type` | | 7 | flags | u8 | 0, otherwise `reserved_flags` | | 8 | length | u32 | payload bytes, <= 262144 (`oversized`) | | 12 | payload | bytes | exactly `length`; short `truncated`, extra `trailing_bytes` | @@ -30,15 +30,16 @@ Payload decoders also reject truncation, trailing bytes, out-of-range enum bytes | 4 | IntentResult | S->C | sequence u64, status u8 (0..7), server_tick u64 | | 5 | EnvironmentView | S->C | view header, environment presentation | | 6 | OrganismView | S->C | view header, organism presentation | -| 7 | Close | both | reason u8 (0..9) | +| 7 | Close | both | reason u8 (0..10) | | 8 | IntentOutcome | S->C | sequence u64, finality u8 (0 committed_no_effect, 1 not_committed), tick u64, commit_sequence u64; committed requires commit_sequence != 0, not_committed requires tick = commit_sequence = 0 | +| 9 | Heartbeat | both | empty (length 0); sent after 1 s without other output; 5 s of silence closes with `timeout` | IntentStatus: 0 admitted, 1 unknown_session, 2 stale_epoch, 3 expired, 4 invalid_vector, 5 replayed, 6 outside_window, 7 rate_limited. CloseReason: 0 normal, 1 unsupported_version, 2 malformed, 3 unauthenticated, 4 expired, 5 duplicate_session, 6 stale_epoch, 7 rate_limited, 8 server_shutdown, -9 capacity. Type 8 and reason 9 were added in M9.7 before any v1 peer was -deployed (ADR-0083); later changes require a new version. +9 capacity, 10 timeout. Types 8-9 and reasons 9-10 were added in M9.7 before any v1 +peer was deployed (ADR-0083); later changes require a new version. View header (40 bytes): session, epoch, server_tick, commit_watermark, baseline (u64). diff --git a/src/network/protocol/admission.cpp b/src/network/protocol/admission.cpp index d630bac..2975eeb 100644 --- a/src/network/protocol/admission.cpp +++ b/src/network/protocol/admission.cpp @@ -103,6 +103,20 @@ IntentResult SessionAdmission::admit(const IntentMessage& intent, std::uint64_t return result; } +std::expected +SessionAdmission::remember(std::span consumed, std::uint64_t now_seconds) { + for (const auto& g : consumed) { + if (g.expires_at <= now_seconds || std::ranges::any_of(grants_, [&](const Remembered& r) { + return r.grant_id == g.grant_id; + })) + continue; + if (grants_.size() >= config_.max_remembered_grants) + return std::unexpected(AdmissionError::grants_full); + grants_.push_back({g.grant_id, g.expires_at}); + } + return {}; +} + std::expected SessionAdmission::close(std::uint64_t session) { if (std::erase_if(sessions_, [&](const Session& s) { return s.id == session; }) == 0) return std::unexpected(AdmissionError::unknown_session); diff --git a/src/network/protocol/codec.cpp b/src/network/protocol/codec.cpp index 337ed41..0b82262 100644 --- a/src/network/protocol/codec.cpp +++ b/src/network/protocol/codec.cpp @@ -5,9 +5,9 @@ namespace evo::network { namespace { -constexpr std::uint8_t last_type = static_cast(MessageType::intent_outcome); +constexpr std::uint8_t last_type = static_cast(MessageType::heartbeat); constexpr std::uint8_t last_status = static_cast(IntentStatus::rate_limited); -constexpr std::uint8_t last_reason = static_cast(CloseReason::capacity); +constexpr std::uint8_t last_reason = static_cast(CloseReason::timeout); constexpr std::uint8_t last_finality = static_cast(IntentFinality::not_committed); constexpr std::uint8_t last_presence = static_cast(client::FieldPresence::present); constexpr std::uint8_t last_moisture = static_cast(world::MoistureStatus::available); @@ -280,6 +280,18 @@ std::vector encode(CloseReason reason) { return framed(MessageType::close, w); } +std::vector encode_heartbeat() { + return encode_frame(MessageType::heartbeat, {}); +} + +std::expected decode_heartbeat(FrameView frame) { + if (frame.type != MessageType::heartbeat) + return std::unexpected(ProtocolError::wrong_type); + if (!frame.payload.empty()) + return std::unexpected(ProtocolError::trailing_bytes); + return {}; +} + std::vector encode(const IntentOutcome& m) { Writer w; w.u64(m.sequence); diff --git a/src/network/session/client.cpp b/src/network/session/client.cpp index 9e5ffa9..5875e10 100644 --- a/src/network/session/client.cpp +++ b/src/network/session/client.cpp @@ -18,15 +18,16 @@ struct SessionClient::Impl { std::size_t next_grant{}; SessionPhase phase{SessionPhase::backoff}; Step step{Step::handshake}; - Clock::time_point retry_at{}, deadline{}; + Clock::time_point retry_at{}, deadline{}, last_received{}, last_sent{}, polled_at{}; std::chrono::milliseconds delay; std::unique_ptr stream; // Declared before the channel that borrows it. std::unique_ptr tls; FrameReader reader; - std::vector out; std::optional session; std::optional view; - std::vector outstanding; // Admitted, awaiting a final outcome. + // Sent and not finally resolved: no IntentResult yet, or admitted without an + // outcome. A connection loss reports every one of them as unknown. + std::vector outstanding; client::EnvironmentLayer environment; client::OrganismLayer organisms; std::vector events; @@ -45,7 +46,6 @@ struct SessionClient::Impl { tls.reset(); stream.reset(); reader = FrameReader{}; - out.clear(); session.reset(); view.reset(); // Keep showing the last presentation, explicitly stale, until a new baseline. @@ -77,9 +77,18 @@ struct SessionClient::Impl { phase = SessionPhase::connecting; step = Step::handshake; deadline = now + config.attempt_deadline; + last_received = last_sent = now; + } + // Every outbound frame goes through the channel's bounded queue. + bool queue(const std::vector& frame, Clock::time_point now) { + if (!tls || !tls->queue(frame)) + return false; + last_sent = now; + return true; } bool frame(const OwnedFrame& f, Clock::time_point now) { const FrameView v{f.type, f.payload}; + last_received = now; if (step == Step::awaiting_welcome) { if (f.type == MessageType::close) { const auto reason = decode_close(v); @@ -132,14 +141,9 @@ struct SessionClient::Impl { disconnect(now, std::nullopt); return false; } - if (r->status == IntentStatus::admitted) { - if (outstanding.size() >= max_outstanding) { - events.push_back( - {ClientEvent::Kind::unknown, {}, {}, {}, {}, outstanding.front()}); - outstanding.erase(outstanding.begin()); - } - outstanding.push_back(r->sequence); - } + // A rejection is final; an admitted intention waits for its outcome. + if (r->status != IntentStatus::admitted) + std::erase(outstanding, r->sequence); events.push_back({ClientEvent::Kind::result, {}, {}, *r, {}, r->sequence}); return true; } @@ -153,6 +157,12 @@ struct SessionClient::Impl { events.push_back({ClientEvent::Kind::outcome, {}, {}, {}, *o, o->sequence}); return true; } + case MessageType::heartbeat: + if (!decode_heartbeat(v)) { + disconnect(now, std::nullopt); + return false; + } + return true; case MessageType::close: { const auto reason = decode_close(v); disconnect(now, reason.value_or(CloseReason::malformed)); @@ -164,18 +174,8 @@ struct SessionClient::Impl { } } void flush(Clock::time_point now) { - std::size_t sent = 0; - while (sent < out.size()) { - const auto n = tls->send(std::span{out}.subspan(sent)); - if (!n) { - disconnect(now, std::nullopt); - return; - } - if (*n == 0) - break; - sent += *n; - } - out.erase(out.begin(), out.begin() + static_cast(sent)); + if (!tls->flush()) + disconnect(now, std::nullopt); } void receive(Clock::time_point now) { std::array buffer{}; @@ -215,7 +215,8 @@ SessionClient::create(SessionClientConfig config) { if (config.host.empty() || config.port == 0 || config.grants.empty() || config.grants.size() > 1024 || config.lead_ticks == 0 || config.lead_ticks > 64 || config.first_retry.count() <= 0 || config.max_retry < config.first_retry || - config.attempt_deadline.count() <= 0) + config.attempt_deadline.count() <= 0 || config.heartbeat_interval.count() <= 0 || + config.liveness_timeout <= 2 * config.heartbeat_interval) return std::unexpected(SessionClientError::configuration); return std::unique_ptr( new SessionClient(std::make_unique(std::move(config)))); @@ -223,6 +224,7 @@ SessionClient::create(SessionClientConfig config) { void SessionClient::poll(std::chrono::steady_clock::time_point now) { auto& m = *impl_; + m.polled_at = now; if (m.phase == SessionPhase::exhausted) return; if (m.phase == SessionPhase::backoff) { @@ -236,6 +238,18 @@ void SessionClient::poll(std::chrono::steady_clock::time_point now) { m.disconnect(now, std::nullopt); return; } + if (m.step == Impl::Step::established) { + // A silent server (paused process, black-holed path) is treated as lost: the + // presentation turns stale and the reconnect policy takes over. + if (now - m.last_received > m.config.liveness_timeout) { + m.disconnect(now, std::nullopt); + return; + } + if (now - m.last_sent >= m.config.heartbeat_interval && !m.queue(encode_heartbeat(), now)) { + m.disconnect(now, std::nullopt); + return; + } + } if (m.step == Impl::Step::handshake) { const auto p = m.tls->handshake(); if (!p) { @@ -251,7 +265,10 @@ void SessionClient::poll(std::chrono::steady_clock::time_point now) { m.disconnect(now, std::nullopt); return; } - m.out.insert(m.out.end(), hello->begin(), hello->end()); + if (!m.queue(*hello, now)) { + m.disconnect(now, std::nullopt); + return; + } m.step = Impl::Step::awaiting_welcome; } m.flush(now); @@ -263,13 +280,18 @@ void SessionClient::poll(std::chrono::steady_clock::time_point now) { bool SessionClient::send(const client::Intention& intention) { auto& m = *impl_; - if (m.phase != SessionPhase::established || !m.session || !m.view) + // Bounded: no session, no view, too many unresolved intentions or a full channel + // queue all refuse rather than buffer (the caller resamples later). + if (m.phase != SessionPhase::established || !m.session || !m.view || + m.outstanding.size() >= max_outstanding) return false; const auto tick = m.view->server_tick + m.config.lead_ticks; const auto bytes = encode(make_intent(m.session->session, m.session->epoch, {intention.sequence, tick, intention.move_x, intention.move_y})); - m.out.insert(m.out.end(), bytes.begin(), bytes.end()); + if (!m.queue(bytes, m.polled_at)) + return false; + m.outstanding.push_back(intention.sequence); return true; } diff --git a/src/network/session/grant_ledger.cpp b/src/network/session/grant_ledger.cpp new file mode 100644 index 0000000..68d95d6 --- /dev/null +++ b/src/network/session/grant_ledger.cpp @@ -0,0 +1,122 @@ +#include "evo/network/grant_ledger.hpp" +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#include +#endif + +namespace evo::network { +namespace { +std::FILE* open_file(const std::filesystem::path& path, const char* mode) { +#ifdef _WIN32 + std::FILE* file = nullptr; + return _wfopen_s(&file, path.c_str(), mode[0] == 'w' ? L"wb" : L"ab") == 0 ? file : nullptr; +#else + return std::fopen(path.c_str(), mode); +#endif +} +bool sync(std::FILE* file) { + if (std::fflush(file) != 0) + return false; +#ifdef _WIN32 + return _commit(_fileno(file)) == 0; +#else + return ::fsync(::fileno(file)) == 0; +#endif +} +// Makes a rename durable on POSIX by syncing the containing directory. +bool sync_directory([[maybe_unused]] const std::filesystem::path& dir) { +#ifdef _WIN32 + return true; // NTFS metadata journaling; MoveFileEx is used with replace semantics. +#else + const int fd = ::open(dir.empty() ? "." : dir.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC); + if (fd < 0) + return false; + const bool ok = ::fsync(fd) == 0; + return ::close(fd) == 0 && ok; +#endif +} +std::string line(ConsumedGrant g) { + constexpr char digits[] = "0123456789abcdef"; + std::string out(16, '0'); + for (int i = 15; i >= 0; --i, g.grant_id >>= 4U) + out[static_cast(i)] = digits[g.grant_id & 0x0FU]; + return out + ' ' + std::to_string(g.expires_at) + '\n'; +} +bool parse(const std::string& text, ConsumedGrant& out) { // One getline() line. + if (text.size() < 18 || text[16] != ' ') + return false; + const auto [a, e1] = std::from_chars(text.data(), text.data() + 16, out.grant_id, 16); + const auto [b, e2] = + std::from_chars(text.data() + 17, text.data() + text.size(), out.expires_at); + return e1 == std::errc{} && a == text.data() + 16 && e2 == std::errc{} && + b == text.data() + text.size() && out.grant_id != 0; +} +} // namespace + +std::expected, LedgerError> +GrantLedger::open(const std::filesystem::path& path, std::uint64_t now_seconds) { + std::vector live; + std::error_code error; + if (std::filesystem::exists(path, error)) { + std::ifstream in{path, std::ios::binary}; + if (!in) + return std::unexpected(LedgerError::io); + std::string text; + std::size_t total = 0; + while (std::getline(in, text)) { + ConsumedGrant g; + if (!parse(text, g)) + return std::unexpected(LedgerError::corrupt); // Never guess: fail closed. + if (++total > max_entries) + return std::unexpected(LedgerError::full); + if (g.expires_at > now_seconds) + live.push_back(g); + } + if (!in.eof()) + return std::unexpected(LedgerError::io); + } else if (error) { + return std::unexpected(LedgerError::io); + } + // Rewrite the unexpired entries atomically, then append to the result. + const auto temporary = std::filesystem::path{path}.concat(".tmp"); + { + std::FILE* out = open_file(temporary, "wb"); + if (out == nullptr) + return std::unexpected(LedgerError::io); + bool ok = true; + for (const auto& g : live) + ok = ok && std::fputs(line(g).c_str(), out) >= 0; + ok = sync(out) && ok; + if (std::fclose(out) != 0 || !ok) + return std::unexpected(LedgerError::io); + } + std::filesystem::rename(temporary, path, error); + if (error || !sync_directory(path.parent_path())) + return std::unexpected(LedgerError::io); + std::FILE* file = open_file(path, "ab"); + if (file == nullptr) + return std::unexpected(LedgerError::io); + return std::unique_ptr(new GrantLedger(file, std::move(live))); +} + +GrantLedger::~GrantLedger() { + std::fclose(file_); +} + +std::expected GrantLedger::record(ConsumedGrant grant) { + if (entries_ >= max_entries) + return std::unexpected(LedgerError::full); + if (std::fputs(line(grant).c_str(), file_) < 0 || !sync(file_)) + return std::unexpected(LedgerError::io); + ++entries_; + return {}; +} +} // namespace evo::network diff --git a/src/network/session/server.cpp b/src/network/session/server.cpp index 8351f6d..6548f42 100644 --- a/src/network/session/server.cpp +++ b/src/network/session/server.cpp @@ -1,5 +1,6 @@ #include "evo/network/session_server.hpp" #include +#include #include namespace evo::network { @@ -7,6 +8,9 @@ namespace { using Clock = std::chrono::steady_clock; constexpr std::size_t max_accepts_per_poll = 4; constexpr std::size_t max_read_per_poll = 65536; +// Carrier ids are 'I','N' + 48-bit epoch + 64-bit server ordinal: distinct for every +// carried intention, independent of client-chosen sequences (which may be any u64). +constexpr std::uint64_t max_carrier_epoch = (std::uint64_t{1} << 48) - 1; CloseReason close_reason(AdmissionError e) { switch (e) { case AdmissionError::duplicate_subject: @@ -24,8 +28,13 @@ CloseReason close_reason(ProtocolError e) { return e == ProtocolError::unsupported_version ? CloseReason::unsupported_version : CloseReason::malformed; } -void append(std::vector& out, const std::vector& bytes) { - out.insert(out.end(), bytes.begin(), bytes.end()); +RequestId carrier_id(std::uint64_t epoch, std::uint64_t ordinal) { + RequestId id{'I', 'N'}; + for (unsigned i = 0; i < 6; ++i) + id.at(2 + i) = static_cast(epoch >> (8U * (5U - i))); + for (unsigned i = 0; i < 8; ++i) + id.at(8 + i) = static_cast(ordinal >> (8U * (7U - i))); + return id; } } // namespace @@ -34,18 +43,16 @@ struct SessionServer::Impl { struct Connection { std::uint64_t id{}; std::unique_ptr stream; // Declared first: outlives the channel. - std::unique_ptr tls; + std::unique_ptr tls; // Owns all queued output (ADR-0083 review fix). State state{State::handshake}; - Clock::time_point deadline; + Clock::time_point deadline, last_received, last_sent, window_start; FrameReader reader; - std::vector out, view; + std::vector view; // Newest unsent view only; replaced, never queued. std::uint64_t session{}, subject{}, epoch{}, expires_at{}; std::size_t window_bytes{}; - Clock::time_point window_start; bool dead{}; }; struct Pending { - RequestId request{}; std::uint64_t intended_tick{}, connection{}, sequence{}; }; struct Carried { @@ -53,13 +60,16 @@ struct SessionServer::Impl { std::uint64_t connection{}, sequence{}; }; - Impl(SessionServerConfig c, DevGrantKey k, std::unique_ptr l) - : config{std::move(c)}, key{std::move(k)}, listener{std::move(l)} {} + Impl(SessionServerConfig c, DevGrantKey k, std::unique_ptr l, + std::unique_ptr g) + : config{std::move(c)}, key{std::move(k)}, listener{std::move(l)}, ledger{std::move(g)} {} SessionServerConfig config; DevGrantKey key; std::unique_ptr listener; + std::unique_ptr ledger; std::optional admission; + bool ledger_rejected{}; // Recorded grants exceed admission capacity: fail closed. std::vector connections; std::deque pending; std::vector carried; @@ -67,8 +77,8 @@ struct SessionServer::Impl { std::optional environment; std::optional organisms; RegionFacts published{}; - std::uint64_t baseline{}, next_connection{1}; - Clock::time_point polled_at{}; // Time of the current poll, for Close deadlines. + std::uint64_t baseline{}, next_connection{1}, next_carrier{1}; + Clock::time_point polled_at{}; Connection* find(std::uint64_t id) { const auto it = std::ranges::find_if(connections, [&](const Connection& c) { @@ -76,16 +86,23 @@ struct SessionServer::Impl { }); return it == connections.end() ? nullptr : &*it; } + // Every outbound frame goes through the channel's bounded queue; a full queue means + // a consumer that cannot keep up, which is dropped rather than buffered further. + void send(Connection& c, const std::vector& frame) { + if (c.dead || !c.tls->queue(frame)) + c.dead = true; + c.last_sent = polled_at; + } void close(Connection& c, CloseReason reason) { if (c.state == State::closing || c.dead) return; - if (c.state == State::hello || c.state == State::established) - append(c.out, encode(reason)); - else - c.dead = true; // No channel yet to say anything over. events.push_back({c.state == State::established ? SessionEvent::Kind::closed : SessionEvent::Kind::refused, c.session, c.subject, reason}); + if (c.state == State::hello || c.state == State::established) + send(c, encode(reason)); + else + c.dead = true; // No channel yet to say anything over. c.state = State::closing; c.deadline = polled_at + std::chrono::seconds{1}; // Bounded flush of the Close frame. c.view.clear(); @@ -96,8 +113,11 @@ struct SessionServer::Impl { return bytes; const ViewHeader header{c.session, c.epoch, published.simulated_tick, published.committed_tick, baseline}; - append(bytes, encode(header, *environment)); - append(bytes, encode(header, *organisms)); + const auto a = encode(header, *environment); + const auto b = encode(header, *organisms); + bytes.reserve(a.size() + b.size()); + bytes.insert(bytes.end(), a.begin(), a.end()); + bytes.insert(bytes.end(), b.begin(), b.end()); return bytes; } void hello(Connection& c, const OwnedFrame& f, const RegionFacts& facts, @@ -115,24 +135,34 @@ struct SessionServer::Impl { close(c, CloseReason::server_shutdown); // No current authority: retry later. return; } + if (facts.epoch > max_carrier_epoch) { + close(c, CloseReason::capacity); // Carrier ids cannot name this epoch: refuse. + return; + } const auto grant = key.verify(message->grant); if (!grant) { close(c, CloseReason::unauthenticated); return; } + c.subject = grant->claims().subject; const auto welcome = admission->open(*grant, now_seconds, facts.simulated_tick, facts.committed_tick); if (!welcome) { - c.subject = grant->claims().subject; close(c, close_reason(welcome.error())); return; } + // Durable before Welcome: a replacement process must refuse this grant too. + if (!ledger->record({grant->claims().grant_id, grant->claims().expires_at})) { + // The grant stays consumed in memory (fail closed); only the session is undone. + static_cast(admission->close(welcome->session).has_value()); + close(c, CloseReason::server_shutdown); + return; + } c.session = welcome->session; - c.subject = grant->claims().subject; c.epoch = welcome->epoch; c.expires_at = welcome->grant_expires_at; c.state = State::established; - append(c.out, encode(*welcome)); + send(c, encode(*welcome)); c.view = views_for(c); // Fresh baseline for every new session. events.push_back({SessionEvent::Kind::opened, c.session, c.subject, {}}); } @@ -152,16 +182,8 @@ struct SessionServer::Impl { result = admission->admit(*m, facts.simulated_tick, now_seconds); } if (result.status == IntentStatus::admitted) - pending.push_back({request_id(c, m->sequence), m->tick, c.id, m->sequence}); - append(c.out, encode(result)); - } - RequestId request_id(const Connection& c, std::uint64_t sequence) const { - RequestId id{'I', 'N', 'T', '1'}; - const std::array parts{c.epoch, c.session, sequence}; - for (std::size_t p = 0; p < parts.size(); ++p) - for (unsigned i = 0; i < 4; ++i) - id.at(4 + 4 * p + i) = static_cast(parts.at(p) >> (8U * (3U - i))); - return id; + pending.push_back({m->tick, c.id, m->sequence}); + send(c, encode(result)); } void receive(Connection& c, const RegionFacts& facts, std::uint64_t now_seconds, Clock::time_point now) { @@ -194,12 +216,17 @@ struct SessionServer::Impl { } rest = rest.subspan(*taken); while (auto f = c.reader.take()) { + c.last_received = now; if (c.state == State::hello) hello(c, *f, facts, now_seconds); else if (f->type == MessageType::intent) intent(c, *f, facts, now_seconds); - else if (f->type == MessageType::close) { + else if (f->type == MessageType::heartbeat) { + if (!decode_heartbeat({f->type, f->payload})) + close(c, CloseReason::malformed); + } else if (f->type == MessageType::close) { c.state = State::closing; // Peer is leaving; nothing to say. + c.deadline = now + std::chrono::seconds{1}; events.push_back({SessionEvent::Kind::closed, c.session, c.subject, CloseReason::normal}); } else @@ -211,27 +238,25 @@ struct SessionServer::Impl { } } void transmit(Connection& c) { - if (c.out.empty() && !c.view.empty() && c.state == State::established) - c.out.swap(c.view); // Only the newest view waits; it is never queued twice. - std::size_t sent = 0; - while (sent < c.out.size()) { - const auto n = c.tls->send(std::span{c.out}.subspan(sent)); - if (!n) { - c.dead = true; - return; - } - if (*n == 0) - break; - sent += *n; + if (c.state == State::established && c.tls->queued() == 0 && !c.view.empty()) { + send(c, c.view); + c.view.clear(); } - c.out.erase(c.out.begin(), c.out.begin() + static_cast(sent)); - if (c.out.size() + c.view.size() > config.max_output_bytes) + if (c.dead) + return; + if (c.tls->queued() + c.view.size() > config.max_output_bytes) { c.dead = true; // Slow consumer: drop rather than buffer without bound. - if (c.state == State::closing && c.out.empty()) { - // Best-effort close_notify: the connection is dropped either way. - static_cast(c.tls->close().has_value()); - c.dead = true; + return; } + if (c.state == State::closing) { + // close() flushes queued frames (the Close frame included) before close_notify. + const auto closed = c.tls->close(); + if (!closed || *closed == TlsProgress::done) + c.dead = true; + return; + } + if (!c.tls->flush()) + c.dead = true; } void step(Connection& c, const RegionFacts& facts, std::uint64_t now_seconds, Clock::time_point now) { @@ -250,10 +275,16 @@ struct SessionServer::Impl { } if ((c.state == State::handshake || c.state == State::hello) && now > c.deadline) close(c, CloseReason::unauthenticated); - if (c.state == State::established && now_seconds >= c.expires_at) - close(c, CloseReason::expired); - if (c.state == State::established && admission && c.epoch != admission->epoch()) - close(c, CloseReason::stale_epoch); + if (c.state == State::established) { + if (now_seconds >= c.expires_at) + close(c, CloseReason::expired); + else if (admission && c.epoch != admission->epoch()) + close(c, CloseReason::stale_epoch); + else if (now - c.last_received > config.liveness_timeout) + close(c, CloseReason::timeout); // Silent peer: free the subject for reconnect. + else if (now - c.last_sent >= config.heartbeat_interval) + send(c, encode_heartbeat()); // Liveness without requiring simulation ticks. + } if (c.state == State::hello || c.state == State::established) receive(c, facts, now_seconds, now); if (!c.dead && c.state != State::handshake) @@ -270,17 +301,44 @@ struct SessionServer::Impl { } std::erase_if(connections, [](const Connection& c) { return c.dead; }); } + void open_admission(const RegionFacts& facts, std::uint64_t now_seconds) { + auto config_with_epoch = config.admission; + config_with_epoch.epoch = facts.epoch; + auto created = SessionAdmission::create(config_with_epoch); + if (!created) + return; + // Grants consumed before this process started stay consumed. + if (!created->remember(ledger->loaded(), now_seconds)) { + ledger_rejected = true; // More recorded grants than admission can hold. + return; + } + admission.emplace(std::move(*created)); + } + void finish(const RequestId& request, const IntentOutcome& outcome) { + const auto it = + std::ranges::find_if(carried, [&](const Carried& c) { return c.request == request; }); + if (it == carried.end()) + return; + auto out = outcome; + out.sequence = it->sequence; // The client's own sequence, carried separately. + if (auto* c = find(it->connection)) + send(*c, encode(out)); + carried.erase(it); + } }; SessionServer::SessionServer(std::unique_ptr impl) : impl_{std::move(impl)} {} SessionServer::~SessionServer() = default; std::expected, SessionServerError> -SessionServer::create(SessionServerConfig config, DevGrantKey key) { +SessionServer::create(SessionServerConfig config, DevGrantKey key, std::uint64_t now_seconds) { if (config.max_connections == 0 || config.max_connections > 64 || config.max_pending_intents == 0 || config.max_carried_intents == 0 || config.max_output_bytes < 2 * (frame_header_bytes + max_frame_payload) || - config.max_input_bytes_per_second < frame_header_bytes + max_grant_bytes + 6) + config.max_output_bytes > TlsChannel::max_queued_bytes || + config.max_input_bytes_per_second < frame_header_bytes + max_grant_bytes + 6 || + config.heartbeat_interval.count() <= 0 || + config.liveness_timeout <= 2 * config.heartbeat_interval || config.grant_ledger.empty()) return std::unexpected(SessionServerError::configuration); // Validate the admission limits now, with a placeholder epoch. auto probe = config.admission; @@ -298,11 +356,14 @@ SessionServer::create(SessionServerConfig config, DevGrantKey key) { } null; if (!TlsChannel::server(config.credentials, null)) return std::unexpected(SessionServerError::credentials); + auto ledger = GrantLedger::open(config.grant_ledger, now_seconds); + if (!ledger) + return std::unexpected(SessionServerError::ledger); auto listener = TcpListener::listen(config.listen); if (!listener) return std::unexpected(SessionServerError::listen); - return std::unique_ptr(new SessionServer( - std::make_unique(std::move(config), std::move(key), std::move(*listener)))); + return std::unique_ptr(new SessionServer(std::make_unique( + std::move(config), std::move(key), std::move(*listener), std::move(*ledger)))); } std::uint16_t SessionServer::port() const noexcept { @@ -313,12 +374,9 @@ void SessionServer::poll(const RegionFacts& facts, std::uint64_t now_seconds, std::chrono::steady_clock::time_point now) { auto& m = *impl_; m.polled_at = now; - if (facts.ready && facts.epoch != 0) { + if (facts.ready && facts.epoch != 0 && !m.ledger_rejected) { if (!m.admission) { - auto config = m.config.admission; - config.epoch = facts.epoch; - if (auto created = SessionAdmission::create(config)) - m.admission.emplace(std::move(*created)); + m.open_admission(facts, now_seconds); } else if (facts.epoch > m.admission->epoch()) { // Strictly newer (checked above), so the advance cannot be refused. static_cast(m.admission->advance_epoch(facts.epoch).has_value()); @@ -340,7 +398,7 @@ void SessionServer::poll(const RegionFacts& facts, std::uint64_t now_seconds, continue; c.tls = std::move(*tls); c.deadline = now + m.config.hello_deadline; - c.window_start = now; + c.window_start = c.last_received = c.last_sent = now; m.connections.push_back(std::move(c)); } for (auto& c : m.connections) @@ -364,13 +422,15 @@ void SessionServer::publish(const client::EnvironmentPresentation& environment, std::optional SessionServer::carry(std::uint64_t tick) { auto& m = *impl_; - if (m.carried.size() >= m.config.max_carried_intents) - return std::nullopt; + if (m.carried.size() >= m.config.max_carried_intents || !m.admission || + m.next_carrier == std::numeric_limits::max()) + return std::nullopt; // Ordinal exhaustion is explicit, never a wrap. const auto it = std::ranges::find_if( m.pending, [&](const Impl::Pending& p) { return p.intended_tick <= tick; }); if (it == m.pending.end()) return std::nullopt; - const Impl::Carried carried{it->request, it->connection, it->sequence}; + const Impl::Carried carried{carrier_id(m.admission->epoch(), m.next_carrier++), it->connection, + it->sequence}; m.pending.erase(it); m.carried.push_back(carried); return carried.request; @@ -378,26 +438,11 @@ std::optional SessionServer::carry(std::uint64_t tick) { void SessionServer::committed(const RequestId& request, std::uint64_t tick, std::uint64_t commit_sequence) { - auto& m = *impl_; - const auto it = std::ranges::find_if( - m.carried, [&](const Impl::Carried& c) { return c.request == request; }); - if (it == m.carried.end()) - return; - if (auto* c = m.find(it->connection)) - append(c->out, encode(IntentOutcome{it->sequence, IntentFinality::committed_no_effect, tick, - commit_sequence})); - m.carried.erase(it); + impl_->finish(request, {0, IntentFinality::committed_no_effect, tick, commit_sequence}); } void SessionServer::not_committed(const RequestId& request) { - auto& m = *impl_; - const auto it = std::ranges::find_if( - m.carried, [&](const Impl::Carried& c) { return c.request == request; }); - if (it == m.carried.end()) - return; - if (auto* c = m.find(it->connection)) - append(c->out, encode(IntentOutcome{it->sequence, IntentFinality::not_committed, 0, 0})); - m.carried.erase(it); + impl_->finish(request, {0, IntentFinality::not_committed, 0, 0}); } std::vector SessionServer::take_events() { diff --git a/src/network/tls/channel.cpp b/src/network/tls/channel.cpp index 2802e2d..44351b4 100644 --- a/src/network/tls/channel.cpp +++ b/src/network/tls/channel.cpp @@ -5,6 +5,7 @@ #include #include #include +#include // The owned user config (third_party/mbedtls/evo_mbedtls_config.h) must be in effect. #if defined(MBEDTLS_SSL_PROTO_TLS1_2) || defined(MBEDTLS_SSL_PROTO_DTLS) || \ @@ -21,6 +22,8 @@ namespace { // ADR-0082 profile: TLS 1.3 only, AEAD suites, ECDHE groups, ephemeral key exchange. constexpr int suites[]{MBEDTLS_TLS1_3_AES_128_GCM_SHA256, MBEDTLS_TLS1_3_CHACHA20_POLY1305_SHA256, 0}; +// One TLS record's plaintext per write call (the TLS 1.3 maximum fragment). +constexpr std::size_t record_plaintext = 16384; constexpr std::uint16_t groups[]{MBEDTLS_SSL_IANA_TLS_GROUP_X25519, MBEDTLS_SSL_IANA_TLS_GROUP_SECP256R1, 0}; const unsigned char* bytes_of(const std::string& text) { @@ -105,6 +108,8 @@ struct TlsChannel::Impl { mbedtls_pk_context key; bool transport_failed{}; std::optional error; + std::vector outbound; // Queued plaintext; the front record may be in flight. + std::size_t in_flight{}; // Length of an interrupted write; 0 when none. }; TlsChannel::TlsChannel(std::unique_ptr impl) : impl_{std::move(impl)} {} @@ -158,15 +163,39 @@ std::expected TlsChannel::handshake() { return impl_->fail_from(result, TlsError::handshake); } -std::expected TlsChannel::send(std::span bytes) { +std::expected TlsChannel::queue(std::span bytes) { if (const auto latched = impl_->error) return std::unexpected(*latched); - const int result = mbedtls_ssl_write(&impl_->ssl, bytes.data(), bytes.size()); - if (result >= 0) - return static_cast(result); - if (result == MBEDTLS_ERR_SSL_WANT_READ || result == MBEDTLS_ERR_SSL_WANT_WRITE) - return 0; - return impl_->fail_from(result, TlsError::record); + if (bytes.size() > max_queued_bytes - impl_->outbound.size()) + return std::unexpected(TlsError::backpressure); // Nothing queued: not latched. + impl_->outbound.insert(impl_->outbound.end(), bytes.begin(), bytes.end()); + return {}; +} + +std::expected TlsChannel::flush() { + auto& m = *impl_; + if (const auto latched = m.error) + return std::unexpected(*latched); + while (!m.outbound.empty()) { + // A record interrupted by WANT_READ/WANT_WRITE keeps its length; bytes queued + // since then sit behind it and do not change the retried call. + if (m.in_flight == 0) + m.in_flight = std::min(m.outbound.size(), record_plaintext); + const int result = mbedtls_ssl_write(&m.ssl, m.outbound.data(), m.in_flight); + if (result == MBEDTLS_ERR_SSL_WANT_READ || result == MBEDTLS_ERR_SSL_WANT_WRITE) + return TlsProgress::want_io; + if (result < 0) + return m.fail_from(result, TlsError::record); + const auto written = std::min(static_cast(result), m.in_flight); + m.outbound.erase(m.outbound.begin(), + m.outbound.begin() + static_cast(written)); + m.in_flight = 0; + } + return TlsProgress::done; +} + +std::size_t TlsChannel::queued() const noexcept { + return impl_->outbound.size(); } std::expected TlsChannel::receive(std::span bytes) { @@ -185,6 +214,9 @@ std::expected TlsChannel::receive(std::span std::expected TlsChannel::close() { if (const auto latched = impl_->error) return std::unexpected(*latched); + // Queued plaintext (including an interrupted record) is sent before close_notify. + if (const auto pending = flush(); !pending || *pending == TlsProgress::want_io) + return pending; const int result = mbedtls_ssl_close_notify(&impl_->ssl); if (result == 0) return TlsProgress::done; diff --git a/tests/fuzz/dev_session_fuzz.cpp b/tests/fuzz/dev_session_fuzz.cpp index a2cbc10..92d9f30 100644 --- a/tests/fuzz/dev_session_fuzz.cpp +++ b/tests/fuzz/dev_session_fuzz.cpp @@ -50,6 +50,10 @@ extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, std::size_t size if (const auto m = decode_intent_outcome(*frame)) again = encode(*m); break; + case MessageType::heartbeat: + if (decode_heartbeat(*frame)) + again = encode_heartbeat(); + break; case MessageType::environment_view: if (const auto m = decode_environment_view(*frame)) again = encode(m->first, m->second); diff --git a/tests/network_protocol.cpp b/tests/network_protocol.cpp index b2ecaaa..3a168c5 100644 --- a/tests/network_protocol.cpp +++ b/tests/network_protocol.cpp @@ -126,6 +126,12 @@ void round_trips() { const auto c = decode_close(frame(encode(CloseReason::stale_epoch))); CHECK(c && *c == CloseReason::stale_epoch); CHECK(decode_close(frame(encode(CloseReason::capacity))) == CloseReason::capacity); + CHECK(decode_close(frame(encode(CloseReason::timeout))) == CloseReason::timeout); + const auto beat = encode_heartbeat(); + CHECK(beat.size() == frame_header_bytes && decode_heartbeat(frame(beat))); + CHECK(refused(decode_heartbeat(frame(with_header(MessageType::heartbeat, 1, 0, 1, 1))), + ProtocolError::trailing_bytes)); + CHECK(refused(decode_heartbeat(frame(encode(CloseReason::normal))), ProtocolError::wrong_type)); const auto out = decode_intent_outcome( frame(encode(IntentOutcome{4, IntentFinality::committed_no_effect, 120, 9}))); CHECK(out && out->sequence == 4 && out->finality == IntentFinality::committed_no_effect && @@ -177,7 +183,7 @@ void frame_rejection() { CHECK(refused(decode_frame(with_header(MessageType{0}, 1, 0, 0, 0)), ProtocolError::unknown_type)); // NOLINTNEXTLINE(clang-analyzer-optin.core.EnumCastOutOfRange) - CHECK(refused(decode_frame(with_header(MessageType{9}, 1, 0, 0, 0)), + CHECK(refused(decode_frame(with_header(MessageType{10}, 1, 0, 0, 0)), ProtocolError::unknown_type)); CHECK(refused(decode_frame(with_header(MessageType::intent, 1, 1, 36, 36)), ProtocolError::reserved_flags)); @@ -207,7 +213,7 @@ void frame_rejection() { status[frame_header_bytes + 8] = 8; CHECK(refused(decode_intent_result(frame(status)), ProtocolError::invalid_value)); auto reason = encode(CloseReason::normal); - reason[frame_header_bytes] = 10; + reason[frame_header_bytes] = 11; CHECK(refused(decode_close(frame(reason)), ProtocolError::invalid_value)); CHECK(refused(decode_welcome(frame(encode(Welcome{}))), ProtocolError::invalid_value)); // A well-formed encoding of an invalid view is rejected by the presentation rules. @@ -261,6 +267,7 @@ void fuzz() { encode(IntentResult{1, IntentStatus::outside_window, 2}), encode(CloseReason::rate_limited), encode(IntentOutcome{3, IntentFinality::committed_no_effect, 4, 5}), + encode_heartbeat(), encode(view_header, environment()), encode(view_header, organisms(3))}; std::size_t accepted = 0; @@ -313,6 +320,10 @@ void fuzz() { if (const auto m = decode_intent_outcome(*f)) again = encode(*m); break; + case MessageType::heartbeat: + if (decode_heartbeat(*f)) + again = encode_heartbeat(); + break; case MessageType::environment_view: if (const auto m = decode_environment_view(*f)) again = encode(m->first, m->second); diff --git a/tests/network_session.cpp b/tests/network_session.cpp index 8d1c985..81bd090 100644 --- a/tests/network_session.cpp +++ b/tests/network_session.cpp @@ -1,11 +1,16 @@ +#include "evo/network/grant_ledger.hpp" #include "evo/network/session_client.hpp" #include "evo/network/session_server.hpp" #include "evo/network/tcp.hpp" #include "network_pki.hpp" #include "network_test.hpp" #include +#include #include +#include +#include #include +#include #include #include #include @@ -57,12 +62,28 @@ client::OrganismPresentation no_organisms(std::uint64_t tick) { return std::move(*v); } +std::filesystem::path fresh_ledger() { + static std::atomic counter{0}; + const auto stamp = Clock::now().time_since_epoch().count(); + return std::filesystem::temp_directory_path() / + ("evo-grant-ledger-" + std::to_string(stamp) + "-" + std::to_string(++counter)); +} struct Harness { explicit Harness(std::size_t max_sessions = 4, - std::chrono::milliseconds hello = std::chrono::milliseconds{3000}) { + std::chrono::milliseconds hello = std::chrono::milliseconds{3000}, + std::function server_tweak = {}, + std::function client_tweak = {}) + : tweak_server{std::move(server_tweak)}, tweak_client{std::move(client_tweak)} { server = make_server(0, max_sessions, hello); port = server->port(); } + ~Harness() { + server.reset(); + std::error_code ignored; + std::filesystem::remove(ledger, ignored); + } + Harness(const Harness&) = delete; + Harness& operator=(const Harness&) = delete; std::unique_ptr make_server(std::uint16_t at, std::size_t max_sessions, std::chrono::milliseconds hello) { SessionServerConfig c; @@ -75,7 +96,10 @@ struct Harness { c.admission.window_ticks = 8; c.admission.max_intents_per_window = 8; c.hello_deadline = hello; - auto s = SessionServer::create(std::move(c), grant_key()); + c.grant_ledger = ledger; // Shared by restarts of this harness's server. + if (tweak_server) + tweak_server(c); + auto s = SessionServer::create(std::move(c), grant_key(), now_seconds); CHECK(s); return std::move(*s); } @@ -88,6 +112,8 @@ struct Harness { c.first_retry = std::chrono::milliseconds{50}; c.max_retry = std::chrono::milliseconds{200}; c.attempt_deadline = std::chrono::milliseconds{3000}; + if (tweak_client) + tweak_client(c); auto s = SessionClient::create(std::move(c)); CHECK(s); clients.push_back(s->get()); @@ -119,6 +145,9 @@ struct Harness { auto e = c.take_events(); into.insert(into.end(), e.begin(), e.end()); } + std::function tweak_server; + std::function tweak_client; + std::filesystem::path ledger = fresh_ledger(); Pki pki; RegionFacts facts{true, 1, 100, 90, 5}; std::unique_ptr server; @@ -167,7 +196,9 @@ void connect_view_intents_and_outcomes() { CHECK(last(events, ClientEvent::Kind::result)->result.status == IntentStatus::admitted); CHECK(!h.server->carry(105)); // Intended tick 102 + 4 = 106. const auto request = h.server->carry(106); - CHECK(must(request)[0] == 'I' && must(request)[3] == '1'); + // 'I','N' + 48-bit epoch (1) + 64-bit server ordinal (first carrier: 1). + CHECK(must(request)[0] == 'I' && must(request)[1] == 'N' && must(request)[7] == 1 && + must(request)[15] == 1); CHECK(!h.server->carry(200)); // Carried once. h.server->committed(must(request), 106, 6); CHECK(h.until([&] { @@ -336,24 +367,42 @@ void abuse() { h.tick(); CHECK(h.until([&] { return c->phase() == SessionPhase::established; })); CHECK(c->grants_left() == 2); - // Flooding: the byte-rate limit closes the connection. The flooding client may see - // a transport reset before it reads the Close frame, so check the server's decision. + // The client never buffers without bound: with the server not answering, at most + // max_outstanding intentions may be unresolved; further sends are refused. CHECK(h.until([&] { return c->latest_view().has_value(); })); - (void)h.server->take_events(); + std::size_t accepted = 0; for (std::uint64_t s = 10; s < 2000; ++s) - CHECK(c->send({s, 0, 0, 0})); + accepted += c->send({s, 0, 0, 0}) ? 1 : 0; + CHECK(accepted == SessionClient::max_outstanding); + + // Flooding: a raw TLS peer (no client-side bounds) exceeds the byte-rate limit after + // a valid Hello; the server closes it. Check the server's decision, since the flooding + // peer may see a reset before it could read the Close frame. + (void)h.server->take_events(); + auto raw = TcpStream::connect("127.0.0.1", h.port); + CHECK(raw); + auto tls = TlsChannel::client({h.pki.authority, server_name}, **raw); + CHECK(tls); + const auto flood_grant = grants(9, 1, 900).front(); + const auto hello = encode(Hello{0, {flood_grant.begin(), flood_grant.end()}}); + CHECK(hello); + bool flooded = false; std::vector server_events; CHECK(h.until([&] { auto e = h.server->take_events(); server_events.insert(server_events.end(), e.begin(), e.end()); + if (const auto p = (*tls)->handshake(); p && *p == TlsProgress::done && !flooded) { + flooded = hello && (*tls)->queue(*hello).has_value(); + const auto beat = encode_heartbeat(); + for (int i = 0; i < 7000 && flooded; ++i) + flooded = (*tls)->queue(beat).has_value(); + } + if (flooded) + static_cast((*tls)->flush().has_value()); // Best effort: may be reset. return std::ranges::any_of(server_events, [](const SessionEvent& x) { return x.kind == SessionEvent::Kind::closed && x.reason == CloseReason::rate_limited; }); })); - CHECK(h.until([&] { - h.collect(*c, events); - return c->phase() != SessionPhase::established; - })); // Plaintext and silent connections are dropped without a session: the server // closes the socket (plaintext at the TLS handshake, silence at the deadline). @@ -381,6 +430,169 @@ void abuse() { CHECK(quiet.until([&] { return quiet.server->connections() == 0; })); CHECK(quiet.server->sessions() == 0); } +// Review M9.4-M9.7 #2: client sequences 2^32 apart must get distinct carriers and +// each final outcome must reach its own sequence. +void carriers_never_alias() { + Harness h; + h.tick(); + auto c = h.client(grants(7, 2, 100)); + std::vector events; + CHECK(h.until([&] { return c->latest_view().has_value(); })); + constexpr std::uint64_t far = (std::uint64_t{1} << 32) + 1; + CHECK(c->send({1, 0, 1000, 0}) && c->send({far, 0, 0, 1000})); + CHECK(h.until([&] { + h.collect(*c, events); + return std::ranges::count_if(events, [](const ClientEvent& e) { + return e.kind == ClientEvent::Kind::result && + e.result.status == IntentStatus::admitted; + }) == 2; + })); + const auto first = h.server->carry(1000); + const auto second = h.server->carry(1000); + CHECK(must(first) != must(second)); + CHECK(must(first)[0] == 'I' && must(first)[1] == 'N' && must(first)[7] == 1); // Epoch 1. + h.server->not_committed(must(second)); + h.server->committed(must(first), 106, 6); + CHECK(h.until([&] { + h.collect(*c, events); + return std::ranges::count_if(events, [](const ClientEvent& e) { + return e.kind == ClientEvent::Kind::outcome; + }) == 2; + })); + for (const auto& e : events) + if (e.kind == ClientEvent::Kind::outcome) + CHECK((e.sequence == far && e.outcome.finality == IntentFinality::not_committed) || + (e.sequence == 1 && e.outcome.finality == IntentFinality::committed_no_effect && + e.outcome.commit_sequence == 6)); +} + +// Review M9.4-M9.7 #3: a grant consumed before a restart stays consumed after it. +void grants_stay_consumed_across_restarts() { + Harness h; + h.tick(); + auto first = h.client(grants(7, 1, 100)); + CHECK(h.until([&] { return first->phase() == SessionPhase::established; })); + h.clients.clear(); + first.reset(); + h.server.reset(); // Process restart: only the ledger file survives. + h.facts = {true, 2, 40, 40, 2}; + h.server = h.make_server(h.port, 4, std::chrono::milliseconds{3000}); + h.tick(); + auto replay = h.client(grants(7, 1, 100)); // The exact consumed grant. + std::vector events; + CHECK(h.until([&] { + h.collect(*replay, events); + return has(events, ClientEvent::Kind::closed); + })); + CHECK(last(events, ClientEvent::Kind::closed)->reason == CloseReason::unauthenticated); + auto fresh = h.client(grants(7, 1, 101)); + CHECK(h.until([&] { return fresh->phase() == SessionPhase::established; })); + CHECK(must(fresh->welcome()).epoch == 2); + + // The ledger fails closed on corruption and drops expired entries when opened. + const auto path = fresh_ledger(); + { + std::ofstream out{path, std::ios::binary}; + out << "not a ledger line\n"; + } + SessionServerConfig bad; + bad.listen = "127.0.0.1:0"; + bad.credentials = h.pki.server; + bad.admission.world = identity; + bad.admission.region = region; + bad.grant_ledger = path; + CHECK( + refused(SessionServer::create(bad, grant_key(), now_seconds), SessionServerError::ledger)); + { + std::ofstream out{path, std::ios::binary | std::ios::trunc}; // LF only. + out << "0000000000000005 " << now_seconds - 1 << "\n" + << "0000000000000006 " << now_seconds + 10 << "\n"; + } + { + const auto ledger = GrantLedger::open(path, now_seconds); + CHECK(ledger && (*ledger)->loaded().size() == 1 && (*ledger)->loaded()[0].grant_id == 6); + CHECK(ledger && (*ledger)->record({7, now_seconds + 20})); + } + const auto reopened = GrantLedger::open(path, now_seconds); + CHECK(reopened && (*reopened)->loaded().size() == 2); + std::error_code ignored; + std::filesystem::remove(path, ignored); +} + +// Review M9.4-M9.7 #4: liveness after Welcome, both directions, without spurious drops. +void liveness() { + const auto fast_server = [](SessionServerConfig& c) { + c.heartbeat_interval = std::chrono::milliseconds{100}; + c.liveness_timeout = std::chrono::milliseconds{500}; + }; + const auto fast_client = [](SessionClientConfig& c) { + c.heartbeat_interval = std::chrono::milliseconds{100}; + c.liveness_timeout = std::chrono::milliseconds{500}; + }; + { + // Healthy but idle: no ticks are published; heartbeats keep both sides alive. + Harness h{4, std::chrono::milliseconds{3000}, fast_server, fast_client}; + h.tick(); + auto c = h.client(grants(7, 2, 100)); + std::vector events; + CHECK(h.until([&] { return c->latest_view().has_value(); })); + const auto start = Clock::now(); + CHECK(!h.until( + [&] { + h.collect(*c, events); + return has(events, ClientEvent::Kind::lost) || + has(events, ClientEvent::Kind::closed); + }, + std::chrono::milliseconds{1500})); + CHECK(Clock::now() - start >= std::chrono::milliseconds{1500}); + CHECK(c->phase() == SessionPhase::established && + c->environment().freshness() == client::Freshness::current); + } + { + // Silent server (socket open, never polled): the client times out, keeps the view + // as stale, reports the sent-but-unresolved intention as unknown, and reconnects. + Harness h{4, std::chrono::milliseconds{3000}, fast_server, fast_client}; + h.tick(); + auto c = h.client(grants(7, 3, 100)); + std::vector events; + CHECK(h.until([&] { return c->latest_view().has_value(); })); + CHECK(c->send({5, 0, 1000, 0})); // Never answered: the server is not polled. + const auto end = Clock::now() + std::chrono::milliseconds{2000}; + while (Clock::now() < end && !has(events, ClientEvent::Kind::lost)) { + c->poll(Clock::now()); + h.collect(*c, events); + std::this_thread::sleep_for(std::chrono::milliseconds{5}); + } + CHECK(has(events, ClientEvent::Kind::lost)); + const auto* unknown = last(events, ClientEvent::Kind::unknown); + CHECK(unknown != nullptr && unknown->sequence == 5); + CHECK(c->environment().freshness() == client::Freshness::stale); + h.tick(); + CHECK(h.until([&] { + return c->phase() == SessionPhase::established && c->latest_view().has_value(); + })); + CHECK(c->environment().freshness() == client::Freshness::current); + } + { + // Silent client: the server closes it with `timeout`, freeing the subject. + Harness h{4, std::chrono::milliseconds{3000}, fast_server, fast_client}; + h.tick(); + auto c = h.client(grants(7, 1, 100)); + CHECK(h.until([&] { return c->phase() == SessionPhase::established; })); + h.clients.clear(); // Stop polling the client; its socket stays open. + std::vector server_events; + CHECK(h.until([&] { + auto e = h.server->take_events(); + server_events.insert(server_events.end(), e.begin(), e.end()); + return h.server->sessions() == 0; + })); + CHECK(std::ranges::any_of(server_events, [](const SessionEvent& e) { + return e.kind == SessionEvent::Kind::closed && e.reason == CloseReason::timeout; + })); + auto again = h.client(grants(7, 1, 200)); // Same subject, fresh grant: admitted. + CHECK(h.until([&] { return again->phase() == SessionPhase::established; })); + } +} } // namespace void session_tests() { @@ -388,5 +600,8 @@ void session_tests() { duplicate_capacity_and_authentication(); stale_epoch_restart_and_reconnect(); abuse(); + carriers_never_alias(); + grants_stay_consumed_across_restarts(); + liveness(); } } // namespace evo::network_test diff --git a/tests/network_tls.cpp b/tests/network_tls.cpp index 2958169..1f2fe9d 100644 --- a/tests/network_tls.cpp +++ b/tests/network_tls.cpp @@ -2,7 +2,9 @@ #include "evo/network/tls_channel.hpp" #include "network_pki.hpp" #include "network_test.hpp" +#include #include +#include #include #include #include @@ -17,6 +19,8 @@ using namespace evo::network; struct Wire { std::deque to_server, to_client; bool fail{}; + // Bytes each side may still write; 0 makes writes "would block" (zero progress). + std::size_t server_budget{SIZE_MAX}, client_budget{SIZE_MAX}; }; class Endpoint final : public ByteTransport { public: @@ -25,7 +29,11 @@ class Endpoint final : public ByteTransport { if (wire_.fail) return std::unexpected(TransportError::failed); auto& out = server_ ? wire_.to_client : wire_.to_server; - const auto count = std::min(bytes.size(), 700); // Force partial writes. + auto& budget = server_ ? wire_.server_budget : wire_.client_budget; + // Force partial writes, and stop entirely when the injected budget runs out. + const auto count = std::min({bytes.size(), std::size_t{700}, budget}); + if (budget != SIZE_MAX) + budget -= count; out.insert(out.end(), bytes.begin(), bytes.begin() + static_cast(count)); return count; } @@ -76,15 +84,15 @@ struct Pair { } std::vector transfer(TlsChannel& from, TlsChannel& to, std::span bytes) { - std::size_t sent = 0; + CHECK(from.queue(bytes)); + return drain(from, to, bytes.size()); + } + // Flushes `from` and reads `to` until `expected` plaintext bytes arrive. + std::vector drain(TlsChannel& from, TlsChannel& to, std::size_t expected) { std::vector received; std::array buffer{}; - for (int i = 0; i < 1000 && received.size() < bytes.size(); ++i) { - if (sent < bytes.size()) { - const auto n = from.send(bytes.subspan(sent)); - CHECK(n); - sent += *n; - } + for (int i = 0; i < 1000 && received.size() < expected; ++i) { + CHECK(from.flush()); const auto n = to.receive(buffer); CHECK(n); received.insert(received.end(), buffer.begin(), @@ -122,7 +130,7 @@ void handshake_and_frames() { for (int i = 0; i < 10 && r && *r == 0; ++i) r = pair.server->receive(buffer); CHECK(refused(r, TlsError::closed)); - CHECK(refused(pair.server->send(buffer), TlsError::closed)); // Latched. + CHECK(refused(pair.server->queue(buffer), TlsError::closed)); // Latched. } void rejections() { @@ -155,8 +163,8 @@ void rejections() { Pair pair{pki.server, {pki.authority, server_name}}; CHECK(pair.handshake()); const std::array data{}; - const auto sent = pair.client->send(data); - CHECK(sent && *sent == data.size()); + CHECK(pair.client->queue(data) && pair.client->flush()); + CHECK(pair.client->queued() == 0); CHECK(pair.wire.to_server.size() > 20); pair.wire.to_server[pair.wire.to_server.size() - 5] ^= 0x01; std::array buffer{}; @@ -181,10 +189,89 @@ void rejections() { CHECK(!r && r.error() == TlsError::handshake); } } +// Review M9.4-M9.7 #1: frames queued while a TLS record is stalled (zero progress, +// or partial progress then zero) must neither be lost nor reordered. +std::vector intents(std::uint64_t first, std::uint64_t count) { + std::vector bytes; + for (std::uint64_t i = 0; i < count; ++i) { + const auto f = encode(IntentMessage{1, 1, first + i, 10, 0, 0}); + bytes.insert(bytes.end(), f.begin(), f.end()); + } + return bytes; +} +std::vector sequences(const std::vector& bytes) { + FrameReader reader; + std::vector out; + std::span rest{bytes}; + while (!rest.empty()) { + const auto taken = reader.feed(rest); + CHECK(taken); + rest = rest.subspan(*taken); + while (auto f = reader.take()) { + const auto m = decode_intent({f->type, f->payload}); + CHECK(m); + out.push_back(m ? m->sequence : 0); + } + } + return out; +} +void interrupted_writes() { + Pki pki; + for (const bool from_client : {true, false}) + for (const std::size_t before_stall : {std::size_t{0}, std::size_t{10}}) { + Pair pair{pki.server, {pki.authority, server_name}}; + CHECK(pair.handshake()); + auto& from = from_client ? *pair.client : *pair.server; + auto& to = from_client ? *pair.server : *pair.client; + auto& budget = from_client ? pair.wire.client_budget : pair.wire.server_budget; + budget = before_stall; // Zero progress, or partial progress then zero. + const auto first = intents(1, 1); + CHECK(from.queue(first)); + const auto stalled = from.flush(); + CHECK(stalled && *stalled == TlsProgress::want_io); + const auto second = intents(2, 3); // Queued while the first record is stuck. + CHECK(from.queue(second) && from.queued() == first.size() + second.size()); + budget = SIZE_MAX; + const auto received = pair.drain(from, to, first.size() + second.size()); + CHECK(received.size() == first.size() + second.size() && from.queued() == 0); + CHECK(sequences(received) == std::vector({1, 2, 3, 4})); + } + // Close while stalled: queued frames are flushed before close_notify. + Pair pair{pki.server, {pki.authority, server_name}}; + CHECK(pair.handshake()); + pair.wire.client_budget = 0; + const auto last = intents(9, 2); + CHECK(pair.client->queue(last)); + const auto pending = pair.client->close(); + CHECK(pending && *pending == TlsProgress::want_io); + pair.wire.client_budget = SIZE_MAX; + std::expected closed = TlsProgress::want_io; + for (int i = 0; i < 50 && closed && *closed == TlsProgress::want_io; ++i) + closed = pair.client->close(); + CHECK(closed && *closed == TlsProgress::done); + std::vector received; + std::array buffer{}; + std::expected r = 0; + for (int i = 0; i < 50 && r; ++i) { + r = pair.server->receive(buffer); + if (r) + received.insert(received.end(), buffer.begin(), + buffer.begin() + static_cast(*r)); + } + CHECK(refused(r, TlsError::closed) && + sequences(received) == std::vector({9, 10})); + // A full queue refuses without queuing or latching. + Pair full{pki.server, {pki.authority, server_name}}; + CHECK(full.handshake()); + const std::vector huge(TlsChannel::max_queued_bytes + 1, 0); + CHECK(refused(full.client->queue(huge), TlsError::backpressure) && full.client->queued() == 0); + CHECK(full.client->queue(intents(1, 1)) && full.client->queued() == intents(1, 1).size()); +} } // namespace void tls_tests() { handshake_and_frames(); rejections(); + interrupted_writes(); } } // namespace evo::network_test diff --git a/tests/region_session_tests.cpp b/tests/region_session_tests.cpp index 2843868..7d12432 100644 --- a/tests/region_session_tests.cpp +++ b/tests/region_session_tests.cpp @@ -50,7 +50,8 @@ struct Credentials { {"--synthetic-inputs", "development-v1", "--session-listen", "127.0.0.1:" + std::to_string(port), "--session-cert", (r.root / "server.pem").string(), "--session-key", - (r.root / "server.key").string(), "--session-grant-key", + (r.root / "server.key").string(), "--session-grant-ledger", + (r.root / "grants.ledger").string(), "--session-grant-key", (r.root / "grant.key").string(), "--session-environment", environment}); } std::vector grants(const ServerRegion& r, std::uint64_t subject, @@ -179,8 +180,9 @@ void connects_acknowledges_and_recovers(const Credentials& c) { const auto* outcome = last(events, ClientEvent::Kind::outcome); CHECK(outcome->outcome.finality == network::IntentFinality::committed_no_effect); first_commit = outcome->outcome.commit_sequence; - // The host acknowledged exactly that request ("INT1...") at that tick and commit. - const std::string prefix = "ack " + hex({'I', 'N', 'T', '1'}); + // The host acknowledged exactly that carrier ('I','N' + 48-bit epoch 1 + ordinal) + // at that tick and commit. + const std::string prefix = "ack " + hex({'I', 'N', 0, 0}) + "00000001"; const auto ack = std::ranges::find_if( server.lines(), [&](const std::string& l) { return starts(l, prefix); }); CHECK(ack != server.lines().end());