Fix M9.4-M9.7 review findings in the dev session stack

- TLS: replace send() with a bounded queue/flush; an interrupted record is
  retried with identical arguments and later frames wait behind it, so no
  queued frame is lost or reordered; close() flushes first.
- Carriers: server-issued 'I','N' | 48-bit epoch | 64-bit ordinal ids
  replace the aliasing low-32-bit packing; client sequences are kept
  separately for replies; out-of-range epochs are refused, never truncated.
- Grants: a durable, fsynced, fail-closed consumed-grant ledger
  (--session-grant-ledger, /objects/dev-session-grants.v1 in dev) keeps
  single-use grants consumed across process and pod restarts.
- Liveness: protocol v1 (undeployed) gains Heartbeat and the timeout close
  reason; both sides heartbeat when idle and treat 5 s of silence as dead.
- Client sends are bounded and unresolved tracking starts at send time.
- ci-client configures with an explicit C compiler.

Each finding has a regression test; ADR-0083, the schema, operations doc
and VALIDATION record the corrections.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-02 05:44:44 -04:00
co-authored by Claude Opus 5.5
parent 8e319322f4
commit c588be237c
26 changed files with 934 additions and 193 deletions
+6 -2
View File
@@ -36,7 +36,9 @@
// //
// Dev sessions (M9.7, ADR-0083; builds with EVO_BUILD_NETWORK only): --session-listen // Dev sessions (M9.7, ADR-0083; builds with EVO_BUILD_NETWORK only): --session-listen
// <numeric address:port> --session-cert F --session-key F --session-grant-key F // <numeric address:port> --session-cert F --session-key F --session-grant-key F
// --session-environment dev serve the TLS 1.3 dev session protocol. They require // --session-grant-ledger F --session-environment dev serve the TLS 1.3 dev session
// protocol. The ledger durably records consumed grants (on a persistent volume), so a
// restarted process refuses them. They require
// --synthetic-inputs: an admitted intention rides the next synthetic input (content // --synthetic-inputs: an admitted intention rides the next synthetic input (content
// unchanged), so its final outcome is the input's durable acknowledgement, with no // unchanged), so its final outcome is the input's durable acknowledgement, with no
// biological effect. The driver then looks ahead 8 ticks instead of 64. Prints // biological effect. The driver then looks ahead 8 ticks instead of 64. Prints
@@ -207,6 +209,8 @@ std::optional<Options> parse(int argc, char** argv) {
o.session.key = value; o.session.key = value;
else if (flag == "--session-grant-key") else if (flag == "--session-grant-key")
o.session.grant_key = value; o.session.grant_key = value;
else if (flag == "--session-grant-ledger")
o.session.grant_ledger = value;
else if (flag == "--session-environment" && else if (flag == "--session-environment" &&
(value == "dev" || value == "stage" || value == "world0")) (value == "dev" || value == "stage" || value == "world0"))
o.session.environment = value; o.session.environment = value;
@@ -559,7 +563,7 @@ int main(int argc, char** argv) {
"[--ops-listen ADDRESS:PORT] [--synthetic-inputs development-v1 " "[--ops-listen ADDRESS:PORT] [--synthetic-inputs development-v1 "
"[--synthetic-until TICK] [--accelerate N] [--session-listen ADDRESS:PORT " "[--synthetic-until TICK] [--accelerate N] [--session-listen ADDRESS:PORT "
"--session-cert F --session-key F --session-grant-key F " "--session-cert F --session-key F --session-grant-key F "
"--session-environment dev]]\n", "--session-grant-ledger F --session-environment dev]]\n",
stderr); stderr);
return 2; return 2;
} }
+17 -13
View File
@@ -22,14 +22,14 @@
namespace evo::server_app { namespace evo::server_app {
struct SessionOptions { struct SessionOptions {
std::string listen, certificate, key, grant_key, environment; std::string listen, certificate, key, grant_key, grant_ledger, environment;
[[nodiscard]] bool any() const noexcept { [[nodiscard]] bool any() const noexcept {
return !listen.empty() || !certificate.empty() || !key.empty() || !grant_key.empty() || return !listen.empty() || !certificate.empty() || !key.empty() || !grant_key.empty() ||
!environment.empty(); !grant_ledger.empty() || !environment.empty();
} }
[[nodiscard]] bool complete() const noexcept { [[nodiscard]] bool complete() const noexcept {
return !listen.empty() && !certificate.empty() && !key.empty() && !grant_key.empty() && return !listen.empty() && !certificate.empty() && !key.empty() && !grant_key.empty() &&
!environment.empty(); !grant_ledger.empty() && !environment.empty();
} }
}; };
// Intentions ride synthetic inputs, so sessions keep the driver's lookahead short. // Intentions ride synthetic inputs, so sessions keep the driver's lookahead short.
@@ -65,11 +65,15 @@ class SessionBridge {
config.admission.intent_window = session_lookahead_ticks; config.admission.intent_window = session_lookahead_ticks;
config.admission.window_ticks = session_lookahead_ticks; config.admission.window_ticks = session_lookahead_ticks;
config.admission.max_intents_per_window = session_lookahead_ticks; config.admission.max_intents_per_window = session_lookahead_ticks;
auto server = network::SessionServer::create(std::move(config), std::move(*grant_key)); config.grant_ledger = o.grant_ledger;
if (!server) auto server = network::SessionServer::create(std::move(config), std::move(*grant_key),
return fail(server.error() == network::SessionServerError::listen wall_seconds());
? "error session-listen" if (!server) {
const auto e = server.error();
return fail(e == network::SessionServerError::listen ? "error session-listen"
: e == network::SessionServerError::ledger ? "error session-grant-ledger"
: "error session-credentials"); : "error session-credentials");
}
bridge.server_ = std::move(*server); bridge.server_ = std::move(*server);
std::printf("session listening %u\n", static_cast<unsigned>(bridge.server_->port())); std::printf("session listening %u\n", static_cast<unsigned>(bridge.server_->port()));
return bridge; return bridge;
@@ -97,12 +101,7 @@ class SessionBridge {
if (state != nullptr && facts.ready && if (state != nullptr && facts.ready &&
(!published_ || *published_ != s.simulated_tick || epoch_ != s.epoch)) (!published_ || *published_ != s.simulated_tick || epoch_ != s.epoch))
publish(community(*state), facts); publish(community(*state), facts);
server_->poll( server_->poll(facts, wall_seconds(), std::chrono::steady_clock::now());
facts,
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count()),
std::chrono::steady_clock::now());
for (const auto& e : server_->take_events()) for (const auto& e : server_->take_events())
std::printf("session %s %llu subject=%llu reason=%u\n", std::printf("session %s %llu subject=%llu reason=%u\n",
e.kind == network::SessionEvent::Kind::opened ? "opened" e.kind == network::SessionEvent::Kind::opened ? "opened"
@@ -114,6 +113,11 @@ class SessionBridge {
} }
private: private:
static std::uint64_t wall_seconds() {
return static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count());
}
static std::optional<SessionBridge> fail(const char* message) { static std::optional<SessionBridge> fail(const char* message) {
std::puts(message); std::puts(message);
return std::nullopt; return std::nullopt;
+1 -1
View File
@@ -41,7 +41,7 @@ target_link_libraries(evo_network_tls PUBLIC evo_network_protocol PRIVATE mbedtl
evo_owned_target(evo_network_tls) evo_owned_target(evo_network_tls)
# M9.7 sessions: non-blocking TCP, the host-agnostic session server and the client. # M9.7 sessions: non-blocking TCP, the host-agnostic session server and the client.
add_library(evo_network_session STATIC src/network/socket/tcp.cpp src/network/session/server.cpp add_library(evo_network_session STATIC src/network/socket/tcp.cpp src/network/session/server.cpp
src/network/session/client.cpp) src/network/session/client.cpp src/network/session/grant_ledger.cpp)
target_link_libraries(evo_network_session PUBLIC evo_network_tls) target_link_libraries(evo_network_session PUBLIC evo_network_tls)
if(WIN32) if(WIN32)
target_link_libraries(evo_network_session PRIVATE ws2_32) target_link_libraries(evo_network_session PRIVATE ws2_32)
+1 -1
View File
@@ -40,7 +40,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
libx11-dev libxext-dev libxrandr-dev libxcursor-dev libxi-dev libxss-dev libxtst-dev xvfb xauth \ libx11-dev libxext-dev libxrandr-dev libxcursor-dev libxi-dev libxss-dev libxtst-dev xvfb xauth \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
RUN cmake -P tools/prepare-sdl3.cmake \ RUN cmake -P tools/prepare-sdl3.cmake \
&& cmake --preset client-probe -DCMAKE_CXX_COMPILER=clang++-19 \ && cmake --preset client-probe -DCMAKE_CXX_COMPILER=clang++-19 -DCMAKE_C_COMPILER=clang-19 \
-DEVO_BUILD_CLIENT=ON -DEVO_ENABLE_TIDY=ON -DSDL_X11=ON -DSDL_WAYLAND=OFF \ -DEVO_BUILD_CLIENT=ON -DEVO_ENABLE_TIDY=ON -DSDL_X11=ON -DSDL_WAYLAND=OFF \
&& cmake --build build/client-probe --target evo-client-probe evo-client evo-client-lifecycle-tests evo-client-environment-tests evo-client-organism-tests --parallel 2 \ && cmake --build build/client-probe --target evo-client-probe evo-client evo-client-lifecycle-tests evo-client-environment-tests evo-client-organism-tests --parallel 2 \
&& ctest --preset client-probe --no-tests=error \ && ctest --preset client-probe --no-tests=error \
@@ -61,10 +61,13 @@ spec:
# Dev sessions are a drill-time override too (tools/ops/session-drill.sh, # Dev sessions are a drill-time override too (tools/ops/session-drill.sh,
# ADR-0083): loopback only, reached by kubectl port-forward, never a Service. # ADR-0083): loopback only, reached by kubectl port-forward, never a Service.
# evo-server refuses any environment but dev and needs the synthetic carrier. # evo-server refuses any environment but dev and needs the synthetic carrier.
# Consumed grants persist on the objects volume (outside the object
# namespaces, ignored by backups) so a restarted pod refuses them.
sessions=() sessions=()
if [ -n "$REGION_SESSION" ]; then if [ -n "$REGION_SESSION" ]; then
sessions=(--session-listen 127.0.0.1:7400 --session-cert /run/session/server.pem sessions=(--session-listen 127.0.0.1:7400 --session-cert /run/session/server.pem
--session-key /run/session/server.key --session-grant-key /run/session/grant.key --session-key /run/session/server.key --session-grant-key /run/session/grant.key
--session-grant-ledger /objects/dev-session-grants.v1
--session-environment "$REGION_SESSION") --session-environment "$REGION_SESSION")
fi fi
exec /usr/local/bin/evo-server --profile lineage "${retained[@]}" "${inputs[@]}" \ exec /usr/local/bin/evo-server --profile lineage "${retained[@]}" "${inputs[@]}" \
+39 -12
View File
@@ -28,9 +28,12 @@ Findings that constrain the design:
**Session server in the region process.** `SessionServer` (src/network/session) is **Session server in the region process.** `SessionServer` (src/network/session) is
host-agnostic and non-blocking; `evo-server` calls it from its existing single-thread host-agnostic and non-blocking; `evo-server` calls it from its existing single-thread
loop (as it does the ops server). It never mutates host state. Per connection: loop (as it does the ops server). It never mutates host state. Per connection:
bounded output (newest view replaces an unsent one; overflow drops the connection), all output goes through the TLS channel's bounded queue (1 MiB; an interrupted TLS
1 s Close flush deadline, 5 s accept-to-Welcome deadline, 64 KiB/s input limit, record is retried with identical arguments, later frames wait behind it; a full queue
session bound to its connection (a connection can only speak for its own session). drops the slow consumer), the newest view replaces an unsent one, 1 s Close flush
deadline, 5 s accept-to-Welcome deadline, 64 KiB/s input limit, heartbeat after 1 s
without output and `timeout` close after 5 s without input, session bound to its
connection (a connection can only speak for its own session).
At most 4 connections/sessions. New epochs close existing sessions (`stale_epoch`) and At most 4 connections/sessions. New epochs close existing sessions (`stale_epoch`) and
discard intentions not yet carried. discard intentions not yet carried.
@@ -42,21 +45,37 @@ committed tick as the durable watermark. Every session starts from the latest vi
**Intentions.** Sessions require `--synthetic-inputs development-v1`. An admitted **Intentions.** Sessions require `--synthetic-inputs development-v1`. An admitted
intention is queued and rides the next synthetic input whose tick is at or after its intention is queued and rides the next synthetic input whose tick is at or after its
intended tick: the input's request id becomes `INT1 | epoch | session | sequence` intended tick under a fresh **server-issued carrier id** `'I','N' | epoch (48 bits)
(low 32 bits each); its **content stays the host-chosen development input**. The | ordinal (64 bits)`, never derived from client values; the client sequence is kept
separately for replies. Epochs beyond 48 bits are refused at Hello (`capacity`) and
ordinal exhaustion stops carrying; nothing wraps or truncates. The input's **content
stays the host-chosen development input**. The
host's FinalAck for that request yields `IntentOutcome{committed_no_effect, tick, host's FinalAck for that request yields `IntentOutcome{committed_no_effect, tick,
commit_sequence}`; a dropped input yields `not_committed`. The driver looks ahead 8 commit_sequence}`; a dropped input yields `not_committed`. The driver looks ahead 8
ticks (instead of 64) when sessions are on, matching the admission window. ticks (instead of 64) when sessions are on, matching the admission window.
**Protocol v1 amendment.** Before any v1 peer was deployed, v1 gained message type 8 **Protocol v1 amendment.** Before any v1 peer was deployed, v1 gained message types 8
`IntentOutcome` and close reason 9 `capacity` (schemas/view/dev-session-v1.md). Once `IntentOutcome` and 9 `Heartbeat` (empty) and close reasons 9 `capacity` and 10
a dev deployment runs, further changes need a new version. `timeout` (schemas/view/dev-session-v1.md). Once a dev deployment runs, further
changes need a new version.
**Grant consumption survives restarts.** Consumed grant ids are appended to a durable
ledger (`--session-grant-ledger`; `/objects/dev-session-grants.v1` in dev, outside the
object namespaces and ignored by backups) and fsynced **before** Welcome; a failed
write refuses the session and keeps the grant consumed in memory. A replacement
process loads unexpired entries into admission at startup. Corrupt content, I/O
failure or more than 65536 entries fail closed. The ledger does not survive a
restore-to-bundle (a rewind) or a lost volume; grants expire within an hour.
**Client.** `SessionClient` connects, verifies the pinned dev CA, consumes one **Client.** `SessionClient` connects, verifies the pinned dev CA, consumes one
single-use grant per Hello, and on Welcome resets both layers (fresh baseline). On single-use grant per Hello (except after `server_shutdown`/`capacity`, which the
loss it keeps the last presentation marked stale, reports outstanding admitted server refuses before recording the grant), and on Welcome resets both layers (fresh
intentions as `unknown`, and retries with capped exponential backoff until grants run baseline). It heartbeats when idle and treats 5 s without any server frame as loss.
out (explicit `exhausted`). Intended tick = newest view tick + 4. The desktop app On loss it keeps the last presentation marked stale, reports every sent-but-unresolved
intention as `unknown` (sent without a result, or admitted without an outcome), and
retries with capped exponential backoff until grants run out (explicit `exhausted`).
`send` is bounded: at most 256 unresolved intentions and the channel queue's capacity;
beyond that it refuses rather than buffers. Intended tick = newest view tick + 4. The desktop app
sends an intention only when its vector changes, at most once per view tick. sends an intention only when its vector changes, at most once per view tick.
`evo-client --connect ... --ca ... --grants ...` replaces the local fixture. `evo-client --connect ... --ca ... --grants ...` replaces the local fixture.
@@ -103,6 +122,14 @@ tokens: anyone holding an unused grant (or the dev key) can open a dev session.
Restart gaps: an intention admitted just before a crash has an unknown outcome until Restart gaps: an intention admitted just before a crash has an unknown outcome until
the client reconnects (reported as such, never assumed). the client reconnects (reported as such, never assumed).
## Review corrections (2026-10-02)
The M9.4-M9.7 review (`.cache/reviews/M9.4-M9.7-review.md`, not in Git) reproduced
and this revision fixes: TLS retry with changed arguments losing queued frames;
carrier ids aliasing sequences 2^32 apart (the original low-32-bit packing above was
unsafe and is withdrawn); consumed grants becoming valid again after a restart; no
liveness after Welcome; and the `ci-client` configure missing its C compiler. Each has
a regression test (engineering/VALIDATION.md).
## Consequences ## Consequences
M9.8 packages the client and qualifies the presentation. M10 replaces the carrier M9.8 packages the client and qualifies the presentation. M10 replaces the carrier
with real intention semantics once organisms move, and reopens transport choice. with real intention semantics once organisms move, and reopens transport choice.
+55
View File
@@ -3509,3 +3509,58 @@ were not executed remotely. Port-forward reachability of the pod-loopback listen
is the documented design, not yet observed. M8-dev/stage remains not passed. The is the documented design, not yet observed. M8-dev/stage remains not passed. The
connected desktop run was hidden/software-rendered; no human visual check of remote connected desktop run was hidden/software-rendered; no human visual check of remote
mode. Intentions have no biological effect by design. mode. Intentions have no biological effect by design.
## M9.4–M9.7 review corrections — 2026-10-02
Responds to the review of `bb0412c..8e31932` (`.cache/reviews/M9.4-M9.7-review.md`,
not in Git). All five findings were confirmed against the code and fixed:
1. **[P1] TLS retry lost queued frames.** `TlsChannel::send` is replaced by a bounded
(1 MiB) `queue` + `flush`: an interrupted record is retried with identical
arguments, later frames wait behind it, `close()` flushes before close_notify, a
full queue returns `backpressure` without queuing. Server and client route every
frame through it. Regression (`interrupted_writes`): zero progress and partial
progress then zero, frames queued during the stall, both directions, exact ordered
sequences; close while stalled; backpressure.
2. **[P1] Carrier ids aliased.** Carriers are server-issued `'I','N' | 48-bit epoch |
64-bit ordinal`; the client sequence is kept separately. Epochs beyond 48 bits are
refused at Hello; ordinal exhaustion stops carrying. ADR-0083's low-32-bit packing
is withdrawn. Regression (`carriers_never_alias`): sequences 1 and 2^32+1 get
distinct ids; `not_committed` and `committed` reach their own sequences.
3. **[P2] Grants reusable after restart.** New durable `GrantLedger` (fsync before
Welcome, atomic prune on open, fail closed on corruption/I-O/65536 entries), loaded
into admission by a replacement process; `--session-grant-ledger` (dev:
`/objects/dev-session-grants.v1`). Regressions: restart with the same ledger
refuses the consumed grant and admits a fresh one; corrupt ledger refuses server
creation; expired entries pruned; and in the end-to-end rehearsal the successor of
a SIGKILLed evo-server refused the first, consumed grant (`closed reason=3`).
4. **[P2] No liveness after Welcome.** Protocol v1 (still undeployed) gains Heartbeat
(type 9) and Close `timeout` (10). Both sides heartbeat after 1 s without output and
treat 5 s of silence as dead (server closes `timeout`, freeing the subject; client
marks stale, reports unknown outcomes, reconnects). Regressions (`liveness`): idle
but healthy session survives 1.5 s with no ticks; a silent server is detected and
reconnected to a fresh baseline; a silent client is closed and its subject reused.
5. **[P1, inherited] ci-client C compiler.** `-DCMAKE_C_COMPILER=clang-19` added; all
other C-enabling fresh configures already name it.
Follow-ups taken: `SessionClient::send` is bounded (256 unresolved, channel queue
capacity) and refuses instead of buffering; unresolved tracking now starts at send,
so a loss before IntentResult is also reported as unknown. The abuse test now floods
through a raw TLS peer (the client can no longer flood) and checks the server's
`rate_limited` decision. Client event-vector bounds remain M9.8 work.
Executed after the fixes: Windows MSVC `network` 1/1 and `client-network` 6/6; Linux
GCC 14 network suite and the **full** PostgreSQL integration suite (all suites,
including the session process test with the ledger and new carrier ids); Clang 19
ASan/UBSan network suite; clang-tidy 19 (warnings as errors) on all changed network,
client and server units (one `string_view::data` finding fixed in the ledger parser);
libFuzzer on the amended codec, 523,479 runs in 61 s, no crash or canonicality abort;
repository format, CI selector fixtures and deploy policy (67 objects, 0 failures).
End-to-end rehearsal: durable ack, SIGKILL, successor Ready in 3472 ms at epoch 2,
grants kept across two `server_shutdown` refusals, second durable ack, consumed grant
refused by the successor, Windows evo-client welcomed (`session=2 epoch=2`), successor
drained with `exit 0`.
Not done: the reviewer's own reproduction binaries target the removed `send` API and
were not rebuilt; equivalent regressions above replace them. The `ci-client` Docker
image and Gitea pipeline were not rerun here, and no cluster operation was performed.
+7 -3
View File
@@ -37,9 +37,13 @@ kubectl -n evolution-dev create secret generic evolution-dev-session \
--from-file=grant.key=$HOME/aevolution-dev-session/grant.key --from-file=grant.key=$HOME/aevolution-dev-session/grant.key
``` ```
The Deployment mounts it optionally (0440). Rotation: create new credentials, The Deployment mounts it optionally (0440). Consumed grants are recorded in
replace the Secret, restart evo-region, redistribute `dev-ca.pem`. Old grants stop `/objects/dev-session-grants.v1` on the region's persistent volume (fsynced before
verifying with the new key. each Welcome), so a restarted pod refuses grants its predecessor used. Corrupt or
unwritable ledgers fail closed (`error session-grant-ledger`, or sessions refused).
Rotation: create new credentials, replace the Secret, restart evo-region and
redistribute `dev-ca.pem`. Old grants stop verifying with the new key.
## 4. Session drill (enables sessions temporarily, then restores Git values) ## 4. Session drill (enables sessions temporarily, then restores Git values)
+9
View File
@@ -4,6 +4,7 @@
#include <cstddef> #include <cstddef>
#include <cstdint> #include <cstdint>
#include <expected> #include <expected>
#include <span>
#include <vector> #include <vector>
// Server-side session admission for the M9.6 dev session (ADR-0082). Transport- // Server-side session admission for the M9.6 dev session (ADR-0082). Transport-
@@ -17,6 +18,10 @@ struct DevGrantClaims {
world::WorldIdentity world; world::WorldIdentity world;
std::uint64_t region{}, issued_at{}, expires_at{}; // Server clock, seconds. std::uint64_t region{}, issued_at{}, expires_at{}; // Server clock, seconds.
}; };
// A grant already used to open a session, remembered until it expires.
struct ConsumedGrant {
std::uint64_t grant_id{}, expires_at{};
};
class DevGrantKey; class DevGrantKey;
// Only DevGrantKey::verify constructs this, so admission cannot be handed // Only DevGrantKey::verify constructs this, so admission cannot be handed
// unauthenticated claims by accident. // unauthenticated claims by accident.
@@ -68,6 +73,10 @@ class SessionAdmission {
[[nodiscard]] IntentResult admit(const IntentMessage& intent, std::uint64_t server_tick, [[nodiscard]] IntentResult admit(const IntentMessage& intent, std::uint64_t server_tick,
std::uint64_t now_seconds); std::uint64_t now_seconds);
[[nodiscard]] std::expected<void, AdmissionError> close(std::uint64_t session); [[nodiscard]] std::expected<void, AdmissionError> close(std::uint64_t session);
// Restores grants consumed before a restart (from a durable ledger), so a replacement
// host refuses them too. Expired entries are skipped; too many fail closed.
[[nodiscard]] std::expected<void, AdmissionError>
remember(std::span<const ConsumedGrant> consumed, std::uint64_t now_seconds);
// New ownership epoch: existing sessions become stale and must reconnect. // New ownership epoch: existing sessions become stale and must reconnect.
[[nodiscard]] std::expected<void, AdmissionError> advance_epoch(std::uint64_t epoch); [[nodiscard]] std::expected<void, AdmissionError> advance_epoch(std::uint64_t epoch);
[[nodiscard]] std::size_t sessions() const noexcept { [[nodiscard]] std::size_t sessions() const noexcept {
+42
View File
@@ -0,0 +1,42 @@
#pragma once
#include "evo/network/admission.hpp"
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <expected>
#include <filesystem>
#include <memory>
#include <span>
#include <utility>
#include <vector>
// Durable record of consumed dev grants (ADR-0083), so a replacement region process
// refuses grants its predecessor already used. Append-only text lines
// "<grant id hex> <expires_at>", flushed and fsynced before the session is welcomed.
// Opening drops expired entries by an atomic rewrite. Corrupt content, I/O failure or
// a full ledger fail closed. Single owner; not a general database.
namespace evo::network {
enum class LedgerError : std::uint8_t { io, corrupt, full };
class GrantLedger {
public:
static constexpr std::size_t max_entries = 65536;
[[nodiscard]] static std::expected<std::unique_ptr<GrantLedger>, LedgerError>
open(const std::filesystem::path& path, std::uint64_t now_seconds);
GrantLedger(const GrantLedger&) = delete;
GrantLedger& operator=(const GrantLedger&) = delete;
~GrantLedger();
// Unexpired entries loaded at open (for SessionAdmission::remember).
[[nodiscard]] std::span<const ConsumedGrant> loaded() const noexcept {
return loaded_;
}
// Durable once this returns success.
[[nodiscard]] std::expected<void, LedgerError> record(ConsumedGrant grant);
private:
GrantLedger(std::FILE* file, std::vector<ConsumedGrant> loaded)
: file_{file}, loaded_{std::move(loaded)}, entries_{loaded_.size()} {}
std::FILE* file_;
std::vector<ConsumedGrant> loaded_;
std::size_t entries_;
};
} // namespace evo::network
+6 -2
View File
@@ -28,7 +28,8 @@ enum class MessageType : std::uint8_t {
environment_view, environment_view,
organism_view, organism_view,
close, close,
intent_outcome intent_outcome,
heartbeat // Empty payload; proves liveness when no other frame flows (ADR-0083).
}; };
enum class ProtocolError : std::uint8_t { enum class ProtocolError : std::uint8_t {
truncated, truncated,
@@ -121,7 +122,8 @@ enum class CloseReason : std::uint8_t {
stale_epoch, stale_epoch,
rate_limited, rate_limited,
server_shutdown, server_shutdown,
capacity capacity,
timeout
}; };
// Final, after the host's durable commit (or loss): the M9.7 dev region has no // Final, after the host's durable commit (or loss): the M9.7 dev region has no
// movable organism, so a committed intention has no biological effect. // movable organism, so a committed intention has no biological effect.
@@ -138,6 +140,7 @@ struct IntentOutcome {
[[nodiscard]] std::vector<std::uint8_t> encode(const IntentResult& m); [[nodiscard]] std::vector<std::uint8_t> encode(const IntentResult& m);
[[nodiscard]] std::vector<std::uint8_t> encode(CloseReason reason); [[nodiscard]] std::vector<std::uint8_t> encode(CloseReason reason);
[[nodiscard]] std::vector<std::uint8_t> encode(const IntentOutcome& m); [[nodiscard]] std::vector<std::uint8_t> encode(const IntentOutcome& m);
[[nodiscard]] std::vector<std::uint8_t> encode_heartbeat();
[[nodiscard]] std::vector<std::uint8_t> encode(const ViewHeader& header, [[nodiscard]] std::vector<std::uint8_t> encode(const ViewHeader& header,
const client::EnvironmentPresentation& view); const client::EnvironmentPresentation& view);
[[nodiscard]] std::vector<std::uint8_t> encode(const ViewHeader& header, [[nodiscard]] std::vector<std::uint8_t> encode(const ViewHeader& header,
@@ -151,6 +154,7 @@ struct IntentOutcome {
[[nodiscard]] std::expected<IntentResult, ProtocolError> decode_intent_result(FrameView frame); [[nodiscard]] std::expected<IntentResult, ProtocolError> decode_intent_result(FrameView frame);
[[nodiscard]] std::expected<CloseReason, ProtocolError> decode_close(FrameView frame); [[nodiscard]] std::expected<CloseReason, ProtocolError> decode_close(FrameView frame);
[[nodiscard]] std::expected<IntentOutcome, ProtocolError> decode_intent_outcome(FrameView frame); [[nodiscard]] std::expected<IntentOutcome, ProtocolError> decode_intent_outcome(FrameView frame);
[[nodiscard]] std::expected<void, ProtocolError> decode_heartbeat(FrameView frame);
[[nodiscard]] std::expected<std::pair<ViewHeader, client::EnvironmentPresentation>, ProtocolError> [[nodiscard]] std::expected<std::pair<ViewHeader, client::EnvironmentPresentation>, ProtocolError>
decode_environment_view(FrameView frame); decode_environment_view(FrameView frame);
[[nodiscard]] std::expected<std::pair<ViewHeader, client::OrganismPresentation>, ProtocolError> [[nodiscard]] std::expected<std::pair<ViewHeader, client::OrganismPresentation>, ProtocolError>
+6 -1
View File
@@ -28,6 +28,9 @@ struct SessionClientConfig {
std::uint32_t lead_ticks{4}; // Intended tick = newest view tick + lead. std::uint32_t lead_ticks{4}; // Intended tick = newest view tick + lead.
std::chrono::milliseconds attempt_deadline{5000}; // Connect to Welcome. std::chrono::milliseconds attempt_deadline{5000}; // Connect to Welcome.
std::chrono::milliseconds first_retry{500}, max_retry{8000}; std::chrono::milliseconds first_retry{500}, max_retry{8000};
// Liveness after Welcome: heartbeat when idle; a silent server past the timeout is
// treated as lost (stale presentation, unknown outcomes, reconnect).
std::chrono::milliseconds heartbeat_interval{1000}, liveness_timeout{5000};
}; };
enum class SessionPhase : std::uint8_t { backoff, connecting, established, exhausted }; enum class SessionPhase : std::uint8_t { backoff, connecting, established, exhausted };
struct ClientEvent { struct ClientEvent {
@@ -56,7 +59,9 @@ class SessionClient {
SessionClient& operator=(const SessionClient&) = delete; SessionClient& operator=(const SessionClient&) = delete;
~SessionClient(); ~SessionClient();
void poll(std::chrono::steady_clock::time_point now); void poll(std::chrono::steady_clock::time_point now);
// Sends when established and a view has arrived; never queued across connections. // Queues one intention when established, a view has arrived, fewer than
// max_outstanding are unresolved and the channel queue has room; otherwise false.
// Never carried across connections.
[[nodiscard]] bool send(const client::Intention& intention); [[nodiscard]] bool send(const client::Intention& intention);
[[nodiscard]] SessionPhase phase() const noexcept; [[nodiscard]] SessionPhase phase() const noexcept;
[[nodiscard]] const client::EnvironmentLayer& environment() const noexcept; [[nodiscard]] const client::EnvironmentLayer& environment() const noexcept;
+11 -3
View File
@@ -1,6 +1,7 @@
#pragma once #pragma once
#include "evo/network/admission.hpp" #include "evo/network/admission.hpp"
#include "evo/network/dev_grant.hpp" #include "evo/network/dev_grant.hpp"
#include "evo/network/grant_ledger.hpp"
#include "evo/network/tcp.hpp" #include "evo/network/tcp.hpp"
#include "evo/network/tls_channel.hpp" #include "evo/network/tls_channel.hpp"
#include <array> #include <array>
@@ -9,6 +10,7 @@
#include <cstdint> #include <cstdint>
#include <deque> #include <deque>
#include <expected> #include <expected>
#include <filesystem>
#include <memory> #include <memory>
#include <optional> #include <optional>
#include <string> #include <string>
@@ -30,13 +32,18 @@ struct SessionServerConfig {
std::size_t max_input_bytes_per_second{65536}; std::size_t max_input_bytes_per_second{65536};
std::size_t max_pending_intents{64}; // Admitted, not yet carried. std::size_t max_pending_intents{64}; // Admitted, not yet carried.
std::size_t max_carried_intents{1024}; // Carried, awaiting durable/lost. std::size_t max_carried_intents{1024}; // Carried, awaiting durable/lost.
// Liveness both ways: a Heartbeat is sent when nothing else was sent for an
// interval; a peer silent for the timeout is closed with `timeout`.
std::chrono::milliseconds heartbeat_interval{1000}, liveness_timeout{5000};
// Durable consumed-grant ledger (required); survives process and pod restarts.
std::filesystem::path grant_ledger;
}; };
// The host's current authority facts, read once per loop iteration. // The host's current authority facts, read once per loop iteration.
struct RegionFacts { struct RegionFacts {
bool ready{}; bool ready{};
std::uint64_t epoch{}, simulated_tick{}, committed_tick{}, commit_sequence{}; std::uint64_t epoch{}, simulated_tick{}, committed_tick{}, commit_sequence{};
}; };
enum class SessionServerError : std::uint8_t { configuration, listen, credentials }; enum class SessionServerError : std::uint8_t { configuration, listen, credentials, ledger };
struct SessionEvent { struct SessionEvent {
enum class Kind : std::uint8_t { opened, closed, refused } kind{}; enum class Kind : std::uint8_t { opened, closed, refused } kind{};
std::uint64_t session{}, subject{}; std::uint64_t session{}, subject{};
@@ -45,7 +52,7 @@ struct SessionEvent {
class SessionServer { class SessionServer {
public: public:
[[nodiscard]] static std::expected<std::unique_ptr<SessionServer>, SessionServerError> [[nodiscard]] static std::expected<std::unique_ptr<SessionServer>, SessionServerError>
create(SessionServerConfig config, DevGrantKey key); create(SessionServerConfig config, DevGrantKey key, std::uint64_t now_seconds);
SessionServer(const SessionServer&) = delete; SessionServer(const SessionServer&) = delete;
SessionServer& operator=(const SessionServer&) = delete; SessionServer& operator=(const SessionServer&) = delete;
~SessionServer(); ~SessionServer();
@@ -59,7 +66,8 @@ class SessionServer {
void publish(const client::EnvironmentPresentation& environment, void publish(const client::EnvironmentPresentation& environment,
const client::OrganismPresentation& organisms, const RegionFacts& facts); const client::OrganismPresentation& organisms, const RegionFacts& facts);
// The oldest admitted intention whose intended tick is <= `tick`, to ride the host // The oldest admitted intention whose intended tick is <= `tick`, to ride the host
// input for `tick`. The host input's content stays host-chosen. // input for `tick`, under a fresh server-issued carrier id ('I','N', 48-bit epoch,
// 64-bit ordinal). The host input's content stays host-chosen.
[[nodiscard]] std::optional<RequestId> carry(std::uint64_t tick); [[nodiscard]] std::optional<RequestId> carry(std::uint64_t tick);
// Host final acknowledgement / drop for a request id; ids not carried here are ignored. // Host final acknowledgement / drop for a request id; ids not carried here are ignored.
void committed(const RequestId& request, std::uint64_t tick, std::uint64_t commit_sequence); void committed(const RequestId& request, std::uint64_t tick, std::uint64_t commit_sequence);
+12 -3
View File
@@ -37,7 +37,8 @@ enum class TlsError : std::uint8_t {
peer_verification, peer_verification,
transport, transport,
closed, closed,
record record,
backpressure
}; };
enum class TlsProgress : std::uint8_t { done, want_io }; enum class TlsProgress : std::uint8_t { done, want_io };
class TlsChannel { class TlsChannel {
@@ -50,9 +51,17 @@ class TlsChannel {
TlsChannel& operator=(const TlsChannel&) = delete; TlsChannel& operator=(const TlsChannel&) = delete;
~TlsChannel(); ~TlsChannel();
[[nodiscard]] std::expected<TlsProgress, TlsError> handshake(); [[nodiscard]] std::expected<TlsProgress, TlsError> handshake();
// Zero means "would block". Errors latch until destruction. // Outbound plaintext is queued, never handed to the caller to retry: a record that
[[nodiscard]] std::expected<std::size_t, TlsError> send(std::span<const std::uint8_t> bytes); // TLS could not finish writing is retried with identical arguments (Mbed TLS
// requires this), and newly queued bytes wait behind it. Bounded; a full queue
// returns `backpressure` and queues nothing. Errors latch until destruction.
static constexpr std::size_t max_queued_bytes = std::size_t{1} << 20;
[[nodiscard]] std::expected<void, TlsError> queue(std::span<const std::uint8_t> bytes);
// Writes queued plaintext until done or the transport would block.
[[nodiscard]] std::expected<TlsProgress, TlsError> flush();
[[nodiscard]] std::size_t queued() const noexcept;
[[nodiscard]] std::expected<std::size_t, TlsError> receive(std::span<std::uint8_t> bytes); [[nodiscard]] std::expected<std::size_t, TlsError> receive(std::span<std::uint8_t> bytes);
// Flushes queued plaintext first, then sends close_notify (want_io until both done).
[[nodiscard]] std::expected<TlsProgress, TlsError> close(); [[nodiscard]] std::expected<TlsProgress, TlsError> close();
// Negotiated "TLSv1.3" and suite name after a completed handshake. // Negotiated "TLSv1.3" and suite name after a completed handshake.
[[nodiscard]] std::string_view version() const noexcept; [[nodiscard]] std::string_view version() const noexcept;
+5 -4
View File
@@ -12,7 +12,7 @@ message re-encodes to identical bytes (fuzz-tested).
|---|---|---|---| |---|---|---|---|
| 0 | magic | u32 | `0x53445645` (bytes `E V D S`) | | 0 | magic | u32 | `0x53445645` (bytes `E V D S`) |
| 4 | version | u16 | exactly 1, otherwise `unsupported_version` | | 4 | version | u16 | exactly 1, otherwise `unsupported_version` |
| 6 | type | u8 | 1..8, otherwise `unknown_type` | | 6 | type | u8 | 1..9, otherwise `unknown_type` |
| 7 | flags | u8 | 0, otherwise `reserved_flags` | | 7 | flags | u8 | 0, otherwise `reserved_flags` |
| 8 | length | u32 | payload bytes, <= 262144 (`oversized`) | | 8 | length | u32 | payload bytes, <= 262144 (`oversized`) |
| 12 | payload | bytes | exactly `length`; short `truncated`, extra `trailing_bytes` | | 12 | payload | bytes | exactly `length`; short `truncated`, extra `trailing_bytes` |
@@ -30,15 +30,16 @@ Payload decoders also reject truncation, trailing bytes, out-of-range enum bytes
| 4 | IntentResult | S->C | sequence u64, status u8 (0..7), server_tick u64 | | 4 | IntentResult | S->C | sequence u64, status u8 (0..7), server_tick u64 |
| 5 | EnvironmentView | S->C | view header, environment presentation | | 5 | EnvironmentView | S->C | view header, environment presentation |
| 6 | OrganismView | S->C | view header, organism presentation | | 6 | OrganismView | S->C | view header, organism presentation |
| 7 | Close | both | reason u8 (0..9) | | 7 | Close | both | reason u8 (0..10) |
| 8 | IntentOutcome | S->C | sequence u64, finality u8 (0 committed_no_effect, 1 not_committed), tick u64, commit_sequence u64; committed requires commit_sequence != 0, not_committed requires tick = commit_sequence = 0 | | 8 | IntentOutcome | S->C | sequence u64, finality u8 (0 committed_no_effect, 1 not_committed), tick u64, commit_sequence u64; committed requires commit_sequence != 0, not_committed requires tick = commit_sequence = 0 |
| 9 | Heartbeat | both | empty (length 0); sent after 1 s without other output; 5 s of silence closes with `timeout` |
IntentStatus: 0 admitted, 1 unknown_session, 2 stale_epoch, 3 expired, IntentStatus: 0 admitted, 1 unknown_session, 2 stale_epoch, 3 expired,
4 invalid_vector, 5 replayed, 6 outside_window, 7 rate_limited. 4 invalid_vector, 5 replayed, 6 outside_window, 7 rate_limited.
CloseReason: 0 normal, 1 unsupported_version, 2 malformed, 3 unauthenticated, CloseReason: 0 normal, 1 unsupported_version, 2 malformed, 3 unauthenticated,
4 expired, 5 duplicate_session, 6 stale_epoch, 7 rate_limited, 8 server_shutdown, 4 expired, 5 duplicate_session, 6 stale_epoch, 7 rate_limited, 8 server_shutdown,
9 capacity. Type 8 and reason 9 were added in M9.7 before any v1 peer was 9 capacity, 10 timeout. Types 8-9 and reasons 9-10 were added in M9.7 before any v1
deployed (ADR-0083); later changes require a new version. peer was deployed (ADR-0083); later changes require a new version.
View header (40 bytes): session, epoch, server_tick, commit_watermark, baseline (u64). View header (40 bytes): session, epoch, server_tick, commit_watermark, baseline (u64).
+14
View File
@@ -103,6 +103,20 @@ IntentResult SessionAdmission::admit(const IntentMessage& intent, std::uint64_t
return result; return result;
} }
std::expected<void, AdmissionError>
SessionAdmission::remember(std::span<const ConsumedGrant> consumed, std::uint64_t now_seconds) {
for (const auto& g : consumed) {
if (g.expires_at <= now_seconds || std::ranges::any_of(grants_, [&](const Remembered& r) {
return r.grant_id == g.grant_id;
}))
continue;
if (grants_.size() >= config_.max_remembered_grants)
return std::unexpected(AdmissionError::grants_full);
grants_.push_back({g.grant_id, g.expires_at});
}
return {};
}
std::expected<void, AdmissionError> SessionAdmission::close(std::uint64_t session) { std::expected<void, AdmissionError> SessionAdmission::close(std::uint64_t session) {
if (std::erase_if(sessions_, [&](const Session& s) { return s.id == session; }) == 0) if (std::erase_if(sessions_, [&](const Session& s) { return s.id == session; }) == 0)
return std::unexpected(AdmissionError::unknown_session); return std::unexpected(AdmissionError::unknown_session);
+14 -2
View File
@@ -5,9 +5,9 @@
namespace evo::network { namespace evo::network {
namespace { namespace {
constexpr std::uint8_t last_type = static_cast<std::uint8_t>(MessageType::intent_outcome); constexpr std::uint8_t last_type = static_cast<std::uint8_t>(MessageType::heartbeat);
constexpr std::uint8_t last_status = static_cast<std::uint8_t>(IntentStatus::rate_limited); constexpr std::uint8_t last_status = static_cast<std::uint8_t>(IntentStatus::rate_limited);
constexpr std::uint8_t last_reason = static_cast<std::uint8_t>(CloseReason::capacity); constexpr std::uint8_t last_reason = static_cast<std::uint8_t>(CloseReason::timeout);
constexpr std::uint8_t last_finality = static_cast<std::uint8_t>(IntentFinality::not_committed); constexpr std::uint8_t last_finality = static_cast<std::uint8_t>(IntentFinality::not_committed);
constexpr std::uint8_t last_presence = static_cast<std::uint8_t>(client::FieldPresence::present); constexpr std::uint8_t last_presence = static_cast<std::uint8_t>(client::FieldPresence::present);
constexpr std::uint8_t last_moisture = static_cast<std::uint8_t>(world::MoistureStatus::available); constexpr std::uint8_t last_moisture = static_cast<std::uint8_t>(world::MoistureStatus::available);
@@ -280,6 +280,18 @@ std::vector<std::uint8_t> encode(CloseReason reason) {
return framed(MessageType::close, w); return framed(MessageType::close, w);
} }
std::vector<std::uint8_t> encode_heartbeat() {
return encode_frame(MessageType::heartbeat, {});
}
std::expected<void, ProtocolError> decode_heartbeat(FrameView frame) {
if (frame.type != MessageType::heartbeat)
return std::unexpected(ProtocolError::wrong_type);
if (!frame.payload.empty())
return std::unexpected(ProtocolError::trailing_bytes);
return {};
}
std::vector<std::uint8_t> encode(const IntentOutcome& m) { std::vector<std::uint8_t> encode(const IntentOutcome& m) {
Writer w; Writer w;
w.u64(m.sequence); w.u64(m.sequence);
+49 -27
View File
@@ -18,15 +18,16 @@ struct SessionClient::Impl {
std::size_t next_grant{}; std::size_t next_grant{};
SessionPhase phase{SessionPhase::backoff}; SessionPhase phase{SessionPhase::backoff};
Step step{Step::handshake}; Step step{Step::handshake};
Clock::time_point retry_at{}, deadline{}; Clock::time_point retry_at{}, deadline{}, last_received{}, last_sent{}, polled_at{};
std::chrono::milliseconds delay; std::chrono::milliseconds delay;
std::unique_ptr<TcpStream> stream; // Declared before the channel that borrows it. std::unique_ptr<TcpStream> stream; // Declared before the channel that borrows it.
std::unique_ptr<TlsChannel> tls; std::unique_ptr<TlsChannel> tls;
FrameReader reader; FrameReader reader;
std::vector<std::uint8_t> out;
std::optional<Welcome> session; std::optional<Welcome> session;
std::optional<ViewHeader> view; std::optional<ViewHeader> view;
std::vector<std::uint64_t> outstanding; // Admitted, awaiting a final outcome. // Sent and not finally resolved: no IntentResult yet, or admitted without an
// outcome. A connection loss reports every one of them as unknown.
std::vector<std::uint64_t> outstanding;
client::EnvironmentLayer environment; client::EnvironmentLayer environment;
client::OrganismLayer organisms; client::OrganismLayer organisms;
std::vector<ClientEvent> events; std::vector<ClientEvent> events;
@@ -45,7 +46,6 @@ struct SessionClient::Impl {
tls.reset(); tls.reset();
stream.reset(); stream.reset();
reader = FrameReader{}; reader = FrameReader{};
out.clear();
session.reset(); session.reset();
view.reset(); view.reset();
// Keep showing the last presentation, explicitly stale, until a new baseline. // Keep showing the last presentation, explicitly stale, until a new baseline.
@@ -77,9 +77,18 @@ struct SessionClient::Impl {
phase = SessionPhase::connecting; phase = SessionPhase::connecting;
step = Step::handshake; step = Step::handshake;
deadline = now + config.attempt_deadline; deadline = now + config.attempt_deadline;
last_received = last_sent = now;
}
// Every outbound frame goes through the channel's bounded queue.
bool queue(const std::vector<std::uint8_t>& frame, Clock::time_point now) {
if (!tls || !tls->queue(frame))
return false;
last_sent = now;
return true;
} }
bool frame(const OwnedFrame& f, Clock::time_point now) { bool frame(const OwnedFrame& f, Clock::time_point now) {
const FrameView v{f.type, f.payload}; const FrameView v{f.type, f.payload};
last_received = now;
if (step == Step::awaiting_welcome) { if (step == Step::awaiting_welcome) {
if (f.type == MessageType::close) { if (f.type == MessageType::close) {
const auto reason = decode_close(v); const auto reason = decode_close(v);
@@ -132,14 +141,9 @@ struct SessionClient::Impl {
disconnect(now, std::nullopt); disconnect(now, std::nullopt);
return false; return false;
} }
if (r->status == IntentStatus::admitted) { // A rejection is final; an admitted intention waits for its outcome.
if (outstanding.size() >= max_outstanding) { if (r->status != IntentStatus::admitted)
events.push_back( std::erase(outstanding, r->sequence);
{ClientEvent::Kind::unknown, {}, {}, {}, {}, outstanding.front()});
outstanding.erase(outstanding.begin());
}
outstanding.push_back(r->sequence);
}
events.push_back({ClientEvent::Kind::result, {}, {}, *r, {}, r->sequence}); events.push_back({ClientEvent::Kind::result, {}, {}, *r, {}, r->sequence});
return true; return true;
} }
@@ -153,6 +157,12 @@ struct SessionClient::Impl {
events.push_back({ClientEvent::Kind::outcome, {}, {}, {}, *o, o->sequence}); events.push_back({ClientEvent::Kind::outcome, {}, {}, {}, *o, o->sequence});
return true; return true;
} }
case MessageType::heartbeat:
if (!decode_heartbeat(v)) {
disconnect(now, std::nullopt);
return false;
}
return true;
case MessageType::close: { case MessageType::close: {
const auto reason = decode_close(v); const auto reason = decode_close(v);
disconnect(now, reason.value_or(CloseReason::malformed)); disconnect(now, reason.value_or(CloseReason::malformed));
@@ -164,18 +174,8 @@ struct SessionClient::Impl {
} }
} }
void flush(Clock::time_point now) { void flush(Clock::time_point now) {
std::size_t sent = 0; if (!tls->flush())
while (sent < out.size()) {
const auto n = tls->send(std::span{out}.subspan(sent));
if (!n) {
disconnect(now, std::nullopt); disconnect(now, std::nullopt);
return;
}
if (*n == 0)
break;
sent += *n;
}
out.erase(out.begin(), out.begin() + static_cast<std::ptrdiff_t>(sent));
} }
void receive(Clock::time_point now) { void receive(Clock::time_point now) {
std::array<std::uint8_t, 16384> buffer{}; std::array<std::uint8_t, 16384> buffer{};
@@ -215,7 +215,8 @@ SessionClient::create(SessionClientConfig config) {
if (config.host.empty() || config.port == 0 || config.grants.empty() || if (config.host.empty() || config.port == 0 || config.grants.empty() ||
config.grants.size() > 1024 || config.lead_ticks == 0 || config.lead_ticks > 64 || config.grants.size() > 1024 || config.lead_ticks == 0 || config.lead_ticks > 64 ||
config.first_retry.count() <= 0 || config.max_retry < config.first_retry || config.first_retry.count() <= 0 || config.max_retry < config.first_retry ||
config.attempt_deadline.count() <= 0) config.attempt_deadline.count() <= 0 || config.heartbeat_interval.count() <= 0 ||
config.liveness_timeout <= 2 * config.heartbeat_interval)
return std::unexpected(SessionClientError::configuration); return std::unexpected(SessionClientError::configuration);
return std::unique_ptr<SessionClient>( return std::unique_ptr<SessionClient>(
new SessionClient(std::make_unique<Impl>(std::move(config)))); new SessionClient(std::make_unique<Impl>(std::move(config))));
@@ -223,6 +224,7 @@ SessionClient::create(SessionClientConfig config) {
void SessionClient::poll(std::chrono::steady_clock::time_point now) { void SessionClient::poll(std::chrono::steady_clock::time_point now) {
auto& m = *impl_; auto& m = *impl_;
m.polled_at = now;
if (m.phase == SessionPhase::exhausted) if (m.phase == SessionPhase::exhausted)
return; return;
if (m.phase == SessionPhase::backoff) { if (m.phase == SessionPhase::backoff) {
@@ -236,6 +238,18 @@ void SessionClient::poll(std::chrono::steady_clock::time_point now) {
m.disconnect(now, std::nullopt); m.disconnect(now, std::nullopt);
return; return;
} }
if (m.step == Impl::Step::established) {
// A silent server (paused process, black-holed path) is treated as lost: the
// presentation turns stale and the reconnect policy takes over.
if (now - m.last_received > m.config.liveness_timeout) {
m.disconnect(now, std::nullopt);
return;
}
if (now - m.last_sent >= m.config.heartbeat_interval && !m.queue(encode_heartbeat(), now)) {
m.disconnect(now, std::nullopt);
return;
}
}
if (m.step == Impl::Step::handshake) { if (m.step == Impl::Step::handshake) {
const auto p = m.tls->handshake(); const auto p = m.tls->handshake();
if (!p) { if (!p) {
@@ -251,7 +265,10 @@ void SessionClient::poll(std::chrono::steady_clock::time_point now) {
m.disconnect(now, std::nullopt); m.disconnect(now, std::nullopt);
return; return;
} }
m.out.insert(m.out.end(), hello->begin(), hello->end()); if (!m.queue(*hello, now)) {
m.disconnect(now, std::nullopt);
return;
}
m.step = Impl::Step::awaiting_welcome; m.step = Impl::Step::awaiting_welcome;
} }
m.flush(now); m.flush(now);
@@ -263,13 +280,18 @@ void SessionClient::poll(std::chrono::steady_clock::time_point now) {
bool SessionClient::send(const client::Intention& intention) { bool SessionClient::send(const client::Intention& intention) {
auto& m = *impl_; auto& m = *impl_;
if (m.phase != SessionPhase::established || !m.session || !m.view) // Bounded: no session, no view, too many unresolved intentions or a full channel
// queue all refuse rather than buffer (the caller resamples later).
if (m.phase != SessionPhase::established || !m.session || !m.view ||
m.outstanding.size() >= max_outstanding)
return false; return false;
const auto tick = m.view->server_tick + m.config.lead_ticks; const auto tick = m.view->server_tick + m.config.lead_ticks;
const auto bytes = const auto bytes =
encode(make_intent(m.session->session, m.session->epoch, encode(make_intent(m.session->session, m.session->epoch,
{intention.sequence, tick, intention.move_x, intention.move_y})); {intention.sequence, tick, intention.move_x, intention.move_y}));
m.out.insert(m.out.end(), bytes.begin(), bytes.end()); if (!m.queue(bytes, m.polled_at))
return false;
m.outstanding.push_back(intention.sequence);
return true; return true;
} }
+122
View File
@@ -0,0 +1,122 @@
#include "evo/network/grant_ledger.hpp"
#include <charconv>
#include <fstream>
#include <string>
#include <string_view>
#include <system_error>
#ifdef _WIN32
#include <io.h>
#else
#include <fcntl.h>
#include <unistd.h>
#endif
namespace evo::network {
namespace {
std::FILE* open_file(const std::filesystem::path& path, const char* mode) {
#ifdef _WIN32
std::FILE* file = nullptr;
return _wfopen_s(&file, path.c_str(), mode[0] == 'w' ? L"wb" : L"ab") == 0 ? file : nullptr;
#else
return std::fopen(path.c_str(), mode);
#endif
}
bool sync(std::FILE* file) {
if (std::fflush(file) != 0)
return false;
#ifdef _WIN32
return _commit(_fileno(file)) == 0;
#else
return ::fsync(::fileno(file)) == 0;
#endif
}
// Makes a rename durable on POSIX by syncing the containing directory.
bool sync_directory([[maybe_unused]] const std::filesystem::path& dir) {
#ifdef _WIN32
return true; // NTFS metadata journaling; MoveFileEx is used with replace semantics.
#else
const int fd = ::open(dir.empty() ? "." : dir.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC);
if (fd < 0)
return false;
const bool ok = ::fsync(fd) == 0;
return ::close(fd) == 0 && ok;
#endif
}
std::string line(ConsumedGrant g) {
constexpr char digits[] = "0123456789abcdef";
std::string out(16, '0');
for (int i = 15; i >= 0; --i, g.grant_id >>= 4U)
out[static_cast<std::size_t>(i)] = digits[g.grant_id & 0x0FU];
return out + ' ' + std::to_string(g.expires_at) + '\n';
}
bool parse(const std::string& text, ConsumedGrant& out) { // One getline() line.
if (text.size() < 18 || text[16] != ' ')
return false;
const auto [a, e1] = std::from_chars(text.data(), text.data() + 16, out.grant_id, 16);
const auto [b, e2] =
std::from_chars(text.data() + 17, text.data() + text.size(), out.expires_at);
return e1 == std::errc{} && a == text.data() + 16 && e2 == std::errc{} &&
b == text.data() + text.size() && out.grant_id != 0;
}
} // namespace
std::expected<std::unique_ptr<GrantLedger>, LedgerError>
GrantLedger::open(const std::filesystem::path& path, std::uint64_t now_seconds) {
std::vector<ConsumedGrant> live;
std::error_code error;
if (std::filesystem::exists(path, error)) {
std::ifstream in{path, std::ios::binary};
if (!in)
return std::unexpected(LedgerError::io);
std::string text;
std::size_t total = 0;
while (std::getline(in, text)) {
ConsumedGrant g;
if (!parse(text, g))
return std::unexpected(LedgerError::corrupt); // Never guess: fail closed.
if (++total > max_entries)
return std::unexpected(LedgerError::full);
if (g.expires_at > now_seconds)
live.push_back(g);
}
if (!in.eof())
return std::unexpected(LedgerError::io);
} else if (error) {
return std::unexpected(LedgerError::io);
}
// Rewrite the unexpired entries atomically, then append to the result.
const auto temporary = std::filesystem::path{path}.concat(".tmp");
{
std::FILE* out = open_file(temporary, "wb");
if (out == nullptr)
return std::unexpected(LedgerError::io);
bool ok = true;
for (const auto& g : live)
ok = ok && std::fputs(line(g).c_str(), out) >= 0;
ok = sync(out) && ok;
if (std::fclose(out) != 0 || !ok)
return std::unexpected(LedgerError::io);
}
std::filesystem::rename(temporary, path, error);
if (error || !sync_directory(path.parent_path()))
return std::unexpected(LedgerError::io);
std::FILE* file = open_file(path, "ab");
if (file == nullptr)
return std::unexpected(LedgerError::io);
return std::unique_ptr<GrantLedger>(new GrantLedger(file, std::move(live)));
}
GrantLedger::~GrantLedger() {
std::fclose(file_);
}
std::expected<void, LedgerError> GrantLedger::record(ConsumedGrant grant) {
if (entries_ >= max_entries)
return std::unexpected(LedgerError::full);
if (std::fputs(line(grant).c_str(), file_) < 0 || !sync(file_))
return std::unexpected(LedgerError::io);
++entries_;
return {};
}
} // namespace evo::network
+125 -80
View File
@@ -1,5 +1,6 @@
#include "evo/network/session_server.hpp" #include "evo/network/session_server.hpp"
#include <algorithm> #include <algorithm>
#include <limits>
#include <utility> #include <utility>
namespace evo::network { namespace evo::network {
@@ -7,6 +8,9 @@ namespace {
using Clock = std::chrono::steady_clock; using Clock = std::chrono::steady_clock;
constexpr std::size_t max_accepts_per_poll = 4; constexpr std::size_t max_accepts_per_poll = 4;
constexpr std::size_t max_read_per_poll = 65536; constexpr std::size_t max_read_per_poll = 65536;
// Carrier ids are 'I','N' + 48-bit epoch + 64-bit server ordinal: distinct for every
// carried intention, independent of client-chosen sequences (which may be any u64).
constexpr std::uint64_t max_carrier_epoch = (std::uint64_t{1} << 48) - 1;
CloseReason close_reason(AdmissionError e) { CloseReason close_reason(AdmissionError e) {
switch (e) { switch (e) {
case AdmissionError::duplicate_subject: case AdmissionError::duplicate_subject:
@@ -24,8 +28,13 @@ CloseReason close_reason(ProtocolError e) {
return e == ProtocolError::unsupported_version ? CloseReason::unsupported_version return e == ProtocolError::unsupported_version ? CloseReason::unsupported_version
: CloseReason::malformed; : CloseReason::malformed;
} }
void append(std::vector<std::uint8_t>& out, const std::vector<std::uint8_t>& bytes) { RequestId carrier_id(std::uint64_t epoch, std::uint64_t ordinal) {
out.insert(out.end(), bytes.begin(), bytes.end()); RequestId id{'I', 'N'};
for (unsigned i = 0; i < 6; ++i)
id.at(2 + i) = static_cast<std::uint8_t>(epoch >> (8U * (5U - i)));
for (unsigned i = 0; i < 8; ++i)
id.at(8 + i) = static_cast<std::uint8_t>(ordinal >> (8U * (7U - i)));
return id;
} }
} // namespace } // namespace
@@ -34,18 +43,16 @@ struct SessionServer::Impl {
struct Connection { struct Connection {
std::uint64_t id{}; std::uint64_t id{};
std::unique_ptr<TcpStream> stream; // Declared first: outlives the channel. std::unique_ptr<TcpStream> stream; // Declared first: outlives the channel.
std::unique_ptr<TlsChannel> tls; std::unique_ptr<TlsChannel> tls; // Owns all queued output (ADR-0083 review fix).
State state{State::handshake}; State state{State::handshake};
Clock::time_point deadline; Clock::time_point deadline, last_received, last_sent, window_start;
FrameReader reader; FrameReader reader;
std::vector<std::uint8_t> out, view; std::vector<std::uint8_t> view; // Newest unsent view only; replaced, never queued.
std::uint64_t session{}, subject{}, epoch{}, expires_at{}; std::uint64_t session{}, subject{}, epoch{}, expires_at{};
std::size_t window_bytes{}; std::size_t window_bytes{};
Clock::time_point window_start;
bool dead{}; bool dead{};
}; };
struct Pending { struct Pending {
RequestId request{};
std::uint64_t intended_tick{}, connection{}, sequence{}; std::uint64_t intended_tick{}, connection{}, sequence{};
}; };
struct Carried { struct Carried {
@@ -53,13 +60,16 @@ struct SessionServer::Impl {
std::uint64_t connection{}, sequence{}; std::uint64_t connection{}, sequence{};
}; };
Impl(SessionServerConfig c, DevGrantKey k, std::unique_ptr<TcpListener> l) Impl(SessionServerConfig c, DevGrantKey k, std::unique_ptr<TcpListener> l,
: config{std::move(c)}, key{std::move(k)}, listener{std::move(l)} {} std::unique_ptr<GrantLedger> g)
: config{std::move(c)}, key{std::move(k)}, listener{std::move(l)}, ledger{std::move(g)} {}
SessionServerConfig config; SessionServerConfig config;
DevGrantKey key; DevGrantKey key;
std::unique_ptr<TcpListener> listener; std::unique_ptr<TcpListener> listener;
std::unique_ptr<GrantLedger> ledger;
std::optional<SessionAdmission> admission; std::optional<SessionAdmission> admission;
bool ledger_rejected{}; // Recorded grants exceed admission capacity: fail closed.
std::vector<Connection> connections; std::vector<Connection> connections;
std::deque<Pending> pending; std::deque<Pending> pending;
std::vector<Carried> carried; std::vector<Carried> carried;
@@ -67,8 +77,8 @@ struct SessionServer::Impl {
std::optional<client::EnvironmentPresentation> environment; std::optional<client::EnvironmentPresentation> environment;
std::optional<client::OrganismPresentation> organisms; std::optional<client::OrganismPresentation> organisms;
RegionFacts published{}; RegionFacts published{};
std::uint64_t baseline{}, next_connection{1}; std::uint64_t baseline{}, next_connection{1}, next_carrier{1};
Clock::time_point polled_at{}; // Time of the current poll, for Close deadlines. Clock::time_point polled_at{};
Connection* find(std::uint64_t id) { Connection* find(std::uint64_t id) {
const auto it = std::ranges::find_if(connections, [&](const Connection& c) { const auto it = std::ranges::find_if(connections, [&](const Connection& c) {
@@ -76,16 +86,23 @@ struct SessionServer::Impl {
}); });
return it == connections.end() ? nullptr : &*it; return it == connections.end() ? nullptr : &*it;
} }
// Every outbound frame goes through the channel's bounded queue; a full queue means
// a consumer that cannot keep up, which is dropped rather than buffered further.
void send(Connection& c, const std::vector<std::uint8_t>& frame) {
if (c.dead || !c.tls->queue(frame))
c.dead = true;
c.last_sent = polled_at;
}
void close(Connection& c, CloseReason reason) { void close(Connection& c, CloseReason reason) {
if (c.state == State::closing || c.dead) if (c.state == State::closing || c.dead)
return; return;
if (c.state == State::hello || c.state == State::established)
append(c.out, encode(reason));
else
c.dead = true; // No channel yet to say anything over.
events.push_back({c.state == State::established ? SessionEvent::Kind::closed events.push_back({c.state == State::established ? SessionEvent::Kind::closed
: SessionEvent::Kind::refused, : SessionEvent::Kind::refused,
c.session, c.subject, reason}); c.session, c.subject, reason});
if (c.state == State::hello || c.state == State::established)
send(c, encode(reason));
else
c.dead = true; // No channel yet to say anything over.
c.state = State::closing; c.state = State::closing;
c.deadline = polled_at + std::chrono::seconds{1}; // Bounded flush of the Close frame. c.deadline = polled_at + std::chrono::seconds{1}; // Bounded flush of the Close frame.
c.view.clear(); c.view.clear();
@@ -96,8 +113,11 @@ struct SessionServer::Impl {
return bytes; return bytes;
const ViewHeader header{c.session, c.epoch, published.simulated_tick, const ViewHeader header{c.session, c.epoch, published.simulated_tick,
published.committed_tick, baseline}; published.committed_tick, baseline};
append(bytes, encode(header, *environment)); const auto a = encode(header, *environment);
append(bytes, encode(header, *organisms)); const auto b = encode(header, *organisms);
bytes.reserve(a.size() + b.size());
bytes.insert(bytes.end(), a.begin(), a.end());
bytes.insert(bytes.end(), b.begin(), b.end());
return bytes; return bytes;
} }
void hello(Connection& c, const OwnedFrame& f, const RegionFacts& facts, void hello(Connection& c, const OwnedFrame& f, const RegionFacts& facts,
@@ -115,24 +135,34 @@ struct SessionServer::Impl {
close(c, CloseReason::server_shutdown); // No current authority: retry later. close(c, CloseReason::server_shutdown); // No current authority: retry later.
return; return;
} }
if (facts.epoch > max_carrier_epoch) {
close(c, CloseReason::capacity); // Carrier ids cannot name this epoch: refuse.
return;
}
const auto grant = key.verify(message->grant); const auto grant = key.verify(message->grant);
if (!grant) { if (!grant) {
close(c, CloseReason::unauthenticated); close(c, CloseReason::unauthenticated);
return; return;
} }
c.subject = grant->claims().subject;
const auto welcome = const auto welcome =
admission->open(*grant, now_seconds, facts.simulated_tick, facts.committed_tick); admission->open(*grant, now_seconds, facts.simulated_tick, facts.committed_tick);
if (!welcome) { if (!welcome) {
c.subject = grant->claims().subject;
close(c, close_reason(welcome.error())); close(c, close_reason(welcome.error()));
return; return;
} }
// Durable before Welcome: a replacement process must refuse this grant too.
if (!ledger->record({grant->claims().grant_id, grant->claims().expires_at})) {
// The grant stays consumed in memory (fail closed); only the session is undone.
static_cast<void>(admission->close(welcome->session).has_value());
close(c, CloseReason::server_shutdown);
return;
}
c.session = welcome->session; c.session = welcome->session;
c.subject = grant->claims().subject;
c.epoch = welcome->epoch; c.epoch = welcome->epoch;
c.expires_at = welcome->grant_expires_at; c.expires_at = welcome->grant_expires_at;
c.state = State::established; c.state = State::established;
append(c.out, encode(*welcome)); send(c, encode(*welcome));
c.view = views_for(c); // Fresh baseline for every new session. c.view = views_for(c); // Fresh baseline for every new session.
events.push_back({SessionEvent::Kind::opened, c.session, c.subject, {}}); events.push_back({SessionEvent::Kind::opened, c.session, c.subject, {}});
} }
@@ -152,16 +182,8 @@ struct SessionServer::Impl {
result = admission->admit(*m, facts.simulated_tick, now_seconds); result = admission->admit(*m, facts.simulated_tick, now_seconds);
} }
if (result.status == IntentStatus::admitted) if (result.status == IntentStatus::admitted)
pending.push_back({request_id(c, m->sequence), m->tick, c.id, m->sequence}); pending.push_back({m->tick, c.id, m->sequence});
append(c.out, encode(result)); send(c, encode(result));
}
RequestId request_id(const Connection& c, std::uint64_t sequence) const {
RequestId id{'I', 'N', 'T', '1'};
const std::array<std::uint64_t, 3> parts{c.epoch, c.session, sequence};
for (std::size_t p = 0; p < parts.size(); ++p)
for (unsigned i = 0; i < 4; ++i)
id.at(4 + 4 * p + i) = static_cast<std::uint8_t>(parts.at(p) >> (8U * (3U - i)));
return id;
} }
void receive(Connection& c, const RegionFacts& facts, std::uint64_t now_seconds, void receive(Connection& c, const RegionFacts& facts, std::uint64_t now_seconds,
Clock::time_point now) { Clock::time_point now) {
@@ -194,12 +216,17 @@ struct SessionServer::Impl {
} }
rest = rest.subspan(*taken); rest = rest.subspan(*taken);
while (auto f = c.reader.take()) { while (auto f = c.reader.take()) {
c.last_received = now;
if (c.state == State::hello) if (c.state == State::hello)
hello(c, *f, facts, now_seconds); hello(c, *f, facts, now_seconds);
else if (f->type == MessageType::intent) else if (f->type == MessageType::intent)
intent(c, *f, facts, now_seconds); intent(c, *f, facts, now_seconds);
else if (f->type == MessageType::close) { else if (f->type == MessageType::heartbeat) {
if (!decode_heartbeat({f->type, f->payload}))
close(c, CloseReason::malformed);
} else if (f->type == MessageType::close) {
c.state = State::closing; // Peer is leaving; nothing to say. c.state = State::closing; // Peer is leaving; nothing to say.
c.deadline = now + std::chrono::seconds{1};
events.push_back({SessionEvent::Kind::closed, c.session, c.subject, events.push_back({SessionEvent::Kind::closed, c.session, c.subject,
CloseReason::normal}); CloseReason::normal});
} else } else
@@ -211,28 +238,26 @@ struct SessionServer::Impl {
} }
} }
void transmit(Connection& c) { void transmit(Connection& c) {
if (c.out.empty() && !c.view.empty() && c.state == State::established) if (c.state == State::established && c.tls->queued() == 0 && !c.view.empty()) {
c.out.swap(c.view); // Only the newest view waits; it is never queued twice. send(c, c.view);
std::size_t sent = 0; c.view.clear();
while (sent < c.out.size()) { }
const auto n = c.tls->send(std::span{c.out}.subspan(sent)); if (c.dead)
if (!n) { return;
if (c.tls->queued() + c.view.size() > config.max_output_bytes) {
c.dead = true; // Slow consumer: drop rather than buffer without bound.
return;
}
if (c.state == State::closing) {
// close() flushes queued frames (the Close frame included) before close_notify.
const auto closed = c.tls->close();
if (!closed || *closed == TlsProgress::done)
c.dead = true; c.dead = true;
return; return;
} }
if (*n == 0) if (!c.tls->flush())
break;
sent += *n;
}
c.out.erase(c.out.begin(), c.out.begin() + static_cast<std::ptrdiff_t>(sent));
if (c.out.size() + c.view.size() > config.max_output_bytes)
c.dead = true; // Slow consumer: drop rather than buffer without bound.
if (c.state == State::closing && c.out.empty()) {
// Best-effort close_notify: the connection is dropped either way.
static_cast<void>(c.tls->close().has_value());
c.dead = true; c.dead = true;
} }
}
void step(Connection& c, const RegionFacts& facts, std::uint64_t now_seconds, void step(Connection& c, const RegionFacts& facts, std::uint64_t now_seconds,
Clock::time_point now) { Clock::time_point now) {
if (c.state == State::handshake) { if (c.state == State::handshake) {
@@ -250,10 +275,16 @@ struct SessionServer::Impl {
} }
if ((c.state == State::handshake || c.state == State::hello) && now > c.deadline) if ((c.state == State::handshake || c.state == State::hello) && now > c.deadline)
close(c, CloseReason::unauthenticated); close(c, CloseReason::unauthenticated);
if (c.state == State::established && now_seconds >= c.expires_at) if (c.state == State::established) {
if (now_seconds >= c.expires_at)
close(c, CloseReason::expired); close(c, CloseReason::expired);
if (c.state == State::established && admission && c.epoch != admission->epoch()) else if (admission && c.epoch != admission->epoch())
close(c, CloseReason::stale_epoch); close(c, CloseReason::stale_epoch);
else if (now - c.last_received > config.liveness_timeout)
close(c, CloseReason::timeout); // Silent peer: free the subject for reconnect.
else if (now - c.last_sent >= config.heartbeat_interval)
send(c, encode_heartbeat()); // Liveness without requiring simulation ticks.
}
if (c.state == State::hello || c.state == State::established) if (c.state == State::hello || c.state == State::established)
receive(c, facts, now_seconds, now); receive(c, facts, now_seconds, now);
if (!c.dead && c.state != State::handshake) if (!c.dead && c.state != State::handshake)
@@ -270,17 +301,44 @@ struct SessionServer::Impl {
} }
std::erase_if(connections, [](const Connection& c) { return c.dead; }); std::erase_if(connections, [](const Connection& c) { return c.dead; });
} }
void open_admission(const RegionFacts& facts, std::uint64_t now_seconds) {
auto config_with_epoch = config.admission;
config_with_epoch.epoch = facts.epoch;
auto created = SessionAdmission::create(config_with_epoch);
if (!created)
return;
// Grants consumed before this process started stay consumed.
if (!created->remember(ledger->loaded(), now_seconds)) {
ledger_rejected = true; // More recorded grants than admission can hold.
return;
}
admission.emplace(std::move(*created));
}
void finish(const RequestId& request, const IntentOutcome& outcome) {
const auto it =
std::ranges::find_if(carried, [&](const Carried& c) { return c.request == request; });
if (it == carried.end())
return;
auto out = outcome;
out.sequence = it->sequence; // The client's own sequence, carried separately.
if (auto* c = find(it->connection))
send(*c, encode(out));
carried.erase(it);
}
}; };
SessionServer::SessionServer(std::unique_ptr<Impl> impl) : impl_{std::move(impl)} {} SessionServer::SessionServer(std::unique_ptr<Impl> impl) : impl_{std::move(impl)} {}
SessionServer::~SessionServer() = default; SessionServer::~SessionServer() = default;
std::expected<std::unique_ptr<SessionServer>, SessionServerError> std::expected<std::unique_ptr<SessionServer>, SessionServerError>
SessionServer::create(SessionServerConfig config, DevGrantKey key) { SessionServer::create(SessionServerConfig config, DevGrantKey key, std::uint64_t now_seconds) {
if (config.max_connections == 0 || config.max_connections > 64 || if (config.max_connections == 0 || config.max_connections > 64 ||
config.max_pending_intents == 0 || config.max_carried_intents == 0 || config.max_pending_intents == 0 || config.max_carried_intents == 0 ||
config.max_output_bytes < 2 * (frame_header_bytes + max_frame_payload) || config.max_output_bytes < 2 * (frame_header_bytes + max_frame_payload) ||
config.max_input_bytes_per_second < frame_header_bytes + max_grant_bytes + 6) config.max_output_bytes > TlsChannel::max_queued_bytes ||
config.max_input_bytes_per_second < frame_header_bytes + max_grant_bytes + 6 ||
config.heartbeat_interval.count() <= 0 ||
config.liveness_timeout <= 2 * config.heartbeat_interval || config.grant_ledger.empty())
return std::unexpected(SessionServerError::configuration); return std::unexpected(SessionServerError::configuration);
// Validate the admission limits now, with a placeholder epoch. // Validate the admission limits now, with a placeholder epoch.
auto probe = config.admission; auto probe = config.admission;
@@ -298,11 +356,14 @@ SessionServer::create(SessionServerConfig config, DevGrantKey key) {
} null; } null;
if (!TlsChannel::server(config.credentials, null)) if (!TlsChannel::server(config.credentials, null))
return std::unexpected(SessionServerError::credentials); return std::unexpected(SessionServerError::credentials);
auto ledger = GrantLedger::open(config.grant_ledger, now_seconds);
if (!ledger)
return std::unexpected(SessionServerError::ledger);
auto listener = TcpListener::listen(config.listen); auto listener = TcpListener::listen(config.listen);
if (!listener) if (!listener)
return std::unexpected(SessionServerError::listen); return std::unexpected(SessionServerError::listen);
return std::unique_ptr<SessionServer>(new SessionServer( return std::unique_ptr<SessionServer>(new SessionServer(std::make_unique<Impl>(
std::make_unique<Impl>(std::move(config), std::move(key), std::move(*listener)))); std::move(config), std::move(key), std::move(*listener), std::move(*ledger))));
} }
std::uint16_t SessionServer::port() const noexcept { std::uint16_t SessionServer::port() const noexcept {
@@ -313,12 +374,9 @@ void SessionServer::poll(const RegionFacts& facts, std::uint64_t now_seconds,
std::chrono::steady_clock::time_point now) { std::chrono::steady_clock::time_point now) {
auto& m = *impl_; auto& m = *impl_;
m.polled_at = now; m.polled_at = now;
if (facts.ready && facts.epoch != 0) { if (facts.ready && facts.epoch != 0 && !m.ledger_rejected) {
if (!m.admission) { if (!m.admission) {
auto config = m.config.admission; m.open_admission(facts, now_seconds);
config.epoch = facts.epoch;
if (auto created = SessionAdmission::create(config))
m.admission.emplace(std::move(*created));
} else if (facts.epoch > m.admission->epoch()) { } else if (facts.epoch > m.admission->epoch()) {
// Strictly newer (checked above), so the advance cannot be refused. // Strictly newer (checked above), so the advance cannot be refused.
static_cast<void>(m.admission->advance_epoch(facts.epoch).has_value()); static_cast<void>(m.admission->advance_epoch(facts.epoch).has_value());
@@ -340,7 +398,7 @@ void SessionServer::poll(const RegionFacts& facts, std::uint64_t now_seconds,
continue; continue;
c.tls = std::move(*tls); c.tls = std::move(*tls);
c.deadline = now + m.config.hello_deadline; c.deadline = now + m.config.hello_deadline;
c.window_start = now; c.window_start = c.last_received = c.last_sent = now;
m.connections.push_back(std::move(c)); m.connections.push_back(std::move(c));
} }
for (auto& c : m.connections) for (auto& c : m.connections)
@@ -364,13 +422,15 @@ void SessionServer::publish(const client::EnvironmentPresentation& environment,
std::optional<RequestId> SessionServer::carry(std::uint64_t tick) { std::optional<RequestId> SessionServer::carry(std::uint64_t tick) {
auto& m = *impl_; auto& m = *impl_;
if (m.carried.size() >= m.config.max_carried_intents) if (m.carried.size() >= m.config.max_carried_intents || !m.admission ||
return std::nullopt; m.next_carrier == std::numeric_limits<std::uint64_t>::max())
return std::nullopt; // Ordinal exhaustion is explicit, never a wrap.
const auto it = std::ranges::find_if( const auto it = std::ranges::find_if(
m.pending, [&](const Impl::Pending& p) { return p.intended_tick <= tick; }); m.pending, [&](const Impl::Pending& p) { return p.intended_tick <= tick; });
if (it == m.pending.end()) if (it == m.pending.end())
return std::nullopt; return std::nullopt;
const Impl::Carried carried{it->request, it->connection, it->sequence}; const Impl::Carried carried{carrier_id(m.admission->epoch(), m.next_carrier++), it->connection,
it->sequence};
m.pending.erase(it); m.pending.erase(it);
m.carried.push_back(carried); m.carried.push_back(carried);
return carried.request; return carried.request;
@@ -378,26 +438,11 @@ std::optional<RequestId> SessionServer::carry(std::uint64_t tick) {
void SessionServer::committed(const RequestId& request, std::uint64_t tick, void SessionServer::committed(const RequestId& request, std::uint64_t tick,
std::uint64_t commit_sequence) { std::uint64_t commit_sequence) {
auto& m = *impl_; impl_->finish(request, {0, IntentFinality::committed_no_effect, tick, commit_sequence});
const auto it = std::ranges::find_if(
m.carried, [&](const Impl::Carried& c) { return c.request == request; });
if (it == m.carried.end())
return;
if (auto* c = m.find(it->connection))
append(c->out, encode(IntentOutcome{it->sequence, IntentFinality::committed_no_effect, tick,
commit_sequence}));
m.carried.erase(it);
} }
void SessionServer::not_committed(const RequestId& request) { void SessionServer::not_committed(const RequestId& request) {
auto& m = *impl_; impl_->finish(request, {0, IntentFinality::not_committed, 0, 0});
const auto it = std::ranges::find_if(
m.carried, [&](const Impl::Carried& c) { return c.request == request; });
if (it == m.carried.end())
return;
if (auto* c = m.find(it->connection))
append(c->out, encode(IntentOutcome{it->sequence, IntentFinality::not_committed, 0, 0}));
m.carried.erase(it);
} }
std::vector<SessionEvent> SessionServer::take_events() { std::vector<SessionEvent> SessionServer::take_events() {
+38 -6
View File
@@ -5,6 +5,7 @@
#include <mbedtls/x509_crt.h> #include <mbedtls/x509_crt.h>
#include <optional> #include <optional>
#include <psa/crypto.h> #include <psa/crypto.h>
#include <vector>
// The owned user config (third_party/mbedtls/evo_mbedtls_config.h) must be in effect. // The owned user config (third_party/mbedtls/evo_mbedtls_config.h) must be in effect.
#if defined(MBEDTLS_SSL_PROTO_TLS1_2) || defined(MBEDTLS_SSL_PROTO_DTLS) || \ #if defined(MBEDTLS_SSL_PROTO_TLS1_2) || defined(MBEDTLS_SSL_PROTO_DTLS) || \
@@ -21,6 +22,8 @@ namespace {
// ADR-0082 profile: TLS 1.3 only, AEAD suites, ECDHE groups, ephemeral key exchange. // ADR-0082 profile: TLS 1.3 only, AEAD suites, ECDHE groups, ephemeral key exchange.
constexpr int suites[]{MBEDTLS_TLS1_3_AES_128_GCM_SHA256, MBEDTLS_TLS1_3_CHACHA20_POLY1305_SHA256, constexpr int suites[]{MBEDTLS_TLS1_3_AES_128_GCM_SHA256, MBEDTLS_TLS1_3_CHACHA20_POLY1305_SHA256,
0}; 0};
// One TLS record's plaintext per write call (the TLS 1.3 maximum fragment).
constexpr std::size_t record_plaintext = 16384;
constexpr std::uint16_t groups[]{MBEDTLS_SSL_IANA_TLS_GROUP_X25519, constexpr std::uint16_t groups[]{MBEDTLS_SSL_IANA_TLS_GROUP_X25519,
MBEDTLS_SSL_IANA_TLS_GROUP_SECP256R1, 0}; MBEDTLS_SSL_IANA_TLS_GROUP_SECP256R1, 0};
const unsigned char* bytes_of(const std::string& text) { const unsigned char* bytes_of(const std::string& text) {
@@ -105,6 +108,8 @@ struct TlsChannel::Impl {
mbedtls_pk_context key; mbedtls_pk_context key;
bool transport_failed{}; bool transport_failed{};
std::optional<TlsError> error; std::optional<TlsError> error;
std::vector<std::uint8_t> outbound; // Queued plaintext; the front record may be in flight.
std::size_t in_flight{}; // Length of an interrupted write; 0 when none.
}; };
TlsChannel::TlsChannel(std::unique_ptr<Impl> impl) : impl_{std::move(impl)} {} TlsChannel::TlsChannel(std::unique_ptr<Impl> impl) : impl_{std::move(impl)} {}
@@ -158,15 +163,39 @@ std::expected<TlsProgress, TlsError> TlsChannel::handshake() {
return impl_->fail_from(result, TlsError::handshake); return impl_->fail_from(result, TlsError::handshake);
} }
std::expected<std::size_t, TlsError> TlsChannel::send(std::span<const std::uint8_t> bytes) { std::expected<void, TlsError> TlsChannel::queue(std::span<const std::uint8_t> bytes) {
if (const auto latched = impl_->error) if (const auto latched = impl_->error)
return std::unexpected(*latched); return std::unexpected(*latched);
const int result = mbedtls_ssl_write(&impl_->ssl, bytes.data(), bytes.size()); if (bytes.size() > max_queued_bytes - impl_->outbound.size())
if (result >= 0) return std::unexpected(TlsError::backpressure); // Nothing queued: not latched.
return static_cast<std::size_t>(result); impl_->outbound.insert(impl_->outbound.end(), bytes.begin(), bytes.end());
return {};
}
std::expected<TlsProgress, TlsError> TlsChannel::flush() {
auto& m = *impl_;
if (const auto latched = m.error)
return std::unexpected(*latched);
while (!m.outbound.empty()) {
// A record interrupted by WANT_READ/WANT_WRITE keeps its length; bytes queued
// since then sit behind it and do not change the retried call.
if (m.in_flight == 0)
m.in_flight = std::min(m.outbound.size(), record_plaintext);
const int result = mbedtls_ssl_write(&m.ssl, m.outbound.data(), m.in_flight);
if (result == MBEDTLS_ERR_SSL_WANT_READ || result == MBEDTLS_ERR_SSL_WANT_WRITE) if (result == MBEDTLS_ERR_SSL_WANT_READ || result == MBEDTLS_ERR_SSL_WANT_WRITE)
return 0; return TlsProgress::want_io;
return impl_->fail_from(result, TlsError::record); if (result < 0)
return m.fail_from(result, TlsError::record);
const auto written = std::min(static_cast<std::size_t>(result), m.in_flight);
m.outbound.erase(m.outbound.begin(),
m.outbound.begin() + static_cast<std::ptrdiff_t>(written));
m.in_flight = 0;
}
return TlsProgress::done;
}
std::size_t TlsChannel::queued() const noexcept {
return impl_->outbound.size();
} }
std::expected<std::size_t, TlsError> TlsChannel::receive(std::span<std::uint8_t> bytes) { std::expected<std::size_t, TlsError> TlsChannel::receive(std::span<std::uint8_t> bytes) {
@@ -185,6 +214,9 @@ std::expected<std::size_t, TlsError> TlsChannel::receive(std::span<std::uint8_t>
std::expected<TlsProgress, TlsError> TlsChannel::close() { std::expected<TlsProgress, TlsError> TlsChannel::close() {
if (const auto latched = impl_->error) if (const auto latched = impl_->error)
return std::unexpected(*latched); return std::unexpected(*latched);
// Queued plaintext (including an interrupted record) is sent before close_notify.
if (const auto pending = flush(); !pending || *pending == TlsProgress::want_io)
return pending;
const int result = mbedtls_ssl_close_notify(&impl_->ssl); const int result = mbedtls_ssl_close_notify(&impl_->ssl);
if (result == 0) if (result == 0)
return TlsProgress::done; return TlsProgress::done;
+4
View File
@@ -50,6 +50,10 @@ extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, std::size_t size
if (const auto m = decode_intent_outcome(*frame)) if (const auto m = decode_intent_outcome(*frame))
again = encode(*m); again = encode(*m);
break; break;
case MessageType::heartbeat:
if (decode_heartbeat(*frame))
again = encode_heartbeat();
break;
case MessageType::environment_view: case MessageType::environment_view:
if (const auto m = decode_environment_view(*frame)) if (const auto m = decode_environment_view(*frame))
again = encode(m->first, m->second); again = encode(m->first, m->second);
+13 -2
View File
@@ -126,6 +126,12 @@ void round_trips() {
const auto c = decode_close(frame(encode(CloseReason::stale_epoch))); const auto c = decode_close(frame(encode(CloseReason::stale_epoch)));
CHECK(c && *c == CloseReason::stale_epoch); CHECK(c && *c == CloseReason::stale_epoch);
CHECK(decode_close(frame(encode(CloseReason::capacity))) == CloseReason::capacity); CHECK(decode_close(frame(encode(CloseReason::capacity))) == CloseReason::capacity);
CHECK(decode_close(frame(encode(CloseReason::timeout))) == CloseReason::timeout);
const auto beat = encode_heartbeat();
CHECK(beat.size() == frame_header_bytes && decode_heartbeat(frame(beat)));
CHECK(refused(decode_heartbeat(frame(with_header(MessageType::heartbeat, 1, 0, 1, 1))),
ProtocolError::trailing_bytes));
CHECK(refused(decode_heartbeat(frame(encode(CloseReason::normal))), ProtocolError::wrong_type));
const auto out = decode_intent_outcome( const auto out = decode_intent_outcome(
frame(encode(IntentOutcome{4, IntentFinality::committed_no_effect, 120, 9}))); frame(encode(IntentOutcome{4, IntentFinality::committed_no_effect, 120, 9})));
CHECK(out && out->sequence == 4 && out->finality == IntentFinality::committed_no_effect && CHECK(out && out->sequence == 4 && out->finality == IntentFinality::committed_no_effect &&
@@ -177,7 +183,7 @@ void frame_rejection() {
CHECK(refused(decode_frame(with_header(MessageType{0}, 1, 0, 0, 0)), CHECK(refused(decode_frame(with_header(MessageType{0}, 1, 0, 0, 0)),
ProtocolError::unknown_type)); ProtocolError::unknown_type));
// NOLINTNEXTLINE(clang-analyzer-optin.core.EnumCastOutOfRange) // NOLINTNEXTLINE(clang-analyzer-optin.core.EnumCastOutOfRange)
CHECK(refused(decode_frame(with_header(MessageType{9}, 1, 0, 0, 0)), CHECK(refused(decode_frame(with_header(MessageType{10}, 1, 0, 0, 0)),
ProtocolError::unknown_type)); ProtocolError::unknown_type));
CHECK(refused(decode_frame(with_header(MessageType::intent, 1, 1, 36, 36)), CHECK(refused(decode_frame(with_header(MessageType::intent, 1, 1, 36, 36)),
ProtocolError::reserved_flags)); ProtocolError::reserved_flags));
@@ -207,7 +213,7 @@ void frame_rejection() {
status[frame_header_bytes + 8] = 8; status[frame_header_bytes + 8] = 8;
CHECK(refused(decode_intent_result(frame(status)), ProtocolError::invalid_value)); CHECK(refused(decode_intent_result(frame(status)), ProtocolError::invalid_value));
auto reason = encode(CloseReason::normal); auto reason = encode(CloseReason::normal);
reason[frame_header_bytes] = 10; reason[frame_header_bytes] = 11;
CHECK(refused(decode_close(frame(reason)), ProtocolError::invalid_value)); CHECK(refused(decode_close(frame(reason)), ProtocolError::invalid_value));
CHECK(refused(decode_welcome(frame(encode(Welcome{}))), ProtocolError::invalid_value)); CHECK(refused(decode_welcome(frame(encode(Welcome{}))), ProtocolError::invalid_value));
// A well-formed encoding of an invalid view is rejected by the presentation rules. // A well-formed encoding of an invalid view is rejected by the presentation rules.
@@ -261,6 +267,7 @@ void fuzz() {
encode(IntentResult{1, IntentStatus::outside_window, 2}), encode(IntentResult{1, IntentStatus::outside_window, 2}),
encode(CloseReason::rate_limited), encode(CloseReason::rate_limited),
encode(IntentOutcome{3, IntentFinality::committed_no_effect, 4, 5}), encode(IntentOutcome{3, IntentFinality::committed_no_effect, 4, 5}),
encode_heartbeat(),
encode(view_header, environment()), encode(view_header, environment()),
encode(view_header, organisms(3))}; encode(view_header, organisms(3))};
std::size_t accepted = 0; std::size_t accepted = 0;
@@ -313,6 +320,10 @@ void fuzz() {
if (const auto m = decode_intent_outcome(*f)) if (const auto m = decode_intent_outcome(*f))
again = encode(*m); again = encode(*m);
break; break;
case MessageType::heartbeat:
if (decode_heartbeat(*f))
again = encode_heartbeat();
break;
case MessageType::environment_view: case MessageType::environment_view:
if (const auto m = decode_environment_view(*f)) if (const auto m = decode_environment_view(*f))
again = encode(m->first, m->second); again = encode(m->first, m->second);
+226 -11
View File
@@ -1,11 +1,16 @@
#include "evo/network/grant_ledger.hpp"
#include "evo/network/session_client.hpp" #include "evo/network/session_client.hpp"
#include "evo/network/session_server.hpp" #include "evo/network/session_server.hpp"
#include "evo/network/tcp.hpp" #include "evo/network/tcp.hpp"
#include "network_pki.hpp" #include "network_pki.hpp"
#include "network_test.hpp" #include "network_test.hpp"
#include <algorithm> #include <algorithm>
#include <atomic>
#include <chrono> #include <chrono>
#include <filesystem>
#include <fstream>
#include <functional> #include <functional>
#include <limits>
#include <string> #include <string>
#include <thread> #include <thread>
#include <vector> #include <vector>
@@ -57,12 +62,28 @@ client::OrganismPresentation no_organisms(std::uint64_t tick) {
return std::move(*v); return std::move(*v);
} }
std::filesystem::path fresh_ledger() {
static std::atomic<unsigned> counter{0};
const auto stamp = Clock::now().time_since_epoch().count();
return std::filesystem::temp_directory_path() /
("evo-grant-ledger-" + std::to_string(stamp) + "-" + std::to_string(++counter));
}
struct Harness { struct Harness {
explicit Harness(std::size_t max_sessions = 4, explicit Harness(std::size_t max_sessions = 4,
std::chrono::milliseconds hello = std::chrono::milliseconds{3000}) { std::chrono::milliseconds hello = std::chrono::milliseconds{3000},
std::function<void(SessionServerConfig&)> server_tweak = {},
std::function<void(SessionClientConfig&)> client_tweak = {})
: tweak_server{std::move(server_tweak)}, tweak_client{std::move(client_tweak)} {
server = make_server(0, max_sessions, hello); server = make_server(0, max_sessions, hello);
port = server->port(); port = server->port();
} }
~Harness() {
server.reset();
std::error_code ignored;
std::filesystem::remove(ledger, ignored);
}
Harness(const Harness&) = delete;
Harness& operator=(const Harness&) = delete;
std::unique_ptr<SessionServer> make_server(std::uint16_t at, std::size_t max_sessions, std::unique_ptr<SessionServer> make_server(std::uint16_t at, std::size_t max_sessions,
std::chrono::milliseconds hello) { std::chrono::milliseconds hello) {
SessionServerConfig c; SessionServerConfig c;
@@ -75,7 +96,10 @@ struct Harness {
c.admission.window_ticks = 8; c.admission.window_ticks = 8;
c.admission.max_intents_per_window = 8; c.admission.max_intents_per_window = 8;
c.hello_deadline = hello; c.hello_deadline = hello;
auto s = SessionServer::create(std::move(c), grant_key()); c.grant_ledger = ledger; // Shared by restarts of this harness's server.
if (tweak_server)
tweak_server(c);
auto s = SessionServer::create(std::move(c), grant_key(), now_seconds);
CHECK(s); CHECK(s);
return std::move(*s); return std::move(*s);
} }
@@ -88,6 +112,8 @@ struct Harness {
c.first_retry = std::chrono::milliseconds{50}; c.first_retry = std::chrono::milliseconds{50};
c.max_retry = std::chrono::milliseconds{200}; c.max_retry = std::chrono::milliseconds{200};
c.attempt_deadline = std::chrono::milliseconds{3000}; c.attempt_deadline = std::chrono::milliseconds{3000};
if (tweak_client)
tweak_client(c);
auto s = SessionClient::create(std::move(c)); auto s = SessionClient::create(std::move(c));
CHECK(s); CHECK(s);
clients.push_back(s->get()); clients.push_back(s->get());
@@ -119,6 +145,9 @@ struct Harness {
auto e = c.take_events(); auto e = c.take_events();
into.insert(into.end(), e.begin(), e.end()); into.insert(into.end(), e.begin(), e.end());
} }
std::function<void(SessionServerConfig&)> tweak_server;
std::function<void(SessionClientConfig&)> tweak_client;
std::filesystem::path ledger = fresh_ledger();
Pki pki; Pki pki;
RegionFacts facts{true, 1, 100, 90, 5}; RegionFacts facts{true, 1, 100, 90, 5};
std::unique_ptr<SessionServer> server; std::unique_ptr<SessionServer> server;
@@ -167,7 +196,9 @@ void connect_view_intents_and_outcomes() {
CHECK(last(events, ClientEvent::Kind::result)->result.status == IntentStatus::admitted); CHECK(last(events, ClientEvent::Kind::result)->result.status == IntentStatus::admitted);
CHECK(!h.server->carry(105)); // Intended tick 102 + 4 = 106. CHECK(!h.server->carry(105)); // Intended tick 102 + 4 = 106.
const auto request = h.server->carry(106); const auto request = h.server->carry(106);
CHECK(must(request)[0] == 'I' && must(request)[3] == '1'); // 'I','N' + 48-bit epoch (1) + 64-bit server ordinal (first carrier: 1).
CHECK(must(request)[0] == 'I' && must(request)[1] == 'N' && must(request)[7] == 1 &&
must(request)[15] == 1);
CHECK(!h.server->carry(200)); // Carried once. CHECK(!h.server->carry(200)); // Carried once.
h.server->committed(must(request), 106, 6); h.server->committed(must(request), 106, 6);
CHECK(h.until([&] { CHECK(h.until([&] {
@@ -336,24 +367,42 @@ void abuse() {
h.tick(); h.tick();
CHECK(h.until([&] { return c->phase() == SessionPhase::established; })); CHECK(h.until([&] { return c->phase() == SessionPhase::established; }));
CHECK(c->grants_left() == 2); CHECK(c->grants_left() == 2);
// Flooding: the byte-rate limit closes the connection. The flooding client may see // The client never buffers without bound: with the server not answering, at most
// a transport reset before it reads the Close frame, so check the server's decision. // max_outstanding intentions may be unresolved; further sends are refused.
CHECK(h.until([&] { return c->latest_view().has_value(); })); CHECK(h.until([&] { return c->latest_view().has_value(); }));
(void)h.server->take_events(); std::size_t accepted = 0;
for (std::uint64_t s = 10; s < 2000; ++s) for (std::uint64_t s = 10; s < 2000; ++s)
CHECK(c->send({s, 0, 0, 0})); accepted += c->send({s, 0, 0, 0}) ? 1 : 0;
CHECK(accepted == SessionClient::max_outstanding);
// Flooding: a raw TLS peer (no client-side bounds) exceeds the byte-rate limit after
// a valid Hello; the server closes it. Check the server's decision, since the flooding
// peer may see a reset before it could read the Close frame.
(void)h.server->take_events();
auto raw = TcpStream::connect("127.0.0.1", h.port);
CHECK(raw);
auto tls = TlsChannel::client({h.pki.authority, server_name}, **raw);
CHECK(tls);
const auto flood_grant = grants(9, 1, 900).front();
const auto hello = encode(Hello{0, {flood_grant.begin(), flood_grant.end()}});
CHECK(hello);
bool flooded = false;
std::vector<SessionEvent> server_events; std::vector<SessionEvent> server_events;
CHECK(h.until([&] { CHECK(h.until([&] {
auto e = h.server->take_events(); auto e = h.server->take_events();
server_events.insert(server_events.end(), e.begin(), e.end()); server_events.insert(server_events.end(), e.begin(), e.end());
if (const auto p = (*tls)->handshake(); p && *p == TlsProgress::done && !flooded) {
flooded = hello && (*tls)->queue(*hello).has_value();
const auto beat = encode_heartbeat();
for (int i = 0; i < 7000 && flooded; ++i)
flooded = (*tls)->queue(beat).has_value();
}
if (flooded)
static_cast<void>((*tls)->flush().has_value()); // Best effort: may be reset.
return std::ranges::any_of(server_events, [](const SessionEvent& x) { return std::ranges::any_of(server_events, [](const SessionEvent& x) {
return x.kind == SessionEvent::Kind::closed && x.reason == CloseReason::rate_limited; return x.kind == SessionEvent::Kind::closed && x.reason == CloseReason::rate_limited;
}); });
})); }));
CHECK(h.until([&] {
h.collect(*c, events);
return c->phase() != SessionPhase::established;
}));
// Plaintext and silent connections are dropped without a session: the server // Plaintext and silent connections are dropped without a session: the server
// closes the socket (plaintext at the TLS handshake, silence at the deadline). // closes the socket (plaintext at the TLS handshake, silence at the deadline).
@@ -381,6 +430,169 @@ void abuse() {
CHECK(quiet.until([&] { return quiet.server->connections() == 0; })); CHECK(quiet.until([&] { return quiet.server->connections() == 0; }));
CHECK(quiet.server->sessions() == 0); CHECK(quiet.server->sessions() == 0);
} }
// Review M9.4-M9.7 #2: client sequences 2^32 apart must get distinct carriers and
// each final outcome must reach its own sequence.
void carriers_never_alias() {
Harness h;
h.tick();
auto c = h.client(grants(7, 2, 100));
std::vector<ClientEvent> events;
CHECK(h.until([&] { return c->latest_view().has_value(); }));
constexpr std::uint64_t far = (std::uint64_t{1} << 32) + 1;
CHECK(c->send({1, 0, 1000, 0}) && c->send({far, 0, 0, 1000}));
CHECK(h.until([&] {
h.collect(*c, events);
return std::ranges::count_if(events, [](const ClientEvent& e) {
return e.kind == ClientEvent::Kind::result &&
e.result.status == IntentStatus::admitted;
}) == 2;
}));
const auto first = h.server->carry(1000);
const auto second = h.server->carry(1000);
CHECK(must(first) != must(second));
CHECK(must(first)[0] == 'I' && must(first)[1] == 'N' && must(first)[7] == 1); // Epoch 1.
h.server->not_committed(must(second));
h.server->committed(must(first), 106, 6);
CHECK(h.until([&] {
h.collect(*c, events);
return std::ranges::count_if(events, [](const ClientEvent& e) {
return e.kind == ClientEvent::Kind::outcome;
}) == 2;
}));
for (const auto& e : events)
if (e.kind == ClientEvent::Kind::outcome)
CHECK((e.sequence == far && e.outcome.finality == IntentFinality::not_committed) ||
(e.sequence == 1 && e.outcome.finality == IntentFinality::committed_no_effect &&
e.outcome.commit_sequence == 6));
}
// Review M9.4-M9.7 #3: a grant consumed before a restart stays consumed after it.
void grants_stay_consumed_across_restarts() {
Harness h;
h.tick();
auto first = h.client(grants(7, 1, 100));
CHECK(h.until([&] { return first->phase() == SessionPhase::established; }));
h.clients.clear();
first.reset();
h.server.reset(); // Process restart: only the ledger file survives.
h.facts = {true, 2, 40, 40, 2};
h.server = h.make_server(h.port, 4, std::chrono::milliseconds{3000});
h.tick();
auto replay = h.client(grants(7, 1, 100)); // The exact consumed grant.
std::vector<ClientEvent> events;
CHECK(h.until([&] {
h.collect(*replay, events);
return has(events, ClientEvent::Kind::closed);
}));
CHECK(last(events, ClientEvent::Kind::closed)->reason == CloseReason::unauthenticated);
auto fresh = h.client(grants(7, 1, 101));
CHECK(h.until([&] { return fresh->phase() == SessionPhase::established; }));
CHECK(must(fresh->welcome()).epoch == 2);
// The ledger fails closed on corruption and drops expired entries when opened.
const auto path = fresh_ledger();
{
std::ofstream out{path, std::ios::binary};
out << "not a ledger line\n";
}
SessionServerConfig bad;
bad.listen = "127.0.0.1:0";
bad.credentials = h.pki.server;
bad.admission.world = identity;
bad.admission.region = region;
bad.grant_ledger = path;
CHECK(
refused(SessionServer::create(bad, grant_key(), now_seconds), SessionServerError::ledger));
{
std::ofstream out{path, std::ios::binary | std::ios::trunc}; // LF only.
out << "0000000000000005 " << now_seconds - 1 << "\n"
<< "0000000000000006 " << now_seconds + 10 << "\n";
}
{
const auto ledger = GrantLedger::open(path, now_seconds);
CHECK(ledger && (*ledger)->loaded().size() == 1 && (*ledger)->loaded()[0].grant_id == 6);
CHECK(ledger && (*ledger)->record({7, now_seconds + 20}));
}
const auto reopened = GrantLedger::open(path, now_seconds);
CHECK(reopened && (*reopened)->loaded().size() == 2);
std::error_code ignored;
std::filesystem::remove(path, ignored);
}
// Review M9.4-M9.7 #4: liveness after Welcome, both directions, without spurious drops.
void liveness() {
const auto fast_server = [](SessionServerConfig& c) {
c.heartbeat_interval = std::chrono::milliseconds{100};
c.liveness_timeout = std::chrono::milliseconds{500};
};
const auto fast_client = [](SessionClientConfig& c) {
c.heartbeat_interval = std::chrono::milliseconds{100};
c.liveness_timeout = std::chrono::milliseconds{500};
};
{
// Healthy but idle: no ticks are published; heartbeats keep both sides alive.
Harness h{4, std::chrono::milliseconds{3000}, fast_server, fast_client};
h.tick();
auto c = h.client(grants(7, 2, 100));
std::vector<ClientEvent> events;
CHECK(h.until([&] { return c->latest_view().has_value(); }));
const auto start = Clock::now();
CHECK(!h.until(
[&] {
h.collect(*c, events);
return has(events, ClientEvent::Kind::lost) ||
has(events, ClientEvent::Kind::closed);
},
std::chrono::milliseconds{1500}));
CHECK(Clock::now() - start >= std::chrono::milliseconds{1500});
CHECK(c->phase() == SessionPhase::established &&
c->environment().freshness() == client::Freshness::current);
}
{
// Silent server (socket open, never polled): the client times out, keeps the view
// as stale, reports the sent-but-unresolved intention as unknown, and reconnects.
Harness h{4, std::chrono::milliseconds{3000}, fast_server, fast_client};
h.tick();
auto c = h.client(grants(7, 3, 100));
std::vector<ClientEvent> events;
CHECK(h.until([&] { return c->latest_view().has_value(); }));
CHECK(c->send({5, 0, 1000, 0})); // Never answered: the server is not polled.
const auto end = Clock::now() + std::chrono::milliseconds{2000};
while (Clock::now() < end && !has(events, ClientEvent::Kind::lost)) {
c->poll(Clock::now());
h.collect(*c, events);
std::this_thread::sleep_for(std::chrono::milliseconds{5});
}
CHECK(has(events, ClientEvent::Kind::lost));
const auto* unknown = last(events, ClientEvent::Kind::unknown);
CHECK(unknown != nullptr && unknown->sequence == 5);
CHECK(c->environment().freshness() == client::Freshness::stale);
h.tick();
CHECK(h.until([&] {
return c->phase() == SessionPhase::established && c->latest_view().has_value();
}));
CHECK(c->environment().freshness() == client::Freshness::current);
}
{
// Silent client: the server closes it with `timeout`, freeing the subject.
Harness h{4, std::chrono::milliseconds{3000}, fast_server, fast_client};
h.tick();
auto c = h.client(grants(7, 1, 100));
CHECK(h.until([&] { return c->phase() == SessionPhase::established; }));
h.clients.clear(); // Stop polling the client; its socket stays open.
std::vector<SessionEvent> server_events;
CHECK(h.until([&] {
auto e = h.server->take_events();
server_events.insert(server_events.end(), e.begin(), e.end());
return h.server->sessions() == 0;
}));
CHECK(std::ranges::any_of(server_events, [](const SessionEvent& e) {
return e.kind == SessionEvent::Kind::closed && e.reason == CloseReason::timeout;
}));
auto again = h.client(grants(7, 1, 200)); // Same subject, fresh grant: admitted.
CHECK(h.until([&] { return again->phase() == SessionPhase::established; }));
}
}
} // namespace } // namespace
void session_tests() { void session_tests() {
@@ -388,5 +600,8 @@ void session_tests() {
duplicate_capacity_and_authentication(); duplicate_capacity_and_authentication();
stale_epoch_restart_and_reconnect(); stale_epoch_restart_and_reconnect();
abuse(); abuse();
carriers_never_alias();
grants_stay_consumed_across_restarts();
liveness();
} }
} // namespace evo::network_test } // namespace evo::network_test
+98 -11
View File
@@ -2,7 +2,9 @@
#include "evo/network/tls_channel.hpp" #include "evo/network/tls_channel.hpp"
#include "network_pki.hpp" #include "network_pki.hpp"
#include "network_test.hpp" #include "network_test.hpp"
#include <algorithm>
#include <array> #include <array>
#include <cstdint>
#include <deque> #include <deque>
#include <mbedtls/x509_crt.h> #include <mbedtls/x509_crt.h>
#include <psa/crypto.h> #include <psa/crypto.h>
@@ -17,6 +19,8 @@ using namespace evo::network;
struct Wire { struct Wire {
std::deque<std::uint8_t> to_server, to_client; std::deque<std::uint8_t> to_server, to_client;
bool fail{}; bool fail{};
// Bytes each side may still write; 0 makes writes "would block" (zero progress).
std::size_t server_budget{SIZE_MAX}, client_budget{SIZE_MAX};
}; };
class Endpoint final : public ByteTransport { class Endpoint final : public ByteTransport {
public: public:
@@ -25,7 +29,11 @@ class Endpoint final : public ByteTransport {
if (wire_.fail) if (wire_.fail)
return std::unexpected(TransportError::failed); return std::unexpected(TransportError::failed);
auto& out = server_ ? wire_.to_client : wire_.to_server; auto& out = server_ ? wire_.to_client : wire_.to_server;
const auto count = std::min<std::size_t>(bytes.size(), 700); // Force partial writes. auto& budget = server_ ? wire_.server_budget : wire_.client_budget;
// Force partial writes, and stop entirely when the injected budget runs out.
const auto count = std::min({bytes.size(), std::size_t{700}, budget});
if (budget != SIZE_MAX)
budget -= count;
out.insert(out.end(), bytes.begin(), bytes.begin() + static_cast<std::ptrdiff_t>(count)); out.insert(out.end(), bytes.begin(), bytes.begin() + static_cast<std::ptrdiff_t>(count));
return count; return count;
} }
@@ -76,15 +84,15 @@ struct Pair {
} }
std::vector<std::uint8_t> transfer(TlsChannel& from, TlsChannel& to, std::vector<std::uint8_t> transfer(TlsChannel& from, TlsChannel& to,
std::span<const std::uint8_t> bytes) { std::span<const std::uint8_t> bytes) {
std::size_t sent = 0; CHECK(from.queue(bytes));
return drain(from, to, bytes.size());
}
// Flushes `from` and reads `to` until `expected` plaintext bytes arrive.
std::vector<std::uint8_t> drain(TlsChannel& from, TlsChannel& to, std::size_t expected) {
std::vector<std::uint8_t> received; std::vector<std::uint8_t> received;
std::array<std::uint8_t, 4096> buffer{}; std::array<std::uint8_t, 4096> buffer{};
for (int i = 0; i < 1000 && received.size() < bytes.size(); ++i) { for (int i = 0; i < 1000 && received.size() < expected; ++i) {
if (sent < bytes.size()) { CHECK(from.flush());
const auto n = from.send(bytes.subspan(sent));
CHECK(n);
sent += *n;
}
const auto n = to.receive(buffer); const auto n = to.receive(buffer);
CHECK(n); CHECK(n);
received.insert(received.end(), buffer.begin(), received.insert(received.end(), buffer.begin(),
@@ -122,7 +130,7 @@ void handshake_and_frames() {
for (int i = 0; i < 10 && r && *r == 0; ++i) for (int i = 0; i < 10 && r && *r == 0; ++i)
r = pair.server->receive(buffer); r = pair.server->receive(buffer);
CHECK(refused(r, TlsError::closed)); CHECK(refused(r, TlsError::closed));
CHECK(refused(pair.server->send(buffer), TlsError::closed)); // Latched. CHECK(refused(pair.server->queue(buffer), TlsError::closed)); // Latched.
} }
void rejections() { void rejections() {
@@ -155,8 +163,8 @@ void rejections() {
Pair pair{pki.server, {pki.authority, server_name}}; Pair pair{pki.server, {pki.authority, server_name}};
CHECK(pair.handshake()); CHECK(pair.handshake());
const std::array<std::uint8_t, 64> data{}; const std::array<std::uint8_t, 64> data{};
const auto sent = pair.client->send(data); CHECK(pair.client->queue(data) && pair.client->flush());
CHECK(sent && *sent == data.size()); CHECK(pair.client->queued() == 0);
CHECK(pair.wire.to_server.size() > 20); CHECK(pair.wire.to_server.size() > 20);
pair.wire.to_server[pair.wire.to_server.size() - 5] ^= 0x01; pair.wire.to_server[pair.wire.to_server.size() - 5] ^= 0x01;
std::array<std::uint8_t, 256> buffer{}; std::array<std::uint8_t, 256> buffer{};
@@ -181,10 +189,89 @@ void rejections() {
CHECK(!r && r.error() == TlsError::handshake); CHECK(!r && r.error() == TlsError::handshake);
} }
} }
// Review M9.4-M9.7 #1: frames queued while a TLS record is stalled (zero progress,
// or partial progress then zero) must neither be lost nor reordered.
std::vector<std::uint8_t> intents(std::uint64_t first, std::uint64_t count) {
std::vector<std::uint8_t> bytes;
for (std::uint64_t i = 0; i < count; ++i) {
const auto f = encode(IntentMessage{1, 1, first + i, 10, 0, 0});
bytes.insert(bytes.end(), f.begin(), f.end());
}
return bytes;
}
std::vector<std::uint64_t> sequences(const std::vector<std::uint8_t>& bytes) {
FrameReader reader;
std::vector<std::uint64_t> out;
std::span<const std::uint8_t> rest{bytes};
while (!rest.empty()) {
const auto taken = reader.feed(rest);
CHECK(taken);
rest = rest.subspan(*taken);
while (auto f = reader.take()) {
const auto m = decode_intent({f->type, f->payload});
CHECK(m);
out.push_back(m ? m->sequence : 0);
}
}
return out;
}
void interrupted_writes() {
Pki pki;
for (const bool from_client : {true, false})
for (const std::size_t before_stall : {std::size_t{0}, std::size_t{10}}) {
Pair pair{pki.server, {pki.authority, server_name}};
CHECK(pair.handshake());
auto& from = from_client ? *pair.client : *pair.server;
auto& to = from_client ? *pair.server : *pair.client;
auto& budget = from_client ? pair.wire.client_budget : pair.wire.server_budget;
budget = before_stall; // Zero progress, or partial progress then zero.
const auto first = intents(1, 1);
CHECK(from.queue(first));
const auto stalled = from.flush();
CHECK(stalled && *stalled == TlsProgress::want_io);
const auto second = intents(2, 3); // Queued while the first record is stuck.
CHECK(from.queue(second) && from.queued() == first.size() + second.size());
budget = SIZE_MAX;
const auto received = pair.drain(from, to, first.size() + second.size());
CHECK(received.size() == first.size() + second.size() && from.queued() == 0);
CHECK(sequences(received) == std::vector<std::uint64_t>({1, 2, 3, 4}));
}
// Close while stalled: queued frames are flushed before close_notify.
Pair pair{pki.server, {pki.authority, server_name}};
CHECK(pair.handshake());
pair.wire.client_budget = 0;
const auto last = intents(9, 2);
CHECK(pair.client->queue(last));
const auto pending = pair.client->close();
CHECK(pending && *pending == TlsProgress::want_io);
pair.wire.client_budget = SIZE_MAX;
std::expected<TlsProgress, TlsError> closed = TlsProgress::want_io;
for (int i = 0; i < 50 && closed && *closed == TlsProgress::want_io; ++i)
closed = pair.client->close();
CHECK(closed && *closed == TlsProgress::done);
std::vector<std::uint8_t> received;
std::array<std::uint8_t, 256> buffer{};
std::expected<std::size_t, TlsError> r = 0;
for (int i = 0; i < 50 && r; ++i) {
r = pair.server->receive(buffer);
if (r)
received.insert(received.end(), buffer.begin(),
buffer.begin() + static_cast<std::ptrdiff_t>(*r));
}
CHECK(refused(r, TlsError::closed) &&
sequences(received) == std::vector<std::uint64_t>({9, 10}));
// A full queue refuses without queuing or latching.
Pair full{pki.server, {pki.authority, server_name}};
CHECK(full.handshake());
const std::vector<std::uint8_t> huge(TlsChannel::max_queued_bytes + 1, 0);
CHECK(refused(full.client->queue(huge), TlsError::backpressure) && full.client->queued() == 0);
CHECK(full.client->queue(intents(1, 1)) && full.client->queued() == intents(1, 1).size());
}
} // namespace } // namespace
void tls_tests() { void tls_tests() {
handshake_and_frames(); handshake_and_frames();
rejections(); rejections();
interrupted_writes();
} }
} // namespace evo::network_test } // namespace evo::network_test
+5 -3
View File
@@ -50,7 +50,8 @@ struct Credentials {
{"--synthetic-inputs", "development-v1", "--session-listen", {"--synthetic-inputs", "development-v1", "--session-listen",
"127.0.0.1:" + std::to_string(port), "--session-cert", "127.0.0.1:" + std::to_string(port), "--session-cert",
(r.root / "server.pem").string(), "--session-key", (r.root / "server.pem").string(), "--session-key",
(r.root / "server.key").string(), "--session-grant-key", (r.root / "server.key").string(), "--session-grant-ledger",
(r.root / "grants.ledger").string(), "--session-grant-key",
(r.root / "grant.key").string(), "--session-environment", environment}); (r.root / "grant.key").string(), "--session-environment", environment});
} }
std::vector<network::DevGrant> grants(const ServerRegion& r, std::uint64_t subject, std::vector<network::DevGrant> grants(const ServerRegion& r, std::uint64_t subject,
@@ -179,8 +180,9 @@ void connects_acknowledges_and_recovers(const Credentials& c) {
const auto* outcome = last(events, ClientEvent::Kind::outcome); const auto* outcome = last(events, ClientEvent::Kind::outcome);
CHECK(outcome->outcome.finality == network::IntentFinality::committed_no_effect); CHECK(outcome->outcome.finality == network::IntentFinality::committed_no_effect);
first_commit = outcome->outcome.commit_sequence; first_commit = outcome->outcome.commit_sequence;
// The host acknowledged exactly that request ("INT1...") at that tick and commit. // The host acknowledged exactly that carrier ('I','N' + 48-bit epoch 1 + ordinal)
const std::string prefix = "ack " + hex({'I', 'N', 'T', '1'}); // at that tick and commit.
const std::string prefix = "ack " + hex({'I', 'N', 0, 0}) + "00000001";
const auto ack = std::ranges::find_if( const auto ack = std::ranges::find_if(
server.lines(), [&](const std::string& l) { return starts(l, prefix); }); server.lines(), [&](const std::string& l) { return starts(l, prefix); });
CHECK(ack != server.lines().end()); CHECK(ack != server.lines().end());