Fix M9.4-M9.7 review findings in the dev session stack

- TLS: replace send() with a bounded queue/flush; an interrupted record is
  retried with identical arguments and later frames wait behind it, so no
  queued frame is lost or reordered; close() flushes first.
- Carriers: server-issued 'I','N' | 48-bit epoch | 64-bit ordinal ids
  replace the aliasing low-32-bit packing; client sequences are kept
  separately for replies; out-of-range epochs are refused, never truncated.
- Grants: a durable, fsynced, fail-closed consumed-grant ledger
  (--session-grant-ledger, /objects/dev-session-grants.v1 in dev) keeps
  single-use grants consumed across process and pod restarts.
- Liveness: protocol v1 (undeployed) gains Heartbeat and the timeout close
  reason; both sides heartbeat when idle and treat 5 s of silence as dead.
- Client sends are bounded and unresolved tracking starts at send time.
- ci-client configures with an explicit C compiler.

Each finding has a regression test; ADR-0083, the schema, operations doc
and VALIDATION record the corrections.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-02 05:44:44 -04:00
co-authored by Claude Opus 5.5
parent 8e319322f4
commit c588be237c
26 changed files with 934 additions and 193 deletions
+6 -2
View File
@@ -36,7 +36,9 @@
//
// Dev sessions (M9.7, ADR-0083; builds with EVO_BUILD_NETWORK only): --session-listen
// <numeric address:port> --session-cert F --session-key F --session-grant-key F
// --session-environment dev serve the TLS 1.3 dev session protocol. They require
// --session-grant-ledger F --session-environment dev serve the TLS 1.3 dev session
// protocol. The ledger durably records consumed grants (on a persistent volume), so a
// restarted process refuses them. They require
// --synthetic-inputs: an admitted intention rides the next synthetic input (content
// unchanged), so its final outcome is the input's durable acknowledgement, with no
// biological effect. The driver then looks ahead 8 ticks instead of 64. Prints
@@ -207,6 +209,8 @@ std::optional<Options> parse(int argc, char** argv) {
o.session.key = value;
else if (flag == "--session-grant-key")
o.session.grant_key = value;
else if (flag == "--session-grant-ledger")
o.session.grant_ledger = value;
else if (flag == "--session-environment" &&
(value == "dev" || value == "stage" || value == "world0"))
o.session.environment = value;
@@ -559,7 +563,7 @@ int main(int argc, char** argv) {
"[--ops-listen ADDRESS:PORT] [--synthetic-inputs development-v1 "
"[--synthetic-until TICK] [--accelerate N] [--session-listen ADDRESS:PORT "
"--session-cert F --session-key F --session-grant-key F "
"--session-environment dev]]\n",
"--session-grant-ledger F --session-environment dev]]\n",
stderr);
return 2;
}
+18 -14
View File
@@ -22,14 +22,14 @@
namespace evo::server_app {
struct SessionOptions {
std::string listen, certificate, key, grant_key, environment;
std::string listen, certificate, key, grant_key, grant_ledger, environment;
[[nodiscard]] bool any() const noexcept {
return !listen.empty() || !certificate.empty() || !key.empty() || !grant_key.empty() ||
!environment.empty();
!grant_ledger.empty() || !environment.empty();
}
[[nodiscard]] bool complete() const noexcept {
return !listen.empty() && !certificate.empty() && !key.empty() && !grant_key.empty() &&
!environment.empty();
!grant_ledger.empty() && !environment.empty();
}
};
// Intentions ride synthetic inputs, so sessions keep the driver's lookahead short.
@@ -65,11 +65,15 @@ class SessionBridge {
config.admission.intent_window = session_lookahead_ticks;
config.admission.window_ticks = session_lookahead_ticks;
config.admission.max_intents_per_window = session_lookahead_ticks;
auto server = network::SessionServer::create(std::move(config), std::move(*grant_key));
if (!server)
return fail(server.error() == network::SessionServerError::listen
? "error session-listen"
: "error session-credentials");
config.grant_ledger = o.grant_ledger;
auto server = network::SessionServer::create(std::move(config), std::move(*grant_key),
wall_seconds());
if (!server) {
const auto e = server.error();
return fail(e == network::SessionServerError::listen ? "error session-listen"
: e == network::SessionServerError::ledger ? "error session-grant-ledger"
: "error session-credentials");
}
bridge.server_ = std::move(*server);
std::printf("session listening %u\n", static_cast<unsigned>(bridge.server_->port()));
return bridge;
@@ -97,12 +101,7 @@ class SessionBridge {
if (state != nullptr && facts.ready &&
(!published_ || *published_ != s.simulated_tick || epoch_ != s.epoch))
publish(community(*state), facts);
server_->poll(
facts,
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count()),
std::chrono::steady_clock::now());
server_->poll(facts, wall_seconds(), std::chrono::steady_clock::now());
for (const auto& e : server_->take_events())
std::printf("session %s %llu subject=%llu reason=%u\n",
e.kind == network::SessionEvent::Kind::opened ? "opened"
@@ -114,6 +113,11 @@ class SessionBridge {
}
private:
static std::uint64_t wall_seconds() {
return static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count());
}
static std::optional<SessionBridge> fail(const char* message) {
std::puts(message);
return std::nullopt;
+1 -1
View File
@@ -41,7 +41,7 @@ target_link_libraries(evo_network_tls PUBLIC evo_network_protocol PRIVATE mbedtl
evo_owned_target(evo_network_tls)
# M9.7 sessions: non-blocking TCP, the host-agnostic session server and the client.
add_library(evo_network_session STATIC src/network/socket/tcp.cpp src/network/session/server.cpp
src/network/session/client.cpp)
src/network/session/client.cpp src/network/session/grant_ledger.cpp)
target_link_libraries(evo_network_session PUBLIC evo_network_tls)
if(WIN32)
target_link_libraries(evo_network_session PRIVATE ws2_32)
+1 -1
View File
@@ -40,7 +40,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
libx11-dev libxext-dev libxrandr-dev libxcursor-dev libxi-dev libxss-dev libxtst-dev xvfb xauth \
&& rm -rf /var/lib/apt/lists/*
RUN cmake -P tools/prepare-sdl3.cmake \
&& cmake --preset client-probe -DCMAKE_CXX_COMPILER=clang++-19 \
&& cmake --preset client-probe -DCMAKE_CXX_COMPILER=clang++-19 -DCMAKE_C_COMPILER=clang-19 \
-DEVO_BUILD_CLIENT=ON -DEVO_ENABLE_TIDY=ON -DSDL_X11=ON -DSDL_WAYLAND=OFF \
&& cmake --build build/client-probe --target evo-client-probe evo-client evo-client-lifecycle-tests evo-client-environment-tests evo-client-organism-tests --parallel 2 \
&& ctest --preset client-probe --no-tests=error \
@@ -61,10 +61,13 @@ spec:
# Dev sessions are a drill-time override too (tools/ops/session-drill.sh,
# ADR-0083): loopback only, reached by kubectl port-forward, never a Service.
# evo-server refuses any environment but dev and needs the synthetic carrier.
# Consumed grants persist on the objects volume (outside the object
# namespaces, ignored by backups) so a restarted pod refuses them.
sessions=()
if [ -n "$REGION_SESSION" ]; then
sessions=(--session-listen 127.0.0.1:7400 --session-cert /run/session/server.pem
--session-key /run/session/server.key --session-grant-key /run/session/grant.key
--session-grant-ledger /objects/dev-session-grants.v1
--session-environment "$REGION_SESSION")
fi
exec /usr/local/bin/evo-server --profile lineage "${retained[@]}" "${inputs[@]}" \
+39 -12
View File
@@ -28,9 +28,12 @@ Findings that constrain the design:
**Session server in the region process.** `SessionServer` (src/network/session) is
host-agnostic and non-blocking; `evo-server` calls it from its existing single-thread
loop (as it does the ops server). It never mutates host state. Per connection:
bounded output (newest view replaces an unsent one; overflow drops the connection),
1 s Close flush deadline, 5 s accept-to-Welcome deadline, 64 KiB/s input limit,
session bound to its connection (a connection can only speak for its own session).
all output goes through the TLS channel's bounded queue (1 MiB; an interrupted TLS
record is retried with identical arguments, later frames wait behind it; a full queue
drops the slow consumer), the newest view replaces an unsent one, 1 s Close flush
deadline, 5 s accept-to-Welcome deadline, 64 KiB/s input limit, heartbeat after 1 s
without output and `timeout` close after 5 s without input, session bound to its
connection (a connection can only speak for its own session).
At most 4 connections/sessions. New epochs close existing sessions (`stale_epoch`) and
discard intentions not yet carried.
@@ -42,21 +45,37 @@ committed tick as the durable watermark. Every session starts from the latest vi
**Intentions.** Sessions require `--synthetic-inputs development-v1`. An admitted
intention is queued and rides the next synthetic input whose tick is at or after its
intended tick: the input's request id becomes `INT1 | epoch | session | sequence`
(low 32 bits each); its **content stays the host-chosen development input**. The
intended tick under a fresh **server-issued carrier id** `'I','N' | epoch (48 bits)
| ordinal (64 bits)`, never derived from client values; the client sequence is kept
separately for replies. Epochs beyond 48 bits are refused at Hello (`capacity`) and
ordinal exhaustion stops carrying; nothing wraps or truncates. The input's **content
stays the host-chosen development input**. The
host's FinalAck for that request yields `IntentOutcome{committed_no_effect, tick,
commit_sequence}`; a dropped input yields `not_committed`. The driver looks ahead 8
ticks (instead of 64) when sessions are on, matching the admission window.
**Protocol v1 amendment.** Before any v1 peer was deployed, v1 gained message type 8
`IntentOutcome` and close reason 9 `capacity` (schemas/view/dev-session-v1.md). Once
a dev deployment runs, further changes need a new version.
**Protocol v1 amendment.** Before any v1 peer was deployed, v1 gained message types 8
`IntentOutcome` and 9 `Heartbeat` (empty) and close reasons 9 `capacity` and 10
`timeout` (schemas/view/dev-session-v1.md). Once a dev deployment runs, further
changes need a new version.
**Grant consumption survives restarts.** Consumed grant ids are appended to a durable
ledger (`--session-grant-ledger`; `/objects/dev-session-grants.v1` in dev, outside the
object namespaces and ignored by backups) and fsynced **before** Welcome; a failed
write refuses the session and keeps the grant consumed in memory. A replacement
process loads unexpired entries into admission at startup. Corrupt content, I/O
failure or more than 65536 entries fail closed. The ledger does not survive a
restore-to-bundle (a rewind) or a lost volume; grants expire within an hour.
**Client.** `SessionClient` connects, verifies the pinned dev CA, consumes one
single-use grant per Hello, and on Welcome resets both layers (fresh baseline). On
loss it keeps the last presentation marked stale, reports outstanding admitted
intentions as `unknown`, and retries with capped exponential backoff until grants run
out (explicit `exhausted`). Intended tick = newest view tick + 4. The desktop app
single-use grant per Hello (except after `server_shutdown`/`capacity`, which the
server refuses before recording the grant), and on Welcome resets both layers (fresh
baseline). It heartbeats when idle and treats 5 s without any server frame as loss.
On loss it keeps the last presentation marked stale, reports every sent-but-unresolved
intention as `unknown` (sent without a result, or admitted without an outcome), and
retries with capped exponential backoff until grants run out (explicit `exhausted`).
`send` is bounded: at most 256 unresolved intentions and the channel queue's capacity;
beyond that it refuses rather than buffers. Intended tick = newest view tick + 4. The desktop app
sends an intention only when its vector changes, at most once per view tick.
`evo-client --connect ... --ca ... --grants ...` replaces the local fixture.
@@ -103,6 +122,14 @@ tokens: anyone holding an unused grant (or the dev key) can open a dev session.
Restart gaps: an intention admitted just before a crash has an unknown outcome until
the client reconnects (reported as such, never assumed).
## Review corrections (2026-10-02)
The M9.4-M9.7 review (`.cache/reviews/M9.4-M9.7-review.md`, not in Git) reproduced
and this revision fixes: TLS retry with changed arguments losing queued frames;
carrier ids aliasing sequences 2^32 apart (the original low-32-bit packing above was
unsafe and is withdrawn); consumed grants becoming valid again after a restart; no
liveness after Welcome; and the `ci-client` configure missing its C compiler. Each has
a regression test (engineering/VALIDATION.md).
## Consequences
M9.8 packages the client and qualifies the presentation. M10 replaces the carrier
with real intention semantics once organisms move, and reopens transport choice.
+55
View File
@@ -3509,3 +3509,58 @@ were not executed remotely. Port-forward reachability of the pod-loopback listen
is the documented design, not yet observed. M8-dev/stage remains not passed. The
connected desktop run was hidden/software-rendered; no human visual check of remote
mode. Intentions have no biological effect by design.
## M9.4–M9.7 review corrections — 2026-10-02
Responds to the review of `bb0412c..8e31932` (`.cache/reviews/M9.4-M9.7-review.md`,
not in Git). All five findings were confirmed against the code and fixed:
1. **[P1] TLS retry lost queued frames.** `TlsChannel::send` is replaced by a bounded
(1 MiB) `queue` + `flush`: an interrupted record is retried with identical
arguments, later frames wait behind it, `close()` flushes before close_notify, a
full queue returns `backpressure` without queuing. Server and client route every
frame through it. Regression (`interrupted_writes`): zero progress and partial
progress then zero, frames queued during the stall, both directions, exact ordered
sequences; close while stalled; backpressure.
2. **[P1] Carrier ids aliased.** Carriers are server-issued `'I','N' | 48-bit epoch |
64-bit ordinal`; the client sequence is kept separately. Epochs beyond 48 bits are
refused at Hello; ordinal exhaustion stops carrying. ADR-0083's low-32-bit packing
is withdrawn. Regression (`carriers_never_alias`): sequences 1 and 2^32+1 get
distinct ids; `not_committed` and `committed` reach their own sequences.
3. **[P2] Grants reusable after restart.** New durable `GrantLedger` (fsync before
Welcome, atomic prune on open, fail closed on corruption/I-O/65536 entries), loaded
into admission by a replacement process; `--session-grant-ledger` (dev:
`/objects/dev-session-grants.v1`). Regressions: restart with the same ledger
refuses the consumed grant and admits a fresh one; corrupt ledger refuses server
creation; expired entries pruned; and in the end-to-end rehearsal the successor of
a SIGKILLed evo-server refused the first, consumed grant (`closed reason=3`).
4. **[P2] No liveness after Welcome.** Protocol v1 (still undeployed) gains Heartbeat
(type 9) and Close `timeout` (10). Both sides heartbeat after 1 s without output and
treat 5 s of silence as dead (server closes `timeout`, freeing the subject; client
marks stale, reports unknown outcomes, reconnects). Regressions (`liveness`): idle
but healthy session survives 1.5 s with no ticks; a silent server is detected and
reconnected to a fresh baseline; a silent client is closed and its subject reused.
5. **[P1, inherited] ci-client C compiler.** `-DCMAKE_C_COMPILER=clang-19` added; all
other C-enabling fresh configures already name it.
Follow-ups taken: `SessionClient::send` is bounded (256 unresolved, channel queue
capacity) and refuses instead of buffering; unresolved tracking now starts at send,
so a loss before IntentResult is also reported as unknown. The abuse test now floods
through a raw TLS peer (the client can no longer flood) and checks the server's
`rate_limited` decision. Client event-vector bounds remain M9.8 work.
Executed after the fixes: Windows MSVC `network` 1/1 and `client-network` 6/6; Linux
GCC 14 network suite and the **full** PostgreSQL integration suite (all suites,
including the session process test with the ledger and new carrier ids); Clang 19
ASan/UBSan network suite; clang-tidy 19 (warnings as errors) on all changed network,
client and server units (one `string_view::data` finding fixed in the ledger parser);
libFuzzer on the amended codec, 523,479 runs in 61 s, no crash or canonicality abort;
repository format, CI selector fixtures and deploy policy (67 objects, 0 failures).
End-to-end rehearsal: durable ack, SIGKILL, successor Ready in 3472 ms at epoch 2,
grants kept across two `server_shutdown` refusals, second durable ack, consumed grant
refused by the successor, Windows evo-client welcomed (`session=2 epoch=2`), successor
drained with `exit 0`.
Not done: the reviewer's own reproduction binaries target the removed `send` API and
were not rebuilt; equivalent regressions above replace them. The `ci-client` Docker
image and Gitea pipeline were not rerun here, and no cluster operation was performed.
+7 -3
View File
@@ -37,9 +37,13 @@ kubectl -n evolution-dev create secret generic evolution-dev-session \
--from-file=grant.key=$HOME/aevolution-dev-session/grant.key
```
The Deployment mounts it optionally (0440). Rotation: create new credentials,
replace the Secret, restart evo-region, redistribute `dev-ca.pem`. Old grants stop
verifying with the new key.
The Deployment mounts it optionally (0440). Consumed grants are recorded in
`/objects/dev-session-grants.v1` on the region's persistent volume (fsynced before
each Welcome), so a restarted pod refuses grants its predecessor used. Corrupt or
unwritable ledgers fail closed (`error session-grant-ledger`, or sessions refused).
Rotation: create new credentials, replace the Secret, restart evo-region and
redistribute `dev-ca.pem`. Old grants stop verifying with the new key.
## 4. Session drill (enables sessions temporarily, then restores Git values)
+9
View File
@@ -4,6 +4,7 @@
#include <cstddef>
#include <cstdint>
#include <expected>
#include <span>
#include <vector>
// Server-side session admission for the M9.6 dev session (ADR-0082). Transport-
@@ -17,6 +18,10 @@ struct DevGrantClaims {
world::WorldIdentity world;
std::uint64_t region{}, issued_at{}, expires_at{}; // Server clock, seconds.
};
// A grant already used to open a session, remembered until it expires.
struct ConsumedGrant {
std::uint64_t grant_id{}, expires_at{};
};
class DevGrantKey;
// Only DevGrantKey::verify constructs this, so admission cannot be handed
// unauthenticated claims by accident.
@@ -68,6 +73,10 @@ class SessionAdmission {
[[nodiscard]] IntentResult admit(const IntentMessage& intent, std::uint64_t server_tick,
std::uint64_t now_seconds);
[[nodiscard]] std::expected<void, AdmissionError> close(std::uint64_t session);
// Restores grants consumed before a restart (from a durable ledger), so a replacement
// host refuses them too. Expired entries are skipped; too many fail closed.
[[nodiscard]] std::expected<void, AdmissionError>
remember(std::span<const ConsumedGrant> consumed, std::uint64_t now_seconds);
// New ownership epoch: existing sessions become stale and must reconnect.
[[nodiscard]] std::expected<void, AdmissionError> advance_epoch(std::uint64_t epoch);
[[nodiscard]] std::size_t sessions() const noexcept {
+42
View File
@@ -0,0 +1,42 @@
#pragma once
#include "evo/network/admission.hpp"
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <expected>
#include <filesystem>
#include <memory>
#include <span>
#include <utility>
#include <vector>
// Durable record of consumed dev grants (ADR-0083), so a replacement region process
// refuses grants its predecessor already used. Append-only text lines
// "<grant id hex> <expires_at>", flushed and fsynced before the session is welcomed.
// Opening drops expired entries by an atomic rewrite. Corrupt content, I/O failure or
// a full ledger fail closed. Single owner; not a general database.
namespace evo::network {
enum class LedgerError : std::uint8_t { io, corrupt, full };
class GrantLedger {
public:
static constexpr std::size_t max_entries = 65536;
[[nodiscard]] static std::expected<std::unique_ptr<GrantLedger>, LedgerError>
open(const std::filesystem::path& path, std::uint64_t now_seconds);
GrantLedger(const GrantLedger&) = delete;
GrantLedger& operator=(const GrantLedger&) = delete;
~GrantLedger();
// Unexpired entries loaded at open (for SessionAdmission::remember).
[[nodiscard]] std::span<const ConsumedGrant> loaded() const noexcept {
return loaded_;
}
// Durable once this returns success.
[[nodiscard]] std::expected<void, LedgerError> record(ConsumedGrant grant);
private:
GrantLedger(std::FILE* file, std::vector<ConsumedGrant> loaded)
: file_{file}, loaded_{std::move(loaded)}, entries_{loaded_.size()} {}
std::FILE* file_;
std::vector<ConsumedGrant> loaded_;
std::size_t entries_;
};
} // namespace evo::network
+6 -2
View File
@@ -28,7 +28,8 @@ enum class MessageType : std::uint8_t {
environment_view,
organism_view,
close,
intent_outcome
intent_outcome,
heartbeat // Empty payload; proves liveness when no other frame flows (ADR-0083).
};
enum class ProtocolError : std::uint8_t {
truncated,
@@ -121,7 +122,8 @@ enum class CloseReason : std::uint8_t {
stale_epoch,
rate_limited,
server_shutdown,
capacity
capacity,
timeout
};
// Final, after the host's durable commit (or loss): the M9.7 dev region has no
// movable organism, so a committed intention has no biological effect.
@@ -138,6 +140,7 @@ struct IntentOutcome {
[[nodiscard]] std::vector<std::uint8_t> encode(const IntentResult& m);
[[nodiscard]] std::vector<std::uint8_t> encode(CloseReason reason);
[[nodiscard]] std::vector<std::uint8_t> encode(const IntentOutcome& m);
[[nodiscard]] std::vector<std::uint8_t> encode_heartbeat();
[[nodiscard]] std::vector<std::uint8_t> encode(const ViewHeader& header,
const client::EnvironmentPresentation& view);
[[nodiscard]] std::vector<std::uint8_t> encode(const ViewHeader& header,
@@ -151,6 +154,7 @@ struct IntentOutcome {
[[nodiscard]] std::expected<IntentResult, ProtocolError> decode_intent_result(FrameView frame);
[[nodiscard]] std::expected<CloseReason, ProtocolError> decode_close(FrameView frame);
[[nodiscard]] std::expected<IntentOutcome, ProtocolError> decode_intent_outcome(FrameView frame);
[[nodiscard]] std::expected<void, ProtocolError> decode_heartbeat(FrameView frame);
[[nodiscard]] std::expected<std::pair<ViewHeader, client::EnvironmentPresentation>, ProtocolError>
decode_environment_view(FrameView frame);
[[nodiscard]] std::expected<std::pair<ViewHeader, client::OrganismPresentation>, ProtocolError>
+6 -1
View File
@@ -28,6 +28,9 @@ struct SessionClientConfig {
std::uint32_t lead_ticks{4}; // Intended tick = newest view tick + lead.
std::chrono::milliseconds attempt_deadline{5000}; // Connect to Welcome.
std::chrono::milliseconds first_retry{500}, max_retry{8000};
// Liveness after Welcome: heartbeat when idle; a silent server past the timeout is
// treated as lost (stale presentation, unknown outcomes, reconnect).
std::chrono::milliseconds heartbeat_interval{1000}, liveness_timeout{5000};
};
enum class SessionPhase : std::uint8_t { backoff, connecting, established, exhausted };
struct ClientEvent {
@@ -56,7 +59,9 @@ class SessionClient {
SessionClient& operator=(const SessionClient&) = delete;
~SessionClient();
void poll(std::chrono::steady_clock::time_point now);
// Sends when established and a view has arrived; never queued across connections.
// Queues one intention when established, a view has arrived, fewer than
// max_outstanding are unresolved and the channel queue has room; otherwise false.
// Never carried across connections.
[[nodiscard]] bool send(const client::Intention& intention);
[[nodiscard]] SessionPhase phase() const noexcept;
[[nodiscard]] const client::EnvironmentLayer& environment() const noexcept;
+11 -3
View File
@@ -1,6 +1,7 @@
#pragma once
#include "evo/network/admission.hpp"
#include "evo/network/dev_grant.hpp"
#include "evo/network/grant_ledger.hpp"
#include "evo/network/tcp.hpp"
#include "evo/network/tls_channel.hpp"
#include <array>
@@ -9,6 +10,7 @@
#include <cstdint>
#include <deque>
#include <expected>
#include <filesystem>
#include <memory>
#include <optional>
#include <string>
@@ -30,13 +32,18 @@ struct SessionServerConfig {
std::size_t max_input_bytes_per_second{65536};
std::size_t max_pending_intents{64}; // Admitted, not yet carried.
std::size_t max_carried_intents{1024}; // Carried, awaiting durable/lost.
// Liveness both ways: a Heartbeat is sent when nothing else was sent for an
// interval; a peer silent for the timeout is closed with `timeout`.
std::chrono::milliseconds heartbeat_interval{1000}, liveness_timeout{5000};
// Durable consumed-grant ledger (required); survives process and pod restarts.
std::filesystem::path grant_ledger;
};
// The host's current authority facts, read once per loop iteration.
struct RegionFacts {
bool ready{};
std::uint64_t epoch{}, simulated_tick{}, committed_tick{}, commit_sequence{};
};
enum class SessionServerError : std::uint8_t { configuration, listen, credentials };
enum class SessionServerError : std::uint8_t { configuration, listen, credentials, ledger };
struct SessionEvent {
enum class Kind : std::uint8_t { opened, closed, refused } kind{};
std::uint64_t session{}, subject{};
@@ -45,7 +52,7 @@ struct SessionEvent {
class SessionServer {
public:
[[nodiscard]] static std::expected<std::unique_ptr<SessionServer>, SessionServerError>
create(SessionServerConfig config, DevGrantKey key);
create(SessionServerConfig config, DevGrantKey key, std::uint64_t now_seconds);
SessionServer(const SessionServer&) = delete;
SessionServer& operator=(const SessionServer&) = delete;
~SessionServer();
@@ -59,7 +66,8 @@ class SessionServer {
void publish(const client::EnvironmentPresentation& environment,
const client::OrganismPresentation& organisms, const RegionFacts& facts);
// The oldest admitted intention whose intended tick is <= `tick`, to ride the host
// input for `tick`. The host input's content stays host-chosen.
// input for `tick`, under a fresh server-issued carrier id ('I','N', 48-bit epoch,
// 64-bit ordinal). The host input's content stays host-chosen.
[[nodiscard]] std::optional<RequestId> carry(std::uint64_t tick);
// Host final acknowledgement / drop for a request id; ids not carried here are ignored.
void committed(const RequestId& request, std::uint64_t tick, std::uint64_t commit_sequence);
+12 -3
View File
@@ -37,7 +37,8 @@ enum class TlsError : std::uint8_t {
peer_verification,
transport,
closed,
record
record,
backpressure
};
enum class TlsProgress : std::uint8_t { done, want_io };
class TlsChannel {
@@ -50,9 +51,17 @@ class TlsChannel {
TlsChannel& operator=(const TlsChannel&) = delete;
~TlsChannel();
[[nodiscard]] std::expected<TlsProgress, TlsError> handshake();
// Zero means "would block". Errors latch until destruction.
[[nodiscard]] std::expected<std::size_t, TlsError> send(std::span<const std::uint8_t> bytes);
// Outbound plaintext is queued, never handed to the caller to retry: a record that
// TLS could not finish writing is retried with identical arguments (Mbed TLS
// requires this), and newly queued bytes wait behind it. Bounded; a full queue
// returns `backpressure` and queues nothing. Errors latch until destruction.
static constexpr std::size_t max_queued_bytes = std::size_t{1} << 20;
[[nodiscard]] std::expected<void, TlsError> queue(std::span<const std::uint8_t> bytes);
// Writes queued plaintext until done or the transport would block.
[[nodiscard]] std::expected<TlsProgress, TlsError> flush();
[[nodiscard]] std::size_t queued() const noexcept;
[[nodiscard]] std::expected<std::size_t, TlsError> receive(std::span<std::uint8_t> bytes);
// Flushes queued plaintext first, then sends close_notify (want_io until both done).
[[nodiscard]] std::expected<TlsProgress, TlsError> close();
// Negotiated "TLSv1.3" and suite name after a completed handshake.
[[nodiscard]] std::string_view version() const noexcept;
+5 -4
View File
@@ -12,7 +12,7 @@ message re-encodes to identical bytes (fuzz-tested).
|---|---|---|---|
| 0 | magic | u32 | `0x53445645` (bytes `E V D S`) |
| 4 | version | u16 | exactly 1, otherwise `unsupported_version` |
| 6 | type | u8 | 1..8, otherwise `unknown_type` |
| 6 | type | u8 | 1..9, otherwise `unknown_type` |
| 7 | flags | u8 | 0, otherwise `reserved_flags` |
| 8 | length | u32 | payload bytes, <= 262144 (`oversized`) |
| 12 | payload | bytes | exactly `length`; short `truncated`, extra `trailing_bytes` |
@@ -30,15 +30,16 @@ Payload decoders also reject truncation, trailing bytes, out-of-range enum bytes
| 4 | IntentResult | S->C | sequence u64, status u8 (0..7), server_tick u64 |
| 5 | EnvironmentView | S->C | view header, environment presentation |
| 6 | OrganismView | S->C | view header, organism presentation |
| 7 | Close | both | reason u8 (0..9) |
| 7 | Close | both | reason u8 (0..10) |
| 8 | IntentOutcome | S->C | sequence u64, finality u8 (0 committed_no_effect, 1 not_committed), tick u64, commit_sequence u64; committed requires commit_sequence != 0, not_committed requires tick = commit_sequence = 0 |
| 9 | Heartbeat | both | empty (length 0); sent after 1 s without other output; 5 s of silence closes with `timeout` |
IntentStatus: 0 admitted, 1 unknown_session, 2 stale_epoch, 3 expired,
4 invalid_vector, 5 replayed, 6 outside_window, 7 rate_limited.
CloseReason: 0 normal, 1 unsupported_version, 2 malformed, 3 unauthenticated,
4 expired, 5 duplicate_session, 6 stale_epoch, 7 rate_limited, 8 server_shutdown,
9 capacity. Type 8 and reason 9 were added in M9.7 before any v1 peer was
deployed (ADR-0083); later changes require a new version.
9 capacity, 10 timeout. Types 8-9 and reasons 9-10 were added in M9.7 before any v1
peer was deployed (ADR-0083); later changes require a new version.
View header (40 bytes): session, epoch, server_tick, commit_watermark, baseline (u64).
+14
View File
@@ -103,6 +103,20 @@ IntentResult SessionAdmission::admit(const IntentMessage& intent, std::uint64_t
return result;
}
std::expected<void, AdmissionError>
SessionAdmission::remember(std::span<const ConsumedGrant> consumed, std::uint64_t now_seconds) {
for (const auto& g : consumed) {
if (g.expires_at <= now_seconds || std::ranges::any_of(grants_, [&](const Remembered& r) {
return r.grant_id == g.grant_id;
}))
continue;
if (grants_.size() >= config_.max_remembered_grants)
return std::unexpected(AdmissionError::grants_full);
grants_.push_back({g.grant_id, g.expires_at});
}
return {};
}
std::expected<void, AdmissionError> SessionAdmission::close(std::uint64_t session) {
if (std::erase_if(sessions_, [&](const Session& s) { return s.id == session; }) == 0)
return std::unexpected(AdmissionError::unknown_session);
+14 -2
View File
@@ -5,9 +5,9 @@
namespace evo::network {
namespace {
constexpr std::uint8_t last_type = static_cast<std::uint8_t>(MessageType::intent_outcome);
constexpr std::uint8_t last_type = static_cast<std::uint8_t>(MessageType::heartbeat);
constexpr std::uint8_t last_status = static_cast<std::uint8_t>(IntentStatus::rate_limited);
constexpr std::uint8_t last_reason = static_cast<std::uint8_t>(CloseReason::capacity);
constexpr std::uint8_t last_reason = static_cast<std::uint8_t>(CloseReason::timeout);
constexpr std::uint8_t last_finality = static_cast<std::uint8_t>(IntentFinality::not_committed);
constexpr std::uint8_t last_presence = static_cast<std::uint8_t>(client::FieldPresence::present);
constexpr std::uint8_t last_moisture = static_cast<std::uint8_t>(world::MoistureStatus::available);
@@ -280,6 +280,18 @@ std::vector<std::uint8_t> encode(CloseReason reason) {
return framed(MessageType::close, w);
}
std::vector<std::uint8_t> encode_heartbeat() {
return encode_frame(MessageType::heartbeat, {});
}
std::expected<void, ProtocolError> decode_heartbeat(FrameView frame) {
if (frame.type != MessageType::heartbeat)
return std::unexpected(ProtocolError::wrong_type);
if (!frame.payload.empty())
return std::unexpected(ProtocolError::trailing_bytes);
return {};
}
std::vector<std::uint8_t> encode(const IntentOutcome& m) {
Writer w;
w.u64(m.sequence);
+50 -28
View File
@@ -18,15 +18,16 @@ struct SessionClient::Impl {
std::size_t next_grant{};
SessionPhase phase{SessionPhase::backoff};
Step step{Step::handshake};
Clock::time_point retry_at{}, deadline{};
Clock::time_point retry_at{}, deadline{}, last_received{}, last_sent{}, polled_at{};
std::chrono::milliseconds delay;
std::unique_ptr<TcpStream> stream; // Declared before the channel that borrows it.
std::unique_ptr<TlsChannel> tls;
FrameReader reader;
std::vector<std::uint8_t> out;
std::optional<Welcome> session;
std::optional<ViewHeader> view;
std::vector<std::uint64_t> outstanding; // Admitted, awaiting a final outcome.
// Sent and not finally resolved: no IntentResult yet, or admitted without an
// outcome. A connection loss reports every one of them as unknown.
std::vector<std::uint64_t> outstanding;
client::EnvironmentLayer environment;
client::OrganismLayer organisms;
std::vector<ClientEvent> events;
@@ -45,7 +46,6 @@ struct SessionClient::Impl {
tls.reset();
stream.reset();
reader = FrameReader{};
out.clear();
session.reset();
view.reset();
// Keep showing the last presentation, explicitly stale, until a new baseline.
@@ -77,9 +77,18 @@ struct SessionClient::Impl {
phase = SessionPhase::connecting;
step = Step::handshake;
deadline = now + config.attempt_deadline;
last_received = last_sent = now;
}
// Every outbound frame goes through the channel's bounded queue.
bool queue(const std::vector<std::uint8_t>& frame, Clock::time_point now) {
if (!tls || !tls->queue(frame))
return false;
last_sent = now;
return true;
}
bool frame(const OwnedFrame& f, Clock::time_point now) {
const FrameView v{f.type, f.payload};
last_received = now;
if (step == Step::awaiting_welcome) {
if (f.type == MessageType::close) {
const auto reason = decode_close(v);
@@ -132,14 +141,9 @@ struct SessionClient::Impl {
disconnect(now, std::nullopt);
return false;
}
if (r->status == IntentStatus::admitted) {
if (outstanding.size() >= max_outstanding) {
events.push_back(
{ClientEvent::Kind::unknown, {}, {}, {}, {}, outstanding.front()});
outstanding.erase(outstanding.begin());
}
outstanding.push_back(r->sequence);
}
// A rejection is final; an admitted intention waits for its outcome.
if (r->status != IntentStatus::admitted)
std::erase(outstanding, r->sequence);
events.push_back({ClientEvent::Kind::result, {}, {}, *r, {}, r->sequence});
return true;
}
@@ -153,6 +157,12 @@ struct SessionClient::Impl {
events.push_back({ClientEvent::Kind::outcome, {}, {}, {}, *o, o->sequence});
return true;
}
case MessageType::heartbeat:
if (!decode_heartbeat(v)) {
disconnect(now, std::nullopt);
return false;
}
return true;
case MessageType::close: {
const auto reason = decode_close(v);
disconnect(now, reason.value_or(CloseReason::malformed));
@@ -164,18 +174,8 @@ struct SessionClient::Impl {
}
}
void flush(Clock::time_point now) {
std::size_t sent = 0;
while (sent < out.size()) {
const auto n = tls->send(std::span{out}.subspan(sent));
if (!n) {
disconnect(now, std::nullopt);
return;
}
if (*n == 0)
break;
sent += *n;
}
out.erase(out.begin(), out.begin() + static_cast<std::ptrdiff_t>(sent));
if (!tls->flush())
disconnect(now, std::nullopt);
}
void receive(Clock::time_point now) {
std::array<std::uint8_t, 16384> buffer{};
@@ -215,7 +215,8 @@ SessionClient::create(SessionClientConfig config) {
if (config.host.empty() || config.port == 0 || config.grants.empty() ||
config.grants.size() > 1024 || config.lead_ticks == 0 || config.lead_ticks > 64 ||
config.first_retry.count() <= 0 || config.max_retry < config.first_retry ||
config.attempt_deadline.count() <= 0)
config.attempt_deadline.count() <= 0 || config.heartbeat_interval.count() <= 0 ||
config.liveness_timeout <= 2 * config.heartbeat_interval)
return std::unexpected(SessionClientError::configuration);
return std::unique_ptr<SessionClient>(
new SessionClient(std::make_unique<Impl>(std::move(config))));
@@ -223,6 +224,7 @@ SessionClient::create(SessionClientConfig config) {
void SessionClient::poll(std::chrono::steady_clock::time_point now) {
auto& m = *impl_;
m.polled_at = now;
if (m.phase == SessionPhase::exhausted)
return;
if (m.phase == SessionPhase::backoff) {
@@ -236,6 +238,18 @@ void SessionClient::poll(std::chrono::steady_clock::time_point now) {
m.disconnect(now, std::nullopt);
return;
}
if (m.step == Impl::Step::established) {
// A silent server (paused process, black-holed path) is treated as lost: the
// presentation turns stale and the reconnect policy takes over.
if (now - m.last_received > m.config.liveness_timeout) {
m.disconnect(now, std::nullopt);
return;
}
if (now - m.last_sent >= m.config.heartbeat_interval && !m.queue(encode_heartbeat(), now)) {
m.disconnect(now, std::nullopt);
return;
}
}
if (m.step == Impl::Step::handshake) {
const auto p = m.tls->handshake();
if (!p) {
@@ -251,7 +265,10 @@ void SessionClient::poll(std::chrono::steady_clock::time_point now) {
m.disconnect(now, std::nullopt);
return;
}
m.out.insert(m.out.end(), hello->begin(), hello->end());
if (!m.queue(*hello, now)) {
m.disconnect(now, std::nullopt);
return;
}
m.step = Impl::Step::awaiting_welcome;
}
m.flush(now);
@@ -263,13 +280,18 @@ void SessionClient::poll(std::chrono::steady_clock::time_point now) {
bool SessionClient::send(const client::Intention& intention) {
auto& m = *impl_;
if (m.phase != SessionPhase::established || !m.session || !m.view)
// Bounded: no session, no view, too many unresolved intentions or a full channel
// queue all refuse rather than buffer (the caller resamples later).
if (m.phase != SessionPhase::established || !m.session || !m.view ||
m.outstanding.size() >= max_outstanding)
return false;
const auto tick = m.view->server_tick + m.config.lead_ticks;
const auto bytes =
encode(make_intent(m.session->session, m.session->epoch,
{intention.sequence, tick, intention.move_x, intention.move_y}));
m.out.insert(m.out.end(), bytes.begin(), bytes.end());
if (!m.queue(bytes, m.polled_at))
return false;
m.outstanding.push_back(intention.sequence);
return true;
}
+122
View File
@@ -0,0 +1,122 @@
#include "evo/network/grant_ledger.hpp"
#include <charconv>
#include <fstream>
#include <string>
#include <string_view>
#include <system_error>
#ifdef _WIN32
#include <io.h>
#else
#include <fcntl.h>
#include <unistd.h>
#endif
namespace evo::network {
namespace {
std::FILE* open_file(const std::filesystem::path& path, const char* mode) {
#ifdef _WIN32
std::FILE* file = nullptr;
return _wfopen_s(&file, path.c_str(), mode[0] == 'w' ? L"wb" : L"ab") == 0 ? file : nullptr;
#else
return std::fopen(path.c_str(), mode);
#endif
}
bool sync(std::FILE* file) {
if (std::fflush(file) != 0)
return false;
#ifdef _WIN32
return _commit(_fileno(file)) == 0;
#else
return ::fsync(::fileno(file)) == 0;
#endif
}
// Makes a rename durable on POSIX by syncing the containing directory.
bool sync_directory([[maybe_unused]] const std::filesystem::path& dir) {
#ifdef _WIN32
return true; // NTFS metadata journaling; MoveFileEx is used with replace semantics.
#else
const int fd = ::open(dir.empty() ? "." : dir.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC);
if (fd < 0)
return false;
const bool ok = ::fsync(fd) == 0;
return ::close(fd) == 0 && ok;
#endif
}
std::string line(ConsumedGrant g) {
constexpr char digits[] = "0123456789abcdef";
std::string out(16, '0');
for (int i = 15; i >= 0; --i, g.grant_id >>= 4U)
out[static_cast<std::size_t>(i)] = digits[g.grant_id & 0x0FU];
return out + ' ' + std::to_string(g.expires_at) + '\n';
}
bool parse(const std::string& text, ConsumedGrant& out) { // One getline() line.
if (text.size() < 18 || text[16] != ' ')
return false;
const auto [a, e1] = std::from_chars(text.data(), text.data() + 16, out.grant_id, 16);
const auto [b, e2] =
std::from_chars(text.data() + 17, text.data() + text.size(), out.expires_at);
return e1 == std::errc{} && a == text.data() + 16 && e2 == std::errc{} &&
b == text.data() + text.size() && out.grant_id != 0;
}
} // namespace
std::expected<std::unique_ptr<GrantLedger>, LedgerError>
GrantLedger::open(const std::filesystem::path& path, std::uint64_t now_seconds) {
std::vector<ConsumedGrant> live;
std::error_code error;
if (std::filesystem::exists(path, error)) {
std::ifstream in{path, std::ios::binary};
if (!in)
return std::unexpected(LedgerError::io);
std::string text;
std::size_t total = 0;
while (std::getline(in, text)) {
ConsumedGrant g;
if (!parse(text, g))
return std::unexpected(LedgerError::corrupt); // Never guess: fail closed.
if (++total > max_entries)
return std::unexpected(LedgerError::full);
if (g.expires_at > now_seconds)
live.push_back(g);
}
if (!in.eof())
return std::unexpected(LedgerError::io);
} else if (error) {
return std::unexpected(LedgerError::io);
}
// Rewrite the unexpired entries atomically, then append to the result.
const auto temporary = std::filesystem::path{path}.concat(".tmp");
{
std::FILE* out = open_file(temporary, "wb");
if (out == nullptr)
return std::unexpected(LedgerError::io);
bool ok = true;
for (const auto& g : live)
ok = ok && std::fputs(line(g).c_str(), out) >= 0;
ok = sync(out) && ok;
if (std::fclose(out) != 0 || !ok)
return std::unexpected(LedgerError::io);
}
std::filesystem::rename(temporary, path, error);
if (error || !sync_directory(path.parent_path()))
return std::unexpected(LedgerError::io);
std::FILE* file = open_file(path, "ab");
if (file == nullptr)
return std::unexpected(LedgerError::io);
return std::unique_ptr<GrantLedger>(new GrantLedger(file, std::move(live)));
}
GrantLedger::~GrantLedger() {
std::fclose(file_);
}
std::expected<void, LedgerError> GrantLedger::record(ConsumedGrant grant) {
if (entries_ >= max_entries)
return std::unexpected(LedgerError::full);
if (std::fputs(line(grant).c_str(), file_) < 0 || !sync(file_))
return std::unexpected(LedgerError::io);
++entries_;
return {};
}
} // namespace evo::network
+128 -83
View File
@@ -1,5 +1,6 @@
#include "evo/network/session_server.hpp"
#include <algorithm>
#include <limits>
#include <utility>
namespace evo::network {
@@ -7,6 +8,9 @@ namespace {
using Clock = std::chrono::steady_clock;
constexpr std::size_t max_accepts_per_poll = 4;
constexpr std::size_t max_read_per_poll = 65536;
// Carrier ids are 'I','N' + 48-bit epoch + 64-bit server ordinal: distinct for every
// carried intention, independent of client-chosen sequences (which may be any u64).
constexpr std::uint64_t max_carrier_epoch = (std::uint64_t{1} << 48) - 1;
CloseReason close_reason(AdmissionError e) {
switch (e) {
case AdmissionError::duplicate_subject:
@@ -24,8 +28,13 @@ CloseReason close_reason(ProtocolError e) {
return e == ProtocolError::unsupported_version ? CloseReason::unsupported_version
: CloseReason::malformed;
}
void append(std::vector<std::uint8_t>& out, const std::vector<std::uint8_t>& bytes) {
out.insert(out.end(), bytes.begin(), bytes.end());
RequestId carrier_id(std::uint64_t epoch, std::uint64_t ordinal) {
RequestId id{'I', 'N'};
for (unsigned i = 0; i < 6; ++i)
id.at(2 + i) = static_cast<std::uint8_t>(epoch >> (8U * (5U - i)));
for (unsigned i = 0; i < 8; ++i)
id.at(8 + i) = static_cast<std::uint8_t>(ordinal >> (8U * (7U - i)));
return id;
}
} // namespace
@@ -34,18 +43,16 @@ struct SessionServer::Impl {
struct Connection {
std::uint64_t id{};
std::unique_ptr<TcpStream> stream; // Declared first: outlives the channel.
std::unique_ptr<TlsChannel> tls;
std::unique_ptr<TlsChannel> tls; // Owns all queued output (ADR-0083 review fix).
State state{State::handshake};
Clock::time_point deadline;
Clock::time_point deadline, last_received, last_sent, window_start;
FrameReader reader;
std::vector<std::uint8_t> out, view;
std::vector<std::uint8_t> view; // Newest unsent view only; replaced, never queued.
std::uint64_t session{}, subject{}, epoch{}, expires_at{};
std::size_t window_bytes{};
Clock::time_point window_start;
bool dead{};
};
struct Pending {
RequestId request{};
std::uint64_t intended_tick{}, connection{}, sequence{};
};
struct Carried {
@@ -53,13 +60,16 @@ struct SessionServer::Impl {
std::uint64_t connection{}, sequence{};
};
Impl(SessionServerConfig c, DevGrantKey k, std::unique_ptr<TcpListener> l)
: config{std::move(c)}, key{std::move(k)}, listener{std::move(l)} {}
Impl(SessionServerConfig c, DevGrantKey k, std::unique_ptr<TcpListener> l,
std::unique_ptr<GrantLedger> g)
: config{std::move(c)}, key{std::move(k)}, listener{std::move(l)}, ledger{std::move(g)} {}
SessionServerConfig config;
DevGrantKey key;
std::unique_ptr<TcpListener> listener;
std::unique_ptr<GrantLedger> ledger;
std::optional<SessionAdmission> admission;
bool ledger_rejected{}; // Recorded grants exceed admission capacity: fail closed.
std::vector<Connection> connections;
std::deque<Pending> pending;
std::vector<Carried> carried;
@@ -67,8 +77,8 @@ struct SessionServer::Impl {
std::optional<client::EnvironmentPresentation> environment;
std::optional<client::OrganismPresentation> organisms;
RegionFacts published{};
std::uint64_t baseline{}, next_connection{1};
Clock::time_point polled_at{}; // Time of the current poll, for Close deadlines.
std::uint64_t baseline{}, next_connection{1}, next_carrier{1};
Clock::time_point polled_at{};
Connection* find(std::uint64_t id) {
const auto it = std::ranges::find_if(connections, [&](const Connection& c) {
@@ -76,16 +86,23 @@ struct SessionServer::Impl {
});
return it == connections.end() ? nullptr : &*it;
}
// Every outbound frame goes through the channel's bounded queue; a full queue means
// a consumer that cannot keep up, which is dropped rather than buffered further.
void send(Connection& c, const std::vector<std::uint8_t>& frame) {
if (c.dead || !c.tls->queue(frame))
c.dead = true;
c.last_sent = polled_at;
}
void close(Connection& c, CloseReason reason) {
if (c.state == State::closing || c.dead)
return;
if (c.state == State::hello || c.state == State::established)
append(c.out, encode(reason));
else
c.dead = true; // No channel yet to say anything over.
events.push_back({c.state == State::established ? SessionEvent::Kind::closed
: SessionEvent::Kind::refused,
c.session, c.subject, reason});
if (c.state == State::hello || c.state == State::established)
send(c, encode(reason));
else
c.dead = true; // No channel yet to say anything over.
c.state = State::closing;
c.deadline = polled_at + std::chrono::seconds{1}; // Bounded flush of the Close frame.
c.view.clear();
@@ -96,8 +113,11 @@ struct SessionServer::Impl {
return bytes;
const ViewHeader header{c.session, c.epoch, published.simulated_tick,
published.committed_tick, baseline};
append(bytes, encode(header, *environment));
append(bytes, encode(header, *organisms));
const auto a = encode(header, *environment);
const auto b = encode(header, *organisms);
bytes.reserve(a.size() + b.size());
bytes.insert(bytes.end(), a.begin(), a.end());
bytes.insert(bytes.end(), b.begin(), b.end());
return bytes;
}
void hello(Connection& c, const OwnedFrame& f, const RegionFacts& facts,
@@ -115,24 +135,34 @@ struct SessionServer::Impl {
close(c, CloseReason::server_shutdown); // No current authority: retry later.
return;
}
if (facts.epoch > max_carrier_epoch) {
close(c, CloseReason::capacity); // Carrier ids cannot name this epoch: refuse.
return;
}
const auto grant = key.verify(message->grant);
if (!grant) {
close(c, CloseReason::unauthenticated);
return;
}
c.subject = grant->claims().subject;
const auto welcome =
admission->open(*grant, now_seconds, facts.simulated_tick, facts.committed_tick);
if (!welcome) {
c.subject = grant->claims().subject;
close(c, close_reason(welcome.error()));
return;
}
// Durable before Welcome: a replacement process must refuse this grant too.
if (!ledger->record({grant->claims().grant_id, grant->claims().expires_at})) {
// The grant stays consumed in memory (fail closed); only the session is undone.
static_cast<void>(admission->close(welcome->session).has_value());
close(c, CloseReason::server_shutdown);
return;
}
c.session = welcome->session;
c.subject = grant->claims().subject;
c.epoch = welcome->epoch;
c.expires_at = welcome->grant_expires_at;
c.state = State::established;
append(c.out, encode(*welcome));
send(c, encode(*welcome));
c.view = views_for(c); // Fresh baseline for every new session.
events.push_back({SessionEvent::Kind::opened, c.session, c.subject, {}});
}
@@ -152,16 +182,8 @@ struct SessionServer::Impl {
result = admission->admit(*m, facts.simulated_tick, now_seconds);
}
if (result.status == IntentStatus::admitted)
pending.push_back({request_id(c, m->sequence), m->tick, c.id, m->sequence});
append(c.out, encode(result));
}
RequestId request_id(const Connection& c, std::uint64_t sequence) const {
RequestId id{'I', 'N', 'T', '1'};
const std::array<std::uint64_t, 3> parts{c.epoch, c.session, sequence};
for (std::size_t p = 0; p < parts.size(); ++p)
for (unsigned i = 0; i < 4; ++i)
id.at(4 + 4 * p + i) = static_cast<std::uint8_t>(parts.at(p) >> (8U * (3U - i)));
return id;
pending.push_back({m->tick, c.id, m->sequence});
send(c, encode(result));
}
void receive(Connection& c, const RegionFacts& facts, std::uint64_t now_seconds,
Clock::time_point now) {
@@ -194,12 +216,17 @@ struct SessionServer::Impl {
}
rest = rest.subspan(*taken);
while (auto f = c.reader.take()) {
c.last_received = now;
if (c.state == State::hello)
hello(c, *f, facts, now_seconds);
else if (f->type == MessageType::intent)
intent(c, *f, facts, now_seconds);
else if (f->type == MessageType::close) {
else if (f->type == MessageType::heartbeat) {
if (!decode_heartbeat({f->type, f->payload}))
close(c, CloseReason::malformed);
} else if (f->type == MessageType::close) {
c.state = State::closing; // Peer is leaving; nothing to say.
c.deadline = now + std::chrono::seconds{1};
events.push_back({SessionEvent::Kind::closed, c.session, c.subject,
CloseReason::normal});
} else
@@ -211,27 +238,25 @@ struct SessionServer::Impl {
}
}
void transmit(Connection& c) {
if (c.out.empty() && !c.view.empty() && c.state == State::established)
c.out.swap(c.view); // Only the newest view waits; it is never queued twice.
std::size_t sent = 0;
while (sent < c.out.size()) {
const auto n = c.tls->send(std::span{c.out}.subspan(sent));
if (!n) {
c.dead = true;
return;
}
if (*n == 0)
break;
sent += *n;
if (c.state == State::established && c.tls->queued() == 0 && !c.view.empty()) {
send(c, c.view);
c.view.clear();
}
c.out.erase(c.out.begin(), c.out.begin() + static_cast<std::ptrdiff_t>(sent));
if (c.out.size() + c.view.size() > config.max_output_bytes)
if (c.dead)
return;
if (c.tls->queued() + c.view.size() > config.max_output_bytes) {
c.dead = true; // Slow consumer: drop rather than buffer without bound.
if (c.state == State::closing && c.out.empty()) {
// Best-effort close_notify: the connection is dropped either way.
static_cast<void>(c.tls->close().has_value());
c.dead = true;
return;
}
if (c.state == State::closing) {
// close() flushes queued frames (the Close frame included) before close_notify.
const auto closed = c.tls->close();
if (!closed || *closed == TlsProgress::done)
c.dead = true;
return;
}
if (!c.tls->flush())
c.dead = true;
}
void step(Connection& c, const RegionFacts& facts, std::uint64_t now_seconds,
Clock::time_point now) {
@@ -250,10 +275,16 @@ struct SessionServer::Impl {
}
if ((c.state == State::handshake || c.state == State::hello) && now > c.deadline)
close(c, CloseReason::unauthenticated);
if (c.state == State::established && now_seconds >= c.expires_at)
close(c, CloseReason::expired);
if (c.state == State::established && admission && c.epoch != admission->epoch())
close(c, CloseReason::stale_epoch);
if (c.state == State::established) {
if (now_seconds >= c.expires_at)
close(c, CloseReason::expired);
else if (admission && c.epoch != admission->epoch())
close(c, CloseReason::stale_epoch);
else if (now - c.last_received > config.liveness_timeout)
close(c, CloseReason::timeout); // Silent peer: free the subject for reconnect.
else if (now - c.last_sent >= config.heartbeat_interval)
send(c, encode_heartbeat()); // Liveness without requiring simulation ticks.
}
if (c.state == State::hello || c.state == State::established)
receive(c, facts, now_seconds, now);
if (!c.dead && c.state != State::handshake)
@@ -270,17 +301,44 @@ struct SessionServer::Impl {
}
std::erase_if(connections, [](const Connection& c) { return c.dead; });
}
void open_admission(const RegionFacts& facts, std::uint64_t now_seconds) {
auto config_with_epoch = config.admission;
config_with_epoch.epoch = facts.epoch;
auto created = SessionAdmission::create(config_with_epoch);
if (!created)
return;
// Grants consumed before this process started stay consumed.
if (!created->remember(ledger->loaded(), now_seconds)) {
ledger_rejected = true; // More recorded grants than admission can hold.
return;
}
admission.emplace(std::move(*created));
}
void finish(const RequestId& request, const IntentOutcome& outcome) {
const auto it =
std::ranges::find_if(carried, [&](const Carried& c) { return c.request == request; });
if (it == carried.end())
return;
auto out = outcome;
out.sequence = it->sequence; // The client's own sequence, carried separately.
if (auto* c = find(it->connection))
send(*c, encode(out));
carried.erase(it);
}
};
SessionServer::SessionServer(std::unique_ptr<Impl> impl) : impl_{std::move(impl)} {}
SessionServer::~SessionServer() = default;
std::expected<std::unique_ptr<SessionServer>, SessionServerError>
SessionServer::create(SessionServerConfig config, DevGrantKey key) {
SessionServer::create(SessionServerConfig config, DevGrantKey key, std::uint64_t now_seconds) {
if (config.max_connections == 0 || config.max_connections > 64 ||
config.max_pending_intents == 0 || config.max_carried_intents == 0 ||
config.max_output_bytes < 2 * (frame_header_bytes + max_frame_payload) ||
config.max_input_bytes_per_second < frame_header_bytes + max_grant_bytes + 6)
config.max_output_bytes > TlsChannel::max_queued_bytes ||
config.max_input_bytes_per_second < frame_header_bytes + max_grant_bytes + 6 ||
config.heartbeat_interval.count() <= 0 ||
config.liveness_timeout <= 2 * config.heartbeat_interval || config.grant_ledger.empty())
return std::unexpected(SessionServerError::configuration);
// Validate the admission limits now, with a placeholder epoch.
auto probe = config.admission;
@@ -298,11 +356,14 @@ SessionServer::create(SessionServerConfig config, DevGrantKey key) {
} null;
if (!TlsChannel::server(config.credentials, null))
return std::unexpected(SessionServerError::credentials);
auto ledger = GrantLedger::open(config.grant_ledger, now_seconds);
if (!ledger)
return std::unexpected(SessionServerError::ledger);
auto listener = TcpListener::listen(config.listen);
if (!listener)
return std::unexpected(SessionServerError::listen);
return std::unique_ptr<SessionServer>(new SessionServer(
std::make_unique<Impl>(std::move(config), std::move(key), std::move(*listener))));
return std::unique_ptr<SessionServer>(new SessionServer(std::make_unique<Impl>(
std::move(config), std::move(key), std::move(*listener), std::move(*ledger))));
}
std::uint16_t SessionServer::port() const noexcept {
@@ -313,12 +374,9 @@ void SessionServer::poll(const RegionFacts& facts, std::uint64_t now_seconds,
std::chrono::steady_clock::time_point now) {
auto& m = *impl_;
m.polled_at = now;
if (facts.ready && facts.epoch != 0) {
if (facts.ready && facts.epoch != 0 && !m.ledger_rejected) {
if (!m.admission) {
auto config = m.config.admission;
config.epoch = facts.epoch;
if (auto created = SessionAdmission::create(config))
m.admission.emplace(std::move(*created));
m.open_admission(facts, now_seconds);
} else if (facts.epoch > m.admission->epoch()) {
// Strictly newer (checked above), so the advance cannot be refused.
static_cast<void>(m.admission->advance_epoch(facts.epoch).has_value());
@@ -340,7 +398,7 @@ void SessionServer::poll(const RegionFacts& facts, std::uint64_t now_seconds,
continue;
c.tls = std::move(*tls);
c.deadline = now + m.config.hello_deadline;
c.window_start = now;
c.window_start = c.last_received = c.last_sent = now;
m.connections.push_back(std::move(c));
}
for (auto& c : m.connections)
@@ -364,13 +422,15 @@ void SessionServer::publish(const client::EnvironmentPresentation& environment,
std::optional<RequestId> SessionServer::carry(std::uint64_t tick) {
auto& m = *impl_;
if (m.carried.size() >= m.config.max_carried_intents)
return std::nullopt;
if (m.carried.size() >= m.config.max_carried_intents || !m.admission ||
m.next_carrier == std::numeric_limits<std::uint64_t>::max())
return std::nullopt; // Ordinal exhaustion is explicit, never a wrap.
const auto it = std::ranges::find_if(
m.pending, [&](const Impl::Pending& p) { return p.intended_tick <= tick; });
if (it == m.pending.end())
return std::nullopt;
const Impl::Carried carried{it->request, it->connection, it->sequence};
const Impl::Carried carried{carrier_id(m.admission->epoch(), m.next_carrier++), it->connection,
it->sequence};
m.pending.erase(it);
m.carried.push_back(carried);
return carried.request;
@@ -378,26 +438,11 @@ std::optional<RequestId> SessionServer::carry(std::uint64_t tick) {
void SessionServer::committed(const RequestId& request, std::uint64_t tick,
std::uint64_t commit_sequence) {
auto& m = *impl_;
const auto it = std::ranges::find_if(
m.carried, [&](const Impl::Carried& c) { return c.request == request; });
if (it == m.carried.end())
return;
if (auto* c = m.find(it->connection))
append(c->out, encode(IntentOutcome{it->sequence, IntentFinality::committed_no_effect, tick,
commit_sequence}));
m.carried.erase(it);
impl_->finish(request, {0, IntentFinality::committed_no_effect, tick, commit_sequence});
}
void SessionServer::not_committed(const RequestId& request) {
auto& m = *impl_;
const auto it = std::ranges::find_if(
m.carried, [&](const Impl::Carried& c) { return c.request == request; });
if (it == m.carried.end())
return;
if (auto* c = m.find(it->connection))
append(c->out, encode(IntentOutcome{it->sequence, IntentFinality::not_committed, 0, 0}));
m.carried.erase(it);
impl_->finish(request, {0, IntentFinality::not_committed, 0, 0});
}
std::vector<SessionEvent> SessionServer::take_events() {
+39 -7
View File
@@ -5,6 +5,7 @@
#include <mbedtls/x509_crt.h>
#include <optional>
#include <psa/crypto.h>
#include <vector>
// The owned user config (third_party/mbedtls/evo_mbedtls_config.h) must be in effect.
#if defined(MBEDTLS_SSL_PROTO_TLS1_2) || defined(MBEDTLS_SSL_PROTO_DTLS) || \
@@ -21,6 +22,8 @@ namespace {
// ADR-0082 profile: TLS 1.3 only, AEAD suites, ECDHE groups, ephemeral key exchange.
constexpr int suites[]{MBEDTLS_TLS1_3_AES_128_GCM_SHA256, MBEDTLS_TLS1_3_CHACHA20_POLY1305_SHA256,
0};
// One TLS record's plaintext per write call (the TLS 1.3 maximum fragment).
constexpr std::size_t record_plaintext = 16384;
constexpr std::uint16_t groups[]{MBEDTLS_SSL_IANA_TLS_GROUP_X25519,
MBEDTLS_SSL_IANA_TLS_GROUP_SECP256R1, 0};
const unsigned char* bytes_of(const std::string& text) {
@@ -105,6 +108,8 @@ struct TlsChannel::Impl {
mbedtls_pk_context key;
bool transport_failed{};
std::optional<TlsError> error;
std::vector<std::uint8_t> outbound; // Queued plaintext; the front record may be in flight.
std::size_t in_flight{}; // Length of an interrupted write; 0 when none.
};
TlsChannel::TlsChannel(std::unique_ptr<Impl> impl) : impl_{std::move(impl)} {}
@@ -158,15 +163,39 @@ std::expected<TlsProgress, TlsError> TlsChannel::handshake() {
return impl_->fail_from(result, TlsError::handshake);
}
std::expected<std::size_t, TlsError> TlsChannel::send(std::span<const std::uint8_t> bytes) {
std::expected<void, TlsError> TlsChannel::queue(std::span<const std::uint8_t> bytes) {
if (const auto latched = impl_->error)
return std::unexpected(*latched);
const int result = mbedtls_ssl_write(&impl_->ssl, bytes.data(), bytes.size());
if (result >= 0)
return static_cast<std::size_t>(result);
if (result == MBEDTLS_ERR_SSL_WANT_READ || result == MBEDTLS_ERR_SSL_WANT_WRITE)
return 0;
return impl_->fail_from(result, TlsError::record);
if (bytes.size() > max_queued_bytes - impl_->outbound.size())
return std::unexpected(TlsError::backpressure); // Nothing queued: not latched.
impl_->outbound.insert(impl_->outbound.end(), bytes.begin(), bytes.end());
return {};
}
std::expected<TlsProgress, TlsError> TlsChannel::flush() {
auto& m = *impl_;
if (const auto latched = m.error)
return std::unexpected(*latched);
while (!m.outbound.empty()) {
// A record interrupted by WANT_READ/WANT_WRITE keeps its length; bytes queued
// since then sit behind it and do not change the retried call.
if (m.in_flight == 0)
m.in_flight = std::min(m.outbound.size(), record_plaintext);
const int result = mbedtls_ssl_write(&m.ssl, m.outbound.data(), m.in_flight);
if (result == MBEDTLS_ERR_SSL_WANT_READ || result == MBEDTLS_ERR_SSL_WANT_WRITE)
return TlsProgress::want_io;
if (result < 0)
return m.fail_from(result, TlsError::record);
const auto written = std::min(static_cast<std::size_t>(result), m.in_flight);
m.outbound.erase(m.outbound.begin(),
m.outbound.begin() + static_cast<std::ptrdiff_t>(written));
m.in_flight = 0;
}
return TlsProgress::done;
}
std::size_t TlsChannel::queued() const noexcept {
return impl_->outbound.size();
}
std::expected<std::size_t, TlsError> TlsChannel::receive(std::span<std::uint8_t> bytes) {
@@ -185,6 +214,9 @@ std::expected<std::size_t, TlsError> TlsChannel::receive(std::span<std::uint8_t>
std::expected<TlsProgress, TlsError> TlsChannel::close() {
if (const auto latched = impl_->error)
return std::unexpected(*latched);
// Queued plaintext (including an interrupted record) is sent before close_notify.
if (const auto pending = flush(); !pending || *pending == TlsProgress::want_io)
return pending;
const int result = mbedtls_ssl_close_notify(&impl_->ssl);
if (result == 0)
return TlsProgress::done;
+4
View File
@@ -50,6 +50,10 @@ extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, std::size_t size
if (const auto m = decode_intent_outcome(*frame))
again = encode(*m);
break;
case MessageType::heartbeat:
if (decode_heartbeat(*frame))
again = encode_heartbeat();
break;
case MessageType::environment_view:
if (const auto m = decode_environment_view(*frame))
again = encode(m->first, m->second);
+13 -2
View File
@@ -126,6 +126,12 @@ void round_trips() {
const auto c = decode_close(frame(encode(CloseReason::stale_epoch)));
CHECK(c && *c == CloseReason::stale_epoch);
CHECK(decode_close(frame(encode(CloseReason::capacity))) == CloseReason::capacity);
CHECK(decode_close(frame(encode(CloseReason::timeout))) == CloseReason::timeout);
const auto beat = encode_heartbeat();
CHECK(beat.size() == frame_header_bytes && decode_heartbeat(frame(beat)));
CHECK(refused(decode_heartbeat(frame(with_header(MessageType::heartbeat, 1, 0, 1, 1))),
ProtocolError::trailing_bytes));
CHECK(refused(decode_heartbeat(frame(encode(CloseReason::normal))), ProtocolError::wrong_type));
const auto out = decode_intent_outcome(
frame(encode(IntentOutcome{4, IntentFinality::committed_no_effect, 120, 9})));
CHECK(out && out->sequence == 4 && out->finality == IntentFinality::committed_no_effect &&
@@ -177,7 +183,7 @@ void frame_rejection() {
CHECK(refused(decode_frame(with_header(MessageType{0}, 1, 0, 0, 0)),
ProtocolError::unknown_type));
// NOLINTNEXTLINE(clang-analyzer-optin.core.EnumCastOutOfRange)
CHECK(refused(decode_frame(with_header(MessageType{9}, 1, 0, 0, 0)),
CHECK(refused(decode_frame(with_header(MessageType{10}, 1, 0, 0, 0)),
ProtocolError::unknown_type));
CHECK(refused(decode_frame(with_header(MessageType::intent, 1, 1, 36, 36)),
ProtocolError::reserved_flags));
@@ -207,7 +213,7 @@ void frame_rejection() {
status[frame_header_bytes + 8] = 8;
CHECK(refused(decode_intent_result(frame(status)), ProtocolError::invalid_value));
auto reason = encode(CloseReason::normal);
reason[frame_header_bytes] = 10;
reason[frame_header_bytes] = 11;
CHECK(refused(decode_close(frame(reason)), ProtocolError::invalid_value));
CHECK(refused(decode_welcome(frame(encode(Welcome{}))), ProtocolError::invalid_value));
// A well-formed encoding of an invalid view is rejected by the presentation rules.
@@ -261,6 +267,7 @@ void fuzz() {
encode(IntentResult{1, IntentStatus::outside_window, 2}),
encode(CloseReason::rate_limited),
encode(IntentOutcome{3, IntentFinality::committed_no_effect, 4, 5}),
encode_heartbeat(),
encode(view_header, environment()),
encode(view_header, organisms(3))};
std::size_t accepted = 0;
@@ -313,6 +320,10 @@ void fuzz() {
if (const auto m = decode_intent_outcome(*f))
again = encode(*m);
break;
case MessageType::heartbeat:
if (decode_heartbeat(*f))
again = encode_heartbeat();
break;
case MessageType::environment_view:
if (const auto m = decode_environment_view(*f))
again = encode(m->first, m->second);
+226 -11
View File
@@ -1,11 +1,16 @@
#include "evo/network/grant_ledger.hpp"
#include "evo/network/session_client.hpp"
#include "evo/network/session_server.hpp"
#include "evo/network/tcp.hpp"
#include "network_pki.hpp"
#include "network_test.hpp"
#include <algorithm>
#include <atomic>
#include <chrono>
#include <filesystem>
#include <fstream>
#include <functional>
#include <limits>
#include <string>
#include <thread>
#include <vector>
@@ -57,12 +62,28 @@ client::OrganismPresentation no_organisms(std::uint64_t tick) {
return std::move(*v);
}
std::filesystem::path fresh_ledger() {
static std::atomic<unsigned> counter{0};
const auto stamp = Clock::now().time_since_epoch().count();
return std::filesystem::temp_directory_path() /
("evo-grant-ledger-" + std::to_string(stamp) + "-" + std::to_string(++counter));
}
struct Harness {
explicit Harness(std::size_t max_sessions = 4,
std::chrono::milliseconds hello = std::chrono::milliseconds{3000}) {
std::chrono::milliseconds hello = std::chrono::milliseconds{3000},
std::function<void(SessionServerConfig&)> server_tweak = {},
std::function<void(SessionClientConfig&)> client_tweak = {})
: tweak_server{std::move(server_tweak)}, tweak_client{std::move(client_tweak)} {
server = make_server(0, max_sessions, hello);
port = server->port();
}
~Harness() {
server.reset();
std::error_code ignored;
std::filesystem::remove(ledger, ignored);
}
Harness(const Harness&) = delete;
Harness& operator=(const Harness&) = delete;
std::unique_ptr<SessionServer> make_server(std::uint16_t at, std::size_t max_sessions,
std::chrono::milliseconds hello) {
SessionServerConfig c;
@@ -75,7 +96,10 @@ struct Harness {
c.admission.window_ticks = 8;
c.admission.max_intents_per_window = 8;
c.hello_deadline = hello;
auto s = SessionServer::create(std::move(c), grant_key());
c.grant_ledger = ledger; // Shared by restarts of this harness's server.
if (tweak_server)
tweak_server(c);
auto s = SessionServer::create(std::move(c), grant_key(), now_seconds);
CHECK(s);
return std::move(*s);
}
@@ -88,6 +112,8 @@ struct Harness {
c.first_retry = std::chrono::milliseconds{50};
c.max_retry = std::chrono::milliseconds{200};
c.attempt_deadline = std::chrono::milliseconds{3000};
if (tweak_client)
tweak_client(c);
auto s = SessionClient::create(std::move(c));
CHECK(s);
clients.push_back(s->get());
@@ -119,6 +145,9 @@ struct Harness {
auto e = c.take_events();
into.insert(into.end(), e.begin(), e.end());
}
std::function<void(SessionServerConfig&)> tweak_server;
std::function<void(SessionClientConfig&)> tweak_client;
std::filesystem::path ledger = fresh_ledger();
Pki pki;
RegionFacts facts{true, 1, 100, 90, 5};
std::unique_ptr<SessionServer> server;
@@ -167,7 +196,9 @@ void connect_view_intents_and_outcomes() {
CHECK(last(events, ClientEvent::Kind::result)->result.status == IntentStatus::admitted);
CHECK(!h.server->carry(105)); // Intended tick 102 + 4 = 106.
const auto request = h.server->carry(106);
CHECK(must(request)[0] == 'I' && must(request)[3] == '1');
// 'I','N' + 48-bit epoch (1) + 64-bit server ordinal (first carrier: 1).
CHECK(must(request)[0] == 'I' && must(request)[1] == 'N' && must(request)[7] == 1 &&
must(request)[15] == 1);
CHECK(!h.server->carry(200)); // Carried once.
h.server->committed(must(request), 106, 6);
CHECK(h.until([&] {
@@ -336,24 +367,42 @@ void abuse() {
h.tick();
CHECK(h.until([&] { return c->phase() == SessionPhase::established; }));
CHECK(c->grants_left() == 2);
// Flooding: the byte-rate limit closes the connection. The flooding client may see
// a transport reset before it reads the Close frame, so check the server's decision.
// The client never buffers without bound: with the server not answering, at most
// max_outstanding intentions may be unresolved; further sends are refused.
CHECK(h.until([&] { return c->latest_view().has_value(); }));
(void)h.server->take_events();
std::size_t accepted = 0;
for (std::uint64_t s = 10; s < 2000; ++s)
CHECK(c->send({s, 0, 0, 0}));
accepted += c->send({s, 0, 0, 0}) ? 1 : 0;
CHECK(accepted == SessionClient::max_outstanding);
// Flooding: a raw TLS peer (no client-side bounds) exceeds the byte-rate limit after
// a valid Hello; the server closes it. Check the server's decision, since the flooding
// peer may see a reset before it could read the Close frame.
(void)h.server->take_events();
auto raw = TcpStream::connect("127.0.0.1", h.port);
CHECK(raw);
auto tls = TlsChannel::client({h.pki.authority, server_name}, **raw);
CHECK(tls);
const auto flood_grant = grants(9, 1, 900).front();
const auto hello = encode(Hello{0, {flood_grant.begin(), flood_grant.end()}});
CHECK(hello);
bool flooded = false;
std::vector<SessionEvent> server_events;
CHECK(h.until([&] {
auto e = h.server->take_events();
server_events.insert(server_events.end(), e.begin(), e.end());
if (const auto p = (*tls)->handshake(); p && *p == TlsProgress::done && !flooded) {
flooded = hello && (*tls)->queue(*hello).has_value();
const auto beat = encode_heartbeat();
for (int i = 0; i < 7000 && flooded; ++i)
flooded = (*tls)->queue(beat).has_value();
}
if (flooded)
static_cast<void>((*tls)->flush().has_value()); // Best effort: may be reset.
return std::ranges::any_of(server_events, [](const SessionEvent& x) {
return x.kind == SessionEvent::Kind::closed && x.reason == CloseReason::rate_limited;
});
}));
CHECK(h.until([&] {
h.collect(*c, events);
return c->phase() != SessionPhase::established;
}));
// Plaintext and silent connections are dropped without a session: the server
// closes the socket (plaintext at the TLS handshake, silence at the deadline).
@@ -381,6 +430,169 @@ void abuse() {
CHECK(quiet.until([&] { return quiet.server->connections() == 0; }));
CHECK(quiet.server->sessions() == 0);
}
// Review M9.4-M9.7 #2: client sequences 2^32 apart must get distinct carriers and
// each final outcome must reach its own sequence.
void carriers_never_alias() {
Harness h;
h.tick();
auto c = h.client(grants(7, 2, 100));
std::vector<ClientEvent> events;
CHECK(h.until([&] { return c->latest_view().has_value(); }));
constexpr std::uint64_t far = (std::uint64_t{1} << 32) + 1;
CHECK(c->send({1, 0, 1000, 0}) && c->send({far, 0, 0, 1000}));
CHECK(h.until([&] {
h.collect(*c, events);
return std::ranges::count_if(events, [](const ClientEvent& e) {
return e.kind == ClientEvent::Kind::result &&
e.result.status == IntentStatus::admitted;
}) == 2;
}));
const auto first = h.server->carry(1000);
const auto second = h.server->carry(1000);
CHECK(must(first) != must(second));
CHECK(must(first)[0] == 'I' && must(first)[1] == 'N' && must(first)[7] == 1); // Epoch 1.
h.server->not_committed(must(second));
h.server->committed(must(first), 106, 6);
CHECK(h.until([&] {
h.collect(*c, events);
return std::ranges::count_if(events, [](const ClientEvent& e) {
return e.kind == ClientEvent::Kind::outcome;
}) == 2;
}));
for (const auto& e : events)
if (e.kind == ClientEvent::Kind::outcome)
CHECK((e.sequence == far && e.outcome.finality == IntentFinality::not_committed) ||
(e.sequence == 1 && e.outcome.finality == IntentFinality::committed_no_effect &&
e.outcome.commit_sequence == 6));
}
// Review M9.4-M9.7 #3: a grant consumed before a restart stays consumed after it.
void grants_stay_consumed_across_restarts() {
Harness h;
h.tick();
auto first = h.client(grants(7, 1, 100));
CHECK(h.until([&] { return first->phase() == SessionPhase::established; }));
h.clients.clear();
first.reset();
h.server.reset(); // Process restart: only the ledger file survives.
h.facts = {true, 2, 40, 40, 2};
h.server = h.make_server(h.port, 4, std::chrono::milliseconds{3000});
h.tick();
auto replay = h.client(grants(7, 1, 100)); // The exact consumed grant.
std::vector<ClientEvent> events;
CHECK(h.until([&] {
h.collect(*replay, events);
return has(events, ClientEvent::Kind::closed);
}));
CHECK(last(events, ClientEvent::Kind::closed)->reason == CloseReason::unauthenticated);
auto fresh = h.client(grants(7, 1, 101));
CHECK(h.until([&] { return fresh->phase() == SessionPhase::established; }));
CHECK(must(fresh->welcome()).epoch == 2);
// The ledger fails closed on corruption and drops expired entries when opened.
const auto path = fresh_ledger();
{
std::ofstream out{path, std::ios::binary};
out << "not a ledger line\n";
}
SessionServerConfig bad;
bad.listen = "127.0.0.1:0";
bad.credentials = h.pki.server;
bad.admission.world = identity;
bad.admission.region = region;
bad.grant_ledger = path;
CHECK(
refused(SessionServer::create(bad, grant_key(), now_seconds), SessionServerError::ledger));
{
std::ofstream out{path, std::ios::binary | std::ios::trunc}; // LF only.
out << "0000000000000005 " << now_seconds - 1 << "\n"
<< "0000000000000006 " << now_seconds + 10 << "\n";
}
{
const auto ledger = GrantLedger::open(path, now_seconds);
CHECK(ledger && (*ledger)->loaded().size() == 1 && (*ledger)->loaded()[0].grant_id == 6);
CHECK(ledger && (*ledger)->record({7, now_seconds + 20}));
}
const auto reopened = GrantLedger::open(path, now_seconds);
CHECK(reopened && (*reopened)->loaded().size() == 2);
std::error_code ignored;
std::filesystem::remove(path, ignored);
}
// Review M9.4-M9.7 #4: liveness after Welcome, both directions, without spurious drops.
void liveness() {
const auto fast_server = [](SessionServerConfig& c) {
c.heartbeat_interval = std::chrono::milliseconds{100};
c.liveness_timeout = std::chrono::milliseconds{500};
};
const auto fast_client = [](SessionClientConfig& c) {
c.heartbeat_interval = std::chrono::milliseconds{100};
c.liveness_timeout = std::chrono::milliseconds{500};
};
{
// Healthy but idle: no ticks are published; heartbeats keep both sides alive.
Harness h{4, std::chrono::milliseconds{3000}, fast_server, fast_client};
h.tick();
auto c = h.client(grants(7, 2, 100));
std::vector<ClientEvent> events;
CHECK(h.until([&] { return c->latest_view().has_value(); }));
const auto start = Clock::now();
CHECK(!h.until(
[&] {
h.collect(*c, events);
return has(events, ClientEvent::Kind::lost) ||
has(events, ClientEvent::Kind::closed);
},
std::chrono::milliseconds{1500}));
CHECK(Clock::now() - start >= std::chrono::milliseconds{1500});
CHECK(c->phase() == SessionPhase::established &&
c->environment().freshness() == client::Freshness::current);
}
{
// Silent server (socket open, never polled): the client times out, keeps the view
// as stale, reports the sent-but-unresolved intention as unknown, and reconnects.
Harness h{4, std::chrono::milliseconds{3000}, fast_server, fast_client};
h.tick();
auto c = h.client(grants(7, 3, 100));
std::vector<ClientEvent> events;
CHECK(h.until([&] { return c->latest_view().has_value(); }));
CHECK(c->send({5, 0, 1000, 0})); // Never answered: the server is not polled.
const auto end = Clock::now() + std::chrono::milliseconds{2000};
while (Clock::now() < end && !has(events, ClientEvent::Kind::lost)) {
c->poll(Clock::now());
h.collect(*c, events);
std::this_thread::sleep_for(std::chrono::milliseconds{5});
}
CHECK(has(events, ClientEvent::Kind::lost));
const auto* unknown = last(events, ClientEvent::Kind::unknown);
CHECK(unknown != nullptr && unknown->sequence == 5);
CHECK(c->environment().freshness() == client::Freshness::stale);
h.tick();
CHECK(h.until([&] {
return c->phase() == SessionPhase::established && c->latest_view().has_value();
}));
CHECK(c->environment().freshness() == client::Freshness::current);
}
{
// Silent client: the server closes it with `timeout`, freeing the subject.
Harness h{4, std::chrono::milliseconds{3000}, fast_server, fast_client};
h.tick();
auto c = h.client(grants(7, 1, 100));
CHECK(h.until([&] { return c->phase() == SessionPhase::established; }));
h.clients.clear(); // Stop polling the client; its socket stays open.
std::vector<SessionEvent> server_events;
CHECK(h.until([&] {
auto e = h.server->take_events();
server_events.insert(server_events.end(), e.begin(), e.end());
return h.server->sessions() == 0;
}));
CHECK(std::ranges::any_of(server_events, [](const SessionEvent& e) {
return e.kind == SessionEvent::Kind::closed && e.reason == CloseReason::timeout;
}));
auto again = h.client(grants(7, 1, 200)); // Same subject, fresh grant: admitted.
CHECK(h.until([&] { return again->phase() == SessionPhase::established; }));
}
}
} // namespace
void session_tests() {
@@ -388,5 +600,8 @@ void session_tests() {
duplicate_capacity_and_authentication();
stale_epoch_restart_and_reconnect();
abuse();
carriers_never_alias();
grants_stay_consumed_across_restarts();
liveness();
}
} // namespace evo::network_test
+98 -11
View File
@@ -2,7 +2,9 @@
#include "evo/network/tls_channel.hpp"
#include "network_pki.hpp"
#include "network_test.hpp"
#include <algorithm>
#include <array>
#include <cstdint>
#include <deque>
#include <mbedtls/x509_crt.h>
#include <psa/crypto.h>
@@ -17,6 +19,8 @@ using namespace evo::network;
struct Wire {
std::deque<std::uint8_t> to_server, to_client;
bool fail{};
// Bytes each side may still write; 0 makes writes "would block" (zero progress).
std::size_t server_budget{SIZE_MAX}, client_budget{SIZE_MAX};
};
class Endpoint final : public ByteTransport {
public:
@@ -25,7 +29,11 @@ class Endpoint final : public ByteTransport {
if (wire_.fail)
return std::unexpected(TransportError::failed);
auto& out = server_ ? wire_.to_client : wire_.to_server;
const auto count = std::min<std::size_t>(bytes.size(), 700); // Force partial writes.
auto& budget = server_ ? wire_.server_budget : wire_.client_budget;
// Force partial writes, and stop entirely when the injected budget runs out.
const auto count = std::min({bytes.size(), std::size_t{700}, budget});
if (budget != SIZE_MAX)
budget -= count;
out.insert(out.end(), bytes.begin(), bytes.begin() + static_cast<std::ptrdiff_t>(count));
return count;
}
@@ -76,15 +84,15 @@ struct Pair {
}
std::vector<std::uint8_t> transfer(TlsChannel& from, TlsChannel& to,
std::span<const std::uint8_t> bytes) {
std::size_t sent = 0;
CHECK(from.queue(bytes));
return drain(from, to, bytes.size());
}
// Flushes `from` and reads `to` until `expected` plaintext bytes arrive.
std::vector<std::uint8_t> drain(TlsChannel& from, TlsChannel& to, std::size_t expected) {
std::vector<std::uint8_t> received;
std::array<std::uint8_t, 4096> buffer{};
for (int i = 0; i < 1000 && received.size() < bytes.size(); ++i) {
if (sent < bytes.size()) {
const auto n = from.send(bytes.subspan(sent));
CHECK(n);
sent += *n;
}
for (int i = 0; i < 1000 && received.size() < expected; ++i) {
CHECK(from.flush());
const auto n = to.receive(buffer);
CHECK(n);
received.insert(received.end(), buffer.begin(),
@@ -122,7 +130,7 @@ void handshake_and_frames() {
for (int i = 0; i < 10 && r && *r == 0; ++i)
r = pair.server->receive(buffer);
CHECK(refused(r, TlsError::closed));
CHECK(refused(pair.server->send(buffer), TlsError::closed)); // Latched.
CHECK(refused(pair.server->queue(buffer), TlsError::closed)); // Latched.
}
void rejections() {
@@ -155,8 +163,8 @@ void rejections() {
Pair pair{pki.server, {pki.authority, server_name}};
CHECK(pair.handshake());
const std::array<std::uint8_t, 64> data{};
const auto sent = pair.client->send(data);
CHECK(sent && *sent == data.size());
CHECK(pair.client->queue(data) && pair.client->flush());
CHECK(pair.client->queued() == 0);
CHECK(pair.wire.to_server.size() > 20);
pair.wire.to_server[pair.wire.to_server.size() - 5] ^= 0x01;
std::array<std::uint8_t, 256> buffer{};
@@ -181,10 +189,89 @@ void rejections() {
CHECK(!r && r.error() == TlsError::handshake);
}
}
// Review M9.4-M9.7 #1: frames queued while a TLS record is stalled (zero progress,
// or partial progress then zero) must neither be lost nor reordered.
std::vector<std::uint8_t> intents(std::uint64_t first, std::uint64_t count) {
std::vector<std::uint8_t> bytes;
for (std::uint64_t i = 0; i < count; ++i) {
const auto f = encode(IntentMessage{1, 1, first + i, 10, 0, 0});
bytes.insert(bytes.end(), f.begin(), f.end());
}
return bytes;
}
std::vector<std::uint64_t> sequences(const std::vector<std::uint8_t>& bytes) {
FrameReader reader;
std::vector<std::uint64_t> out;
std::span<const std::uint8_t> rest{bytes};
while (!rest.empty()) {
const auto taken = reader.feed(rest);
CHECK(taken);
rest = rest.subspan(*taken);
while (auto f = reader.take()) {
const auto m = decode_intent({f->type, f->payload});
CHECK(m);
out.push_back(m ? m->sequence : 0);
}
}
return out;
}
void interrupted_writes() {
Pki pki;
for (const bool from_client : {true, false})
for (const std::size_t before_stall : {std::size_t{0}, std::size_t{10}}) {
Pair pair{pki.server, {pki.authority, server_name}};
CHECK(pair.handshake());
auto& from = from_client ? *pair.client : *pair.server;
auto& to = from_client ? *pair.server : *pair.client;
auto& budget = from_client ? pair.wire.client_budget : pair.wire.server_budget;
budget = before_stall; // Zero progress, or partial progress then zero.
const auto first = intents(1, 1);
CHECK(from.queue(first));
const auto stalled = from.flush();
CHECK(stalled && *stalled == TlsProgress::want_io);
const auto second = intents(2, 3); // Queued while the first record is stuck.
CHECK(from.queue(second) && from.queued() == first.size() + second.size());
budget = SIZE_MAX;
const auto received = pair.drain(from, to, first.size() + second.size());
CHECK(received.size() == first.size() + second.size() && from.queued() == 0);
CHECK(sequences(received) == std::vector<std::uint64_t>({1, 2, 3, 4}));
}
// Close while stalled: queued frames are flushed before close_notify.
Pair pair{pki.server, {pki.authority, server_name}};
CHECK(pair.handshake());
pair.wire.client_budget = 0;
const auto last = intents(9, 2);
CHECK(pair.client->queue(last));
const auto pending = pair.client->close();
CHECK(pending && *pending == TlsProgress::want_io);
pair.wire.client_budget = SIZE_MAX;
std::expected<TlsProgress, TlsError> closed = TlsProgress::want_io;
for (int i = 0; i < 50 && closed && *closed == TlsProgress::want_io; ++i)
closed = pair.client->close();
CHECK(closed && *closed == TlsProgress::done);
std::vector<std::uint8_t> received;
std::array<std::uint8_t, 256> buffer{};
std::expected<std::size_t, TlsError> r = 0;
for (int i = 0; i < 50 && r; ++i) {
r = pair.server->receive(buffer);
if (r)
received.insert(received.end(), buffer.begin(),
buffer.begin() + static_cast<std::ptrdiff_t>(*r));
}
CHECK(refused(r, TlsError::closed) &&
sequences(received) == std::vector<std::uint64_t>({9, 10}));
// A full queue refuses without queuing or latching.
Pair full{pki.server, {pki.authority, server_name}};
CHECK(full.handshake());
const std::vector<std::uint8_t> huge(TlsChannel::max_queued_bytes + 1, 0);
CHECK(refused(full.client->queue(huge), TlsError::backpressure) && full.client->queued() == 0);
CHECK(full.client->queue(intents(1, 1)) && full.client->queued() == intents(1, 1).size());
}
} // namespace
void tls_tests() {
handshake_and_frames();
rejections();
interrupted_writes();
}
} // namespace evo::network_test
+5 -3
View File
@@ -50,7 +50,8 @@ struct Credentials {
{"--synthetic-inputs", "development-v1", "--session-listen",
"127.0.0.1:" + std::to_string(port), "--session-cert",
(r.root / "server.pem").string(), "--session-key",
(r.root / "server.key").string(), "--session-grant-key",
(r.root / "server.key").string(), "--session-grant-ledger",
(r.root / "grants.ledger").string(), "--session-grant-key",
(r.root / "grant.key").string(), "--session-environment", environment});
}
std::vector<network::DevGrant> grants(const ServerRegion& r, std::uint64_t subject,
@@ -179,8 +180,9 @@ void connects_acknowledges_and_recovers(const Credentials& c) {
const auto* outcome = last(events, ClientEvent::Kind::outcome);
CHECK(outcome->outcome.finality == network::IntentFinality::committed_no_effect);
first_commit = outcome->outcome.commit_sequence;
// The host acknowledged exactly that request ("INT1...") at that tick and commit.
const std::string prefix = "ack " + hex({'I', 'N', 'T', '1'});
// The host acknowledged exactly that carrier ('I','N' + 48-bit epoch 1 + ordinal)
// at that tick and commit.
const std::string prefix = "ack " + hex({'I', 'N', 0, 0}) + "00000001";
const auto ack = std::ranges::find_if(
server.lines(), [&](const std::string& l) { return starts(l, prefix); });
CHECK(ack != server.lines().end());