diff --git a/CMakeLists.txt b/CMakeLists.txt index 1afab90..ac1278b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -139,7 +139,25 @@ if(EVO_BUILD_QUIC) message(FATAL_ERROR "EVO_BUILD_QUIC requires EVO_BUILD_NETWORK") endif() include(cmake/Quic.cmake) + # M10.2b transport: bounded channels and exact CA pinning over msquic. + add_library(evo_network_quic STATIC src/network/transport/quic_endpoint.cpp) + target_include_directories(evo_network_quic PUBLIC "${PROJECT_SOURCE_DIR}/include") + target_link_libraries(evo_network_quic PUBLIC evo_msquic PRIVATE mbedtls) + evo_owned_target(evo_network_quic) if(BUILD_TESTING) + # Owned impairment relay (tests and measurements only). + add_library(evo_network_impair STATIC tools/network/impair/udp_relay.cpp) + target_include_directories(evo_network_impair PUBLIC "${PROJECT_SOURCE_DIR}/tools/network/impair") + if(WIN32) + target_link_libraries(evo_network_impair PUBLIC ws2_32) + endif() + evo_owned_target(evo_network_impair) + add_executable(evo-quic-transport-tests tests/quic_transport.cpp) + target_link_libraries(evo-quic-transport-tests PRIVATE evo_network_quic evo_network_impair evo_network_tls) + evo_owned_target(evo-quic-transport-tests) + evo_msquic_runtime(evo-quic-transport-tests) + add_test(NAME quic-transport COMMAND evo-quic-transport-tests) + set_tests_properties(quic-transport PROPERTIES TIMEOUT 180 LABELS "quic") add_executable(evo-quic-smoke-tests tests/quic_smoke.cpp) target_link_libraries(evo-quic-smoke-tests PRIVATE evo_msquic evo_network_tls) evo_owned_target(evo-quic-smoke-tests) diff --git a/CMakePresets.json b/CMakePresets.json index 7e4fba6..bf24dea 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -24,7 +24,7 @@ {"name": "client", "configurePreset": "client", "targets": ["evo-client-state-tests", "evo-client", "evo-client-lifecycle-tests", "evo-client-environment-tests", "evo-client-organism-tests", "evo-client-ui-model-tests", "evo-client-panel-tests", "evo-client-bindings-tests"]}, {"name": "client-probe", "configurePreset": "client-probe", "targets": ["evo-client-probe"]}, {"name": "network", "configurePreset": "network", "targets": ["evo-client-state-tests", "evo-network-tests", "evo-dev-session"]}, - {"name": "quic", "configurePreset": "quic", "targets": ["evo-quic-smoke-tests", "evo-network-tests"]}, + {"name": "quic", "configurePreset": "quic", "targets": ["evo-quic-smoke-tests", "evo-quic-transport-tests", "evo-network-tests"]}, {"name": "client-network", "configurePreset": "client-network", "targets": ["evo-client-state-tests", "evo-client", "evo-client-lifecycle-tests", "evo-client-environment-tests", "evo-client-organism-tests", "evo-client-ui-model-tests", "evo-client-panel-tests", "evo-client-bindings-tests", "evo-network-tests", "evo-dev-session"]}, {"name": "client-package", "configurePreset": "client-package", "targets": ["evo-client-state-tests", "evo-client", "evo-client-lifecycle-tests", "evo-client-environment-tests", "evo-client-organism-tests", "evo-client-ui-model-tests", "evo-client-panel-tests", "evo-client-bindings-tests", "evo-network-tests", "evo-dev-session"]} ], diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 8b2a655..c70adf7 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -517,8 +517,8 @@ bounded replication and recovery under loss, overload and region transfer. **Status (2026-10-03):** 10.1 ratified (ADR-0089, schemas/network/m10-budgets-v1.md, networking/M10_TIMING_TRACES.md); targets are not yet qualified on the reference hardware (10.9). 10.2 split: 10.2a selection, pins and build of msquic/OpenSSL done -(ADR-0090); 10.2b bounded channels, exact CA pinning and the impairment relay, and 10.2c -loss/head-of-line measurements follow. 10.3-10.9 planned. +(ADR-0090); 10.2b bounded channels, exact CA pinning and the impairment relay done; +10.2c loss/head-of-line measurements follow. 10.3-10.9 planned. **Entry:** M9 client/session evidence and M7/M8 dev/stage durability. Read NETWORK_ARCHITECTURE, PERSISTENCE_ARCHITECTURE and ADR-0017. The M9 transport is diff --git a/docs/STATUS.md b/docs/STATUS.md index 1dd6537..2d5df6d 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -10,8 +10,10 @@ traces (`evo-dev-session trace`, networking/M10_TIMING_TRACES.md) measured admis p99 <= 5.2 ms, durable p95 about 1.4 s and 50 ms view cadence. Finding: genome v1 and metabolism use per-tick rates, so the quantum fixes biological rates until M11.1 converts or ratifies them. M10.2a selects QUIC with msquic 2.6.2 and OpenSSL 3.5.9 LTS -(ADR-0090), pinned and building on Windows and Linux with a loopback smoke test. Next: -M10.2b bounded channels, exact CA pinning and the owned impairment relay. +(ADR-0090), pinned and building on Windows and Linux with a loopback smoke test. M10.2b adds +the bounded QUIC endpoint (datagram and reliable channels, exact CA pinning with the owned +check authoritative, minimum-MTU start) and the owned impairment relay, tested on both +platforms. Next: M10.2c loss and head-of-line measurements. ## Previous gate - 2026-10-02 diff --git a/docs/adr/0090-m10-quic-transport.md b/docs/adr/0090-m10-quic-transport.md index 34d6e37..2068a96 100644 --- a/docs/adr/0090-m10-quic-transport.md +++ b/docs/adr/0090-m10-quic-transport.md @@ -92,3 +92,31 @@ When the client adopts QUIC, the Windows package (ADR-0086) adds `msquic.dll` to allowlist and the msquic (MIT) and OpenSSL (Apache-2.0) notices. OpenSSL and msquic security releases update the pin, checksums and this ADR. UDP-blocked networks are served only by the dev fallback for now. + +## M10.2b implementation (2026-10-03) +`evo_network_quic` (`include/evo/network/quic_transport.hpp`) provides one `Endpoint` +(server or client) owned by one thread that calls `poll()`: +- **Channels:** realtime DATAGRAMs (payload <= min(1,200, the peer's current maximum); + inbound newest-wins per connection, outbound in-flight cap refuses) and one + client-opened bidirectional stream carrying 32-bit length-framed messages (<= 64 KiB, + once, in order). Outbound reliable bytes not yet completed and inbound bytes awaiting + `poll()` are bounded; inbound overflow closes the connection (`overloaded`), an + oversized or malformed frame closes it (`protocol`), a second peer stream is refused. + Closures carry a cause: local, peer, idle, untrusted, overloaded, protocol, transport. +- **Exact pinning closes the 10.2a gap:** the client defers OpenSSL's verdict, receives the + peer leaf as DER and returns pending; `poll()` verifies it with Mbed TLS against only + the configured CA and name and completes validation. The owned check is the sole + authority; OpenSSL's verdict is advisory (counted when it disagrees). Reason: dev CAs + share a subject name, so with another CA in `SSL_CERT_FILE` OpenSSL's issuer lookup + failed a genuine server; the environment must neither add nor remove trust. +- **Threading rule:** callbacks touch state only under one mutex and never call msquic + while holding it (msquic may run callbacks inline); handles are closed only by the + owning thread. Mbed TLS runs only on the owning thread. +- **Path MTU:** msquic assumes a 1,288-byte IP MTU by default (1,260-byte IPv4 UDP + payloads), which a compliant 1,280-byte path drops. `MinimumMtu` is set to 1,280; path + discovery only probes upward, and the 1,200-byte datagram budget fits that path. +- **Settings:** idle 5 s, keep-alive 1 s, handshake 5 s, flow-control windows equal to the + inbound reliable bound, one peer bidirectional stream, no unidirectional streams. +- **Impairment relay** (`tools/network/impair`, tests and measurements only): seeded + loss, duplication, reordering, corruption, delay/jitter, payload-size limit and live + reconfiguration (outage). diff --git a/docs/architecture/NETWORK_ARCHITECTURE.md b/docs/architecture/NETWORK_ARCHITECTURE.md index 811286c..4973c56 100644 --- a/docs/architecture/NETWORK_ARCHITECTURE.md +++ b/docs/architecture/NETWORK_ARCHITECTURE.md @@ -3,7 +3,8 @@ Status: locked server authority and untrusted clients; transport/reliability are proposals for M10. M10 timing and workload budgets: ADR-0089 and [m10-budgets-v1](../../schemas/network/m10-budgets-v1.md). M10 transport: QUIC via msquic -with OpenSSL (ADR-0090); channels and impairment tests follow in M10.2b. The scoped M9 dev session (TLS 1.3, protocol v1, dev-only grants) +with OpenSSL (ADR-0090): bounded DATAGRAM and reliable-stream channels with exact CA +pinning (`evo_network_quic`, M10.2b); loss/head-of-line measurements follow in M10.2c. The scoped M9 dev session (TLS 1.3, protocol v1, dev-only grants) is specified in [DEV_SESSION_PROTOCOL](../networking/DEV_SESSION_PROTOCOL.md) and ADR-0082. M0 has no networking. Steam distribution follows later through an identity-provider adapter; core account IDs are independent of Steam IDs. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index aac28bd..8da2a4d 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -4039,3 +4039,36 @@ explicitly). Security finding recorded in ADR-0090: OpenSSL's built-in verificat trusts its default verify paths; exact pinning is 10.2b work. Not done: bounded channels, impairment relay, loss/HOL measurements, CI integration of the quic preset, sanitizer builds of msquic, packaging of msquic.dll. + + +## 2026-10-03 — M10.2b bounded QUIC channels, exact pinning, impairment relay (ADR-0090) + +Implemented `evo_network_quic` (Endpoint, MessageReassembler, framing), +`evo_network_impair` (UdpRelay) and `evo-quic-transport-tests` (label `quic`). +Tests: framing with 2,000 random-length messages under random chunking, oversized +frame refusal, empty messages; 100 ordered messages each way, datagrams each way, local +size limits, unknown connection, outbound reliable budget (busy), peer protocol +violation closing the connection; exact pinning (attacker CA via `SSL_CERT_FILE` +refused, genuine server still accepted with that environment, wrong name refused); +impairment through the relay (10 % loss, 10 % duplication, 20 % reordering, 5 % +corruption, 5 ms + 0-10 ms jitter both ways): 200/200 messages exactly once in order, +datagrams never duplicated and partly lost (262-290 of 300 across runs); 1,252-byte UDP +path: 1,200-byte datagrams and 1,100-byte messages cross it (6 larger PMTUD probes +dropped); total outage: both sides idle-closed after about 1.3 s (1 s idle timeout); +bounds: newest-wins datagram queue (at most 4 kept), connection cap refusal, reliable +inbound overflow closing as overloaded; 500 random UDP datagrams at the listener produce +no events and a genuine client still connects; clean local/peer close. + +Found and fixed during development: a possible deadlock (msquic calls under our lock; +callbacks can run inline), OpenSSL's issuer lookup failing a genuine server when another +same-named CA was in `SSL_CERT_FILE` (owned pin made authoritative), msquic's 1,288-byte +default MTU exceeding a 1,280-byte path (MinimumMtu 1280), a flaky coverage assertion +(2 % corruption could produce none), two test bugs found by UBSan/assertions, Windows +`NOMINMAX`/`` portability, and clang-tidy findings. + +Executed: Linux GCC 14 preset `quic` CTest 3/3 and transport tests 5/5 repeated runs; +Clang 19 ASan/UBSan (Debug, leak detection) transport tests; clang-tidy on endpoint, +relay and tests; format check; Windows MSVC 19.44 preset `quic` 3/3 and transport tests +5/5 repeated runs. +Not done: loss/head-of-line measurements and TCP comparison (10.2c), msquic itself under +sanitizers, CI integration of the quic preset, integration into the region host/client. diff --git a/include/evo/network/quic_transport.hpp b/include/evo/network/quic_transport.hpp new file mode 100644 index 0000000..a3bfd55 --- /dev/null +++ b/include/evo/network/quic_transport.hpp @@ -0,0 +1,142 @@ +#pragma once +// M10.2b QUIC transport (ADR-0090) over msquic: one endpoint (server or client) owned by +// one thread, which calls poll(). msquic's worker threads only append to bounded queues +// under a mutex; nothing they do touches simulation or presentation state. +// +// Channels per connection: +// - realtime: QUIC DATAGRAM frames for superseding state and intentions. Unreliable and +// congestion-controlled; payload <= min(limits.datagram_payload, the peer's current +// maximum). Inbound keeps the newest `inbound_datagrams`; older ones are dropped and +// counted. Sends beyond `outbound_datagrams` in flight are refused (the caller drops). +// - reliable: one bidirectional stream opened by the client; messages framed with a +// 32-bit little-endian length, each <= limits.reliable_message, delivered once, in +// order. Outbound bytes not yet acknowledged by msquic are bounded (sends refuse beyond); +// inbound bytes awaiting poll() are bounded (overflow closes the connection). +// +// Trust: the client pins exactly one CA (PEM) and one server name. The owned check (Mbed +// TLS, on the owning thread) is the sole authority: the leaf must verify against only the +// pinned CA for that name. OpenSSL's deferred verdict is advisory (counted when it +// disagrees), so OpenSSL default verify paths and environment variables can neither add +// nor remove trust (ADR-0090 "exact pinning"). +#include +#include +#include +#include +#include +#include +#include +#include + +namespace evo::network::quic { +// ADR-0089 budgets; tests may narrow them. +struct Limits { + std::size_t datagram_payload{1200}; + std::size_t reliable_message{std::size_t{64} * 1024}; + std::size_t inbound_datagrams{64}; + std::size_t inbound_reliable_bytes{std::size_t{256} * 1024}; + std::size_t outbound_reliable_bytes{std::size_t{256} * 1024}; + std::size_t outbound_datagrams{64}; + std::size_t max_connections{32}; + std::size_t max_events{4096}; // Endpoint-wide queue between msquic threads and poll(). + std::chrono::milliseconds idle_timeout{5000}, keep_alive{1000}, handshake_timeout{5000}; +}; +using ConnectionId = std::uint64_t; // Endpoint-local, never reused. +enum class EventKind : std::uint8_t { + connected, // Handshake complete (client: certificate pinned and accepted). + datagram, // Realtime payload. + message, // Reliable message. + closed, // Connection gone; `cause` says why. No further events for it. +}; +enum class CloseCause : std::uint8_t { + local, // close() or endpoint shutdown. + peer, // The peer closed. + idle, // Idle timeout (liveness lost). + untrusted, // Certificate/name refused (either side's view). + overloaded, // A bounded inbound queue overflowed; the connection was closed. + protocol, // Malformed reliable framing or oversized message from the peer. + transport, // Any other transport failure. +}; +struct Event { + EventKind kind{}; + ConnectionId connection{}; + std::vector bytes; + CloseCause cause{}; +}; +struct Counters { + std::uint64_t datagrams_dropped_inbound{}, datagrams_refused_outbound{}, + messages_refused_outbound{}; + std::uint64_t connections_refused{}, events_dropped{}; + std::uint64_t openssl_disagreements{}; // OpenSSL refused a chain the pinned check accepted. +}; +enum class EndpointError : std::uint8_t { configuration, library, credentials, listen }; +enum class SendError : std::uint8_t { + unknown_connection, + not_connected, + too_large, + busy, + transport +}; + +struct ServerConfig { + std::string listen_address{"127.0.0.1"}; + std::uint16_t port{}; // 0: ephemeral (see Endpoint::port()). + std::string certificate_pem_file, key_pem_file; + Limits limits{}; +}; +struct ClientConfig { + std::string server_address{"127.0.0.1"}; + std::uint16_t port{}; + std::string server_name; // SNI and the pinned name. + std::string authority_pem; // The one CA trusted (PEM text, not a path). + std::string authority_pem_file; // The same CA as a file, for OpenSSL's own check. + Limits limits{}; +}; + +class Endpoint { + public: + [[nodiscard]] static std::expected, EndpointError> + serve(const ServerConfig& config); + [[nodiscard]] static std::expected, EndpointError> + connect(const ClientConfig& config); + Endpoint(const Endpoint&) = delete; + Endpoint& operator=(const Endpoint&) = delete; + ~Endpoint(); // Closes every connection and waits for msquic to finish with them. + + // Drains events (at most limits.max_events) and completes pending certificate checks. + [[nodiscard]] std::vector poll(); + [[nodiscard]] std::expected send_datagram(ConnectionId id, + std::span bytes); + [[nodiscard]] std::expected send_message(ConnectionId id, + std::span bytes); + void close(ConnectionId id); + // Current maximum datagram payload for a connection (0 if none yet). + [[nodiscard]] std::size_t datagram_limit(ConnectionId id) const; + [[nodiscard]] std::uint16_t port() const noexcept; + [[nodiscard]] Counters counters() const; + + struct Impl; + + private: + explicit Endpoint(std::unique_ptr impl); + std::unique_ptr impl_; +}; + +// Reliable-stream framing, exposed for tests and fuzzing: a reassembler that accepts +// arbitrary byte chunks and yields complete messages, refusing oversized lengths. +class MessageReassembler { + public: + explicit MessageReassembler(std::size_t max_message) : max_{max_message} {} + // Appends bytes; complete messages are appended to `out`. False on a frame longer + // than the limit (the stream is then unusable). + [[nodiscard]] bool feed(std::span bytes, + std::vector>& out); + [[nodiscard]] std::size_t buffered() const noexcept { + return buffer_.size(); + } + + private: + std::size_t max_; + std::vector buffer_; +}; +[[nodiscard]] std::vector frame_message(std::span payload); +} // namespace evo::network::quic diff --git a/src/network/transport/quic_endpoint.cpp b/src/network/transport/quic_endpoint.cpp new file mode 100644 index 0000000..81e2d65 --- /dev/null +++ b/src/network/transport/quic_endpoint.cpp @@ -0,0 +1,744 @@ +// M10.2b QUIC endpoint over msquic (ADR-0090). Threading: msquic invokes the callbacks +// below on its worker threads; they only touch `Impl` under `mutex` and append to bounded +// queues. msquic may run callbacks inline from API calls, so no msquic function is ever +// called while `mutex` is held: decisions are made under the lock, calls after it. +// Every msquic handle is closed by the owning thread (poll()/~Endpoint), so a handle used +// by send_*() or certificate completion is never freed underneath it. Mbed TLS (exact +// pinning) is only used on the owning thread. +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX // msquic.h includes ; keep std::min/std::max usable. +#endif +#endif + +#include "evo/network/quic_transport.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace evo::network::quic { +namespace { +constexpr std::string_view alpn_text = "evo-m10"; +constexpr std::uint32_t frame_header = 4; + +std::uint32_t read_u32(const std::uint8_t* p) { + return static_cast(p[0]) | (static_cast(p[1]) << 8U) | + (static_cast(p[2]) << 16U) | (static_cast(p[3]) << 24U); +} +bool tls_or_certificate_failure(QUIC_STATUS status) { + const auto value = static_cast(status); + const auto alert_base = static_cast(QUIC_STATUS_TLS_ALERT(0)); + return (value & ~0xffU) == alert_base || status == QUIC_STATUS_CERT_EXPIRED || + status == QUIC_STATUS_CERT_UNTRUSTED_ROOT || status == QUIC_STATUS_CERT_NO_CERT; +} +// Leaf (DER) verified against exactly one CA (PEM) for one name; nothing else trusted. +bool pinned(std::span leaf, const std::string& authority_pem, + const std::string& name) { + if (psa_crypto_init() != PSA_SUCCESS || leaf.empty()) + return false; + mbedtls_x509_crt ca; + mbedtls_x509_crt peer; + mbedtls_x509_crt_init(&ca); + mbedtls_x509_crt_init(&peer); + std::vector pem(authority_pem.begin(), authority_pem.end()); + pem.push_back('\0'); + std::uint32_t flags = 0; + const bool ok = + mbedtls_x509_crt_parse(&ca, pem.data(), pem.size()) == 0 && + mbedtls_x509_crt_parse_der(&peer, leaf.data(), leaf.size()) == 0 && + mbedtls_x509_crt_verify(&peer, &ca, nullptr, name.c_str(), &flags, nullptr, nullptr) == 0 && + flags == 0; + mbedtls_x509_crt_free(&peer); + mbedtls_x509_crt_free(&ca); + return ok; +} +} // namespace + +bool MessageReassembler::feed(std::span bytes, + std::vector>& out) { + buffer_.insert(buffer_.end(), bytes.begin(), bytes.end()); + std::size_t at = 0; + while (buffer_.size() - at >= frame_header) { + const std::size_t length = read_u32(buffer_.data() + at); + if (length > max_) + return false; + if (buffer_.size() - at - frame_header < length) + break; + const auto* begin = buffer_.data() + at + frame_header; + out.emplace_back(begin, begin + length); + at += frame_header + length; + } + buffer_.erase(buffer_.begin(), buffer_.begin() + static_cast(at)); + return true; +} +std::vector frame_message(std::span payload) { + const auto n = static_cast(payload.size()); + std::vector out{static_cast(n), static_cast(n >> 8U), + static_cast(n >> 16U), + static_cast(n >> 24U)}; + out.insert(out.end(), payload.begin(), payload.end()); + return out; +} + +struct SendBuffer { + QUIC_BUFFER quic{}; + std::vector bytes; + ConnectionId connection{}; +}; + +struct Endpoint::Impl { + struct Conn { + Impl* owner{}; + ConnectionId id{}; + HQUIC handle{}, stream{}; + bool handshake{}, connected{}, shutdown_complete{}, stream_complete{}, datagrams{}; + std::size_t datagram_max{}; + std::size_t inbound_datagrams{}, inbound_bytes{}; + std::size_t outbound_datagrams{}, outbound_bytes{}; + std::optional cause; + MessageReassembler reassembler{0}; + }; + struct PendingCertificate { + ConnectionId id{}; + std::vector leaf; + bool openssl_ok{}; + }; + const QUIC_API_TABLE* api{}; + HQUIC registration{}, configuration{}, listener{}; + bool is_server{}; + Limits limits; + std::string authority_pem, server_name; + std::uint16_t bound_port{}; + std::array alpn_bytes{}; + QUIC_BUFFER alpn{}; + + mutable std::mutex mutex; + std::condition_variable changed; + std::map> conns; + ConnectionId next_id{1}; + std::deque events; + std::deque certificates; + Counters counters; + + // Under `mutex`. Datagrams: newest wins per connection; other events refuse to drop + // and instead mark the connection overloaded (returns true: the caller shuts it down + // after releasing the lock). + [[nodiscard]] bool push(Conn& c, Event e) { + if (e.kind == EventKind::datagram) { + if (c.inbound_datagrams >= limits.inbound_datagrams) { + const auto oldest = std::find_if(events.begin(), events.end(), [&](const Event& x) { + return x.kind == EventKind::datagram && x.connection == c.id; + }); + if (oldest != events.end()) { + events.erase(oldest); + --c.inbound_datagrams; + } + ++counters.datagrams_dropped_inbound; + } + if (events.size() >= limits.max_events) { + ++counters.datagrams_dropped_inbound; + return false; + } + ++c.inbound_datagrams; + } else if (e.kind != EventKind::closed && events.size() >= limits.max_events) { + ++counters.events_dropped; + return mark(c, CloseCause::overloaded); + } + events.push_back(std::move(e)); + changed.notify_all(); + return false; + } + // Under `mutex`: records the first cause; true when the caller must shut down. + static bool mark(Conn& c, CloseCause cause) { + if (!c.cause) + c.cause = cause; + return !c.shutdown_complete; + } + // Without `mutex`: asynchronous close; the outcome arrives as SHUTDOWN_COMPLETE. + void shutdown(HQUIC handle, CloseCause cause) const { + api->ConnectionShutdown(handle, QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, + static_cast(cause) + 1); + } + + static QUIC_STATUS QUIC_API on_stream(HQUIC /*stream*/, void* context, + QUIC_STREAM_EVENT* event) { + auto& c = *static_cast(context); + auto& self = *c.owner; + std::optional close; + HQUIC handle = nullptr; + { + const std::scoped_lock lock{self.mutex}; + handle = c.handle; + close = self.stream_event(c, event); + } + if (close) + self.shutdown(handle, *close); + return QUIC_STATUS_SUCCESS; + } + // Under `mutex`; returns a cause when the connection must be shut down. + std::optional stream_event(Conn& c, QUIC_STREAM_EVENT* event) { + auto& self = *this; + switch (event->Type) { + case QUIC_STREAM_EVENT_RECEIVE: { + std::vector> messages; + for (std::uint32_t i = 0; i < event->RECEIVE.BufferCount; ++i) { + const auto& b = event->RECEIVE.Buffers[i]; + c.inbound_bytes += b.Length; + if (!c.reassembler.feed({b.Buffer, b.Length}, messages)) + return mark(c, CloseCause::protocol) ? std::optional{CloseCause::protocol} + : std::nullopt; + } + if (c.inbound_bytes > self.limits.inbound_reliable_bytes) + return mark(c, CloseCause::overloaded) ? std::optional{CloseCause::overloaded} + : std::nullopt; + for (auto& m : messages) + if (self.push(c, {EventKind::message, c.id, std::move(m), {}})) + return CloseCause::overloaded; + break; + } + case QUIC_STREAM_EVENT_SEND_COMPLETE: { + std::unique_ptr done{ + static_cast(event->SEND_COMPLETE.ClientContext)}; + if (done) + c.outbound_bytes -= std::min(c.outbound_bytes, done->bytes.size()); + break; + } + case QUIC_STREAM_EVENT_PEER_SEND_SHUTDOWN: + case QUIC_STREAM_EVENT_PEER_SEND_ABORTED: + case QUIC_STREAM_EVENT_PEER_RECEIVE_ABORTED: + // The reliable channel lives exactly as long as the connection. + return mark(c, CloseCause::peer) ? std::optional{CloseCause::peer} : std::nullopt; + case QUIC_STREAM_EVENT_SHUTDOWN_COMPLETE: + c.stream_complete = true; + self.changed.notify_all(); + break; + default: + break; + } + return std::nullopt; + } + + static QUIC_STATUS QUIC_API on_connection(HQUIC handle, void* context, + QUIC_CONNECTION_EVENT* event) { + auto& c = *static_cast(context); + auto& self = *c.owner; + if (event->Type == QUIC_CONNECTION_EVENT_CONNECTED && !self.is_server) { + // The client opens the one reliable stream and announces it immediately; these + // calls may run stream callbacks inline, so the lock is not held. + HQUIC stream = nullptr; + const bool opened = QUIC_SUCCEEDED( + self.api->StreamOpen(handle, QUIC_STREAM_OPEN_FLAG_NONE, on_stream, &c, &stream)); + if (opened) { + const std::scoped_lock lock{self.mutex}; + c.stream = stream; + } + if (!opened || + QUIC_FAILED(self.api->StreamStart(stream, QUIC_STREAM_START_FLAG_IMMEDIATE))) { + { + const std::scoped_lock lock{self.mutex}; + (void)mark(c, CloseCause::transport); + } + self.shutdown(handle, CloseCause::transport); + return QUIC_STATUS_SUCCESS; + } + } + if (event->Type == QUIC_CONNECTION_EVENT_PEER_STREAM_STARTED) { + bool accept = false; + { + const std::scoped_lock lock{self.mutex}; + accept = self.is_server && c.stream == nullptr; + if (accept) + c.stream = event->PEER_STREAM_STARTED.Stream; + } + if (!accept) { + // Exactly one client-opened stream is allowed. + self.api->StreamShutdown(event->PEER_STREAM_STARTED.Stream, + QUIC_STREAM_SHUTDOWN_FLAG_ABORT, 1); + { + const std::scoped_lock lock{self.mutex}; + (void)mark(c, CloseCause::protocol); + } + self.shutdown(handle, CloseCause::protocol); + return QUIC_STATUS_SUCCESS; + } + self.api->SetCallbackHandler(event->PEER_STREAM_STARTED.Stream, + reinterpret_cast(on_stream), &c); + } + std::optional close; + QUIC_STATUS result = QUIC_STATUS_SUCCESS; + { + const std::scoped_lock lock{self.mutex}; + close = self.connection_event(c, event, result); + } + if (close) + self.shutdown(handle, *close); + return result; + } + // Under `mutex`; returns a cause when the connection must be shut down. + std::optional connection_event(Conn& c, QUIC_CONNECTION_EVENT* event, + QUIC_STATUS& result) { + auto& self = *this; + const auto connected = [&]() -> std::optional { + c.connected = true; + if (self.push(c, {EventKind::connected, c.id, {}, {}})) + return CloseCause::overloaded; + return std::nullopt; + }; + switch (event->Type) { + case QUIC_CONNECTION_EVENT_PEER_CERTIFICATE_RECEIVED: { + // Client only: copied and checked on the owning thread (exact pinning). + const auto* leaf = + static_cast(event->PEER_CERTIFICATE_RECEIVED.Certificate); + PendingCertificate p{ + c.id, {}, QUIC_SUCCEEDED(event->PEER_CERTIFICATE_RECEIVED.DeferredStatus)}; + if (leaf != nullptr && leaf->Buffer != nullptr) + p.leaf.assign(leaf->Buffer, leaf->Buffer + leaf->Length); + self.certificates.push_back(std::move(p)); + self.changed.notify_all(); + result = QUIC_STATUS_PENDING; + break; + } + case QUIC_CONNECTION_EVENT_CONNECTED: + c.handshake = true; + // Client: the stream is open (above). Server: wait for the client's stream. + if (!self.is_server || c.stream != nullptr) + return connected(); + break; + case QUIC_CONNECTION_EVENT_PEER_STREAM_STARTED: + if (c.handshake && !c.connected) + return connected(); + break; + case QUIC_CONNECTION_EVENT_DATAGRAM_STATE_CHANGED: + c.datagrams = event->DATAGRAM_STATE_CHANGED.SendEnabled != 0; + c.datagram_max = event->DATAGRAM_STATE_CHANGED.MaxSendLength; + break; + case QUIC_CONNECTION_EVENT_DATAGRAM_RECEIVED: { + const auto* b = event->DATAGRAM_RECEIVED.Buffer; + if (b->Length <= self.limits.datagram_payload) + (void)self.push( + c, {EventKind::datagram, c.id, {b->Buffer, b->Buffer + b->Length}, {}}); + break; + } + case QUIC_CONNECTION_EVENT_DATAGRAM_SEND_STATE_CHANGED: + if (QUIC_DATAGRAM_SEND_STATE_IS_FINAL(event->DATAGRAM_SEND_STATE_CHANGED.State)) { + delete static_cast(event->DATAGRAM_SEND_STATE_CHANGED.ClientContext); + c.outbound_datagrams -= std::min(c.outbound_datagrams, 1); + } + break; + case QUIC_CONNECTION_EVENT_SHUTDOWN_INITIATED_BY_TRANSPORT: { + const auto status = event->SHUTDOWN_INITIATED_BY_TRANSPORT.Status; + if (!c.cause) + c.cause = status == QUIC_STATUS_CONNECTION_IDLE ? CloseCause::idle + : tls_or_certificate_failure(status) ? CloseCause::untrusted + : CloseCause::transport; + break; + } + case QUIC_CONNECTION_EVENT_SHUTDOWN_INITIATED_BY_PEER: + if (!c.cause) + c.cause = CloseCause::peer; + break; + case QUIC_CONNECTION_EVENT_SHUTDOWN_COMPLETE: + c.shutdown_complete = true; + c.connected = false; + (void)self.push(c, + {EventKind::closed, c.id, {}, c.cause.value_or(CloseCause::transport)}); + break; + default: + break; + } + return std::nullopt; + } + + static QUIC_STATUS QUIC_API on_listener(HQUIC /*listener*/, void* context, + QUIC_LISTENER_EVENT* event) { + if (event->Type != QUIC_LISTENER_EVENT_NEW_CONNECTION) + return QUIC_STATUS_SUCCESS; + auto& self = *static_cast(context); + Conn* c = nullptr; + { + const std::scoped_lock lock{self.mutex}; + if (self.conns.size() >= self.limits.max_connections) { + ++self.counters.connections_refused; + return QUIC_STATUS_CONNECTION_REFUSED; + } + c = self.adopt(event->NEW_CONNECTION.Connection); + } + self.api->SetCallbackHandler(c->handle, reinterpret_cast(on_connection), c); + return self.api->ConnectionSetConfiguration(c->handle, self.configuration); + } + + // Under `mutex`. + Conn* adopt(HQUIC handle) { + auto c = std::make_unique(); + c->owner = this; + c->id = next_id++; + c->handle = handle; + c->reassembler = MessageReassembler{limits.reliable_message}; + auto* raw = c.get(); + conns.emplace(raw->id, std::move(c)); + return raw; + } + + QUIC_SETTINGS settings() const { + QUIC_SETTINGS s{}; + s.IdleTimeoutMs = static_cast(limits.idle_timeout.count()); + s.IsSet.IdleTimeoutMs = TRUE; + s.HandshakeIdleTimeoutMs = static_cast(limits.handshake_timeout.count()); + s.IsSet.HandshakeIdleTimeoutMs = TRUE; + s.KeepAliveIntervalMs = static_cast(limits.keep_alive.count()); + s.IsSet.KeepAliveIntervalMs = TRUE; + s.DatagramReceiveEnabled = TRUE; + s.IsSet.DatagramReceiveEnabled = TRUE; + // msquic otherwise assumes a 1288-byte IP MTU (1260-byte IPv4 UDP payloads), which + // a compliant 1280-byte path drops. Start at the IPv6 minimum path MTU and only + // probe upward; the 1,200-byte datagram budget (ADR-0089) fits it. + s.MinimumMtu = 1280; + s.IsSet.MinimumMtu = TRUE; + s.PeerBidiStreamCount = is_server ? 1 : 0; // Only the client opens the reliable stream. + s.IsSet.PeerBidiStreamCount = TRUE; + s.PeerUnidiStreamCount = 0; + s.IsSet.PeerUnidiStreamCount = TRUE; + // Flow control bounds what a peer may push before poll() drains it. + s.StreamRecvWindowDefault = + static_cast(std::max(limits.inbound_reliable_bytes, 4096)); + s.IsSet.StreamRecvWindowDefault = TRUE; + s.ConnFlowControlWindow = + static_cast(std::max(limits.inbound_reliable_bytes, 4096)); + s.IsSet.ConnFlowControlWindow = TRUE; + return s; + } + + bool open(bool server) { + is_server = server; + std::copy(alpn_text.begin(), alpn_text.end(), alpn_bytes.begin()); + alpn = {static_cast(alpn_bytes.size()), alpn_bytes.data()}; + if (QUIC_FAILED(MsQuicOpen2(&api))) + return false; + const QUIC_REGISTRATION_CONFIG reg{"aevolution", QUIC_EXECUTION_PROFILE_LOW_LATENCY}; + if (QUIC_FAILED(api->RegistrationOpen(®, ®istration))) + return false; + const auto s = settings(); + return QUIC_SUCCEEDED( + api->ConfigurationOpen(registration, &alpn, 1, &s, sizeof(s), nullptr, &configuration)); + } + + // Owning thread: completes pending certificate checks outside the lock. + void verify_certificates() { + std::deque work; + { + const std::scoped_lock lock{mutex}; + work.swap(certificates); + } + for (const auto& p : work) { + // The owned check is the sole authority: OpenSSL's verdict (which its default + // paths and environment can influence either way) is advisory only. + const bool ok = pinned(p.leaf, authority_pem, server_name); + if (!p.openssl_ok && ok) { + const std::scoped_lock lock{mutex}; + ++counters.openssl_disagreements; + } + HQUIC handle = nullptr; + { + const std::scoped_lock lock{mutex}; + const auto it = conns.find(p.id); + if (it == conns.end()) + continue; + handle = it->second->handle; + if (!ok && !it->second->cause) + it->second->cause = CloseCause::untrusted; + } + api->ConnectionCertificateValidationComplete(handle, ok ? TRUE : FALSE, + ok ? QUIC_TLS_ALERT_CODE_SUCCESS + : QUIC_TLS_ALERT_CODE_BAD_CERTIFICATE); + } + } + + // Owning thread: closes msquic handles of finished connections. + void release(std::unique_ptr c) { + if (c->stream != nullptr) + api->StreamClose(c->stream); + api->ConnectionClose(c->handle); + } + + void shutdown_all() { + std::vector> all; + std::vector open, pending; + { + const std::scoped_lock lock{mutex}; + for (auto& [id, c] : conns) + if (!c->shutdown_complete) + open.push_back(c->handle); + for (const auto& p : certificates) + if (const auto it = conns.find(p.id); it != conns.end()) + pending.push_back(it->second->handle); + certificates.clear(); + } + // Pending certificate checks would hold a connection open; refuse them first. + for (auto* h : pending) + api->ConnectionCertificateValidationComplete(h, FALSE, + QUIC_TLS_ALERT_CODE_BAD_CERTIFICATE); + for (auto* h : open) + api->ConnectionShutdown(h, QUIC_CONNECTION_SHUTDOWN_FLAG_SILENT, 0); + { + std::unique_lock lock{mutex}; + changed.wait_for(lock, std::chrono::seconds{10}, [&] { + return std::all_of(conns.begin(), conns.end(), [](const auto& e) { + return e.second->shutdown_complete && + (e.second->stream == nullptr || e.second->stream_complete); + }); + }); + for (auto& [id, c] : conns) + all.push_back(std::move(c)); + conns.clear(); + } + for (auto& c : all) + release(std::move(c)); + } +}; + +Endpoint::Endpoint(std::unique_ptr impl) : impl_{std::move(impl)} {} + +Endpoint::~Endpoint() { + auto& p = *impl_; + if (p.api == nullptr) + return; + if (p.listener != nullptr) + p.api->ListenerStop(p.listener); + p.shutdown_all(); + if (p.listener != nullptr) + p.api->ListenerClose(p.listener); + if (p.configuration != nullptr) + p.api->ConfigurationClose(p.configuration); + if (p.registration != nullptr) + p.api->RegistrationClose(p.registration); + MsQuicClose(p.api); +} + +std::expected, EndpointError> +Endpoint::serve(const ServerConfig& config) { + if (config.certificate_pem_file.empty() || config.key_pem_file.empty() || + config.limits.max_connections == 0) + return std::unexpected(EndpointError::configuration); + auto impl = std::make_unique(); + impl->limits = config.limits; + std::unique_ptr endpoint{new Endpoint{std::move(impl)}}; + auto& p = *endpoint->impl_; + if (!p.open(true)) + return std::unexpected(EndpointError::library); + QUIC_CERTIFICATE_FILE file{config.key_pem_file.c_str(), config.certificate_pem_file.c_str()}; + QUIC_CREDENTIAL_CONFIG credential{}; + credential.Type = QUIC_CREDENTIAL_TYPE_CERTIFICATE_FILE; + credential.CertificateFile = &file; + if (QUIC_FAILED(p.api->ConfigurationLoadCredential(p.configuration, &credential))) + return std::unexpected(EndpointError::credentials); + QUIC_ADDR address{}; + if (!QuicAddrFromString(config.listen_address.c_str(), config.port, &address) || + QUIC_FAILED(p.api->ListenerOpen(p.registration, Impl::on_listener, &p, &p.listener)) || + QUIC_FAILED(p.api->ListenerStart(p.listener, &p.alpn, 1, &address))) + return std::unexpected(EndpointError::listen); + std::uint32_t size = sizeof(address); + if (QUIC_FAILED( + p.api->GetParam(p.listener, QUIC_PARAM_LISTENER_LOCAL_ADDRESS, &size, &address))) + return std::unexpected(EndpointError::listen); + p.bound_port = QuicAddrGetPort(&address); + return endpoint; +} + +std::expected, EndpointError> +Endpoint::connect(const ClientConfig& config) { + if (config.server_name.empty() || config.authority_pem.empty() || + config.authority_pem_file.empty() || config.port == 0) + return std::unexpected(EndpointError::configuration); + auto impl = std::make_unique(); + impl->limits = config.limits; + impl->authority_pem = config.authority_pem; + impl->server_name = config.server_name; + std::unique_ptr endpoint{new Endpoint{std::move(impl)}}; + auto& p = *endpoint->impl_; + if (!p.open(false)) + return std::unexpected(EndpointError::library); + QUIC_CREDENTIAL_CONFIG credential{}; + credential.Type = QUIC_CREDENTIAL_TYPE_NONE; + // OpenSSL's verifier everywhere, its result deferred to us, the chain in portable DER + // form: the connection completes only after the owned exact-pinning check. + constexpr auto flags = + static_cast(QUIC_CREDENTIAL_FLAG_CLIENT) | + static_cast(QUIC_CREDENTIAL_FLAG_SET_CA_CERTIFICATE_FILE) | + static_cast(QUIC_CREDENTIAL_FLAG_USE_TLS_BUILTIN_CERTIFICATE_VALIDATION) | + static_cast(QUIC_CREDENTIAL_FLAG_INDICATE_CERTIFICATE_RECEIVED) | + static_cast(QUIC_CREDENTIAL_FLAG_DEFER_CERTIFICATE_VALIDATION) | + static_cast(QUIC_CREDENTIAL_FLAG_USE_PORTABLE_CERTIFICATES); + credential.Flags = static_cast( + flags); // NOLINT(clang-analyzer-optin.core.EnumCastOutOfRange) + credential.CaCertificateFile = config.authority_pem_file.c_str(); + if (QUIC_FAILED(p.api->ConfigurationLoadCredential(p.configuration, &credential))) + return std::unexpected(EndpointError::credentials); + HQUIC handle = nullptr; + Impl::Conn* c = nullptr; + if (QUIC_FAILED(p.api->ConnectionOpen(p.registration, Impl::on_connection, nullptr, &handle))) + return std::unexpected(EndpointError::library); + { + const std::scoped_lock lock{p.mutex}; + c = p.adopt(handle); + } + p.api->SetCallbackHandler(handle, reinterpret_cast(Impl::on_connection), c); + QUIC_ADDR remote{}; + if (!QuicAddrFromString(config.server_address.c_str(), config.port, &remote) || + QUIC_FAILED( + p.api->SetParam(handle, QUIC_PARAM_CONN_REMOTE_ADDRESS, sizeof(remote), &remote)) || + QUIC_FAILED(p.api->ConnectionStart(handle, p.configuration, QUIC_ADDRESS_FAMILY_UNSPEC, + config.server_name.c_str(), config.port))) + return std::unexpected(EndpointError::configuration); + p.bound_port = config.port; + return endpoint; +} + +std::vector Endpoint::poll() { + auto& p = *impl_; + p.verify_certificates(); + std::vector out; + std::vector> finished; + { + const std::scoped_lock lock{p.mutex}; + while (!p.events.empty() && out.size() < p.limits.max_events) { + auto e = std::move(p.events.front()); + p.events.pop_front(); + if (const auto it = p.conns.find(e.connection); it != p.conns.end()) { + auto& c = *it->second; + if (e.kind == EventKind::datagram) + --c.inbound_datagrams; + else if (e.kind == EventKind::message) + c.inbound_bytes -= std::min(c.inbound_bytes, e.bytes.size() + frame_header); + else if (e.kind == EventKind::closed && + (c.stream == nullptr || c.stream_complete || c.shutdown_complete)) { + finished.push_back(std::move(it->second)); + p.conns.erase(it); + } + } + out.push_back(std::move(e)); + } + } + for (auto& c : finished) + p.release(std::move(c)); + return out; +} + +std::expected Endpoint::send_datagram(ConnectionId id, + std::span bytes) { + auto& p = *impl_; + auto buffer = std::make_unique(); + HQUIC handle = nullptr; + { + const std::scoped_lock lock{p.mutex}; + const auto it = p.conns.find(id); + if (it == p.conns.end()) + return std::unexpected(SendError::unknown_connection); + auto& c = *it->second; + if (!c.connected || !c.datagrams) + return std::unexpected(SendError::not_connected); + if (bytes.size() > std::min(p.limits.datagram_payload, c.datagram_max)) { + ++p.counters.datagrams_refused_outbound; + return std::unexpected(SendError::too_large); + } + if (c.outbound_datagrams >= p.limits.outbound_datagrams) { + ++p.counters.datagrams_refused_outbound; + return std::unexpected(SendError::busy); + } + ++c.outbound_datagrams; + handle = c.handle; + } + buffer->bytes.assign(bytes.begin(), bytes.end()); + buffer->connection = id; + buffer->quic = {static_cast(buffer->bytes.size()), buffer->bytes.data()}; + auto* raw = buffer.release(); + if (QUIC_FAILED(p.api->DatagramSend(handle, &raw->quic, 1, QUIC_SEND_FLAG_NONE, raw))) { + delete raw; + const std::scoped_lock lock{p.mutex}; + if (const auto it = p.conns.find(id); it != p.conns.end()) + it->second->outbound_datagrams -= + std::min(it->second->outbound_datagrams, 1); + return std::unexpected(SendError::transport); + } + return {}; +} + +std::expected Endpoint::send_message(ConnectionId id, + std::span bytes) { + auto& p = *impl_; + if (bytes.size() > p.limits.reliable_message) { + const std::scoped_lock lock{p.mutex}; + ++p.counters.messages_refused_outbound; + return std::unexpected(SendError::too_large); + } + auto buffer = std::make_unique(); + buffer->bytes = frame_message(bytes); + buffer->connection = id; + buffer->quic = {static_cast(buffer->bytes.size()), buffer->bytes.data()}; + HQUIC stream = nullptr; + { + const std::scoped_lock lock{p.mutex}; + const auto it = p.conns.find(id); + if (it == p.conns.end()) + return std::unexpected(SendError::unknown_connection); + auto& c = *it->second; + if (!c.connected || c.stream == nullptr) + return std::unexpected(SendError::not_connected); + if (c.outbound_bytes + buffer->bytes.size() > p.limits.outbound_reliable_bytes) { + ++p.counters.messages_refused_outbound; + return std::unexpected(SendError::busy); + } + c.outbound_bytes += buffer->bytes.size(); + stream = c.stream; + } + const auto size = buffer->bytes.size(); + auto* raw = buffer.release(); + if (QUIC_FAILED(p.api->StreamSend(stream, &raw->quic, 1, QUIC_SEND_FLAG_NONE, raw))) { + delete raw; + const std::scoped_lock lock{p.mutex}; + if (const auto it = p.conns.find(id); it != p.conns.end()) + it->second->outbound_bytes -= std::min(it->second->outbound_bytes, size); + return std::unexpected(SendError::transport); + } + return {}; +} + +void Endpoint::close(ConnectionId id) { + auto& p = *impl_; + HQUIC handle = nullptr; + { + const std::scoped_lock lock{p.mutex}; + if (const auto it = p.conns.find(id); + it != p.conns.end() && Impl::mark(*it->second, CloseCause::local)) + handle = it->second->handle; + } + if (handle != nullptr) + p.shutdown(handle, CloseCause::local); +} + +std::size_t Endpoint::datagram_limit(ConnectionId id) const { + const auto& p = *impl_; + const std::scoped_lock lock{p.mutex}; + const auto it = p.conns.find(id); + return it == p.conns.end() || !it->second->datagrams + ? 0 + : std::min(p.limits.datagram_payload, it->second->datagram_max); +} + +std::uint16_t Endpoint::port() const noexcept { + return impl_->bound_port; +} + +Counters Endpoint::counters() const { + const std::scoped_lock lock{impl_->mutex}; + return impl_->counters; +} +} // namespace evo::network::quic diff --git a/tests/quic_transport.cpp b/tests/quic_transport.cpp new file mode 100644 index 0000000..fde99b8 --- /dev/null +++ b/tests/quic_transport.cpp @@ -0,0 +1,575 @@ +// M10.2b QUIC transport tests (ADR-0090): framing, bounded channels, exact CA pinning, +// and behaviour through the owned impairment relay (loss, duplication, reordering, +// jitter, MTU, corruption, outage), plus malformed peers and limits. +#include "evo/network/quic_transport.hpp" +#include "evo/network/dev_credentials.hpp" +#include "udp_relay.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include +#else +#include +#include +#include +#include +#endif + +namespace { +using namespace evo::network; +using namespace evo::network::quic; +using Clock = std::chrono::steady_clock; +void check(bool ok, std::source_location at = std::source_location::current()) { + if (!ok) + throw std::runtime_error("quic transport line " + std::to_string(at.line())); +} +#define CHECK(...) check(static_cast(__VA_ARGS__)) +constexpr std::string_view name = "dev-region.evolution.test"; + +struct Files { + std::filesystem::path dir; + std::string ca_pem; + std::filesystem::path ca, certificate, key; +}; +Files credentials(const std::string& tag) { + const auto made = generate_dev_credentials(DeploymentEnvironment::dev, name, 1, + std::chrono::system_clock::now()); + CHECK(made.has_value()); + Files f; + f.dir = std::filesystem::temp_directory_path() / ("evo-quic-transport-" + tag); + std::filesystem::remove_all(f.dir); + std::filesystem::create_directories(f.dir); + f.ca_pem = made->authority_pem; + f.ca = f.dir / "ca.pem"; + f.certificate = f.dir / "server.pem"; + f.key = f.dir / "server.key"; + for (const auto& [path, text] : {std::pair{f.ca, made->authority_pem}, + std::pair{f.certificate, made->server_certificate_pem}, + std::pair{f.key, made->server_key_pem}}) { + std::ofstream out{path, std::ios::binary | std::ios::trunc}; + out << text; + CHECK(out.good()); + } + return f; +} +void set_env(const char* key, const std::string& value) { +#ifdef _WIN32 + CHECK(_putenv_s(key, value.c_str()) == 0); +#else + CHECK(value.empty() ? ::unsetenv(key) == 0 : ::setenv(key, value.c_str(), 1) == 0); +#endif +} + +struct Pair { + std::unique_ptr server, client; + std::vector server_events, client_events; + ConnectionId server_id{}, client_id{}; + void pump() { + for (auto& e : server->poll()) + server_events.push_back(std::move(e)); + for (auto& e : client->poll()) + client_events.push_back(std::move(e)); + } + bool until(const std::function& done, + std::chrono::milliseconds limit = std::chrono::milliseconds{10000}) { + const auto end = Clock::now() + limit; + while (Clock::now() < end) { + pump(); + if (done()) + return true; + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + } + pump(); + return done(); + } + static const Event* find(const std::vector& events, EventKind kind) { + const auto it = std::find_if(events.begin(), events.end(), + [&](const Event& e) { return e.kind == kind; }); + return it == events.end() ? nullptr : &*it; + } + bool connected() { + const bool ok = until([&] { + return find(server_events, EventKind::connected) && + find(client_events, EventKind::connected); + }); + if (ok) { + server_id = find(server_events, EventKind::connected)->connection; + client_id = find(client_events, EventKind::connected)->connection; + } + return ok; + } + std::vector> payloads(bool at_server, EventKind kind) const { + std::vector> out; + for (const auto& e : at_server ? server_events : client_events) + if (e.kind == kind) + out.push_back(e.bytes); + return out; + } +}; +std::unique_ptr serve(const Files& f, Limits limits = {}) { + ServerConfig c; + c.certificate_pem_file = f.certificate.string(); + c.key_pem_file = f.key.string(); + c.limits = limits; + auto made = Endpoint::serve(c); + CHECK(made.has_value()); + return std::move(*made); +} +std::unique_ptr dial(const Files& trusted, std::uint16_t port, Limits limits = {}, + std::string server = std::string{name}) { + ClientConfig c; + c.port = port; + c.server_name = std::move(server); + c.authority_pem = trusted.ca_pem; + c.authority_pem_file = trusted.ca.string(); + c.limits = limits; + auto made = Endpoint::connect(c); + CHECK(made.has_value()); + return std::move(*made); +} +std::vector numbered(std::uint32_t n, std::size_t size) { + std::vector v(std::max(size, 4), static_cast(n * 7U)); + v[0] = static_cast(n); + v[1] = static_cast(n >> 8U); + v[2] = static_cast(n >> 16U); + v[3] = static_cast(n >> 24U); + return v; +} +std::uint32_t number(const std::vector& v) { + return static_cast(v[0]) | (static_cast(v[1]) << 8U) | + (static_cast(v[2]) << 16U) | (static_cast(v[3]) << 24U); +} + +void framing() { + std::vector> out; + MessageReassembler r{1024}; + std::vector stream; + std::uint64_t state = 42; + const auto next = [&] { + state = state * 6364136223846793005ULL + 1442695040888963407ULL; + return static_cast(state >> 33U); + }; + std::vector> sent; + for (int i = 0; i < 2000; ++i) { + std::vector m(next() % 1025, static_cast(i)); + const auto framed = frame_message(m); + stream.insert(stream.end(), framed.begin(), framed.end()); + sent.push_back(std::move(m)); + } + std::size_t at = 0; + while (at < stream.size()) { // Random chunking, including 1-byte pieces. + const std::size_t n = std::min(stream.size() - at, 1 + next() % 3000); + CHECK(r.feed({stream.data() + at, n}, out)); + at += n; + } + CHECK(out == sent && r.buffered() == 0); + MessageReassembler small{16}; + std::vector> none; + const auto oversized = frame_message(std::vector(17, 1)); + CHECK(!small.feed(oversized, none) && none.empty()); // The stream is unusable afterwards. + MessageReassembler fresh{16}; + CHECK(fresh.feed(frame_message({}), none) && none.size() == 1 && + none[0].empty()); // Empty messages are valid. +} + +void channels() { + const auto f = credentials("channels"); + Pair p; + Limits server_limits; + server_limits.reliable_message = 1024; + p.server = serve(f, server_limits); + Limits client_limits; + client_limits.outbound_reliable_bytes = std::size_t{8} * 1024; + p.client = dial(f, p.server->port(), client_limits); + CHECK(p.connected()); + // Reliable: 100 ordered messages each way. + for (std::uint32_t i = 0; i < 100; ++i) { + CHECK(p.client->send_message(p.client_id, numbered(i, 32)).has_value() || p.until([&] { + return p.client->send_message(p.client_id, numbered(i, 32)).has_value(); + })); + CHECK(p.server->send_message(p.server_id, numbered(i, 32)).has_value()); + } + CHECK(p.until([&] { + return p.payloads(true, EventKind::message).size() == 100 && + p.payloads(false, EventKind::message).size() == 100; + })); + for (std::uint32_t i = 0; i < 100; ++i) { + CHECK(number(p.payloads(true, EventKind::message)[i]) == i); + CHECK(number(p.payloads(false, EventKind::message)[i]) == i); + } + // Realtime: datagrams both ways; size limits enforced locally. + CHECK(p.until([&] { + return p.client->datagram_limit(p.client_id) > 0 && + p.server->datagram_limit(p.server_id) > 0; + })); + CHECK(p.client->datagram_limit(p.client_id) <= 1200); + CHECK(p.client->send_datagram(p.client_id, numbered(7, 100)).has_value()); + CHECK(p.server->send_datagram(p.server_id, numbered(8, 100)).has_value()); + CHECK(p.until([&] { + return !p.payloads(true, EventKind::datagram).empty() && + !p.payloads(false, EventKind::datagram).empty(); + })); + CHECK(p.client->send_datagram(p.client_id, std::vector(1201, 1)).error() == + SendError::too_large); + CHECK( + p.client + ->send_message(p.client_id, std::vector((std::size_t{64} * 1024) + 1, 1)) + .error() == SendError::too_large); + CHECK(p.client->send_message(p.client_id + 99, numbered(1, 4)).error() == + SendError::unknown_connection); + // Outbound reliable budget (8 KiB): a burst is refused rather than buffered without bound. + bool busy = false; + for (int i = 0; i < 64 && !busy; ++i) + busy = p.client->send_message(p.client_id, std::vector(1000, 2)) + .error_or(SendError::transport) == SendError::busy; + CHECK(busy && p.client->counters().messages_refused_outbound > 0); + // The server's 1 KiB message limit: a larger peer message is a protocol violation. + CHECK(p.until([&] { + return p.client->send_message(p.client_id, std::vector(2000, 3)).has_value(); + })); + CHECK(p.until([&] { + const auto* closed = Pair::find(p.server_events, EventKind::closed); + return closed != nullptr && closed->cause == CloseCause::protocol; + })); + CHECK(p.until([&] { return Pair::find(p.client_events, EventKind::closed) != nullptr; })); + std::filesystem::remove_all(f.dir); +} + +void pinning() { + const auto good = credentials("pin-good"); + const auto attacker = credentials("pin-attacker"); // Same name, another CA. + // OpenSSL would also trust SSL_CERT_FILE; the owned pin must not. + set_env("SSL_CERT_FILE", attacker.ca.string()); + { + Pair p; + p.server = serve(attacker); + p.client = dial(good, p.server->port()); + CHECK(p.until([&] { + const auto* closed = Pair::find(p.client_events, EventKind::closed); + return closed != nullptr && closed->cause == CloseCause::untrusted; + })); + CHECK(Pair::find(p.client_events, EventKind::connected) == nullptr); + CHECK(Pair::find(p.server_events, EventKind::connected) == nullptr); + } + { + // The genuine server still connects with the same environment. + Pair p; + p.server = serve(good); + p.client = dial(good, p.server->port()); + const bool ok = p.connected(); + if (!ok) + if (const auto* c = Pair::find(p.client_events, EventKind::closed)) + std::fprintf(stderr, + "genuine server with SSL_CERT_FILE set: client closed, cause %d\n", + static_cast(c->cause)); + CHECK(ok); + } + set_env("SSL_CERT_FILE", ""); + { + Pair p; + p.server = serve(good); + p.client = dial(good, p.server->port(), {}, "wrong.evolution.test"); + CHECK(p.until([&] { + const auto* closed = Pair::find(p.client_events, EventKind::closed); + return closed != nullptr && closed->cause == CloseCause::untrusted; + })); + } + std::filesystem::remove_all(good.dir); + std::filesystem::remove_all(attacker.dir); +} + +void impaired() { + const auto f = credentials("impaired"); + impair::Impairment rough; + rough.loss = 0.10; + rough.duplicate = 0.10; + rough.reorder = 0.20; + rough.corrupt = 0.05; + rough.delay = std::chrono::milliseconds{5}; + rough.jitter = std::chrono::milliseconds{10}; + Pair p; + p.server = serve(f); + const auto relay = impair::UdpRelay::start(p.server->port(), rough, rough, 0xE70); + CHECK(relay != nullptr); + p.client = dial(f, relay->port()); + CHECK(p.connected()); + constexpr std::uint32_t messages = 200, datagrams = 300; + std::uint32_t sent = 0; + for (std::uint32_t i = 0; i < datagrams; ++i) { + if (!p.client->send_datagram( + p.client_id, + numbered(i, 200))) { /* Busy or too large: dropped like network loss. */ + } // Busy/too large: dropped like loss. + if (sent < messages && p.client->send_message(p.client_id, numbered(sent, 300)).has_value()) + ++sent; + p.pump(); + std::this_thread::sleep_for(std::chrono::milliseconds{2}); + } + CHECK(p.until( + [&] { + while (sent < messages && + p.client->send_message(p.client_id, numbered(sent, 300)).has_value()) + ++sent; + return p.payloads(true, EventKind::message).size() == messages; + }, + std::chrono::milliseconds{30000})); + const auto got = p.payloads(true, EventKind::message); + for (std::uint32_t i = 0; i < messages; ++i) + CHECK(number(got[i]) == i); // Exactly once, in order, despite loss/duplication/reordering. + std::set seen; + for (const auto& d : p.payloads(true, EventKind::datagram)) + CHECK(seen.insert(number(d)).second); // Never duplicated to the application. + CHECK(!seen.empty() && seen.size() < datagrams); // Some lost, none invented. + const auto counters = relay->counters(); + CHECK(counters.dropped > 0 && counters.duplicated > 0 && counters.reordered > 0 && + counters.corrupted > 0); + std::printf("quic impaired: %zu/%u datagrams, %u/%u messages; relay forwarded=%llu " + "dropped=%llu dup=%llu " + "reordered=%llu corrupted=%llu\n", + seen.size(), datagrams, messages, messages, + static_cast(counters.forwarded), + static_cast(counters.dropped), + static_cast(counters.duplicated), + static_cast(counters.reordered), + static_cast(counters.corrupted)); + std::filesystem::remove_all(f.dir); +} + +void mtu() { + const auto f = credentials("mtu"); + impair::Impairment narrow; + narrow.max_payload = 1252; // An IPv6 minimum path (1280) less IP/UDP headers. + Pair p; + p.server = serve(f); + const auto relay = impair::UdpRelay::start(p.server->port(), narrow, narrow, 7); + CHECK(relay != nullptr); + p.client = dial(f, relay->port()); + CHECK(p.connected()); + CHECK(p.until([&] { return p.client->datagram_limit(p.client_id) > 0; })); + // A full 1,200-byte budget datagram crosses a minimum path both ways and keeps + // crossing it (path discovery may probe larger packets; the path drops only those). + CHECK(p.connected()); + const auto limit = p.client->datagram_limit(p.client_id); + CHECK(limit == 1200); + CHECK(p.client->send_datagram(p.client_id, std::vector(limit + 1, 1)).error() == + SendError::too_large); + for (int i = 0; i < 20; ++i) { + if (!p.client->send_datagram( + p.client_id, + std::vector( + 1200, static_cast( + i)))) { /* Busy or too large: dropped like network loss. */ + } + if (!p.server->send_datagram( + p.server_id, + std::vector( + 1200, static_cast( + i)))) { /* Busy or too large: dropped like network loss. */ + } + CHECK(p.client->send_message(p.client_id, numbered(static_cast(i), 1100)) + .has_value()); + p.pump(); + std::this_thread::sleep_for(std::chrono::milliseconds{50}); + } + CHECK(p.until([&] { + return p.payloads(true, EventKind::message).size() == 20 && + !p.payloads(true, EventKind::datagram).empty() && + !p.payloads(false, EventKind::datagram).empty(); + })); + CHECK(p.payloads(true, EventKind::datagram)[0].size() == 1200); + CHECK(Pair::find(p.client_events, EventKind::closed) == nullptr); + std::printf("quic mtu: 1200-byte datagrams over a 1252-byte UDP path (%llu larger probe " + "packets dropped)\n", + static_cast(relay->counters().oversized)); + std::filesystem::remove_all(f.dir); +} + +void outage() { + const auto f = credentials("outage"); + Limits fast; + fast.idle_timeout = std::chrono::milliseconds{1000}; + fast.keep_alive = std::chrono::milliseconds{300}; + Pair p; + p.server = serve(f, fast); + const auto relay = impair::UdpRelay::start(p.server->port(), {}, {}, 9); + CHECK(relay != nullptr); + p.client = dial(f, relay->port(), fast); + CHECK(p.connected()); + impair::Impairment dead; + dead.loss = 1.0; + relay->set(dead, dead); + const auto start = Clock::now(); + CHECK(p.until( + [&] { + const auto* s = Pair::find(p.server_events, EventKind::closed); + const auto* c = Pair::find(p.client_events, EventKind::closed); + return s != nullptr && c != nullptr && s->cause == CloseCause::idle && + c->cause == CloseCause::idle; + }, + std::chrono::milliseconds{8000})); + std::printf( + "quic outage: both sides idle-closed after %lld ms\n", + static_cast( + std::chrono::duration_cast(Clock::now() - start).count())); + std::filesystem::remove_all(f.dir); +} + +void bounds() { + const auto f = credentials("bounds"); + Limits tight; + tight.inbound_datagrams = 4; + tight.inbound_reliable_bytes = 4096; + tight.max_connections = 1; + Pair p; + p.server = serve(f, tight); + p.client = dial(f, p.server->port()); + CHECK(p.until([&] { return Pair::find(p.client_events, EventKind::connected) != nullptr; })); + p.client_id = Pair::find(p.client_events, EventKind::connected)->connection; + CHECK(p.until([&] { + for (auto& e : p.client->poll()) + p.client_events.push_back(std::move(e)); + return p.client->datagram_limit(p.client_id) > 0; + })); + // Datagrams pile up while the server does not poll: only the newest four survive. + for (std::uint32_t i = 0; i < 50; ++i) { + if (!p.client->send_datagram( + p.client_id, numbered(i, 64))) { /* Busy or too large: dropped like network loss. */ + } + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + } + std::this_thread::sleep_for(std::chrono::milliseconds{200}); + auto server_events = p.server->poll(); + std::vector kept; + for (const auto& e : server_events) + if (e.kind == EventKind::datagram) + kept.push_back(number(e.bytes)); + p.server_events.insert(p.server_events.end(), server_events.begin(), server_events.end()); + CHECK(!kept.empty() && kept.size() <= 4 && std::is_sorted(kept.begin(), kept.end())); + CHECK(p.server->counters().datagrams_dropped_inbound > 0); + const auto* accepted = Pair::find(p.server_events, EventKind::connected); + CHECK(accepted != nullptr); + p.server_id = accepted->connection; + // A second client exceeds max_connections and is refused. + auto extra = dial(f, p.server->port()); + bool refused = false; + for (int i = 0; i < 5000 && !refused; ++i) { + for (const auto& e : extra->poll()) + refused = refused || e.kind == EventKind::closed; + (void)p.server->poll(); // Keep the first connection serviced. + std::this_thread::sleep_for(std::chrono::milliseconds{1}); + } + CHECK(refused && p.server->counters().connections_refused > 0); + // Reliable bytes beyond 4 KiB awaiting poll close the connection as overloaded. + for (int i = 0; i < 40; ++i) + if (!p.client->send_message( + p.client_id, std::vector( + 1000, 4))) { /* Busy or too large: dropped like network loss. */ + } + const auto end = Clock::now() + std::chrono::seconds{10}; + const Event* closed = nullptr; + std::vector late; + while (Clock::now() < end && closed == nullptr) { + std::this_thread::sleep_for(std::chrono::milliseconds{50}); // Server lags behind. + for (auto& e : p.server->poll()) + late.push_back(std::move(e)); + closed = Pair::find(late, EventKind::closed); + (void)p.client->poll(); + } + CHECK(closed != nullptr && closed->cause == CloseCause::overloaded); + std::filesystem::remove_all(f.dir); +} + +void garbage_and_close() { + const auto f = credentials("garbage"); + Pair p; + p.server = serve(f); + // Random UDP datagrams at the listener: ignored, nothing surfaces. +#ifdef _WIN32 + using Native = SOCKET; +#else + using Native = int; +#endif + const Native s = ::socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); +#ifdef _WIN32 + if (s == INVALID_SOCKET) + throw std::runtime_error("quic transport: no UDP socket"); +#else + if (s < 0) + throw std::runtime_error("quic transport: no UDP socket"); +#endif + sockaddr_in to{}; + to.sin_family = AF_INET; + to.sin_port = htons(p.server->port()); + to.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + std::uint64_t state = 1; + for (std::size_t i = 0; i < 500; ++i) { + std::vector junk(1 + (i * 37) % 1400); + for (auto& b : junk) { + state = state * 6364136223846793005ULL + 1; + b = static_cast(state >> 56U); + } +#ifdef _WIN32 + const auto length = static_cast(junk.size()); +#else + const auto length = junk.size(); +#endif + (void)::sendto(s, junk.data(), length, 0, reinterpret_cast(&to), + sizeof to); + } +#ifdef _WIN32 + closesocket(s); +#else + ::close(s); +#endif + std::this_thread::sleep_for(std::chrono::milliseconds{200}); + CHECK(p.server->poll().empty()); + // The listener still serves a genuine client; then a clean local close. + p.client = dial(f, p.server->port()); + CHECK(p.connected()); + p.client->close(p.client_id); + CHECK(p.until([&] { + const auto* c = Pair::find(p.client_events, EventKind::closed); + const auto* s2 = Pair::find(p.server_events, EventKind::closed); + return c != nullptr && s2 != nullptr && c->cause == CloseCause::local && + s2->cause == CloseCause::peer; + })); + CHECK(p.client->send_message(p.client_id, numbered(1, 4)).error() == + SendError::unknown_connection); + std::filesystem::remove_all(f.dir); +} +} // namespace + +int main() try { + framing(); + channels(); + pinning(); + impaired(); + mtu(); + outage(); + bounds(); + garbage_and_close(); + std::puts("quic transport tests passed"); + return 0; +} catch (const std::exception& e) { + std::fprintf(stderr, "%s\n", e.what()); + return 1; +} diff --git a/tools/network/impair/udp_relay.cpp b/tools/network/impair/udp_relay.cpp new file mode 100644 index 0000000..12ccaba --- /dev/null +++ b/tools/network/impair/udp_relay.cpp @@ -0,0 +1,252 @@ +#include "udp_relay.hpp" +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include +#else +#include +#include +#include +#include +#include +#include +#endif + +namespace evo::network::impair { +namespace { +#ifdef _WIN32 +using Native = SOCKET; +constexpr Native invalid = INVALID_SOCKET; +void close_native(Native s) { + closesocket(s); +} +bool startup() { + static std::once_flag once; + static bool ok = false; + std::call_once(once, [] { + WSADATA data{}; + ok = WSAStartup(MAKEWORD(2, 2), &data) == 0; + }); + return ok; +} +int wait_readable(std::array& fds, int ms) { + return WSAPoll(fds.data(), static_cast(fds.size()), ms); +} +using Length = int; +#else +using Native = int; +constexpr Native invalid = -1; +void close_native(Native s) { + ::close(s); +} +bool startup() { + return true; +} +int wait_readable(std::array& fds, int ms) { + return ::poll(fds.data(), fds.size(), ms); +} +using Length = std::size_t; +#endif +using Clock = std::chrono::steady_clock; + +struct SplitMix64 { + std::uint64_t state; + std::uint64_t next() { + std::uint64_t z = (state += 0x9E3779B97F4A7C15ULL); + z = (z ^ (z >> 30U)) * 0xBF58476D1CE4E5B9ULL; + z = (z ^ (z >> 27U)) * 0x94D049BB133111EBULL; + return z ^ (z >> 31U); + } + double unit() { + return static_cast(next() >> 11U) * 0x1.0p-53; + } +}; +Native bound_udp(std::uint16_t port) { + const Native s = ::socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + if (s == invalid) + return invalid; + sockaddr_in local{}; + local.sin_family = AF_INET; + local.sin_port = htons(port); + local.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + if (::bind(s, reinterpret_cast(&local), sizeof local) != 0) { + close_native(s); + return invalid; + } + return s; +} +} // namespace + +struct UdpRelay::Impl { + Native client_side{invalid}, server_side{invalid}; + std::uint16_t port{}; + sockaddr_in target{}; + std::optional client; + mutable std::mutex mutex; + Impairment up, down; + RelayCounters counters; + SplitMix64 rng{0}; + std::atomic stop{false}; + std::thread thread; + struct Pending { + Clock::time_point due; + std::uint64_t order; + bool upstream; + std::vector bytes; + bool operator>(const Pending& o) const { + return due != o.due ? due > o.due : order > o.order; + } + }; + std::priority_queue, std::greater<>> queue; + std::uint64_t order{}; + + // Under `mutex`: applies one direction's impairment to a received packet. + void admit(bool upstream, std::vector bytes, Clock::time_point now) { + const auto& rule = upstream ? up : down; + if (bytes.size() > rule.max_payload) { + ++counters.oversized; + return; + } + if (rng.unit() < rule.loss) { + ++counters.dropped; + return; + } + if (!bytes.empty() && rng.unit() < rule.corrupt) { + bytes[rng.next() % bytes.size()] ^= static_cast(1U + (rng.next() % 255U)); + ++counters.corrupted; + } + const auto jitter = + rule.jitter.count() > 0 + ? std::chrono::milliseconds{static_cast( + rng.next() % static_cast(rule.jitter.count() + 1))} + : std::chrono::milliseconds{0}; + auto due = now + rule.delay + jitter; + if (rng.unit() < rule.reorder) { + due += rule.reorder_delay; + ++counters.reordered; + } + const int copies = rng.unit() < rule.duplicate ? 2 : 1; + if (copies == 2) + ++counters.duplicated; + for (int i = 0; i < copies; ++i) + queue.push({due, order++, upstream, bytes}); + } + + void run() { + std::array buffer{}; + while (!stop.load()) { + std::array fds{}; + fds[0].fd = client_side; + fds[0].events = POLLIN; + fds[1].fd = server_side; + fds[1].events = POLLIN; + (void)wait_readable(fds, 1); + const auto now = Clock::now(); + for (int side = 0; side < 2; ++side) { + if ((fds[static_cast(side)].revents & POLLIN) == 0) + continue; + sockaddr_in from{}; + socklen_t from_size = sizeof from; + const auto n = ::recvfrom(side == 0 ? client_side : server_side, + reinterpret_cast(buffer.data()), + static_cast(buffer.size()), 0, + reinterpret_cast(&from), &from_size); + if (n <= 0) + continue; + const std::lock_guard lock{mutex}; + if (side == 0 && !client) + client = from; // The first client owns the relay. + if (side == 0 && (from.sin_port != client->sin_port || + from.sin_addr.s_addr != client->sin_addr.s_addr)) + continue; + admit(side == 0, {buffer.data(), buffer.data() + n}, now); + } + std::vector ready; + { + const std::lock_guard lock{mutex}; + while (!queue.empty() && queue.top().due <= Clock::now()) { + ready.push_back(queue.top()); + queue.pop(); + } + } + for (const auto& p : ready) { + const Native s = p.upstream ? server_side : client_side; + const sockaddr_in* to = p.upstream ? &target : (client ? &*client : nullptr); + if (to == nullptr) + continue; + (void)::sendto(s, reinterpret_cast(p.bytes.data()), + static_cast(p.bytes.size()), 0, + reinterpret_cast(to), sizeof *to); + const std::lock_guard lock{mutex}; + ++counters.forwarded; + } + } + } +}; + +UdpRelay::UdpRelay(std::unique_ptr impl) : impl_{std::move(impl)} {} + +UdpRelay::~UdpRelay() { + impl_->stop.store(true); + if (impl_->thread.joinable()) + impl_->thread.join(); + if (impl_->client_side != invalid) + close_native(impl_->client_side); + if (impl_->server_side != invalid) + close_native(impl_->server_side); +} + +std::unique_ptr UdpRelay::start(std::uint16_t target_port, Impairment upstream, + Impairment downstream, std::uint64_t seed) { + if (!startup() || target_port == 0) + return nullptr; + auto impl = std::make_unique(); + impl->client_side = bound_udp(0); + impl->server_side = bound_udp(0); + if (impl->client_side == invalid || impl->server_side == invalid) + return nullptr; + sockaddr_in local{}; + socklen_t size = sizeof local; + if (::getsockname(impl->client_side, reinterpret_cast(&local), &size) != 0) + return nullptr; + impl->port = ntohs(local.sin_port); + impl->target.sin_family = AF_INET; + impl->target.sin_port = htons(target_port); + impl->target.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + impl->up = upstream; + impl->down = downstream; + impl->rng.state = seed; + std::unique_ptr relay{new UdpRelay{std::move(impl)}}; + auto* raw = relay->impl_.get(); + raw->thread = std::thread{[raw] { raw->run(); }}; + return relay; +} + +std::uint16_t UdpRelay::port() const noexcept { + return impl_->port; +} + +void UdpRelay::set(Impairment upstream, Impairment downstream) { + const std::lock_guard lock{impl_->mutex}; + impl_->up = upstream; + impl_->down = downstream; +} + +RelayCounters UdpRelay::counters() const { + const std::lock_guard lock{impl_->mutex}; + return impl_->counters; +} +} // namespace evo::network::impair diff --git a/tools/network/impair/udp_relay.hpp b/tools/network/impair/udp_relay.hpp new file mode 100644 index 0000000..b72edb8 --- /dev/null +++ b/tools/network/impair/udp_relay.hpp @@ -0,0 +1,43 @@ +#pragma once +// M10.2 owned UDP impairment relay (ADR-0090) for tests and measurements; never shipped. +// Listens on 127.0.0.1: for one client and forwards to 127.0.0.1:, applying +// seeded loss, duplication, reordering, delay/jitter, a payload-size (MTU) limit and +// corruption to each packet in each direction. Impairment decisions come from SplitMix64 +// in packet order; wall-clock delivery times are not deterministic. Runs its own thread. +#include +#include +#include + +namespace evo::network::impair { +struct Impairment { + double loss{}; // Probability a packet is dropped. + double duplicate{}; // Probability a packet is sent twice. + double reorder{}; // Probability a packet is held back by `reorder_delay`. + double corrupt{}; // Probability one byte is flipped. + std::chrono::milliseconds delay{}, jitter{}, reorder_delay{10}; + std::size_t max_payload{65507}; // Larger UDP payloads are dropped (path MTU). +}; +struct RelayCounters { + std::uint64_t forwarded{}, dropped{}, duplicated{}, reordered{}, corrupted{}, oversized{}; +}; +class UdpRelay { + public: + // Upstream (client -> server) and downstream (server -> client) impairments. + [[nodiscard]] static std::unique_ptr start(std::uint16_t target_port, + Impairment upstream, Impairment downstream, + std::uint64_t seed); + UdpRelay(const UdpRelay&) = delete; + UdpRelay& operator=(const UdpRelay&) = delete; + ~UdpRelay(); + [[nodiscard]] std::uint16_t port() const noexcept; + // Changes impairments while running (e.g. a total outage to exercise idle timeout). + void set(Impairment upstream, Impairment downstream); + [[nodiscard]] RelayCounters counters() const; + + struct Impl; + + private: + explicit UdpRelay(std::unique_ptr impl); + std::unique_ptr impl_; +}; +} // namespace evo::network::impair