Add M10.2b bounded QUIC channels with exact CA pinning
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 1m47s
Foundation / checks (network) (push) Successful in 8m11s
Foundation / checks (client) (push) Successful in 12m20s
Foundation / checks (tsan) (push) Successful in 7m44s
Foundation / checks (checks) (push) Successful in 31m12s
Foundation / checks (postgres) (push) Successful in 33m27s
Foundation / image (push) Successful in 2m14s
Foundation / postgres-tools (push) Successful in 6m24s
Foundation / server-image (push) Successful in 18m6s

evo_network_quic wraps msquic in a single-owner Endpoint: newest-wins datagram
channel and one length-framed reliable stream, both bounded, with explicit close
causes. The client pins exactly one CA and name with an owned Mbed TLS check that
is the sole authority (OpenSSL's verdict advisory), callbacks never call msquic
under the lock, and MinimumMtu starts at 1280 so 1,200-byte datagrams fit minimum
paths. An owned seeded UDP impairment relay drives loss, duplication, reordering,
corruption, MTU and outage tests, passing on MSVC, GCC and Clang ASan/UBSan.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-03 16:11:59 -04:00
co-authored by Claude Opus 5.5
parent 3c421cd0cf
commit ca762bff75
12 changed files with 1844 additions and 6 deletions
+744
View File
@@ -0,0 +1,744 @@
// M10.2b QUIC endpoint over msquic (ADR-0090). Threading: msquic invokes the callbacks
// below on its worker threads; they only touch `Impl` under `mutex` and append to bounded
// queues. msquic may run callbacks inline from API calls, so no msquic function is ever
// called while `mutex` is held: decisions are made under the lock, calls after it.
// Every msquic handle is closed by the owning thread (poll()/~Endpoint), so a handle used
// by send_*() or certificate completion is never freed underneath it. Mbed TLS (exact
// pinning) is only used on the owning thread.
#ifdef _WIN32
#ifndef NOMINMAX
#define NOMINMAX // msquic.h includes <windows.h>; keep std::min/std::max usable.
#endif
#endif
#include "evo/network/quic_transport.hpp"
#include <algorithm>
#include <array>
#include <condition_variable>
#include <cstring>
#include <deque>
#include <map>
#include <mbedtls/x509_crt.h>
#include <msquic.h>
#include <mutex>
#include <optional>
#include <psa/crypto.h>
#include <utility>
namespace evo::network::quic {
namespace {
constexpr std::string_view alpn_text = "evo-m10";
constexpr std::uint32_t frame_header = 4;
std::uint32_t read_u32(const std::uint8_t* p) {
return static_cast<std::uint32_t>(p[0]) | (static_cast<std::uint32_t>(p[1]) << 8U) |
(static_cast<std::uint32_t>(p[2]) << 16U) | (static_cast<std::uint32_t>(p[3]) << 24U);
}
bool tls_or_certificate_failure(QUIC_STATUS status) {
const auto value = static_cast<std::uint32_t>(status);
const auto alert_base = static_cast<std::uint32_t>(QUIC_STATUS_TLS_ALERT(0));
return (value & ~0xffU) == alert_base || status == QUIC_STATUS_CERT_EXPIRED ||
status == QUIC_STATUS_CERT_UNTRUSTED_ROOT || status == QUIC_STATUS_CERT_NO_CERT;
}
// Leaf (DER) verified against exactly one CA (PEM) for one name; nothing else trusted.
bool pinned(std::span<const std::uint8_t> leaf, const std::string& authority_pem,
const std::string& name) {
if (psa_crypto_init() != PSA_SUCCESS || leaf.empty())
return false;
mbedtls_x509_crt ca;
mbedtls_x509_crt peer;
mbedtls_x509_crt_init(&ca);
mbedtls_x509_crt_init(&peer);
std::vector<unsigned char> pem(authority_pem.begin(), authority_pem.end());
pem.push_back('\0');
std::uint32_t flags = 0;
const bool ok =
mbedtls_x509_crt_parse(&ca, pem.data(), pem.size()) == 0 &&
mbedtls_x509_crt_parse_der(&peer, leaf.data(), leaf.size()) == 0 &&
mbedtls_x509_crt_verify(&peer, &ca, nullptr, name.c_str(), &flags, nullptr, nullptr) == 0 &&
flags == 0;
mbedtls_x509_crt_free(&peer);
mbedtls_x509_crt_free(&ca);
return ok;
}
} // namespace
bool MessageReassembler::feed(std::span<const std::uint8_t> bytes,
std::vector<std::vector<std::uint8_t>>& out) {
buffer_.insert(buffer_.end(), bytes.begin(), bytes.end());
std::size_t at = 0;
while (buffer_.size() - at >= frame_header) {
const std::size_t length = read_u32(buffer_.data() + at);
if (length > max_)
return false;
if (buffer_.size() - at - frame_header < length)
break;
const auto* begin = buffer_.data() + at + frame_header;
out.emplace_back(begin, begin + length);
at += frame_header + length;
}
buffer_.erase(buffer_.begin(), buffer_.begin() + static_cast<std::ptrdiff_t>(at));
return true;
}
std::vector<std::uint8_t> frame_message(std::span<const std::uint8_t> payload) {
const auto n = static_cast<std::uint32_t>(payload.size());
std::vector<std::uint8_t> out{static_cast<std::uint8_t>(n), static_cast<std::uint8_t>(n >> 8U),
static_cast<std::uint8_t>(n >> 16U),
static_cast<std::uint8_t>(n >> 24U)};
out.insert(out.end(), payload.begin(), payload.end());
return out;
}
struct SendBuffer {
QUIC_BUFFER quic{};
std::vector<std::uint8_t> bytes;
ConnectionId connection{};
};
struct Endpoint::Impl {
struct Conn {
Impl* owner{};
ConnectionId id{};
HQUIC handle{}, stream{};
bool handshake{}, connected{}, shutdown_complete{}, stream_complete{}, datagrams{};
std::size_t datagram_max{};
std::size_t inbound_datagrams{}, inbound_bytes{};
std::size_t outbound_datagrams{}, outbound_bytes{};
std::optional<CloseCause> cause;
MessageReassembler reassembler{0};
};
struct PendingCertificate {
ConnectionId id{};
std::vector<std::uint8_t> leaf;
bool openssl_ok{};
};
const QUIC_API_TABLE* api{};
HQUIC registration{}, configuration{}, listener{};
bool is_server{};
Limits limits;
std::string authority_pem, server_name;
std::uint16_t bound_port{};
std::array<std::uint8_t, alpn_text.size()> alpn_bytes{};
QUIC_BUFFER alpn{};
mutable std::mutex mutex;
std::condition_variable changed;
std::map<ConnectionId, std::unique_ptr<Conn>> conns;
ConnectionId next_id{1};
std::deque<Event> events;
std::deque<PendingCertificate> certificates;
Counters counters;
// Under `mutex`. Datagrams: newest wins per connection; other events refuse to drop
// and instead mark the connection overloaded (returns true: the caller shuts it down
// after releasing the lock).
[[nodiscard]] bool push(Conn& c, Event e) {
if (e.kind == EventKind::datagram) {
if (c.inbound_datagrams >= limits.inbound_datagrams) {
const auto oldest = std::find_if(events.begin(), events.end(), [&](const Event& x) {
return x.kind == EventKind::datagram && x.connection == c.id;
});
if (oldest != events.end()) {
events.erase(oldest);
--c.inbound_datagrams;
}
++counters.datagrams_dropped_inbound;
}
if (events.size() >= limits.max_events) {
++counters.datagrams_dropped_inbound;
return false;
}
++c.inbound_datagrams;
} else if (e.kind != EventKind::closed && events.size() >= limits.max_events) {
++counters.events_dropped;
return mark(c, CloseCause::overloaded);
}
events.push_back(std::move(e));
changed.notify_all();
return false;
}
// Under `mutex`: records the first cause; true when the caller must shut down.
static bool mark(Conn& c, CloseCause cause) {
if (!c.cause)
c.cause = cause;
return !c.shutdown_complete;
}
// Without `mutex`: asynchronous close; the outcome arrives as SHUTDOWN_COMPLETE.
void shutdown(HQUIC handle, CloseCause cause) const {
api->ConnectionShutdown(handle, QUIC_CONNECTION_SHUTDOWN_FLAG_NONE,
static_cast<QUIC_UINT62>(cause) + 1);
}
static QUIC_STATUS QUIC_API on_stream(HQUIC /*stream*/, void* context,
QUIC_STREAM_EVENT* event) {
auto& c = *static_cast<Conn*>(context);
auto& self = *c.owner;
std::optional<CloseCause> close;
HQUIC handle = nullptr;
{
const std::scoped_lock lock{self.mutex};
handle = c.handle;
close = self.stream_event(c, event);
}
if (close)
self.shutdown(handle, *close);
return QUIC_STATUS_SUCCESS;
}
// Under `mutex`; returns a cause when the connection must be shut down.
std::optional<CloseCause> stream_event(Conn& c, QUIC_STREAM_EVENT* event) {
auto& self = *this;
switch (event->Type) {
case QUIC_STREAM_EVENT_RECEIVE: {
std::vector<std::vector<std::uint8_t>> messages;
for (std::uint32_t i = 0; i < event->RECEIVE.BufferCount; ++i) {
const auto& b = event->RECEIVE.Buffers[i];
c.inbound_bytes += b.Length;
if (!c.reassembler.feed({b.Buffer, b.Length}, messages))
return mark(c, CloseCause::protocol) ? std::optional{CloseCause::protocol}
: std::nullopt;
}
if (c.inbound_bytes > self.limits.inbound_reliable_bytes)
return mark(c, CloseCause::overloaded) ? std::optional{CloseCause::overloaded}
: std::nullopt;
for (auto& m : messages)
if (self.push(c, {EventKind::message, c.id, std::move(m), {}}))
return CloseCause::overloaded;
break;
}
case QUIC_STREAM_EVENT_SEND_COMPLETE: {
std::unique_ptr<SendBuffer> done{
static_cast<SendBuffer*>(event->SEND_COMPLETE.ClientContext)};
if (done)
c.outbound_bytes -= std::min(c.outbound_bytes, done->bytes.size());
break;
}
case QUIC_STREAM_EVENT_PEER_SEND_SHUTDOWN:
case QUIC_STREAM_EVENT_PEER_SEND_ABORTED:
case QUIC_STREAM_EVENT_PEER_RECEIVE_ABORTED:
// The reliable channel lives exactly as long as the connection.
return mark(c, CloseCause::peer) ? std::optional{CloseCause::peer} : std::nullopt;
case QUIC_STREAM_EVENT_SHUTDOWN_COMPLETE:
c.stream_complete = true;
self.changed.notify_all();
break;
default:
break;
}
return std::nullopt;
}
static QUIC_STATUS QUIC_API on_connection(HQUIC handle, void* context,
QUIC_CONNECTION_EVENT* event) {
auto& c = *static_cast<Conn*>(context);
auto& self = *c.owner;
if (event->Type == QUIC_CONNECTION_EVENT_CONNECTED && !self.is_server) {
// The client opens the one reliable stream and announces it immediately; these
// calls may run stream callbacks inline, so the lock is not held.
HQUIC stream = nullptr;
const bool opened = QUIC_SUCCEEDED(
self.api->StreamOpen(handle, QUIC_STREAM_OPEN_FLAG_NONE, on_stream, &c, &stream));
if (opened) {
const std::scoped_lock lock{self.mutex};
c.stream = stream;
}
if (!opened ||
QUIC_FAILED(self.api->StreamStart(stream, QUIC_STREAM_START_FLAG_IMMEDIATE))) {
{
const std::scoped_lock lock{self.mutex};
(void)mark(c, CloseCause::transport);
}
self.shutdown(handle, CloseCause::transport);
return QUIC_STATUS_SUCCESS;
}
}
if (event->Type == QUIC_CONNECTION_EVENT_PEER_STREAM_STARTED) {
bool accept = false;
{
const std::scoped_lock lock{self.mutex};
accept = self.is_server && c.stream == nullptr;
if (accept)
c.stream = event->PEER_STREAM_STARTED.Stream;
}
if (!accept) {
// Exactly one client-opened stream is allowed.
self.api->StreamShutdown(event->PEER_STREAM_STARTED.Stream,
QUIC_STREAM_SHUTDOWN_FLAG_ABORT, 1);
{
const std::scoped_lock lock{self.mutex};
(void)mark(c, CloseCause::protocol);
}
self.shutdown(handle, CloseCause::protocol);
return QUIC_STATUS_SUCCESS;
}
self.api->SetCallbackHandler(event->PEER_STREAM_STARTED.Stream,
reinterpret_cast<void*>(on_stream), &c);
}
std::optional<CloseCause> close;
QUIC_STATUS result = QUIC_STATUS_SUCCESS;
{
const std::scoped_lock lock{self.mutex};
close = self.connection_event(c, event, result);
}
if (close)
self.shutdown(handle, *close);
return result;
}
// Under `mutex`; returns a cause when the connection must be shut down.
std::optional<CloseCause> connection_event(Conn& c, QUIC_CONNECTION_EVENT* event,
QUIC_STATUS& result) {
auto& self = *this;
const auto connected = [&]() -> std::optional<CloseCause> {
c.connected = true;
if (self.push(c, {EventKind::connected, c.id, {}, {}}))
return CloseCause::overloaded;
return std::nullopt;
};
switch (event->Type) {
case QUIC_CONNECTION_EVENT_PEER_CERTIFICATE_RECEIVED: {
// Client only: copied and checked on the owning thread (exact pinning).
const auto* leaf =
static_cast<const QUIC_BUFFER*>(event->PEER_CERTIFICATE_RECEIVED.Certificate);
PendingCertificate p{
c.id, {}, QUIC_SUCCEEDED(event->PEER_CERTIFICATE_RECEIVED.DeferredStatus)};
if (leaf != nullptr && leaf->Buffer != nullptr)
p.leaf.assign(leaf->Buffer, leaf->Buffer + leaf->Length);
self.certificates.push_back(std::move(p));
self.changed.notify_all();
result = QUIC_STATUS_PENDING;
break;
}
case QUIC_CONNECTION_EVENT_CONNECTED:
c.handshake = true;
// Client: the stream is open (above). Server: wait for the client's stream.
if (!self.is_server || c.stream != nullptr)
return connected();
break;
case QUIC_CONNECTION_EVENT_PEER_STREAM_STARTED:
if (c.handshake && !c.connected)
return connected();
break;
case QUIC_CONNECTION_EVENT_DATAGRAM_STATE_CHANGED:
c.datagrams = event->DATAGRAM_STATE_CHANGED.SendEnabled != 0;
c.datagram_max = event->DATAGRAM_STATE_CHANGED.MaxSendLength;
break;
case QUIC_CONNECTION_EVENT_DATAGRAM_RECEIVED: {
const auto* b = event->DATAGRAM_RECEIVED.Buffer;
if (b->Length <= self.limits.datagram_payload)
(void)self.push(
c, {EventKind::datagram, c.id, {b->Buffer, b->Buffer + b->Length}, {}});
break;
}
case QUIC_CONNECTION_EVENT_DATAGRAM_SEND_STATE_CHANGED:
if (QUIC_DATAGRAM_SEND_STATE_IS_FINAL(event->DATAGRAM_SEND_STATE_CHANGED.State)) {
delete static_cast<SendBuffer*>(event->DATAGRAM_SEND_STATE_CHANGED.ClientContext);
c.outbound_datagrams -= std::min<std::size_t>(c.outbound_datagrams, 1);
}
break;
case QUIC_CONNECTION_EVENT_SHUTDOWN_INITIATED_BY_TRANSPORT: {
const auto status = event->SHUTDOWN_INITIATED_BY_TRANSPORT.Status;
if (!c.cause)
c.cause = status == QUIC_STATUS_CONNECTION_IDLE ? CloseCause::idle
: tls_or_certificate_failure(status) ? CloseCause::untrusted
: CloseCause::transport;
break;
}
case QUIC_CONNECTION_EVENT_SHUTDOWN_INITIATED_BY_PEER:
if (!c.cause)
c.cause = CloseCause::peer;
break;
case QUIC_CONNECTION_EVENT_SHUTDOWN_COMPLETE:
c.shutdown_complete = true;
c.connected = false;
(void)self.push(c,
{EventKind::closed, c.id, {}, c.cause.value_or(CloseCause::transport)});
break;
default:
break;
}
return std::nullopt;
}
static QUIC_STATUS QUIC_API on_listener(HQUIC /*listener*/, void* context,
QUIC_LISTENER_EVENT* event) {
if (event->Type != QUIC_LISTENER_EVENT_NEW_CONNECTION)
return QUIC_STATUS_SUCCESS;
auto& self = *static_cast<Impl*>(context);
Conn* c = nullptr;
{
const std::scoped_lock lock{self.mutex};
if (self.conns.size() >= self.limits.max_connections) {
++self.counters.connections_refused;
return QUIC_STATUS_CONNECTION_REFUSED;
}
c = self.adopt(event->NEW_CONNECTION.Connection);
}
self.api->SetCallbackHandler(c->handle, reinterpret_cast<void*>(on_connection), c);
return self.api->ConnectionSetConfiguration(c->handle, self.configuration);
}
// Under `mutex`.
Conn* adopt(HQUIC handle) {
auto c = std::make_unique<Conn>();
c->owner = this;
c->id = next_id++;
c->handle = handle;
c->reassembler = MessageReassembler{limits.reliable_message};
auto* raw = c.get();
conns.emplace(raw->id, std::move(c));
return raw;
}
QUIC_SETTINGS settings() const {
QUIC_SETTINGS s{};
s.IdleTimeoutMs = static_cast<std::uint64_t>(limits.idle_timeout.count());
s.IsSet.IdleTimeoutMs = TRUE;
s.HandshakeIdleTimeoutMs = static_cast<std::uint64_t>(limits.handshake_timeout.count());
s.IsSet.HandshakeIdleTimeoutMs = TRUE;
s.KeepAliveIntervalMs = static_cast<std::uint32_t>(limits.keep_alive.count());
s.IsSet.KeepAliveIntervalMs = TRUE;
s.DatagramReceiveEnabled = TRUE;
s.IsSet.DatagramReceiveEnabled = TRUE;
// msquic otherwise assumes a 1288-byte IP MTU (1260-byte IPv4 UDP payloads), which
// a compliant 1280-byte path drops. Start at the IPv6 minimum path MTU and only
// probe upward; the 1,200-byte datagram budget (ADR-0089) fits it.
s.MinimumMtu = 1280;
s.IsSet.MinimumMtu = TRUE;
s.PeerBidiStreamCount = is_server ? 1 : 0; // Only the client opens the reliable stream.
s.IsSet.PeerBidiStreamCount = TRUE;
s.PeerUnidiStreamCount = 0;
s.IsSet.PeerUnidiStreamCount = TRUE;
// Flow control bounds what a peer may push before poll() drains it.
s.StreamRecvWindowDefault =
static_cast<std::uint32_t>(std::max<std::size_t>(limits.inbound_reliable_bytes, 4096));
s.IsSet.StreamRecvWindowDefault = TRUE;
s.ConnFlowControlWindow =
static_cast<std::uint32_t>(std::max<std::size_t>(limits.inbound_reliable_bytes, 4096));
s.IsSet.ConnFlowControlWindow = TRUE;
return s;
}
bool open(bool server) {
is_server = server;
std::copy(alpn_text.begin(), alpn_text.end(), alpn_bytes.begin());
alpn = {static_cast<std::uint32_t>(alpn_bytes.size()), alpn_bytes.data()};
if (QUIC_FAILED(MsQuicOpen2(&api)))
return false;
const QUIC_REGISTRATION_CONFIG reg{"aevolution", QUIC_EXECUTION_PROFILE_LOW_LATENCY};
if (QUIC_FAILED(api->RegistrationOpen(&reg, &registration)))
return false;
const auto s = settings();
return QUIC_SUCCEEDED(
api->ConfigurationOpen(registration, &alpn, 1, &s, sizeof(s), nullptr, &configuration));
}
// Owning thread: completes pending certificate checks outside the lock.
void verify_certificates() {
std::deque<PendingCertificate> work;
{
const std::scoped_lock lock{mutex};
work.swap(certificates);
}
for (const auto& p : work) {
// The owned check is the sole authority: OpenSSL's verdict (which its default
// paths and environment can influence either way) is advisory only.
const bool ok = pinned(p.leaf, authority_pem, server_name);
if (!p.openssl_ok && ok) {
const std::scoped_lock lock{mutex};
++counters.openssl_disagreements;
}
HQUIC handle = nullptr;
{
const std::scoped_lock lock{mutex};
const auto it = conns.find(p.id);
if (it == conns.end())
continue;
handle = it->second->handle;
if (!ok && !it->second->cause)
it->second->cause = CloseCause::untrusted;
}
api->ConnectionCertificateValidationComplete(handle, ok ? TRUE : FALSE,
ok ? QUIC_TLS_ALERT_CODE_SUCCESS
: QUIC_TLS_ALERT_CODE_BAD_CERTIFICATE);
}
}
// Owning thread: closes msquic handles of finished connections.
void release(std::unique_ptr<Conn> c) {
if (c->stream != nullptr)
api->StreamClose(c->stream);
api->ConnectionClose(c->handle);
}
void shutdown_all() {
std::vector<std::unique_ptr<Conn>> all;
std::vector<HQUIC> open, pending;
{
const std::scoped_lock lock{mutex};
for (auto& [id, c] : conns)
if (!c->shutdown_complete)
open.push_back(c->handle);
for (const auto& p : certificates)
if (const auto it = conns.find(p.id); it != conns.end())
pending.push_back(it->second->handle);
certificates.clear();
}
// Pending certificate checks would hold a connection open; refuse them first.
for (auto* h : pending)
api->ConnectionCertificateValidationComplete(h, FALSE,
QUIC_TLS_ALERT_CODE_BAD_CERTIFICATE);
for (auto* h : open)
api->ConnectionShutdown(h, QUIC_CONNECTION_SHUTDOWN_FLAG_SILENT, 0);
{
std::unique_lock lock{mutex};
changed.wait_for(lock, std::chrono::seconds{10}, [&] {
return std::all_of(conns.begin(), conns.end(), [](const auto& e) {
return e.second->shutdown_complete &&
(e.second->stream == nullptr || e.second->stream_complete);
});
});
for (auto& [id, c] : conns)
all.push_back(std::move(c));
conns.clear();
}
for (auto& c : all)
release(std::move(c));
}
};
Endpoint::Endpoint(std::unique_ptr<Impl> impl) : impl_{std::move(impl)} {}
Endpoint::~Endpoint() {
auto& p = *impl_;
if (p.api == nullptr)
return;
if (p.listener != nullptr)
p.api->ListenerStop(p.listener);
p.shutdown_all();
if (p.listener != nullptr)
p.api->ListenerClose(p.listener);
if (p.configuration != nullptr)
p.api->ConfigurationClose(p.configuration);
if (p.registration != nullptr)
p.api->RegistrationClose(p.registration);
MsQuicClose(p.api);
}
std::expected<std::unique_ptr<Endpoint>, EndpointError>
Endpoint::serve(const ServerConfig& config) {
if (config.certificate_pem_file.empty() || config.key_pem_file.empty() ||
config.limits.max_connections == 0)
return std::unexpected(EndpointError::configuration);
auto impl = std::make_unique<Impl>();
impl->limits = config.limits;
std::unique_ptr<Endpoint> endpoint{new Endpoint{std::move(impl)}};
auto& p = *endpoint->impl_;
if (!p.open(true))
return std::unexpected(EndpointError::library);
QUIC_CERTIFICATE_FILE file{config.key_pem_file.c_str(), config.certificate_pem_file.c_str()};
QUIC_CREDENTIAL_CONFIG credential{};
credential.Type = QUIC_CREDENTIAL_TYPE_CERTIFICATE_FILE;
credential.CertificateFile = &file;
if (QUIC_FAILED(p.api->ConfigurationLoadCredential(p.configuration, &credential)))
return std::unexpected(EndpointError::credentials);
QUIC_ADDR address{};
if (!QuicAddrFromString(config.listen_address.c_str(), config.port, &address) ||
QUIC_FAILED(p.api->ListenerOpen(p.registration, Impl::on_listener, &p, &p.listener)) ||
QUIC_FAILED(p.api->ListenerStart(p.listener, &p.alpn, 1, &address)))
return std::unexpected(EndpointError::listen);
std::uint32_t size = sizeof(address);
if (QUIC_FAILED(
p.api->GetParam(p.listener, QUIC_PARAM_LISTENER_LOCAL_ADDRESS, &size, &address)))
return std::unexpected(EndpointError::listen);
p.bound_port = QuicAddrGetPort(&address);
return endpoint;
}
std::expected<std::unique_ptr<Endpoint>, EndpointError>
Endpoint::connect(const ClientConfig& config) {
if (config.server_name.empty() || config.authority_pem.empty() ||
config.authority_pem_file.empty() || config.port == 0)
return std::unexpected(EndpointError::configuration);
auto impl = std::make_unique<Impl>();
impl->limits = config.limits;
impl->authority_pem = config.authority_pem;
impl->server_name = config.server_name;
std::unique_ptr<Endpoint> endpoint{new Endpoint{std::move(impl)}};
auto& p = *endpoint->impl_;
if (!p.open(false))
return std::unexpected(EndpointError::library);
QUIC_CREDENTIAL_CONFIG credential{};
credential.Type = QUIC_CREDENTIAL_TYPE_NONE;
// OpenSSL's verifier everywhere, its result deferred to us, the chain in portable DER
// form: the connection completes only after the owned exact-pinning check.
constexpr auto flags =
static_cast<std::uint32_t>(QUIC_CREDENTIAL_FLAG_CLIENT) |
static_cast<std::uint32_t>(QUIC_CREDENTIAL_FLAG_SET_CA_CERTIFICATE_FILE) |
static_cast<std::uint32_t>(QUIC_CREDENTIAL_FLAG_USE_TLS_BUILTIN_CERTIFICATE_VALIDATION) |
static_cast<std::uint32_t>(QUIC_CREDENTIAL_FLAG_INDICATE_CERTIFICATE_RECEIVED) |
static_cast<std::uint32_t>(QUIC_CREDENTIAL_FLAG_DEFER_CERTIFICATE_VALIDATION) |
static_cast<std::uint32_t>(QUIC_CREDENTIAL_FLAG_USE_PORTABLE_CERTIFICATES);
credential.Flags = static_cast<QUIC_CREDENTIAL_FLAGS>(
flags); // NOLINT(clang-analyzer-optin.core.EnumCastOutOfRange)
credential.CaCertificateFile = config.authority_pem_file.c_str();
if (QUIC_FAILED(p.api->ConfigurationLoadCredential(p.configuration, &credential)))
return std::unexpected(EndpointError::credentials);
HQUIC handle = nullptr;
Impl::Conn* c = nullptr;
if (QUIC_FAILED(p.api->ConnectionOpen(p.registration, Impl::on_connection, nullptr, &handle)))
return std::unexpected(EndpointError::library);
{
const std::scoped_lock lock{p.mutex};
c = p.adopt(handle);
}
p.api->SetCallbackHandler(handle, reinterpret_cast<void*>(Impl::on_connection), c);
QUIC_ADDR remote{};
if (!QuicAddrFromString(config.server_address.c_str(), config.port, &remote) ||
QUIC_FAILED(
p.api->SetParam(handle, QUIC_PARAM_CONN_REMOTE_ADDRESS, sizeof(remote), &remote)) ||
QUIC_FAILED(p.api->ConnectionStart(handle, p.configuration, QUIC_ADDRESS_FAMILY_UNSPEC,
config.server_name.c_str(), config.port)))
return std::unexpected(EndpointError::configuration);
p.bound_port = config.port;
return endpoint;
}
std::vector<Event> Endpoint::poll() {
auto& p = *impl_;
p.verify_certificates();
std::vector<Event> out;
std::vector<std::unique_ptr<Impl::Conn>> finished;
{
const std::scoped_lock lock{p.mutex};
while (!p.events.empty() && out.size() < p.limits.max_events) {
auto e = std::move(p.events.front());
p.events.pop_front();
if (const auto it = p.conns.find(e.connection); it != p.conns.end()) {
auto& c = *it->second;
if (e.kind == EventKind::datagram)
--c.inbound_datagrams;
else if (e.kind == EventKind::message)
c.inbound_bytes -= std::min(c.inbound_bytes, e.bytes.size() + frame_header);
else if (e.kind == EventKind::closed &&
(c.stream == nullptr || c.stream_complete || c.shutdown_complete)) {
finished.push_back(std::move(it->second));
p.conns.erase(it);
}
}
out.push_back(std::move(e));
}
}
for (auto& c : finished)
p.release(std::move(c));
return out;
}
std::expected<void, SendError> Endpoint::send_datagram(ConnectionId id,
std::span<const std::uint8_t> bytes) {
auto& p = *impl_;
auto buffer = std::make_unique<SendBuffer>();
HQUIC handle = nullptr;
{
const std::scoped_lock lock{p.mutex};
const auto it = p.conns.find(id);
if (it == p.conns.end())
return std::unexpected(SendError::unknown_connection);
auto& c = *it->second;
if (!c.connected || !c.datagrams)
return std::unexpected(SendError::not_connected);
if (bytes.size() > std::min(p.limits.datagram_payload, c.datagram_max)) {
++p.counters.datagrams_refused_outbound;
return std::unexpected(SendError::too_large);
}
if (c.outbound_datagrams >= p.limits.outbound_datagrams) {
++p.counters.datagrams_refused_outbound;
return std::unexpected(SendError::busy);
}
++c.outbound_datagrams;
handle = c.handle;
}
buffer->bytes.assign(bytes.begin(), bytes.end());
buffer->connection = id;
buffer->quic = {static_cast<std::uint32_t>(buffer->bytes.size()), buffer->bytes.data()};
auto* raw = buffer.release();
if (QUIC_FAILED(p.api->DatagramSend(handle, &raw->quic, 1, QUIC_SEND_FLAG_NONE, raw))) {
delete raw;
const std::scoped_lock lock{p.mutex};
if (const auto it = p.conns.find(id); it != p.conns.end())
it->second->outbound_datagrams -=
std::min<std::size_t>(it->second->outbound_datagrams, 1);
return std::unexpected(SendError::transport);
}
return {};
}
std::expected<void, SendError> Endpoint::send_message(ConnectionId id,
std::span<const std::uint8_t> bytes) {
auto& p = *impl_;
if (bytes.size() > p.limits.reliable_message) {
const std::scoped_lock lock{p.mutex};
++p.counters.messages_refused_outbound;
return std::unexpected(SendError::too_large);
}
auto buffer = std::make_unique<SendBuffer>();
buffer->bytes = frame_message(bytes);
buffer->connection = id;
buffer->quic = {static_cast<std::uint32_t>(buffer->bytes.size()), buffer->bytes.data()};
HQUIC stream = nullptr;
{
const std::scoped_lock lock{p.mutex};
const auto it = p.conns.find(id);
if (it == p.conns.end())
return std::unexpected(SendError::unknown_connection);
auto& c = *it->second;
if (!c.connected || c.stream == nullptr)
return std::unexpected(SendError::not_connected);
if (c.outbound_bytes + buffer->bytes.size() > p.limits.outbound_reliable_bytes) {
++p.counters.messages_refused_outbound;
return std::unexpected(SendError::busy);
}
c.outbound_bytes += buffer->bytes.size();
stream = c.stream;
}
const auto size = buffer->bytes.size();
auto* raw = buffer.release();
if (QUIC_FAILED(p.api->StreamSend(stream, &raw->quic, 1, QUIC_SEND_FLAG_NONE, raw))) {
delete raw;
const std::scoped_lock lock{p.mutex};
if (const auto it = p.conns.find(id); it != p.conns.end())
it->second->outbound_bytes -= std::min(it->second->outbound_bytes, size);
return std::unexpected(SendError::transport);
}
return {};
}
void Endpoint::close(ConnectionId id) {
auto& p = *impl_;
HQUIC handle = nullptr;
{
const std::scoped_lock lock{p.mutex};
if (const auto it = p.conns.find(id);
it != p.conns.end() && Impl::mark(*it->second, CloseCause::local))
handle = it->second->handle;
}
if (handle != nullptr)
p.shutdown(handle, CloseCause::local);
}
std::size_t Endpoint::datagram_limit(ConnectionId id) const {
const auto& p = *impl_;
const std::scoped_lock lock{p.mutex};
const auto it = p.conns.find(id);
return it == p.conns.end() || !it->second->datagrams
? 0
: std::min(p.limits.datagram_payload, it->second->datagram_max);
}
std::uint16_t Endpoint::port() const noexcept {
return impl_->bound_port;
}
Counters Endpoint::counters() const {
const std::scoped_lock lock{impl_->mutex};
return impl_->counters;
}
} // namespace evo::network::quic