Fix M9.4-M9.7 review findings in the dev session stack

- TLS: replace send() with a bounded queue/flush; an interrupted record is
  retried with identical arguments and later frames wait behind it, so no
  queued frame is lost or reordered; close() flushes first.
- Carriers: server-issued 'I','N' | 48-bit epoch | 64-bit ordinal ids
  replace the aliasing low-32-bit packing; client sequences are kept
  separately for replies; out-of-range epochs are refused, never truncated.
- Grants: a durable, fsynced, fail-closed consumed-grant ledger
  (--session-grant-ledger, /objects/dev-session-grants.v1 in dev) keeps
  single-use grants consumed across process and pod restarts.
- Liveness: protocol v1 (undeployed) gains Heartbeat and the timeout close
  reason; both sides heartbeat when idle and treat 5 s of silence as dead.
- Client sends are bounded and unresolved tracking starts at send time.
- ci-client configures with an explicit C compiler.

Each finding has a regression test; ADR-0083, the schema, operations doc
and VALIDATION record the corrections.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-02 05:44:44 -04:00
co-authored by Claude Opus 5.5
parent 8e319322f4
commit c588be237c
26 changed files with 934 additions and 193 deletions
+14
View File
@@ -103,6 +103,20 @@ IntentResult SessionAdmission::admit(const IntentMessage& intent, std::uint64_t
return result;
}
std::expected<void, AdmissionError>
SessionAdmission::remember(std::span<const ConsumedGrant> consumed, std::uint64_t now_seconds) {
for (const auto& g : consumed) {
if (g.expires_at <= now_seconds || std::ranges::any_of(grants_, [&](const Remembered& r) {
return r.grant_id == g.grant_id;
}))
continue;
if (grants_.size() >= config_.max_remembered_grants)
return std::unexpected(AdmissionError::grants_full);
grants_.push_back({g.grant_id, g.expires_at});
}
return {};
}
std::expected<void, AdmissionError> SessionAdmission::close(std::uint64_t session) {
if (std::erase_if(sessions_, [&](const Session& s) { return s.id == session; }) == 0)
return std::unexpected(AdmissionError::unknown_session);
+14 -2
View File
@@ -5,9 +5,9 @@
namespace evo::network {
namespace {
constexpr std::uint8_t last_type = static_cast<std::uint8_t>(MessageType::intent_outcome);
constexpr std::uint8_t last_type = static_cast<std::uint8_t>(MessageType::heartbeat);
constexpr std::uint8_t last_status = static_cast<std::uint8_t>(IntentStatus::rate_limited);
constexpr std::uint8_t last_reason = static_cast<std::uint8_t>(CloseReason::capacity);
constexpr std::uint8_t last_reason = static_cast<std::uint8_t>(CloseReason::timeout);
constexpr std::uint8_t last_finality = static_cast<std::uint8_t>(IntentFinality::not_committed);
constexpr std::uint8_t last_presence = static_cast<std::uint8_t>(client::FieldPresence::present);
constexpr std::uint8_t last_moisture = static_cast<std::uint8_t>(world::MoistureStatus::available);
@@ -280,6 +280,18 @@ std::vector<std::uint8_t> encode(CloseReason reason) {
return framed(MessageType::close, w);
}
std::vector<std::uint8_t> encode_heartbeat() {
return encode_frame(MessageType::heartbeat, {});
}
std::expected<void, ProtocolError> decode_heartbeat(FrameView frame) {
if (frame.type != MessageType::heartbeat)
return std::unexpected(ProtocolError::wrong_type);
if (!frame.payload.empty())
return std::unexpected(ProtocolError::trailing_bytes);
return {};
}
std::vector<std::uint8_t> encode(const IntentOutcome& m) {
Writer w;
w.u64(m.sequence);
+50 -28
View File
@@ -18,15 +18,16 @@ struct SessionClient::Impl {
std::size_t next_grant{};
SessionPhase phase{SessionPhase::backoff};
Step step{Step::handshake};
Clock::time_point retry_at{}, deadline{};
Clock::time_point retry_at{}, deadline{}, last_received{}, last_sent{}, polled_at{};
std::chrono::milliseconds delay;
std::unique_ptr<TcpStream> stream; // Declared before the channel that borrows it.
std::unique_ptr<TlsChannel> tls;
FrameReader reader;
std::vector<std::uint8_t> out;
std::optional<Welcome> session;
std::optional<ViewHeader> view;
std::vector<std::uint64_t> outstanding; // Admitted, awaiting a final outcome.
// Sent and not finally resolved: no IntentResult yet, or admitted without an
// outcome. A connection loss reports every one of them as unknown.
std::vector<std::uint64_t> outstanding;
client::EnvironmentLayer environment;
client::OrganismLayer organisms;
std::vector<ClientEvent> events;
@@ -45,7 +46,6 @@ struct SessionClient::Impl {
tls.reset();
stream.reset();
reader = FrameReader{};
out.clear();
session.reset();
view.reset();
// Keep showing the last presentation, explicitly stale, until a new baseline.
@@ -77,9 +77,18 @@ struct SessionClient::Impl {
phase = SessionPhase::connecting;
step = Step::handshake;
deadline = now + config.attempt_deadline;
last_received = last_sent = now;
}
// Every outbound frame goes through the channel's bounded queue.
bool queue(const std::vector<std::uint8_t>& frame, Clock::time_point now) {
if (!tls || !tls->queue(frame))
return false;
last_sent = now;
return true;
}
bool frame(const OwnedFrame& f, Clock::time_point now) {
const FrameView v{f.type, f.payload};
last_received = now;
if (step == Step::awaiting_welcome) {
if (f.type == MessageType::close) {
const auto reason = decode_close(v);
@@ -132,14 +141,9 @@ struct SessionClient::Impl {
disconnect(now, std::nullopt);
return false;
}
if (r->status == IntentStatus::admitted) {
if (outstanding.size() >= max_outstanding) {
events.push_back(
{ClientEvent::Kind::unknown, {}, {}, {}, {}, outstanding.front()});
outstanding.erase(outstanding.begin());
}
outstanding.push_back(r->sequence);
}
// A rejection is final; an admitted intention waits for its outcome.
if (r->status != IntentStatus::admitted)
std::erase(outstanding, r->sequence);
events.push_back({ClientEvent::Kind::result, {}, {}, *r, {}, r->sequence});
return true;
}
@@ -153,6 +157,12 @@ struct SessionClient::Impl {
events.push_back({ClientEvent::Kind::outcome, {}, {}, {}, *o, o->sequence});
return true;
}
case MessageType::heartbeat:
if (!decode_heartbeat(v)) {
disconnect(now, std::nullopt);
return false;
}
return true;
case MessageType::close: {
const auto reason = decode_close(v);
disconnect(now, reason.value_or(CloseReason::malformed));
@@ -164,18 +174,8 @@ struct SessionClient::Impl {
}
}
void flush(Clock::time_point now) {
std::size_t sent = 0;
while (sent < out.size()) {
const auto n = tls->send(std::span{out}.subspan(sent));
if (!n) {
disconnect(now, std::nullopt);
return;
}
if (*n == 0)
break;
sent += *n;
}
out.erase(out.begin(), out.begin() + static_cast<std::ptrdiff_t>(sent));
if (!tls->flush())
disconnect(now, std::nullopt);
}
void receive(Clock::time_point now) {
std::array<std::uint8_t, 16384> buffer{};
@@ -215,7 +215,8 @@ SessionClient::create(SessionClientConfig config) {
if (config.host.empty() || config.port == 0 || config.grants.empty() ||
config.grants.size() > 1024 || config.lead_ticks == 0 || config.lead_ticks > 64 ||
config.first_retry.count() <= 0 || config.max_retry < config.first_retry ||
config.attempt_deadline.count() <= 0)
config.attempt_deadline.count() <= 0 || config.heartbeat_interval.count() <= 0 ||
config.liveness_timeout <= 2 * config.heartbeat_interval)
return std::unexpected(SessionClientError::configuration);
return std::unique_ptr<SessionClient>(
new SessionClient(std::make_unique<Impl>(std::move(config))));
@@ -223,6 +224,7 @@ SessionClient::create(SessionClientConfig config) {
void SessionClient::poll(std::chrono::steady_clock::time_point now) {
auto& m = *impl_;
m.polled_at = now;
if (m.phase == SessionPhase::exhausted)
return;
if (m.phase == SessionPhase::backoff) {
@@ -236,6 +238,18 @@ void SessionClient::poll(std::chrono::steady_clock::time_point now) {
m.disconnect(now, std::nullopt);
return;
}
if (m.step == Impl::Step::established) {
// A silent server (paused process, black-holed path) is treated as lost: the
// presentation turns stale and the reconnect policy takes over.
if (now - m.last_received > m.config.liveness_timeout) {
m.disconnect(now, std::nullopt);
return;
}
if (now - m.last_sent >= m.config.heartbeat_interval && !m.queue(encode_heartbeat(), now)) {
m.disconnect(now, std::nullopt);
return;
}
}
if (m.step == Impl::Step::handshake) {
const auto p = m.tls->handshake();
if (!p) {
@@ -251,7 +265,10 @@ void SessionClient::poll(std::chrono::steady_clock::time_point now) {
m.disconnect(now, std::nullopt);
return;
}
m.out.insert(m.out.end(), hello->begin(), hello->end());
if (!m.queue(*hello, now)) {
m.disconnect(now, std::nullopt);
return;
}
m.step = Impl::Step::awaiting_welcome;
}
m.flush(now);
@@ -263,13 +280,18 @@ void SessionClient::poll(std::chrono::steady_clock::time_point now) {
bool SessionClient::send(const client::Intention& intention) {
auto& m = *impl_;
if (m.phase != SessionPhase::established || !m.session || !m.view)
// Bounded: no session, no view, too many unresolved intentions or a full channel
// queue all refuse rather than buffer (the caller resamples later).
if (m.phase != SessionPhase::established || !m.session || !m.view ||
m.outstanding.size() >= max_outstanding)
return false;
const auto tick = m.view->server_tick + m.config.lead_ticks;
const auto bytes =
encode(make_intent(m.session->session, m.session->epoch,
{intention.sequence, tick, intention.move_x, intention.move_y}));
m.out.insert(m.out.end(), bytes.begin(), bytes.end());
if (!m.queue(bytes, m.polled_at))
return false;
m.outstanding.push_back(intention.sequence);
return true;
}
+122
View File
@@ -0,0 +1,122 @@
#include "evo/network/grant_ledger.hpp"
#include <charconv>
#include <fstream>
#include <string>
#include <string_view>
#include <system_error>
#ifdef _WIN32
#include <io.h>
#else
#include <fcntl.h>
#include <unistd.h>
#endif
namespace evo::network {
namespace {
std::FILE* open_file(const std::filesystem::path& path, const char* mode) {
#ifdef _WIN32
std::FILE* file = nullptr;
return _wfopen_s(&file, path.c_str(), mode[0] == 'w' ? L"wb" : L"ab") == 0 ? file : nullptr;
#else
return std::fopen(path.c_str(), mode);
#endif
}
bool sync(std::FILE* file) {
if (std::fflush(file) != 0)
return false;
#ifdef _WIN32
return _commit(_fileno(file)) == 0;
#else
return ::fsync(::fileno(file)) == 0;
#endif
}
// Makes a rename durable on POSIX by syncing the containing directory.
bool sync_directory([[maybe_unused]] const std::filesystem::path& dir) {
#ifdef _WIN32
return true; // NTFS metadata journaling; MoveFileEx is used with replace semantics.
#else
const int fd = ::open(dir.empty() ? "." : dir.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC);
if (fd < 0)
return false;
const bool ok = ::fsync(fd) == 0;
return ::close(fd) == 0 && ok;
#endif
}
std::string line(ConsumedGrant g) {
constexpr char digits[] = "0123456789abcdef";
std::string out(16, '0');
for (int i = 15; i >= 0; --i, g.grant_id >>= 4U)
out[static_cast<std::size_t>(i)] = digits[g.grant_id & 0x0FU];
return out + ' ' + std::to_string(g.expires_at) + '\n';
}
bool parse(const std::string& text, ConsumedGrant& out) { // One getline() line.
if (text.size() < 18 || text[16] != ' ')
return false;
const auto [a, e1] = std::from_chars(text.data(), text.data() + 16, out.grant_id, 16);
const auto [b, e2] =
std::from_chars(text.data() + 17, text.data() + text.size(), out.expires_at);
return e1 == std::errc{} && a == text.data() + 16 && e2 == std::errc{} &&
b == text.data() + text.size() && out.grant_id != 0;
}
} // namespace
std::expected<std::unique_ptr<GrantLedger>, LedgerError>
GrantLedger::open(const std::filesystem::path& path, std::uint64_t now_seconds) {
std::vector<ConsumedGrant> live;
std::error_code error;
if (std::filesystem::exists(path, error)) {
std::ifstream in{path, std::ios::binary};
if (!in)
return std::unexpected(LedgerError::io);
std::string text;
std::size_t total = 0;
while (std::getline(in, text)) {
ConsumedGrant g;
if (!parse(text, g))
return std::unexpected(LedgerError::corrupt); // Never guess: fail closed.
if (++total > max_entries)
return std::unexpected(LedgerError::full);
if (g.expires_at > now_seconds)
live.push_back(g);
}
if (!in.eof())
return std::unexpected(LedgerError::io);
} else if (error) {
return std::unexpected(LedgerError::io);
}
// Rewrite the unexpired entries atomically, then append to the result.
const auto temporary = std::filesystem::path{path}.concat(".tmp");
{
std::FILE* out = open_file(temporary, "wb");
if (out == nullptr)
return std::unexpected(LedgerError::io);
bool ok = true;
for (const auto& g : live)
ok = ok && std::fputs(line(g).c_str(), out) >= 0;
ok = sync(out) && ok;
if (std::fclose(out) != 0 || !ok)
return std::unexpected(LedgerError::io);
}
std::filesystem::rename(temporary, path, error);
if (error || !sync_directory(path.parent_path()))
return std::unexpected(LedgerError::io);
std::FILE* file = open_file(path, "ab");
if (file == nullptr)
return std::unexpected(LedgerError::io);
return std::unique_ptr<GrantLedger>(new GrantLedger(file, std::move(live)));
}
GrantLedger::~GrantLedger() {
std::fclose(file_);
}
std::expected<void, LedgerError> GrantLedger::record(ConsumedGrant grant) {
if (entries_ >= max_entries)
return std::unexpected(LedgerError::full);
if (std::fputs(line(grant).c_str(), file_) < 0 || !sync(file_))
return std::unexpected(LedgerError::io);
++entries_;
return {};
}
} // namespace evo::network
+128 -83
View File
@@ -1,5 +1,6 @@
#include "evo/network/session_server.hpp"
#include <algorithm>
#include <limits>
#include <utility>
namespace evo::network {
@@ -7,6 +8,9 @@ namespace {
using Clock = std::chrono::steady_clock;
constexpr std::size_t max_accepts_per_poll = 4;
constexpr std::size_t max_read_per_poll = 65536;
// Carrier ids are 'I','N' + 48-bit epoch + 64-bit server ordinal: distinct for every
// carried intention, independent of client-chosen sequences (which may be any u64).
constexpr std::uint64_t max_carrier_epoch = (std::uint64_t{1} << 48) - 1;
CloseReason close_reason(AdmissionError e) {
switch (e) {
case AdmissionError::duplicate_subject:
@@ -24,8 +28,13 @@ CloseReason close_reason(ProtocolError e) {
return e == ProtocolError::unsupported_version ? CloseReason::unsupported_version
: CloseReason::malformed;
}
void append(std::vector<std::uint8_t>& out, const std::vector<std::uint8_t>& bytes) {
out.insert(out.end(), bytes.begin(), bytes.end());
RequestId carrier_id(std::uint64_t epoch, std::uint64_t ordinal) {
RequestId id{'I', 'N'};
for (unsigned i = 0; i < 6; ++i)
id.at(2 + i) = static_cast<std::uint8_t>(epoch >> (8U * (5U - i)));
for (unsigned i = 0; i < 8; ++i)
id.at(8 + i) = static_cast<std::uint8_t>(ordinal >> (8U * (7U - i)));
return id;
}
} // namespace
@@ -34,18 +43,16 @@ struct SessionServer::Impl {
struct Connection {
std::uint64_t id{};
std::unique_ptr<TcpStream> stream; // Declared first: outlives the channel.
std::unique_ptr<TlsChannel> tls;
std::unique_ptr<TlsChannel> tls; // Owns all queued output (ADR-0083 review fix).
State state{State::handshake};
Clock::time_point deadline;
Clock::time_point deadline, last_received, last_sent, window_start;
FrameReader reader;
std::vector<std::uint8_t> out, view;
std::vector<std::uint8_t> view; // Newest unsent view only; replaced, never queued.
std::uint64_t session{}, subject{}, epoch{}, expires_at{};
std::size_t window_bytes{};
Clock::time_point window_start;
bool dead{};
};
struct Pending {
RequestId request{};
std::uint64_t intended_tick{}, connection{}, sequence{};
};
struct Carried {
@@ -53,13 +60,16 @@ struct SessionServer::Impl {
std::uint64_t connection{}, sequence{};
};
Impl(SessionServerConfig c, DevGrantKey k, std::unique_ptr<TcpListener> l)
: config{std::move(c)}, key{std::move(k)}, listener{std::move(l)} {}
Impl(SessionServerConfig c, DevGrantKey k, std::unique_ptr<TcpListener> l,
std::unique_ptr<GrantLedger> g)
: config{std::move(c)}, key{std::move(k)}, listener{std::move(l)}, ledger{std::move(g)} {}
SessionServerConfig config;
DevGrantKey key;
std::unique_ptr<TcpListener> listener;
std::unique_ptr<GrantLedger> ledger;
std::optional<SessionAdmission> admission;
bool ledger_rejected{}; // Recorded grants exceed admission capacity: fail closed.
std::vector<Connection> connections;
std::deque<Pending> pending;
std::vector<Carried> carried;
@@ -67,8 +77,8 @@ struct SessionServer::Impl {
std::optional<client::EnvironmentPresentation> environment;
std::optional<client::OrganismPresentation> organisms;
RegionFacts published{};
std::uint64_t baseline{}, next_connection{1};
Clock::time_point polled_at{}; // Time of the current poll, for Close deadlines.
std::uint64_t baseline{}, next_connection{1}, next_carrier{1};
Clock::time_point polled_at{};
Connection* find(std::uint64_t id) {
const auto it = std::ranges::find_if(connections, [&](const Connection& c) {
@@ -76,16 +86,23 @@ struct SessionServer::Impl {
});
return it == connections.end() ? nullptr : &*it;
}
// Every outbound frame goes through the channel's bounded queue; a full queue means
// a consumer that cannot keep up, which is dropped rather than buffered further.
void send(Connection& c, const std::vector<std::uint8_t>& frame) {
if (c.dead || !c.tls->queue(frame))
c.dead = true;
c.last_sent = polled_at;
}
void close(Connection& c, CloseReason reason) {
if (c.state == State::closing || c.dead)
return;
if (c.state == State::hello || c.state == State::established)
append(c.out, encode(reason));
else
c.dead = true; // No channel yet to say anything over.
events.push_back({c.state == State::established ? SessionEvent::Kind::closed
: SessionEvent::Kind::refused,
c.session, c.subject, reason});
if (c.state == State::hello || c.state == State::established)
send(c, encode(reason));
else
c.dead = true; // No channel yet to say anything over.
c.state = State::closing;
c.deadline = polled_at + std::chrono::seconds{1}; // Bounded flush of the Close frame.
c.view.clear();
@@ -96,8 +113,11 @@ struct SessionServer::Impl {
return bytes;
const ViewHeader header{c.session, c.epoch, published.simulated_tick,
published.committed_tick, baseline};
append(bytes, encode(header, *environment));
append(bytes, encode(header, *organisms));
const auto a = encode(header, *environment);
const auto b = encode(header, *organisms);
bytes.reserve(a.size() + b.size());
bytes.insert(bytes.end(), a.begin(), a.end());
bytes.insert(bytes.end(), b.begin(), b.end());
return bytes;
}
void hello(Connection& c, const OwnedFrame& f, const RegionFacts& facts,
@@ -115,24 +135,34 @@ struct SessionServer::Impl {
close(c, CloseReason::server_shutdown); // No current authority: retry later.
return;
}
if (facts.epoch > max_carrier_epoch) {
close(c, CloseReason::capacity); // Carrier ids cannot name this epoch: refuse.
return;
}
const auto grant = key.verify(message->grant);
if (!grant) {
close(c, CloseReason::unauthenticated);
return;
}
c.subject = grant->claims().subject;
const auto welcome =
admission->open(*grant, now_seconds, facts.simulated_tick, facts.committed_tick);
if (!welcome) {
c.subject = grant->claims().subject;
close(c, close_reason(welcome.error()));
return;
}
// Durable before Welcome: a replacement process must refuse this grant too.
if (!ledger->record({grant->claims().grant_id, grant->claims().expires_at})) {
// The grant stays consumed in memory (fail closed); only the session is undone.
static_cast<void>(admission->close(welcome->session).has_value());
close(c, CloseReason::server_shutdown);
return;
}
c.session = welcome->session;
c.subject = grant->claims().subject;
c.epoch = welcome->epoch;
c.expires_at = welcome->grant_expires_at;
c.state = State::established;
append(c.out, encode(*welcome));
send(c, encode(*welcome));
c.view = views_for(c); // Fresh baseline for every new session.
events.push_back({SessionEvent::Kind::opened, c.session, c.subject, {}});
}
@@ -152,16 +182,8 @@ struct SessionServer::Impl {
result = admission->admit(*m, facts.simulated_tick, now_seconds);
}
if (result.status == IntentStatus::admitted)
pending.push_back({request_id(c, m->sequence), m->tick, c.id, m->sequence});
append(c.out, encode(result));
}
RequestId request_id(const Connection& c, std::uint64_t sequence) const {
RequestId id{'I', 'N', 'T', '1'};
const std::array<std::uint64_t, 3> parts{c.epoch, c.session, sequence};
for (std::size_t p = 0; p < parts.size(); ++p)
for (unsigned i = 0; i < 4; ++i)
id.at(4 + 4 * p + i) = static_cast<std::uint8_t>(parts.at(p) >> (8U * (3U - i)));
return id;
pending.push_back({m->tick, c.id, m->sequence});
send(c, encode(result));
}
void receive(Connection& c, const RegionFacts& facts, std::uint64_t now_seconds,
Clock::time_point now) {
@@ -194,12 +216,17 @@ struct SessionServer::Impl {
}
rest = rest.subspan(*taken);
while (auto f = c.reader.take()) {
c.last_received = now;
if (c.state == State::hello)
hello(c, *f, facts, now_seconds);
else if (f->type == MessageType::intent)
intent(c, *f, facts, now_seconds);
else if (f->type == MessageType::close) {
else if (f->type == MessageType::heartbeat) {
if (!decode_heartbeat({f->type, f->payload}))
close(c, CloseReason::malformed);
} else if (f->type == MessageType::close) {
c.state = State::closing; // Peer is leaving; nothing to say.
c.deadline = now + std::chrono::seconds{1};
events.push_back({SessionEvent::Kind::closed, c.session, c.subject,
CloseReason::normal});
} else
@@ -211,27 +238,25 @@ struct SessionServer::Impl {
}
}
void transmit(Connection& c) {
if (c.out.empty() && !c.view.empty() && c.state == State::established)
c.out.swap(c.view); // Only the newest view waits; it is never queued twice.
std::size_t sent = 0;
while (sent < c.out.size()) {
const auto n = c.tls->send(std::span{c.out}.subspan(sent));
if (!n) {
c.dead = true;
return;
}
if (*n == 0)
break;
sent += *n;
if (c.state == State::established && c.tls->queued() == 0 && !c.view.empty()) {
send(c, c.view);
c.view.clear();
}
c.out.erase(c.out.begin(), c.out.begin() + static_cast<std::ptrdiff_t>(sent));
if (c.out.size() + c.view.size() > config.max_output_bytes)
if (c.dead)
return;
if (c.tls->queued() + c.view.size() > config.max_output_bytes) {
c.dead = true; // Slow consumer: drop rather than buffer without bound.
if (c.state == State::closing && c.out.empty()) {
// Best-effort close_notify: the connection is dropped either way.
static_cast<void>(c.tls->close().has_value());
c.dead = true;
return;
}
if (c.state == State::closing) {
// close() flushes queued frames (the Close frame included) before close_notify.
const auto closed = c.tls->close();
if (!closed || *closed == TlsProgress::done)
c.dead = true;
return;
}
if (!c.tls->flush())
c.dead = true;
}
void step(Connection& c, const RegionFacts& facts, std::uint64_t now_seconds,
Clock::time_point now) {
@@ -250,10 +275,16 @@ struct SessionServer::Impl {
}
if ((c.state == State::handshake || c.state == State::hello) && now > c.deadline)
close(c, CloseReason::unauthenticated);
if (c.state == State::established && now_seconds >= c.expires_at)
close(c, CloseReason::expired);
if (c.state == State::established && admission && c.epoch != admission->epoch())
close(c, CloseReason::stale_epoch);
if (c.state == State::established) {
if (now_seconds >= c.expires_at)
close(c, CloseReason::expired);
else if (admission && c.epoch != admission->epoch())
close(c, CloseReason::stale_epoch);
else if (now - c.last_received > config.liveness_timeout)
close(c, CloseReason::timeout); // Silent peer: free the subject for reconnect.
else if (now - c.last_sent >= config.heartbeat_interval)
send(c, encode_heartbeat()); // Liveness without requiring simulation ticks.
}
if (c.state == State::hello || c.state == State::established)
receive(c, facts, now_seconds, now);
if (!c.dead && c.state != State::handshake)
@@ -270,17 +301,44 @@ struct SessionServer::Impl {
}
std::erase_if(connections, [](const Connection& c) { return c.dead; });
}
void open_admission(const RegionFacts& facts, std::uint64_t now_seconds) {
auto config_with_epoch = config.admission;
config_with_epoch.epoch = facts.epoch;
auto created = SessionAdmission::create(config_with_epoch);
if (!created)
return;
// Grants consumed before this process started stay consumed.
if (!created->remember(ledger->loaded(), now_seconds)) {
ledger_rejected = true; // More recorded grants than admission can hold.
return;
}
admission.emplace(std::move(*created));
}
void finish(const RequestId& request, const IntentOutcome& outcome) {
const auto it =
std::ranges::find_if(carried, [&](const Carried& c) { return c.request == request; });
if (it == carried.end())
return;
auto out = outcome;
out.sequence = it->sequence; // The client's own sequence, carried separately.
if (auto* c = find(it->connection))
send(*c, encode(out));
carried.erase(it);
}
};
SessionServer::SessionServer(std::unique_ptr<Impl> impl) : impl_{std::move(impl)} {}
SessionServer::~SessionServer() = default;
std::expected<std::unique_ptr<SessionServer>, SessionServerError>
SessionServer::create(SessionServerConfig config, DevGrantKey key) {
SessionServer::create(SessionServerConfig config, DevGrantKey key, std::uint64_t now_seconds) {
if (config.max_connections == 0 || config.max_connections > 64 ||
config.max_pending_intents == 0 || config.max_carried_intents == 0 ||
config.max_output_bytes < 2 * (frame_header_bytes + max_frame_payload) ||
config.max_input_bytes_per_second < frame_header_bytes + max_grant_bytes + 6)
config.max_output_bytes > TlsChannel::max_queued_bytes ||
config.max_input_bytes_per_second < frame_header_bytes + max_grant_bytes + 6 ||
config.heartbeat_interval.count() <= 0 ||
config.liveness_timeout <= 2 * config.heartbeat_interval || config.grant_ledger.empty())
return std::unexpected(SessionServerError::configuration);
// Validate the admission limits now, with a placeholder epoch.
auto probe = config.admission;
@@ -298,11 +356,14 @@ SessionServer::create(SessionServerConfig config, DevGrantKey key) {
} null;
if (!TlsChannel::server(config.credentials, null))
return std::unexpected(SessionServerError::credentials);
auto ledger = GrantLedger::open(config.grant_ledger, now_seconds);
if (!ledger)
return std::unexpected(SessionServerError::ledger);
auto listener = TcpListener::listen(config.listen);
if (!listener)
return std::unexpected(SessionServerError::listen);
return std::unique_ptr<SessionServer>(new SessionServer(
std::make_unique<Impl>(std::move(config), std::move(key), std::move(*listener))));
return std::unique_ptr<SessionServer>(new SessionServer(std::make_unique<Impl>(
std::move(config), std::move(key), std::move(*listener), std::move(*ledger))));
}
std::uint16_t SessionServer::port() const noexcept {
@@ -313,12 +374,9 @@ void SessionServer::poll(const RegionFacts& facts, std::uint64_t now_seconds,
std::chrono::steady_clock::time_point now) {
auto& m = *impl_;
m.polled_at = now;
if (facts.ready && facts.epoch != 0) {
if (facts.ready && facts.epoch != 0 && !m.ledger_rejected) {
if (!m.admission) {
auto config = m.config.admission;
config.epoch = facts.epoch;
if (auto created = SessionAdmission::create(config))
m.admission.emplace(std::move(*created));
m.open_admission(facts, now_seconds);
} else if (facts.epoch > m.admission->epoch()) {
// Strictly newer (checked above), so the advance cannot be refused.
static_cast<void>(m.admission->advance_epoch(facts.epoch).has_value());
@@ -340,7 +398,7 @@ void SessionServer::poll(const RegionFacts& facts, std::uint64_t now_seconds,
continue;
c.tls = std::move(*tls);
c.deadline = now + m.config.hello_deadline;
c.window_start = now;
c.window_start = c.last_received = c.last_sent = now;
m.connections.push_back(std::move(c));
}
for (auto& c : m.connections)
@@ -364,13 +422,15 @@ void SessionServer::publish(const client::EnvironmentPresentation& environment,
std::optional<RequestId> SessionServer::carry(std::uint64_t tick) {
auto& m = *impl_;
if (m.carried.size() >= m.config.max_carried_intents)
return std::nullopt;
if (m.carried.size() >= m.config.max_carried_intents || !m.admission ||
m.next_carrier == std::numeric_limits<std::uint64_t>::max())
return std::nullopt; // Ordinal exhaustion is explicit, never a wrap.
const auto it = std::ranges::find_if(
m.pending, [&](const Impl::Pending& p) { return p.intended_tick <= tick; });
if (it == m.pending.end())
return std::nullopt;
const Impl::Carried carried{it->request, it->connection, it->sequence};
const Impl::Carried carried{carrier_id(m.admission->epoch(), m.next_carrier++), it->connection,
it->sequence};
m.pending.erase(it);
m.carried.push_back(carried);
return carried.request;
@@ -378,26 +438,11 @@ std::optional<RequestId> SessionServer::carry(std::uint64_t tick) {
void SessionServer::committed(const RequestId& request, std::uint64_t tick,
std::uint64_t commit_sequence) {
auto& m = *impl_;
const auto it = std::ranges::find_if(
m.carried, [&](const Impl::Carried& c) { return c.request == request; });
if (it == m.carried.end())
return;
if (auto* c = m.find(it->connection))
append(c->out, encode(IntentOutcome{it->sequence, IntentFinality::committed_no_effect, tick,
commit_sequence}));
m.carried.erase(it);
impl_->finish(request, {0, IntentFinality::committed_no_effect, tick, commit_sequence});
}
void SessionServer::not_committed(const RequestId& request) {
auto& m = *impl_;
const auto it = std::ranges::find_if(
m.carried, [&](const Impl::Carried& c) { return c.request == request; });
if (it == m.carried.end())
return;
if (auto* c = m.find(it->connection))
append(c->out, encode(IntentOutcome{it->sequence, IntentFinality::not_committed, 0, 0}));
m.carried.erase(it);
impl_->finish(request, {0, IntentFinality::not_committed, 0, 0});
}
std::vector<SessionEvent> SessionServer::take_events() {
+39 -7
View File
@@ -5,6 +5,7 @@
#include <mbedtls/x509_crt.h>
#include <optional>
#include <psa/crypto.h>
#include <vector>
// The owned user config (third_party/mbedtls/evo_mbedtls_config.h) must be in effect.
#if defined(MBEDTLS_SSL_PROTO_TLS1_2) || defined(MBEDTLS_SSL_PROTO_DTLS) || \
@@ -21,6 +22,8 @@ namespace {
// ADR-0082 profile: TLS 1.3 only, AEAD suites, ECDHE groups, ephemeral key exchange.
constexpr int suites[]{MBEDTLS_TLS1_3_AES_128_GCM_SHA256, MBEDTLS_TLS1_3_CHACHA20_POLY1305_SHA256,
0};
// One TLS record's plaintext per write call (the TLS 1.3 maximum fragment).
constexpr std::size_t record_plaintext = 16384;
constexpr std::uint16_t groups[]{MBEDTLS_SSL_IANA_TLS_GROUP_X25519,
MBEDTLS_SSL_IANA_TLS_GROUP_SECP256R1, 0};
const unsigned char* bytes_of(const std::string& text) {
@@ -105,6 +108,8 @@ struct TlsChannel::Impl {
mbedtls_pk_context key;
bool transport_failed{};
std::optional<TlsError> error;
std::vector<std::uint8_t> outbound; // Queued plaintext; the front record may be in flight.
std::size_t in_flight{}; // Length of an interrupted write; 0 when none.
};
TlsChannel::TlsChannel(std::unique_ptr<Impl> impl) : impl_{std::move(impl)} {}
@@ -158,15 +163,39 @@ std::expected<TlsProgress, TlsError> TlsChannel::handshake() {
return impl_->fail_from(result, TlsError::handshake);
}
std::expected<std::size_t, TlsError> TlsChannel::send(std::span<const std::uint8_t> bytes) {
std::expected<void, TlsError> TlsChannel::queue(std::span<const std::uint8_t> bytes) {
if (const auto latched = impl_->error)
return std::unexpected(*latched);
const int result = mbedtls_ssl_write(&impl_->ssl, bytes.data(), bytes.size());
if (result >= 0)
return static_cast<std::size_t>(result);
if (result == MBEDTLS_ERR_SSL_WANT_READ || result == MBEDTLS_ERR_SSL_WANT_WRITE)
return 0;
return impl_->fail_from(result, TlsError::record);
if (bytes.size() > max_queued_bytes - impl_->outbound.size())
return std::unexpected(TlsError::backpressure); // Nothing queued: not latched.
impl_->outbound.insert(impl_->outbound.end(), bytes.begin(), bytes.end());
return {};
}
std::expected<TlsProgress, TlsError> TlsChannel::flush() {
auto& m = *impl_;
if (const auto latched = m.error)
return std::unexpected(*latched);
while (!m.outbound.empty()) {
// A record interrupted by WANT_READ/WANT_WRITE keeps its length; bytes queued
// since then sit behind it and do not change the retried call.
if (m.in_flight == 0)
m.in_flight = std::min(m.outbound.size(), record_plaintext);
const int result = mbedtls_ssl_write(&m.ssl, m.outbound.data(), m.in_flight);
if (result == MBEDTLS_ERR_SSL_WANT_READ || result == MBEDTLS_ERR_SSL_WANT_WRITE)
return TlsProgress::want_io;
if (result < 0)
return m.fail_from(result, TlsError::record);
const auto written = std::min(static_cast<std::size_t>(result), m.in_flight);
m.outbound.erase(m.outbound.begin(),
m.outbound.begin() + static_cast<std::ptrdiff_t>(written));
m.in_flight = 0;
}
return TlsProgress::done;
}
std::size_t TlsChannel::queued() const noexcept {
return impl_->outbound.size();
}
std::expected<std::size_t, TlsError> TlsChannel::receive(std::span<std::uint8_t> bytes) {
@@ -185,6 +214,9 @@ std::expected<std::size_t, TlsError> TlsChannel::receive(std::span<std::uint8_t>
std::expected<TlsProgress, TlsError> TlsChannel::close() {
if (const auto latched = impl_->error)
return std::unexpected(*latched);
// Queued plaintext (including an interrupted record) is sent before close_notify.
if (const auto pending = flush(); !pending || *pending == TlsProgress::want_io)
return pending;
const int result = mbedtls_ssl_close_notify(&impl_->ssl);
if (result == 0)
return TlsProgress::done;