Fix M9.4-M9.7 review findings in the dev session stack
- TLS: replace send() with a bounded queue/flush; an interrupted record is retried with identical arguments and later frames wait behind it, so no queued frame is lost or reordered; close() flushes first. - Carriers: server-issued 'I','N' | 48-bit epoch | 64-bit ordinal ids replace the aliasing low-32-bit packing; client sequences are kept separately for replies; out-of-range epochs are refused, never truncated. - Grants: a durable, fsynced, fail-closed consumed-grant ledger (--session-grant-ledger, /objects/dev-session-grants.v1 in dev) keeps single-use grants consumed across process and pod restarts. - Liveness: protocol v1 (undeployed) gains Heartbeat and the timeout close reason; both sides heartbeat when idle and treat 5 s of silence as dead. - Client sends are bounded and unresolved tracking starts at send time. - ci-client configures with an explicit C compiler. Each finding has a regression test; ADR-0083, the schema, operations doc and VALIDATION record the corrections. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -103,6 +103,20 @@ IntentResult SessionAdmission::admit(const IntentMessage& intent, std::uint64_t
|
||||
return result;
|
||||
}
|
||||
|
||||
std::expected<void, AdmissionError>
|
||||
SessionAdmission::remember(std::span<const ConsumedGrant> consumed, std::uint64_t now_seconds) {
|
||||
for (const auto& g : consumed) {
|
||||
if (g.expires_at <= now_seconds || std::ranges::any_of(grants_, [&](const Remembered& r) {
|
||||
return r.grant_id == g.grant_id;
|
||||
}))
|
||||
continue;
|
||||
if (grants_.size() >= config_.max_remembered_grants)
|
||||
return std::unexpected(AdmissionError::grants_full);
|
||||
grants_.push_back({g.grant_id, g.expires_at});
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
std::expected<void, AdmissionError> SessionAdmission::close(std::uint64_t session) {
|
||||
if (std::erase_if(sessions_, [&](const Session& s) { return s.id == session; }) == 0)
|
||||
return std::unexpected(AdmissionError::unknown_session);
|
||||
|
||||
@@ -5,9 +5,9 @@
|
||||
|
||||
namespace evo::network {
|
||||
namespace {
|
||||
constexpr std::uint8_t last_type = static_cast<std::uint8_t>(MessageType::intent_outcome);
|
||||
constexpr std::uint8_t last_type = static_cast<std::uint8_t>(MessageType::heartbeat);
|
||||
constexpr std::uint8_t last_status = static_cast<std::uint8_t>(IntentStatus::rate_limited);
|
||||
constexpr std::uint8_t last_reason = static_cast<std::uint8_t>(CloseReason::capacity);
|
||||
constexpr std::uint8_t last_reason = static_cast<std::uint8_t>(CloseReason::timeout);
|
||||
constexpr std::uint8_t last_finality = static_cast<std::uint8_t>(IntentFinality::not_committed);
|
||||
constexpr std::uint8_t last_presence = static_cast<std::uint8_t>(client::FieldPresence::present);
|
||||
constexpr std::uint8_t last_moisture = static_cast<std::uint8_t>(world::MoistureStatus::available);
|
||||
@@ -280,6 +280,18 @@ std::vector<std::uint8_t> encode(CloseReason reason) {
|
||||
return framed(MessageType::close, w);
|
||||
}
|
||||
|
||||
std::vector<std::uint8_t> encode_heartbeat() {
|
||||
return encode_frame(MessageType::heartbeat, {});
|
||||
}
|
||||
|
||||
std::expected<void, ProtocolError> decode_heartbeat(FrameView frame) {
|
||||
if (frame.type != MessageType::heartbeat)
|
||||
return std::unexpected(ProtocolError::wrong_type);
|
||||
if (!frame.payload.empty())
|
||||
return std::unexpected(ProtocolError::trailing_bytes);
|
||||
return {};
|
||||
}
|
||||
|
||||
std::vector<std::uint8_t> encode(const IntentOutcome& m) {
|
||||
Writer w;
|
||||
w.u64(m.sequence);
|
||||
|
||||
@@ -18,15 +18,16 @@ struct SessionClient::Impl {
|
||||
std::size_t next_grant{};
|
||||
SessionPhase phase{SessionPhase::backoff};
|
||||
Step step{Step::handshake};
|
||||
Clock::time_point retry_at{}, deadline{};
|
||||
Clock::time_point retry_at{}, deadline{}, last_received{}, last_sent{}, polled_at{};
|
||||
std::chrono::milliseconds delay;
|
||||
std::unique_ptr<TcpStream> stream; // Declared before the channel that borrows it.
|
||||
std::unique_ptr<TlsChannel> tls;
|
||||
FrameReader reader;
|
||||
std::vector<std::uint8_t> out;
|
||||
std::optional<Welcome> session;
|
||||
std::optional<ViewHeader> view;
|
||||
std::vector<std::uint64_t> outstanding; // Admitted, awaiting a final outcome.
|
||||
// Sent and not finally resolved: no IntentResult yet, or admitted without an
|
||||
// outcome. A connection loss reports every one of them as unknown.
|
||||
std::vector<std::uint64_t> outstanding;
|
||||
client::EnvironmentLayer environment;
|
||||
client::OrganismLayer organisms;
|
||||
std::vector<ClientEvent> events;
|
||||
@@ -45,7 +46,6 @@ struct SessionClient::Impl {
|
||||
tls.reset();
|
||||
stream.reset();
|
||||
reader = FrameReader{};
|
||||
out.clear();
|
||||
session.reset();
|
||||
view.reset();
|
||||
// Keep showing the last presentation, explicitly stale, until a new baseline.
|
||||
@@ -77,9 +77,18 @@ struct SessionClient::Impl {
|
||||
phase = SessionPhase::connecting;
|
||||
step = Step::handshake;
|
||||
deadline = now + config.attempt_deadline;
|
||||
last_received = last_sent = now;
|
||||
}
|
||||
// Every outbound frame goes through the channel's bounded queue.
|
||||
bool queue(const std::vector<std::uint8_t>& frame, Clock::time_point now) {
|
||||
if (!tls || !tls->queue(frame))
|
||||
return false;
|
||||
last_sent = now;
|
||||
return true;
|
||||
}
|
||||
bool frame(const OwnedFrame& f, Clock::time_point now) {
|
||||
const FrameView v{f.type, f.payload};
|
||||
last_received = now;
|
||||
if (step == Step::awaiting_welcome) {
|
||||
if (f.type == MessageType::close) {
|
||||
const auto reason = decode_close(v);
|
||||
@@ -132,14 +141,9 @@ struct SessionClient::Impl {
|
||||
disconnect(now, std::nullopt);
|
||||
return false;
|
||||
}
|
||||
if (r->status == IntentStatus::admitted) {
|
||||
if (outstanding.size() >= max_outstanding) {
|
||||
events.push_back(
|
||||
{ClientEvent::Kind::unknown, {}, {}, {}, {}, outstanding.front()});
|
||||
outstanding.erase(outstanding.begin());
|
||||
}
|
||||
outstanding.push_back(r->sequence);
|
||||
}
|
||||
// A rejection is final; an admitted intention waits for its outcome.
|
||||
if (r->status != IntentStatus::admitted)
|
||||
std::erase(outstanding, r->sequence);
|
||||
events.push_back({ClientEvent::Kind::result, {}, {}, *r, {}, r->sequence});
|
||||
return true;
|
||||
}
|
||||
@@ -153,6 +157,12 @@ struct SessionClient::Impl {
|
||||
events.push_back({ClientEvent::Kind::outcome, {}, {}, {}, *o, o->sequence});
|
||||
return true;
|
||||
}
|
||||
case MessageType::heartbeat:
|
||||
if (!decode_heartbeat(v)) {
|
||||
disconnect(now, std::nullopt);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
case MessageType::close: {
|
||||
const auto reason = decode_close(v);
|
||||
disconnect(now, reason.value_or(CloseReason::malformed));
|
||||
@@ -164,18 +174,8 @@ struct SessionClient::Impl {
|
||||
}
|
||||
}
|
||||
void flush(Clock::time_point now) {
|
||||
std::size_t sent = 0;
|
||||
while (sent < out.size()) {
|
||||
const auto n = tls->send(std::span{out}.subspan(sent));
|
||||
if (!n) {
|
||||
disconnect(now, std::nullopt);
|
||||
return;
|
||||
}
|
||||
if (*n == 0)
|
||||
break;
|
||||
sent += *n;
|
||||
}
|
||||
out.erase(out.begin(), out.begin() + static_cast<std::ptrdiff_t>(sent));
|
||||
if (!tls->flush())
|
||||
disconnect(now, std::nullopt);
|
||||
}
|
||||
void receive(Clock::time_point now) {
|
||||
std::array<std::uint8_t, 16384> buffer{};
|
||||
@@ -215,7 +215,8 @@ SessionClient::create(SessionClientConfig config) {
|
||||
if (config.host.empty() || config.port == 0 || config.grants.empty() ||
|
||||
config.grants.size() > 1024 || config.lead_ticks == 0 || config.lead_ticks > 64 ||
|
||||
config.first_retry.count() <= 0 || config.max_retry < config.first_retry ||
|
||||
config.attempt_deadline.count() <= 0)
|
||||
config.attempt_deadline.count() <= 0 || config.heartbeat_interval.count() <= 0 ||
|
||||
config.liveness_timeout <= 2 * config.heartbeat_interval)
|
||||
return std::unexpected(SessionClientError::configuration);
|
||||
return std::unique_ptr<SessionClient>(
|
||||
new SessionClient(std::make_unique<Impl>(std::move(config))));
|
||||
@@ -223,6 +224,7 @@ SessionClient::create(SessionClientConfig config) {
|
||||
|
||||
void SessionClient::poll(std::chrono::steady_clock::time_point now) {
|
||||
auto& m = *impl_;
|
||||
m.polled_at = now;
|
||||
if (m.phase == SessionPhase::exhausted)
|
||||
return;
|
||||
if (m.phase == SessionPhase::backoff) {
|
||||
@@ -236,6 +238,18 @@ void SessionClient::poll(std::chrono::steady_clock::time_point now) {
|
||||
m.disconnect(now, std::nullopt);
|
||||
return;
|
||||
}
|
||||
if (m.step == Impl::Step::established) {
|
||||
// A silent server (paused process, black-holed path) is treated as lost: the
|
||||
// presentation turns stale and the reconnect policy takes over.
|
||||
if (now - m.last_received > m.config.liveness_timeout) {
|
||||
m.disconnect(now, std::nullopt);
|
||||
return;
|
||||
}
|
||||
if (now - m.last_sent >= m.config.heartbeat_interval && !m.queue(encode_heartbeat(), now)) {
|
||||
m.disconnect(now, std::nullopt);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (m.step == Impl::Step::handshake) {
|
||||
const auto p = m.tls->handshake();
|
||||
if (!p) {
|
||||
@@ -251,7 +265,10 @@ void SessionClient::poll(std::chrono::steady_clock::time_point now) {
|
||||
m.disconnect(now, std::nullopt);
|
||||
return;
|
||||
}
|
||||
m.out.insert(m.out.end(), hello->begin(), hello->end());
|
||||
if (!m.queue(*hello, now)) {
|
||||
m.disconnect(now, std::nullopt);
|
||||
return;
|
||||
}
|
||||
m.step = Impl::Step::awaiting_welcome;
|
||||
}
|
||||
m.flush(now);
|
||||
@@ -263,13 +280,18 @@ void SessionClient::poll(std::chrono::steady_clock::time_point now) {
|
||||
|
||||
bool SessionClient::send(const client::Intention& intention) {
|
||||
auto& m = *impl_;
|
||||
if (m.phase != SessionPhase::established || !m.session || !m.view)
|
||||
// Bounded: no session, no view, too many unresolved intentions or a full channel
|
||||
// queue all refuse rather than buffer (the caller resamples later).
|
||||
if (m.phase != SessionPhase::established || !m.session || !m.view ||
|
||||
m.outstanding.size() >= max_outstanding)
|
||||
return false;
|
||||
const auto tick = m.view->server_tick + m.config.lead_ticks;
|
||||
const auto bytes =
|
||||
encode(make_intent(m.session->session, m.session->epoch,
|
||||
{intention.sequence, tick, intention.move_x, intention.move_y}));
|
||||
m.out.insert(m.out.end(), bytes.begin(), bytes.end());
|
||||
if (!m.queue(bytes, m.polled_at))
|
||||
return false;
|
||||
m.outstanding.push_back(intention.sequence);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
#include "evo/network/grant_ledger.hpp"
|
||||
#include <charconv>
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <system_error>
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <io.h>
|
||||
#else
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
namespace evo::network {
|
||||
namespace {
|
||||
std::FILE* open_file(const std::filesystem::path& path, const char* mode) {
|
||||
#ifdef _WIN32
|
||||
std::FILE* file = nullptr;
|
||||
return _wfopen_s(&file, path.c_str(), mode[0] == 'w' ? L"wb" : L"ab") == 0 ? file : nullptr;
|
||||
#else
|
||||
return std::fopen(path.c_str(), mode);
|
||||
#endif
|
||||
}
|
||||
bool sync(std::FILE* file) {
|
||||
if (std::fflush(file) != 0)
|
||||
return false;
|
||||
#ifdef _WIN32
|
||||
return _commit(_fileno(file)) == 0;
|
||||
#else
|
||||
return ::fsync(::fileno(file)) == 0;
|
||||
#endif
|
||||
}
|
||||
// Makes a rename durable on POSIX by syncing the containing directory.
|
||||
bool sync_directory([[maybe_unused]] const std::filesystem::path& dir) {
|
||||
#ifdef _WIN32
|
||||
return true; // NTFS metadata journaling; MoveFileEx is used with replace semantics.
|
||||
#else
|
||||
const int fd = ::open(dir.empty() ? "." : dir.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC);
|
||||
if (fd < 0)
|
||||
return false;
|
||||
const bool ok = ::fsync(fd) == 0;
|
||||
return ::close(fd) == 0 && ok;
|
||||
#endif
|
||||
}
|
||||
std::string line(ConsumedGrant g) {
|
||||
constexpr char digits[] = "0123456789abcdef";
|
||||
std::string out(16, '0');
|
||||
for (int i = 15; i >= 0; --i, g.grant_id >>= 4U)
|
||||
out[static_cast<std::size_t>(i)] = digits[g.grant_id & 0x0FU];
|
||||
return out + ' ' + std::to_string(g.expires_at) + '\n';
|
||||
}
|
||||
bool parse(const std::string& text, ConsumedGrant& out) { // One getline() line.
|
||||
if (text.size() < 18 || text[16] != ' ')
|
||||
return false;
|
||||
const auto [a, e1] = std::from_chars(text.data(), text.data() + 16, out.grant_id, 16);
|
||||
const auto [b, e2] =
|
||||
std::from_chars(text.data() + 17, text.data() + text.size(), out.expires_at);
|
||||
return e1 == std::errc{} && a == text.data() + 16 && e2 == std::errc{} &&
|
||||
b == text.data() + text.size() && out.grant_id != 0;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::expected<std::unique_ptr<GrantLedger>, LedgerError>
|
||||
GrantLedger::open(const std::filesystem::path& path, std::uint64_t now_seconds) {
|
||||
std::vector<ConsumedGrant> live;
|
||||
std::error_code error;
|
||||
if (std::filesystem::exists(path, error)) {
|
||||
std::ifstream in{path, std::ios::binary};
|
||||
if (!in)
|
||||
return std::unexpected(LedgerError::io);
|
||||
std::string text;
|
||||
std::size_t total = 0;
|
||||
while (std::getline(in, text)) {
|
||||
ConsumedGrant g;
|
||||
if (!parse(text, g))
|
||||
return std::unexpected(LedgerError::corrupt); // Never guess: fail closed.
|
||||
if (++total > max_entries)
|
||||
return std::unexpected(LedgerError::full);
|
||||
if (g.expires_at > now_seconds)
|
||||
live.push_back(g);
|
||||
}
|
||||
if (!in.eof())
|
||||
return std::unexpected(LedgerError::io);
|
||||
} else if (error) {
|
||||
return std::unexpected(LedgerError::io);
|
||||
}
|
||||
// Rewrite the unexpired entries atomically, then append to the result.
|
||||
const auto temporary = std::filesystem::path{path}.concat(".tmp");
|
||||
{
|
||||
std::FILE* out = open_file(temporary, "wb");
|
||||
if (out == nullptr)
|
||||
return std::unexpected(LedgerError::io);
|
||||
bool ok = true;
|
||||
for (const auto& g : live)
|
||||
ok = ok && std::fputs(line(g).c_str(), out) >= 0;
|
||||
ok = sync(out) && ok;
|
||||
if (std::fclose(out) != 0 || !ok)
|
||||
return std::unexpected(LedgerError::io);
|
||||
}
|
||||
std::filesystem::rename(temporary, path, error);
|
||||
if (error || !sync_directory(path.parent_path()))
|
||||
return std::unexpected(LedgerError::io);
|
||||
std::FILE* file = open_file(path, "ab");
|
||||
if (file == nullptr)
|
||||
return std::unexpected(LedgerError::io);
|
||||
return std::unique_ptr<GrantLedger>(new GrantLedger(file, std::move(live)));
|
||||
}
|
||||
|
||||
GrantLedger::~GrantLedger() {
|
||||
std::fclose(file_);
|
||||
}
|
||||
|
||||
std::expected<void, LedgerError> GrantLedger::record(ConsumedGrant grant) {
|
||||
if (entries_ >= max_entries)
|
||||
return std::unexpected(LedgerError::full);
|
||||
if (std::fputs(line(grant).c_str(), file_) < 0 || !sync(file_))
|
||||
return std::unexpected(LedgerError::io);
|
||||
++entries_;
|
||||
return {};
|
||||
}
|
||||
} // namespace evo::network
|
||||
+128
-83
@@ -1,5 +1,6 @@
|
||||
#include "evo/network/session_server.hpp"
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <utility>
|
||||
|
||||
namespace evo::network {
|
||||
@@ -7,6 +8,9 @@ namespace {
|
||||
using Clock = std::chrono::steady_clock;
|
||||
constexpr std::size_t max_accepts_per_poll = 4;
|
||||
constexpr std::size_t max_read_per_poll = 65536;
|
||||
// Carrier ids are 'I','N' + 48-bit epoch + 64-bit server ordinal: distinct for every
|
||||
// carried intention, independent of client-chosen sequences (which may be any u64).
|
||||
constexpr std::uint64_t max_carrier_epoch = (std::uint64_t{1} << 48) - 1;
|
||||
CloseReason close_reason(AdmissionError e) {
|
||||
switch (e) {
|
||||
case AdmissionError::duplicate_subject:
|
||||
@@ -24,8 +28,13 @@ CloseReason close_reason(ProtocolError e) {
|
||||
return e == ProtocolError::unsupported_version ? CloseReason::unsupported_version
|
||||
: CloseReason::malformed;
|
||||
}
|
||||
void append(std::vector<std::uint8_t>& out, const std::vector<std::uint8_t>& bytes) {
|
||||
out.insert(out.end(), bytes.begin(), bytes.end());
|
||||
RequestId carrier_id(std::uint64_t epoch, std::uint64_t ordinal) {
|
||||
RequestId id{'I', 'N'};
|
||||
for (unsigned i = 0; i < 6; ++i)
|
||||
id.at(2 + i) = static_cast<std::uint8_t>(epoch >> (8U * (5U - i)));
|
||||
for (unsigned i = 0; i < 8; ++i)
|
||||
id.at(8 + i) = static_cast<std::uint8_t>(ordinal >> (8U * (7U - i)));
|
||||
return id;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
@@ -34,18 +43,16 @@ struct SessionServer::Impl {
|
||||
struct Connection {
|
||||
std::uint64_t id{};
|
||||
std::unique_ptr<TcpStream> stream; // Declared first: outlives the channel.
|
||||
std::unique_ptr<TlsChannel> tls;
|
||||
std::unique_ptr<TlsChannel> tls; // Owns all queued output (ADR-0083 review fix).
|
||||
State state{State::handshake};
|
||||
Clock::time_point deadline;
|
||||
Clock::time_point deadline, last_received, last_sent, window_start;
|
||||
FrameReader reader;
|
||||
std::vector<std::uint8_t> out, view;
|
||||
std::vector<std::uint8_t> view; // Newest unsent view only; replaced, never queued.
|
||||
std::uint64_t session{}, subject{}, epoch{}, expires_at{};
|
||||
std::size_t window_bytes{};
|
||||
Clock::time_point window_start;
|
||||
bool dead{};
|
||||
};
|
||||
struct Pending {
|
||||
RequestId request{};
|
||||
std::uint64_t intended_tick{}, connection{}, sequence{};
|
||||
};
|
||||
struct Carried {
|
||||
@@ -53,13 +60,16 @@ struct SessionServer::Impl {
|
||||
std::uint64_t connection{}, sequence{};
|
||||
};
|
||||
|
||||
Impl(SessionServerConfig c, DevGrantKey k, std::unique_ptr<TcpListener> l)
|
||||
: config{std::move(c)}, key{std::move(k)}, listener{std::move(l)} {}
|
||||
Impl(SessionServerConfig c, DevGrantKey k, std::unique_ptr<TcpListener> l,
|
||||
std::unique_ptr<GrantLedger> g)
|
||||
: config{std::move(c)}, key{std::move(k)}, listener{std::move(l)}, ledger{std::move(g)} {}
|
||||
|
||||
SessionServerConfig config;
|
||||
DevGrantKey key;
|
||||
std::unique_ptr<TcpListener> listener;
|
||||
std::unique_ptr<GrantLedger> ledger;
|
||||
std::optional<SessionAdmission> admission;
|
||||
bool ledger_rejected{}; // Recorded grants exceed admission capacity: fail closed.
|
||||
std::vector<Connection> connections;
|
||||
std::deque<Pending> pending;
|
||||
std::vector<Carried> carried;
|
||||
@@ -67,8 +77,8 @@ struct SessionServer::Impl {
|
||||
std::optional<client::EnvironmentPresentation> environment;
|
||||
std::optional<client::OrganismPresentation> organisms;
|
||||
RegionFacts published{};
|
||||
std::uint64_t baseline{}, next_connection{1};
|
||||
Clock::time_point polled_at{}; // Time of the current poll, for Close deadlines.
|
||||
std::uint64_t baseline{}, next_connection{1}, next_carrier{1};
|
||||
Clock::time_point polled_at{};
|
||||
|
||||
Connection* find(std::uint64_t id) {
|
||||
const auto it = std::ranges::find_if(connections, [&](const Connection& c) {
|
||||
@@ -76,16 +86,23 @@ struct SessionServer::Impl {
|
||||
});
|
||||
return it == connections.end() ? nullptr : &*it;
|
||||
}
|
||||
// Every outbound frame goes through the channel's bounded queue; a full queue means
|
||||
// a consumer that cannot keep up, which is dropped rather than buffered further.
|
||||
void send(Connection& c, const std::vector<std::uint8_t>& frame) {
|
||||
if (c.dead || !c.tls->queue(frame))
|
||||
c.dead = true;
|
||||
c.last_sent = polled_at;
|
||||
}
|
||||
void close(Connection& c, CloseReason reason) {
|
||||
if (c.state == State::closing || c.dead)
|
||||
return;
|
||||
if (c.state == State::hello || c.state == State::established)
|
||||
append(c.out, encode(reason));
|
||||
else
|
||||
c.dead = true; // No channel yet to say anything over.
|
||||
events.push_back({c.state == State::established ? SessionEvent::Kind::closed
|
||||
: SessionEvent::Kind::refused,
|
||||
c.session, c.subject, reason});
|
||||
if (c.state == State::hello || c.state == State::established)
|
||||
send(c, encode(reason));
|
||||
else
|
||||
c.dead = true; // No channel yet to say anything over.
|
||||
c.state = State::closing;
|
||||
c.deadline = polled_at + std::chrono::seconds{1}; // Bounded flush of the Close frame.
|
||||
c.view.clear();
|
||||
@@ -96,8 +113,11 @@ struct SessionServer::Impl {
|
||||
return bytes;
|
||||
const ViewHeader header{c.session, c.epoch, published.simulated_tick,
|
||||
published.committed_tick, baseline};
|
||||
append(bytes, encode(header, *environment));
|
||||
append(bytes, encode(header, *organisms));
|
||||
const auto a = encode(header, *environment);
|
||||
const auto b = encode(header, *organisms);
|
||||
bytes.reserve(a.size() + b.size());
|
||||
bytes.insert(bytes.end(), a.begin(), a.end());
|
||||
bytes.insert(bytes.end(), b.begin(), b.end());
|
||||
return bytes;
|
||||
}
|
||||
void hello(Connection& c, const OwnedFrame& f, const RegionFacts& facts,
|
||||
@@ -115,24 +135,34 @@ struct SessionServer::Impl {
|
||||
close(c, CloseReason::server_shutdown); // No current authority: retry later.
|
||||
return;
|
||||
}
|
||||
if (facts.epoch > max_carrier_epoch) {
|
||||
close(c, CloseReason::capacity); // Carrier ids cannot name this epoch: refuse.
|
||||
return;
|
||||
}
|
||||
const auto grant = key.verify(message->grant);
|
||||
if (!grant) {
|
||||
close(c, CloseReason::unauthenticated);
|
||||
return;
|
||||
}
|
||||
c.subject = grant->claims().subject;
|
||||
const auto welcome =
|
||||
admission->open(*grant, now_seconds, facts.simulated_tick, facts.committed_tick);
|
||||
if (!welcome) {
|
||||
c.subject = grant->claims().subject;
|
||||
close(c, close_reason(welcome.error()));
|
||||
return;
|
||||
}
|
||||
// Durable before Welcome: a replacement process must refuse this grant too.
|
||||
if (!ledger->record({grant->claims().grant_id, grant->claims().expires_at})) {
|
||||
// The grant stays consumed in memory (fail closed); only the session is undone.
|
||||
static_cast<void>(admission->close(welcome->session).has_value());
|
||||
close(c, CloseReason::server_shutdown);
|
||||
return;
|
||||
}
|
||||
c.session = welcome->session;
|
||||
c.subject = grant->claims().subject;
|
||||
c.epoch = welcome->epoch;
|
||||
c.expires_at = welcome->grant_expires_at;
|
||||
c.state = State::established;
|
||||
append(c.out, encode(*welcome));
|
||||
send(c, encode(*welcome));
|
||||
c.view = views_for(c); // Fresh baseline for every new session.
|
||||
events.push_back({SessionEvent::Kind::opened, c.session, c.subject, {}});
|
||||
}
|
||||
@@ -152,16 +182,8 @@ struct SessionServer::Impl {
|
||||
result = admission->admit(*m, facts.simulated_tick, now_seconds);
|
||||
}
|
||||
if (result.status == IntentStatus::admitted)
|
||||
pending.push_back({request_id(c, m->sequence), m->tick, c.id, m->sequence});
|
||||
append(c.out, encode(result));
|
||||
}
|
||||
RequestId request_id(const Connection& c, std::uint64_t sequence) const {
|
||||
RequestId id{'I', 'N', 'T', '1'};
|
||||
const std::array<std::uint64_t, 3> parts{c.epoch, c.session, sequence};
|
||||
for (std::size_t p = 0; p < parts.size(); ++p)
|
||||
for (unsigned i = 0; i < 4; ++i)
|
||||
id.at(4 + 4 * p + i) = static_cast<std::uint8_t>(parts.at(p) >> (8U * (3U - i)));
|
||||
return id;
|
||||
pending.push_back({m->tick, c.id, m->sequence});
|
||||
send(c, encode(result));
|
||||
}
|
||||
void receive(Connection& c, const RegionFacts& facts, std::uint64_t now_seconds,
|
||||
Clock::time_point now) {
|
||||
@@ -194,12 +216,17 @@ struct SessionServer::Impl {
|
||||
}
|
||||
rest = rest.subspan(*taken);
|
||||
while (auto f = c.reader.take()) {
|
||||
c.last_received = now;
|
||||
if (c.state == State::hello)
|
||||
hello(c, *f, facts, now_seconds);
|
||||
else if (f->type == MessageType::intent)
|
||||
intent(c, *f, facts, now_seconds);
|
||||
else if (f->type == MessageType::close) {
|
||||
else if (f->type == MessageType::heartbeat) {
|
||||
if (!decode_heartbeat({f->type, f->payload}))
|
||||
close(c, CloseReason::malformed);
|
||||
} else if (f->type == MessageType::close) {
|
||||
c.state = State::closing; // Peer is leaving; nothing to say.
|
||||
c.deadline = now + std::chrono::seconds{1};
|
||||
events.push_back({SessionEvent::Kind::closed, c.session, c.subject,
|
||||
CloseReason::normal});
|
||||
} else
|
||||
@@ -211,27 +238,25 @@ struct SessionServer::Impl {
|
||||
}
|
||||
}
|
||||
void transmit(Connection& c) {
|
||||
if (c.out.empty() && !c.view.empty() && c.state == State::established)
|
||||
c.out.swap(c.view); // Only the newest view waits; it is never queued twice.
|
||||
std::size_t sent = 0;
|
||||
while (sent < c.out.size()) {
|
||||
const auto n = c.tls->send(std::span{c.out}.subspan(sent));
|
||||
if (!n) {
|
||||
c.dead = true;
|
||||
return;
|
||||
}
|
||||
if (*n == 0)
|
||||
break;
|
||||
sent += *n;
|
||||
if (c.state == State::established && c.tls->queued() == 0 && !c.view.empty()) {
|
||||
send(c, c.view);
|
||||
c.view.clear();
|
||||
}
|
||||
c.out.erase(c.out.begin(), c.out.begin() + static_cast<std::ptrdiff_t>(sent));
|
||||
if (c.out.size() + c.view.size() > config.max_output_bytes)
|
||||
if (c.dead)
|
||||
return;
|
||||
if (c.tls->queued() + c.view.size() > config.max_output_bytes) {
|
||||
c.dead = true; // Slow consumer: drop rather than buffer without bound.
|
||||
if (c.state == State::closing && c.out.empty()) {
|
||||
// Best-effort close_notify: the connection is dropped either way.
|
||||
static_cast<void>(c.tls->close().has_value());
|
||||
c.dead = true;
|
||||
return;
|
||||
}
|
||||
if (c.state == State::closing) {
|
||||
// close() flushes queued frames (the Close frame included) before close_notify.
|
||||
const auto closed = c.tls->close();
|
||||
if (!closed || *closed == TlsProgress::done)
|
||||
c.dead = true;
|
||||
return;
|
||||
}
|
||||
if (!c.tls->flush())
|
||||
c.dead = true;
|
||||
}
|
||||
void step(Connection& c, const RegionFacts& facts, std::uint64_t now_seconds,
|
||||
Clock::time_point now) {
|
||||
@@ -250,10 +275,16 @@ struct SessionServer::Impl {
|
||||
}
|
||||
if ((c.state == State::handshake || c.state == State::hello) && now > c.deadline)
|
||||
close(c, CloseReason::unauthenticated);
|
||||
if (c.state == State::established && now_seconds >= c.expires_at)
|
||||
close(c, CloseReason::expired);
|
||||
if (c.state == State::established && admission && c.epoch != admission->epoch())
|
||||
close(c, CloseReason::stale_epoch);
|
||||
if (c.state == State::established) {
|
||||
if (now_seconds >= c.expires_at)
|
||||
close(c, CloseReason::expired);
|
||||
else if (admission && c.epoch != admission->epoch())
|
||||
close(c, CloseReason::stale_epoch);
|
||||
else if (now - c.last_received > config.liveness_timeout)
|
||||
close(c, CloseReason::timeout); // Silent peer: free the subject for reconnect.
|
||||
else if (now - c.last_sent >= config.heartbeat_interval)
|
||||
send(c, encode_heartbeat()); // Liveness without requiring simulation ticks.
|
||||
}
|
||||
if (c.state == State::hello || c.state == State::established)
|
||||
receive(c, facts, now_seconds, now);
|
||||
if (!c.dead && c.state != State::handshake)
|
||||
@@ -270,17 +301,44 @@ struct SessionServer::Impl {
|
||||
}
|
||||
std::erase_if(connections, [](const Connection& c) { return c.dead; });
|
||||
}
|
||||
void open_admission(const RegionFacts& facts, std::uint64_t now_seconds) {
|
||||
auto config_with_epoch = config.admission;
|
||||
config_with_epoch.epoch = facts.epoch;
|
||||
auto created = SessionAdmission::create(config_with_epoch);
|
||||
if (!created)
|
||||
return;
|
||||
// Grants consumed before this process started stay consumed.
|
||||
if (!created->remember(ledger->loaded(), now_seconds)) {
|
||||
ledger_rejected = true; // More recorded grants than admission can hold.
|
||||
return;
|
||||
}
|
||||
admission.emplace(std::move(*created));
|
||||
}
|
||||
void finish(const RequestId& request, const IntentOutcome& outcome) {
|
||||
const auto it =
|
||||
std::ranges::find_if(carried, [&](const Carried& c) { return c.request == request; });
|
||||
if (it == carried.end())
|
||||
return;
|
||||
auto out = outcome;
|
||||
out.sequence = it->sequence; // The client's own sequence, carried separately.
|
||||
if (auto* c = find(it->connection))
|
||||
send(*c, encode(out));
|
||||
carried.erase(it);
|
||||
}
|
||||
};
|
||||
|
||||
SessionServer::SessionServer(std::unique_ptr<Impl> impl) : impl_{std::move(impl)} {}
|
||||
SessionServer::~SessionServer() = default;
|
||||
|
||||
std::expected<std::unique_ptr<SessionServer>, SessionServerError>
|
||||
SessionServer::create(SessionServerConfig config, DevGrantKey key) {
|
||||
SessionServer::create(SessionServerConfig config, DevGrantKey key, std::uint64_t now_seconds) {
|
||||
if (config.max_connections == 0 || config.max_connections > 64 ||
|
||||
config.max_pending_intents == 0 || config.max_carried_intents == 0 ||
|
||||
config.max_output_bytes < 2 * (frame_header_bytes + max_frame_payload) ||
|
||||
config.max_input_bytes_per_second < frame_header_bytes + max_grant_bytes + 6)
|
||||
config.max_output_bytes > TlsChannel::max_queued_bytes ||
|
||||
config.max_input_bytes_per_second < frame_header_bytes + max_grant_bytes + 6 ||
|
||||
config.heartbeat_interval.count() <= 0 ||
|
||||
config.liveness_timeout <= 2 * config.heartbeat_interval || config.grant_ledger.empty())
|
||||
return std::unexpected(SessionServerError::configuration);
|
||||
// Validate the admission limits now, with a placeholder epoch.
|
||||
auto probe = config.admission;
|
||||
@@ -298,11 +356,14 @@ SessionServer::create(SessionServerConfig config, DevGrantKey key) {
|
||||
} null;
|
||||
if (!TlsChannel::server(config.credentials, null))
|
||||
return std::unexpected(SessionServerError::credentials);
|
||||
auto ledger = GrantLedger::open(config.grant_ledger, now_seconds);
|
||||
if (!ledger)
|
||||
return std::unexpected(SessionServerError::ledger);
|
||||
auto listener = TcpListener::listen(config.listen);
|
||||
if (!listener)
|
||||
return std::unexpected(SessionServerError::listen);
|
||||
return std::unique_ptr<SessionServer>(new SessionServer(
|
||||
std::make_unique<Impl>(std::move(config), std::move(key), std::move(*listener))));
|
||||
return std::unique_ptr<SessionServer>(new SessionServer(std::make_unique<Impl>(
|
||||
std::move(config), std::move(key), std::move(*listener), std::move(*ledger))));
|
||||
}
|
||||
|
||||
std::uint16_t SessionServer::port() const noexcept {
|
||||
@@ -313,12 +374,9 @@ void SessionServer::poll(const RegionFacts& facts, std::uint64_t now_seconds,
|
||||
std::chrono::steady_clock::time_point now) {
|
||||
auto& m = *impl_;
|
||||
m.polled_at = now;
|
||||
if (facts.ready && facts.epoch != 0) {
|
||||
if (facts.ready && facts.epoch != 0 && !m.ledger_rejected) {
|
||||
if (!m.admission) {
|
||||
auto config = m.config.admission;
|
||||
config.epoch = facts.epoch;
|
||||
if (auto created = SessionAdmission::create(config))
|
||||
m.admission.emplace(std::move(*created));
|
||||
m.open_admission(facts, now_seconds);
|
||||
} else if (facts.epoch > m.admission->epoch()) {
|
||||
// Strictly newer (checked above), so the advance cannot be refused.
|
||||
static_cast<void>(m.admission->advance_epoch(facts.epoch).has_value());
|
||||
@@ -340,7 +398,7 @@ void SessionServer::poll(const RegionFacts& facts, std::uint64_t now_seconds,
|
||||
continue;
|
||||
c.tls = std::move(*tls);
|
||||
c.deadline = now + m.config.hello_deadline;
|
||||
c.window_start = now;
|
||||
c.window_start = c.last_received = c.last_sent = now;
|
||||
m.connections.push_back(std::move(c));
|
||||
}
|
||||
for (auto& c : m.connections)
|
||||
@@ -364,13 +422,15 @@ void SessionServer::publish(const client::EnvironmentPresentation& environment,
|
||||
|
||||
std::optional<RequestId> SessionServer::carry(std::uint64_t tick) {
|
||||
auto& m = *impl_;
|
||||
if (m.carried.size() >= m.config.max_carried_intents)
|
||||
return std::nullopt;
|
||||
if (m.carried.size() >= m.config.max_carried_intents || !m.admission ||
|
||||
m.next_carrier == std::numeric_limits<std::uint64_t>::max())
|
||||
return std::nullopt; // Ordinal exhaustion is explicit, never a wrap.
|
||||
const auto it = std::ranges::find_if(
|
||||
m.pending, [&](const Impl::Pending& p) { return p.intended_tick <= tick; });
|
||||
if (it == m.pending.end())
|
||||
return std::nullopt;
|
||||
const Impl::Carried carried{it->request, it->connection, it->sequence};
|
||||
const Impl::Carried carried{carrier_id(m.admission->epoch(), m.next_carrier++), it->connection,
|
||||
it->sequence};
|
||||
m.pending.erase(it);
|
||||
m.carried.push_back(carried);
|
||||
return carried.request;
|
||||
@@ -378,26 +438,11 @@ std::optional<RequestId> SessionServer::carry(std::uint64_t tick) {
|
||||
|
||||
void SessionServer::committed(const RequestId& request, std::uint64_t tick,
|
||||
std::uint64_t commit_sequence) {
|
||||
auto& m = *impl_;
|
||||
const auto it = std::ranges::find_if(
|
||||
m.carried, [&](const Impl::Carried& c) { return c.request == request; });
|
||||
if (it == m.carried.end())
|
||||
return;
|
||||
if (auto* c = m.find(it->connection))
|
||||
append(c->out, encode(IntentOutcome{it->sequence, IntentFinality::committed_no_effect, tick,
|
||||
commit_sequence}));
|
||||
m.carried.erase(it);
|
||||
impl_->finish(request, {0, IntentFinality::committed_no_effect, tick, commit_sequence});
|
||||
}
|
||||
|
||||
void SessionServer::not_committed(const RequestId& request) {
|
||||
auto& m = *impl_;
|
||||
const auto it = std::ranges::find_if(
|
||||
m.carried, [&](const Impl::Carried& c) { return c.request == request; });
|
||||
if (it == m.carried.end())
|
||||
return;
|
||||
if (auto* c = m.find(it->connection))
|
||||
append(c->out, encode(IntentOutcome{it->sequence, IntentFinality::not_committed, 0, 0}));
|
||||
m.carried.erase(it);
|
||||
impl_->finish(request, {0, IntentFinality::not_committed, 0, 0});
|
||||
}
|
||||
|
||||
std::vector<SessionEvent> SessionServer::take_events() {
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#include <mbedtls/x509_crt.h>
|
||||
#include <optional>
|
||||
#include <psa/crypto.h>
|
||||
#include <vector>
|
||||
|
||||
// The owned user config (third_party/mbedtls/evo_mbedtls_config.h) must be in effect.
|
||||
#if defined(MBEDTLS_SSL_PROTO_TLS1_2) || defined(MBEDTLS_SSL_PROTO_DTLS) || \
|
||||
@@ -21,6 +22,8 @@ namespace {
|
||||
// ADR-0082 profile: TLS 1.3 only, AEAD suites, ECDHE groups, ephemeral key exchange.
|
||||
constexpr int suites[]{MBEDTLS_TLS1_3_AES_128_GCM_SHA256, MBEDTLS_TLS1_3_CHACHA20_POLY1305_SHA256,
|
||||
0};
|
||||
// One TLS record's plaintext per write call (the TLS 1.3 maximum fragment).
|
||||
constexpr std::size_t record_plaintext = 16384;
|
||||
constexpr std::uint16_t groups[]{MBEDTLS_SSL_IANA_TLS_GROUP_X25519,
|
||||
MBEDTLS_SSL_IANA_TLS_GROUP_SECP256R1, 0};
|
||||
const unsigned char* bytes_of(const std::string& text) {
|
||||
@@ -105,6 +108,8 @@ struct TlsChannel::Impl {
|
||||
mbedtls_pk_context key;
|
||||
bool transport_failed{};
|
||||
std::optional<TlsError> error;
|
||||
std::vector<std::uint8_t> outbound; // Queued plaintext; the front record may be in flight.
|
||||
std::size_t in_flight{}; // Length of an interrupted write; 0 when none.
|
||||
};
|
||||
|
||||
TlsChannel::TlsChannel(std::unique_ptr<Impl> impl) : impl_{std::move(impl)} {}
|
||||
@@ -158,15 +163,39 @@ std::expected<TlsProgress, TlsError> TlsChannel::handshake() {
|
||||
return impl_->fail_from(result, TlsError::handshake);
|
||||
}
|
||||
|
||||
std::expected<std::size_t, TlsError> TlsChannel::send(std::span<const std::uint8_t> bytes) {
|
||||
std::expected<void, TlsError> TlsChannel::queue(std::span<const std::uint8_t> bytes) {
|
||||
if (const auto latched = impl_->error)
|
||||
return std::unexpected(*latched);
|
||||
const int result = mbedtls_ssl_write(&impl_->ssl, bytes.data(), bytes.size());
|
||||
if (result >= 0)
|
||||
return static_cast<std::size_t>(result);
|
||||
if (result == MBEDTLS_ERR_SSL_WANT_READ || result == MBEDTLS_ERR_SSL_WANT_WRITE)
|
||||
return 0;
|
||||
return impl_->fail_from(result, TlsError::record);
|
||||
if (bytes.size() > max_queued_bytes - impl_->outbound.size())
|
||||
return std::unexpected(TlsError::backpressure); // Nothing queued: not latched.
|
||||
impl_->outbound.insert(impl_->outbound.end(), bytes.begin(), bytes.end());
|
||||
return {};
|
||||
}
|
||||
|
||||
std::expected<TlsProgress, TlsError> TlsChannel::flush() {
|
||||
auto& m = *impl_;
|
||||
if (const auto latched = m.error)
|
||||
return std::unexpected(*latched);
|
||||
while (!m.outbound.empty()) {
|
||||
// A record interrupted by WANT_READ/WANT_WRITE keeps its length; bytes queued
|
||||
// since then sit behind it and do not change the retried call.
|
||||
if (m.in_flight == 0)
|
||||
m.in_flight = std::min(m.outbound.size(), record_plaintext);
|
||||
const int result = mbedtls_ssl_write(&m.ssl, m.outbound.data(), m.in_flight);
|
||||
if (result == MBEDTLS_ERR_SSL_WANT_READ || result == MBEDTLS_ERR_SSL_WANT_WRITE)
|
||||
return TlsProgress::want_io;
|
||||
if (result < 0)
|
||||
return m.fail_from(result, TlsError::record);
|
||||
const auto written = std::min(static_cast<std::size_t>(result), m.in_flight);
|
||||
m.outbound.erase(m.outbound.begin(),
|
||||
m.outbound.begin() + static_cast<std::ptrdiff_t>(written));
|
||||
m.in_flight = 0;
|
||||
}
|
||||
return TlsProgress::done;
|
||||
}
|
||||
|
||||
std::size_t TlsChannel::queued() const noexcept {
|
||||
return impl_->outbound.size();
|
||||
}
|
||||
|
||||
std::expected<std::size_t, TlsError> TlsChannel::receive(std::span<std::uint8_t> bytes) {
|
||||
@@ -185,6 +214,9 @@ std::expected<std::size_t, TlsError> TlsChannel::receive(std::span<std::uint8_t>
|
||||
std::expected<TlsProgress, TlsError> TlsChannel::close() {
|
||||
if (const auto latched = impl_->error)
|
||||
return std::unexpected(*latched);
|
||||
// Queued plaintext (including an interrupted record) is sent before close_notify.
|
||||
if (const auto pending = flush(); !pending || *pending == TlsProgress::want_io)
|
||||
return pending;
|
||||
const int result = mbedtls_ssl_close_notify(&impl_->ssl);
|
||||
if (result == 0)
|
||||
return TlsProgress::done;
|
||||
|
||||
Reference in New Issue
Block a user