Fix M9.4-M9.7 review findings in the dev session stack
- TLS: replace send() with a bounded queue/flush; an interrupted record is retried with identical arguments and later frames wait behind it, so no queued frame is lost or reordered; close() flushes first. - Carriers: server-issued 'I','N' | 48-bit epoch | 64-bit ordinal ids replace the aliasing low-32-bit packing; client sequences are kept separately for replies; out-of-range epochs are refused, never truncated. - Grants: a durable, fsynced, fail-closed consumed-grant ledger (--session-grant-ledger, /objects/dev-session-grants.v1 in dev) keeps single-use grants consumed across process and pod restarts. - Liveness: protocol v1 (undeployed) gains Heartbeat and the timeout close reason; both sides heartbeat when idle and treat 5 s of silence as dead. - Client sends are bounded and unresolved tracking starts at send time. - ci-client configures with an explicit C compiler. Each finding has a regression test; ADR-0083, the schema, operations doc and VALIDATION record the corrections. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+226
-11
@@ -1,11 +1,16 @@
|
||||
#include "evo/network/grant_ledger.hpp"
|
||||
#include "evo/network/session_client.hpp"
|
||||
#include "evo/network/session_server.hpp"
|
||||
#include "evo/network/tcp.hpp"
|
||||
#include "network_pki.hpp"
|
||||
#include "network_test.hpp"
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <functional>
|
||||
#include <limits>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
@@ -57,12 +62,28 @@ client::OrganismPresentation no_organisms(std::uint64_t tick) {
|
||||
return std::move(*v);
|
||||
}
|
||||
|
||||
std::filesystem::path fresh_ledger() {
|
||||
static std::atomic<unsigned> counter{0};
|
||||
const auto stamp = Clock::now().time_since_epoch().count();
|
||||
return std::filesystem::temp_directory_path() /
|
||||
("evo-grant-ledger-" + std::to_string(stamp) + "-" + std::to_string(++counter));
|
||||
}
|
||||
struct Harness {
|
||||
explicit Harness(std::size_t max_sessions = 4,
|
||||
std::chrono::milliseconds hello = std::chrono::milliseconds{3000}) {
|
||||
std::chrono::milliseconds hello = std::chrono::milliseconds{3000},
|
||||
std::function<void(SessionServerConfig&)> server_tweak = {},
|
||||
std::function<void(SessionClientConfig&)> client_tweak = {})
|
||||
: tweak_server{std::move(server_tweak)}, tweak_client{std::move(client_tweak)} {
|
||||
server = make_server(0, max_sessions, hello);
|
||||
port = server->port();
|
||||
}
|
||||
~Harness() {
|
||||
server.reset();
|
||||
std::error_code ignored;
|
||||
std::filesystem::remove(ledger, ignored);
|
||||
}
|
||||
Harness(const Harness&) = delete;
|
||||
Harness& operator=(const Harness&) = delete;
|
||||
std::unique_ptr<SessionServer> make_server(std::uint16_t at, std::size_t max_sessions,
|
||||
std::chrono::milliseconds hello) {
|
||||
SessionServerConfig c;
|
||||
@@ -75,7 +96,10 @@ struct Harness {
|
||||
c.admission.window_ticks = 8;
|
||||
c.admission.max_intents_per_window = 8;
|
||||
c.hello_deadline = hello;
|
||||
auto s = SessionServer::create(std::move(c), grant_key());
|
||||
c.grant_ledger = ledger; // Shared by restarts of this harness's server.
|
||||
if (tweak_server)
|
||||
tweak_server(c);
|
||||
auto s = SessionServer::create(std::move(c), grant_key(), now_seconds);
|
||||
CHECK(s);
|
||||
return std::move(*s);
|
||||
}
|
||||
@@ -88,6 +112,8 @@ struct Harness {
|
||||
c.first_retry = std::chrono::milliseconds{50};
|
||||
c.max_retry = std::chrono::milliseconds{200};
|
||||
c.attempt_deadline = std::chrono::milliseconds{3000};
|
||||
if (tweak_client)
|
||||
tweak_client(c);
|
||||
auto s = SessionClient::create(std::move(c));
|
||||
CHECK(s);
|
||||
clients.push_back(s->get());
|
||||
@@ -119,6 +145,9 @@ struct Harness {
|
||||
auto e = c.take_events();
|
||||
into.insert(into.end(), e.begin(), e.end());
|
||||
}
|
||||
std::function<void(SessionServerConfig&)> tweak_server;
|
||||
std::function<void(SessionClientConfig&)> tweak_client;
|
||||
std::filesystem::path ledger = fresh_ledger();
|
||||
Pki pki;
|
||||
RegionFacts facts{true, 1, 100, 90, 5};
|
||||
std::unique_ptr<SessionServer> server;
|
||||
@@ -167,7 +196,9 @@ void connect_view_intents_and_outcomes() {
|
||||
CHECK(last(events, ClientEvent::Kind::result)->result.status == IntentStatus::admitted);
|
||||
CHECK(!h.server->carry(105)); // Intended tick 102 + 4 = 106.
|
||||
const auto request = h.server->carry(106);
|
||||
CHECK(must(request)[0] == 'I' && must(request)[3] == '1');
|
||||
// 'I','N' + 48-bit epoch (1) + 64-bit server ordinal (first carrier: 1).
|
||||
CHECK(must(request)[0] == 'I' && must(request)[1] == 'N' && must(request)[7] == 1 &&
|
||||
must(request)[15] == 1);
|
||||
CHECK(!h.server->carry(200)); // Carried once.
|
||||
h.server->committed(must(request), 106, 6);
|
||||
CHECK(h.until([&] {
|
||||
@@ -336,24 +367,42 @@ void abuse() {
|
||||
h.tick();
|
||||
CHECK(h.until([&] { return c->phase() == SessionPhase::established; }));
|
||||
CHECK(c->grants_left() == 2);
|
||||
// Flooding: the byte-rate limit closes the connection. The flooding client may see
|
||||
// a transport reset before it reads the Close frame, so check the server's decision.
|
||||
// The client never buffers without bound: with the server not answering, at most
|
||||
// max_outstanding intentions may be unresolved; further sends are refused.
|
||||
CHECK(h.until([&] { return c->latest_view().has_value(); }));
|
||||
(void)h.server->take_events();
|
||||
std::size_t accepted = 0;
|
||||
for (std::uint64_t s = 10; s < 2000; ++s)
|
||||
CHECK(c->send({s, 0, 0, 0}));
|
||||
accepted += c->send({s, 0, 0, 0}) ? 1 : 0;
|
||||
CHECK(accepted == SessionClient::max_outstanding);
|
||||
|
||||
// Flooding: a raw TLS peer (no client-side bounds) exceeds the byte-rate limit after
|
||||
// a valid Hello; the server closes it. Check the server's decision, since the flooding
|
||||
// peer may see a reset before it could read the Close frame.
|
||||
(void)h.server->take_events();
|
||||
auto raw = TcpStream::connect("127.0.0.1", h.port);
|
||||
CHECK(raw);
|
||||
auto tls = TlsChannel::client({h.pki.authority, server_name}, **raw);
|
||||
CHECK(tls);
|
||||
const auto flood_grant = grants(9, 1, 900).front();
|
||||
const auto hello = encode(Hello{0, {flood_grant.begin(), flood_grant.end()}});
|
||||
CHECK(hello);
|
||||
bool flooded = false;
|
||||
std::vector<SessionEvent> server_events;
|
||||
CHECK(h.until([&] {
|
||||
auto e = h.server->take_events();
|
||||
server_events.insert(server_events.end(), e.begin(), e.end());
|
||||
if (const auto p = (*tls)->handshake(); p && *p == TlsProgress::done && !flooded) {
|
||||
flooded = hello && (*tls)->queue(*hello).has_value();
|
||||
const auto beat = encode_heartbeat();
|
||||
for (int i = 0; i < 7000 && flooded; ++i)
|
||||
flooded = (*tls)->queue(beat).has_value();
|
||||
}
|
||||
if (flooded)
|
||||
static_cast<void>((*tls)->flush().has_value()); // Best effort: may be reset.
|
||||
return std::ranges::any_of(server_events, [](const SessionEvent& x) {
|
||||
return x.kind == SessionEvent::Kind::closed && x.reason == CloseReason::rate_limited;
|
||||
});
|
||||
}));
|
||||
CHECK(h.until([&] {
|
||||
h.collect(*c, events);
|
||||
return c->phase() != SessionPhase::established;
|
||||
}));
|
||||
|
||||
// Plaintext and silent connections are dropped without a session: the server
|
||||
// closes the socket (plaintext at the TLS handshake, silence at the deadline).
|
||||
@@ -381,6 +430,169 @@ void abuse() {
|
||||
CHECK(quiet.until([&] { return quiet.server->connections() == 0; }));
|
||||
CHECK(quiet.server->sessions() == 0);
|
||||
}
|
||||
// Review M9.4-M9.7 #2: client sequences 2^32 apart must get distinct carriers and
|
||||
// each final outcome must reach its own sequence.
|
||||
void carriers_never_alias() {
|
||||
Harness h;
|
||||
h.tick();
|
||||
auto c = h.client(grants(7, 2, 100));
|
||||
std::vector<ClientEvent> events;
|
||||
CHECK(h.until([&] { return c->latest_view().has_value(); }));
|
||||
constexpr std::uint64_t far = (std::uint64_t{1} << 32) + 1;
|
||||
CHECK(c->send({1, 0, 1000, 0}) && c->send({far, 0, 0, 1000}));
|
||||
CHECK(h.until([&] {
|
||||
h.collect(*c, events);
|
||||
return std::ranges::count_if(events, [](const ClientEvent& e) {
|
||||
return e.kind == ClientEvent::Kind::result &&
|
||||
e.result.status == IntentStatus::admitted;
|
||||
}) == 2;
|
||||
}));
|
||||
const auto first = h.server->carry(1000);
|
||||
const auto second = h.server->carry(1000);
|
||||
CHECK(must(first) != must(second));
|
||||
CHECK(must(first)[0] == 'I' && must(first)[1] == 'N' && must(first)[7] == 1); // Epoch 1.
|
||||
h.server->not_committed(must(second));
|
||||
h.server->committed(must(first), 106, 6);
|
||||
CHECK(h.until([&] {
|
||||
h.collect(*c, events);
|
||||
return std::ranges::count_if(events, [](const ClientEvent& e) {
|
||||
return e.kind == ClientEvent::Kind::outcome;
|
||||
}) == 2;
|
||||
}));
|
||||
for (const auto& e : events)
|
||||
if (e.kind == ClientEvent::Kind::outcome)
|
||||
CHECK((e.sequence == far && e.outcome.finality == IntentFinality::not_committed) ||
|
||||
(e.sequence == 1 && e.outcome.finality == IntentFinality::committed_no_effect &&
|
||||
e.outcome.commit_sequence == 6));
|
||||
}
|
||||
|
||||
// Review M9.4-M9.7 #3: a grant consumed before a restart stays consumed after it.
|
||||
void grants_stay_consumed_across_restarts() {
|
||||
Harness h;
|
||||
h.tick();
|
||||
auto first = h.client(grants(7, 1, 100));
|
||||
CHECK(h.until([&] { return first->phase() == SessionPhase::established; }));
|
||||
h.clients.clear();
|
||||
first.reset();
|
||||
h.server.reset(); // Process restart: only the ledger file survives.
|
||||
h.facts = {true, 2, 40, 40, 2};
|
||||
h.server = h.make_server(h.port, 4, std::chrono::milliseconds{3000});
|
||||
h.tick();
|
||||
auto replay = h.client(grants(7, 1, 100)); // The exact consumed grant.
|
||||
std::vector<ClientEvent> events;
|
||||
CHECK(h.until([&] {
|
||||
h.collect(*replay, events);
|
||||
return has(events, ClientEvent::Kind::closed);
|
||||
}));
|
||||
CHECK(last(events, ClientEvent::Kind::closed)->reason == CloseReason::unauthenticated);
|
||||
auto fresh = h.client(grants(7, 1, 101));
|
||||
CHECK(h.until([&] { return fresh->phase() == SessionPhase::established; }));
|
||||
CHECK(must(fresh->welcome()).epoch == 2);
|
||||
|
||||
// The ledger fails closed on corruption and drops expired entries when opened.
|
||||
const auto path = fresh_ledger();
|
||||
{
|
||||
std::ofstream out{path, std::ios::binary};
|
||||
out << "not a ledger line\n";
|
||||
}
|
||||
SessionServerConfig bad;
|
||||
bad.listen = "127.0.0.1:0";
|
||||
bad.credentials = h.pki.server;
|
||||
bad.admission.world = identity;
|
||||
bad.admission.region = region;
|
||||
bad.grant_ledger = path;
|
||||
CHECK(
|
||||
refused(SessionServer::create(bad, grant_key(), now_seconds), SessionServerError::ledger));
|
||||
{
|
||||
std::ofstream out{path, std::ios::binary | std::ios::trunc}; // LF only.
|
||||
out << "0000000000000005 " << now_seconds - 1 << "\n"
|
||||
<< "0000000000000006 " << now_seconds + 10 << "\n";
|
||||
}
|
||||
{
|
||||
const auto ledger = GrantLedger::open(path, now_seconds);
|
||||
CHECK(ledger && (*ledger)->loaded().size() == 1 && (*ledger)->loaded()[0].grant_id == 6);
|
||||
CHECK(ledger && (*ledger)->record({7, now_seconds + 20}));
|
||||
}
|
||||
const auto reopened = GrantLedger::open(path, now_seconds);
|
||||
CHECK(reopened && (*reopened)->loaded().size() == 2);
|
||||
std::error_code ignored;
|
||||
std::filesystem::remove(path, ignored);
|
||||
}
|
||||
|
||||
// Review M9.4-M9.7 #4: liveness after Welcome, both directions, without spurious drops.
|
||||
void liveness() {
|
||||
const auto fast_server = [](SessionServerConfig& c) {
|
||||
c.heartbeat_interval = std::chrono::milliseconds{100};
|
||||
c.liveness_timeout = std::chrono::milliseconds{500};
|
||||
};
|
||||
const auto fast_client = [](SessionClientConfig& c) {
|
||||
c.heartbeat_interval = std::chrono::milliseconds{100};
|
||||
c.liveness_timeout = std::chrono::milliseconds{500};
|
||||
};
|
||||
{
|
||||
// Healthy but idle: no ticks are published; heartbeats keep both sides alive.
|
||||
Harness h{4, std::chrono::milliseconds{3000}, fast_server, fast_client};
|
||||
h.tick();
|
||||
auto c = h.client(grants(7, 2, 100));
|
||||
std::vector<ClientEvent> events;
|
||||
CHECK(h.until([&] { return c->latest_view().has_value(); }));
|
||||
const auto start = Clock::now();
|
||||
CHECK(!h.until(
|
||||
[&] {
|
||||
h.collect(*c, events);
|
||||
return has(events, ClientEvent::Kind::lost) ||
|
||||
has(events, ClientEvent::Kind::closed);
|
||||
},
|
||||
std::chrono::milliseconds{1500}));
|
||||
CHECK(Clock::now() - start >= std::chrono::milliseconds{1500});
|
||||
CHECK(c->phase() == SessionPhase::established &&
|
||||
c->environment().freshness() == client::Freshness::current);
|
||||
}
|
||||
{
|
||||
// Silent server (socket open, never polled): the client times out, keeps the view
|
||||
// as stale, reports the sent-but-unresolved intention as unknown, and reconnects.
|
||||
Harness h{4, std::chrono::milliseconds{3000}, fast_server, fast_client};
|
||||
h.tick();
|
||||
auto c = h.client(grants(7, 3, 100));
|
||||
std::vector<ClientEvent> events;
|
||||
CHECK(h.until([&] { return c->latest_view().has_value(); }));
|
||||
CHECK(c->send({5, 0, 1000, 0})); // Never answered: the server is not polled.
|
||||
const auto end = Clock::now() + std::chrono::milliseconds{2000};
|
||||
while (Clock::now() < end && !has(events, ClientEvent::Kind::lost)) {
|
||||
c->poll(Clock::now());
|
||||
h.collect(*c, events);
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds{5});
|
||||
}
|
||||
CHECK(has(events, ClientEvent::Kind::lost));
|
||||
const auto* unknown = last(events, ClientEvent::Kind::unknown);
|
||||
CHECK(unknown != nullptr && unknown->sequence == 5);
|
||||
CHECK(c->environment().freshness() == client::Freshness::stale);
|
||||
h.tick();
|
||||
CHECK(h.until([&] {
|
||||
return c->phase() == SessionPhase::established && c->latest_view().has_value();
|
||||
}));
|
||||
CHECK(c->environment().freshness() == client::Freshness::current);
|
||||
}
|
||||
{
|
||||
// Silent client: the server closes it with `timeout`, freeing the subject.
|
||||
Harness h{4, std::chrono::milliseconds{3000}, fast_server, fast_client};
|
||||
h.tick();
|
||||
auto c = h.client(grants(7, 1, 100));
|
||||
CHECK(h.until([&] { return c->phase() == SessionPhase::established; }));
|
||||
h.clients.clear(); // Stop polling the client; its socket stays open.
|
||||
std::vector<SessionEvent> server_events;
|
||||
CHECK(h.until([&] {
|
||||
auto e = h.server->take_events();
|
||||
server_events.insert(server_events.end(), e.begin(), e.end());
|
||||
return h.server->sessions() == 0;
|
||||
}));
|
||||
CHECK(std::ranges::any_of(server_events, [](const SessionEvent& e) {
|
||||
return e.kind == SessionEvent::Kind::closed && e.reason == CloseReason::timeout;
|
||||
}));
|
||||
auto again = h.client(grants(7, 1, 200)); // Same subject, fresh grant: admitted.
|
||||
CHECK(h.until([&] { return again->phase() == SessionPhase::established; }));
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void session_tests() {
|
||||
@@ -388,5 +600,8 @@ void session_tests() {
|
||||
duplicate_capacity_and_authentication();
|
||||
stale_epoch_restart_and_reconnect();
|
||||
abuse();
|
||||
carriers_never_alias();
|
||||
grants_stay_consumed_across_restarts();
|
||||
liveness();
|
||||
}
|
||||
} // namespace evo::network_test
|
||||
|
||||
Reference in New Issue
Block a user