// M10.2c head-of-line and loss measurement (ADR-0090): the same 20 Hz state stream sent // on the QUIC DATAGRAM channel and on the single ordered reliable stream (the TCP-like // case), through the owned impairment relay, across an RTT x loss matrix. Reliable // 16 KiB messages once per second exercise multi-packet (fragmented) delivery. One // process and one monotonic clock: latencies are one-way, staleness is the age of the // newest state the client holds, sampled every 5 ms. Development reference only. // // evo-quic-hol-measure [--seconds 2..120] [--out FILE.csv] #include "evo/network/dev_credentials.hpp" #include "evo/network/quic_transport.hpp" #include "udp_relay.hpp" #ifdef _WIN32 #ifndef NOMINMAX #define NOMINMAX #endif #include #include #endif #include #include #include #include #include #include #include #include #include #include #include namespace { using namespace evo::network; using namespace evo::network::quic; using Clock = std::chrono::steady_clock; struct Config { int rtt_ms; double loss; }; struct Series { std::vector latency_ms, staleness_ms; std::uint64_t sent{}, received{}, refused{}; }; double percentile(std::vector v, double p) { if (v.empty()) return -1; std::sort(v.begin(), v.end()); const auto rank = static_cast(p / 100.0 * static_cast(v.size()) + 0.999999); return v[std::clamp(rank, 1, v.size()) - 1]; } std::vector stamp(std::uint32_t seq, std::int64_t sent_us, std::size_t size) { std::vector v(size, 0); const auto t = static_cast(sent_us); for (std::size_t i = 0; i < 4; ++i) v[i] = static_cast(seq >> (8U * i)); for (std::size_t i = 0; i < 8; ++i) v[4 + i] = static_cast(t >> (8U * i)); return v; } std::int64_t sent_at(const std::vector& v) { std::uint64_t t = 0; for (std::size_t i = 0; i < 8; ++i) t |= static_cast(v[4 + i]) << (8U * i); return static_cast(t); } std::uint32_t seq_of(const std::vector& v) { return static_cast(v[0]) | (static_cast(v[1]) << 8U) | (static_cast(v[2]) << 16U) | (static_cast(v[3]) << 24U); } struct Result { Config config{}; Series datagram, stream, bulk; impair::RelayCounters relay{}; bool connected{}; }; Result run(const Config& config, int seconds, const std::filesystem::path& dir, const std::string& ca_pem) { Result r; r.config = config; ServerConfig sc; sc.certificate_pem_file = (dir / "server.pem").string(); sc.key_pem_file = (dir / "server.key").string(); auto server = Endpoint::serve(sc); if (!server) return r; impair::Impairment way; way.delay = std::chrono::milliseconds{config.rtt_ms / 2}; way.jitter = std::chrono::milliseconds{std::max(1, config.rtt_ms / 20)}; way.loss = config.loss; way.reorder = 0.02; way.duplicate = 0.01; auto relay = impair::UdpRelay::start((*server)->port(), way, way, 0xA11CE + static_cast(config.rtt_ms)); if (!relay) return r; ClientConfig cc; cc.port = relay->port(); cc.server_name = "dev-region.evolution.test"; cc.authority_pem = ca_pem; cc.authority_pem_file = (dir / "ca.pem").string(); auto client = Endpoint::connect(cc); if (!client) return r; ConnectionId server_id = 0; bool client_ready = false; const auto connect_end = Clock::now() + std::chrono::seconds{15}; while (Clock::now() < connect_end && (server_id == 0 || !client_ready)) { for (const auto& e : (*server)->poll()) if (e.kind == EventKind::connected) server_id = e.connection; for (const auto& e : (*client)->poll()) if (e.kind == EventKind::connected) client_ready = true; std::this_thread::sleep_for(std::chrono::milliseconds{1}); } if (server_id == 0 || !client_ready) return r; // Measure only once the datagram channel is negotiated (earlier sends would be refused). while (Clock::now() < connect_end && (*server)->datagram_limit(server_id) == 0) { (void)(*server)->poll(); (void)(*client)->poll(); std::this_thread::sleep_for(std::chrono::milliseconds{1}); } if ((*server)->datagram_limit(server_id) == 0) return r; r.connected = true; const auto origin = Clock::now(); const auto us = [&](Clock::time_point t) { return std::chrono::duration_cast(t - origin).count(); }; const auto end = origin + std::chrono::seconds{seconds}; auto next_state = origin; auto next_bulk = origin + std::chrono::milliseconds{500}; auto next_sample = origin; std::uint32_t seq = 0, bulk_seq = 0; std::int64_t newest_datagram = -1, newest_stream = -1; std::uint32_t newest_datagram_seq = 0, newest_stream_seq = 0; while (Clock::now() < end) { const auto now = Clock::now(); if (now >= next_state) { // 20 Hz state on both channels (300 bytes). const auto t = us(now); ++seq; ++r.datagram.sent; ++r.stream.sent; if (!(*server)->send_datagram(server_id, stamp(seq, t, 300))) ++r.datagram.refused; if (!(*server)->send_message(server_id, stamp(seq, t, 300))) ++r.stream.refused; next_state += std::chrono::milliseconds{50}; } if (now >= next_bulk) { // Once per second a multi-packet reliable message. ++r.bulk.sent; if (!(*server)->send_message( server_id, stamp(1000000 + ++bulk_seq, us(now), std::size_t{16} * 1024))) ++r.bulk.refused; next_bulk += std::chrono::seconds{1}; } (void)(*server)->poll(); for (const auto& e : (*client)->poll()) { const double latency = static_cast(us(Clock::now()) - sent_at(e.bytes)) / 1000.0; if (e.kind == EventKind::datagram) { ++r.datagram.received; r.datagram.latency_ms.push_back(latency); if (seq_of(e.bytes) > newest_datagram_seq) { newest_datagram_seq = seq_of(e.bytes); newest_datagram = sent_at(e.bytes); } } else if (e.kind == EventKind::message && e.bytes.size() > 1000) { ++r.bulk.received; r.bulk.latency_ms.push_back(latency); } else if (e.kind == EventKind::message) { ++r.stream.received; r.stream.latency_ms.push_back(latency); newest_stream_seq = std::max(newest_stream_seq, seq_of(e.bytes)); newest_stream = sent_at(e.bytes); } } if (now >= next_sample) { const auto t = static_cast(us(now)); if (newest_datagram >= 0) r.datagram.staleness_ms.push_back((t - static_cast(newest_datagram)) / 1000.0); if (newest_stream >= 0) r.stream.staleness_ms.push_back((t - static_cast(newest_stream)) / 1000.0); next_sample += std::chrono::milliseconds{5}; } std::this_thread::sleep_for(std::chrono::microseconds{200}); } r.relay = relay->counters(); return r; } void print(const char* channel, const Result& r, const Series& s) { std::printf( "hol rtt=%dms loss=%.0f%% %-8s sent=%llu recv=%llu refused=%llu latency p50=%.1f p95=%.1f " "p99=%.1f " "max=%.1f staleness p50=%.1f p95=%.1f p99=%.1f\n", r.config.rtt_ms, r.config.loss * 100, channel, static_cast(s.sent), static_cast(s.received), static_cast(s.refused), percentile(s.latency_ms, 50), percentile(s.latency_ms, 95), percentile(s.latency_ms, 99), s.latency_ms.empty() ? -1.0 : *std::max_element(s.latency_ms.begin(), s.latency_ms.end()), percentile(s.staleness_ms, 50), percentile(s.staleness_ms, 95), percentile(s.staleness_ms, 99)); } } // namespace int main(int argc, char** argv) { int seconds = 15; std::string out; for (int i = 1; i + 1 < argc; i += 2) { const std::string_view flag{argv[i]}; const std::string_view value{argv[i + 1]}; if (flag == "--seconds") { const auto [p, ec] = std::from_chars(value.data(), value.data() + value.size(), seconds); if (ec != std::errc{} || seconds < 2 || seconds > 120) return 2; } else if (flag == "--out") { out = value; } else { std::fputs("usage: evo-quic-hol-measure [--seconds 2..120] [--out FILE.csv]\n", stderr); return 2; } } #ifdef _WIN32 timeBeginPeriod(1); // The measurement loop's short sleeps (see the relay). #endif const auto creds = generate_dev_credentials(DeploymentEnvironment::dev, "dev-region.evolution.test", 1, std::chrono::system_clock::now()); if (!creds) return 1; const auto dir = std::filesystem::temp_directory_path() / "evo-quic-hol"; std::filesystem::remove_all(dir); std::filesystem::create_directories(dir); for (const auto& [name, text] : std::map{{"ca.pem", creds->authority_pem}, {"server.pem", creds->server_certificate_pem}, {"server.key", creds->server_key_pem}}) { std::ofstream f{dir / name, std::ios::binary | std::ios::trunc}; f << text; } std::ofstream csv; if (!out.empty()) { csv.open(out, std::ios::trunc); csv << "rtt_ms,loss,channel,kind,value_ms\n"; } int failures = 0; for (const int rtt : {2, 50, 100}) for (const double loss : {0.0, 0.01, 0.05, 0.10}) { const auto r = run({rtt, loss}, seconds, dir, creds->authority_pem); if (!r.connected) { std::printf("hol rtt=%dms loss=%.0f%% connection failed\n", rtt, loss * 100); ++failures; continue; } print("datagram", r, r.datagram); print("stream", r, r.stream); print("bulk16k", r, r.bulk); std::printf("hol rtt=%dms loss=%.0f%% relay forwarded=%llu dropped=%llu reordered=%llu " "duplicated=%llu\n", rtt, loss * 100, static_cast(r.relay.forwarded), static_cast(r.relay.dropped), static_cast(r.relay.reordered), static_cast(r.relay.duplicated)); std::fflush(stdout); if (csv) for (const auto& [channel, series] : {std::pair{"datagram", &r.datagram}, std::pair{"stream", &r.stream}, std::pair{"bulk16k", &r.bulk}}) { for (const auto v : series->latency_ms) csv << rtt << ',' << loss << ',' << channel << ",latency," << v << '\n'; for (const auto v : series->staleness_ms) csv << rtt << ',' << loss << ',' << channel << ",staleness," << v << '\n'; } } std::filesystem::remove_all(dir); return failures == 0 ? 0 : 1; }