#pragma once // M10.4c QUIC session listener for evo-server (ADR-0091, ADR-0092). Compiled only with // EVO_SERVER_M10 (EVO_BUILD_QUIC with the PostgreSQL build); otherwise the flags are refused // at parse time. Opt-in, development and test only: nothing deploys it yet. // // Each loop iteration after the host's step: the sessions see the host's authority; actions // scheduled for upcoming ticks are submitted to the host, which commits them with the batch // of the tick they ride; the host's action acknowledgements and drops become outcomes. // Session control and action records use their own connection under the host's lease. // M10.5 (ADR-0093): once per simulated tick every session's snapshot is built from the // region's own stores for its interest (default: the region's first habitat site, radius 7). // M10.6 (ADR-0094): sessions' probe markers move within the habitat sites' bounding cells, // or within --m10-area. M10.8c (ADR-0096): with --m10-handoff-key, markers are durable probe // records and a marker crossing into a --m10-neighbour's area is handed over with a ticket. // M10.9a (ADR-0097): the host's step and this bridge's work per simulated tick feed a load // governor; above its thresholds grant logins are refused and replication is shed. #include "evo/persistence/postgres.hpp" #include "evo/persistence/region_lease.hpp" #include "evo/server/ops.hpp" #include #include #include #include #include #ifdef EVO_SERVER_M10 #include "evo/client/environment_view.hpp" #include "evo/network/handoff_ticket.hpp" #include "evo/processes/lineage_region.hpp" #include "evo/server/load_governor.hpp" #include "evo/server/pg_slot.hpp" #include "evo/server/postgres_action_ledger.hpp" #include "evo/server/postgres_control_ledger.hpp" #include "evo/server/postgres_handoff_ledger.hpp" #include "evo/server/region_host.hpp" #include "evo/server/region_sessions.hpp" #include "evo/server/session_window.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #endif namespace evo::server_app { struct M10Options { std::string listen, certificate, key, issuer, environment, session_window; // M10.5c: dev only, 60..14400 s; 0 keeps the default (300 s, ADR-0091). std::uint64_t max_grant_lifetime{}; // M10.9 (ADR-0097, dev only): `--m10-load-governor off` keeps measuring and exposing the // work but never refuses or sheds (process tests whose host speed is unknown). bool load_governor{true}; // M10.8c (ADR-0096, dev only): this region's marker area in cells "X0,Y0,X1,Y1" // (inclusive), its handoff key (32-byte P-256 scalar; key id = region id) and up to 8 // neighbours "REGION=X0,Y0,X1,Y1=HOST:PORT=SERVER_NAME=PUBLIC_KEY_FILE". std::string area, handoff_key; std::vector neighbours; [[nodiscard]] bool any() const noexcept { return !listen.empty() || !certificate.empty() || !key.empty() || !issuer.empty() || !environment.empty() || !session_window.empty() || max_grant_lifetime != 0 || !area.empty() || !handoff_key.empty() || !neighbours.empty() || !load_governor; } [[nodiscard]] bool complete() const noexcept { return !listen.empty() && !certificate.empty() && !key.empty() && !issuer.empty() && !environment.empty() && !session_window.empty(); } }; // Actions are handed to the host as soon as they are scheduled (at most 8 ticks ahead). inline constexpr std::uint64_t m10_carry_ahead_ticks = 8; #ifdef EVO_SERVER_M10 inline constexpr bool m10_available = true; template class M10Bridge { public: // Empty bridge without M10 options; nullopt on a startup failure (the caller exits). static std::optional open(const M10Options& o, Host& host, persistence::RegionKey key, std::uint64_t frame, const persistence::PgOptions& pg) { M10Bridge bridge; if (!o.any()) return bridge; const auto window = server::SessionWindow::open(o.session_window, wall_seconds()); if (!window) return fail(window.error() == server::SessionWindow::Error::expired ? "error m10-window-expired" : window.error() == server::SessionWindow::Error::too_long ? "error m10-window-too-long" : "error m10-window"); const auto colon = o.listen.rfind(':'); std::uint16_t port = 0; if (colon == std::string::npos || std::from_chars(o.listen.data() + colon + 1, o.listen.data() + o.listen.size(), port) .ec != std::errc{}) return fail("error m10-listen"); const auto issuer = read(o.issuer); if (issuer.size() != network::grant_public_key_bytes) return fail("error m10-issuer"); server::RegionSessionsConfig config; config.transport.listen_address = o.listen.substr(0, colon); config.transport.port = port; config.transport.certificate_pem_file = o.certificate; config.transport.key_pem_file = o.key; config.control.environment = o.environment == "dev" ? network::DeploymentEnvironment::dev : o.environment == "stage" ? network::DeploymentEnvironment::stage : network::DeploymentEnvironment::world0; if (o.max_grant_lifetime != 0) { // Longer grants let a tester's bundle outlive a coffee break; never outside dev. if (config.control.environment != network::DeploymentEnvironment::dev) return fail("error m10-grant-lifetime"); config.control.max_grant_lifetime = o.max_grant_lifetime; } if (!o.load_governor && config.control.environment != network::DeploymentEnvironment::dev) return fail("error m10-load-governor"); std::optional> area_cells; if (!o.area.empty() && !(area_cells = cells_of(o.area))) return fail("error m10-area"); if ((!o.handoff_key.empty() || !o.neighbours.empty()) && (config.control.environment != network::DeploymentEnvironment::dev || o.handoff_key.empty() || o.neighbours.empty() || !area_cells)) return fail("error m10-handoff"); // Dev only; key, neighbours and area together. config.control.world = {key.world, frame}; config.control.region = key.region; config.control.epoch = 1; // Follows the host's epoch from the first poll. server::IssuerKey issuer_key; issuer_key.key_id = 1; std::copy(issuer.begin(), issuer.end(), issuer_key.public_key.begin()); config.issuer_keys.push_back(issuer_key); bridge.host_ = &host; bridge.window_ = *window; bridge.slot_ = std::make_unique(pg, std::chrono::seconds{2}); const auto lease = [h = &host] { return h->authority(); }; bridge.control_ = std::make_unique(*bridge.slot_, key, lease); bridge.actions_ = std::make_unique(*bridge.slot_, key, lease); config.ledger = bridge.control_.get(); config.actions = bridge.actions_.get(); bridge.area_cells_ = area_cells; bridge.governed_ = o.load_governor; if (!o.load_governor) std::puts("m10 load governor off"); if (!o.handoff_key.empty()) { const auto scalar = read(o.handoff_key); auto signer = network::HandoffSigningKey::import( config.control.environment, static_cast(key.region), std::span{reinterpret_cast(scalar.data()), scalar.size()}); if (!signer) return fail("error m10-handoff-key"); config.handoff_key = std::make_shared(std::move(*signer)); for (const auto& spec : o.neighbours) { auto n = neighbour_of(spec); if (!n || n->first.address.region == key.region) return fail("error m10-neighbour"); config.handoff_keys.push_back({n->first.address.region, static_cast(n->first.address.region), n->second}); config.neighbours.push_back(std::move(n->first)); } bridge.probes_ = std::make_unique(*bridge.slot_, key, lease); bridge.handoff_ = std::make_unique(*bridge.slot_, key, lease); config.probes = bridge.probes_.get(); config.handoff = bridge.handoff_.get(); std::printf("m10 handoff neighbours=%zu\n", config.neighbours.size()); } auto sessions = server::RegionSessions::create(std::move(config)); if (!sessions) return fail(sessions.error() == server::RegionSessionsError::keys ? "error m10-issuer" : sessions.error() == server::RegionSessionsError::transport ? "error m10-listen" : "error m10-config"); bridge.sessions_ = std::move(*sessions); std::printf("m10 listening %u\n", static_cast(bridge.sessions_->port())); return bridge; } [[nodiscard]] bool enabled() const noexcept { return sessions_ != nullptr; } [[nodiscard]] bool accepting(std::uint64_t now_seconds) { if (!sessions_ || window_closed_) return sessions_ != nullptr && !window_closed_; if (window_ && window_->accepting(now_seconds)) return sessions_ != nullptr; window_closed_ = true; sessions_->shutdown(std::chrono::steady_clock::now()); std::printf("m10 window closed activation=%s\n", window_->activation_id().c_str()); return false; } // `host_work`: this iteration's host step, measured by the caller. void after_step(std::chrono::nanoseconds host_work) { if (!sessions_ || !accepting(wall_seconds())) return; const auto started = std::chrono::steady_clock::now(); auto& host = *host_; const auto& s = host.status(); const server::HostFacts facts{s.ready && s.phase == server::HostPhase::running, s.epoch, s.simulated_tick}; sessions_->poll(facts, wall_seconds(), std::chrono::steady_clock::now()); if (facts.ready && host.state() != nullptr && s.simulated_tick > replicated_) { replicated_ = s.simulated_tick; const auto& c = community(*host.state()); if (!default_interest_) if (const auto& sites = c.resources().fields().sites(); !sites.empty()) { const auto v = sites.front().voxel; default_interest_ = network::SessionInterest{v.x, v.y, v.z, 7}; sessions_->set_default_interest(*default_interest_); set_motion_area(sites, v); } sessions_->replicate(s.simulated_tick, [&](const network::SessionInterest& in) { return source(c, in, s.simulated_tick); }); } if (facts.ready) for (const auto& a : sessions_->carry(s.simulated_tick + m10_carry_ahead_ticks)) { const persistence::ActionRecord record{a.account.bytes, a.action_id, a.tick, static_cast(a.kind)}; if (!host.submit_action(record)) // Never reached the host: final. sessions_->dropped(std::span{&a, 1}, true); } std::map> committed; for (const auto& ack : host.take_action_acks()) committed[ack.commit_sequence].push_back(carried(ack.record)); for (const auto& [sequence, actions] : committed) { sessions_->committed(actions, sequence); std::printf("m10 committed actions=%zu commit=%llu\n", actions.size(), static_cast(sequence)); } for (const auto& d : host.take_dropped_actions()) { const auto a = carried(d.record); sessions_->dropped(std::span{&a, 1}, d.final); std::printf("m10 dropped action=%llu final=%d\n", static_cast(d.record.action_id), d.final ? 1 : 0); } for (const auto& e : sessions_->take_events()) std::printf("m10 %s session=%llu account=%s reason=%u\n", e.kind == server::RegionSessionEvent::Kind::opened ? "opened" : e.kind == server::RegionSessionEvent::Kind::closed ? "closed" : "refused", static_cast(e.session), e.account.c_str(), static_cast(e.reason)); measure(host_work, std::chrono::steady_clock::now() - started, s); } // The listener's load for /metrics. void metrics(server::OpsCounters& c) const { c.m10_enabled = sessions_ != nullptr; if (!sessions_) return; c.m10_sessions = sessions_->sessions(); c.m10_refused_logins = sessions_->refused_for_load(); c.m10_load = governor_.state(); if (!governed_) { // Measured only: nothing is refused or shed. c.m10_load.refusing = false; c.m10_load.shed = 0; } } void shutdown() { if (sessions_) sessions_->shutdown(std::chrono::steady_clock::now()); } private: // Busy time accumulates until the simulated tick advances by k; it then counts as k equal // per-tick samples (ADR-0097). A recovery's jump starts a new count instead. void measure(std::chrono::nanoseconds host_work, std::chrono::nanoseconds session_work, const server::HostStatus& s) { busy_ += host_work + session_work; session_busy_ += session_work; if (s.simulated_tick <= measured_tick_) return; const auto ticks = s.simulated_tick - measured_tick_; const bool counted = measured_tick_ != 0 && ticks <= server::load_window_ticks; measured_tick_ = s.simulated_tick; const auto per = [&](std::chrono::nanoseconds total) { return static_cast( std::chrono::duration_cast(total).count() / static_cast(ticks)); }; const auto per_tick = per(busy_), session_per_tick = per(session_busy_); busy_ = session_busy_ = {}; if (!counted) return; const auto now_us = static_cast(std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) .count()); bool changed = false; for (std::uint64_t i = 0; i < ticks; ++i) changed = governor_.observe(per_tick, session_per_tick, s.tick_debt, now_us) || changed; report(s, now_us); if (!changed || !governed_) return; const auto& load = governor_.state(); sessions_->set_load(load.refusing, load.shed); std::printf("m10 load refusing=%d shed=%u p95_us=%llu\n", load.refusing ? 1 : 0, static_cast(load.shed), static_cast(load.p95_us)); } // M10.9d (ADR-0097): every 10 s, the load as /metrics would show it, in the log (drills // read it with `kubectl logs --timestamps` where the ops port is not reachable). void report(const server::HostStatus& s, std::uint64_t now_us) { if (reported_us_ != 0 && now_us - reported_us_ < 10'000'000) return; reported_us_ = now_us; const auto& load = governor_.state(); std::printf("m10 load-report sessions=%zu p95_us=%llu p99_us=%llu max_us=%llu " "session_p95_us=%llu session_p99_us=%llu refusing=%d shed=%u tick_debt=%llu\n", sessions_->sessions(), static_cast(load.p95_us), static_cast(load.p99_us), static_cast(load.max_us), static_cast(load.session_p95_us), static_cast(load.session_p99_us), governed_ && load.refusing ? 1 : 0, governed_ ? load.shed : 0U, static_cast(s.tick_debt)); } static const processes::EnvironmentalCommunity& community(const processes::EnvironmentalCommunity& c) { return c; } static const processes::EnvironmentalCommunity& community(const processes::LineageRegion& r) { return r.community(); } // Probe markers (ADR-0094, EXPERIMENT) move within the habitat sites' bounding cells of // the default slice and spawn at the centre of its first site. template void set_motion_area(const Sites& sites, world::VoxelAddress first) { constexpr auto cell = network::motion::micrometres_per_cell; network::motion::Area area{first.x * cell, first.y * cell, (first.x * cell) + cell - 1, (first.y * cell) + cell - 1, first.z}; if (area_cells_) { // --m10-area: this region's cells, inclusive. const auto& c = *area_cells_; area = {c[0] * cell, c[1] * cell, (c[2] * cell) + cell - 1, (c[3] * cell) + cell - 1, first.z}; if (sessions_->set_motion_area(area, ((c[0] + c[2] + 1) * cell) / 2, ((c[1] + c[3] + 1) * cell) / 2)) std::printf("m10 motion area %lld %lld %lld %lld z=%lld\n", static_cast(area.min_x), static_cast(area.min_y), static_cast(area.max_x), static_cast(area.max_y), static_cast(area.z)); return; } for (const auto& site : sites) if (site.voxel.z == first.z) { area.min_x = std::min(area.min_x, site.voxel.x * cell); area.min_y = std::min(area.min_y, site.voxel.y * cell); area.max_x = std::max(area.max_x, (site.voxel.x * cell) + cell - 1); area.max_y = std::max(area.max_y, (site.voxel.y * cell) + cell - 1); } if (sessions_->set_motion_area(area, (first.x * cell) + (cell / 2), (first.y * cell) + (cell / 2))) std::printf("m10 motion area %lld %lld %lld %lld z=%lld\n", static_cast(area.min_x), static_cast(area.min_y), static_cast(area.max_x), static_cast(area.max_y), static_cast(area.z)); } // A read-only copy of the region's stores within one interest: the habitat sites of its // square in its slice. The region has no placed individuals: organisms stay empty. network::ReplicationSource source(const processes::EnvironmentalCommunity& c, const network::SessionInterest& in, std::uint64_t tick) { network::ReplicationSource src; src.mask = world::all_environment_fields; const auto& fields = c.resources().fields(); std::vector footprint; for (const auto& site : fields.sites()) { const auto v = site.voxel; if (v.z == in.z && v.x >= in.x - in.radius && v.x <= in.x + in.radius && v.y >= in.y - in.radius && v.y <= in.y + in.radius && footprint.size() < world::max_environment_footprint) footprint.push_back(v); } if (footprint.empty()) return src; const auto view = world::query_environment(fields, c.resources().terrain(), tick, footprint, world::all_environment_fields); if (!view) { if (!warned_) std::printf("m10 view-unavailable %u\n", static_cast(view.error())); warned_ = true; return src; } src.identity = view->identity(); src.field_revision = view->field_revision(); if (const auto cells = client::present_environment(*view, in.z)) src.environment.assign(cells->cells().begin(), cells->cells().end()); return src; } // "X0,Y0,X1,Y1" in cells, X0 <= X1 and Y0 <= Y1, each within +-2^40. static std::optional> cells_of(std::string_view text) { std::array v{}; for (std::size_t i = 0; i < v.size(); ++i) { const auto comma = text.find(','); const auto part = text.substr(0, comma); const auto [end, ec] = std::from_chars(part.data(), part.data() + part.size(), v.at(i)); if (ec != std::errc{} || end != part.data() + part.size() || (comma == std::string_view::npos) != (i + 1 == v.size()) || v.at(i) < -(std::int64_t{1} << 40) || v.at(i) > (std::int64_t{1} << 40)) return std::nullopt; text.remove_prefix(comma == std::string_view::npos ? text.size() : comma + 1); } if (v[0] > v[2] || v[1] > v[3]) return std::nullopt; return v; } // "REGION=X0,Y0,X1,Y1=HOST:PORT=SERVER_NAME=PUBLIC_KEY_FILE". static std::optional> neighbour_of(std::string_view spec) { std::array part{}; for (std::size_t i = 0; i < part.size(); ++i) { const auto eq = spec.find('='); if ((eq == std::string_view::npos) != (i + 1 == part.size())) return std::nullopt; part.at(i) = spec.substr(0, eq); spec.remove_prefix(eq == std::string_view::npos ? spec.size() : eq + 1); } std::uint64_t region = 0; if (const auto [end, ec] = std::from_chars(part[0].data(), part[0].data() + part[0].size(), region); ec != std::errc{} || end != part[0].data() + part[0].size() || region == 0) return std::nullopt; const auto cells = cells_of(part[1]); const auto key = read(std::string{part[4]}); if (!cells || part[2].empty() || part[3].empty() || part[2].size() > 64 || part[3].size() > 64 || key.size() != network::grant_public_key_bytes) return std::nullopt; constexpr auto cell = network::motion::micrometres_per_cell; const auto& c = *cells; server::Neighbour n{ {region, std::string{part[2]}, std::string{part[3]}}, {c[0] * cell, c[1] * cell, (c[2] * cell) + cell - 1, (c[3] * cell) + cell - 1, 0}}; network::GrantPublicKey pub{}; std::copy(key.begin(), key.end(), pub.begin()); return std::pair{std::move(n), pub}; } static network::CarriedAction carried(const persistence::ActionRecord& r) { network::CarriedAction a; a.account.bytes = r.account; a.action_id = r.action_id; a.tick = r.tick; a.kind = network::ActionKind::probe; return a; } static std::uint64_t wall_seconds() { return static_cast(std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count()); } static std::optional fail(const char* message) { std::puts(message); return std::nullopt; } static std::string read(const std::string& path) { std::ifstream file{path, std::ios::binary}; return {std::istreambuf_iterator{file}, std::istreambuf_iterator{}}; } Host* host_{}; // Heap-held so the sessions' pointers stay valid when the bridge moves. std::unique_ptr slot_; std::unique_ptr control_; std::unique_ptr actions_; server::LoadGovernor governor_{*server::LoadGovernor::create({})}; bool governed_{true}; std::uint64_t reported_us_{}; std::chrono::nanoseconds busy_{}, session_busy_{}; std::uint64_t measured_tick_{}; std::unique_ptr probes_; // Handoff mode only. std::unique_ptr handoff_; // Handoff mode only. std::optional> area_cells_; std::unique_ptr sessions_; // Destroyed first: uses the ledgers. std::optional window_; bool window_closed_{}; std::uint64_t replicated_{}; std::optional default_interest_; bool warned_{}; }; #else inline constexpr bool m10_available = false; template class M10Bridge { public: static std::optional open(const M10Options&, Host&, persistence::RegionKey, std::uint64_t, const persistence::PgOptions&) { return M10Bridge{}; } [[nodiscard]] bool enabled() const noexcept { return false; } [[nodiscard]] bool accepting(std::uint64_t) const noexcept { return false; } void after_step(std::chrono::nanoseconds) {} void metrics(server::OpsCounters& c) const { c.m10_enabled = false; } void shutdown() {} }; #endif } // namespace evo::server_app