Wire M10.8c-1 region handoff: protocol v4, durable probes, crossing, ghosts
Foundation / checks (deploy) (push) Successful in 3m29s
Foundation / checks (ecs) (push) Successful in 2m0s
Foundation / checks (client) (push) Successful in 12m51s
Foundation / checks (network) (push) Successful in 13m0s
Foundation / checks (tsan) (push) Successful in 7m50s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 32m4s
Foundation / checks (postgres) (push) Canceled after 19m13s

ADR-0096. Session protocol v4 (v3 is deployed and frozen) adds HandoffOffer,
SessionRedirect, close reasons handed_off and moved, and ghost marker records.
Sessions use the durable probe record: logins open it where it was saved
(redirected when it lives elsewhere), and session ends save it. A marker
crossing into a neighbour's area is frozen, prepared, its control released
and a signed ticket offered; uncommitted transfers are aborted after the
ticket's lifetime. The destination shows arriving probes as ghosts (schema
v13 incoming_probe_transfers) and opens the marker where it crossed.

Refreshes the dev session window.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-06 21:24:42 -04:00
co-authored by Claude Opus 5.5
parent 97a3756e46
commit a7a3d12283
39 changed files with 1315 additions and 68 deletions
+7 -4
View File
@@ -105,7 +105,8 @@ std::vector<PresentedMarker> MarkerInterpolator::at(double render_tick) {
const auto& newest = snapshots_.back();
const auto show = [&](const Snapshot& s) {
for (const auto& m : s.markers)
out.push_back({m.id, static_cast<double>(m.state.x), static_cast<double>(m.state.y)});
out.push_back({m.id, static_cast<double>(m.state.x), static_cast<double>(m.state.y),
false, false, m.ghost});
};
if (render_tick >= static_cast<double>(newest.tick)) {
// Beyond the newest snapshot: continue the held vector briefly, then freeze (stale).
@@ -116,7 +117,7 @@ std::vector<PresentedMarker> MarkerInterpolator::at(double render_tick) {
out.push_back({m.id,
static_cast<double>(m.state.x) + motion_um(m.state.vector.x, ticks),
static_cast<double>(m.state.y) + motion_um(m.state.vector.y, ticks),
ticks > 0, stale});
ticks > 0, stale, m.ghost});
stats_.extrapolated_frames += ticks > 0 ? 1 : 0;
stats_.stale_frames += stale ? 1 : 0;
return out;
@@ -142,14 +143,16 @@ std::vector<PresentedMarker> MarkerInterpolator::at(double render_tick) {
const auto next = std::ranges::lower_bound(b->markers, m.id, {}, &RemoteMarker::id);
if (next == b->markers.end() || next->id != m.id) {
// Removed by b: shown where it was until b's tick, never moved toward a removal.
out.push_back({m.id, static_cast<double>(m.state.x), static_cast<double>(m.state.y)});
out.push_back({m.id, static_cast<double>(m.state.x), static_cast<double>(m.state.y),
false, false, m.ghost});
continue;
}
const auto lerp = [t](std::int64_t from, std::int64_t to) {
return static_cast<double>(from) +
((static_cast<double>(to) - static_cast<double>(from)) * t);
};
out.push_back({m.id, lerp(m.state.x, next->state.x), lerp(m.state.y, next->state.y)});
out.push_back({m.id, lerp(m.state.x, next->state.x), lerp(m.state.y, next->state.y), false,
false, next->ghost});
}
return out; // Markers new in b appear from b's tick.
}
+19 -1
View File
@@ -25,7 +25,9 @@ struct RegionSessionClient::Impl {
std::vector<network::ActionOutcome> action_outcomes;
std::optional<network::SnapshotAssembler> snapshots;
std::uint64_t snapshot_chunks{}, snapshot_bytes{};
static constexpr std::size_t max_kept = 1024; // Per kind, until taken; oldest dropped.
std::optional<network::HandoffOffer> handoff; // v4: continue at another region.
std::optional<network::RegionAddress> redirect; // v4: the probe is owned elsewhere.
static constexpr std::size_t max_kept = 1024; // Per kind, until taken; oldest dropped.
template <typename T> static void keep(std::vector<T>& list, const T& item) {
if (list.size() >= max_kept)
@@ -95,6 +97,10 @@ struct RegionSessionClient::Impl {
if (const auto* m = std::get_if<network::SessionClose>(&*decoded))
return end(m->reason, false);
if (state == RegionSessionState::awaiting_welcome) {
if (const auto* m = std::get_if<network::SessionRedirect>(&*decoded)) {
redirect = m->owner; // A Close (moved) follows.
return;
}
if (const auto* m = std::get_if<network::SessionWelcome>(&*decoded)) {
welcome = *m;
snapshots.emplace(world::WorldIdentity{m->world, m->frame}, m->epoch);
@@ -115,6 +121,10 @@ struct RegionSessionClient::Impl {
}
if (const auto* m = std::get_if<network::ActionOutcome>(&*decoded))
return keep(action_outcomes, *m);
if (const auto* m = std::get_if<network::HandoffOffer>(&*decoded)) {
handoff = *m; // A Close (handed_off) follows.
return;
}
}
end(SessionCloseReason::protocol, true);
}
@@ -258,6 +268,14 @@ std::uint64_t RegionSessionClient::snapshot_bytes() const noexcept {
return impl_->snapshot_bytes;
}
std::optional<network::HandoffOffer> RegionSessionClient::take_handoff() {
return std::exchange(impl_->handoff, std::nullopt);
}
const std::optional<network::RegionAddress>& RegionSessionClient::redirect() const noexcept {
return impl_->redirect;
}
RegionSessionState RegionSessionClient::state() const noexcept {
return impl_->state;
}
+55 -3
View File
@@ -41,6 +41,11 @@ bool ProbeMarkers::set_area(const Area& area, std::int64_t spawn_x, std::int64_t
}
std::expected<std::uint64_t, MarkerError> ProbeMarkers::open(std::uint64_t session) {
return open_at(session, spawn_x_, spawn_y_);
}
std::expected<std::uint64_t, MarkerError> ProbeMarkers::open_at(std::uint64_t session,
std::int64_t x, std::int64_t y) {
if (!area_)
return std::unexpected(MarkerError::no_area);
if (session == 0 ||
@@ -51,12 +56,47 @@ std::expected<std::uint64_t, MarkerError> ProbeMarkers::open(std::uint64_t sessi
Entry e;
e.marker.id = next_id_++;
e.marker.session = session;
e.marker.state = {spawn_x_, spawn_y_, {}};
e.marker.state = {
std::clamp(x, area_->min_x, area_->max_x), std::clamp(y, area_->min_y, area_->max_y), {}};
entries_.push_back(std::move(e));
publish();
return entries_.back().marker.id;
}
void ProbeMarkers::place(std::uint64_t session, std::int64_t x, std::int64_t y) {
if (!area_)
return;
for (auto& e : entries_)
if (e.marker.session == session) {
e.marker.state.x = std::clamp(x, area_->min_x, area_->max_x);
e.marker.state.y = std::clamp(y, area_->min_y, area_->max_y);
}
publish();
}
void ProbeMarkers::freeze(std::uint64_t session) {
for (auto& e : entries_)
if (e.marker.session == session) {
e.marker.frozen = true;
e.marker.state.vector = {};
e.scheduled.clear();
}
publish();
}
void ProbeMarkers::set_ghosts(std::span<const Ghost> ghosts) {
std::vector<GhostEntry> next;
for (const auto& g : ghosts) {
if (next.size() >= max_markers_)
break;
const auto known = std::ranges::find_if(
ghosts_, [&](const GhostEntry& e) { return e.ghost.transfer == g.transfer; });
next.push_back({g, known == ghosts_.end() ? next_id_++ : known->id});
}
ghosts_ = std::move(next);
publish();
}
void ProbeMarkers::close(std::uint64_t session) {
std::erase_if(entries_, [&](const Entry& e) { return e.marker.session == session; });
publish();
@@ -64,6 +104,7 @@ void ProbeMarkers::close(std::uint64_t session) {
void ProbeMarkers::clear() noexcept {
entries_.clear();
ghosts_.clear();
markers_.clear();
tick_ = 0;
}
@@ -75,6 +116,8 @@ std::expected<void, MarkerError> ProbeMarkers::schedule(std::uint64_t session,
std::ranges::find_if(entries_, [&](const Entry& e) { return e.marker.session == session; });
if (it == entries_.end())
return std::unexpected(MarkerError::unknown_session);
if (it->marker.frozen)
return std::unexpected(MarkerError::invalid);
if (!valid(vector) || tick <= tick_ || sequence <= it->marker.applied_sequence ||
(!it->scheduled.empty() && sequence <= it->scheduled.back().sequence))
return std::unexpected(MarkerError::invalid);
@@ -97,6 +140,8 @@ void ProbeMarkers::advance(std::uint64_t tick) {
const auto first = tick - tick_ > max_catch_up ? tick - max_catch_up + 1 : tick_ + 1;
for (auto& e : entries_) {
auto& m = e.marker;
if (m.frozen)
continue;
// Intentions for ticks that can no longer be stepped still set the held vector.
while (!e.scheduled.empty() && e.scheduled.front().tick < first) {
m.state.vector = e.scheduled.front().vector;
@@ -117,8 +162,8 @@ void ProbeMarkers::advance(std::uint64_t tick) {
}
const Marker* ProbeMarkers::of(std::uint64_t session) const noexcept {
const auto it =
std::ranges::find_if(markers_, [&](const Marker& m) { return m.session == session; });
const auto it = std::ranges::find_if(
markers_, [&](const Marker& m) { return !m.ghost && m.session == session; });
return it == markers_.end() ? nullptr : &*it;
}
@@ -126,6 +171,13 @@ void ProbeMarkers::publish() {
markers_.clear();
for (const auto& e : entries_)
markers_.push_back(e.marker);
for (const auto& g : ghosts_) {
Marker m;
m.id = g.id;
m.state = {g.ghost.x, g.ghost.y, {}};
m.ghost = true;
markers_.push_back(m);
}
std::ranges::sort(markers_, {}, &Marker::id);
}
} // namespace evo::network::motion
+6 -5
View File
@@ -150,7 +150,7 @@ std::vector<std::uint8_t> encode_record(const motion::Marker& m, std::int64_t z,
put_signed(out, z);
put(out, static_cast<std::uint16_t>(m.state.vector.x), 2);
put(out, static_cast<std::uint16_t>(m.state.vector.y), 2);
out.push_back(own_area != nullptr ? 1 : 0);
out.push_back(own_area != nullptr ? 1 : m.ghost ? 2 : 0); // v4: 2 is a ghost.
if (own_area != nullptr) {
put(out, m.applied_sequence, 8);
for (const auto v : {own_area->min_x, own_area->min_y, own_area->max_x, own_area->max_y})
@@ -261,7 +261,7 @@ std::vector<std::pair<std::uint64_t, std::vector<SnapshotChunk>>> ReplicationSer
// every marker first and makes it due every tick it changed.
if (src.marker_area)
for (const auto& m : src.markers) {
const bool own = m.session == s.id;
const bool own = !m.ghost && m.session == s.id;
if (!own && (src.marker_area->z != in.z ||
distance(cell_of(m.state.x), cell_of(m.state.y), in) > in.radius))
continue;
@@ -502,10 +502,11 @@ SnapshotAssembler::accept(const SnapshotChunk& chunk) {
m.z = r.i64();
m.state.vector = {static_cast<std::int16_t>(r.u(2)),
static_cast<std::int16_t>(r.u(2))};
const auto own = r.u(1);
if (own > 1)
const auto kind_byte = r.u(1);
if (kind_byte > 2)
return std::unexpected(AssemblyError::malformed);
m.own = own == 1;
m.own = kind_byte == 1;
m.ghost = kind_byte == 2;
if (m.own) {
m.applied_sequence = r.u(8);
m.area = {r.i64(), r.i64(), r.i64(), r.i64(), m.z};
+78 -1
View File
@@ -1,5 +1,7 @@
#include "evo/network/session_protocol.hpp"
#include <algorithm>
#include <optional>
#include <string>
namespace evo::network {
namespace {
@@ -19,6 +21,9 @@ constexpr std::uint8_t max_admit_status = static_cast<std::uint8_t>(AdmitStatus:
constexpr std::size_t interest_bytes = (3 * std::size_t{8}) + 2;
constexpr std::size_t snapshot_ack_bytes = 2 * std::size_t{8};
constexpr std::uint8_t max_snapshot_chunks = 16;
constexpr std::size_t address_bytes = 8 + (2 * session_address_bytes);
constexpr std::size_t handoff_offer_bytes = 16 + address_bytes + session_grant_wire_bytes;
constexpr std::size_t redirect_bytes = address_bytes;
void put(std::vector<std::uint8_t>& out, std::uint64_t v, std::size_t count) {
for (std::size_t i = 0; i < count; ++i)
@@ -41,7 +46,44 @@ std::vector<std::uint8_t> header(SessionMessageType type, std::size_t payload) {
}
bool wire_reason(std::uint64_t v) {
return v >= static_cast<std::uint8_t>(SessionCloseReason::client_closed) &&
v <= static_cast<std::uint8_t>(SessionCloseReason::shutdown);
v <= static_cast<std::uint8_t>(SessionCloseReason::moved);
}
// Printable ASCII without spaces, then zero padding to the field size.
bool valid_text(const std::string& s) {
return !s.empty() && s.size() <= session_address_bytes &&
std::ranges::all_of(s, [](char ch) { return ch > 0x20 && ch < 0x7F; });
}
void put_text(std::vector<std::uint8_t>& out, const std::string& s) {
for (std::size_t i = 0; i < session_address_bytes; ++i)
out.push_back(i < s.size() ? static_cast<std::uint8_t>(s[i]) : std::uint8_t{0});
}
std::optional<std::string> text_at(std::span<const std::uint8_t> in, std::size_t at) {
std::string s;
std::size_t i = 0;
for (; i < session_address_bytes && in[at + i] != 0; ++i)
s.push_back(static_cast<char>(in[at + i]));
for (; i < session_address_bytes; ++i)
if (in[at + i] != 0)
return std::nullopt; // Bytes after the terminator must be padding.
if (!valid_text(s))
return std::nullopt;
return s;
}
void put_address(std::vector<std::uint8_t>& out, const RegionAddress& a) {
put(out, a.region, 8);
put_text(out, a.endpoint);
put_text(out, a.server_name);
}
std::optional<RegionAddress> address_at(std::span<const std::uint8_t> in, std::size_t at) {
RegionAddress a;
a.region = get(in, at, 8);
auto endpoint = text_at(in, at + 8);
auto name = text_at(in, at + 8 + session_address_bytes);
if (a.region == 0 || !endpoint || !name)
return std::nullopt;
a.endpoint = std::move(*endpoint);
a.server_name = std::move(*name);
return a;
}
WireGrant grant_at(std::span<const std::uint8_t> in, std::size_t at) {
WireGrant g{};
@@ -157,6 +199,21 @@ std::vector<std::uint8_t> encode_session(const SessionMessage& message) {
put(out, m.tick, 8);
return out;
}
// Invalid text fields are encoded as given; the receiver refuses them.
std::vector<std::uint8_t> operator()(const HandoffOffer& m) const {
auto out = header(SessionMessageType::handoff_offer, handoff_offer_bytes);
out.insert(out.end(), m.transfer.begin(), m.transfer.end());
put_address(out, m.destination);
out.insert(out.end(), m.ticket.begin(), m.ticket.end());
out.resize(session_header_bytes + handoff_offer_bytes); // Overlong text truncates.
return out;
}
std::vector<std::uint8_t> operator()(const SessionRedirect& m) const {
auto out = header(SessionMessageType::redirect, redirect_bytes);
put_address(out, m.owner);
out.resize(session_header_bytes + redirect_bytes);
return out;
}
};
return std::visit(Encoder{}, message);
}
@@ -306,6 +363,26 @@ decode_session(std::span<const std::uint8_t> bytes) {
return std::unexpected(SessionCodecError::wrong_length);
return SnapshotAck{get(payload, 0, 8), get(payload, 8, 8)};
}
case static_cast<std::uint8_t>(SessionMessageType::handoff_offer): {
if (!expect(handoff_offer_bytes))
return std::unexpected(SessionCodecError::wrong_length);
HandoffOffer m;
std::copy_n(payload.begin(), m.transfer.size(), m.transfer.begin());
auto destination = address_at(payload, 16);
if (!destination || std::ranges::all_of(m.transfer, [](auto b) { return b == 0; }))
return std::unexpected(SessionCodecError::invalid_value);
m.destination = std::move(*destination);
m.ticket = grant_at(payload, 16 + address_bytes);
return m;
}
case static_cast<std::uint8_t>(SessionMessageType::redirect): {
if (!expect(redirect_bytes))
return std::unexpected(SessionCodecError::wrong_length);
auto owner = address_at(payload, 0);
if (!owner)
return std::unexpected(SessionCodecError::invalid_value);
return SessionRedirect{std::move(*owner)};
}
default:
return std::unexpected(SessionCodecError::unknown_type);
}
+27
View File
@@ -721,4 +721,31 @@ find_probe_transfer(Postgres& db, RegionKey key, const LeaseGrant& lease, const
return std::unexpected{ProbeStoreFailure{ProbeStoreError::unknown, {}}};
return t;
}
std::expected<std::vector<IncomingTransfer>, ProbeStoreFailure>
incoming_probe_transfers(Postgres& db, RegionKey key, const LeaseGrant& lease,
milliseconds timeout) {
const std::array<std::string, 4> p{std::to_string(key.world), std::to_string(key.region),
uuid_text(lease.holder), std::to_string(lease.epoch)};
auto rows = db.query("SELECT * FROM evolution.incoming_probe_transfers($1::evolution.u64, "
"$2::evolution.u64, $3::uuid, $4::evolution.u64)",
p, timeout);
if (!rows)
return std::unexpected{probe_failure(rows.error())};
std::vector<IncomingTransfer> out;
for (const auto& r : *rows) {
if (r.size() != 6)
return std::unexpected{ProbeStoreFailure{ProbeStoreError::unknown, {}}};
const auto transfer = parse_uuid(r[0]);
const auto account = parse_uuid(r[1]);
const auto source = parse_integer<std::uint64_t>(r[2]);
const auto x = parse_signed(r[3]);
const auto y = parse_signed(r[4]);
const auto z = parse_signed(r[5]);
if (!transfer || !account || !source || !x || !y || !z)
return std::unexpected{ProbeStoreFailure{ProbeStoreError::unknown, {}}};
out.push_back({*transfer, *account, *source, *x, *y, *z});
}
return out;
}
} // namespace evo::persistence
+104 -2
View File
@@ -27,7 +27,7 @@ HandoffLedgerError ledger_error(const persistence::ProbeStoreFailure& f) {
}
} // namespace
std::expected<network::ControlLease, HandoffLedgerError>
std::expected<network::HandoffAdmission, HandoffLedgerError>
PostgresHandoffLedger::admit(const network::TicketClaims& ticket, std::uint64_t session) {
const auto lease = lease_();
if (!lease)
@@ -65,6 +65,108 @@ PostgresHandoffLedger::admit(const network::TicketClaims& ticket, std::uint64_t
return std::unexpected(lease_fenced(control.error().cause) ? HandoffLedgerError::fenced
: HandoffLedgerError::refused);
}
return network::ControlLease{control->generation, control->wait, control->granted};
return network::HandoffAdmission{
network::ControlLease{control->generation, control->wait, control->granted},
committed->generation + 1, committed->x, committed->y};
}
namespace {
ProbeLedgerError probe_error(const persistence::ProbeStoreFailure& f) {
return f.kind == ProbeStoreError::fenced && lease_fenced(f.cause)
? ProbeLedgerError::fenced
: ProbeLedgerError::unavailable;
}
} // namespace
std::expected<ProbeHome, ProbeLedgerError>
PostgresProbeLedger::open(const network::AccountId& account, std::int64_t x, std::int64_t y,
std::int64_t z) {
const auto lease = lease_();
if (!lease)
return std::unexpected(ProbeLedgerError::fenced);
auto* db = db_->get();
if (!db)
return std::unexpected(ProbeLedgerError::unavailable);
const auto r =
persistence::open_probe(*db, key_, *lease, account.bytes, x, y, z, inventory, timeout_);
if (!r) {
if (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown)
db_->broken();
return std::unexpected(probe_error(r.error()));
}
return ProbeHome{r->region, r->generation, r->x, r->y, r->transfer.has_value()};
}
void PostgresProbeLedger::save(const network::AccountId& account, std::uint64_t generation,
std::int64_t x, std::int64_t y, std::int64_t z) {
const auto lease = lease_();
if (!lease)
return;
auto* db = db_->get();
if (!db)
return;
if (const auto r = persistence::save_probe(*db, key_, *lease, account.bytes, generation, x, y,
z, timeout_);
!r &&
(r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown))
db_->broken();
}
std::expected<void, ProbeLedgerError>
PostgresProbeLedger::prepare(const network::AccountId& account, std::uint64_t generation,
const network::TransferId& transfer, std::uint64_t destination,
std::int64_t x, std::int64_t y, std::int64_t z) {
const auto lease = lease_();
if (!lease)
return std::unexpected(ProbeLedgerError::fenced);
auto* db = db_->get();
if (!db)
return std::unexpected(ProbeLedgerError::unavailable);
const auto r = persistence::prepare_probe_transfer(*db, key_, *lease, account.bytes, generation,
transfer, destination, x, y, z, timeout_);
if (!r) {
if (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown)
db_->broken();
return std::unexpected(probe_error(r.error()));
}
if (r->state != persistence::TransferState::prepared)
return std::unexpected(ProbeLedgerError::unavailable); // A retry found it decided.
return {};
}
std::expected<bool, ProbeLedgerError>
PostgresProbeLedger::abort(const network::TransferId& transfer) {
const auto lease = lease_();
if (!lease)
return std::unexpected(ProbeLedgerError::fenced);
auto* db = db_->get();
if (!db)
return std::unexpected(ProbeLedgerError::unavailable);
const auto r = persistence::abort_probe_transfer(*db, key_, *lease, transfer, timeout_);
if (!r) {
if (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown)
db_->broken();
return std::unexpected(probe_error(r.error()));
}
return r->state == persistence::TransferState::aborted;
}
std::expected<std::vector<IncomingProbe>, ProbeLedgerError> PostgresProbeLedger::incoming() {
const auto lease = lease_();
if (!lease)
return std::unexpected(ProbeLedgerError::fenced);
auto* db = db_->get();
if (!db)
return std::unexpected(ProbeLedgerError::unavailable);
const auto r = persistence::incoming_probe_transfers(*db, key_, *lease, timeout_);
if (!r) {
if (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown)
db_->broken();
return std::unexpected(probe_error(r.error()));
}
std::vector<IncomingProbe> out;
for (const auto& t : *r)
out.push_back({t.transfer, t.x, t.y});
return out;
}
} // namespace evo::server
+175 -5
View File
@@ -80,6 +80,12 @@ struct RegionSessions::Impl {
std::uint64_t generation{};
Clock::time_point control_deadline, next_renew;
network::SessionWelcome welcome;
// Durable probe (probe records only): its ownership generation and position.
std::optional<ProbeHome> probe;
};
struct Outgoing {
network::TransferId transfer{};
std::uint64_t deadline{}; // Wall seconds: abort unless committed by then.
};
Impl(RegionSessionsConfig c, std::unique_ptr<network::quic::Endpoint> e,
@@ -96,7 +102,11 @@ struct RegionSessions::Impl {
network::SessionControl control;
network::IntentAdmission intents;
network::ReplicationServer replication;
network::motion::ProbeMarkers markers; // Ephemeral probe markers (ADR-0094).
network::motion::ProbeMarkers markers; // Probe markers (ADR-0094, durable: ADR-0096).
std::optional<network::motion::Area> own_area; // This region's part of the marker area.
std::int64_t spawn_x{}, spawn_y{}; // Where new probes start.
std::vector<Outgoing> outgoing; // Handed off, not yet known committed.
Clock::time_point next_ghosts{};
std::optional<network::SessionInterest> default_interest;
HostFacts facts;
std::vector<Connection> connections;
@@ -140,8 +150,10 @@ struct RegionSessions::Impl {
if (c.state == State::established) {
intents.close(c.session);
replication.close(c.session);
save_probe(c);
markers.close(c.session);
}
c.probe.reset();
if (c.session != 0) // May already be released (superseded, expired, epoch).
(void)control.close(c.session).has_value();
// A superseded session's durable control already moved to its successor.
@@ -153,6 +165,49 @@ struct RegionSessions::Impl {
c.deadline = now + config.close_flush;
}
// Best effort, at session end: the durable probe's position (a frozen or moved probe is
// fenced by the store and stays as it was).
void save_probe(const Connection& c) {
if (!config.probes || !c.probe)
return;
if (const auto* m = markers.of(c.session); m && !m->frozen && own_area)
config.probes->save(c.account_id, c.probe->generation, m->state.x, m->state.y,
own_area->z);
}
// After the durable control claim of a grant login: where the account's probe lives.
// False when the session must not open here (the connection is then closed).
bool place_probe(Connection& c, const network::AccountId& account,
const std::optional<network::ControlLease>& lease, Clock::time_point now) {
if (!config.probes || !own_area)
return true;
const auto home = config.probes->open(account, spawn_x, spawn_y, own_area->z);
const auto refuse = [&](SessionCloseReason why) {
if (lease)
config.ledger->release(account, lease->generation);
close(c, why, now);
return false;
};
if (!home)
return refuse(home.error() == ProbeLedgerError::fenced ? SessionCloseReason::shutdown
: SessionCloseReason::capacity);
if (home->owner != config.control.region) {
const auto n = std::ranges::find_if(config.neighbours, [&](const Neighbour& x) {
return x.address.region == home->owner;
});
if (n != config.neighbours.end())
send(c, network::SessionRedirect{n->address});
return refuse(SessionCloseReason::moved);
}
if (home->frozen) // A transfer is open: decided by its commit or abort, retry later.
return refuse(SessionCloseReason::capacity);
c.probe = *home;
// After an aborted handoff the probe was saved where it crossed: back inside, so the
// new session does not cross again at once.
c.probe->x = std::clamp(c.probe->x, own_area->min_x, own_area->max_x);
c.probe->y = std::clamp(c.probe->y, own_area->min_y, own_area->max_y);
return true;
}
void hello(Connection& c, const network::SessionHello& m, std::uint64_t now_seconds,
Clock::time_point now) {
if (m.required_features != 0)
@@ -178,6 +233,8 @@ struct RegionSessions::Impl {
return close(c, reason_for(claimed.error()), now);
lease = *claimed;
}
if (!place_probe(c, claims.account, lease, now))
return;
admitted(c, control.open(*grant, now_seconds), lease, promised, claims.account, now_seconds,
now);
}
@@ -201,7 +258,10 @@ struct RegionSessions::Impl {
auto moved = config.handoff->admit(claims, promised);
if (!moved)
return close(c, reason_for(moved.error()), now);
lease = *moved;
lease = moved->lease;
if (config.probes)
c.probe =
ProbeHome{config.control.region, moved->generation, moved->x, moved->y, false};
} else if (config.ledger) {
return close(c, SessionCloseReason::capacity, now); // Misconfigured: fail closed.
}
@@ -257,7 +317,9 @@ struct RegionSessions::Impl {
if (default_interest)
(void)replication.set_interest(c.session, *default_interest).has_value();
if (markers.area()) // Full or refused: the session simply has no marker.
(void)markers.open(c.session).has_value();
(void)(c.probe ? markers.open_at(c.session, c.probe->x, c.probe->y)
: markers.open(c.session))
.has_value();
c.state = State::established;
c.welcome.server_tick = facts.simulated_tick;
send(c, c.welcome);
@@ -333,6 +395,95 @@ struct RegionSessions::Impl {
send_datagram(c, receipt);
}
[[nodiscard]] bool handoffs_enabled() const {
return config.probes != nullptr && config.handoff_key != nullptr &&
!config.neighbours.empty() && own_area.has_value();
}
// Markers that left this region's area: into a neighbour's area they are handed over,
// anywhere else they are put back at the edge.
void crossings(std::uint64_t now_seconds, Clock::time_point now) {
if (!own_area)
return;
const auto inside = [](const network::motion::Area& a, std::int64_t x, std::int64_t y) {
return x >= a.min_x && x <= a.max_x && y >= a.min_y && y <= a.max_y;
};
for (auto& c : connections) {
if (c.state != State::established)
continue;
const auto* m = markers.of(c.session);
if (m == nullptr || m->frozen || inside(*own_area, m->state.x, m->state.y))
continue;
const auto n = std::ranges::find_if(config.neighbours, [&](const Neighbour& x) {
return inside(x.area, m->state.x, m->state.y);
});
if (n == config.neighbours.end() || !handoffs_enabled() || !c.probe) {
const auto x = std::clamp(m->state.x, own_area->min_x, own_area->max_x);
const auto y = std::clamp(m->state.y, own_area->min_y, own_area->max_y);
markers.place(c.session, x, y); // Back at the edge (`m` is stale after this).
continue;
}
hand_off(c, *n, now_seconds, now);
}
}
// The source side of a crossing: freeze, prepare durably, release control, offer the
// ticket, close. On any failure the probe stays here, at the edge.
void hand_off(Connection& c, const Neighbour& n, std::uint64_t now_seconds,
Clock::time_point now) {
if (!c.probe || !own_area) // Checked by the caller (handoffs_enabled, c.probe).
return;
const auto generation = c.probe->generation;
const auto z = own_area->z;
markers.freeze(c.session);
const auto* m = markers.of(c.session);
network::TransferId transfer{};
if (m == nullptr || !network::random_bytes(transfer))
return close(c, SessionCloseReason::shutdown, now);
const auto x = m->state.x, y = m->state.y;
if (const auto prepared = config.probes->prepare(c.account_id, generation, transfer,
n.address.region, x, y, z);
!prepared)
return close(c, SessionCloseReason::shutdown, now);
c.probe.reset(); // Frozen under the transfer: never saved here again.
network::TicketClaims claims;
claims.transfer = transfer;
claims.account = c.account_id;
claims.world = config.control.world;
claims.source = config.control.region;
claims.destination = n.address.region;
claims.expires_at = static_cast<std::uint32_t>(now_seconds + config.ticket_lifetime);
claims.session_expires_at = static_cast<std::uint32_t>(c.welcome.expires_at);
const auto ticket = config.handoff_key->sign(claims);
// Control goes first so the destination need not wait it out.
if (config.ledger && c.generation != 0)
config.ledger->release(c.account_id, c.generation);
c.generation = 0;
outgoing.push_back({transfer, now_seconds + config.ticket_lifetime + 5});
if (ticket)
send(c, network::HandoffOffer{transfer, n.address, *ticket});
close(c, SessionCloseReason::handed_off, now);
}
// Transfers whose client never arrived are aborted (the probe resumes here); committed
// ones are forgotten. Undecided calls retry next poll.
void expire_outgoing(std::uint64_t now_seconds) {
std::erase_if(outgoing, [&](const Outgoing& o) {
if (now_seconds < o.deadline)
return false;
return config.probes->abort(o.transfer).has_value();
});
}
void refresh_ghosts(Clock::time_point now) {
if (now < next_ghosts)
return;
next_ghosts = now + config.ghost_poll;
const auto incoming = config.probes->incoming();
if (!incoming)
return; // Keep the current ghosts until the next answer.
std::vector<network::motion::Ghost> ghosts;
for (const auto& p : *incoming)
ghosts.push_back({p.transfer, p.x, p.y});
markers.set_ghosts(ghosts);
}
void deliver(const std::vector<network::AccountOutcome>& outcomes) {
for (const auto& o : outcomes)
if (const auto session = intents.session_of(o.account))
@@ -394,6 +545,8 @@ std::expected<std::unique_ptr<RegionSessions>, RegionSessionsError>
RegionSessions::create(RegionSessionsConfig config) {
// Connections beyond sessions are headroom for logins and closes still flushing.
if (config.issuer_keys.empty() || (config.handoff != nullptr && config.ledger == nullptr) ||
(config.handoff_key != nullptr && config.probes == nullptr) ||
config.ticket_lifetime == 0 || config.ticket_lifetime > network::max_ticket_lifetime ||
config.transport.limits.max_connections < config.control.max_sessions ||
config.hello_deadline.count() <= 0 || config.close_flush.count() <= 0 ||
config.renew_interval.count() <= 0 || config.safety_margin.count() <= 0)
@@ -451,8 +604,14 @@ void RegionSessions::poll(const HostFacts& facts, std::uint64_t now_seconds,
m.markers.clear(); // The new epoch may restart at an earlier tick.
}
// Markers reach the simulated tick before this poll's intentions are admitted.
if (facts.ready)
if (facts.ready) {
m.markers.advance(facts.simulated_tick);
m.crossings(now_seconds, now);
if (m.config.probes) {
m.expire_outgoing(now_seconds);
m.refresh_ghosts(now);
}
}
for (auto& e : m.endpoint->poll()) {
using network::quic::EventKind;
if (e.kind == EventKind::connected) {
@@ -471,6 +630,7 @@ void RegionSessions::poll(const HostFacts& facts, std::uint64_t now_seconds,
if (it->state == Impl::State::established) {
m.intents.close(it->session);
m.replication.close(it->session);
m.save_probe(*it);
m.markers.close(it->session);
}
if (it->state == Impl::State::pending) {
@@ -563,8 +723,18 @@ network::ReplicationStats RegionSessions::replication_stats() const {
bool RegionSessions::set_motion_area(const network::motion::Area& area, std::int64_t spawn_x,
std::int64_t spawn_y) {
auto& m = *impl_;
if (!m.markers.set_area(area, spawn_x, spawn_y))
auto movement = area;
for (const auto& n : m.config.neighbours) {
movement.min_x = std::min(movement.min_x, n.area.min_x);
movement.min_y = std::min(movement.min_y, n.area.min_y);
movement.max_x = std::max(movement.max_x, n.area.max_x);
movement.max_y = std::max(movement.max_y, n.area.max_y);
}
if (!network::motion::valid(area) || !m.markers.set_area(movement, spawn_x, spawn_y))
return false;
m.own_area = area;
m.spawn_x = std::clamp(spawn_x, area.min_x, area.max_x);
m.spawn_y = std::clamp(spawn_y, area.min_y, area.max_y);
for (const auto& c : m.connections)
if (c.state == Impl::State::established && !m.markers.of(c.session))
(void)m.markers.open(c.session).has_value();