Wire M10.8c-1 region handoff: protocol v4, durable probes, crossing, ghosts
Foundation / checks (deploy) (push) Successful in 3m29s
Foundation / checks (ecs) (push) Successful in 2m0s
Foundation / checks (client) (push) Successful in 12m51s
Foundation / checks (network) (push) Successful in 13m0s
Foundation / checks (tsan) (push) Successful in 7m50s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 32m4s
Foundation / checks (postgres) (push) Canceled after 19m13s
Foundation / checks (deploy) (push) Successful in 3m29s
Foundation / checks (ecs) (push) Successful in 2m0s
Foundation / checks (client) (push) Successful in 12m51s
Foundation / checks (network) (push) Successful in 13m0s
Foundation / checks (tsan) (push) Successful in 7m50s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 32m4s
Foundation / checks (postgres) (push) Canceled after 19m13s
ADR-0096. Session protocol v4 (v3 is deployed and frozen) adds HandoffOffer, SessionRedirect, close reasons handed_off and moved, and ghost marker records. Sessions use the durable probe record: logins open it where it was saved (redirected when it lives elsewhere), and session ends save it. A marker crossing into a neighbour's area is frozen, prepared, its control released and a signed ticket offered; uncommitted transfers are aborted after the ticket's lifetime. The destination shows arriving probes as ghosts (schema v13 incoming_probe_transfers) and opens the marker where it crossed. Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -105,7 +105,8 @@ std::vector<PresentedMarker> MarkerInterpolator::at(double render_tick) {
|
||||
const auto& newest = snapshots_.back();
|
||||
const auto show = [&](const Snapshot& s) {
|
||||
for (const auto& m : s.markers)
|
||||
out.push_back({m.id, static_cast<double>(m.state.x), static_cast<double>(m.state.y)});
|
||||
out.push_back({m.id, static_cast<double>(m.state.x), static_cast<double>(m.state.y),
|
||||
false, false, m.ghost});
|
||||
};
|
||||
if (render_tick >= static_cast<double>(newest.tick)) {
|
||||
// Beyond the newest snapshot: continue the held vector briefly, then freeze (stale).
|
||||
@@ -116,7 +117,7 @@ std::vector<PresentedMarker> MarkerInterpolator::at(double render_tick) {
|
||||
out.push_back({m.id,
|
||||
static_cast<double>(m.state.x) + motion_um(m.state.vector.x, ticks),
|
||||
static_cast<double>(m.state.y) + motion_um(m.state.vector.y, ticks),
|
||||
ticks > 0, stale});
|
||||
ticks > 0, stale, m.ghost});
|
||||
stats_.extrapolated_frames += ticks > 0 ? 1 : 0;
|
||||
stats_.stale_frames += stale ? 1 : 0;
|
||||
return out;
|
||||
@@ -142,14 +143,16 @@ std::vector<PresentedMarker> MarkerInterpolator::at(double render_tick) {
|
||||
const auto next = std::ranges::lower_bound(b->markers, m.id, {}, &RemoteMarker::id);
|
||||
if (next == b->markers.end() || next->id != m.id) {
|
||||
// Removed by b: shown where it was until b's tick, never moved toward a removal.
|
||||
out.push_back({m.id, static_cast<double>(m.state.x), static_cast<double>(m.state.y)});
|
||||
out.push_back({m.id, static_cast<double>(m.state.x), static_cast<double>(m.state.y),
|
||||
false, false, m.ghost});
|
||||
continue;
|
||||
}
|
||||
const auto lerp = [t](std::int64_t from, std::int64_t to) {
|
||||
return static_cast<double>(from) +
|
||||
((static_cast<double>(to) - static_cast<double>(from)) * t);
|
||||
};
|
||||
out.push_back({m.id, lerp(m.state.x, next->state.x), lerp(m.state.y, next->state.y)});
|
||||
out.push_back({m.id, lerp(m.state.x, next->state.x), lerp(m.state.y, next->state.y), false,
|
||||
false, next->ghost});
|
||||
}
|
||||
return out; // Markers new in b appear from b's tick.
|
||||
}
|
||||
|
||||
@@ -25,7 +25,9 @@ struct RegionSessionClient::Impl {
|
||||
std::vector<network::ActionOutcome> action_outcomes;
|
||||
std::optional<network::SnapshotAssembler> snapshots;
|
||||
std::uint64_t snapshot_chunks{}, snapshot_bytes{};
|
||||
static constexpr std::size_t max_kept = 1024; // Per kind, until taken; oldest dropped.
|
||||
std::optional<network::HandoffOffer> handoff; // v4: continue at another region.
|
||||
std::optional<network::RegionAddress> redirect; // v4: the probe is owned elsewhere.
|
||||
static constexpr std::size_t max_kept = 1024; // Per kind, until taken; oldest dropped.
|
||||
|
||||
template <typename T> static void keep(std::vector<T>& list, const T& item) {
|
||||
if (list.size() >= max_kept)
|
||||
@@ -95,6 +97,10 @@ struct RegionSessionClient::Impl {
|
||||
if (const auto* m = std::get_if<network::SessionClose>(&*decoded))
|
||||
return end(m->reason, false);
|
||||
if (state == RegionSessionState::awaiting_welcome) {
|
||||
if (const auto* m = std::get_if<network::SessionRedirect>(&*decoded)) {
|
||||
redirect = m->owner; // A Close (moved) follows.
|
||||
return;
|
||||
}
|
||||
if (const auto* m = std::get_if<network::SessionWelcome>(&*decoded)) {
|
||||
welcome = *m;
|
||||
snapshots.emplace(world::WorldIdentity{m->world, m->frame}, m->epoch);
|
||||
@@ -115,6 +121,10 @@ struct RegionSessionClient::Impl {
|
||||
}
|
||||
if (const auto* m = std::get_if<network::ActionOutcome>(&*decoded))
|
||||
return keep(action_outcomes, *m);
|
||||
if (const auto* m = std::get_if<network::HandoffOffer>(&*decoded)) {
|
||||
handoff = *m; // A Close (handed_off) follows.
|
||||
return;
|
||||
}
|
||||
}
|
||||
end(SessionCloseReason::protocol, true);
|
||||
}
|
||||
@@ -258,6 +268,14 @@ std::uint64_t RegionSessionClient::snapshot_bytes() const noexcept {
|
||||
return impl_->snapshot_bytes;
|
||||
}
|
||||
|
||||
std::optional<network::HandoffOffer> RegionSessionClient::take_handoff() {
|
||||
return std::exchange(impl_->handoff, std::nullopt);
|
||||
}
|
||||
|
||||
const std::optional<network::RegionAddress>& RegionSessionClient::redirect() const noexcept {
|
||||
return impl_->redirect;
|
||||
}
|
||||
|
||||
RegionSessionState RegionSessionClient::state() const noexcept {
|
||||
return impl_->state;
|
||||
}
|
||||
|
||||
@@ -41,6 +41,11 @@ bool ProbeMarkers::set_area(const Area& area, std::int64_t spawn_x, std::int64_t
|
||||
}
|
||||
|
||||
std::expected<std::uint64_t, MarkerError> ProbeMarkers::open(std::uint64_t session) {
|
||||
return open_at(session, spawn_x_, spawn_y_);
|
||||
}
|
||||
|
||||
std::expected<std::uint64_t, MarkerError> ProbeMarkers::open_at(std::uint64_t session,
|
||||
std::int64_t x, std::int64_t y) {
|
||||
if (!area_)
|
||||
return std::unexpected(MarkerError::no_area);
|
||||
if (session == 0 ||
|
||||
@@ -51,12 +56,47 @@ std::expected<std::uint64_t, MarkerError> ProbeMarkers::open(std::uint64_t sessi
|
||||
Entry e;
|
||||
e.marker.id = next_id_++;
|
||||
e.marker.session = session;
|
||||
e.marker.state = {spawn_x_, spawn_y_, {}};
|
||||
e.marker.state = {
|
||||
std::clamp(x, area_->min_x, area_->max_x), std::clamp(y, area_->min_y, area_->max_y), {}};
|
||||
entries_.push_back(std::move(e));
|
||||
publish();
|
||||
return entries_.back().marker.id;
|
||||
}
|
||||
|
||||
void ProbeMarkers::place(std::uint64_t session, std::int64_t x, std::int64_t y) {
|
||||
if (!area_)
|
||||
return;
|
||||
for (auto& e : entries_)
|
||||
if (e.marker.session == session) {
|
||||
e.marker.state.x = std::clamp(x, area_->min_x, area_->max_x);
|
||||
e.marker.state.y = std::clamp(y, area_->min_y, area_->max_y);
|
||||
}
|
||||
publish();
|
||||
}
|
||||
|
||||
void ProbeMarkers::freeze(std::uint64_t session) {
|
||||
for (auto& e : entries_)
|
||||
if (e.marker.session == session) {
|
||||
e.marker.frozen = true;
|
||||
e.marker.state.vector = {};
|
||||
e.scheduled.clear();
|
||||
}
|
||||
publish();
|
||||
}
|
||||
|
||||
void ProbeMarkers::set_ghosts(std::span<const Ghost> ghosts) {
|
||||
std::vector<GhostEntry> next;
|
||||
for (const auto& g : ghosts) {
|
||||
if (next.size() >= max_markers_)
|
||||
break;
|
||||
const auto known = std::ranges::find_if(
|
||||
ghosts_, [&](const GhostEntry& e) { return e.ghost.transfer == g.transfer; });
|
||||
next.push_back({g, known == ghosts_.end() ? next_id_++ : known->id});
|
||||
}
|
||||
ghosts_ = std::move(next);
|
||||
publish();
|
||||
}
|
||||
|
||||
void ProbeMarkers::close(std::uint64_t session) {
|
||||
std::erase_if(entries_, [&](const Entry& e) { return e.marker.session == session; });
|
||||
publish();
|
||||
@@ -64,6 +104,7 @@ void ProbeMarkers::close(std::uint64_t session) {
|
||||
|
||||
void ProbeMarkers::clear() noexcept {
|
||||
entries_.clear();
|
||||
ghosts_.clear();
|
||||
markers_.clear();
|
||||
tick_ = 0;
|
||||
}
|
||||
@@ -75,6 +116,8 @@ std::expected<void, MarkerError> ProbeMarkers::schedule(std::uint64_t session,
|
||||
std::ranges::find_if(entries_, [&](const Entry& e) { return e.marker.session == session; });
|
||||
if (it == entries_.end())
|
||||
return std::unexpected(MarkerError::unknown_session);
|
||||
if (it->marker.frozen)
|
||||
return std::unexpected(MarkerError::invalid);
|
||||
if (!valid(vector) || tick <= tick_ || sequence <= it->marker.applied_sequence ||
|
||||
(!it->scheduled.empty() && sequence <= it->scheduled.back().sequence))
|
||||
return std::unexpected(MarkerError::invalid);
|
||||
@@ -97,6 +140,8 @@ void ProbeMarkers::advance(std::uint64_t tick) {
|
||||
const auto first = tick - tick_ > max_catch_up ? tick - max_catch_up + 1 : tick_ + 1;
|
||||
for (auto& e : entries_) {
|
||||
auto& m = e.marker;
|
||||
if (m.frozen)
|
||||
continue;
|
||||
// Intentions for ticks that can no longer be stepped still set the held vector.
|
||||
while (!e.scheduled.empty() && e.scheduled.front().tick < first) {
|
||||
m.state.vector = e.scheduled.front().vector;
|
||||
@@ -117,8 +162,8 @@ void ProbeMarkers::advance(std::uint64_t tick) {
|
||||
}
|
||||
|
||||
const Marker* ProbeMarkers::of(std::uint64_t session) const noexcept {
|
||||
const auto it =
|
||||
std::ranges::find_if(markers_, [&](const Marker& m) { return m.session == session; });
|
||||
const auto it = std::ranges::find_if(
|
||||
markers_, [&](const Marker& m) { return !m.ghost && m.session == session; });
|
||||
return it == markers_.end() ? nullptr : &*it;
|
||||
}
|
||||
|
||||
@@ -126,6 +171,13 @@ void ProbeMarkers::publish() {
|
||||
markers_.clear();
|
||||
for (const auto& e : entries_)
|
||||
markers_.push_back(e.marker);
|
||||
for (const auto& g : ghosts_) {
|
||||
Marker m;
|
||||
m.id = g.id;
|
||||
m.state = {g.ghost.x, g.ghost.y, {}};
|
||||
m.ghost = true;
|
||||
markers_.push_back(m);
|
||||
}
|
||||
std::ranges::sort(markers_, {}, &Marker::id);
|
||||
}
|
||||
} // namespace evo::network::motion
|
||||
|
||||
@@ -150,7 +150,7 @@ std::vector<std::uint8_t> encode_record(const motion::Marker& m, std::int64_t z,
|
||||
put_signed(out, z);
|
||||
put(out, static_cast<std::uint16_t>(m.state.vector.x), 2);
|
||||
put(out, static_cast<std::uint16_t>(m.state.vector.y), 2);
|
||||
out.push_back(own_area != nullptr ? 1 : 0);
|
||||
out.push_back(own_area != nullptr ? 1 : m.ghost ? 2 : 0); // v4: 2 is a ghost.
|
||||
if (own_area != nullptr) {
|
||||
put(out, m.applied_sequence, 8);
|
||||
for (const auto v : {own_area->min_x, own_area->min_y, own_area->max_x, own_area->max_y})
|
||||
@@ -261,7 +261,7 @@ std::vector<std::pair<std::uint64_t, std::vector<SnapshotChunk>>> ReplicationSer
|
||||
// every marker first and makes it due every tick it changed.
|
||||
if (src.marker_area)
|
||||
for (const auto& m : src.markers) {
|
||||
const bool own = m.session == s.id;
|
||||
const bool own = !m.ghost && m.session == s.id;
|
||||
if (!own && (src.marker_area->z != in.z ||
|
||||
distance(cell_of(m.state.x), cell_of(m.state.y), in) > in.radius))
|
||||
continue;
|
||||
@@ -502,10 +502,11 @@ SnapshotAssembler::accept(const SnapshotChunk& chunk) {
|
||||
m.z = r.i64();
|
||||
m.state.vector = {static_cast<std::int16_t>(r.u(2)),
|
||||
static_cast<std::int16_t>(r.u(2))};
|
||||
const auto own = r.u(1);
|
||||
if (own > 1)
|
||||
const auto kind_byte = r.u(1);
|
||||
if (kind_byte > 2)
|
||||
return std::unexpected(AssemblyError::malformed);
|
||||
m.own = own == 1;
|
||||
m.own = kind_byte == 1;
|
||||
m.ghost = kind_byte == 2;
|
||||
if (m.own) {
|
||||
m.applied_sequence = r.u(8);
|
||||
m.area = {r.i64(), r.i64(), r.i64(), r.i64(), m.z};
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
#include "evo/network/session_protocol.hpp"
|
||||
#include <algorithm>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
namespace evo::network {
|
||||
namespace {
|
||||
@@ -19,6 +21,9 @@ constexpr std::uint8_t max_admit_status = static_cast<std::uint8_t>(AdmitStatus:
|
||||
constexpr std::size_t interest_bytes = (3 * std::size_t{8}) + 2;
|
||||
constexpr std::size_t snapshot_ack_bytes = 2 * std::size_t{8};
|
||||
constexpr std::uint8_t max_snapshot_chunks = 16;
|
||||
constexpr std::size_t address_bytes = 8 + (2 * session_address_bytes);
|
||||
constexpr std::size_t handoff_offer_bytes = 16 + address_bytes + session_grant_wire_bytes;
|
||||
constexpr std::size_t redirect_bytes = address_bytes;
|
||||
|
||||
void put(std::vector<std::uint8_t>& out, std::uint64_t v, std::size_t count) {
|
||||
for (std::size_t i = 0; i < count; ++i)
|
||||
@@ -41,7 +46,44 @@ std::vector<std::uint8_t> header(SessionMessageType type, std::size_t payload) {
|
||||
}
|
||||
bool wire_reason(std::uint64_t v) {
|
||||
return v >= static_cast<std::uint8_t>(SessionCloseReason::client_closed) &&
|
||||
v <= static_cast<std::uint8_t>(SessionCloseReason::shutdown);
|
||||
v <= static_cast<std::uint8_t>(SessionCloseReason::moved);
|
||||
}
|
||||
// Printable ASCII without spaces, then zero padding to the field size.
|
||||
bool valid_text(const std::string& s) {
|
||||
return !s.empty() && s.size() <= session_address_bytes &&
|
||||
std::ranges::all_of(s, [](char ch) { return ch > 0x20 && ch < 0x7F; });
|
||||
}
|
||||
void put_text(std::vector<std::uint8_t>& out, const std::string& s) {
|
||||
for (std::size_t i = 0; i < session_address_bytes; ++i)
|
||||
out.push_back(i < s.size() ? static_cast<std::uint8_t>(s[i]) : std::uint8_t{0});
|
||||
}
|
||||
std::optional<std::string> text_at(std::span<const std::uint8_t> in, std::size_t at) {
|
||||
std::string s;
|
||||
std::size_t i = 0;
|
||||
for (; i < session_address_bytes && in[at + i] != 0; ++i)
|
||||
s.push_back(static_cast<char>(in[at + i]));
|
||||
for (; i < session_address_bytes; ++i)
|
||||
if (in[at + i] != 0)
|
||||
return std::nullopt; // Bytes after the terminator must be padding.
|
||||
if (!valid_text(s))
|
||||
return std::nullopt;
|
||||
return s;
|
||||
}
|
||||
void put_address(std::vector<std::uint8_t>& out, const RegionAddress& a) {
|
||||
put(out, a.region, 8);
|
||||
put_text(out, a.endpoint);
|
||||
put_text(out, a.server_name);
|
||||
}
|
||||
std::optional<RegionAddress> address_at(std::span<const std::uint8_t> in, std::size_t at) {
|
||||
RegionAddress a;
|
||||
a.region = get(in, at, 8);
|
||||
auto endpoint = text_at(in, at + 8);
|
||||
auto name = text_at(in, at + 8 + session_address_bytes);
|
||||
if (a.region == 0 || !endpoint || !name)
|
||||
return std::nullopt;
|
||||
a.endpoint = std::move(*endpoint);
|
||||
a.server_name = std::move(*name);
|
||||
return a;
|
||||
}
|
||||
WireGrant grant_at(std::span<const std::uint8_t> in, std::size_t at) {
|
||||
WireGrant g{};
|
||||
@@ -157,6 +199,21 @@ std::vector<std::uint8_t> encode_session(const SessionMessage& message) {
|
||||
put(out, m.tick, 8);
|
||||
return out;
|
||||
}
|
||||
// Invalid text fields are encoded as given; the receiver refuses them.
|
||||
std::vector<std::uint8_t> operator()(const HandoffOffer& m) const {
|
||||
auto out = header(SessionMessageType::handoff_offer, handoff_offer_bytes);
|
||||
out.insert(out.end(), m.transfer.begin(), m.transfer.end());
|
||||
put_address(out, m.destination);
|
||||
out.insert(out.end(), m.ticket.begin(), m.ticket.end());
|
||||
out.resize(session_header_bytes + handoff_offer_bytes); // Overlong text truncates.
|
||||
return out;
|
||||
}
|
||||
std::vector<std::uint8_t> operator()(const SessionRedirect& m) const {
|
||||
auto out = header(SessionMessageType::redirect, redirect_bytes);
|
||||
put_address(out, m.owner);
|
||||
out.resize(session_header_bytes + redirect_bytes);
|
||||
return out;
|
||||
}
|
||||
};
|
||||
return std::visit(Encoder{}, message);
|
||||
}
|
||||
@@ -306,6 +363,26 @@ decode_session(std::span<const std::uint8_t> bytes) {
|
||||
return std::unexpected(SessionCodecError::wrong_length);
|
||||
return SnapshotAck{get(payload, 0, 8), get(payload, 8, 8)};
|
||||
}
|
||||
case static_cast<std::uint8_t>(SessionMessageType::handoff_offer): {
|
||||
if (!expect(handoff_offer_bytes))
|
||||
return std::unexpected(SessionCodecError::wrong_length);
|
||||
HandoffOffer m;
|
||||
std::copy_n(payload.begin(), m.transfer.size(), m.transfer.begin());
|
||||
auto destination = address_at(payload, 16);
|
||||
if (!destination || std::ranges::all_of(m.transfer, [](auto b) { return b == 0; }))
|
||||
return std::unexpected(SessionCodecError::invalid_value);
|
||||
m.destination = std::move(*destination);
|
||||
m.ticket = grant_at(payload, 16 + address_bytes);
|
||||
return m;
|
||||
}
|
||||
case static_cast<std::uint8_t>(SessionMessageType::redirect): {
|
||||
if (!expect(redirect_bytes))
|
||||
return std::unexpected(SessionCodecError::wrong_length);
|
||||
auto owner = address_at(payload, 0);
|
||||
if (!owner)
|
||||
return std::unexpected(SessionCodecError::invalid_value);
|
||||
return SessionRedirect{std::move(*owner)};
|
||||
}
|
||||
default:
|
||||
return std::unexpected(SessionCodecError::unknown_type);
|
||||
}
|
||||
|
||||
@@ -721,4 +721,31 @@ find_probe_transfer(Postgres& db, RegionKey key, const LeaseGrant& lease, const
|
||||
return std::unexpected{ProbeStoreFailure{ProbeStoreError::unknown, {}}};
|
||||
return t;
|
||||
}
|
||||
|
||||
std::expected<std::vector<IncomingTransfer>, ProbeStoreFailure>
|
||||
incoming_probe_transfers(Postgres& db, RegionKey key, const LeaseGrant& lease,
|
||||
milliseconds timeout) {
|
||||
const std::array<std::string, 4> p{std::to_string(key.world), std::to_string(key.region),
|
||||
uuid_text(lease.holder), std::to_string(lease.epoch)};
|
||||
auto rows = db.query("SELECT * FROM evolution.incoming_probe_transfers($1::evolution.u64, "
|
||||
"$2::evolution.u64, $3::uuid, $4::evolution.u64)",
|
||||
p, timeout);
|
||||
if (!rows)
|
||||
return std::unexpected{probe_failure(rows.error())};
|
||||
std::vector<IncomingTransfer> out;
|
||||
for (const auto& r : *rows) {
|
||||
if (r.size() != 6)
|
||||
return std::unexpected{ProbeStoreFailure{ProbeStoreError::unknown, {}}};
|
||||
const auto transfer = parse_uuid(r[0]);
|
||||
const auto account = parse_uuid(r[1]);
|
||||
const auto source = parse_integer<std::uint64_t>(r[2]);
|
||||
const auto x = parse_signed(r[3]);
|
||||
const auto y = parse_signed(r[4]);
|
||||
const auto z = parse_signed(r[5]);
|
||||
if (!transfer || !account || !source || !x || !y || !z)
|
||||
return std::unexpected{ProbeStoreFailure{ProbeStoreError::unknown, {}}};
|
||||
out.push_back({*transfer, *account, *source, *x, *y, *z});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
|
||||
@@ -27,7 +27,7 @@ HandoffLedgerError ledger_error(const persistence::ProbeStoreFailure& f) {
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::expected<network::ControlLease, HandoffLedgerError>
|
||||
std::expected<network::HandoffAdmission, HandoffLedgerError>
|
||||
PostgresHandoffLedger::admit(const network::TicketClaims& ticket, std::uint64_t session) {
|
||||
const auto lease = lease_();
|
||||
if (!lease)
|
||||
@@ -65,6 +65,108 @@ PostgresHandoffLedger::admit(const network::TicketClaims& ticket, std::uint64_t
|
||||
return std::unexpected(lease_fenced(control.error().cause) ? HandoffLedgerError::fenced
|
||||
: HandoffLedgerError::refused);
|
||||
}
|
||||
return network::ControlLease{control->generation, control->wait, control->granted};
|
||||
return network::HandoffAdmission{
|
||||
network::ControlLease{control->generation, control->wait, control->granted},
|
||||
committed->generation + 1, committed->x, committed->y};
|
||||
}
|
||||
|
||||
namespace {
|
||||
ProbeLedgerError probe_error(const persistence::ProbeStoreFailure& f) {
|
||||
return f.kind == ProbeStoreError::fenced && lease_fenced(f.cause)
|
||||
? ProbeLedgerError::fenced
|
||||
: ProbeLedgerError::unavailable;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::expected<ProbeHome, ProbeLedgerError>
|
||||
PostgresProbeLedger::open(const network::AccountId& account, std::int64_t x, std::int64_t y,
|
||||
std::int64_t z) {
|
||||
const auto lease = lease_();
|
||||
if (!lease)
|
||||
return std::unexpected(ProbeLedgerError::fenced);
|
||||
auto* db = db_->get();
|
||||
if (!db)
|
||||
return std::unexpected(ProbeLedgerError::unavailable);
|
||||
const auto r =
|
||||
persistence::open_probe(*db, key_, *lease, account.bytes, x, y, z, inventory, timeout_);
|
||||
if (!r) {
|
||||
if (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown)
|
||||
db_->broken();
|
||||
return std::unexpected(probe_error(r.error()));
|
||||
}
|
||||
return ProbeHome{r->region, r->generation, r->x, r->y, r->transfer.has_value()};
|
||||
}
|
||||
|
||||
void PostgresProbeLedger::save(const network::AccountId& account, std::uint64_t generation,
|
||||
std::int64_t x, std::int64_t y, std::int64_t z) {
|
||||
const auto lease = lease_();
|
||||
if (!lease)
|
||||
return;
|
||||
auto* db = db_->get();
|
||||
if (!db)
|
||||
return;
|
||||
if (const auto r = persistence::save_probe(*db, key_, *lease, account.bytes, generation, x, y,
|
||||
z, timeout_);
|
||||
!r &&
|
||||
(r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown))
|
||||
db_->broken();
|
||||
}
|
||||
|
||||
std::expected<void, ProbeLedgerError>
|
||||
PostgresProbeLedger::prepare(const network::AccountId& account, std::uint64_t generation,
|
||||
const network::TransferId& transfer, std::uint64_t destination,
|
||||
std::int64_t x, std::int64_t y, std::int64_t z) {
|
||||
const auto lease = lease_();
|
||||
if (!lease)
|
||||
return std::unexpected(ProbeLedgerError::fenced);
|
||||
auto* db = db_->get();
|
||||
if (!db)
|
||||
return std::unexpected(ProbeLedgerError::unavailable);
|
||||
const auto r = persistence::prepare_probe_transfer(*db, key_, *lease, account.bytes, generation,
|
||||
transfer, destination, x, y, z, timeout_);
|
||||
if (!r) {
|
||||
if (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown)
|
||||
db_->broken();
|
||||
return std::unexpected(probe_error(r.error()));
|
||||
}
|
||||
if (r->state != persistence::TransferState::prepared)
|
||||
return std::unexpected(ProbeLedgerError::unavailable); // A retry found it decided.
|
||||
return {};
|
||||
}
|
||||
|
||||
std::expected<bool, ProbeLedgerError>
|
||||
PostgresProbeLedger::abort(const network::TransferId& transfer) {
|
||||
const auto lease = lease_();
|
||||
if (!lease)
|
||||
return std::unexpected(ProbeLedgerError::fenced);
|
||||
auto* db = db_->get();
|
||||
if (!db)
|
||||
return std::unexpected(ProbeLedgerError::unavailable);
|
||||
const auto r = persistence::abort_probe_transfer(*db, key_, *lease, transfer, timeout_);
|
||||
if (!r) {
|
||||
if (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown)
|
||||
db_->broken();
|
||||
return std::unexpected(probe_error(r.error()));
|
||||
}
|
||||
return r->state == persistence::TransferState::aborted;
|
||||
}
|
||||
|
||||
std::expected<std::vector<IncomingProbe>, ProbeLedgerError> PostgresProbeLedger::incoming() {
|
||||
const auto lease = lease_();
|
||||
if (!lease)
|
||||
return std::unexpected(ProbeLedgerError::fenced);
|
||||
auto* db = db_->get();
|
||||
if (!db)
|
||||
return std::unexpected(ProbeLedgerError::unavailable);
|
||||
const auto r = persistence::incoming_probe_transfers(*db, key_, *lease, timeout_);
|
||||
if (!r) {
|
||||
if (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown)
|
||||
db_->broken();
|
||||
return std::unexpected(probe_error(r.error()));
|
||||
}
|
||||
std::vector<IncomingProbe> out;
|
||||
for (const auto& t : *r)
|
||||
out.push_back({t.transfer, t.x, t.y});
|
||||
return out;
|
||||
}
|
||||
} // namespace evo::server
|
||||
|
||||
@@ -80,6 +80,12 @@ struct RegionSessions::Impl {
|
||||
std::uint64_t generation{};
|
||||
Clock::time_point control_deadline, next_renew;
|
||||
network::SessionWelcome welcome;
|
||||
// Durable probe (probe records only): its ownership generation and position.
|
||||
std::optional<ProbeHome> probe;
|
||||
};
|
||||
struct Outgoing {
|
||||
network::TransferId transfer{};
|
||||
std::uint64_t deadline{}; // Wall seconds: abort unless committed by then.
|
||||
};
|
||||
|
||||
Impl(RegionSessionsConfig c, std::unique_ptr<network::quic::Endpoint> e,
|
||||
@@ -96,7 +102,11 @@ struct RegionSessions::Impl {
|
||||
network::SessionControl control;
|
||||
network::IntentAdmission intents;
|
||||
network::ReplicationServer replication;
|
||||
network::motion::ProbeMarkers markers; // Ephemeral probe markers (ADR-0094).
|
||||
network::motion::ProbeMarkers markers; // Probe markers (ADR-0094, durable: ADR-0096).
|
||||
std::optional<network::motion::Area> own_area; // This region's part of the marker area.
|
||||
std::int64_t spawn_x{}, spawn_y{}; // Where new probes start.
|
||||
std::vector<Outgoing> outgoing; // Handed off, not yet known committed.
|
||||
Clock::time_point next_ghosts{};
|
||||
std::optional<network::SessionInterest> default_interest;
|
||||
HostFacts facts;
|
||||
std::vector<Connection> connections;
|
||||
@@ -140,8 +150,10 @@ struct RegionSessions::Impl {
|
||||
if (c.state == State::established) {
|
||||
intents.close(c.session);
|
||||
replication.close(c.session);
|
||||
save_probe(c);
|
||||
markers.close(c.session);
|
||||
}
|
||||
c.probe.reset();
|
||||
if (c.session != 0) // May already be released (superseded, expired, epoch).
|
||||
(void)control.close(c.session).has_value();
|
||||
// A superseded session's durable control already moved to its successor.
|
||||
@@ -153,6 +165,49 @@ struct RegionSessions::Impl {
|
||||
c.deadline = now + config.close_flush;
|
||||
}
|
||||
|
||||
// Best effort, at session end: the durable probe's position (a frozen or moved probe is
|
||||
// fenced by the store and stays as it was).
|
||||
void save_probe(const Connection& c) {
|
||||
if (!config.probes || !c.probe)
|
||||
return;
|
||||
if (const auto* m = markers.of(c.session); m && !m->frozen && own_area)
|
||||
config.probes->save(c.account_id, c.probe->generation, m->state.x, m->state.y,
|
||||
own_area->z);
|
||||
}
|
||||
// After the durable control claim of a grant login: where the account's probe lives.
|
||||
// False when the session must not open here (the connection is then closed).
|
||||
bool place_probe(Connection& c, const network::AccountId& account,
|
||||
const std::optional<network::ControlLease>& lease, Clock::time_point now) {
|
||||
if (!config.probes || !own_area)
|
||||
return true;
|
||||
const auto home = config.probes->open(account, spawn_x, spawn_y, own_area->z);
|
||||
const auto refuse = [&](SessionCloseReason why) {
|
||||
if (lease)
|
||||
config.ledger->release(account, lease->generation);
|
||||
close(c, why, now);
|
||||
return false;
|
||||
};
|
||||
if (!home)
|
||||
return refuse(home.error() == ProbeLedgerError::fenced ? SessionCloseReason::shutdown
|
||||
: SessionCloseReason::capacity);
|
||||
if (home->owner != config.control.region) {
|
||||
const auto n = std::ranges::find_if(config.neighbours, [&](const Neighbour& x) {
|
||||
return x.address.region == home->owner;
|
||||
});
|
||||
if (n != config.neighbours.end())
|
||||
send(c, network::SessionRedirect{n->address});
|
||||
return refuse(SessionCloseReason::moved);
|
||||
}
|
||||
if (home->frozen) // A transfer is open: decided by its commit or abort, retry later.
|
||||
return refuse(SessionCloseReason::capacity);
|
||||
c.probe = *home;
|
||||
// After an aborted handoff the probe was saved where it crossed: back inside, so the
|
||||
// new session does not cross again at once.
|
||||
c.probe->x = std::clamp(c.probe->x, own_area->min_x, own_area->max_x);
|
||||
c.probe->y = std::clamp(c.probe->y, own_area->min_y, own_area->max_y);
|
||||
return true;
|
||||
}
|
||||
|
||||
void hello(Connection& c, const network::SessionHello& m, std::uint64_t now_seconds,
|
||||
Clock::time_point now) {
|
||||
if (m.required_features != 0)
|
||||
@@ -178,6 +233,8 @@ struct RegionSessions::Impl {
|
||||
return close(c, reason_for(claimed.error()), now);
|
||||
lease = *claimed;
|
||||
}
|
||||
if (!place_probe(c, claims.account, lease, now))
|
||||
return;
|
||||
admitted(c, control.open(*grant, now_seconds), lease, promised, claims.account, now_seconds,
|
||||
now);
|
||||
}
|
||||
@@ -201,7 +258,10 @@ struct RegionSessions::Impl {
|
||||
auto moved = config.handoff->admit(claims, promised);
|
||||
if (!moved)
|
||||
return close(c, reason_for(moved.error()), now);
|
||||
lease = *moved;
|
||||
lease = moved->lease;
|
||||
if (config.probes)
|
||||
c.probe =
|
||||
ProbeHome{config.control.region, moved->generation, moved->x, moved->y, false};
|
||||
} else if (config.ledger) {
|
||||
return close(c, SessionCloseReason::capacity, now); // Misconfigured: fail closed.
|
||||
}
|
||||
@@ -257,7 +317,9 @@ struct RegionSessions::Impl {
|
||||
if (default_interest)
|
||||
(void)replication.set_interest(c.session, *default_interest).has_value();
|
||||
if (markers.area()) // Full or refused: the session simply has no marker.
|
||||
(void)markers.open(c.session).has_value();
|
||||
(void)(c.probe ? markers.open_at(c.session, c.probe->x, c.probe->y)
|
||||
: markers.open(c.session))
|
||||
.has_value();
|
||||
c.state = State::established;
|
||||
c.welcome.server_tick = facts.simulated_tick;
|
||||
send(c, c.welcome);
|
||||
@@ -333,6 +395,95 @@ struct RegionSessions::Impl {
|
||||
send_datagram(c, receipt);
|
||||
}
|
||||
|
||||
[[nodiscard]] bool handoffs_enabled() const {
|
||||
return config.probes != nullptr && config.handoff_key != nullptr &&
|
||||
!config.neighbours.empty() && own_area.has_value();
|
||||
}
|
||||
// Markers that left this region's area: into a neighbour's area they are handed over,
|
||||
// anywhere else they are put back at the edge.
|
||||
void crossings(std::uint64_t now_seconds, Clock::time_point now) {
|
||||
if (!own_area)
|
||||
return;
|
||||
const auto inside = [](const network::motion::Area& a, std::int64_t x, std::int64_t y) {
|
||||
return x >= a.min_x && x <= a.max_x && y >= a.min_y && y <= a.max_y;
|
||||
};
|
||||
for (auto& c : connections) {
|
||||
if (c.state != State::established)
|
||||
continue;
|
||||
const auto* m = markers.of(c.session);
|
||||
if (m == nullptr || m->frozen || inside(*own_area, m->state.x, m->state.y))
|
||||
continue;
|
||||
const auto n = std::ranges::find_if(config.neighbours, [&](const Neighbour& x) {
|
||||
return inside(x.area, m->state.x, m->state.y);
|
||||
});
|
||||
if (n == config.neighbours.end() || !handoffs_enabled() || !c.probe) {
|
||||
const auto x = std::clamp(m->state.x, own_area->min_x, own_area->max_x);
|
||||
const auto y = std::clamp(m->state.y, own_area->min_y, own_area->max_y);
|
||||
markers.place(c.session, x, y); // Back at the edge (`m` is stale after this).
|
||||
continue;
|
||||
}
|
||||
hand_off(c, *n, now_seconds, now);
|
||||
}
|
||||
}
|
||||
// The source side of a crossing: freeze, prepare durably, release control, offer the
|
||||
// ticket, close. On any failure the probe stays here, at the edge.
|
||||
void hand_off(Connection& c, const Neighbour& n, std::uint64_t now_seconds,
|
||||
Clock::time_point now) {
|
||||
if (!c.probe || !own_area) // Checked by the caller (handoffs_enabled, c.probe).
|
||||
return;
|
||||
const auto generation = c.probe->generation;
|
||||
const auto z = own_area->z;
|
||||
markers.freeze(c.session);
|
||||
const auto* m = markers.of(c.session);
|
||||
network::TransferId transfer{};
|
||||
if (m == nullptr || !network::random_bytes(transfer))
|
||||
return close(c, SessionCloseReason::shutdown, now);
|
||||
const auto x = m->state.x, y = m->state.y;
|
||||
if (const auto prepared = config.probes->prepare(c.account_id, generation, transfer,
|
||||
n.address.region, x, y, z);
|
||||
!prepared)
|
||||
return close(c, SessionCloseReason::shutdown, now);
|
||||
c.probe.reset(); // Frozen under the transfer: never saved here again.
|
||||
network::TicketClaims claims;
|
||||
claims.transfer = transfer;
|
||||
claims.account = c.account_id;
|
||||
claims.world = config.control.world;
|
||||
claims.source = config.control.region;
|
||||
claims.destination = n.address.region;
|
||||
claims.expires_at = static_cast<std::uint32_t>(now_seconds + config.ticket_lifetime);
|
||||
claims.session_expires_at = static_cast<std::uint32_t>(c.welcome.expires_at);
|
||||
const auto ticket = config.handoff_key->sign(claims);
|
||||
// Control goes first so the destination need not wait it out.
|
||||
if (config.ledger && c.generation != 0)
|
||||
config.ledger->release(c.account_id, c.generation);
|
||||
c.generation = 0;
|
||||
outgoing.push_back({transfer, now_seconds + config.ticket_lifetime + 5});
|
||||
if (ticket)
|
||||
send(c, network::HandoffOffer{transfer, n.address, *ticket});
|
||||
close(c, SessionCloseReason::handed_off, now);
|
||||
}
|
||||
// Transfers whose client never arrived are aborted (the probe resumes here); committed
|
||||
// ones are forgotten. Undecided calls retry next poll.
|
||||
void expire_outgoing(std::uint64_t now_seconds) {
|
||||
std::erase_if(outgoing, [&](const Outgoing& o) {
|
||||
if (now_seconds < o.deadline)
|
||||
return false;
|
||||
return config.probes->abort(o.transfer).has_value();
|
||||
});
|
||||
}
|
||||
void refresh_ghosts(Clock::time_point now) {
|
||||
if (now < next_ghosts)
|
||||
return;
|
||||
next_ghosts = now + config.ghost_poll;
|
||||
const auto incoming = config.probes->incoming();
|
||||
if (!incoming)
|
||||
return; // Keep the current ghosts until the next answer.
|
||||
std::vector<network::motion::Ghost> ghosts;
|
||||
for (const auto& p : *incoming)
|
||||
ghosts.push_back({p.transfer, p.x, p.y});
|
||||
markers.set_ghosts(ghosts);
|
||||
}
|
||||
|
||||
void deliver(const std::vector<network::AccountOutcome>& outcomes) {
|
||||
for (const auto& o : outcomes)
|
||||
if (const auto session = intents.session_of(o.account))
|
||||
@@ -394,6 +545,8 @@ std::expected<std::unique_ptr<RegionSessions>, RegionSessionsError>
|
||||
RegionSessions::create(RegionSessionsConfig config) {
|
||||
// Connections beyond sessions are headroom for logins and closes still flushing.
|
||||
if (config.issuer_keys.empty() || (config.handoff != nullptr && config.ledger == nullptr) ||
|
||||
(config.handoff_key != nullptr && config.probes == nullptr) ||
|
||||
config.ticket_lifetime == 0 || config.ticket_lifetime > network::max_ticket_lifetime ||
|
||||
config.transport.limits.max_connections < config.control.max_sessions ||
|
||||
config.hello_deadline.count() <= 0 || config.close_flush.count() <= 0 ||
|
||||
config.renew_interval.count() <= 0 || config.safety_margin.count() <= 0)
|
||||
@@ -451,8 +604,14 @@ void RegionSessions::poll(const HostFacts& facts, std::uint64_t now_seconds,
|
||||
m.markers.clear(); // The new epoch may restart at an earlier tick.
|
||||
}
|
||||
// Markers reach the simulated tick before this poll's intentions are admitted.
|
||||
if (facts.ready)
|
||||
if (facts.ready) {
|
||||
m.markers.advance(facts.simulated_tick);
|
||||
m.crossings(now_seconds, now);
|
||||
if (m.config.probes) {
|
||||
m.expire_outgoing(now_seconds);
|
||||
m.refresh_ghosts(now);
|
||||
}
|
||||
}
|
||||
for (auto& e : m.endpoint->poll()) {
|
||||
using network::quic::EventKind;
|
||||
if (e.kind == EventKind::connected) {
|
||||
@@ -471,6 +630,7 @@ void RegionSessions::poll(const HostFacts& facts, std::uint64_t now_seconds,
|
||||
if (it->state == Impl::State::established) {
|
||||
m.intents.close(it->session);
|
||||
m.replication.close(it->session);
|
||||
m.save_probe(*it);
|
||||
m.markers.close(it->session);
|
||||
}
|
||||
if (it->state == Impl::State::pending) {
|
||||
@@ -563,8 +723,18 @@ network::ReplicationStats RegionSessions::replication_stats() const {
|
||||
bool RegionSessions::set_motion_area(const network::motion::Area& area, std::int64_t spawn_x,
|
||||
std::int64_t spawn_y) {
|
||||
auto& m = *impl_;
|
||||
if (!m.markers.set_area(area, spawn_x, spawn_y))
|
||||
auto movement = area;
|
||||
for (const auto& n : m.config.neighbours) {
|
||||
movement.min_x = std::min(movement.min_x, n.area.min_x);
|
||||
movement.min_y = std::min(movement.min_y, n.area.min_y);
|
||||
movement.max_x = std::max(movement.max_x, n.area.max_x);
|
||||
movement.max_y = std::max(movement.max_y, n.area.max_y);
|
||||
}
|
||||
if (!network::motion::valid(area) || !m.markers.set_area(movement, spawn_x, spawn_y))
|
||||
return false;
|
||||
m.own_area = area;
|
||||
m.spawn_x = std::clamp(spawn_x, area.min_x, area.max_x);
|
||||
m.spawn_y = std::clamp(spawn_y, area.min_y, area.max_y);
|
||||
for (const auto& c : m.connections)
|
||||
if (c.state == Impl::State::established && !m.markers.of(c.session))
|
||||
(void)m.markers.open(c.session).has_value();
|
||||
|
||||
Reference in New Issue
Block a user