Wire M10.8c-1 region handoff: protocol v4, durable probes, crossing, ghosts
Foundation / checks (deploy) (push) Successful in 3m29s
Foundation / checks (ecs) (push) Successful in 2m0s
Foundation / checks (client) (push) Successful in 12m51s
Foundation / checks (network) (push) Successful in 13m0s
Foundation / checks (tsan) (push) Successful in 7m50s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 32m4s
Foundation / checks (postgres) (push) Canceled after 19m13s

ADR-0096. Session protocol v4 (v3 is deployed and frozen) adds HandoffOffer,
SessionRedirect, close reasons handed_off and moved, and ghost marker records.
Sessions use the durable probe record: logins open it where it was saved
(redirected when it lives elsewhere), and session ends save it. A marker
crossing into a neighbour's area is frozen, prepared, its control released
and a signed ticket offered; uncommitted transfers are aborted after the
ticket's lifetime. The destination shows arriving probes as ghosts (schema
v13 incoming_probe_transfers) and opens the marker where it crossed.

Refreshes the dev session window.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-06 21:24:42 -04:00
co-authored by Claude Opus 5.5
parent 97a3756e46
commit a7a3d12283
39 changed files with 1315 additions and 68 deletions
+104 -2
View File
@@ -27,7 +27,7 @@ HandoffLedgerError ledger_error(const persistence::ProbeStoreFailure& f) {
}
} // namespace
std::expected<network::ControlLease, HandoffLedgerError>
std::expected<network::HandoffAdmission, HandoffLedgerError>
PostgresHandoffLedger::admit(const network::TicketClaims& ticket, std::uint64_t session) {
const auto lease = lease_();
if (!lease)
@@ -65,6 +65,108 @@ PostgresHandoffLedger::admit(const network::TicketClaims& ticket, std::uint64_t
return std::unexpected(lease_fenced(control.error().cause) ? HandoffLedgerError::fenced
: HandoffLedgerError::refused);
}
return network::ControlLease{control->generation, control->wait, control->granted};
return network::HandoffAdmission{
network::ControlLease{control->generation, control->wait, control->granted},
committed->generation + 1, committed->x, committed->y};
}
namespace {
ProbeLedgerError probe_error(const persistence::ProbeStoreFailure& f) {
return f.kind == ProbeStoreError::fenced && lease_fenced(f.cause)
? ProbeLedgerError::fenced
: ProbeLedgerError::unavailable;
}
} // namespace
std::expected<ProbeHome, ProbeLedgerError>
PostgresProbeLedger::open(const network::AccountId& account, std::int64_t x, std::int64_t y,
std::int64_t z) {
const auto lease = lease_();
if (!lease)
return std::unexpected(ProbeLedgerError::fenced);
auto* db = db_->get();
if (!db)
return std::unexpected(ProbeLedgerError::unavailable);
const auto r =
persistence::open_probe(*db, key_, *lease, account.bytes, x, y, z, inventory, timeout_);
if (!r) {
if (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown)
db_->broken();
return std::unexpected(probe_error(r.error()));
}
return ProbeHome{r->region, r->generation, r->x, r->y, r->transfer.has_value()};
}
void PostgresProbeLedger::save(const network::AccountId& account, std::uint64_t generation,
std::int64_t x, std::int64_t y, std::int64_t z) {
const auto lease = lease_();
if (!lease)
return;
auto* db = db_->get();
if (!db)
return;
if (const auto r = persistence::save_probe(*db, key_, *lease, account.bytes, generation, x, y,
z, timeout_);
!r &&
(r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown))
db_->broken();
}
std::expected<void, ProbeLedgerError>
PostgresProbeLedger::prepare(const network::AccountId& account, std::uint64_t generation,
const network::TransferId& transfer, std::uint64_t destination,
std::int64_t x, std::int64_t y, std::int64_t z) {
const auto lease = lease_();
if (!lease)
return std::unexpected(ProbeLedgerError::fenced);
auto* db = db_->get();
if (!db)
return std::unexpected(ProbeLedgerError::unavailable);
const auto r = persistence::prepare_probe_transfer(*db, key_, *lease, account.bytes, generation,
transfer, destination, x, y, z, timeout_);
if (!r) {
if (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown)
db_->broken();
return std::unexpected(probe_error(r.error()));
}
if (r->state != persistence::TransferState::prepared)
return std::unexpected(ProbeLedgerError::unavailable); // A retry found it decided.
return {};
}
std::expected<bool, ProbeLedgerError>
PostgresProbeLedger::abort(const network::TransferId& transfer) {
const auto lease = lease_();
if (!lease)
return std::unexpected(ProbeLedgerError::fenced);
auto* db = db_->get();
if (!db)
return std::unexpected(ProbeLedgerError::unavailable);
const auto r = persistence::abort_probe_transfer(*db, key_, *lease, transfer, timeout_);
if (!r) {
if (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown)
db_->broken();
return std::unexpected(probe_error(r.error()));
}
return r->state == persistence::TransferState::aborted;
}
std::expected<std::vector<IncomingProbe>, ProbeLedgerError> PostgresProbeLedger::incoming() {
const auto lease = lease_();
if (!lease)
return std::unexpected(ProbeLedgerError::fenced);
auto* db = db_->get();
if (!db)
return std::unexpected(ProbeLedgerError::unavailable);
const auto r = persistence::incoming_probe_transfers(*db, key_, *lease, timeout_);
if (!r) {
if (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown)
db_->broken();
return std::unexpected(probe_error(r.error()));
}
std::vector<IncomingProbe> out;
for (const auto& t : *r)
out.push_back({t.transfer, t.x, t.y});
return out;
}
} // namespace evo::server
+175 -5
View File
@@ -80,6 +80,12 @@ struct RegionSessions::Impl {
std::uint64_t generation{};
Clock::time_point control_deadline, next_renew;
network::SessionWelcome welcome;
// Durable probe (probe records only): its ownership generation and position.
std::optional<ProbeHome> probe;
};
struct Outgoing {
network::TransferId transfer{};
std::uint64_t deadline{}; // Wall seconds: abort unless committed by then.
};
Impl(RegionSessionsConfig c, std::unique_ptr<network::quic::Endpoint> e,
@@ -96,7 +102,11 @@ struct RegionSessions::Impl {
network::SessionControl control;
network::IntentAdmission intents;
network::ReplicationServer replication;
network::motion::ProbeMarkers markers; // Ephemeral probe markers (ADR-0094).
network::motion::ProbeMarkers markers; // Probe markers (ADR-0094, durable: ADR-0096).
std::optional<network::motion::Area> own_area; // This region's part of the marker area.
std::int64_t spawn_x{}, spawn_y{}; // Where new probes start.
std::vector<Outgoing> outgoing; // Handed off, not yet known committed.
Clock::time_point next_ghosts{};
std::optional<network::SessionInterest> default_interest;
HostFacts facts;
std::vector<Connection> connections;
@@ -140,8 +150,10 @@ struct RegionSessions::Impl {
if (c.state == State::established) {
intents.close(c.session);
replication.close(c.session);
save_probe(c);
markers.close(c.session);
}
c.probe.reset();
if (c.session != 0) // May already be released (superseded, expired, epoch).
(void)control.close(c.session).has_value();
// A superseded session's durable control already moved to its successor.
@@ -153,6 +165,49 @@ struct RegionSessions::Impl {
c.deadline = now + config.close_flush;
}
// Best effort, at session end: the durable probe's position (a frozen or moved probe is
// fenced by the store and stays as it was).
void save_probe(const Connection& c) {
if (!config.probes || !c.probe)
return;
if (const auto* m = markers.of(c.session); m && !m->frozen && own_area)
config.probes->save(c.account_id, c.probe->generation, m->state.x, m->state.y,
own_area->z);
}
// After the durable control claim of a grant login: where the account's probe lives.
// False when the session must not open here (the connection is then closed).
bool place_probe(Connection& c, const network::AccountId& account,
const std::optional<network::ControlLease>& lease, Clock::time_point now) {
if (!config.probes || !own_area)
return true;
const auto home = config.probes->open(account, spawn_x, spawn_y, own_area->z);
const auto refuse = [&](SessionCloseReason why) {
if (lease)
config.ledger->release(account, lease->generation);
close(c, why, now);
return false;
};
if (!home)
return refuse(home.error() == ProbeLedgerError::fenced ? SessionCloseReason::shutdown
: SessionCloseReason::capacity);
if (home->owner != config.control.region) {
const auto n = std::ranges::find_if(config.neighbours, [&](const Neighbour& x) {
return x.address.region == home->owner;
});
if (n != config.neighbours.end())
send(c, network::SessionRedirect{n->address});
return refuse(SessionCloseReason::moved);
}
if (home->frozen) // A transfer is open: decided by its commit or abort, retry later.
return refuse(SessionCloseReason::capacity);
c.probe = *home;
// After an aborted handoff the probe was saved where it crossed: back inside, so the
// new session does not cross again at once.
c.probe->x = std::clamp(c.probe->x, own_area->min_x, own_area->max_x);
c.probe->y = std::clamp(c.probe->y, own_area->min_y, own_area->max_y);
return true;
}
void hello(Connection& c, const network::SessionHello& m, std::uint64_t now_seconds,
Clock::time_point now) {
if (m.required_features != 0)
@@ -178,6 +233,8 @@ struct RegionSessions::Impl {
return close(c, reason_for(claimed.error()), now);
lease = *claimed;
}
if (!place_probe(c, claims.account, lease, now))
return;
admitted(c, control.open(*grant, now_seconds), lease, promised, claims.account, now_seconds,
now);
}
@@ -201,7 +258,10 @@ struct RegionSessions::Impl {
auto moved = config.handoff->admit(claims, promised);
if (!moved)
return close(c, reason_for(moved.error()), now);
lease = *moved;
lease = moved->lease;
if (config.probes)
c.probe =
ProbeHome{config.control.region, moved->generation, moved->x, moved->y, false};
} else if (config.ledger) {
return close(c, SessionCloseReason::capacity, now); // Misconfigured: fail closed.
}
@@ -257,7 +317,9 @@ struct RegionSessions::Impl {
if (default_interest)
(void)replication.set_interest(c.session, *default_interest).has_value();
if (markers.area()) // Full or refused: the session simply has no marker.
(void)markers.open(c.session).has_value();
(void)(c.probe ? markers.open_at(c.session, c.probe->x, c.probe->y)
: markers.open(c.session))
.has_value();
c.state = State::established;
c.welcome.server_tick = facts.simulated_tick;
send(c, c.welcome);
@@ -333,6 +395,95 @@ struct RegionSessions::Impl {
send_datagram(c, receipt);
}
[[nodiscard]] bool handoffs_enabled() const {
return config.probes != nullptr && config.handoff_key != nullptr &&
!config.neighbours.empty() && own_area.has_value();
}
// Markers that left this region's area: into a neighbour's area they are handed over,
// anywhere else they are put back at the edge.
void crossings(std::uint64_t now_seconds, Clock::time_point now) {
if (!own_area)
return;
const auto inside = [](const network::motion::Area& a, std::int64_t x, std::int64_t y) {
return x >= a.min_x && x <= a.max_x && y >= a.min_y && y <= a.max_y;
};
for (auto& c : connections) {
if (c.state != State::established)
continue;
const auto* m = markers.of(c.session);
if (m == nullptr || m->frozen || inside(*own_area, m->state.x, m->state.y))
continue;
const auto n = std::ranges::find_if(config.neighbours, [&](const Neighbour& x) {
return inside(x.area, m->state.x, m->state.y);
});
if (n == config.neighbours.end() || !handoffs_enabled() || !c.probe) {
const auto x = std::clamp(m->state.x, own_area->min_x, own_area->max_x);
const auto y = std::clamp(m->state.y, own_area->min_y, own_area->max_y);
markers.place(c.session, x, y); // Back at the edge (`m` is stale after this).
continue;
}
hand_off(c, *n, now_seconds, now);
}
}
// The source side of a crossing: freeze, prepare durably, release control, offer the
// ticket, close. On any failure the probe stays here, at the edge.
void hand_off(Connection& c, const Neighbour& n, std::uint64_t now_seconds,
Clock::time_point now) {
if (!c.probe || !own_area) // Checked by the caller (handoffs_enabled, c.probe).
return;
const auto generation = c.probe->generation;
const auto z = own_area->z;
markers.freeze(c.session);
const auto* m = markers.of(c.session);
network::TransferId transfer{};
if (m == nullptr || !network::random_bytes(transfer))
return close(c, SessionCloseReason::shutdown, now);
const auto x = m->state.x, y = m->state.y;
if (const auto prepared = config.probes->prepare(c.account_id, generation, transfer,
n.address.region, x, y, z);
!prepared)
return close(c, SessionCloseReason::shutdown, now);
c.probe.reset(); // Frozen under the transfer: never saved here again.
network::TicketClaims claims;
claims.transfer = transfer;
claims.account = c.account_id;
claims.world = config.control.world;
claims.source = config.control.region;
claims.destination = n.address.region;
claims.expires_at = static_cast<std::uint32_t>(now_seconds + config.ticket_lifetime);
claims.session_expires_at = static_cast<std::uint32_t>(c.welcome.expires_at);
const auto ticket = config.handoff_key->sign(claims);
// Control goes first so the destination need not wait it out.
if (config.ledger && c.generation != 0)
config.ledger->release(c.account_id, c.generation);
c.generation = 0;
outgoing.push_back({transfer, now_seconds + config.ticket_lifetime + 5});
if (ticket)
send(c, network::HandoffOffer{transfer, n.address, *ticket});
close(c, SessionCloseReason::handed_off, now);
}
// Transfers whose client never arrived are aborted (the probe resumes here); committed
// ones are forgotten. Undecided calls retry next poll.
void expire_outgoing(std::uint64_t now_seconds) {
std::erase_if(outgoing, [&](const Outgoing& o) {
if (now_seconds < o.deadline)
return false;
return config.probes->abort(o.transfer).has_value();
});
}
void refresh_ghosts(Clock::time_point now) {
if (now < next_ghosts)
return;
next_ghosts = now + config.ghost_poll;
const auto incoming = config.probes->incoming();
if (!incoming)
return; // Keep the current ghosts until the next answer.
std::vector<network::motion::Ghost> ghosts;
for (const auto& p : *incoming)
ghosts.push_back({p.transfer, p.x, p.y});
markers.set_ghosts(ghosts);
}
void deliver(const std::vector<network::AccountOutcome>& outcomes) {
for (const auto& o : outcomes)
if (const auto session = intents.session_of(o.account))
@@ -394,6 +545,8 @@ std::expected<std::unique_ptr<RegionSessions>, RegionSessionsError>
RegionSessions::create(RegionSessionsConfig config) {
// Connections beyond sessions are headroom for logins and closes still flushing.
if (config.issuer_keys.empty() || (config.handoff != nullptr && config.ledger == nullptr) ||
(config.handoff_key != nullptr && config.probes == nullptr) ||
config.ticket_lifetime == 0 || config.ticket_lifetime > network::max_ticket_lifetime ||
config.transport.limits.max_connections < config.control.max_sessions ||
config.hello_deadline.count() <= 0 || config.close_flush.count() <= 0 ||
config.renew_interval.count() <= 0 || config.safety_margin.count() <= 0)
@@ -451,8 +604,14 @@ void RegionSessions::poll(const HostFacts& facts, std::uint64_t now_seconds,
m.markers.clear(); // The new epoch may restart at an earlier tick.
}
// Markers reach the simulated tick before this poll's intentions are admitted.
if (facts.ready)
if (facts.ready) {
m.markers.advance(facts.simulated_tick);
m.crossings(now_seconds, now);
if (m.config.probes) {
m.expire_outgoing(now_seconds);
m.refresh_ghosts(now);
}
}
for (auto& e : m.endpoint->poll()) {
using network::quic::EventKind;
if (e.kind == EventKind::connected) {
@@ -471,6 +630,7 @@ void RegionSessions::poll(const HostFacts& facts, std::uint64_t now_seconds,
if (it->state == Impl::State::established) {
m.intents.close(it->session);
m.replication.close(it->session);
m.save_probe(*it);
m.markers.close(it->session);
}
if (it->state == Impl::State::pending) {
@@ -563,8 +723,18 @@ network::ReplicationStats RegionSessions::replication_stats() const {
bool RegionSessions::set_motion_area(const network::motion::Area& area, std::int64_t spawn_x,
std::int64_t spawn_y) {
auto& m = *impl_;
if (!m.markers.set_area(area, spawn_x, spawn_y))
auto movement = area;
for (const auto& n : m.config.neighbours) {
movement.min_x = std::min(movement.min_x, n.area.min_x);
movement.min_y = std::min(movement.min_y, n.area.min_y);
movement.max_x = std::max(movement.max_x, n.area.max_x);
movement.max_y = std::max(movement.max_y, n.area.max_y);
}
if (!network::motion::valid(area) || !m.markers.set_area(movement, spawn_x, spawn_y))
return false;
m.own_area = area;
m.spawn_x = std::clamp(spawn_x, area.min_x, area.max_x);
m.spawn_y = std::clamp(spawn_y, area.min_y, area.max_y);
for (const auto& c : m.connections)
if (c.state == Impl::State::established && !m.markers.of(c.session))
(void)m.markers.open(c.session).has_value();