Wire M10.8c-1 region handoff: protocol v4, durable probes, crossing, ghosts
Foundation / checks (deploy) (push) Successful in 3m29s
Foundation / checks (ecs) (push) Successful in 2m0s
Foundation / checks (client) (push) Successful in 12m51s
Foundation / checks (network) (push) Successful in 13m0s
Foundation / checks (tsan) (push) Successful in 7m50s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 32m4s
Foundation / checks (postgres) (push) Canceled after 19m13s
Foundation / checks (deploy) (push) Successful in 3m29s
Foundation / checks (ecs) (push) Successful in 2m0s
Foundation / checks (client) (push) Successful in 12m51s
Foundation / checks (network) (push) Successful in 13m0s
Foundation / checks (tsan) (push) Successful in 7m50s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 32m4s
Foundation / checks (postgres) (push) Canceled after 19m13s
ADR-0096. Session protocol v4 (v3 is deployed and frozen) adds HandoffOffer, SessionRedirect, close reasons handed_off and moved, and ghost marker records. Sessions use the durable probe record: logins open it where it was saved (redirected when it lives elsewhere), and session ends save it. A marker crossing into a neighbour's area is frozen, prepared, its control released and a signed ticket offered; uncommitted transfers are aborted after the ticket's lifetime. The destination shows arriving probes as ghosts (schema v13 incoming_probe_transfers) and opens the marker where it crossed. Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -27,7 +27,7 @@ HandoffLedgerError ledger_error(const persistence::ProbeStoreFailure& f) {
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::expected<network::ControlLease, HandoffLedgerError>
|
||||
std::expected<network::HandoffAdmission, HandoffLedgerError>
|
||||
PostgresHandoffLedger::admit(const network::TicketClaims& ticket, std::uint64_t session) {
|
||||
const auto lease = lease_();
|
||||
if (!lease)
|
||||
@@ -65,6 +65,108 @@ PostgresHandoffLedger::admit(const network::TicketClaims& ticket, std::uint64_t
|
||||
return std::unexpected(lease_fenced(control.error().cause) ? HandoffLedgerError::fenced
|
||||
: HandoffLedgerError::refused);
|
||||
}
|
||||
return network::ControlLease{control->generation, control->wait, control->granted};
|
||||
return network::HandoffAdmission{
|
||||
network::ControlLease{control->generation, control->wait, control->granted},
|
||||
committed->generation + 1, committed->x, committed->y};
|
||||
}
|
||||
|
||||
namespace {
|
||||
ProbeLedgerError probe_error(const persistence::ProbeStoreFailure& f) {
|
||||
return f.kind == ProbeStoreError::fenced && lease_fenced(f.cause)
|
||||
? ProbeLedgerError::fenced
|
||||
: ProbeLedgerError::unavailable;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::expected<ProbeHome, ProbeLedgerError>
|
||||
PostgresProbeLedger::open(const network::AccountId& account, std::int64_t x, std::int64_t y,
|
||||
std::int64_t z) {
|
||||
const auto lease = lease_();
|
||||
if (!lease)
|
||||
return std::unexpected(ProbeLedgerError::fenced);
|
||||
auto* db = db_->get();
|
||||
if (!db)
|
||||
return std::unexpected(ProbeLedgerError::unavailable);
|
||||
const auto r =
|
||||
persistence::open_probe(*db, key_, *lease, account.bytes, x, y, z, inventory, timeout_);
|
||||
if (!r) {
|
||||
if (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown)
|
||||
db_->broken();
|
||||
return std::unexpected(probe_error(r.error()));
|
||||
}
|
||||
return ProbeHome{r->region, r->generation, r->x, r->y, r->transfer.has_value()};
|
||||
}
|
||||
|
||||
void PostgresProbeLedger::save(const network::AccountId& account, std::uint64_t generation,
|
||||
std::int64_t x, std::int64_t y, std::int64_t z) {
|
||||
const auto lease = lease_();
|
||||
if (!lease)
|
||||
return;
|
||||
auto* db = db_->get();
|
||||
if (!db)
|
||||
return;
|
||||
if (const auto r = persistence::save_probe(*db, key_, *lease, account.bytes, generation, x, y,
|
||||
z, timeout_);
|
||||
!r &&
|
||||
(r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown))
|
||||
db_->broken();
|
||||
}
|
||||
|
||||
std::expected<void, ProbeLedgerError>
|
||||
PostgresProbeLedger::prepare(const network::AccountId& account, std::uint64_t generation,
|
||||
const network::TransferId& transfer, std::uint64_t destination,
|
||||
std::int64_t x, std::int64_t y, std::int64_t z) {
|
||||
const auto lease = lease_();
|
||||
if (!lease)
|
||||
return std::unexpected(ProbeLedgerError::fenced);
|
||||
auto* db = db_->get();
|
||||
if (!db)
|
||||
return std::unexpected(ProbeLedgerError::unavailable);
|
||||
const auto r = persistence::prepare_probe_transfer(*db, key_, *lease, account.bytes, generation,
|
||||
transfer, destination, x, y, z, timeout_);
|
||||
if (!r) {
|
||||
if (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown)
|
||||
db_->broken();
|
||||
return std::unexpected(probe_error(r.error()));
|
||||
}
|
||||
if (r->state != persistence::TransferState::prepared)
|
||||
return std::unexpected(ProbeLedgerError::unavailable); // A retry found it decided.
|
||||
return {};
|
||||
}
|
||||
|
||||
std::expected<bool, ProbeLedgerError>
|
||||
PostgresProbeLedger::abort(const network::TransferId& transfer) {
|
||||
const auto lease = lease_();
|
||||
if (!lease)
|
||||
return std::unexpected(ProbeLedgerError::fenced);
|
||||
auto* db = db_->get();
|
||||
if (!db)
|
||||
return std::unexpected(ProbeLedgerError::unavailable);
|
||||
const auto r = persistence::abort_probe_transfer(*db, key_, *lease, transfer, timeout_);
|
||||
if (!r) {
|
||||
if (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown)
|
||||
db_->broken();
|
||||
return std::unexpected(probe_error(r.error()));
|
||||
}
|
||||
return r->state == persistence::TransferState::aborted;
|
||||
}
|
||||
|
||||
std::expected<std::vector<IncomingProbe>, ProbeLedgerError> PostgresProbeLedger::incoming() {
|
||||
const auto lease = lease_();
|
||||
if (!lease)
|
||||
return std::unexpected(ProbeLedgerError::fenced);
|
||||
auto* db = db_->get();
|
||||
if (!db)
|
||||
return std::unexpected(ProbeLedgerError::unavailable);
|
||||
const auto r = persistence::incoming_probe_transfers(*db, key_, *lease, timeout_);
|
||||
if (!r) {
|
||||
if (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown)
|
||||
db_->broken();
|
||||
return std::unexpected(probe_error(r.error()));
|
||||
}
|
||||
std::vector<IncomingProbe> out;
|
||||
for (const auto& t : *r)
|
||||
out.push_back({t.transfer, t.x, t.y});
|
||||
return out;
|
||||
}
|
||||
} // namespace evo::server
|
||||
|
||||
@@ -80,6 +80,12 @@ struct RegionSessions::Impl {
|
||||
std::uint64_t generation{};
|
||||
Clock::time_point control_deadline, next_renew;
|
||||
network::SessionWelcome welcome;
|
||||
// Durable probe (probe records only): its ownership generation and position.
|
||||
std::optional<ProbeHome> probe;
|
||||
};
|
||||
struct Outgoing {
|
||||
network::TransferId transfer{};
|
||||
std::uint64_t deadline{}; // Wall seconds: abort unless committed by then.
|
||||
};
|
||||
|
||||
Impl(RegionSessionsConfig c, std::unique_ptr<network::quic::Endpoint> e,
|
||||
@@ -96,7 +102,11 @@ struct RegionSessions::Impl {
|
||||
network::SessionControl control;
|
||||
network::IntentAdmission intents;
|
||||
network::ReplicationServer replication;
|
||||
network::motion::ProbeMarkers markers; // Ephemeral probe markers (ADR-0094).
|
||||
network::motion::ProbeMarkers markers; // Probe markers (ADR-0094, durable: ADR-0096).
|
||||
std::optional<network::motion::Area> own_area; // This region's part of the marker area.
|
||||
std::int64_t spawn_x{}, spawn_y{}; // Where new probes start.
|
||||
std::vector<Outgoing> outgoing; // Handed off, not yet known committed.
|
||||
Clock::time_point next_ghosts{};
|
||||
std::optional<network::SessionInterest> default_interest;
|
||||
HostFacts facts;
|
||||
std::vector<Connection> connections;
|
||||
@@ -140,8 +150,10 @@ struct RegionSessions::Impl {
|
||||
if (c.state == State::established) {
|
||||
intents.close(c.session);
|
||||
replication.close(c.session);
|
||||
save_probe(c);
|
||||
markers.close(c.session);
|
||||
}
|
||||
c.probe.reset();
|
||||
if (c.session != 0) // May already be released (superseded, expired, epoch).
|
||||
(void)control.close(c.session).has_value();
|
||||
// A superseded session's durable control already moved to its successor.
|
||||
@@ -153,6 +165,49 @@ struct RegionSessions::Impl {
|
||||
c.deadline = now + config.close_flush;
|
||||
}
|
||||
|
||||
// Best effort, at session end: the durable probe's position (a frozen or moved probe is
|
||||
// fenced by the store and stays as it was).
|
||||
void save_probe(const Connection& c) {
|
||||
if (!config.probes || !c.probe)
|
||||
return;
|
||||
if (const auto* m = markers.of(c.session); m && !m->frozen && own_area)
|
||||
config.probes->save(c.account_id, c.probe->generation, m->state.x, m->state.y,
|
||||
own_area->z);
|
||||
}
|
||||
// After the durable control claim of a grant login: where the account's probe lives.
|
||||
// False when the session must not open here (the connection is then closed).
|
||||
bool place_probe(Connection& c, const network::AccountId& account,
|
||||
const std::optional<network::ControlLease>& lease, Clock::time_point now) {
|
||||
if (!config.probes || !own_area)
|
||||
return true;
|
||||
const auto home = config.probes->open(account, spawn_x, spawn_y, own_area->z);
|
||||
const auto refuse = [&](SessionCloseReason why) {
|
||||
if (lease)
|
||||
config.ledger->release(account, lease->generation);
|
||||
close(c, why, now);
|
||||
return false;
|
||||
};
|
||||
if (!home)
|
||||
return refuse(home.error() == ProbeLedgerError::fenced ? SessionCloseReason::shutdown
|
||||
: SessionCloseReason::capacity);
|
||||
if (home->owner != config.control.region) {
|
||||
const auto n = std::ranges::find_if(config.neighbours, [&](const Neighbour& x) {
|
||||
return x.address.region == home->owner;
|
||||
});
|
||||
if (n != config.neighbours.end())
|
||||
send(c, network::SessionRedirect{n->address});
|
||||
return refuse(SessionCloseReason::moved);
|
||||
}
|
||||
if (home->frozen) // A transfer is open: decided by its commit or abort, retry later.
|
||||
return refuse(SessionCloseReason::capacity);
|
||||
c.probe = *home;
|
||||
// After an aborted handoff the probe was saved where it crossed: back inside, so the
|
||||
// new session does not cross again at once.
|
||||
c.probe->x = std::clamp(c.probe->x, own_area->min_x, own_area->max_x);
|
||||
c.probe->y = std::clamp(c.probe->y, own_area->min_y, own_area->max_y);
|
||||
return true;
|
||||
}
|
||||
|
||||
void hello(Connection& c, const network::SessionHello& m, std::uint64_t now_seconds,
|
||||
Clock::time_point now) {
|
||||
if (m.required_features != 0)
|
||||
@@ -178,6 +233,8 @@ struct RegionSessions::Impl {
|
||||
return close(c, reason_for(claimed.error()), now);
|
||||
lease = *claimed;
|
||||
}
|
||||
if (!place_probe(c, claims.account, lease, now))
|
||||
return;
|
||||
admitted(c, control.open(*grant, now_seconds), lease, promised, claims.account, now_seconds,
|
||||
now);
|
||||
}
|
||||
@@ -201,7 +258,10 @@ struct RegionSessions::Impl {
|
||||
auto moved = config.handoff->admit(claims, promised);
|
||||
if (!moved)
|
||||
return close(c, reason_for(moved.error()), now);
|
||||
lease = *moved;
|
||||
lease = moved->lease;
|
||||
if (config.probes)
|
||||
c.probe =
|
||||
ProbeHome{config.control.region, moved->generation, moved->x, moved->y, false};
|
||||
} else if (config.ledger) {
|
||||
return close(c, SessionCloseReason::capacity, now); // Misconfigured: fail closed.
|
||||
}
|
||||
@@ -257,7 +317,9 @@ struct RegionSessions::Impl {
|
||||
if (default_interest)
|
||||
(void)replication.set_interest(c.session, *default_interest).has_value();
|
||||
if (markers.area()) // Full or refused: the session simply has no marker.
|
||||
(void)markers.open(c.session).has_value();
|
||||
(void)(c.probe ? markers.open_at(c.session, c.probe->x, c.probe->y)
|
||||
: markers.open(c.session))
|
||||
.has_value();
|
||||
c.state = State::established;
|
||||
c.welcome.server_tick = facts.simulated_tick;
|
||||
send(c, c.welcome);
|
||||
@@ -333,6 +395,95 @@ struct RegionSessions::Impl {
|
||||
send_datagram(c, receipt);
|
||||
}
|
||||
|
||||
[[nodiscard]] bool handoffs_enabled() const {
|
||||
return config.probes != nullptr && config.handoff_key != nullptr &&
|
||||
!config.neighbours.empty() && own_area.has_value();
|
||||
}
|
||||
// Markers that left this region's area: into a neighbour's area they are handed over,
|
||||
// anywhere else they are put back at the edge.
|
||||
void crossings(std::uint64_t now_seconds, Clock::time_point now) {
|
||||
if (!own_area)
|
||||
return;
|
||||
const auto inside = [](const network::motion::Area& a, std::int64_t x, std::int64_t y) {
|
||||
return x >= a.min_x && x <= a.max_x && y >= a.min_y && y <= a.max_y;
|
||||
};
|
||||
for (auto& c : connections) {
|
||||
if (c.state != State::established)
|
||||
continue;
|
||||
const auto* m = markers.of(c.session);
|
||||
if (m == nullptr || m->frozen || inside(*own_area, m->state.x, m->state.y))
|
||||
continue;
|
||||
const auto n = std::ranges::find_if(config.neighbours, [&](const Neighbour& x) {
|
||||
return inside(x.area, m->state.x, m->state.y);
|
||||
});
|
||||
if (n == config.neighbours.end() || !handoffs_enabled() || !c.probe) {
|
||||
const auto x = std::clamp(m->state.x, own_area->min_x, own_area->max_x);
|
||||
const auto y = std::clamp(m->state.y, own_area->min_y, own_area->max_y);
|
||||
markers.place(c.session, x, y); // Back at the edge (`m` is stale after this).
|
||||
continue;
|
||||
}
|
||||
hand_off(c, *n, now_seconds, now);
|
||||
}
|
||||
}
|
||||
// The source side of a crossing: freeze, prepare durably, release control, offer the
|
||||
// ticket, close. On any failure the probe stays here, at the edge.
|
||||
void hand_off(Connection& c, const Neighbour& n, std::uint64_t now_seconds,
|
||||
Clock::time_point now) {
|
||||
if (!c.probe || !own_area) // Checked by the caller (handoffs_enabled, c.probe).
|
||||
return;
|
||||
const auto generation = c.probe->generation;
|
||||
const auto z = own_area->z;
|
||||
markers.freeze(c.session);
|
||||
const auto* m = markers.of(c.session);
|
||||
network::TransferId transfer{};
|
||||
if (m == nullptr || !network::random_bytes(transfer))
|
||||
return close(c, SessionCloseReason::shutdown, now);
|
||||
const auto x = m->state.x, y = m->state.y;
|
||||
if (const auto prepared = config.probes->prepare(c.account_id, generation, transfer,
|
||||
n.address.region, x, y, z);
|
||||
!prepared)
|
||||
return close(c, SessionCloseReason::shutdown, now);
|
||||
c.probe.reset(); // Frozen under the transfer: never saved here again.
|
||||
network::TicketClaims claims;
|
||||
claims.transfer = transfer;
|
||||
claims.account = c.account_id;
|
||||
claims.world = config.control.world;
|
||||
claims.source = config.control.region;
|
||||
claims.destination = n.address.region;
|
||||
claims.expires_at = static_cast<std::uint32_t>(now_seconds + config.ticket_lifetime);
|
||||
claims.session_expires_at = static_cast<std::uint32_t>(c.welcome.expires_at);
|
||||
const auto ticket = config.handoff_key->sign(claims);
|
||||
// Control goes first so the destination need not wait it out.
|
||||
if (config.ledger && c.generation != 0)
|
||||
config.ledger->release(c.account_id, c.generation);
|
||||
c.generation = 0;
|
||||
outgoing.push_back({transfer, now_seconds + config.ticket_lifetime + 5});
|
||||
if (ticket)
|
||||
send(c, network::HandoffOffer{transfer, n.address, *ticket});
|
||||
close(c, SessionCloseReason::handed_off, now);
|
||||
}
|
||||
// Transfers whose client never arrived are aborted (the probe resumes here); committed
|
||||
// ones are forgotten. Undecided calls retry next poll.
|
||||
void expire_outgoing(std::uint64_t now_seconds) {
|
||||
std::erase_if(outgoing, [&](const Outgoing& o) {
|
||||
if (now_seconds < o.deadline)
|
||||
return false;
|
||||
return config.probes->abort(o.transfer).has_value();
|
||||
});
|
||||
}
|
||||
void refresh_ghosts(Clock::time_point now) {
|
||||
if (now < next_ghosts)
|
||||
return;
|
||||
next_ghosts = now + config.ghost_poll;
|
||||
const auto incoming = config.probes->incoming();
|
||||
if (!incoming)
|
||||
return; // Keep the current ghosts until the next answer.
|
||||
std::vector<network::motion::Ghost> ghosts;
|
||||
for (const auto& p : *incoming)
|
||||
ghosts.push_back({p.transfer, p.x, p.y});
|
||||
markers.set_ghosts(ghosts);
|
||||
}
|
||||
|
||||
void deliver(const std::vector<network::AccountOutcome>& outcomes) {
|
||||
for (const auto& o : outcomes)
|
||||
if (const auto session = intents.session_of(o.account))
|
||||
@@ -394,6 +545,8 @@ std::expected<std::unique_ptr<RegionSessions>, RegionSessionsError>
|
||||
RegionSessions::create(RegionSessionsConfig config) {
|
||||
// Connections beyond sessions are headroom for logins and closes still flushing.
|
||||
if (config.issuer_keys.empty() || (config.handoff != nullptr && config.ledger == nullptr) ||
|
||||
(config.handoff_key != nullptr && config.probes == nullptr) ||
|
||||
config.ticket_lifetime == 0 || config.ticket_lifetime > network::max_ticket_lifetime ||
|
||||
config.transport.limits.max_connections < config.control.max_sessions ||
|
||||
config.hello_deadline.count() <= 0 || config.close_flush.count() <= 0 ||
|
||||
config.renew_interval.count() <= 0 || config.safety_margin.count() <= 0)
|
||||
@@ -451,8 +604,14 @@ void RegionSessions::poll(const HostFacts& facts, std::uint64_t now_seconds,
|
||||
m.markers.clear(); // The new epoch may restart at an earlier tick.
|
||||
}
|
||||
// Markers reach the simulated tick before this poll's intentions are admitted.
|
||||
if (facts.ready)
|
||||
if (facts.ready) {
|
||||
m.markers.advance(facts.simulated_tick);
|
||||
m.crossings(now_seconds, now);
|
||||
if (m.config.probes) {
|
||||
m.expire_outgoing(now_seconds);
|
||||
m.refresh_ghosts(now);
|
||||
}
|
||||
}
|
||||
for (auto& e : m.endpoint->poll()) {
|
||||
using network::quic::EventKind;
|
||||
if (e.kind == EventKind::connected) {
|
||||
@@ -471,6 +630,7 @@ void RegionSessions::poll(const HostFacts& facts, std::uint64_t now_seconds,
|
||||
if (it->state == Impl::State::established) {
|
||||
m.intents.close(it->session);
|
||||
m.replication.close(it->session);
|
||||
m.save_probe(*it);
|
||||
m.markers.close(it->session);
|
||||
}
|
||||
if (it->state == Impl::State::pending) {
|
||||
@@ -563,8 +723,18 @@ network::ReplicationStats RegionSessions::replication_stats() const {
|
||||
bool RegionSessions::set_motion_area(const network::motion::Area& area, std::int64_t spawn_x,
|
||||
std::int64_t spawn_y) {
|
||||
auto& m = *impl_;
|
||||
if (!m.markers.set_area(area, spawn_x, spawn_y))
|
||||
auto movement = area;
|
||||
for (const auto& n : m.config.neighbours) {
|
||||
movement.min_x = std::min(movement.min_x, n.area.min_x);
|
||||
movement.min_y = std::min(movement.min_y, n.area.min_y);
|
||||
movement.max_x = std::max(movement.max_x, n.area.max_x);
|
||||
movement.max_y = std::max(movement.max_y, n.area.max_y);
|
||||
}
|
||||
if (!network::motion::valid(area) || !m.markers.set_area(movement, spawn_x, spawn_y))
|
||||
return false;
|
||||
m.own_area = area;
|
||||
m.spawn_x = std::clamp(spawn_x, area.min_x, area.max_x);
|
||||
m.spawn_y = std::clamp(spawn_y, area.min_y, area.max_y);
|
||||
for (const auto& c : m.connections)
|
||||
if (c.state == Impl::State::established && !m.markers.of(c.session))
|
||||
(void)m.markers.open(c.session).has_value();
|
||||
|
||||
Reference in New Issue
Block a user