Add M10.8d two-process handoff fault matrix and recovery fixes
Foundation / checks (deploy) (push) Successful in 3m20s
Foundation / checks (ecs) (push) Successful in 1m57s
Foundation / checks (client) (push) Failing after 10m4s
Foundation / checks (network) (push) Successful in 13m3s
Foundation / checks (tsan) (push) Successful in 7m53s
Foundation / checks (checks) (push) Successful in 32m18s
Foundation / checks (postgres) (push) Successful in 43m26s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
Foundation / checks (deploy) (push) Successful in 3m20s
Foundation / checks (ecs) (push) Successful in 1m57s
Foundation / checks (client) (push) Failing after 10m4s
Foundation / checks (network) (push) Successful in 13m3s
Foundation / checks (tsan) (push) Successful in 7m53s
Foundation / checks (checks) (push) Successful in 32m18s
Foundation / checks (postgres) (push) Successful in 43m26s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
ADR-0096. Two real evo-server processes on PostgreSQL (regions 7/161 and 7/162, adjacent areas) run crossings and crossings back, SIGKILL of the source after prepare and of the destination after stage and after commit, duplicate and stale ticket retries, a client that never arrives and a destination that is down. After every scenario SQL checks one owner per probe, the conserved inventory and frozen-iff-open-transfer. The matrix found two recovery gaps, now fixed: - a source restarted after a prepare left the probe frozen for good (its abort deadline was in memory): schema v14 overdue_probe_transfers and a session-layer sweep abort open outgoing transfers past the ticket lifetime plus 10 s by the database clock; - a ticket replayed at a restarted destination after the probe moved on took control back: admission of a committed transfer now requires the record as that commit left it. M10.8 is complete. Refreshes the dev session window. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -53,6 +53,17 @@ PostgresHandoffLedger::admit(const network::TicketClaims& ticket, std::uint64_t
|
||||
return fail(committed.error());
|
||||
if (committed->state != persistence::TransferState::committed)
|
||||
return std::unexpected(HandoffLedgerError::refused); // Aborted meanwhile.
|
||||
// M10.8d: a retried ticket (e.g. after this region restarted and forgot it was used) is
|
||||
// honoured only while the probe is still as that commit left it: owned here at the next
|
||||
// generation and not frozen. Once it moved on, the ticket must not take control back.
|
||||
const auto record =
|
||||
persistence::open_probe(*db, key_, *lease, ticket.account.bytes, committed->x, committed->y,
|
||||
committed->z, PostgresProbeLedger::inventory, timeout_);
|
||||
if (!record)
|
||||
return fail(record.error());
|
||||
if (record->region != key_.region || record->generation != committed->generation + 1 ||
|
||||
record->transfer)
|
||||
return std::unexpected(HandoffLedgerError::refused);
|
||||
const auto control = persistence::handoff_account_control(
|
||||
*db, key_, *lease, ticket.account.bytes, ticket.transfer, session, duration_, timeout_);
|
||||
if (!control) {
|
||||
@@ -169,4 +180,21 @@ std::expected<std::vector<IncomingProbe>, ProbeLedgerError> PostgresProbeLedger:
|
||||
out.push_back({t.transfer, t.x, t.y});
|
||||
return out;
|
||||
}
|
||||
|
||||
std::expected<std::vector<network::TransferId>, ProbeLedgerError>
|
||||
PostgresProbeLedger::overdue(std::chrono::seconds age) {
|
||||
const auto lease = lease_();
|
||||
if (!lease)
|
||||
return std::unexpected(ProbeLedgerError::fenced);
|
||||
auto* db = db_->get();
|
||||
if (!db)
|
||||
return std::unexpected(ProbeLedgerError::unavailable);
|
||||
auto r = persistence::overdue_probe_transfers(*db, key_, *lease, age, timeout_);
|
||||
if (!r) {
|
||||
if (r.error().kind == ProbeStoreError::store || r.error().kind == ProbeStoreError::unknown)
|
||||
db_->broken();
|
||||
return std::unexpected(probe_error(r.error()));
|
||||
}
|
||||
return std::vector<network::TransferId>(r->begin(), r->end());
|
||||
}
|
||||
} // namespace evo::server
|
||||
|
||||
@@ -106,7 +106,7 @@ struct RegionSessions::Impl {
|
||||
std::optional<network::motion::Area> own_area; // This region's part of the marker area.
|
||||
std::int64_t spawn_x{}, spawn_y{}; // Where new probes start.
|
||||
std::vector<Outgoing> outgoing; // Handed off, not yet known committed.
|
||||
Clock::time_point next_ghosts{};
|
||||
Clock::time_point next_ghosts{}, next_overdue{};
|
||||
std::optional<network::SessionInterest> default_interest;
|
||||
HostFacts facts;
|
||||
std::vector<Connection> connections;
|
||||
@@ -471,6 +471,20 @@ struct RegionSessions::Impl {
|
||||
return config.probes->abort(o.transfer).has_value();
|
||||
});
|
||||
}
|
||||
// Recovery (M10.8d): the in-memory deadlines die with the process, so outgoing transfers
|
||||
// still open well after any ticket for them expired are aborted from the store's list.
|
||||
// Abort is serialized with the destination's commit: a committed one stays committed.
|
||||
void abort_overdue(Clock::time_point now) {
|
||||
if (now < next_overdue)
|
||||
return;
|
||||
next_overdue = now + config.overdue_poll;
|
||||
const auto overdue =
|
||||
config.probes->overdue(std::chrono::seconds{config.ticket_lifetime + 10});
|
||||
if (!overdue)
|
||||
return; // Retried next poll.
|
||||
for (const auto& t : *overdue)
|
||||
(void)config.probes->abort(t).has_value();
|
||||
}
|
||||
void refresh_ghosts(Clock::time_point now) {
|
||||
if (now < next_ghosts)
|
||||
return;
|
||||
@@ -549,7 +563,8 @@ RegionSessions::create(RegionSessionsConfig config) {
|
||||
config.ticket_lifetime == 0 || config.ticket_lifetime > network::max_ticket_lifetime ||
|
||||
config.transport.limits.max_connections < config.control.max_sessions ||
|
||||
config.hello_deadline.count() <= 0 || config.close_flush.count() <= 0 ||
|
||||
config.renew_interval.count() <= 0 || config.safety_margin.count() <= 0)
|
||||
config.renew_interval.count() <= 0 || config.safety_margin.count() <= 0 ||
|
||||
config.overdue_poll.count() <= 0)
|
||||
return std::unexpected(RegionSessionsError::configuration);
|
||||
auto control = network::SessionControl::create(config.control);
|
||||
if (!control)
|
||||
@@ -609,6 +624,7 @@ void RegionSessions::poll(const HostFacts& facts, std::uint64_t now_seconds,
|
||||
m.crossings(now_seconds, now);
|
||||
if (m.config.probes) {
|
||||
m.expire_outgoing(now_seconds);
|
||||
m.abort_overdue(now);
|
||||
m.refresh_ghosts(now);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user