From 07d12a4d82f1d60d4ce362d76eef5d65f95496ff Mon Sep 17 00:00:00 2001 From: dingo Date: Thu, 8 Oct 2026 21:35:43 -0400 Subject: [PATCH] M10.9d: batch durable action lookups (one per poll, one per commit) The fourth dev drill showed a synchronous PostgreSQL round trip per action on the simulation thread: at admission (uniqueness) and again at commit (first-record rule). About 12 lookups landed on each commit tick (commit ticks 58-70 ms, session p99 30-39 ms). - ActionLedger::find_all; persistence::find_region_actions applies the schema v11 find_region_action to many pairs in one statement (no migration). - IntentAdmission::admit/verify: actions only the ledger can refuse wait for one verify per poll; receipts still go out in the same poll. committed() asks once per commit. - Tests: deferred_actions (network_control), find_region_actions and find_all (Postgres). - ADR-0097 fourth run, ADR-0092 amendment, VALIDATION; refresh the dev session window. Co-Authored-By: Claude Opus 5.5 --- .../region/dev-session-route/listener.yaml | 2 +- ...092-m10-intentions-and-durable-outcomes.md | 17 ++ .../0097-m10-load-and-abuse-qualification.md | 31 ++++ docs/engineering/VALIDATION.md | 24 +++ include/evo/network/intent_admission.hpp | 31 +++- .../evo/persistence/region_lease_store.hpp | 9 + include/evo/server/postgres_action_ledger.hpp | 3 + src/network/protocol/intent_admission.cpp | 169 ++++++++++++++---- src/persistence/region_lease_store.cpp | 44 +++++ src/server/session/postgres_action_ledger.cpp | 34 ++++ src/server/session/region_sessions.cpp | 25 ++- tests/network_control.cpp | 68 ++++++- tests/region_action_tests.cpp | 18 ++ 13 files changed, 430 insertions(+), 45 deletions(-) diff --git a/deploy/kubernetes/region/dev-session-route/listener.yaml b/deploy/kubernetes/region/dev-session-route/listener.yaml index cf462f7..6ef363a 100644 --- a/deploy/kubernetes/region/dev-session-route/listener.yaml +++ b/deploy/kubernetes/region/dev-session-route/listener.yaml @@ -12,7 +12,7 @@ spec: env: - {name: REGION_SESSION, value: "dev"} - {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"} - - {name: REGION_SESSION_WINDOW, value: "dev-20261008T215407-71fa:1791510247"} + - {name: REGION_SESSION_WINDOW, value: "dev-20261009T012919-51fc:1791523159"} - {name: REGION_M10_MAX_GRANT_LIFETIME, value: "14400"} ports: # QUIC (ADR-0090): UDP only; nothing listens on TCP 7400. diff --git a/docs/adr/0092-m10-intentions-and-durable-outcomes.md b/docs/adr/0092-m10-intentions-and-durable-outcomes.md index 4b065ef..0c15a14 100644 --- a/docs/adr/0092-m10-intentions-and-durable-outcomes.md +++ b/docs/adr/0092-m10-intentions-and-durable-outcomes.md @@ -129,3 +129,20 @@ It is cumulative, so no admitted intention is ever missing from a later watermar learns "simulated" up to 3 ticks (150 ms) later. Receipts are unchanged: one per intention. Reason: on the reference worker, datagram sends dominated the session layer, and the per-tick watermark was a quarter of them. + +## Amendment (2026-10-09, M10.9d, ADR-0097): batched durable lookups +The durable checks are unchanged; only how many store round trips they cost changes. +- **Admission.** In evo-server, an action that passes every in-memory check waits for the + poll's single `verify`, which asks the ledger about all waiting actions at once and + sends their receipts in the same poll. + - While it waits, the action counts toward the unresolved limit. + - A repeat of it is a `duplicate`. + - It is never carried. + - A store failure, lost readiness or a new epoch before `verify` refuses it as + admission would (`not_ready`, `stale_epoch`). +- **Commit.** A commit's outcomes come from one request for all of its actions. The + first-record rule still decides each one. +- **API.** `IntentAdmission::action` keeps the immediate single lookup for callers that + want it. +- **Reason:** one round trip per action on the simulation thread dominated the session + layer on the reference worker (ADR-0097, fourth run). diff --git a/docs/adr/0097-m10-load-and-abuse-qualification.md b/docs/adr/0097-m10-load-and-abuse-qualification.md index d303514..52341f3 100644 --- a/docs/adr/0097-m10-load-and-abuse-qualification.md +++ b/docs/adr/0097-m10-load-and-abuse-qualification.md @@ -387,3 +387,34 @@ the trust), then the drill ran at 21:00–21:05 UTC and was restored. scope here. - The drill summary now judges by budgets v2 and prints commit and other-tick maxima and the poll and replicate averages. + +## 10.9d fourth reference run (2026-10-09, image `754446fe…`) +Drill at 01:00–01:06 UTC, restored. It is the first run whose actions were all new (no +duplicates), so the first in which actions actually committed (about 12 per commit, one +commit about every 1.05 s). +- **Client side:** 24 sessions within every client budget (response p95 111 ms; 20 Hz; + no starvation; `duplicate_actions=0`). Outcome p95 about 1.2 s: the action waits for + the commit of the batch it rides. The governor refused logins beyond 24. +- **Server side, not within budgets v2:** + - commit ticks 58–70 ms (budget 50); + - session-layer p99 30–39 ms (1.5–1.6 ms per session), and 3–5.6 ms with a single + session; + - other ticks at most 12–19 ms; no tick debt. +- **Cause:** a synchronous PostgreSQL round trip per action on the simulation thread. + Admission looked up each new action's durable record (uniqueness across epochs, + processes and regions), and each commit looked up every committed action again, so the + first-record rule could hold. About 12 lookups land on each commit tick (≈ 35 ms on the + dev database), and each incoming action costs one round trip on whichever tick it + arrives. Locally a round trip is a fraction of a millisecond, which is why the profile + missed it. Drill 3 never reached the commit path. +- **Fix (no schema change):** + - `ActionLedger::find_all` answers many (account, action) pairs in one statement: the + schema v11 `find_region_action`, lease check included, applied to each pair through + `unnest … CROSS JOIN LATERAL`. + - `IntentAdmission::admit` holds an action that only the ledger can still refuse, and + one `verify` per poll confirms every waiting action with one request. Its receipt is + still sent in the same poll. + - `committed` asks once per commit. + - The store cost is now at most one round trip per poll plus one per commit, whatever + the number of sessions or actions. +- **Next:** a fifth run with this fix, then the declaration. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index cc496d5..7eac939 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -5254,3 +5254,27 @@ Executed: Not done: - shellcheck: not installed in this WSL. - The fourth drill (needs the new image, a fresh bundle and the go-ahead). + +## 2026-10-09 — M10.9d fourth evolution-dev drill, batched durable lookups + +Fourth drill (image `754446fe…`, 01:00–01:06 UTC, restored Ready): +- 24 sessions within the client budgets; +- the server was not within budgets v2 (commit ticks 58–70 ms, session p99 30–39 ms). + +The cause was one synchronous store round trip per action, at admission and again at +commit (ADR-0097, fourth run). Fix: +- `ActionLedger::find_all` and `find_region_actions` (one statement over the schema v11 + function, no migration); +- `IntentAdmission::admit`/`verify`, with one verify per poll in `RegionSessions`; +- one request per commit. + +Executed: +- WSL Release `quic`: build, plus the quic, network, ops-contracts and session-fuzz CTest + (6/6), passed. +- The 18 Postgres suites, passed. They include the new `find_region_actions` checks: + order, absent pairs, empty, fenced, and `PostgresActionLedger::find_all`. +- `deferred_actions` in `network_control`. +- clang-tidy 19 on the changed sources, clean after two fixes. +- Windows `debug` 60/60, `quic` 5/5, `client-network` 15/15. + +Not done: the fifth drill (needs the new image, a fresh bundle and the go-ahead). diff --git a/include/evo/network/intent_admission.hpp b/include/evo/network/intent_admission.hpp index 2876572..714509a 100644 --- a/include/evo/network/intent_admission.hpp +++ b/include/evo/network/intent_admission.hpp @@ -54,8 +54,14 @@ struct AccountOutcome { enum class IntentConfigError : std::uint8_t { invalid }; // Durable action records (10.4b): the region host implements it over its lease (schema v11); // tests use fakes. `find` answers with the committed outcome, nothing when the action was -// never committed, or `unavailable` when it cannot decide. +// never committed, or `unavailable` when it cannot decide. `find_all` answers several in +// order, all or nothing (M10.9d, ADR-0097: one store round trip instead of one per action); +// the default asks `find` for each. enum class ActionLedgerError : std::uint8_t { unavailable }; +struct ActionKey { + AccountId account; + std::uint64_t action_id{}; +}; class ActionLedger { public: ActionLedger() = default; @@ -64,6 +70,9 @@ class ActionLedger { virtual ~ActionLedger() = default; [[nodiscard]] virtual std::expected, ActionLedgerError> find(const AccountId& account, std::uint64_t action_id) = 0; + [[nodiscard]] virtual std::expected>, + ActionLedgerError> + find_all(std::span keys); protected: ActionLedger(ActionLedger&&) = default; @@ -87,6 +96,13 @@ class IntentAdmission { const AdmissionFacts& facts); [[nodiscard]] ActionReceipt action(std::uint64_t session, const SessionAction& m, const AdmissionFacts& facts); + // The same admission with the durable uniqueness check deferred (M10.9d): nothing when the + // action awaits `verify`, which confirms every waiting action with one ledger request and + // returns their receipts by session. Without a ledger, `admit` always answers at once. + [[nodiscard]] std::optional admit(std::uint64_t session, const SessionAction& m, + const AdmissionFacts& facts); + [[nodiscard]] std::vector> + verify(const AdmissionFacts& facts); // An action of the session's account: final, pending or unknown. With a ledger, an action // whose batch outcome was unknown is decided from its durable record once this authority // is ready, and an action no longer in memory is looked up. @@ -101,7 +117,7 @@ class IntentAdmission { // Scheduled actions whose tick is <= `tick`: they ride that tick's batch from now on. [[nodiscard]] std::vector carry(std::uint64_t tick); // These carried actions are durable in `commit_sequence`; `tick` is the tick each rode. - // With a ledger, each outcome is taken from the action's durable record. + // With a ledger, each outcome is taken from the action's durable record (one request). [[nodiscard]] std::vector committed(std::span actions, std::uint64_t commit_sequence); // The host dropped these carried actions. Final: never committed. Otherwise their batch's @@ -127,11 +143,14 @@ class IntentAdmission { std::deque pending; std::uint64_t watermark{}, sent_watermark{}, sent_at_tick{}; }; - enum class Stage : std::uint8_t { scheduled, carried }; + // verifying: admitted in memory, awaiting the durable uniqueness check (`verify`). + enum class Stage : std::uint8_t { verifying, scheduled, carried }; struct Action { CarriedAction carried; Stage stage{}; bool unknown{}; // Carried in a batch whose outcome was unknown when authority was lost. + // While verifying: the admitting session, and the epoch and tick of its admission. + std::uint64_t session{}, epoch{}, admitted_at{}; }; explicit IntentAdmission(IntentLimits limits) : limits_{limits} {} // A ledger answer flattened: unavailable (or no ledger), absent, or committed. @@ -141,6 +160,12 @@ class IntentAdmission { ActionOutcome outcome; }; [[nodiscard]] Found lookup(const AccountId& account, std::uint64_t action_id) const; + // The in-memory checks of an action; `check` is set when only the ledger can still + // refuse it. + [[nodiscard]] ActionReceipt screen(Session& s, const SessionAction& m, + const AdmissionFacts& facts, bool& check); + // Applies a durable answer to a receipt that passed `screen`. + void decide(const AccountId& account, ActionReceipt& r, const Found& found); Session* find(std::uint64_t session); const Session* find(std::uint64_t session) const; void count_window(Session& s, std::uint64_t server_tick) const; diff --git a/include/evo/persistence/region_lease_store.hpp b/include/evo/persistence/region_lease_store.hpp index 7916fba..3405a4b 100644 --- a/include/evo/persistence/region_lease_store.hpp +++ b/include/evo/persistence/region_lease_store.hpp @@ -105,6 +105,15 @@ struct CommittedAction { [[nodiscard]] std::expected, LeaseStoreFailure> find_region_action(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& account, std::uint64_t action_id, std::chrono::milliseconds timeout); +// The same for several (account, action) pairs in one statement, answered in order +// (M10.9d, ADR-0097). Empty asks nothing. +struct ActionRef { + Uuid account{}; + std::uint64_t action_id{}; +}; +[[nodiscard]] std::expected>, LeaseStoreFailure> +find_region_actions(Postgres& db, RegionKey key, const LeaseGrant& lease, + std::span actions, std::chrono::milliseconds timeout); struct CommittedDecision { std::uint64_t commit_sequence{}; diff --git a/include/evo/server/postgres_action_ledger.hpp b/include/evo/server/postgres_action_ledger.hpp index 6af8921..1ef7ed3 100644 --- a/include/evo/server/postgres_action_ledger.hpp +++ b/include/evo/server/postgres_action_ledger.hpp @@ -20,6 +20,9 @@ class PostgresActionLedger final : public network::ActionLedger { : db_{&db}, key_{key}, lease_{std::move(lease)}, timeout_{timeout} {} [[nodiscard]] std::expected, network::ActionLedgerError> find(const network::AccountId& account, std::uint64_t action_id) override; + [[nodiscard]] std::expected>, + network::ActionLedgerError> + find_all(std::span keys) override; // One statement. private: PgSlot* db_; diff --git a/src/network/protocol/intent_admission.cpp b/src/network/protocol/intent_admission.cpp index cdfd512..f154726 100644 --- a/src/network/protocol/intent_admission.cpp +++ b/src/network/protocol/intent_admission.cpp @@ -108,19 +108,15 @@ IntentionReceipt IntentAdmission::intention(std::uint64_t session, const Session return r; } -ActionReceipt IntentAdmission::action(std::uint64_t session, const SessionAction& m, - const AdmissionFacts& facts) { +ActionReceipt IntentAdmission::screen(Session& s, const SessionAction& m, + const AdmissionFacts& facts, bool& check) { ActionReceipt r{m.action_id, AdmitStatus::admitted, facts.server_tick, m.tick}; - auto* s = find(session); - if (!s) { - r.status = AdmitStatus::unknown_session; - return r; - } - count_window(*s, facts.server_tick); - const bool over = s->action_attempts >= limits_.max_actions_per_window; + check = false; + count_window(s, facts.server_tick); + const bool over = s.action_attempts >= limits_.max_actions_per_window; if (!over) - ++s->action_attempts; - const auto& account = s->account; + ++s.action_attempts; + const auto& account = s.account; const auto known_action = std::ranges::find_if(actions_, [&](const Action& a) { return a.carried.account == account && a.carried.action_id == m.action_id; }); @@ -129,7 +125,7 @@ ActionReceipt IntentAdmission::action(std::uint64_t session, const SessionAction }); if (over) r.status = AdmitStatus::rate_limited; - else if (m.session != session) + else if (m.session != s.id) r.status = AdmitStatus::unknown_session; else if (!facts.ready) r.status = AdmitStatus::not_ready; @@ -144,24 +140,107 @@ ActionReceipt IntentAdmission::action(std::uint64_t session, const SessionAction known_action != actions_.end() ? known_action->carried.tick : known_final->outcome.tick; } else if (!in_window(m.tick, facts.server_tick, limits_.intent_window)) r.status = AdmitStatus::outside_window; - else if (unresolved_of(*s) >= limits_.max_unresolved) + else if (unresolved_of(s) >= limits_.max_unresolved) r.status = AdmitStatus::too_many_unresolved; - else if (ledger_) { - // Committed earlier (another epoch, process or region): never admitted again. - const auto found = lookup(account, m.action_id); - if (found.state == Record::unavailable) { - r.status = AdmitStatus::not_ready; // Uniqueness cannot be confirmed now. - } else if (found.state == Record::committed) { - r.status = AdmitStatus::duplicate; - r.tick = found.outcome.tick; - remember(account, found.outcome); - } - } - if (r.status == AdmitStatus::admitted) - actions_.push_back({{account, m.action_id, m.tick, ActionKind::probe}, Stage::scheduled}); + else + check = ledger_ != nullptr; return r; } +void IntentAdmission::decide(const AccountId& account, ActionReceipt& r, const Found& found) { + // Committed earlier (another epoch, process or region): never admitted again. + if (found.state == Record::unavailable) { + r.status = AdmitStatus::not_ready; // Uniqueness cannot be confirmed now. + } else if (found.state == Record::committed) { + r.status = AdmitStatus::duplicate; + r.tick = found.outcome.tick; + remember(account, found.outcome); + } +} + +ActionReceipt IntentAdmission::action(std::uint64_t session, const SessionAction& m, + const AdmissionFacts& facts) { + auto* s = find(session); + if (!s) + return {m.action_id, AdmitStatus::unknown_session, facts.server_tick, m.tick}; + bool check = false; + auto r = screen(*s, m, facts, check); + if (check) + decide(s->account, r, lookup(s->account, m.action_id)); + if (r.status == AdmitStatus::admitted) + actions_.push_back( + {{s->account, m.action_id, m.tick, ActionKind::probe}, Stage::scheduled}); + return r; +} + +std::optional IntentAdmission::admit(std::uint64_t session, const SessionAction& m, + const AdmissionFacts& facts) { + auto* s = find(session); + if (!s) + return ActionReceipt{m.action_id, AdmitStatus::unknown_session, facts.server_tick, m.tick}; + bool check = false; + const auto r = screen(*s, m, facts, check); + if (r.status != AdmitStatus::admitted) + return r; + // Held like an admitted action (duplicates and the unresolved limit see it), but never + // carried until `verify` confirms it. + actions_.push_back({{s->account, m.action_id, m.tick, ActionKind::probe}, + check ? Stage::verifying : Stage::scheduled, + false, + session, + facts.epoch, + facts.server_tick}); + if (check) + return std::nullopt; + return r; +} + +std::vector> +IntentAdmission::verify(const AdmissionFacts& facts) { + std::vector> out; + std::vector keys; + for (const auto& a : actions_) + if (a.stage == Stage::verifying) + keys.push_back({a.carried.account, a.carried.action_id}); + if (keys.empty()) + return out; + // Authority lost or moved on since admission: refused as admission would refuse it now. + const bool current = facts.ready; + std::expected>, ActionLedgerError> answers = + std::unexpected(ActionLedgerError::unavailable); + if (current && ledger_ != nullptr) + answers = ledger_->find_all(keys); + if (answers.has_value() && answers->size() != keys.size()) + answers = std::unexpected(ActionLedgerError::unavailable); + std::size_t i = 0; + for (auto& a : actions_) { + if (a.stage != Stage::verifying) + continue; + const auto index = i++; + ActionReceipt r{a.carried.action_id, AdmitStatus::admitted, a.admitted_at, a.carried.tick}; + if (!current) { + r.status = AdmitStatus::not_ready; + } else if (a.epoch != facts.epoch) { + r.status = AdmitStatus::stale_epoch; + } else { + Found found; + if (answers.has_value()) { + const auto& record = (*answers)[index]; + found.state = record.has_value() ? Record::committed : Record::absent; + if (record.has_value()) + found.outcome = *record; + } + decide(a.carried.account, r, found); + } + if (r.status == AdmitStatus::admitted) + a.stage = Stage::scheduled; + out.emplace_back(a.session, r); + } + // Every refused one leaves (a duplicate is remembered among the finals by `decide`). + std::erase_if(actions_, [](const Action& a) { return a.stage == Stage::verifying; }); + return out; +} + ActionOutcome IntentAdmission::query(std::uint64_t session, const ActionQuery& m, const AdmissionFacts& facts) { ActionOutcome o{m.action_id, ActionResult::unknown, 0, 0}; @@ -242,13 +321,26 @@ void IntentAdmission::remember(const AccountId& account, const ActionOutcome& ou std::vector IntentAdmission::committed(std::span actions, std::uint64_t commit_sequence) { std::vector out; - for (const auto& c : actions) { + // The durable record is the authority: an id carried by two batches (an account that + // moved regions with the action pending) keeps the record of the first commit. One + // request for the whole commit; unanswered, the host's own report stands. + std::expected>, ActionLedgerError> records = + std::unexpected(ActionLedgerError::unavailable); + if (ledger_ != nullptr && !actions.empty()) { + std::vector keys; + keys.reserve(actions.size()); + for (const auto& c : actions) + keys.push_back({c.account, c.action_id}); + records = ledger_->find_all(keys); + if (records.has_value() && records->size() != actions.size()) + records = std::unexpected(ActionLedgerError::unavailable); + } + for (std::size_t i = 0; i < actions.size(); ++i) { + const auto& c = actions[i]; ActionOutcome o{c.action_id, ActionResult::committed_no_effect, c.tick, commit_sequence}; - // The durable record is the authority: an id carried by two batches (an account that - // moved regions with the action pending) keeps the record of the first commit. - if (ledger_) - if (const auto found = lookup(c.account, c.action_id); found.state == Record::committed) - o = found.outcome; + if (records.has_value()) + if (const auto& record = (*records)[i]; record.has_value()) + o = *record; std::erase_if(actions_, [&](const Action& a) { return a.carried.account == c.account && a.carried.action_id == c.action_id; }); @@ -297,6 +389,19 @@ std::vector IntentAdmission::abandon() { return out; } +std::expected>, ActionLedgerError> +ActionLedger::find_all(std::span keys) { + std::vector> out; + out.reserve(keys.size()); + for (const auto& k : keys) { + auto found = find(k.account, k.action_id); + if (!found.has_value()) + return std::unexpected(found.error()); + out.push_back(*found); + } + return out; +} + IntentAdmission::Found IntentAdmission::lookup(const AccountId& account, std::uint64_t action_id) const { Found f; diff --git a/src/persistence/region_lease_store.cpp b/src/persistence/region_lease_store.cpp index 3abdcda..ad6a779 100644 --- a/src/persistence/region_lease_store.cpp +++ b/src/persistence/region_lease_store.cpp @@ -353,6 +353,50 @@ find_region_action(Postgres& db, RegionKey key, const LeaseGrant& lease, const U return std::optional{CommittedAction{*region, *tick, *sequence}}; } +std::expected>, LeaseStoreFailure> +find_region_actions(Postgres& db, RegionKey key, const LeaseGrant& lease, + std::span actions, milliseconds timeout) { + std::vector> out(actions.size()); + if (actions.empty()) + return out; + // Array literals of canonical uuids and decimal ids: no quoting is needed. + std::string accounts = "{", ids = "{"; + for (std::size_t i = 0; i < actions.size(); ++i) { + accounts += (i == 0 ? "" : ",") + uuid_text(actions[i].account); + ids += (i == 0 ? "" : ",") + std::to_string(actions[i].action_id); + } + accounts += "}"; + ids += "}"; + const std::array p{std::to_string(key.world), + std::to_string(key.region), + uuid_text(lease.holder), + std::to_string(lease.epoch), + accounts, + ids}; + // The schema v11 function per pair (its lease check included); rows name their pair. + auto rows = db.query("SELECT x.n, f.action_region, f.action_tick, f.action_commit " + "FROM unnest($5::uuid[], $6::numeric[]) WITH ORDINALITY AS x(a, i, n) " + "CROSS JOIN LATERAL evolution.find_region_action($1::evolution.u64, " + "$2::evolution.u64, $3::uuid, $4::evolution.u64, x.a, " + "x.i::evolution.u64) AS f", + p, timeout); + if (!rows) + return std::unexpected{classify(rows.error())}; + for (const auto& r : *rows) { + if (r.size() != 4) + return std::unexpected{malformed(false)}; + const auto n = parse_integer(r[0]); + const auto region = parse_integer(r[1]); + const auto tick = parse_integer(r[2]); + const auto sequence = parse_integer(r[3]); + if (!n || *n == 0 || *n > actions.size() || out[*n - 1].has_value() || !region || !tick || + !sequence) + return std::unexpected{malformed(false)}; + out[*n - 1] = CommittedAction{*region, *tick, *sequence}; + } + return out; +} + namespace { std::expected, LeaseStoreFailure> decisions_from(const PgRows& rows) { diff --git a/src/server/session/postgres_action_ledger.cpp b/src/server/session/postgres_action_ledger.cpp index 63412f5..53002c7 100644 --- a/src/server/session/postgres_action_ledger.cpp +++ b/src/server/session/postgres_action_ledger.cpp @@ -25,4 +25,38 @@ PostgresActionLedger::find(const network::AccountId& account, std::uint64_t acti network::ActionOutcome{action_id, network::ActionResult::committed_no_effect, record.value().tick, record.value().commit_sequence}}; } + +std::expected>, network::ActionLedgerError> +PostgresActionLedger::find_all(std::span keys) { + const auto lease = lease_(); + if (!lease) + return std::unexpected(network::ActionLedgerError::unavailable); + auto* db = db_->get(); + if (!db) + return std::unexpected(network::ActionLedgerError::unavailable); + std::vector refs; + refs.reserve(keys.size()); + for (const auto& k : keys) + refs.push_back({k.account.bytes, k.action_id}); + const auto found = persistence::find_region_actions(*db, key_, *lease, refs, timeout_); + if (!found.has_value()) { + const auto kind = found.error().kind; + if (kind == persistence::LeaseStoreError::store || + kind == persistence::LeaseStoreError::unknown) + db_->broken(); // Reconnect next time. + return std::unexpected(network::ActionLedgerError::unavailable); + } + std::vector> out; + out.reserve(keys.size()); + for (std::size_t i = 0; i < keys.size(); ++i) { + const auto& record = (*found)[i]; + if (record.has_value()) + out.emplace_back(network::ActionOutcome{keys[i].action_id, + network::ActionResult::committed_no_effect, + record->tick, record->commit_sequence}); + else + out.emplace_back(); + } + return out; +} } // namespace evo::server diff --git a/src/server/session/region_sessions.cpp b/src/server/session/region_sessions.cpp index 82185f4..4e9948f 100644 --- a/src/server/session/region_sessions.cpp +++ b/src/server/session/region_sessions.cpp @@ -504,6 +504,16 @@ struct RegionSessions::Impl { markers.set_ghosts(ghosts); } + void answer(Connection& c, const network::ActionReceipt& receipt) { + send(c, receipt); + // A duplicate of a final action: answer with the outcome as well. + if (receipt.status == network::AdmitStatus::duplicate) + if (const auto o = intents.query(c.session, {receipt.action_id}, admission()); + o.result == network::ActionResult::committed_no_effect || + o.result == network::ActionResult::not_committed) + send(c, o); + } + void deliver(const std::vector& outcomes) { for (const auto& o : outcomes) if (const auto session = intents.session_of(o.account)) @@ -532,14 +542,9 @@ struct RegionSessions::Impl { if (const auto* m = std::get_if(&*decoded)) return refresh(c, *m, now_seconds, now); if (const auto* m = std::get_if(&*decoded)) { - const auto receipt = intents.action(c.session, *m, admission()); - send(c, receipt); - // A duplicate of a final action: answer with the outcome as well. - if (receipt.status == network::AdmitStatus::duplicate) - if (const auto o = intents.query(c.session, {m->action_id}, admission()); - o.result == network::ActionResult::committed_no_effect || - o.result == network::ActionResult::not_committed) - send(c, o); + // A durable uniqueness check waits for this poll's single `verify` (M10.9d). + if (const auto receipt = intents.admit(c.session, *m, admission())) + answer(c, *receipt); return; } if (const auto* m = std::get_if(&*decoded)) @@ -676,6 +681,10 @@ void RegionSessions::poll(const HostFacts& facts, std::uint64_t now_seconds, m.datagram(*c, e.bytes, now); } } + // One ledger request for every action admitted by this poll's messages. + for (const auto& [session, receipt] : m.intents.verify(m.admission())) + if (auto* c = m.by_session(session); c && c->state == Impl::State::established) + m.answer(*c, receipt); for (const auto& [session, watermark] : m.intents.simulated(m.admission())) if (auto* c = m.by_session(session); c && c->state == Impl::State::established) m.send_datagram(*c, watermark); diff --git a/tests/network_control.cpp b/tests/network_control.cpp index 5e000aa..f5d3130 100644 --- a/tests/network_control.cpp +++ b/tests/network_control.cpp @@ -427,11 +427,76 @@ struct FakeActions final : ActionLedger { return std::optional{outcome}; return std::optional{}; } + std::expected>, ActionLedgerError> + find_all(std::span keys) override { + ++batches; + return ActionLedger::find_all(keys); // Counts each pair as a lookup too. + } std::vector> records; bool unavailable{}; - int lookups{}; + int lookups{}, batches{}; }; +// M10.9d: admission whose durable uniqueness check waits for one `verify` per poll, and a +// commit decided with one ledger request. +void deferred_actions() { + auto a = *IntentAdmission::create(IntentLimits{}); + FakeActions ledger; + a.attach(&ledger); + const auto alice = account(1), bob = account(2); + CHECK(a.open(1, alice) && a.open(2, bob)); + AdmissionFacts f{true, 5, 100}; + const auto admit = [&](std::uint64_t session, std::uint64_t id, std::uint64_t tick) { + return a.admit(session, SessionAction{session, 5, id, 1, tick}, f); + }; + ledger.records.emplace_back(alice, ActionOutcome{9, ActionResult::committed_no_effect, 40, 3}); + CHECK(!admit(1, 9, 101) && !admit(1, 20, 101) && !admit(2, 30, 102)); + // In-memory refusals answer at once; a repeat of a waiting action is a duplicate. + CHECK(admit(1, 20, 101) == ActionReceipt{20, AdmitStatus::duplicate, 100, 101}); + CHECK(admit(1, 21, 150) == ActionReceipt{21, AdmitStatus::outside_window, 100, 150}); + CHECK(a.carry(110).empty() && ledger.lookups == 0); // Nothing rides before `verify`. + f.server_tick = 101; + auto receipts = a.verify(f); + CHECK(ledger.batches == 1 && ledger.lookups == 3 && receipts.size() == 3); + CHECK(receipts[0] == + std::pair{std::uint64_t{1}, ActionReceipt{9, AdmitStatus::duplicate, 100, 40}}); + CHECK(receipts[1] == + std::pair{std::uint64_t{1}, ActionReceipt{20, AdmitStatus::admitted, 100, 101}}); + CHECK(receipts[2] == + std::pair{std::uint64_t{2}, ActionReceipt{30, AdmitStatus::admitted, 100, 102}}); + CHECK(a.query(1, {9}, f).result == ActionResult::committed_no_effect); + CHECK(a.verify(f).empty() && ledger.batches == 1); // Nothing waiting: nothing asked. + // One request for a whole commit. + const auto riding = a.carry(102); + CHECK(riding.size() == 2); + const auto outcomes = a.committed(riding, 6); + CHECK(ledger.batches == 2 && outcomes.size() == 2 && + outcomes[0].outcome == ActionOutcome{20, ActionResult::committed_no_effect, 101, 6}); + // Unconfirmed uniqueness, lost readiness and a new epoch refuse what was waiting. + f.server_tick = 110; // A new rate window. + ledger.unavailable = true; + CHECK(!admit(1, 22, 111)); + receipts = a.verify(f); + CHECK(receipts.size() == 1 && receipts[0].second.status == AdmitStatus::not_ready); + ledger.unavailable = false; + CHECK(a.unresolved(1) == 0); + CHECK(!admit(1, 23, 111)); + f.ready = false; + receipts = a.verify(f); + CHECK(receipts.size() == 1 && receipts[0].second.status == AdmitStatus::not_ready); + f.ready = true; + CHECK(!admit(1, 24, 111)); + f.epoch = 6; + receipts = a.verify(f); + CHECK(receipts.size() == 1 && receipts[0].second.status == AdmitStatus::stale_epoch); + // Without a ledger nothing waits. + auto plain = *IntentAdmission::create(IntentLimits{}); + CHECK(plain.open(1, alice)); + CHECK(plain.admit(1, SessionAction{1, 5, 7, 1, 101}, AdmissionFacts{true, 5, 100}) == + ActionReceipt{7, AdmitStatus::admitted, 100, 101}); + CHECK(plain.verify(AdmissionFacts{true, 5, 100}).empty()); +} + void durable_actions() { auto a = *IntentAdmission::create(IntentLimits{}); FakeActions ledger; @@ -777,5 +842,6 @@ void control_tests() { session_codec(); intent_admission(); durable_actions(); + deferred_actions(); } } // namespace evo::network_test diff --git a/tests/region_action_tests.cpp b/tests/region_action_tests.cpp index 1ab8d1a..0428e5b 100644 --- a/tests/region_action_tests.cpp +++ b/tests/region_action_tests.cpp @@ -166,6 +166,15 @@ void store_rules() { std::optional{CommittedAction{131, 10, 1}})); CHECK(holds(find_region_action(*db, r.key, *lease, alice, 3, 5s), std::optional{})); + // M10.9d: the same answers for several pairs in one statement, in the order asked. + const std::vector asked{{alice, 3}, {bob, 1}, {alice, 2}, {alice, 1}, {bob, 7}}; + auto many = find_region_actions(*db, r.key, *lease, asked, 5s); + CHECK(many && + *many == std::vector>{ + std::nullopt, CommittedAction{131, 3, 1}, CommittedAction{131, 10, 1}, + CommittedAction{131, 3, 1}, std::nullopt}); + CHECK(holds(find_region_actions(*db, r.key, *lease, {}, 5s), + std::vector>{})); // Exact retry and resolution include the action block. const auto block = encode_actions(actions, 0, 10); @@ -223,6 +232,9 @@ void store_rules() { CHECK(later); auto fenced = find_region_action(*db, r.key, *lease, alice, 1, 5s); CHECK(!fenced && fenced.error().kind == LeaseStoreError::fenced); + const std::vector one{{alice, 1}}; + auto fenced_many = find_region_actions(*db, r.key, *lease, one, 5s); + CHECK(!fenced_many && fenced_many.error().kind == LeaseStoreError::fenced); CHECK(holds(find_region_action(*db, r.key, *later, alice, 3, 5s), std::optional{CommittedAction{131, 12, 2}})); #ifdef EVO_POSTGRES_SESSIONS @@ -236,8 +248,14 @@ void store_rules() { std::optional{evo::network::ActionOutcome{ 3, evo::network::ActionResult::committed_no_effect, 12, 2}})); CHECK(holds(ledger.find(who, 99), std::optional{})); + const std::vector keys{{who, 99}, {who, 3}}; + CHECK(holds(ledger.find_all(keys), + std::vector>{ + std::nullopt, evo::network::ActionOutcome{ + 3, evo::network::ActionResult::committed_no_effect, 12, 2}})); current.reset(); CHECK(!ledger.find(who, 3)); + CHECK(!ledger.find_all(keys)); #endif CHECK(release_region_lease(*db, r.key, *later, 5s)); }