From 69f3ad7cf8196feab4f59b5904c31b896ee522cf Mon Sep 17 00:00:00 2001 From: dingo Date: Sat, 3 Oct 2026 22:53:38 -0400 Subject: [PATCH] Add M10.4b durable action records (schema v11) Each batch's actions are recorded in the same statement as its checkpoint: one region_actions row per (world, account, action id) and the block in an append-only table, compared on exact retries and resolution. The region host attaches actions to the ticks they ride and reports acknowledgements and drops, keeping unknown-outcome batches unresolved. Outcomes come from the durable record via ActionLedger, so crashes before or after a commit, unknown publications and cross-region moves all resolve to one answer; a repeated id keeps its first record instead of stalling a region. ADR-0092. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 8 +- docs/ROADMAP.md | 5 +- docs/STATUS.md | 7 +- ...092-m10-intentions-and-durable-outcomes.md | 30 ++ docs/engineering/VALIDATION.md | 39 ++ docs/infrastructure/POSTGRESQL.md | 10 + include/evo/network/intent_admission.hpp | 49 ++- include/evo/persistence/checkpoint_store.hpp | 7 +- include/evo/persistence/region_batch.hpp | 16 + .../evo/persistence/region_lease_store.hpp | 25 ++ include/evo/server/postgres_action_ledger.hpp | 29 ++ include/evo/server/region_host.hpp | 25 ++ include/evo/server/region_sessions.hpp | 12 +- migrations/0011_region_actions.sql | 170 ++++++++ migrations/apply.sql | 29 +- schemas/region-actions-v1.md | 27 ++ src/network/protocol/intent_admission.cpp | 103 ++++- src/persistence/checkpoint_store.cpp | 25 +- src/persistence/region_batch.cpp | 37 ++ src/persistence/region_lease_store.cpp | 71 +++ src/server/region_host.cpp | 54 ++- src/server/session/postgres_action_ledger.cpp | 20 + src/server/session/region_sessions.cpp | 14 +- tests/network_control.cpp | 107 ++++- tests/postgres_tests.cpp | 2 + tests/quic_sessions.cpp | 2 +- tests/region_action_tests.cpp | 412 ++++++++++++++++++ tests/region_checkpoint_store_tests.cpp | 2 +- tests/region_migration_tests.cpp | 3 +- tools/ci/postgres-check.sh | 16 +- tools/ci/upgrade-check.sh | 4 +- tools/migrate-postgres.sh | 4 +- 32 files changed, 1289 insertions(+), 75 deletions(-) create mode 100644 include/evo/server/postgres_action_ledger.hpp create mode 100644 migrations/0011_region_actions.sql create mode 100644 schemas/region-actions-v1.md create mode 100644 src/server/session/postgres_action_ledger.cpp create mode 100644 tests/region_action_tests.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 2dae047..7e5ccc4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -220,8 +220,10 @@ if(EVO_BUILD_POSTGRES) endif() evo_owned_target(evo-server) if(EVO_BUILD_NETWORK) - # M10.3c durable account control for region sessions (ADR-0091, schema v10). - add_library(evo_control_ledger_postgres STATIC src/server/session/postgres_control_ledger.cpp) + # M10.3c durable account control and M10.4b durable action records for region sessions + # (ADR-0091 schema v10, ADR-0092 schema v11). + add_library(evo_control_ledger_postgres STATIC src/server/session/postgres_control_ledger.cpp + src/server/session/postgres_action_ledger.cpp) target_link_libraries(evo_control_ledger_postgres PUBLIC evo::postgres evo_network_protocol) evo_owned_target(evo_control_ledger_postgres) endif() @@ -232,7 +234,7 @@ if(EVO_BUILD_POSTGRES) if(BUILD_TESTING) # Contention tests use one thread and one connection per contender. add_executable(evo-postgres-tests tests/postgres_tests.cpp tests/region_lease_store_tests.cpp - tests/account_control_store_tests.cpp + tests/account_control_store_tests.cpp tests/region_action_tests.cpp tests/region_checkpoint_store_tests.cpp tests/region_host_tests.cpp tests/region_server_process_tests.cpp tests/region_recovery_tests.cpp tests/region_ops_tests.cpp tests/region_clone_tests.cpp diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index ba7ddfe..fdca89b 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -522,8 +522,9 @@ hardware (10.9). 10.2 split: 10.2a selection, pins and build of msquic/OpenSSL d 10.3 split (ADR-0091): 10.3a account ids, ECDSA grants, identity adapter and region session control done; 10.3b QUIC session login (schemas/network/session-v1.md) done; 10.3c durable per-world account control (schema v10) done. 10.4 split (ADR-0092): -10.4a intention/action protocol and admission done; 10.4b durable action records and -10.4c evo-server wiring with a headless client follow. 10.5-10.9 planned. +10.4a intention/action protocol and admission done; 10.4b durable action records +(schema v11, same transaction as the checkpoint) done; 10.4c evo-server wiring with a +headless client follows. 10.5-10.9 planned. **Entry:** M9 client/session evidence and M7/M8 dev/stage durability. Read NETWORK_ARCHITECTURE, PERSISTENCE_ARCHITECTURE and ADR-0017. The M9 transport is diff --git a/docs/STATUS.md b/docs/STATUS.md index 261d242..41ba2d4 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -28,8 +28,11 @@ control is fenced or cannot be renewed. M10.4a (ADR-0092) adds intentions (datagram receipts and a cumulative simulated watermark) and account-scoped actions with stable ids, receipts and final outcomes to session-v1, with admission (window, rate, sequence, duplicates, unresolved bound) and client support; intentions and -actions have no biological effect yet. Next: M10.4b durable action records (migration -v11) and M10.4c evo-server wiring. +actions have no biological effect yet. M10.4b makes +action outcomes durable: schema v11 records each batch's actions in the same statement as +its checkpoint, the host attaches actions to the ticks they ride, and outcomes are read +from the durable record after crashes before or after a commit, unknown publications and +cross-region moves. Next: M10.4c evo-server wiring with a headless client. ## Previous gate - 2026-10-02 diff --git a/docs/adr/0092-m10-intentions-and-durable-outcomes.md b/docs/adr/0092-m10-intentions-and-durable-outcomes.md index 101b832..057c213 100644 --- a/docs/adr/0092-m10-intentions-and-durable-outcomes.md +++ b/docs/adr/0092-m10-intentions-and-durable-outcomes.md @@ -59,3 +59,33 @@ the wire field must never be cast from it). - The simulated watermark says the targeted tick ran while the intention was scheduled; with no movement rule it changed nothing. 10.6 gives intentions an effect. - Nothing is wired into evo-server yet (10.4c); nothing deploys M10 sessions. + +## M10.4b durable action records (2026-10-03) +Schema v11 (`migrations/0011_region_actions.sql`, [region-actions-v1](../../schemas/region-actions-v1.md)): +- **Same transaction as the checkpoint.** The region host attaches each submitted action to + the first simulated tick at or after its scheduled tick and publishes the batch's action + block with the three-argument `publish_region_checkpoint`; the records exist exactly when + the checkpoint does. Committed history stays immutable: blocks live in their own + append-only table, and an exact retry or resolution must repeat the block. +- **Outcomes come from the record.** `find_region_action` (under the caller's exact region + lease) returns the committed record of (world, account, action id). Found: final + `committed_no_effect` with that record's tick and commit. Absent once this authority is + recovered and ready: never committed by any earlier epoch (its commits are final). +- **One record per id, without stalling a region.** Found during testing: an account that + moves regions while its action is pending in the old one can have the same id admitted by + the new region (the record does not exist yet). Refusing the second commit would have + quarantined a healthy region. The database instead keeps the first record and lets the + later batch commit; `IntentAdmission::committed` reports each outcome from the record, so + every region answers the same tick and commit for an id. +- **Host semantics.** `submit_action` (bounded: 256 queued plus riding), `ActionAck` per + committed action, `DroppedAction` with `final` = certainly never committed. Actions of a + batch whose publication outcome was unknown when authority was lost are not final; the + session layer keeps them pending until the record decides. +- **Session layer.** `ActionLedger` (interface) with `PostgresActionLedger`; admission + checks the record before admitting an id not in memory (`duplicate` with its outcome, or + `not_ready` when the record cannot be read); queries resolve unknown-outcome and evicted + actions from the record once ready. `RegionSessions::committed/dropped` take the host's + reports. +- Limits: action rows are retained (no pruning policy yet); clones (ADR-0067) do not copy + them; the two-argument publication of a commit that has an action block would still + return its receipt (the adapter never does this). diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 0c22e4c..e872ccd 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -4237,3 +4237,42 @@ tests; format check; Windows MSVC 19.44 preset `quic` build and CTest 4/4 (binar timestamps checked). Not done: durable action records and crash resolution (10.4b), evo-server wiring and process kill tests (10.4c). + +## 2026-10-03 — M10.4b durable action records (ADR-0092, schema v11) + +Implemented migration v11 (`region_actions`, `region_commit_actions`, `check_action_block`, +`commit_actions`, three-argument publication, six-argument resolution, +`find_region_action`; apply chain and audit), `encode_actions`, the store overloads, +actions in `PreparedCommit`/publication/resolution, host `submit_action` with +`ActionAck`/`DroppedAction`, `ActionLedger` in admission and `PostgresActionLedger`. +Database tests (`tests/region_action_tests.cpp`, regions 7/131-135, commit ids tagged 0xA7): +publication with actions and lookups (including another account's and an absent id); +exact retry and resolution with the block; a changed block conflicting (retry and +resolution); nine invalid blocks refused (tick outside, tick 0, kind, zero account, zero id, +unsorted, duplicate, one id at two ticks, wrong length) and the encoder refusing the same; +a later batch carrying an already recorded id committing with the first record kept; a +batch without actions; lookups fenced with an old lease; the PostgreSQL action ledger. +Host: acknowledgement with the tick each action rode, late action riding the next tick, +duplicate, invalid and full submissions, not ready; crash before the transaction (successor +finds no record), crash after the commit before any acknowledgement (record durable, +successor at commit 1); unknown publication outcome resolved by exact bytes including the +actions. Unit tests: duplicate from the record, unavailable record refusing admission, +refused attempts costing no lookup, exact commit tick, final and unknown drops, unknown +decided once ready (committed or not committed), evicted outcome found on record. +`postgres-check.sh`: history 1..11, future-version probe 12, v11 checksum drift, audit +refusing SELECT on both new tables and EXECUTE on `check_action_block`. + +Found and fixed during development: updating `region_commits` with the block was refused by +the immutable-history trigger (blocks now live in their own append-only table); a repeated +action id would have failed the later commit and quarantined a healthy region (now the first +record is kept and outcomes come from it); the encoder and SQL accepted one id at two ticks +in one block; test fixtures colliding with other tests' world, commit ids and accounts. + +Executed: Linux GCC 14 `tools/ci/postgres-check.sh` complete (18 integration and rehearsal +suites, migration and audit probes); clang-tidy 19 on the changed persistence, host, +ledger, session and test sources (Clang builds with network+PostgreSQL and with quic); +format check; GCC preset `quic` CTest 4/4, session tests 10/10 repeated; Clang 19 ASan/UBSan +session and network tests; Windows MSVC 19.44 preset `quic` build and CTest 4/4 (binaries +rebuilt, timestamps checked). +Not done: evo-server wiring, headless client and process kill tests with real sessions +(10.4c); ASan of the PostgreSQL suite; a real two-region move with a pending action. diff --git a/docs/infrastructure/POSTGRESQL.md b/docs/infrastructure/POSTGRESQL.md index d1811ee..d802075 100644 --- a/docs/infrastructure/POSTGRESQL.md +++ b/docs/infrastructure/POSTGRESQL.md @@ -190,6 +190,16 @@ renewal requires the same generation, region and epoch and an unexpired control (`EVL12` otherwise). The privilege audit includes all three functions. Lock order: upgrade lock (shared), region lease row, account control row. +## Action records (schema v11, ADR-0092) + +`publish_region_checkpoint(manifest, decisions, actions)` records a batch's action block +([region-actions-v1](../../schemas/region-actions-v1.md)) in the same statement as its +checkpoint: one `evolution.region_actions` row per new (world, account, action id), and the +block in the append-only `region_commit_actions` (committed history itself is never +updated). Exact retries and `resolve_region_commit(..., actions)` compare the block. +`find_region_action` reads one record under the caller's exact region lease. The privilege +audit covers all three functions; the tables and `check_action_block` are owner-only. + ## Host lease discipline Hosts use `LeaseAuthority` with `LeasePolicy` defaults: 20 s lease, 5 s renewal diff --git a/include/evo/network/intent_admission.hpp b/include/evo/network/intent_admission.hpp index 835017a..9b35d4a 100644 --- a/include/evo/network/intent_admission.hpp +++ b/include/evo/network/intent_admission.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -45,11 +46,32 @@ struct AccountOutcome { ActionOutcome outcome; }; enum class IntentConfigError : std::uint8_t { invalid }; +// Durable action records (10.4b): the region host implements it over its lease (schema v11); +// tests use fakes. `find` answers with the committed outcome, nothing when the action was +// never committed, or `unavailable` when it cannot decide. +enum class ActionLedgerError : std::uint8_t { unavailable }; +class ActionLedger { + public: + ActionLedger() = default; + ActionLedger(const ActionLedger&) = delete; + ActionLedger& operator=(const ActionLedger&) = delete; + virtual ~ActionLedger() = default; + [[nodiscard]] virtual std::expected, ActionLedgerError> + find(const AccountId& account, std::uint64_t action_id) = 0; + + protected: + ActionLedger(ActionLedger&&) = default; + ActionLedger& operator=(ActionLedger&&) = default; +}; class IntentAdmission { public: [[nodiscard]] static std::expected create(IntentLimits limits); + // Durable lookups for duplicates and unresolved actions; null keeps memory only. + void attach(ActionLedger* ledger) noexcept { + ledger_ = ledger; + } // Session lifecycle, driven by session control (after Welcome, on any close). [[nodiscard]] bool open(std::uint64_t session, const AccountId& account); void close(std::uint64_t session); @@ -59,18 +81,26 @@ class IntentAdmission { const AdmissionFacts& facts); [[nodiscard]] ActionReceipt action(std::uint64_t session, const SessionAction& m, const AdmissionFacts& facts); - // What this process knows about an action of the session's account. - [[nodiscard]] ActionOutcome query(std::uint64_t session, const ActionQuery& m) const; + // An action of the session's account: final, pending or unknown. With a ledger, an action + // whose batch outcome was unknown is decided from its durable record once this authority + // is ready, and an action no longer in memory is looked up. + [[nodiscard]] ActionOutcome query(std::uint64_t session, const ActionQuery& m, + const AdmissionFacts& facts); // Watermarks to send after the host simulated through facts.server_tick. [[nodiscard]] std::vector> simulated(const AdmissionFacts& facts); // Scheduled actions whose tick is <= `tick`: they ride that tick's batch from now on. [[nodiscard]] std::vector carry(std::uint64_t tick); - // A commit covering ticks up to `end_tick` is durable: its carried actions are final. - [[nodiscard]] std::vector committed(std::uint64_t end_tick, + // These carried actions are durable in `commit_sequence`; `tick` is the tick each rode. + // With a ledger, each outcome is taken from the action's durable record. + [[nodiscard]] std::vector committed(std::span actions, std::uint64_t commit_sequence); + // The host dropped these carried actions. Final: never committed. Otherwise their batch's + // outcome was unknown: they stay pending until their durable record decides them. + [[nodiscard]] std::vector dropped(std::span actions, + bool final); // Authority lost or epoch advanced: actions never carried are final (not_committed); - // carried ones stay unresolved until a durable record decides them (10.4b). + // carried ones stay pending until committed, dropped or decided by their record. [[nodiscard]] std::vector abandon(); // The current session of an account, for delivering outcomes. [[nodiscard]] std::optional session_of(const AccountId& account) const; @@ -92,8 +122,16 @@ class IntentAdmission { struct Action { CarriedAction carried; Stage stage{}; + bool unknown{}; // Carried in a batch whose outcome was unknown when authority was lost. }; explicit IntentAdmission(IntentLimits limits) : limits_{limits} {} + // A ledger answer flattened: unavailable (or no ledger), absent, or committed. + enum class Record : std::uint8_t { unavailable, absent, committed }; + struct Found { + Record state{Record::unavailable}; + ActionOutcome outcome; + }; + [[nodiscard]] Found lookup(const AccountId& account, std::uint64_t action_id) const; Session* find(std::uint64_t session); const Session* find(std::uint64_t session) const; void count_window(Session& s, std::uint64_t server_tick) const; @@ -103,5 +141,6 @@ class IntentAdmission { std::vector sessions_; std::vector actions_; std::deque finals_; + ActionLedger* ledger_{}; }; } // namespace evo::network diff --git a/include/evo/persistence/checkpoint_store.hpp b/include/evo/persistence/checkpoint_store.hpp index 042cce0..0ca0ca9 100644 --- a/include/evo/persistence/checkpoint_store.hpp +++ b/include/evo/persistence/checkpoint_store.hpp @@ -34,6 +34,7 @@ struct PreparedCommit { Uuid commit{}; std::vector manifest; std::vector decisions; + std::vector actions; // Schema v11 block; empty: published without actions. }; struct PublicationFailure { PublicationError kind{}; @@ -92,7 +93,8 @@ publish_environmental_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease, const CommittedCheckpoint& predecessor, const RegionBatch& batch, Uuid commit, const Sha256Digest& build, std::size_t restore_budget, - std::chrono::milliseconds timeout, const PublishHooks& hooks = {}); + std::chrono::milliseconds timeout, const PublishHooks& hooks = {}, + std::span actions = {}); // Loads and fully verifies the committed head before any readiness (ADR-0065): // manifest digest and chain fields, retained build, payload length/SHA-256 and full // restore; then the head's tail: its trace object binds to the committed @@ -119,7 +121,8 @@ publish_lineage_batch(Postgres& db, const FilesystemObjectStore& objects, const const CommittedCheckpoint& predecessor, const RegionBatch& batch, Uuid commit, const Sha256Digest& build, std::size_t restore_budget, - std::chrono::milliseconds timeout, const PublishHooks& hooks = {}); + std::chrono::milliseconds timeout, const PublishHooks& hooks = {}, + std::span actions = {}); [[nodiscard]] std::expected recover_lineage_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, std::span retained_builds, diff --git a/include/evo/persistence/region_batch.hpp b/include/evo/persistence/region_batch.hpp index 15767b7..d135ba8 100644 --- a/include/evo/persistence/region_batch.hpp +++ b/include/evo/persistence/region_batch.hpp @@ -57,6 +57,22 @@ inline constexpr std::size_t max_batch_decisions = 8; return trace_header_bytes + (static_cast(ticks) * trace_step_bytes); } +// M10.4b durable action records (schema v11, ADR-0092): the actions that rode a batch, +// committed in the same statement as its checkpoint. Not part of the trace or manifest. +inline constexpr std::size_t action_record_bytes = 34; +inline constexpr std::size_t max_batch_actions = 1024; +struct ActionRecord { + Uuid account{}; + std::uint64_t action_id{}, tick{}; + std::uint16_t kind{1}; // 1 = probe (no effect). + friend bool operator==(const ActionRecord&, const ActionRecord&) = default; +}; +// Canonical block: sorted by (tick, account, id); refuses zero accounts or ids, other kinds, +// ticks outside (begin, end], duplicates and more than max_batch_actions records. +[[nodiscard]] std::expected, BatchError> +encode_actions(std::span actions, std::uint64_t begin_tick, + std::uint64_t end_tick); + // One step per tick, begin_tick+1..end_tick, at most 64. [[nodiscard]] std::expected, BatchError> encode_trace(const TraceHeader& header, std::span steps); diff --git a/include/evo/persistence/region_lease_store.hpp b/include/evo/persistence/region_lease_store.hpp index fb482a6..7916fba 100644 --- a/include/evo/persistence/region_lease_store.hpp +++ b/include/evo/persistence/region_lease_store.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -67,6 +68,14 @@ publish_region_checkpoint(Postgres& db, std::span manifest, std::span decisions, std::chrono::milliseconds timeout); +// Schema v11: the same publication with the batch's action block (encode_actions), recorded +// in the same statement. The block is part of the commit identity. An action id already on +// record in the world keeps its first record (find_region_action is the authority). +[[nodiscard]] std::expected +publish_region_checkpoint(Postgres& db, std::span manifest, + std::span decisions, + std::span actions, std::chrono::milliseconds timeout); + enum class Resolution : std::uint8_t { committed, // Exact bytes are durable; receipt holds the head. Grants no authority. absent, // Not committed now; proves nothing about a request still in flight. @@ -81,6 +90,22 @@ resolve_region_commit(Postgres& db, RegionKey key, const Uuid& commit, std::span manifest, std::span decisions, std::chrono::milliseconds timeout); +// Schema v11 resolution, comparing the action block too. +[[nodiscard]] std::expected +resolve_region_commit(Postgres& db, RegionKey key, const Uuid& commit, + std::span manifest, + std::span decisions, + std::span actions, std::chrono::milliseconds timeout); +// The committed record of an account's action in the lease's world (schema v11), under the +// caller's exact lease. Nothing when the action was never committed. +struct CommittedAction { + std::uint64_t region{}, tick{}, commit_sequence{}; + friend bool operator==(const CommittedAction&, const CommittedAction&) = default; +}; +[[nodiscard]] std::expected, LeaseStoreFailure> +find_region_action(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& account, + std::uint64_t action_id, std::chrono::milliseconds timeout); + struct CommittedDecision { std::uint64_t commit_sequence{}; BatchDecision decision{}; diff --git a/include/evo/server/postgres_action_ledger.hpp b/include/evo/server/postgres_action_ledger.hpp new file mode 100644 index 0000000..43ad08c --- /dev/null +++ b/include/evo/server/postgres_action_ledger.hpp @@ -0,0 +1,29 @@ +#pragma once +#include "evo/network/intent_admission.hpp" +#include "evo/persistence/region_lease_store.hpp" +#include +#include +#include +#include + +// M10.4b durable action records over the region's PostgreSQL lease (ADR-0092, schema v11). +// Linux only. Owned by the region host's thread, on the host's own connection. `lease` +// returns the host's current region lease, or nothing while it holds none; every lookup +// then reports `unavailable`, as does any store failure. +namespace evo::server { +class PostgresActionLedger final : public network::ActionLedger { + public: + using LeaseSource = std::function()>; + PostgresActionLedger(persistence::Postgres& db, persistence::RegionKey key, LeaseSource lease, + std::chrono::milliseconds timeout = std::chrono::milliseconds{2000}) + : db_{&db}, key_{key}, lease_{std::move(lease)}, timeout_{timeout} {} + [[nodiscard]] std::expected, network::ActionLedgerError> + find(const network::AccountId& account, std::uint64_t action_id) override; + + private: + persistence::Postgres* db_; + persistence::RegionKey key_; + LeaseSource lease_; + std::chrono::milliseconds timeout_; +}; +} // namespace evo::server diff --git a/include/evo/server/region_host.hpp b/include/evo/server/region_host.hpp index f01c385..c2f2fc1 100644 --- a/include/evo/server/region_host.hpp +++ b/include/evo/server/region_host.hpp @@ -50,6 +50,20 @@ struct Dropped { TickInput input{}; Unacknowledged reason{}; }; +// M10.4b (ADR-0092): an action is durable with the batch that simulated its tick. +struct ActionAck { + persistence::ActionRecord record{}; // `tick` is the tick it rode. + std::uint64_t commit_sequence{}; + friend bool operator==(const ActionAck&, const ActionAck&) = default; +}; +// `final`: certainly never committed. Otherwise its batch's publication outcome was unknown +// when authority was lost; the durable record decides (find_region_action) once a later +// authority has recovered. +struct DroppedAction { + persistence::ActionRecord record{}; + bool final{}; + Unacknowledged reason{}; +}; // Commit boundaries for fault injection (M7.7 kill tests). enum class CrashPoint : std::uint8_t { @@ -68,6 +82,7 @@ struct HostConfig { std::chrono::milliseconds flush_interval{1000}; // Commit a non-empty batch this often. std::size_t max_queued_inputs{64}; std::size_t max_queued_bytes{65536}; + std::size_t max_queued_actions{256}; // Queued plus riding the open batch. std::uint32_t max_catch_up_ticks{8}; // Per step; debt is never skipped. persistence::Sha256Digest build{}; std::vector retained_builds; @@ -95,6 +110,10 @@ template class RegionHost { // Provisional admission only; nothing is final until a FinalAck. [[nodiscard]] std::expected submit(const TickInput& input); + // An action rides the first simulated tick >= record.tick and is committed with that + // batch, in the same statement (schema v11). Final only through an ActionAck. + [[nodiscard]] std::expected + submit_action(const persistence::ActionRecord& record); // Advances the state machine once: lease upkeep, recovery, resolution, commits // and up to max_catch_up_ticks ticks. Blocks at most for bounded I/O. const HostStatus& step(SchedulerTime now); @@ -104,6 +123,8 @@ template class RegionHost { [[nodiscard]] std::vector take_acks(); [[nodiscard]] std::vector take_outcomes(); [[nodiscard]] std::vector take_dropped(); + [[nodiscard]] std::vector take_action_acks(); + [[nodiscard]] std::vector take_dropped_actions(); [[nodiscard]] const HostStatus& status() const noexcept { return status_; } @@ -148,6 +169,10 @@ template class RegionHost { std::vector acks_; std::vector outcomes_; std::vector dropped_; + std::deque action_queue_; + std::vector batch_actions_; // Riding the open batch. + std::vector action_acks_; + std::vector dropped_actions_; SchedulerTime origin_{}, batch_started_{}; // Store failures back off: a failed commit is retried after retry_delay_, which // doubles to a cap and resets on the next confirmed commit. diff --git a/include/evo/server/region_sessions.hpp b/include/evo/server/region_sessions.hpp index 616fdf1..ca19e31 100644 --- a/include/evo/server/region_sessions.hpp +++ b/include/evo/server/region_sessions.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -45,6 +46,8 @@ struct RegionSessionsConfig { network::ControlLedger* ledger{}; std::chrono::milliseconds renew_interval{5000}, safety_margin{2000}; network::IntentLimits intents{}; // max_sessions follows control.max_sessions. + // Durable action records (10.4b), owned by the host; null keeps outcomes in memory only. + network::ActionLedger* actions{}; }; // The region host's authority, read once per poll. struct HostFacts { @@ -74,9 +77,12 @@ class RegionSessions { // Actions scheduled for ticks <= `tick`, to be recorded durably with the batch that // simulates that tick (10.4b). Each action is returned once. [[nodiscard]] std::vector carry(std::uint64_t tick); - // A commit covering ticks up to `end_tick` is durable: carried actions become final and - // their outcomes are sent to each account's current session. - void committed(std::uint64_t end_tick, std::uint64_t commit_sequence); + // These carried actions are durable in `commit_sequence` (each with the tick it rode): + // their outcomes go to each account's current session. + void committed(std::span actions, std::uint64_t commit_sequence); + // The host dropped these carried actions: final ones are reported not_committed; the + // others await their durable record (their batch outcome was unknown). + void dropped(std::span actions, bool final); // Closes every connection with `shutdown` (flushed by later polls). void shutdown(std::chrono::steady_clock::time_point now); [[nodiscard]] std::vector take_events(); diff --git a/migrations/0011_region_actions.sql b/migrations/0011_region_actions.sql new file mode 100644 index 0000000..02c6c94 --- /dev/null +++ b/migrations/0011_region_actions.sql @@ -0,0 +1,170 @@ +-- M10.4b durable action records (ADR-0092). An action rides the batch that simulates its +-- tick and becomes durable in the same statement as that batch's checkpoint. The action +-- block is part of the commit's identity: an exact retry must repeat it byte for byte. +-- +-- Action block: N records of 34 bytes, strictly ascending by (tick, account, action id): +-- account uuid (16 raw bytes), action id u64 LE (nonzero), tick u64 LE in +-- (begin_tick, end_tick], kind u16 LE (1 = probe). At most 1024 records. +-- (world, account, action id) is unique across the world: an id is recorded once, by the +-- first commit that carries it. A later batch carrying the same id (an account that moved +-- regions while its action was pending in the old one) commits normally without a second +-- record; the session layer reports outcomes from the record, never from the batch. +-- Committed history stays immutable: each commit's block is kept in its own append-only +-- table (a commit without a row has an empty block). +CREATE TABLE evolution.region_commit_actions ( + commit_id uuid PRIMARY KEY REFERENCES evolution.region_commits(commit_id), + actions bytea NOT NULL CHECK (octet_length(actions) > 0) +); +REVOKE ALL ON evolution.region_commit_actions FROM PUBLIC; + +CREATE TABLE evolution.region_actions ( + world_id evolution.u64 NOT NULL, + account_id uuid NOT NULL CHECK (account_id <> '00000000-0000-0000-0000-000000000000'), + action_id evolution.u64 NOT NULL CHECK (action_id > 0), + region_id evolution.u64 NOT NULL, + tick evolution.u64 NOT NULL CHECK (tick > 0), + kind smallint NOT NULL CHECK (kind = 1), + commit_sequence evolution.u64 NOT NULL CHECK (commit_sequence > 0), + PRIMARY KEY (world_id, account_id, action_id) +); +REVOKE ALL ON evolution.region_actions FROM PUBLIC; + +-- The action block a commit was published with (empty when none). +CREATE FUNCTION evolution.commit_actions(p_commit uuid) +RETURNS bytea LANGUAGE sql STABLE SECURITY INVOKER SET search_path = pg_catalog, pg_temp AS $$ + SELECT coalesce((SELECT a.actions FROM evolution.region_commit_actions a + WHERE a.commit_id = p_commit), ''::bytea); +$$; + +-- Validates an action block against a manifest's tick range; returns the record count. +CREATE FUNCTION evolution.check_action_block(p_actions bytea, p_begin numeric, p_end numeric) +RETURNS integer LANGUAGE plpgsql IMMUTABLE SECURITY INVOKER +SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + n integer := coalesce(octet_length(p_actions), -1); + account bytea; + id numeric; + tick numeric; + kind numeric; + prev_tick numeric := -1; + prev_account bytea := ''::bytea; + prev_id numeric := -1; + seen text[] := '{}'; +BEGIN + IF n < 0 OR n % 34 <> 0 OR n / 34 > 1024 THEN + RAISE EXCEPTION 'invalid action block' USING ERRCODE = '22023'; + END IF; + FOR i IN 0 .. n / 34 - 1 LOOP + account := substring(p_actions FROM 34 * i + 1 FOR 16); + id := evolution.manifest_uint(p_actions, 34 * i + 16, 8); + tick := evolution.manifest_uint(p_actions, 34 * i + 24, 8); + kind := evolution.manifest_uint(p_actions, 34 * i + 32, 2); + -- Canonical order: strictly ascending (tick, account bytes, id); no duplicates. + IF account = decode(repeat('00', 16), 'hex') OR id = 0 OR kind <> 1 + OR tick <= p_begin OR tick > p_end + OR (tick, account, id) <= (prev_tick, prev_account, prev_id) THEN + RAISE EXCEPTION 'invalid action record' USING ERRCODE = '22023'; + END IF; + -- An action id appears once in a block, whatever its tick. + IF encode(account, 'hex') || ':' || id::text = ANY (seen) THEN + RAISE EXCEPTION 'invalid action record' USING ERRCODE = '22023'; + END IF; + seen := seen || (encode(account, 'hex') || ':' || id::text); + prev_tick := tick; + prev_account := account; + prev_id := id; + END LOOP; + RETURN n / 34; +END $$; + +CREATE FUNCTION evolution.publish_region_checkpoint(p_manifest bytea, p_decisions bytea, + p_actions bytea) +RETURNS TABLE (region_frame evolution.u64, head_sequence evolution.u64, head_commit uuid, + head_digest text, head_tick evolution.u64) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +DECLARE + c evolution.checkpoint_manifest := evolution.parse_checkpoint_manifest(p_manifest); + existing boolean; + r record; + n integer; +BEGIN + n := evolution.check_action_block(p_actions, c.begin_tick, c.end_tick); + -- An exact retry repeats the actions too; any difference is a reused identity. + existing := EXISTS (SELECT 1 FROM evolution.region_commits x WHERE x.commit_id = c.commit_id); + IF existing AND evolution.commit_actions(c.commit_id) <> p_actions THEN + RAISE EXCEPTION 'commit identity reused' USING ERRCODE = 'EVL07'; + END IF; + -- The checkpoint itself: lease, head, decisions and exact-retry receipt (schema v5). + SELECT * INTO r FROM evolution.publish_region_checkpoint(p_manifest, p_decisions); + IF NOT FOUND THEN + RAISE EXCEPTION 'publication returned no head' USING ERRCODE = 'XX000'; + END IF; + IF NOT existing THEN -- First publication, not a retry: record the actions. + FOR i IN 0 .. n - 1 LOOP + INSERT INTO evolution.region_actions(world_id, account_id, action_id, region_id, tick, + kind, commit_sequence) + VALUES (c.world_id, encode(substring(p_actions FROM 34 * i + 1 FOR 16), 'hex')::uuid, + evolution.manifest_uint(p_actions, 34 * i + 16, 8), c.region_id, + evolution.manifest_uint(p_actions, 34 * i + 24, 8), 1, c.commit_sequence) + ON CONFLICT (world_id, account_id, action_id) DO NOTHING; + END LOOP; + IF n > 0 THEN + INSERT INTO evolution.region_commit_actions(commit_id, actions) + VALUES (c.commit_id, p_actions); + END IF; + END IF; + RETURN QUERY SELECT r.region_frame, r.head_sequence, r.head_commit, r.head_digest, + r.head_tick; +END $$; + +-- Resolution after an ambiguous publication, including the action block. +CREATE FUNCTION evolution.resolve_region_commit(p_world evolution.u64, p_region evolution.u64, + p_commit uuid, p_manifest bytea, p_decisions bytea, p_actions bytea) +RETURNS TABLE (status text, region_frame evolution.u64, head_sequence evolution.u64, + head_commit uuid, head_digest text, head_tick evolution.u64) +LANGUAGE plpgsql STABLE SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +BEGIN + IF p_actions IS NULL THEN + RAISE EXCEPTION 'invalid resolution request' USING ERRCODE = '22023'; + END IF; + IF EXISTS (SELECT 1 FROM evolution.region_commits x WHERE x.commit_id = p_commit) + AND evolution.commit_actions(p_commit) <> p_actions THEN + RETURN QUERY SELECT 'conflict'::text, NULL::evolution.u64, NULL::evolution.u64, NULL::uuid, + NULL::text, NULL::evolution.u64; + RETURN; + END IF; + RETURN QUERY SELECT * FROM evolution.resolve_region_commit(p_world, p_region, p_commit, + p_manifest, p_decisions); +END $$; + +-- The committed record of one action of an account in the caller's world, under the +-- caller's exact region lease. No row: the action is not committed (and, once the caller's +-- authority is recovered and ready, never was by an earlier epoch). +CREATE FUNCTION evolution.find_region_action(p_world evolution.u64, p_region evolution.u64, + p_holder uuid, p_epoch evolution.u64, p_account uuid, p_action evolution.u64) +RETURNS TABLE (action_region evolution.u64, action_tick evolution.u64, + action_commit evolution.u64) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$ +BEGIN + IF p_account IS NULL OR p_action IS NULL THEN + RAISE EXCEPTION 'invalid action request' USING ERRCODE = '22023'; + END IF; + PERFORM evolution.check_region_lease(p_world, p_region, p_holder, p_epoch); + RETURN QUERY SELECT a.region_id, a.tick, a.commit_sequence FROM evolution.region_actions a + WHERE a.world_id = p_world AND a.account_id = p_account AND a.action_id = p_action; +END $$; + +REVOKE ALL ON FUNCTION evolution.check_action_block(bytea, numeric, numeric), + evolution.commit_actions(uuid), + evolution.publish_region_checkpoint(bytea, bytea, bytea), + evolution.resolve_region_commit(evolution.u64, evolution.u64, uuid, bytea, bytea, bytea), + evolution.find_region_action(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, + evolution.u64) + FROM PUBLIC; +GRANT EXECUTE ON FUNCTION + evolution.publish_region_checkpoint(bytea, bytea, bytea), + evolution.resolve_region_commit(evolution.u64, evolution.u64, uuid, bytea, bytea, bytea), + evolution.find_region_action(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, + evolution.u64) + TO :"runtime_role"; +INSERT INTO evolution.schema_migrations(version, sha256) VALUES (11, :'migration_sha_11'); diff --git a/migrations/apply.sql b/migrations/apply.sql index a3352ad..e77dc58 100644 --- a/migrations/apply.sql +++ b/migrations/apply.sql @@ -45,28 +45,41 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') || ',3:' || :'migration_sha_3' || ',4:' || :'migration_sha_4' || ',5:' || :'migration_sha_5' || ',6:' || :'migration_sha_6' || ',7:' || :'migration_sha_7' || ',8:' || :'migration_sha_8' - || ',9:' || :'migration_sha_9' || ',10:' || :'migration_sha_10' AS at_v10 \gset - \if :at_v10 + || ',9:' || :'migration_sha_9' || ',10:' || :'migration_sha_10' AS at_v10, + :'history' = '1:' || :'migration_sha' || ',2:' || :'migration_sha_2' + || ',3:' || :'migration_sha_3' || ',4:' || :'migration_sha_4' + || ',5:' || :'migration_sha_5' || ',6:' || :'migration_sha_6' + || ',7:' || :'migration_sha_7' || ',8:' || :'migration_sha_8' + || ',9:' || :'migration_sha_9' || ',10:' || :'migration_sha_10' + || ',11:' || :'migration_sha_11' AS at_v11 \gset + \if :at_v11 + \elif :at_v10 + \ir 0011_region_actions.sql \elif :at_v9 \ir 0010_account_control.sql + \ir 0011_region_actions.sql \elif :at_v8 \ir 0009_serialized_authority.sql \ir 0010_account_control.sql + \ir 0011_region_actions.sql \elif :at_v7 \ir 0008_upgrade_barrier.sql \ir 0009_serialized_authority.sql \ir 0010_account_control.sql + \ir 0011_region_actions.sql \elif :at_v6 \ir 0007_region_clones.sql \ir 0008_upgrade_barrier.sql \ir 0009_serialized_authority.sql \ir 0010_account_control.sql + \ir 0011_region_actions.sql \elif :at_v5 \ir 0006_commit_lookup.sql \ir 0007_region_clones.sql \ir 0008_upgrade_barrier.sql \ir 0009_serialized_authority.sql \ir 0010_account_control.sql + \ir 0011_region_actions.sql \elif :at_v4 \ir pre_v5_guard.sql \ir 0005_batch_decisions.sql @@ -75,6 +88,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0008_upgrade_barrier.sql \ir 0009_serialized_authority.sql \ir 0010_account_control.sql + \ir 0011_region_actions.sql \elif :at_v3 \ir 0004_lineage_profile.sql \ir pre_v5_guard.sql @@ -84,6 +98,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0008_upgrade_barrier.sql \ir 0009_serialized_authority.sql \ir 0010_account_control.sql + \ir 0011_region_actions.sql \elif :at_v2 \ir 0003_region_checkpoints.sql \ir 0004_lineage_profile.sql @@ -94,6 +109,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0008_upgrade_barrier.sql \ir 0009_serialized_authority.sql \ir 0010_account_control.sql + \ir 0011_region_actions.sql \elif :at_v1 \ir 0002_region_fencing.sql \ir 0003_region_checkpoints.sql @@ -105,6 +121,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0008_upgrade_barrier.sql \ir 0009_serialized_authority.sql \ir 0010_account_control.sql + \ir 0011_region_actions.sql \else \echo 'Unknown or changed migration history' DO $$ BEGIN RAISE EXCEPTION 'unknown or changed migration history'; END $$; @@ -121,6 +138,7 @@ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname='evolution') \ir 0008_upgrade_barrier.sql \ir 0009_serialized_authority.sql \ir 0010_account_control.sql + \ir 0011_region_actions.sql \endif -- Effective-privilege audit on every run, including no-ops (ADR-0060). Attributes -- alone are insufficient: predefined roles such as pg_write_all_data, PUBLIC @@ -155,7 +173,10 @@ BEGIN ('evolution.load_region_commit(evolution.u64,evolution.u64,evolution.u64)'::regprocedure::oid), ('evolution.claim_account_control(evolution.u64,evolution.u64,uuid,evolution.u64,uuid,evolution.u64,evolution.u64,integer)'::regprocedure::oid), ('evolution.renew_account_control(evolution.u64,evolution.u64,uuid,evolution.u64,uuid,evolution.u64,integer)'::regprocedure::oid), - ('evolution.release_account_control(evolution.u64,evolution.u64,uuid,evolution.u64,uuid,evolution.u64)'::regprocedure::oid) + ('evolution.release_account_control(evolution.u64,evolution.u64,uuid,evolution.u64,uuid,evolution.u64)'::regprocedure::oid), + ('evolution.publish_region_checkpoint(bytea,bytea,bytea)'::regprocedure::oid), + ('evolution.resolve_region_commit(evolution.u64,evolution.u64,uuid,bytea,bytea,bytea)'::regprocedure::oid), + ('evolution.find_region_action(evolution.u64,evolution.u64,uuid,evolution.u64,uuid,evolution.u64)'::regprocedure::oid) ), violations(role_oid, reason) AS ( SELECT r.oid, 'privileged attribute or owner identity' FROM pg_catalog.pg_roles r JOIN closure c ON c.oid = r.oid @@ -179,7 +200,7 @@ BEGIN OR EXISTS (SELECT 1 FROM pg_catalog.pg_namespace n WHERE n.nspname = 'public' AND pg_catalog.has_schema_privilege(c.oid, n.oid, 'CREATE')) UNION ALL - -- Only the fifteen guarded functions are executable; provisioning/internal are not. + -- Only the eighteen guarded functions are executable; provisioning/internal are not. SELECT c.oid, 'EXECUTE outside the fenced function set' FROM closure c CROSS JOIN pg_catalog.pg_proc p WHERE p.pronamespace = 'evolution'::regnamespace diff --git a/schemas/region-actions-v1.md b/schemas/region-actions-v1.md new file mode 100644 index 0000000..21a0ff5 --- /dev/null +++ b/schemas/region-actions-v1.md @@ -0,0 +1,27 @@ +# Region action block, version 1 + +Durable record of the actions that rode one checkpoint batch (ADR-0092, schema v11, +migrations/0011_region_actions.sql). Encoded by `encode_actions` +(include/evo/persistence/region_batch.hpp), checked again by +`evolution.check_action_block`, published by the three-argument +`publish_region_checkpoint` in the same statement as the batch's checkpoint. The block is +not part of the trace or manifest, so simulation evidence and replay are unchanged; it is +part of the commit's identity (an exact retry or resolution must repeat it byte for byte). + +N records of 34 bytes, integers little-endian, N <= 1024: + +| Offset | Size | Field | Rule | +|---|---|---|---| +| 0 | 16 | account | AccountId bytes, not all zero | +| 16 | 8 | action_id | nonzero | +| 24 | 8 | tick | the tick the action rode, in (begin_tick, end_tick] of the batch | +| 32 | 2 | kind | 1 (probe, no effect) | + +Records are strictly ascending by (tick, account bytes, action_id), and an +(account, action_id) pair appears once. An empty block publishes exactly as before v11. + +Each record also becomes a row of `evolution.region_actions` keyed by +(world, account, action_id). The first commit that carries an id records it; a later batch +carrying the same id (an account that moved regions while the action was pending in the +old one) commits normally and adds no second record. `find_region_action` is therefore the +authority for an action's outcome. diff --git a/src/network/protocol/intent_admission.cpp b/src/network/protocol/intent_admission.cpp index 9010593..bdfd551 100644 --- a/src/network/protocol/intent_admission.cpp +++ b/src/network/protocol/intent_admission.cpp @@ -145,28 +145,60 @@ ActionReceipt IntentAdmission::action(std::uint64_t session, const SessionAction r.status = AdmitStatus::outside_window; else if (unresolved_of(*s) >= limits_.max_unresolved) r.status = AdmitStatus::too_many_unresolved; + else if (ledger_) { + // Committed earlier (another epoch, process or region): never admitted again. + const auto found = lookup(account, m.action_id); + if (found.state == Record::unavailable) { + r.status = AdmitStatus::not_ready; // Uniqueness cannot be confirmed now. + } else if (found.state == Record::committed) { + r.status = AdmitStatus::duplicate; + r.tick = found.outcome.tick; + remember(account, found.outcome); + } + } if (r.status == AdmitStatus::admitted) actions_.push_back({{account, m.action_id, m.tick, ActionKind::probe}, Stage::scheduled}); return r; } -ActionOutcome IntentAdmission::query(std::uint64_t session, const ActionQuery& m) const { +ActionOutcome IntentAdmission::query(std::uint64_t session, const ActionQuery& m, + const AdmissionFacts& facts) { ActionOutcome o{m.action_id, ActionResult::unknown, 0, 0}; const auto* s = find(session); if (!s) return o; + const auto account = s->account; const auto pending = std::ranges::find_if(actions_, [&](const Action& a) { - return a.carried.account == s->account && a.carried.action_id == m.action_id; + return a.carried.account == account && a.carried.action_id == m.action_id; }); if (pending != actions_.end()) { o.result = ActionResult::pending; o.tick = pending->carried.tick; + // A batch outcome left unknown by a lost authority is decided by the durable record + // once this authority is ready: every earlier epoch's commits are then final. + if (pending->unknown && ledger_ && facts.ready) + if (const auto found = lookup(account, m.action_id); + found.state != Record::unavailable) { + o = found.state == Record::committed + ? found.outcome + : ActionOutcome{m.action_id, ActionResult::not_committed, 0, 0}; + actions_.erase(pending); + remember(account, o); + } return o; } const auto final = std::ranges::find_if(finals_, [&](const AccountOutcome& f) { - return f.account == s->account && f.outcome.action_id == m.action_id; + return f.account == account && f.outcome.action_id == m.action_id; }); - return final != finals_.end() ? final->outcome : o; + if (final != finals_.end()) + return final->outcome; + // No longer in memory: a committed action is still on record. + if (ledger_ && facts.ready) + if (const auto found = lookup(account, m.action_id); found.state == Record::committed) { + remember(account, found.outcome); + return found.outcome; + } + return o; } std::vector> @@ -206,23 +238,51 @@ void IntentAdmission::remember(const AccountId& account, const ActionOutcome& ou finals_.pop_front(); } -std::vector IntentAdmission::committed(std::uint64_t end_tick, +std::vector IntentAdmission::committed(std::span actions, std::uint64_t commit_sequence) { std::vector out; - std::erase_if(actions_, [&](const Action& a) { - if (a.stage != Stage::carried || a.carried.tick > end_tick) - return false; - const ActionOutcome o{a.carried.action_id, ActionResult::committed_no_effect, - a.carried.tick, commit_sequence}; - out.push_back({a.carried.account, o}); - remember(a.carried.account, o); - return true; - }); + for (const auto& c : actions) { + ActionOutcome o{c.action_id, ActionResult::committed_no_effect, c.tick, commit_sequence}; + // The durable record is the authority: an id carried by two batches (an account that + // moved regions with the action pending) keeps the record of the first commit. + if (ledger_) + if (const auto found = lookup(c.account, c.action_id); found.state == Record::committed) + o = found.outcome; + std::erase_if(actions_, [&](const Action& a) { + return a.carried.account == c.account && a.carried.action_id == c.action_id; + }); + out.push_back({c.account, o}); + remember(c.account, o); + } + return out; +} + +std::vector IntentAdmission::dropped(std::span actions, + bool final) { + std::vector out; + for (const auto& c : actions) { + const auto it = std::ranges::find_if(actions_, [&](const Action& a) { + return a.carried.account == c.account && a.carried.action_id == c.action_id; + }); + if (!final) { + if (it != actions_.end()) + it->unknown = true; + continue; + } + if (it != actions_.end()) + actions_.erase(it); + const ActionOutcome o{c.action_id, ActionResult::not_committed, 0, 0}; + out.push_back({c.account, o}); + remember(c.account, o); + } return out; } std::vector IntentAdmission::abandon() { std::vector out; + for (auto& a : actions_) + if (a.stage == Stage::carried) + a.unknown = true; // Decided by a commit report, a drop report or its record. std::erase_if(actions_, [&](const Action& a) { if (a.stage != Stage::scheduled) return false; @@ -236,6 +296,21 @@ std::vector IntentAdmission::abandon() { return out; } +IntentAdmission::Found IntentAdmission::lookup(const AccountId& account, + std::uint64_t action_id) const { + Found f; + if (!ledger_) + return f; + const auto answer = ledger_->find(account, action_id); + if (!answer.has_value()) + return f; + const auto& record = answer.value(); + f.state = record.has_value() ? Record::committed : Record::absent; + if (record.has_value()) + f.outcome = record.value(); + return f; +} + std::optional IntentAdmission::session_of(const AccountId& account) const { const auto it = std::ranges::find_if(sessions_, [&](const Session& s) { return s.account == account; }); diff --git a/src/persistence/checkpoint_store.cpp b/src/persistence/checkpoint_store.cpp index e42e1b7..0b1ade2 100644 --- a/src/persistence/checkpoint_store.cpp +++ b/src/persistence/checkpoint_store.cpp @@ -49,7 +49,7 @@ publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGra const CommittedCheckpoint& predecessor, const RegionBatch& batch, Uuid commit, const Sha256Digest& build, std::size_t budget, milliseconds timeout, - const PublishHooks& hooks) { + const PublishHooks& hooks, std::span actions) { const auto previous = decode_manifest(predecessor.manifest); if (!previous || previous->profile != Codec::profile || !batch.sealed() || batch.steps().empty() || batch.begin_tick() != previous->end_tick) @@ -81,7 +81,8 @@ publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGra batch.begin_tick(), batch.end_tick()}; auto trace = encode_trace(header, batch.steps()); - if (!decision_bytes || !trace) + auto action_bytes = encode_actions(actions, batch.begin_tick(), batch.end_tick()); + if (!decision_bytes || !trace || !action_bytes) return std::unexpected{fail(PublicationError::candidate)}; const auto payload_digest = sha256(*payload); const auto trace_digest = sha256(*trace); @@ -104,10 +105,10 @@ publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGra if (auto stored = put_verified(objects, *trace, {*trace_digest, trace->size()}); !stored) return std::unexpected{stored.error()}; - PreparedCommit prepared{previous->key, commit, *bytes, *decision_bytes}; + PreparedCommit prepared{previous->key, commit, *bytes, *decision_bytes, *action_bytes}; if (hooks.before_transaction) hooks.before_transaction(); - auto published = publish_region_checkpoint(db, *bytes, *decision_bytes, timeout); + auto published = retry_prepared(db, prepared, timeout); auto committed = confirm(published, std::move(*bytes), *manifest, std::move(prepared)); if (!committed) return std::unexpected{committed.error()}; @@ -128,9 +129,10 @@ publish_environmental_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease, const CommittedCheckpoint& predecessor, const RegionBatch& batch, Uuid commit, const Sha256Digest& build, std::size_t restore_budget, - milliseconds timeout, const PublishHooks& hooks) { + milliseconds timeout, const PublishHooks& hooks, + std::span actions) { return publish_batch(db, objects, lease, predecessor, batch, commit, build, - restore_budget, timeout, hooks); + restore_budget, timeout, hooks, actions); } std::expected recover_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, @@ -152,9 +154,9 @@ publish_lineage_batch(Postgres& db, const FilesystemObjectStore& objects, const const CommittedCheckpoint& predecessor, const RegionBatch& batch, Uuid commit, const Sha256Digest& build, std::size_t restore_budget, milliseconds timeout, - const PublishHooks& hooks) { + const PublishHooks& hooks, std::span actions) { return publish_batch(db, objects, lease, predecessor, batch, commit, build, - restore_budget, timeout, hooks); + restore_budget, timeout, hooks, actions); } std::expected recover_lineage_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, @@ -165,11 +167,18 @@ recover_lineage_checkpoint(Postgres& db, const FilesystemObjectStore& objects, R std::expected resolve_prepared(Postgres& db, const PreparedCommit& prepared, milliseconds timeout) { + if (!prepared.actions.empty()) + return resolve_region_commit(db, prepared.key, prepared.commit, prepared.manifest, + prepared.decisions, prepared.actions, timeout); return resolve_region_commit(db, prepared.key, prepared.commit, prepared.manifest, prepared.decisions, timeout); } std::expected retry_prepared(Postgres& db, const PreparedCommit& prepared, milliseconds timeout) { + // Without actions the schema v5 publication is used, exactly as before v11. + if (!prepared.actions.empty()) + return publish_region_checkpoint(db, prepared.manifest, prepared.decisions, + prepared.actions, timeout); return publish_region_checkpoint(db, prepared.manifest, prepared.decisions, timeout); } } // namespace evo::persistence diff --git a/src/persistence/region_batch.cpp b/src/persistence/region_batch.cpp index 5c0b654..9ea7b52 100644 --- a/src/persistence/region_batch.cpp +++ b/src/persistence/region_batch.cpp @@ -1,5 +1,7 @@ #include "evo/persistence/region_batch.hpp" #include +#include +#include namespace evo::persistence { namespace { @@ -223,4 +225,39 @@ std::vector derive_decisions(std::span steps, } return out; } +std::expected, BatchError> +encode_actions(std::span actions, std::uint64_t begin_tick, + std::uint64_t end_tick) { + if (actions.size() > max_batch_actions) + return std::unexpected{BatchError::format}; + std::vector sorted(actions.begin(), actions.end()); + const auto key = [](const ActionRecord& a) { + return std::tuple{a.tick, a.account, a.action_id}; + }; + std::ranges::sort(sorted, {}, key); + // An action id appears once, whatever its tick. + std::vector> ids; + ids.reserve(sorted.size()); + for (const auto& a : sorted) + ids.emplace_back(a.account, a.action_id); + std::ranges::sort(ids); + if (std::ranges::adjacent_find(ids) != ids.end()) + return std::unexpected{BatchError::format}; + std::vector out; + out.reserve(sorted.size() * action_record_bytes); + for (std::size_t i = 0; i < sorted.size(); ++i) { + const auto& a = sorted[i]; + if (a.account == Uuid{} || a.action_id == 0 || a.kind != 1 || a.tick <= begin_tick || + a.tick > end_tick || (i > 0 && key(sorted[i - 1]) == key(a))) + return std::unexpected{BatchError::format}; + out.insert(out.end(), a.account.begin(), a.account.end()); + for (unsigned b = 0; b < 8; ++b) + out.push_back(static_cast(a.action_id >> (8U * b))); + for (unsigned b = 0; b < 8; ++b) + out.push_back(static_cast(a.tick >> (8U * b))); + out.push_back(static_cast(a.kind)); + out.push_back(static_cast(a.kind >> 8U)); + } + return out; +} } // namespace evo::persistence diff --git a/src/persistence/region_lease_store.cpp b/src/persistence/region_lease_store.cpp index 6fafca9..0903524 100644 --- a/src/persistence/region_lease_store.cpp +++ b/src/persistence/region_lease_store.cpp @@ -281,6 +281,77 @@ resolve_region_commit(Postgres& db, RegionKey key, const Uuid& commit, return ResolvedCommit{Resolution::committed, *head}; } +std::expected +publish_region_checkpoint(Postgres& db, std::span manifest, + std::span decisions, + std::span actions, milliseconds timeout) { + const std::array p{bytea(manifest), bytea(decisions), bytea(actions)}; + auto rows = db.query( + "SELECT * FROM evolution.publish_region_checkpoint($1::bytea, $2::bytea, $3::bytea)", p, + timeout); + if (!rows) + return std::unexpected{classify(rows.error())}; + auto head = parse_head(*rows); + if (!head) + return std::unexpected{malformed(true)}; + return *head; +} + +std::expected +resolve_region_commit(Postgres& db, RegionKey key, const Uuid& commit, + std::span manifest, + std::span decisions, + std::span actions, milliseconds timeout) { + const std::array p{std::to_string(key.world), std::to_string(key.region), + uuid_text(commit), bytea(manifest), + bytea(decisions), bytea(actions)}; + auto rows = db.query("SELECT * FROM evolution.resolve_region_commit($1::evolution.u64, " + "$2::evolution.u64, $3::uuid, $4::bytea, $5::bytea, $6::bytea)", + p, timeout); + if (!rows) + return std::unexpected{classify(rows.error())}; + if (rows->size() != 1 || (*rows)[0].size() != 6) + return std::unexpected{malformed(false)}; + const auto& status_text = (*rows)[0][0]; + if (!status_text) + return std::unexpected{malformed(false)}; + const auto status = *status_text; + if (status == "absent") + return ResolvedCommit{Resolution::absent, {}}; + if (status == "conflict") + return ResolvedCommit{Resolution::conflict, {}}; + const PgRows receipt{ + std::vector>((*rows)[0].begin() + 1, (*rows)[0].end())}; + auto head = status == "committed" ? parse_head(receipt) : std::nullopt; + if (!head) + return std::unexpected{malformed(false)}; + return ResolvedCommit{Resolution::committed, *head}; +} + +std::expected, LeaseStoreFailure> +find_region_action(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& account, + std::uint64_t action_id, milliseconds timeout) { + const std::array p{std::to_string(key.world), std::to_string(key.region), + uuid_text(lease.holder), std::to_string(lease.epoch), + uuid_text(account), std::to_string(action_id)}; + auto rows = db.query("SELECT * FROM evolution.find_region_action($1::evolution.u64, " + "$2::evolution.u64, $3::uuid, $4::evolution.u64, $5::uuid, " + "$6::evolution.u64)", + p, timeout); + if (!rows) + return std::unexpected{classify(rows.error())}; + if (rows->empty()) + return std::optional{}; + if (rows->size() != 1 || (*rows)[0].size() != 3) + return std::unexpected{malformed(false)}; + const auto region = parse_integer((*rows)[0][0]); + const auto tick = parse_integer((*rows)[0][1]); + const auto sequence = parse_integer((*rows)[0][2]); + if (!region || !tick || !sequence) + return std::unexpected{malformed(false)}; + return std::optional{CommittedAction{*region, *tick, *sequence}}; +} + namespace { std::expected, LeaseStoreFailure> decisions_from(const PgRows& rows) { diff --git a/src/server/region_host.cpp b/src/server/region_host.cpp index 1135ac3..7ea898a 100644 --- a/src/server/region_host.cpp +++ b/src/server/region_host.cpp @@ -100,6 +100,27 @@ std::expected RegionHost::submit(const TickInput& i return {}; } +template +std::expected +RegionHost::submit_action(const persistence::ActionRecord& record) { + if (record.account == persistence::Uuid{} || record.action_id == 0 || record.kind != 1 || + record.tick == 0) + return std::unexpected{AdmissionError::invalid}; + if (draining_) + return std::unexpected{AdmissionError::draining}; + if (!status_.ready) + return std::unexpected{AdmissionError::not_ready}; + const auto same = [&](const persistence::ActionRecord& a) { + return a.account == record.account && a.action_id == record.action_id; + }; + if (std::ranges::any_of(action_queue_, same) || std::ranges::any_of(batch_actions_, same)) + return std::unexpected{AdmissionError::duplicate}; + if (action_queue_.size() + batch_actions_.size() >= config_.max_queued_actions) + return std::unexpected{AdmissionError::full}; + action_queue_.push_back(record); + return {}; +} + template bool RegionHost::database() { if (db_) return true; @@ -118,6 +139,13 @@ void RegionHost::lose_authority(Unacknowledged reason, HostPhase next) { dropped_.push_back({a.input, reason}); for (auto& q : queue_) dropped_.push_back({q, reason}); + // Actions of a batch whose publication outcome is unknown may be durable: not final. + for (const auto& a : batch_actions_) + dropped_actions_.push_back({a, !prepared_.has_value(), reason}); + for (const auto& a : action_queue_) + dropped_actions_.push_back({a, true, reason}); + batch_actions_.clear(); + action_queue_.clear(); applied_.clear(); queue_.clear(); batch_.reset(); // References state_: reset first. @@ -248,6 +276,9 @@ void RegionHost::confirmed(const persistence::PublishedBatch& published, applied_.clear(); for (const auto& d : published.decisions) outcomes_.push_back({d, head.sequence}); + for (const auto& a : batch_actions_) + action_acks_.push_back({a, head.sequence}); + batch_actions_.clear(); prepared_.reset(); batch_started_ = now; retry_at_ = {}; @@ -272,9 +303,9 @@ template void RegionHost::commit(SchedulerTime now) { persistence::PublishHooks hooks; if (crash) hooks.before_transaction = [&crash] { crash(CrashPoint::before_transaction); }; - auto published = - Ops::publish(*db_, *objects_, *grant, *committed_, *batch_, commit_id, config_.build, - config_.restore_budget, config_.lease.commit_timeout, hooks); + auto published = Ops::publish(*db_, *objects_, *grant, *committed_, *batch_, commit_id, + config_.build, config_.restore_budget, + config_.lease.commit_timeout, hooks, batch_actions_); if (published) { confirmed(*published, now); return; @@ -387,6 +418,14 @@ template void RegionHost::simulate(SchedulerTime now) { continue; } applied_.push_back({input, report->tick}); + // Actions due at or before this tick ride it (late ones ride the first tick run). + std::erase_if(action_queue_, [&](persistence::ActionRecord action) { + if (action.tick > report->tick) + return false; + action.tick = report->tick; + batch_actions_.push_back(action); + return true; + }); } } @@ -482,6 +521,9 @@ template void RegionHost::drain() { for (auto& q : queue_) dropped_.push_back({q, Unacknowledged::drained}); queue_.clear(); + for (const auto& a : action_queue_) + dropped_actions_.push_back({a, true, Unacknowledged::drained}); + action_queue_.clear(); if (status_.phase == HostPhase::running) status_.phase = HostPhase::draining; else if (status_.phase == HostPhase::acquiring) @@ -498,6 +540,12 @@ template std::vector RegionHost::take_outc template std::vector RegionHost::take_dropped() { return std::exchange(dropped_, {}); } +template std::vector RegionHost::take_action_acks() { + return std::exchange(action_acks_, {}); +} +template std::vector RegionHost::take_dropped_actions() { + return std::exchange(dropped_actions_, {}); +} template class RegionHost; template class RegionHost; diff --git a/src/server/session/postgres_action_ledger.cpp b/src/server/session/postgres_action_ledger.cpp new file mode 100644 index 0000000..41b3f0f --- /dev/null +++ b/src/server/session/postgres_action_ledger.cpp @@ -0,0 +1,20 @@ +#include "evo/server/postgres_action_ledger.hpp" + +namespace evo::server { +std::expected, network::ActionLedgerError> +PostgresActionLedger::find(const network::AccountId& account, std::uint64_t action_id) { + const auto lease = lease_(); + if (!lease) + return std::unexpected(network::ActionLedgerError::unavailable); + const auto found = + persistence::find_region_action(*db_, key_, *lease, account.bytes, action_id, timeout_); + if (!found.has_value()) + return std::unexpected(network::ActionLedgerError::unavailable); + const auto& record = found.value(); + if (!record.has_value()) + return std::optional{}; + return std::optional{ + network::ActionOutcome{action_id, network::ActionResult::committed_no_effect, + record.value().tick, record.value().commit_sequence}}; +} +} // namespace evo::server diff --git a/src/server/session/region_sessions.cpp b/src/server/session/region_sessions.cpp index cd581c0..261170d 100644 --- a/src/server/session/region_sessions.cpp +++ b/src/server/session/region_sessions.cpp @@ -282,14 +282,14 @@ struct RegionSessions::Impl { send(c, receipt); // A duplicate of a final action: answer with the outcome as well. if (receipt.status == network::AdmitStatus::duplicate) - if (const auto o = intents.query(c.session, {m->action_id}); + if (const auto o = intents.query(c.session, {m->action_id}, admission()); o.result == network::ActionResult::committed_no_effect || o.result == network::ActionResult::not_committed) send(c, o); return; } if (const auto* m = std::get_if(&*decoded)) - return send(c, intents.query(c.session, *m)); + return send(c, intents.query(c.session, *m, admission())); if (const auto* m = std::get_if(&*decoded); m && m->reason == SessionCloseReason::client_closed) { close(c, SessionCloseReason::client_closed, now); @@ -324,6 +324,7 @@ RegionSessions::create(RegionSessionsConfig config) { auto intents = network::IntentAdmission::create(config.intents); if (!intents) return std::unexpected(RegionSessionsError::configuration); + intents->attach(config.actions); auto endpoint = network::quic::Endpoint::serve(config.transport); if (!endpoint) return std::unexpected(RegionSessionsError::transport); @@ -407,8 +408,13 @@ std::vector RegionSessions::carry(std::uint64_t tick) { return impl_->intents.carry(tick); } -void RegionSessions::committed(std::uint64_t end_tick, std::uint64_t commit_sequence) { - impl_->deliver(impl_->intents.committed(end_tick, commit_sequence)); +void RegionSessions::committed(std::span actions, + std::uint64_t commit_sequence) { + impl_->deliver(impl_->intents.committed(actions, commit_sequence)); +} + +void RegionSessions::dropped(std::span actions, bool final) { + impl_->deliver(impl_->intents.dropped(actions, final)); } void RegionSessions::shutdown(Clock::time_point now) { diff --git a/tests/network_control.cpp b/tests/network_control.cpp index b427feb..a2c8efc 100644 --- a/tests/network_control.cpp +++ b/tests/network_control.cpp @@ -412,6 +412,85 @@ void session_codec() { CHECK(refused(decode_session(encode_session(w)), SessionCodecError::invalid_value)); } } +// Scripted durable action records (10.4b). +struct FakeActions final : ActionLedger { + std::expected, ActionLedgerError> + find(const AccountId& who, std::uint64_t action_id) override { + ++lookups; + if (unavailable) + return std::unexpected(ActionLedgerError::unavailable); + for (const auto& [account, outcome] : records) + if (account == who && outcome.action_id == action_id) + return std::optional{outcome}; + return std::optional{}; + } + std::vector> records; + bool unavailable{}; + int lookups{}; +}; + +void durable_actions() { + auto a = *IntentAdmission::create(IntentLimits{}); + FakeActions ledger; + a.attach(&ledger); + const auto alice = account(1); + CHECK(a.open(1, alice)); + AdmissionFacts f{true, 5, 100}; + const auto action = [&](std::uint64_t id, std::uint64_t tick) { + return a.action(1, SessionAction{1, 5, id, 1, tick}, f); + }; + // Committed by an earlier epoch, process or region: a duplicate with its outcome. + const ActionOutcome old{9, ActionResult::committed_no_effect, 40, 3}; + ledger.records.emplace_back(alice, old); + auto r = action(9, 101); + CHECK(r.status == AdmitStatus::duplicate && r.tick == 40); + CHECK(a.query(1, {9}, f) == old); + // Uniqueness that cannot be confirmed is not admitted. + ledger.unavailable = true; + CHECK(action(10, 101).status == AdmitStatus::not_ready); + ledger.unavailable = false; + const int before = ledger.lookups; + CHECK(action(10, 101).status == AdmitStatus::admitted && ledger.lookups == before + 1); + // A refused attempt never costs a lookup. + CHECK(action(11, 150).status == AdmitStatus::outside_window && ledger.lookups == before + 1); + // Exact commit: the tick reported is the tick the action rode. + auto riding = a.carry(101); + CHECK(riding.size() == 1); + riding[0].tick = 102; + auto outcomes = a.committed(riding, 6); + CHECK(outcomes.size() == 1 && + outcomes[0].outcome == ActionOutcome{10, ActionResult::committed_no_effect, 102, 6}); + // A final drop is not_committed; an unknown one stays pending until its record decides. + f.server_tick = 108; + CHECK(action(12, 109).status == AdmitStatus::admitted); + CHECK(action(13, 109).status == AdmitStatus::admitted); + riding = a.carry(109); + CHECK(riding.size() == 2); + outcomes = a.dropped(std::span{riding}.first(1), true); + CHECK(outcomes.size() == 1 && outcomes[0].outcome.result == ActionResult::not_committed); + CHECK(a.dropped(std::span{riding}.subspan(1), false).empty()); + f.ready = false; // Recovering: nothing is decided yet. + CHECK(a.query(1, {13}, f).result == ActionResult::pending); + f.ready = true; + ledger.records.emplace_back(alice, + ActionOutcome{13, ActionResult::committed_no_effect, 109, 7}); + CHECK(a.query(1, {13}, f) == ActionOutcome{13, ActionResult::committed_no_effect, 109, 7}); + // The same for an action whose record is absent once the new authority is ready. + f.server_tick = 116; + CHECK(action(14, 117).status == AdmitStatus::admitted); + riding = a.carry(117); + CHECK(a.dropped(riding, false).empty()); + CHECK(a.query(1, {14}, f) == ActionOutcome{14, ActionResult::not_committed, 0, 0}); + // Outcomes evicted from memory are found again on record. + auto small = IntentLimits{}; + small.max_finals = 1; + auto b = *IntentAdmission::create(small); + b.attach(&ledger); + CHECK(b.open(1, alice)); + CHECK(b.query(1, {13}, f).result == ActionResult::committed_no_effect); + CHECK(b.query(1, {99}, f).result == ActionResult::unknown); +} + void intent_admission() { IntentLimits limits; limits.max_unresolved = 6; @@ -465,7 +544,7 @@ void intent_admission() { f.server_tick = 200; auto r = action(1, 10, 203); CHECK(r.status == AdmitStatus::admitted && r.tick == 203 && r.server_tick == 200); - CHECK(a.query(1, {10}).result == ActionResult::pending); + CHECK(a.query(1, {10}, f).result == ActionResult::pending); CHECK(action(1, 0, 203).status == AdmitStatus::invalid_action); CHECK(action(1, 11, 203, 9).status == AdmitStatus::invalid_action); { // Wire kind 257 must not truncate to a known kind. @@ -481,23 +560,23 @@ void intent_admission() { const auto carried = a.carry(203); CHECK(carried.size() == 1 && carried[0] == CarriedAction{alice, 10, 203, ActionKind::probe}); CHECK(a.carry(203).empty()); // Each action is carried once. - CHECK(a.committed(202, 7).empty()); - auto outcomes = a.committed(203, 8); + CHECK(a.committed({}, 7).empty()); + auto outcomes = a.committed(carried, 8); CHECK(outcomes.size() == 1 && outcomes[0].account == alice && outcomes[0].outcome == ActionOutcome{10, ActionResult::committed_no_effect, 203, 8}); - CHECK(a.query(1, {10}) == outcomes[0].outcome); - CHECK(a.query(2, {10}).result == ActionResult::unknown); // Bob's account has no action 10. + CHECK(a.query(1, {10}, f) == outcomes[0].outcome); + CHECK(a.query(2, {10}, f).result == ActionResult::unknown); // Bob's account has no action 10. // A reconnect of the same account (new session) sees the same outcome. a.close(1); - CHECK(a.query(1, {10}).result == ActionResult::unknown); // Closed session asks nothing. + CHECK(a.query(1, {10}, f).result == ActionResult::unknown); // Closed session asks nothing. CHECK(a.open(3, alice) && a.session_of(alice) == 3u); - CHECK(a.query(3, {10}) == outcomes[0].outcome); + CHECK(a.query(3, {10}, f) == outcomes[0].outcome); f.server_tick = 300; CHECK(action(3, 10, 301).status == AdmitStatus::duplicate); // Never admitted twice. // Unresolved cap counts intentions and actions; abandon finalizes only uncarried actions. CHECK(action(3, 20, 301).status == AdmitStatus::admitted); CHECK(action(3, 21, 308).status == AdmitStatus::admitted); - (void)a.carry(301); + const auto riding = a.carry(301); for (std::uint64_t i = 0; i < 4; ++i) CHECK(a.intention(3, SessionIntention{3, 5, i + 1, 302 + i, 0, 0}, f).status == AdmitStatus::admitted); @@ -507,18 +586,17 @@ void intent_admission() { outcomes = a.abandon(); CHECK(outcomes.size() == 1 && outcomes[0].outcome == ActionOutcome{21, ActionResult::not_committed, 0, 0}); - CHECK(a.query(3, {20}).result == ActionResult::pending); // Carried: awaits its record. + CHECK(a.query(3, {20}, f).result == ActionResult::pending); // Carried: awaits its record. CHECK(a.unresolved(3) == 1); - CHECK(a.committed(301, 9).size() == 1); // Its batch turned out durable. + CHECK(a.committed(riding, 9).size() == 1); // Its batch turned out durable. // Final outcomes are kept within a bound; older ones become unknown here. for (std::uint64_t id = 30; id < 34; ++id) { f.server_tick += 8; CHECK(action(3, id, f.server_tick + 1).status == AdmitStatus::admitted); - (void)a.carry(f.server_tick + 1); - CHECK(a.committed(f.server_tick + 1, id).size() == 1); + CHECK(a.committed(a.carry(f.server_tick + 1), id).size() == 1); } - CHECK(a.query(3, {10}).result == ActionResult::unknown); - CHECK(a.query(3, {33}).result == ActionResult::committed_no_effect); + CHECK(a.query(3, {10}, f).result == ActionResult::unknown); + CHECK(a.query(3, {33}, f).result == ActionResult::committed_no_effect); } } // namespace @@ -528,5 +606,6 @@ void control_tests() { control_rules(); session_codec(); intent_admission(); + durable_actions(); } } // namespace evo::network_test diff --git a/tests/postgres_tests.cpp b/tests/postgres_tests.cpp index baf6329..49bff9d 100644 --- a/tests/postgres_tests.cpp +++ b/tests/postgres_tests.cpp @@ -5,6 +5,7 @@ #include void run_region_lease_store_tests(); void run_account_control_store_tests(); +void run_region_action_tests(); void run_region_checkpoint_store_tests(); void run_region_host_tests(); void run_region_server_process_tests(); @@ -118,6 +119,7 @@ int main() { std::puts("postgres adapter integration passed"); run_region_lease_store_tests(); run_account_control_store_tests(); + run_region_action_tests(); std::puts("region lease fencing integration passed"); run_region_checkpoint_store_tests(); std::puts("region checkpoint publication integration passed"); diff --git a/tests/quic_sessions.cpp b/tests/quic_sessions.cpp index 94e9d49..11955f6 100644 --- a/tests/quic_sessions.cpp +++ b/tests/quic_sessions.cpp @@ -406,7 +406,7 @@ void intentions_and_actions(const Files& f) { CHECK(h.server->carry(102).empty()); const auto carried = h.server->carry(103); CHECK(carried.size() == 1 && carried[0] == CarriedAction{alice, 7, 103, ActionKind::probe}); - h.server->committed(103, 4); + h.server->committed(carried, 4); CHECK(h.until([&] { return !(outcomes = a->take_action_outcomes()).empty(); })); CHECK(outcomes[0] == ActionOutcome{7, ActionResult::committed_no_effect, 103, 4}); // A retry gets a duplicate receipt and the same final outcome. diff --git a/tests/region_action_tests.cpp b/tests/region_action_tests.cpp new file mode 100644 index 0000000..bf519a1 --- /dev/null +++ b/tests/region_action_tests.cpp @@ -0,0 +1,412 @@ +// Real PostgreSQL M10.4b durable action records (ADR-0092, schema v11): actions committed in +// the same statement as their batch's checkpoint, exact retries and resolution including +// the action block, invalid blocks, an action id never committed twice, and host crashes +// before and after the commit. Regions 7/131..135 and commit ids tagged 0xA7 are used only +// here. Run by +// tools/ci/postgres-check.sh. +#include "evo/persistence/checkpoint_store.hpp" +#include "evo/persistence/region_lease_store.hpp" +#include "evo/server/region_host.hpp" +#include "fixtures/environmental_region.hpp" +#ifdef EVO_POSTGRES_SESSIONS +#include "evo/server/postgres_action_ledger.hpp" +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include + +void run_region_action_tests(); + +namespace { +using namespace evo::persistence; +using namespace evo::server; +using namespace std::chrono_literals; +using evo::processes::LineageRegion; +namespace fs = std::filesystem; +void check(bool value, std::source_location at = std::source_location::current()) { + if (!value) + throw std::runtime_error("region action test line " + std::to_string(at.line())); +} +#define CHECK(...) check(static_cast(__VA_ARGS__)) +template bool holds(const std::expected& e, const T& v) { + return e.has_value() && *e == v; +} +constexpr std::uint64_t world = 7; // The fixtures' world. +constexpr std::uint64_t base_region = 130; +constexpr std::size_t budget = 8000000; + +std::string env(const char* key) { + const auto* value = std::getenv(key); + if (!value) + throw std::runtime_error("missing test environment"); + return value; +} +PgOptions options(const std::string& user) { + return {env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}}; +} +std::unique_ptr connect_as(const std::string& user) { + auto made = Postgres::connect(options(user), 5s); + CHECK(made); + return std::move(*made); +} +template std::array filled(std::uint8_t value) { + std::array out{}; + out.fill(value); + return out; +} +const Sha256Digest build = filled<32>(0xB1); +const Uuid alice = filled<16>(0xA1), bob = filled<16>(0xB0); +const Uuid carol = filled<16>(0xC1), dave = filled<16>(0xD0); // Host tests only. +// Commit ids are unique across the database: tag this file's. +Uuid tagged(std::uint8_t n) { + Uuid id = filled<16>(0xA7); + id.back() = n; + return id; +} + +struct Region { + RegionKey key; + fs::path root; + CommittedCheckpoint genesis; +}; +Region provisioned(std::uint64_t region) { + region += base_region; + Region r{{world, region}, fs::path(env("PGHOST")) / ("actions-" + std::to_string(region)), {}}; + fs::remove_all(r.root); + fs::create_directory(r.root); + fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace); + auto owner = connect_as(env("PGUSER")); + CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " + "(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING", + {}, 5s)); + auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes); + CHECK(objects); + auto genesis = provision_lineage_region( + *owner, **objects, r.key, evo::test::region_identity.frame, *evo::test::lineage_region(), + tagged(static_cast(region)), build, budget, 5s); + if (!genesis) + std::fprintf(stderr, "provision failed: kind %d sqlstate %s\n", + static_cast(genesis.error().kind), + genesis.error().store.cause.sqlstate.data()); + CHECK(genesis); + r.genesis = *genesis; + return r; +} +std::vector record(const Uuid& account, std::uint64_t id, std::uint64_t tick, + std::uint16_t kind = 1) { + std::vector out(account.begin(), account.end()); + for (unsigned b = 0; b < 8; ++b) + out.push_back(static_cast(id >> (8U * b))); + for (unsigned b = 0; b < 8; ++b) + out.push_back(static_cast(tick >> (8U * b))); + out.push_back(static_cast(kind)); + out.push_back(static_cast(kind >> 8U)); + return out; +} +std::vector join(std::initializer_list> parts) { + std::vector out; + for (const auto& p : parts) + out.insert(out.end(), p.begin(), p.end()); + return out; +} +// Owner read of the durable record: (region, tick, commit) or empty. +std::string stored(Postgres& owner, const Uuid& account, std::uint64_t id) { + const std::array p{std::string{"\\x"} + + [&] { + std::string h; + constexpr char digits[] = "0123456789abcdef"; + for (const auto b : account) { + h.push_back(digits[b >> 4U]); + h.push_back(digits[b & 0xFU]); + } + return h; + }(), + std::to_string(id)}; + auto rows = owner.query("SELECT region_id || ':' || tick || ':' || commit_sequence FROM " + "evolution.region_actions WHERE world_id = 7 AND account_id = " + "encode($1::bytea, 'hex')::uuid AND action_id = $2::evolution.u64", + p, 5s); + CHECK(rows); + return rows->empty() ? std::string{} : (*rows)[0][0].value_or(""); +} + +void store_rules() { + auto r = provisioned(1); + auto db = connect_as(env("EVO_PG_RUNTIME_USER")); + auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes); + CHECK(objects); + auto lease = acquire_region_lease(*db, r.key, build, 20000ms, 5s); + CHECK(lease); + auto state = evo::test::lineage_region(); + RegionBatch batch{*state}; + for (std::uint64_t tick = 1; tick <= 10; ++tick) + CHECK(batch.advance(evo::test::region_input(tick, false))); + batch.seal(); + const std::vector actions{{alice, 2, 10, 1}, {bob, 1, 3, 1}, {alice, 1, 3, 1}}; + auto first = publish_lineage_batch(*db, **objects, *lease, r.genesis, batch, tagged(0x01), + build, budget, 5s, {}, actions); + if (!first) + std::fprintf(stderr, "publish failed: kind %d store %d sqlstate %s\n", + static_cast(first.error().kind), + static_cast(first.error().store.kind), + first.error().store.cause.sqlstate.data()); + CHECK(first && first->committed.head.sequence == 1); + CHECK(batch.confirm(10)); + auto found = find_region_action(*db, r.key, *lease, alice, 1, 5s); + CHECK(found && *found == CommittedAction{131, 3, 1}); + CHECK(holds(find_region_action(*db, r.key, *lease, bob, 1, 5s), + std::optional{CommittedAction{131, 3, 1}})); + CHECK(holds(find_region_action(*db, r.key, *lease, alice, 2, 5s), + std::optional{CommittedAction{131, 10, 1}})); + CHECK(holds(find_region_action(*db, r.key, *lease, alice, 3, 5s), + std::optional{})); + + // Exact retry and resolution include the action block. + const auto block = encode_actions(actions, 0, 10); + CHECK(block); + PreparedCommit prepared{r.key, tagged(0x01), first->committed.manifest, {}, *block}; + CHECK(retry_prepared(*db, prepared, 5s) == first->committed.head); + auto resolved = resolve_prepared(*db, prepared, 5s); + CHECK(resolved && resolved->status == Resolution::committed); + auto changed = prepared; + changed.actions = join({record(alice, 1, 3), record(alice, 2, 10)}); // One missing. + auto retried = retry_prepared(*db, changed, 5s); + CHECK(!retried && retried.error().kind == LeaseStoreError::commit_conflict); + resolved = resolve_prepared(*db, changed, 5s); + CHECK(resolved && resolved->status == Resolution::conflict); + // Every invalid block is refused before anything else. + for (const auto& bad : + {record(alice, 9, 11), record(alice, 9, 0), record(alice, 9, 5, 2), record(Uuid{}, 9, 5), + record(alice, 0, 5), join({record(bob, 9, 5), record(alice, 9, 5)}), + join({record(alice, 9, 5), record(alice, 9, 5)}), + join({record(alice, 9, 6), record(alice, 9, 5)}), + join({record(alice, 9, 5), record(alice, 9, 6)}), // One id at two ticks. + std::vector(33, 1)}) { + auto p = prepared; + p.actions = bad; + retried = retry_prepared(*db, p, 5s); + CHECK(!retried && retried.error().kind == LeaseStoreError::invalid); + } + // The local encoder refuses the same blocks it would send. + CHECK(!encode_actions(std::vector{{alice, 9, 11, 1}}, 0, 10)); + CHECK(!encode_actions(std::vector{{alice, 9, 5, 1}, {alice, 9, 6, 1}}, 0, 10)); + + // An action id is recorded once in a world: a later batch carrying it commits, and the + // first record stands. + RegionBatch next{*state}; + for (std::uint64_t tick = 11; tick <= 20; ++tick) + CHECK(next.advance(evo::test::region_input(tick, false))); + next.seal(); + const std::vector repeat{{alice, 1, 12, 1}, {alice, 3, 12, 1}}; + auto second = publish_lineage_batch(*db, **objects, *lease, first->committed, next, + tagged(0x03), build, budget, 5s, {}, repeat); + CHECK(second && second->committed.head.sequence == 2); + CHECK(holds(find_region_action(*db, r.key, *lease, alice, 1, 5s), + std::optional{CommittedAction{131, 3, 1}})); + // A batch without actions still publishes as before v11. + CHECK(next.confirm(20)); + RegionBatch plain{*state}; + for (std::uint64_t tick = 21; tick <= 25; ++tick) + CHECK(plain.advance(evo::test::region_input(tick, false))); + plain.seal(); + CHECK(publish_lineage_batch(*db, **objects, *lease, second->committed, plain, tagged(0x04), + build, budget, 5s)); + // Lookups are fenced like everything else under the lease. + CHECK(release_region_lease(*db, r.key, *lease, 5s)); + auto later = acquire_region_lease(*db, r.key, build, 20000ms, 5s); + CHECK(later); + auto fenced = find_region_action(*db, r.key, *lease, alice, 1, 5s); + CHECK(!fenced && fenced.error().kind == LeaseStoreError::fenced); + CHECK(holds(find_region_action(*db, r.key, *later, alice, 3, 5s), + std::optional{CommittedAction{131, 12, 2}})); +#ifdef EVO_POSTGRES_SESSIONS + // The session layer's ledger over the same lookup. + std::optional current = *later; + PostgresActionLedger ledger{*db, r.key, [&] { return current; }}; + evo::network::AccountId who; + who.bytes = alice; + CHECK(holds(ledger.find(who, 3), + std::optional{evo::network::ActionOutcome{ + 3, evo::network::ActionResult::committed_no_effect, 12, 2}})); + CHECK(holds(ledger.find(who, 99), std::optional{})); + current.reset(); + CHECK(!ledger.find(who, 3)); +#endif + CHECK(release_region_lease(*db, r.key, *later, 5s)); +} + +// Host-level: fast lease (4 s), 1 s commits, as in the host tests. +const LeasePolicy fast{4000ms, 800ms, 400ms, 400ms, 1000ms}; +struct Crash {}; +HostConfig config(const Region& r) { + HostConfig c; + c.key = r.key; + c.frame = evo::test::region_identity.frame; + c.lease = fast; + c.build = build; + c.retained_builds = {build}; + c.max_queued_actions = 3; + return c; +} +std::unique_ptr> host(const Region& r, HostConfig c) { + auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes); + CHECK(objects); + auto made = RegionHost::create( + std::move(c), [] { return Postgres::connect(options(env("EVO_PG_RUNTIME_USER")), 2s); }, + std::move(*objects)); + CHECK(made); + return std::move(*made); +} +TickInput input(std::uint64_t tick, std::uint8_t tag) { + Uuid request{}; + request[0] = tag; + for (unsigned i = 0; i < 8; ++i) + request[8 + i] = static_cast((tick >> (8U * i)) & 0xFFU); + return {request, evo::test::region_input(tick, false)}; +} +template +const HostStatus& step_until(RegionHost& h, SchedulerTime& t, Done done, + int limit = 1000) { + for (int i = 0; i < limit; ++i) { + t += 50ms; + const auto& s = h.step(t); + if (done(s)) + return s; + std::this_thread::sleep_for(10ms); + } + CHECK(false); + return h.status(); +} + +void host_actions() { + auto owner = connect_as(env("PGUSER")); + { + // Normal path: acknowledged with the tick each rode; late actions ride the next tick. + const auto r = provisioned(2); + auto h = host(r, config(r)); + CHECK(h->submit_action({carol, 1, 3, 1}).error() == AdmissionError::not_ready); + SchedulerTime t = std::chrono::steady_clock::now(); + CHECK(h->step(t).ready); + CHECK(h->submit_action({carol, 1, 3, 1})); + CHECK(h->submit_action({carol, 1, 4, 1}).error() == AdmissionError::duplicate); + CHECK(h->submit_action({carol, 0, 4, 1}).error() == AdmissionError::invalid); + CHECK(h->submit_action({carol, 5, 4, 2}).error() == AdmissionError::invalid); + CHECK(h->submit_action({dave, 1, 1, 1})); + CHECK(h->submit_action({dave, 2, 2, 1})); + CHECK(h->submit_action({dave, 3, 2, 1}).error() == AdmissionError::full); + for (std::uint64_t tick = 1; tick <= 30; ++tick) + CHECK(h->submit(input(tick, 0x71))); + step_until(*h, t, [](const HostStatus& s) { return s.committed_sequence >= 1; }); + auto acks = h->take_action_acks(); + CHECK(acks.size() == 3); + for (const auto& a : acks) + CHECK(a.commit_sequence == 1 && + a.record.tick == (a.record.account == dave ? a.record.action_id : 3u)); + CHECK(stored(*owner, carol, 1) == "132:3:1"); + // Submitted after tick 20 ran for tick 5: rides the first tick simulated next. + const auto ran = h->status().simulated_tick; + CHECK(h->submit_action({carol, 7, 5, 1})); + step_until(*h, t, [](const HostStatus& s) { return s.committed_sequence >= 2; }); + acks = h->take_action_acks(); + CHECK(acks.size() == 1 && acks[0].record.tick == ran + 1); + CHECK(h->take_dropped_actions().empty()); + } + { + // Crash before the transaction: never committed; a successor finds no record. + const auto r = provisioned(3); + bool armed = false; + auto c = config(r); + c.crash_hook = [&](CrashPoint p) { + if (armed && p == CrashPoint::before_transaction) + throw Crash{}; + }; + auto h = host(r, c); + SchedulerTime t = std::chrono::steady_clock::now(); + CHECK(h->step(t).ready); + for (std::uint64_t tick = 1; tick <= 30; ++tick) + CHECK(h->submit(input(tick, 0x72))); + CHECK(h->submit_action({carol, 10, 5, 1})); + armed = true; + bool crashed = false; + try { + step_until(*h, t, [](const HostStatus& s) { return s.committed_sequence >= 1; }); + } catch (const Crash&) { + crashed = true; + } + CHECK(crashed); + h.reset(); + auto successor = host(r, config(r)); + step_until(*successor, t, [](const HostStatus& s) { return s.ready; }); + CHECK(successor->status().committed_sequence == 0 && stored(*owner, carol, 10).empty()); + } + { + // Crash after the commit, before any acknowledgement: the record is durable. + const auto r = provisioned(4); + bool armed = false; + auto c = config(r); + c.crash_hook = [&](CrashPoint p) { + if (armed && p == CrashPoint::after_commit) + throw Crash{}; + }; + auto h = host(r, c); + SchedulerTime t = std::chrono::steady_clock::now(); + CHECK(h->step(t).ready); + for (std::uint64_t tick = 1; tick <= 30; ++tick) + CHECK(h->submit(input(tick, 0x73))); + CHECK(h->submit_action({carol, 11, 5, 1})); + armed = true; + bool crashed = false; + try { + step_until(*h, t, [](const HostStatus& s) { return s.committed_sequence >= 1; }); + } catch (const Crash&) { + crashed = true; + } + CHECK(crashed && h->take_action_acks().empty()); + h.reset(); + CHECK(stored(*owner, carol, 11) == "134:5:1"); + auto successor = host(r, config(r)); + step_until(*successor, t, [](const HostStatus& s) { return s.ready; }); + CHECK(successor->status().committed_sequence == 1); + CHECK(successor->submit_action({carol, 11, 40, 1})); // The host does not deduplicate + // across processes: admission does (find_region_action), and the database refuses a + // second commit of the id (store_rules). + } + { + // Unknown publication outcome: resolved by the exact bytes, actions included. + const auto r = provisioned(5); + auto h = host(r, config(r)); + SchedulerTime t = std::chrono::steady_clock::now(); + CHECK(h->step(t).ready); + for (std::uint64_t tick = 1; tick <= 30; ++tick) + CHECK(h->submit(input(tick, 0x74))); + CHECK(h->submit_action({dave, 12, 2, 1})); + step_until(*h, t, [](const HostStatus& s) { return s.simulated_tick >= 2; }); + auto lock = connect_as(env("PGUSER")); + const std::array key{std::to_string(r.key.world), + std::to_string(r.key.region)}; + CHECK(lock->query("BEGIN", {}, 5s)); + CHECK(lock->query("SELECT 1 FROM evolution.region_leases WHERE world_id = " + "$1::evolution.u64 AND region_id = $2::evolution.u64 FOR UPDATE", + key, 5s)); + step_until(*h, t, [](const HostStatus& s) { return s.phase == HostPhase::resolving; }); + CHECK(h->take_action_acks().empty() && h->take_dropped_actions().empty()); + CHECK(lock->query("ROLLBACK", {}, 5s)); + step_until(*h, t, [](const HostStatus& s) { return s.phase == HostPhase::running; }); + const auto acks = h->take_action_acks(); + CHECK(acks.size() == 1 && acks[0].record.action_id == 12 && acks[0].commit_sequence == 1); + CHECK(stored(*owner, dave, 12) == "135:2:1"); + } +} +} // namespace + +void run_region_action_tests() { + store_rules(); + host_actions(); + std::puts("region action integration passed"); +} diff --git a/tests/region_checkpoint_store_tests.cpp b/tests/region_checkpoint_store_tests.cpp index d199ac0..eb10a7e 100644 --- a/tests/region_checkpoint_store_tests.cpp +++ b/tests/region_checkpoint_store_tests.cpp @@ -582,7 +582,7 @@ void durable_outcomes_and_ambiguity() { // receipt and records nothing twice; resolution confirms it under any epoch. auto death_block = encode_decisions(died->decisions, 0, 20); CHECK(death_block); - const PreparedCommit lost{r.key, filled<16>(0xF1), died->committed.manifest, *death_block}; + const PreparedCommit lost{r.key, filled<16>(0xF1), died->committed.manifest, *death_block, {}}; auto retried = retry_prepared(*r.runtime, lost, 5s); CHECK(retried && *retried == died->committed.head); auto resolved = resolve_prepared(*r.runtime, lost, 5s); diff --git a/tests/region_migration_tests.cpp b/tests/region_migration_tests.cpp index f26c19e..4b51f53 100644 --- a/tests/region_migration_tests.cpp +++ b/tests/region_migration_tests.cpp @@ -176,6 +176,7 @@ void upgrade_across_migration_and_barrier() { " -v migration_sha_7=$(s 0007_region_clones.sql) -v migration_sha_8=invalid" " -v migration_sha_9=$(s 0009_serialized_authority.sql)" " -v migration_sha_10=$(s 0010_account_control.sql)" + " -v migration_sha_11=$(s 0011_region_actions.sql)" " -v runtime_role=evo_runtime_test -f migrations/apply.sql" " > /dev/null 2>&1") != 0); CHECK(history(*owner_db) == "1,2,3,4,5,6,7"); @@ -191,7 +192,7 @@ void upgrade_across_migration_and_barrier() { CHECK(shell("PGDATABASE=" + database + " EVO_PG_RUNTIME_ROLE=evo_runtime_test bash \"$EVO_SOURCE_ROOT/" "tools/migrate-postgres.sh\" > /dev/null") == 0); - CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8,9,10"); + CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8,9,10,11"); CHECK(renew_region_lease(*runtime, r.key, *lease, 20000ms, 5s)); confirm(publish_next(20)); // 41..60, now v8. CHECK(head.head.end_tick == 60); diff --git a/tools/ci/postgres-check.sh b/tools/ci/postgres-check.sh index 2e65ffc..beeed1d 100644 --- a/tools/ci/postgres-check.sh +++ b/tools/ci/postgres-check.sh @@ -61,7 +61,7 @@ PGDATABASE=upgrade_path psql -X -v ON_ERROR_STOP=1 -1 -v "migration_sha=$v1_sha" -f "$root/migrations/0001_region_bootstrap.sql" > /dev/null test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',') FROM evolution.schema_migrations")" = 1 PGDATABASE=upgrade_path bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10 +test "$(PGDATABASE=upgrade_path psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11 # v2 heads have no payload binding: v2->v3 fails closed and leaves history at v2. v2_sha="$(sha256sum "$root/migrations/0002_region_fencing.sql")" v2_sha="${v2_sha%% *}" @@ -82,7 +82,7 @@ PGDATABASE=v3_commits psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '01000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32), 'hex'))" > /dev/null PGDATABASE=v3_commits bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 10:1,2,3,4,5,6,7,8,9,10 +test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) || ':' || string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 11:1,2,3,4,5,6,7,8,9,10,11 test "$(PGDATABASE=v3_commits psql -XAtc "SELECT count(*) FROM evolution.region_commits")" = 1 # Review P1: a v4 database holding a published (possibly terminal) checkpoint has # no durable decisions for it. The v5 upgrade must refuse before changing anything. @@ -136,7 +136,7 @@ PGDATABASE=v6_populated psql -X -v ON_ERROR_STOP=1 \ -c "INSERT INTO evolution.retained_builds VALUES (decode(repeat('b1', 32), 'hex'))" \ -c "SELECT evolution.provision_region_checkpoint(decode('4145564f52434d31' || '02000000' || '01000000' || '0100000000000000' || '0100000000000000' || '0900000000000000' || repeat('11', 16) || repeat('00', 8) || repeat('00', 16) || repeat('00', 32) || repeat('00', 16) || repeat('00', 16) || repeat('00', 8) || '50c3000000000000' || '020000000200000001000000010000000100000001000000010000000100000001000000' || repeat('b1', 32) || '00000000' || '00000000' || '01000000' || '6400000000000000' || repeat('a0', 32) || repeat('00', 80), 'hex'))" > /dev/null PGDATABASE=v6_populated bash "$root/tools/migrate-postgres.sh" -test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10 +test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(version::text, ',' ORDER BY version) FROM evolution.schema_migrations")" = 1,2,3,4,5,6,7,8,9,10,11 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT count(*) FROM evolution.region_commits c, LATERAL evolution.parse_checkpoint_manifest(c.manifest) m WHERE m.commit_sequence = 0 AND m.predecessor_commit IS NULL")" = 1 test "$(PGDATABASE=v6_populated psql -XAtc "SELECT string_agg(conname, ',') FROM pg_constraint WHERE conrelid = 'evolution.region_heads'::regclass AND contype = 'c' AND pg_get_constraintdef(oid) LIKE '%end_tick%'")" = region_heads_published_tick # Inherited privileges bypass fencing, so every migrate run audits effective rights. @@ -169,14 +169,20 @@ expect_audit_failure 'GRANT EXECUTE ON FUNCTION evolution.check_region_lease(evo expect_audit_failure 'ALTER ROLE evo_runtime_login CREATEDB' 'ALTER ROLE evo_runtime_login NOCREATEDB' expect_audit_failure 'GRANT EXECUTE ON FUNCTION evolution.provision_region_checkpoint(bytea) TO evo_runtime_test' 'REVOKE EXECUTE ON FUNCTION evolution.provision_region_checkpoint(bytea) FROM evo_runtime_test' expect_audit_failure 'GRANT SELECT ON evolution.region_commits TO evo_runtime_test' 'REVOKE SELECT ON evolution.region_commits FROM evo_runtime_test' +# Schema v11: action records are reachable only through the fenced functions. +expect_audit_failure 'GRANT SELECT ON evolution.region_actions TO evo_runtime_test' 'REVOKE SELECT ON evolution.region_actions FROM evo_runtime_test' +expect_audit_failure 'GRANT SELECT ON evolution.region_commit_actions TO evo_runtime_test' 'REVOKE SELECT ON evolution.region_commit_actions FROM evo_runtime_test' +expect_audit_failure 'GRANT EXECUTE ON FUNCTION evolution.check_action_block(bytea, numeric, numeric) TO evo_runtime_test' 'REVOKE EXECUTE ON FUNCTION evolution.check_action_block(bytea, numeric, numeric) FROM evo_runtime_test' # Schema v10: account controls are reachable only through the three fenced functions. expect_audit_failure 'GRANT SELECT ON evolution.account_controls TO evo_runtime_test' 'REVOKE SELECT ON evolution.account_controls FROM evo_runtime_test' expect_audit_failure 'REVOKE EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) FROM evo_runtime_test' 'GRANT EXECUTE ON FUNCTION evolution.claim_account_control(evolution.u64, evolution.u64, uuid, evolution.u64, uuid, evolution.u64, evolution.u64, integer) TO evo_runtime_test' bash "$root/tools/migrate-postgres.sh" # Restored configuration passes the audit again. # Unknown future versions and checksum drift must not be repaired silently. -psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=11 WHERE version=10' +psql -X -v ON_ERROR_STOP=1 -c 'UPDATE evolution.schema_migrations SET version=12 WHERE version=11' +if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=11 WHERE version=12" +psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=11" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi -psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET version=10 WHERE version=11" psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=10" if bash "$root/tools/migrate-postgres.sh"; then exit 1; fi psql -X -v ON_ERROR_STOP=1 -c "UPDATE evolution.schema_migrations SET sha256=repeat('0',64) WHERE version=9" diff --git a/tools/ci/upgrade-check.sh b/tools/ci/upgrade-check.sh index 5bb6967..22cda3f 100644 --- a/tools/ci/upgrade-check.sh +++ b/tools/ci/upgrade-check.sh @@ -4,7 +4,7 @@ # exported). Real evo-server processes on world 500: # 1. Failed migration: a migration that takes the migration lock, alters a history # table, holds its locks for 3 s and then fails runs while build b1 serves. -# Nothing it did remains (schema still v10, no new column); the host keeps its +# Nothing it did remains (schema still v11, no new column); the host keeps its # authority, and every acknowledgement stays durable. # 2. Barrier (ADR-0068 operator order): build b2 is retained, and a b2 host is started # while b1 still holds the lease (it waits, never ready). upgrade_barrier(b1) fences @@ -61,7 +61,7 @@ SQL [ "$code" != 0 ] && grep -q 'division by zero' "$work/migration.log" || fail "the rehearsal migration did not fail" [ "$(q -c "SELECT count(*) FROM information_schema.columns WHERE table_name = 'region_commits' AND column_name = 'rehearsal_column'")" = 0 ] || fail "the failed migration left a column" -[ "$(q -c 'SELECT max(version) FROM evolution.schema_migrations')" = 10 ] || fail "schema version moved" +[ "$(q -c 'SELECT max(version) FROM evolution.schema_migrations')" = 11 ] || fail "schema version moved" wait_ack "$work/b1.log" $((before + 40)) || fail "b1 did not resume after the failed migration" kill -0 "$b1_pid" 2> /dev/null || fail "b1 exited during the failed migration" bash "$root/tools/migrate-postgres.sh" > /dev/null || fail "migrate after the failure" diff --git a/tools/migrate-postgres.sh b/tools/migrate-postgres.sh index bb752f1..3347c2d 100644 --- a/tools/migrate-postgres.sh +++ b/tools/migrate-postgres.sh @@ -32,11 +32,13 @@ digest_9="$(sha256sum "$root/migrations/0009_serialized_authority.sql")" digest_9="${digest_9%% *}" digest_10="$(sha256sum "$root/migrations/0010_account_control.sql")" digest_10="${digest_10%% *}" +digest_11="$(sha256sum "$root/migrations/0011_region_actions.sql")" +digest_11="${digest_11%% *}" psql -X --no-password --set=ON_ERROR_STOP=1 --set="migration_sha=$digest" \ --set="migration_sha_2=$digest_2" --set="migration_sha_3=$digest_3" \ --set="migration_sha_4=$digest_4" --set="migration_sha_5=$digest_5" \ --set="migration_sha_6=$digest_6" --set="migration_sha_7=$digest_7" \ --set="migration_sha_8=$digest_8" --set="migration_sha_9=$digest_9" \ - --set="migration_sha_10=$digest_10" \ + --set="migration_sha_10=$digest_10" --set="migration_sha_11=$digest_11" \ --set="runtime_role=$runtime_role" \ --file="$root/migrations/apply.sql"