Add M10.4b durable action records (schema v11)
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 1m47s
Foundation / checks (network) (push) Successful in 9m30s
Foundation / checks (client) (push) Successful in 12m19s
Foundation / checks (tsan) (push) Successful in 7m43s
Foundation / checks (checks) (push) Successful in 31m29s
Foundation / checks (postgres) (push) Successful in 36m39s
Foundation / postgres-tools (push) Successful in 19s
Foundation / image (push) Successful in 2m15s
Foundation / server-image (push) Successful in 20m37s
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 1m47s
Foundation / checks (network) (push) Successful in 9m30s
Foundation / checks (client) (push) Successful in 12m19s
Foundation / checks (tsan) (push) Successful in 7m43s
Foundation / checks (checks) (push) Successful in 31m29s
Foundation / checks (postgres) (push) Successful in 36m39s
Foundation / postgres-tools (push) Successful in 19s
Foundation / image (push) Successful in 2m15s
Foundation / server-image (push) Successful in 20m37s
Each batch's actions are recorded in the same statement as its checkpoint: one region_actions row per (world, account, action id) and the block in an append-only table, compared on exact retries and resolution. The region host attaches actions to the ticks they ride and reports acknowledgements and drops, keeping unknown-outcome batches unresolved. Outcomes come from the durable record via ActionLedger, so crashes before or after a commit, unknown publications and cross-region moves all resolve to one answer; a repeated id keeps its first record instead of stalling a region. ADR-0092. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -6,6 +6,7 @@
|
||||
#include <deque>
|
||||
#include <expected>
|
||||
#include <optional>
|
||||
#include <span>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
@@ -45,11 +46,32 @@ struct AccountOutcome {
|
||||
ActionOutcome outcome;
|
||||
};
|
||||
enum class IntentConfigError : std::uint8_t { invalid };
|
||||
// Durable action records (10.4b): the region host implements it over its lease (schema v11);
|
||||
// tests use fakes. `find` answers with the committed outcome, nothing when the action was
|
||||
// never committed, or `unavailable` when it cannot decide.
|
||||
enum class ActionLedgerError : std::uint8_t { unavailable };
|
||||
class ActionLedger {
|
||||
public:
|
||||
ActionLedger() = default;
|
||||
ActionLedger(const ActionLedger&) = delete;
|
||||
ActionLedger& operator=(const ActionLedger&) = delete;
|
||||
virtual ~ActionLedger() = default;
|
||||
[[nodiscard]] virtual std::expected<std::optional<ActionOutcome>, ActionLedgerError>
|
||||
find(const AccountId& account, std::uint64_t action_id) = 0;
|
||||
|
||||
protected:
|
||||
ActionLedger(ActionLedger&&) = default;
|
||||
ActionLedger& operator=(ActionLedger&&) = default;
|
||||
};
|
||||
|
||||
class IntentAdmission {
|
||||
public:
|
||||
[[nodiscard]] static std::expected<IntentAdmission, IntentConfigError>
|
||||
create(IntentLimits limits);
|
||||
// Durable lookups for duplicates and unresolved actions; null keeps memory only.
|
||||
void attach(ActionLedger* ledger) noexcept {
|
||||
ledger_ = ledger;
|
||||
}
|
||||
// Session lifecycle, driven by session control (after Welcome, on any close).
|
||||
[[nodiscard]] bool open(std::uint64_t session, const AccountId& account);
|
||||
void close(std::uint64_t session);
|
||||
@@ -59,18 +81,26 @@ class IntentAdmission {
|
||||
const AdmissionFacts& facts);
|
||||
[[nodiscard]] ActionReceipt action(std::uint64_t session, const SessionAction& m,
|
||||
const AdmissionFacts& facts);
|
||||
// What this process knows about an action of the session's account.
|
||||
[[nodiscard]] ActionOutcome query(std::uint64_t session, const ActionQuery& m) const;
|
||||
// An action of the session's account: final, pending or unknown. With a ledger, an action
|
||||
// whose batch outcome was unknown is decided from its durable record once this authority
|
||||
// is ready, and an action no longer in memory is looked up.
|
||||
[[nodiscard]] ActionOutcome query(std::uint64_t session, const ActionQuery& m,
|
||||
const AdmissionFacts& facts);
|
||||
// Watermarks to send after the host simulated through facts.server_tick.
|
||||
[[nodiscard]] std::vector<std::pair<std::uint64_t, SimulatedThrough>>
|
||||
simulated(const AdmissionFacts& facts);
|
||||
// Scheduled actions whose tick is <= `tick`: they ride that tick's batch from now on.
|
||||
[[nodiscard]] std::vector<CarriedAction> carry(std::uint64_t tick);
|
||||
// A commit covering ticks up to `end_tick` is durable: its carried actions are final.
|
||||
[[nodiscard]] std::vector<AccountOutcome> committed(std::uint64_t end_tick,
|
||||
// These carried actions are durable in `commit_sequence`; `tick` is the tick each rode.
|
||||
// With a ledger, each outcome is taken from the action's durable record.
|
||||
[[nodiscard]] std::vector<AccountOutcome> committed(std::span<const CarriedAction> actions,
|
||||
std::uint64_t commit_sequence);
|
||||
// The host dropped these carried actions. Final: never committed. Otherwise their batch's
|
||||
// outcome was unknown: they stay pending until their durable record decides them.
|
||||
[[nodiscard]] std::vector<AccountOutcome> dropped(std::span<const CarriedAction> actions,
|
||||
bool final);
|
||||
// Authority lost or epoch advanced: actions never carried are final (not_committed);
|
||||
// carried ones stay unresolved until a durable record decides them (10.4b).
|
||||
// carried ones stay pending until committed, dropped or decided by their record.
|
||||
[[nodiscard]] std::vector<AccountOutcome> abandon();
|
||||
// The current session of an account, for delivering outcomes.
|
||||
[[nodiscard]] std::optional<std::uint64_t> session_of(const AccountId& account) const;
|
||||
@@ -92,8 +122,16 @@ class IntentAdmission {
|
||||
struct Action {
|
||||
CarriedAction carried;
|
||||
Stage stage{};
|
||||
bool unknown{}; // Carried in a batch whose outcome was unknown when authority was lost.
|
||||
};
|
||||
explicit IntentAdmission(IntentLimits limits) : limits_{limits} {}
|
||||
// A ledger answer flattened: unavailable (or no ledger), absent, or committed.
|
||||
enum class Record : std::uint8_t { unavailable, absent, committed };
|
||||
struct Found {
|
||||
Record state{Record::unavailable};
|
||||
ActionOutcome outcome;
|
||||
};
|
||||
[[nodiscard]] Found lookup(const AccountId& account, std::uint64_t action_id) const;
|
||||
Session* find(std::uint64_t session);
|
||||
const Session* find(std::uint64_t session) const;
|
||||
void count_window(Session& s, std::uint64_t server_tick) const;
|
||||
@@ -103,5 +141,6 @@ class IntentAdmission {
|
||||
std::vector<Session> sessions_;
|
||||
std::vector<Action> actions_;
|
||||
std::deque<AccountOutcome> finals_;
|
||||
ActionLedger* ledger_{};
|
||||
};
|
||||
} // namespace evo::network
|
||||
|
||||
@@ -34,6 +34,7 @@ struct PreparedCommit {
|
||||
Uuid commit{};
|
||||
std::vector<std::uint8_t> manifest;
|
||||
std::vector<std::uint8_t> decisions;
|
||||
std::vector<std::uint8_t> actions; // Schema v11 block; empty: published without actions.
|
||||
};
|
||||
struct PublicationFailure {
|
||||
PublicationError kind{};
|
||||
@@ -92,7 +93,8 @@ publish_environmental_batch(Postgres& db, const FilesystemObjectStore& objects,
|
||||
const LeaseGrant& lease, const CommittedCheckpoint& predecessor,
|
||||
const RegionBatch<processes::EnvironmentalCommunity>& batch,
|
||||
Uuid commit, const Sha256Digest& build, std::size_t restore_budget,
|
||||
std::chrono::milliseconds timeout, const PublishHooks& hooks = {});
|
||||
std::chrono::milliseconds timeout, const PublishHooks& hooks = {},
|
||||
std::span<const ActionRecord> actions = {});
|
||||
// Loads and fully verifies the committed head before any readiness (ADR-0065):
|
||||
// manifest digest and chain fields, retained build, payload length/SHA-256 and full
|
||||
// restore; then the head's tail: its trace object binds to the committed
|
||||
@@ -119,7 +121,8 @@ publish_lineage_batch(Postgres& db, const FilesystemObjectStore& objects, const
|
||||
const CommittedCheckpoint& predecessor,
|
||||
const RegionBatch<processes::LineageRegion>& batch, Uuid commit,
|
||||
const Sha256Digest& build, std::size_t restore_budget,
|
||||
std::chrono::milliseconds timeout, const PublishHooks& hooks = {});
|
||||
std::chrono::milliseconds timeout, const PublishHooks& hooks = {},
|
||||
std::span<const ActionRecord> actions = {});
|
||||
[[nodiscard]] std::expected<RecoveredLineageCheckpoint, RecoveryFailure>
|
||||
recover_lineage_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
std::span<const Sha256Digest> retained_builds,
|
||||
|
||||
@@ -57,6 +57,22 @@ inline constexpr std::size_t max_batch_decisions = 8;
|
||||
return trace_header_bytes + (static_cast<std::size_t>(ticks) * trace_step_bytes);
|
||||
}
|
||||
|
||||
// M10.4b durable action records (schema v11, ADR-0092): the actions that rode a batch,
|
||||
// committed in the same statement as its checkpoint. Not part of the trace or manifest.
|
||||
inline constexpr std::size_t action_record_bytes = 34;
|
||||
inline constexpr std::size_t max_batch_actions = 1024;
|
||||
struct ActionRecord {
|
||||
Uuid account{};
|
||||
std::uint64_t action_id{}, tick{};
|
||||
std::uint16_t kind{1}; // 1 = probe (no effect).
|
||||
friend bool operator==(const ActionRecord&, const ActionRecord&) = default;
|
||||
};
|
||||
// Canonical block: sorted by (tick, account, id); refuses zero accounts or ids, other kinds,
|
||||
// ticks outside (begin, end], duplicates and more than max_batch_actions records.
|
||||
[[nodiscard]] std::expected<std::vector<std::uint8_t>, BatchError>
|
||||
encode_actions(std::span<const ActionRecord> actions, std::uint64_t begin_tick,
|
||||
std::uint64_t end_tick);
|
||||
|
||||
// One step per tick, begin_tick+1..end_tick, at most 64.
|
||||
[[nodiscard]] std::expected<std::vector<std::uint8_t>, BatchError>
|
||||
encode_trace(const TraceHeader& header, std::span<const TraceStep> steps);
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <expected>
|
||||
#include <optional>
|
||||
#include <span>
|
||||
#include <vector>
|
||||
|
||||
@@ -67,6 +68,14 @@ publish_region_checkpoint(Postgres& db, std::span<const std::uint8_t> manifest,
|
||||
std::span<const std::uint8_t> decisions,
|
||||
std::chrono::milliseconds timeout);
|
||||
|
||||
// Schema v11: the same publication with the batch's action block (encode_actions), recorded
|
||||
// in the same statement. The block is part of the commit identity. An action id already on
|
||||
// record in the world keeps its first record (find_region_action is the authority).
|
||||
[[nodiscard]] std::expected<RegionHead, LeaseStoreFailure>
|
||||
publish_region_checkpoint(Postgres& db, std::span<const std::uint8_t> manifest,
|
||||
std::span<const std::uint8_t> decisions,
|
||||
std::span<const std::uint8_t> actions, std::chrono::milliseconds timeout);
|
||||
|
||||
enum class Resolution : std::uint8_t {
|
||||
committed, // Exact bytes are durable; receipt holds the head. Grants no authority.
|
||||
absent, // Not committed now; proves nothing about a request still in flight.
|
||||
@@ -81,6 +90,22 @@ resolve_region_commit(Postgres& db, RegionKey key, const Uuid& commit,
|
||||
std::span<const std::uint8_t> manifest,
|
||||
std::span<const std::uint8_t> decisions, std::chrono::milliseconds timeout);
|
||||
|
||||
// Schema v11 resolution, comparing the action block too.
|
||||
[[nodiscard]] std::expected<ResolvedCommit, LeaseStoreFailure>
|
||||
resolve_region_commit(Postgres& db, RegionKey key, const Uuid& commit,
|
||||
std::span<const std::uint8_t> manifest,
|
||||
std::span<const std::uint8_t> decisions,
|
||||
std::span<const std::uint8_t> actions, std::chrono::milliseconds timeout);
|
||||
// The committed record of an account's action in the lease's world (schema v11), under the
|
||||
// caller's exact lease. Nothing when the action was never committed.
|
||||
struct CommittedAction {
|
||||
std::uint64_t region{}, tick{}, commit_sequence{};
|
||||
friend bool operator==(const CommittedAction&, const CommittedAction&) = default;
|
||||
};
|
||||
[[nodiscard]] std::expected<std::optional<CommittedAction>, LeaseStoreFailure>
|
||||
find_region_action(Postgres& db, RegionKey key, const LeaseGrant& lease, const Uuid& account,
|
||||
std::uint64_t action_id, std::chrono::milliseconds timeout);
|
||||
|
||||
struct CommittedDecision {
|
||||
std::uint64_t commit_sequence{};
|
||||
BatchDecision decision{};
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
#pragma once
|
||||
#include "evo/network/intent_admission.hpp"
|
||||
#include "evo/persistence/region_lease_store.hpp"
|
||||
#include <chrono>
|
||||
#include <functional>
|
||||
#include <optional>
|
||||
#include <utility>
|
||||
|
||||
// M10.4b durable action records over the region's PostgreSQL lease (ADR-0092, schema v11).
|
||||
// Linux only. Owned by the region host's thread, on the host's own connection. `lease`
|
||||
// returns the host's current region lease, or nothing while it holds none; every lookup
|
||||
// then reports `unavailable`, as does any store failure.
|
||||
namespace evo::server {
|
||||
class PostgresActionLedger final : public network::ActionLedger {
|
||||
public:
|
||||
using LeaseSource = std::function<std::optional<persistence::LeaseGrant>()>;
|
||||
PostgresActionLedger(persistence::Postgres& db, persistence::RegionKey key, LeaseSource lease,
|
||||
std::chrono::milliseconds timeout = std::chrono::milliseconds{2000})
|
||||
: db_{&db}, key_{key}, lease_{std::move(lease)}, timeout_{timeout} {}
|
||||
[[nodiscard]] std::expected<std::optional<network::ActionOutcome>, network::ActionLedgerError>
|
||||
find(const network::AccountId& account, std::uint64_t action_id) override;
|
||||
|
||||
private:
|
||||
persistence::Postgres* db_;
|
||||
persistence::RegionKey key_;
|
||||
LeaseSource lease_;
|
||||
std::chrono::milliseconds timeout_;
|
||||
};
|
||||
} // namespace evo::server
|
||||
@@ -50,6 +50,20 @@ struct Dropped {
|
||||
TickInput input{};
|
||||
Unacknowledged reason{};
|
||||
};
|
||||
// M10.4b (ADR-0092): an action is durable with the batch that simulated its tick.
|
||||
struct ActionAck {
|
||||
persistence::ActionRecord record{}; // `tick` is the tick it rode.
|
||||
std::uint64_t commit_sequence{};
|
||||
friend bool operator==(const ActionAck&, const ActionAck&) = default;
|
||||
};
|
||||
// `final`: certainly never committed. Otherwise its batch's publication outcome was unknown
|
||||
// when authority was lost; the durable record decides (find_region_action) once a later
|
||||
// authority has recovered.
|
||||
struct DroppedAction {
|
||||
persistence::ActionRecord record{};
|
||||
bool final{};
|
||||
Unacknowledged reason{};
|
||||
};
|
||||
|
||||
// Commit boundaries for fault injection (M7.7 kill tests).
|
||||
enum class CrashPoint : std::uint8_t {
|
||||
@@ -68,6 +82,7 @@ struct HostConfig {
|
||||
std::chrono::milliseconds flush_interval{1000}; // Commit a non-empty batch this often.
|
||||
std::size_t max_queued_inputs{64};
|
||||
std::size_t max_queued_bytes{65536};
|
||||
std::size_t max_queued_actions{256}; // Queued plus riding the open batch.
|
||||
std::uint32_t max_catch_up_ticks{8}; // Per step; debt is never skipped.
|
||||
persistence::Sha256Digest build{};
|
||||
std::vector<persistence::Sha256Digest> retained_builds;
|
||||
@@ -95,6 +110,10 @@ template <typename State> class RegionHost {
|
||||
|
||||
// Provisional admission only; nothing is final until a FinalAck.
|
||||
[[nodiscard]] std::expected<void, AdmissionError> submit(const TickInput& input);
|
||||
// An action rides the first simulated tick >= record.tick and is committed with that
|
||||
// batch, in the same statement (schema v11). Final only through an ActionAck.
|
||||
[[nodiscard]] std::expected<void, AdmissionError>
|
||||
submit_action(const persistence::ActionRecord& record);
|
||||
// Advances the state machine once: lease upkeep, recovery, resolution, commits
|
||||
// and up to max_catch_up_ticks ticks. Blocks at most for bounded I/O.
|
||||
const HostStatus& step(SchedulerTime now);
|
||||
@@ -104,6 +123,8 @@ template <typename State> class RegionHost {
|
||||
[[nodiscard]] std::vector<FinalAck> take_acks();
|
||||
[[nodiscard]] std::vector<FinalOutcome> take_outcomes();
|
||||
[[nodiscard]] std::vector<Dropped> take_dropped();
|
||||
[[nodiscard]] std::vector<ActionAck> take_action_acks();
|
||||
[[nodiscard]] std::vector<DroppedAction> take_dropped_actions();
|
||||
[[nodiscard]] const HostStatus& status() const noexcept {
|
||||
return status_;
|
||||
}
|
||||
@@ -148,6 +169,10 @@ template <typename State> class RegionHost {
|
||||
std::vector<FinalAck> acks_;
|
||||
std::vector<FinalOutcome> outcomes_;
|
||||
std::vector<Dropped> dropped_;
|
||||
std::deque<persistence::ActionRecord> action_queue_;
|
||||
std::vector<persistence::ActionRecord> batch_actions_; // Riding the open batch.
|
||||
std::vector<ActionAck> action_acks_;
|
||||
std::vector<DroppedAction> dropped_actions_;
|
||||
SchedulerTime origin_{}, batch_started_{};
|
||||
// Store failures back off: a failed commit is retried after retry_delay_, which
|
||||
// doubles to a cap and resets on the next confirmed commit.
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
#include <expected>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <span>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
@@ -45,6 +46,8 @@ struct RegionSessionsConfig {
|
||||
network::ControlLedger* ledger{};
|
||||
std::chrono::milliseconds renew_interval{5000}, safety_margin{2000};
|
||||
network::IntentLimits intents{}; // max_sessions follows control.max_sessions.
|
||||
// Durable action records (10.4b), owned by the host; null keeps outcomes in memory only.
|
||||
network::ActionLedger* actions{};
|
||||
};
|
||||
// The region host's authority, read once per poll.
|
||||
struct HostFacts {
|
||||
@@ -74,9 +77,12 @@ class RegionSessions {
|
||||
// Actions scheduled for ticks <= `tick`, to be recorded durably with the batch that
|
||||
// simulates that tick (10.4b). Each action is returned once.
|
||||
[[nodiscard]] std::vector<network::CarriedAction> carry(std::uint64_t tick);
|
||||
// A commit covering ticks up to `end_tick` is durable: carried actions become final and
|
||||
// their outcomes are sent to each account's current session.
|
||||
void committed(std::uint64_t end_tick, std::uint64_t commit_sequence);
|
||||
// These carried actions are durable in `commit_sequence` (each with the tick it rode):
|
||||
// their outcomes go to each account's current session.
|
||||
void committed(std::span<const network::CarriedAction> actions, std::uint64_t commit_sequence);
|
||||
// The host dropped these carried actions: final ones are reported not_committed; the
|
||||
// others await their durable record (their batch outcome was unknown).
|
||||
void dropped(std::span<const network::CarriedAction> actions, bool final);
|
||||
// Closes every connection with `shutdown` (flushed by later polls).
|
||||
void shutdown(std::chrono::steady_clock::time_point now);
|
||||
[[nodiscard]] std::vector<RegionSessionEvent> take_events();
|
||||
|
||||
Reference in New Issue
Block a user