Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 1m47s
Foundation / checks (network) (push) Successful in 9m30s
Foundation / checks (client) (push) Successful in 12m19s
Foundation / checks (tsan) (push) Successful in 7m43s
Foundation / checks (checks) (push) Successful in 31m29s
Foundation / checks (postgres) (push) Successful in 36m39s
Foundation / postgres-tools (push) Successful in 19s
Foundation / image (push) Successful in 2m15s
Foundation / server-image (push) Successful in 20m37s
Each batch's actions are recorded in the same statement as its checkpoint: one region_actions row per (world, account, action id) and the block in an append-only table, compared on exact retries and resolution. The region host attaches actions to the ticks they ride and reports acknowledgements and drops, keeping unknown-outcome batches unresolved. Outcomes come from the durable record via ActionLedger, so crashes before or after a commit, unknown publications and cross-region moves all resolve to one answer; a repeated id keeps its first record instead of stalling a region. ADR-0092. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
189 lines
8.0 KiB
C++
189 lines
8.0 KiB
C++
#pragma once
|
|
#include "evo/persistence/checkpoint_store.hpp"
|
|
#include "evo/persistence/region_batch.hpp"
|
|
#include "evo/persistence/region_lease.hpp"
|
|
#include "evo/server/ops.hpp"
|
|
#include <chrono>
|
|
#include <cstddef>
|
|
#include <cstdint>
|
|
#include <deque>
|
|
#include <expected>
|
|
#include <functional>
|
|
#include <memory>
|
|
#include <optional>
|
|
#include <vector>
|
|
|
|
// M7.6 authoritative region host (ADR-0064). Linux; composes the fenced lease,
|
|
// verified recovery and batch publication. One owner, one thread: every call runs
|
|
// on the caller's thread and all I/O is bounded by the lease policy's deadlines.
|
|
// While a batch commits the region is stopped, by design (ADR-0058).
|
|
namespace evo::server {
|
|
// One tick's host input, from the (future) gateway or a local feed.
|
|
struct TickInput {
|
|
persistence::Uuid request{}; // Caller identity for acknowledgement; nonzero.
|
|
processes::EnvironmentalInput input{};
|
|
friend bool operator==(const TickInput&, const TickInput&) = default;
|
|
};
|
|
inline constexpr std::size_t tick_input_bytes = 16 + 72; // Canonical accounting size.
|
|
|
|
enum class AdmissionError : std::uint8_t {
|
|
not_ready, // No current authority or not recovered: nothing is accepted.
|
|
draining, // Shutdown in progress.
|
|
full, // Backpressure: bounded backlog (inputs or bytes) is full.
|
|
duplicate, // Request identity already queued or pending in this host.
|
|
invalid, // Zero request identity.
|
|
};
|
|
// Final: the input's tick is in a durably committed checkpoint.
|
|
struct FinalAck {
|
|
persistence::Uuid request{};
|
|
std::uint64_t tick{};
|
|
std::uint64_t commit_sequence{};
|
|
friend bool operator==(const FinalAck&, const FinalAck&) = default;
|
|
};
|
|
// Final: the permanent outcome is durable (and in the outbox).
|
|
struct FinalOutcome {
|
|
persistence::BatchDecision decision{};
|
|
std::uint64_t commit_sequence{};
|
|
friend bool operator==(const FinalOutcome&, const FinalOutcome&) = default;
|
|
};
|
|
struct Dropped {
|
|
TickInput input{};
|
|
Unacknowledged reason{};
|
|
};
|
|
// M10.4b (ADR-0092): an action is durable with the batch that simulated its tick.
|
|
struct ActionAck {
|
|
persistence::ActionRecord record{}; // `tick` is the tick it rode.
|
|
std::uint64_t commit_sequence{};
|
|
friend bool operator==(const ActionAck&, const ActionAck&) = default;
|
|
};
|
|
// `final`: certainly never committed. Otherwise its batch's publication outcome was unknown
|
|
// when authority was lost; the durable record decides (find_region_action) once a later
|
|
// authority has recovered.
|
|
struct DroppedAction {
|
|
persistence::ActionRecord record{};
|
|
bool final{};
|
|
Unacknowledged reason{};
|
|
};
|
|
|
|
// Commit boundaries for fault injection (M7.7 kill tests).
|
|
enum class CrashPoint : std::uint8_t {
|
|
before_upload, // Batch sealed; nothing uploaded or sent.
|
|
before_transaction, // Payload and trace uploaded and verified; manifest not sent.
|
|
after_commit, // Commit confirmed; no acknowledgement or outcome emitted yet.
|
|
};
|
|
using Connect =
|
|
std::function<std::expected<std::unique_ptr<persistence::Postgres>, persistence::PgFailure>()>;
|
|
struct HostConfig {
|
|
persistence::RegionKey key{};
|
|
std::uint64_t frame{};
|
|
persistence::LeasePolicy lease{};
|
|
// Region quantum; must equal the committed checkpoint's tick_microseconds.
|
|
std::chrono::microseconds tick_quantum{50000};
|
|
std::chrono::milliseconds flush_interval{1000}; // Commit a non-empty batch this often.
|
|
std::size_t max_queued_inputs{64};
|
|
std::size_t max_queued_bytes{65536};
|
|
std::size_t max_queued_actions{256}; // Queued plus riding the open batch.
|
|
std::uint32_t max_catch_up_ticks{8}; // Per step; debt is never skipped.
|
|
persistence::Sha256Digest build{};
|
|
std::vector<persistence::Sha256Digest> retained_builds;
|
|
std::size_t restore_budget{8000000};
|
|
// Test-only: called at each commit boundary (e.g. to SIGKILL the process).
|
|
// Production configurations leave it empty.
|
|
std::function<void(CrashPoint)> crash_hook;
|
|
};
|
|
enum class HostConfigError : std::uint8_t { invalid };
|
|
|
|
// Scheduling time comes from the caller (monotonic, may be scaled in tests); lease
|
|
// deadlines always use LeaseClock (real, suspend-aware) and database time.
|
|
using SchedulerTime = std::chrono::steady_clock::time_point;
|
|
|
|
template <typename State> class RegionHost {
|
|
public:
|
|
[[nodiscard]] static std::expected<std::unique_ptr<RegionHost>, HostConfigError>
|
|
create(HostConfig config, Connect connect,
|
|
std::unique_ptr<persistence::FilesystemObjectStore> objects);
|
|
RegionHost(const RegionHost&) = delete;
|
|
RegionHost& operator=(const RegionHost&) = delete;
|
|
RegionHost(RegionHost&&) = delete;
|
|
RegionHost& operator=(RegionHost&&) = delete;
|
|
~RegionHost();
|
|
|
|
// Provisional admission only; nothing is final until a FinalAck.
|
|
[[nodiscard]] std::expected<void, AdmissionError> submit(const TickInput& input);
|
|
// An action rides the first simulated tick >= record.tick and is committed with that
|
|
// batch, in the same statement (schema v11). Final only through an ActionAck.
|
|
[[nodiscard]] std::expected<void, AdmissionError>
|
|
submit_action(const persistence::ActionRecord& record);
|
|
// Advances the state machine once: lease upkeep, recovery, resolution, commits
|
|
// and up to max_catch_up_ticks ticks. Blocks at most for bounded I/O.
|
|
const HostStatus& step(SchedulerTime now);
|
|
// Stops admission, drops the backlog, commits pending ticks, releases the lease.
|
|
void drain();
|
|
|
|
[[nodiscard]] std::vector<FinalAck> take_acks();
|
|
[[nodiscard]] std::vector<FinalOutcome> take_outcomes();
|
|
[[nodiscard]] std::vector<Dropped> take_dropped();
|
|
[[nodiscard]] std::vector<ActionAck> take_action_acks();
|
|
[[nodiscard]] std::vector<DroppedAction> take_dropped_actions();
|
|
[[nodiscard]] const HostStatus& status() const noexcept {
|
|
return status_;
|
|
}
|
|
// Read-only, for operational reporting (free space against the reserve).
|
|
[[nodiscard]] const persistence::FilesystemObjectStore& objects() const noexcept {
|
|
return *objects_;
|
|
}
|
|
// Read-only view for diagnostics/tests; null unless recovered.
|
|
[[nodiscard]] const State* state() const noexcept {
|
|
return state_.get();
|
|
}
|
|
|
|
private:
|
|
struct Applied {
|
|
TickInput input;
|
|
std::uint64_t tick{};
|
|
};
|
|
RegionHost(HostConfig config, Connect connect,
|
|
std::unique_ptr<persistence::FilesystemObjectStore> objects);
|
|
bool database();
|
|
void acquire(SchedulerTime now);
|
|
void upkeep();
|
|
void simulate(SchedulerTime now);
|
|
void commit(SchedulerTime now);
|
|
void resolve(SchedulerTime now);
|
|
void confirmed(const persistence::PublishedBatch& batch, SchedulerTime now);
|
|
void lose_authority(Unacknowledged reason, HostPhase next);
|
|
void refresh(SchedulerTime now);
|
|
[[nodiscard]] persistence::Uuid next_commit_id() const;
|
|
|
|
HostConfig config_;
|
|
Connect connect_;
|
|
std::unique_ptr<persistence::FilesystemObjectStore> objects_;
|
|
std::unique_ptr<persistence::Postgres> db_;
|
|
persistence::LeaseAuthority authority_;
|
|
std::unique_ptr<State> state_;
|
|
std::optional<persistence::RegionBatch<State>> batch_;
|
|
std::optional<persistence::CommittedCheckpoint> committed_;
|
|
std::optional<persistence::PreparedCommit> prepared_;
|
|
std::deque<TickInput> queue_;
|
|
std::vector<Applied> applied_;
|
|
std::vector<FinalAck> acks_;
|
|
std::vector<FinalOutcome> outcomes_;
|
|
std::vector<Dropped> dropped_;
|
|
std::deque<persistence::ActionRecord> action_queue_;
|
|
std::vector<persistence::ActionRecord> batch_actions_; // Riding the open batch.
|
|
std::vector<ActionAck> action_acks_;
|
|
std::vector<DroppedAction> dropped_actions_;
|
|
SchedulerTime origin_{}, batch_started_{};
|
|
// Store failures back off: a failed commit is retried after retry_delay_, which
|
|
// doubles to a cap and resets on the next confirmed commit.
|
|
SchedulerTime retry_at_{};
|
|
std::chrono::milliseconds retry_delay_{};
|
|
std::uint64_t origin_tick_{};
|
|
bool draining_{};
|
|
HostStatus status_{};
|
|
};
|
|
|
|
extern template class RegionHost<processes::EnvironmentalCommunity>;
|
|
extern template class RegionHost<processes::LineageRegion>;
|
|
} // namespace evo::server
|