Files
aevolution/include/evo/server/region_host.hpp
T
dingoandClaude Opus 5.5 4a900b0bd3 fix(server): back off commit retries after store failures; sharpen fault drill
The first dev drill showed a full volume driving ~140 refused publications a
second. Commit retries now back off 100 ms doubling to a 2 s cap, reset on the
next confirmed commit. fault-drill.sh reports the empty-volume pod's exit code,
restarts and logs, and prints node-specific iptables rules for db-outage.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 15:12:58 -04:00

164 lines
6.7 KiB
C++

#pragma once
#include "evo/persistence/checkpoint_store.hpp"
#include "evo/persistence/region_batch.hpp"
#include "evo/persistence/region_lease.hpp"
#include "evo/server/ops.hpp"
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <deque>
#include <expected>
#include <functional>
#include <memory>
#include <optional>
#include <vector>
// M7.6 authoritative region host (ADR-0064). Linux; composes the fenced lease,
// verified recovery and batch publication. One owner, one thread: every call runs
// on the caller's thread and all I/O is bounded by the lease policy's deadlines.
// While a batch commits the region is stopped, by design (ADR-0058).
namespace evo::server {
// One tick's host input, from the (future) gateway or a local feed.
struct TickInput {
persistence::Uuid request{}; // Caller identity for acknowledgement; nonzero.
processes::EnvironmentalInput input{};
friend bool operator==(const TickInput&, const TickInput&) = default;
};
inline constexpr std::size_t tick_input_bytes = 16 + 72; // Canonical accounting size.
enum class AdmissionError : std::uint8_t {
not_ready, // No current authority or not recovered: nothing is accepted.
draining, // Shutdown in progress.
full, // Backpressure: bounded backlog (inputs or bytes) is full.
duplicate, // Request identity already queued or pending in this host.
invalid, // Zero request identity.
};
// Final: the input's tick is in a durably committed checkpoint.
struct FinalAck {
persistence::Uuid request{};
std::uint64_t tick{};
std::uint64_t commit_sequence{};
friend bool operator==(const FinalAck&, const FinalAck&) = default;
};
// Final: the permanent outcome is durable (and in the outbox).
struct FinalOutcome {
persistence::BatchDecision decision{};
std::uint64_t commit_sequence{};
friend bool operator==(const FinalOutcome&, const FinalOutcome&) = default;
};
struct Dropped {
TickInput input{};
Unacknowledged reason{};
};
// Commit boundaries for fault injection (M7.7 kill tests).
enum class CrashPoint : std::uint8_t {
before_upload, // Batch sealed; nothing uploaded or sent.
before_transaction, // Payload and trace uploaded and verified; manifest not sent.
after_commit, // Commit confirmed; no acknowledgement or outcome emitted yet.
};
using Connect =
std::function<std::expected<std::unique_ptr<persistence::Postgres>, persistence::PgFailure>()>;
struct HostConfig {
persistence::RegionKey key{};
std::uint64_t frame{};
persistence::LeasePolicy lease{};
// Region quantum; must equal the committed checkpoint's tick_microseconds.
std::chrono::microseconds tick_quantum{50000};
std::chrono::milliseconds flush_interval{1000}; // Commit a non-empty batch this often.
std::size_t max_queued_inputs{64};
std::size_t max_queued_bytes{65536};
std::uint32_t max_catch_up_ticks{8}; // Per step; debt is never skipped.
persistence::Sha256Digest build{};
std::vector<persistence::Sha256Digest> retained_builds;
std::size_t restore_budget{8000000};
// Test-only: called at each commit boundary (e.g. to SIGKILL the process).
// Production configurations leave it empty.
std::function<void(CrashPoint)> crash_hook;
};
enum class HostConfigError : std::uint8_t { invalid };
// Scheduling time comes from the caller (monotonic, may be scaled in tests); lease
// deadlines always use LeaseClock (real, suspend-aware) and database time.
using SchedulerTime = std::chrono::steady_clock::time_point;
template <typename State> class RegionHost {
public:
[[nodiscard]] static std::expected<std::unique_ptr<RegionHost>, HostConfigError>
create(HostConfig config, Connect connect,
std::unique_ptr<persistence::FilesystemObjectStore> objects);
RegionHost(const RegionHost&) = delete;
RegionHost& operator=(const RegionHost&) = delete;
RegionHost(RegionHost&&) = delete;
RegionHost& operator=(RegionHost&&) = delete;
~RegionHost();
// Provisional admission only; nothing is final until a FinalAck.
[[nodiscard]] std::expected<void, AdmissionError> submit(const TickInput& input);
// Advances the state machine once: lease upkeep, recovery, resolution, commits
// and up to max_catch_up_ticks ticks. Blocks at most for bounded I/O.
const HostStatus& step(SchedulerTime now);
// Stops admission, drops the backlog, commits pending ticks, releases the lease.
void drain();
[[nodiscard]] std::vector<FinalAck> take_acks();
[[nodiscard]] std::vector<FinalOutcome> take_outcomes();
[[nodiscard]] std::vector<Dropped> take_dropped();
[[nodiscard]] const HostStatus& status() const noexcept {
return status_;
}
// Read-only, for operational reporting (free space against the reserve).
[[nodiscard]] const persistence::FilesystemObjectStore& objects() const noexcept {
return *objects_;
}
// Read-only view for diagnostics/tests; null unless recovered.
[[nodiscard]] const State* state() const noexcept {
return state_.get();
}
private:
struct Applied {
TickInput input;
std::uint64_t tick{};
};
RegionHost(HostConfig config, Connect connect,
std::unique_ptr<persistence::FilesystemObjectStore> objects);
bool database();
void acquire(SchedulerTime now);
void upkeep();
void simulate(SchedulerTime now);
void commit(SchedulerTime now);
void resolve(SchedulerTime now);
void confirmed(const persistence::PublishedBatch& batch, SchedulerTime now);
void lose_authority(Unacknowledged reason, HostPhase next);
void refresh(SchedulerTime now);
[[nodiscard]] persistence::Uuid next_commit_id() const;
HostConfig config_;
Connect connect_;
std::unique_ptr<persistence::FilesystemObjectStore> objects_;
std::unique_ptr<persistence::Postgres> db_;
persistence::LeaseAuthority authority_;
std::unique_ptr<State> state_;
std::optional<persistence::RegionBatch<State>> batch_;
std::optional<persistence::CommittedCheckpoint> committed_;
std::optional<persistence::PreparedCommit> prepared_;
std::deque<TickInput> queue_;
std::vector<Applied> applied_;
std::vector<FinalAck> acks_;
std::vector<FinalOutcome> outcomes_;
std::vector<Dropped> dropped_;
SchedulerTime origin_{}, batch_started_{};
// Store failures back off: a failed commit is retried after retry_delay_, which
// doubles to a cap and resets on the next confirmed commit.
SchedulerTime retry_at_{};
std::chrono::milliseconds retry_delay_{};
std::uint64_t origin_tick_{};
bool draining_{};
HostStatus status_{};
};
extern template class RegionHost<processes::EnvironmentalCommunity>;
extern template class RegionHost<processes::LineageRegion>;
} // namespace evo::server