An AEvolution-only Prometheus 3.13.3 and Alertmanager 0.34.1, pinned by digest and delivered by Argo (ADR-0072), scrape each region's operations Service through static targets (no API access, no RBAC); the region namespaces admit port 9100 from Prometheus only. Sixteen alert rules cover authority, lease, commits, backlog and object capacity and are proven by promtool injection tests in CI with tools copied from the same pinned images. Two new gauges expose object volume free space and the reserve. Alerts route to a null receiver until a destination is chosen; no broker is adopted and the database outbox stays. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
160 lines
6.5 KiB
C++
160 lines
6.5 KiB
C++
#pragma once
|
|
#include "evo/persistence/checkpoint_store.hpp"
|
|
#include "evo/persistence/region_batch.hpp"
|
|
#include "evo/persistence/region_lease.hpp"
|
|
#include "evo/server/ops.hpp"
|
|
#include <chrono>
|
|
#include <cstddef>
|
|
#include <cstdint>
|
|
#include <deque>
|
|
#include <expected>
|
|
#include <functional>
|
|
#include <memory>
|
|
#include <optional>
|
|
#include <vector>
|
|
|
|
// M7.6 authoritative region host (ADR-0064). Linux; composes the fenced lease,
|
|
// verified recovery and batch publication. One owner, one thread: every call runs
|
|
// on the caller's thread and all I/O is bounded by the lease policy's deadlines.
|
|
// While a batch commits the region is stopped, by design (ADR-0058).
|
|
namespace evo::server {
|
|
// One tick's host input, from the (future) gateway or a local feed.
|
|
struct TickInput {
|
|
persistence::Uuid request{}; // Caller identity for acknowledgement; nonzero.
|
|
processes::EnvironmentalInput input{};
|
|
friend bool operator==(const TickInput&, const TickInput&) = default;
|
|
};
|
|
inline constexpr std::size_t tick_input_bytes = 16 + 72; // Canonical accounting size.
|
|
|
|
enum class AdmissionError : std::uint8_t {
|
|
not_ready, // No current authority or not recovered: nothing is accepted.
|
|
draining, // Shutdown in progress.
|
|
full, // Backpressure: bounded backlog (inputs or bytes) is full.
|
|
duplicate, // Request identity already queued or pending in this host.
|
|
invalid, // Zero request identity.
|
|
};
|
|
// Final: the input's tick is in a durably committed checkpoint.
|
|
struct FinalAck {
|
|
persistence::Uuid request{};
|
|
std::uint64_t tick{};
|
|
std::uint64_t commit_sequence{};
|
|
friend bool operator==(const FinalAck&, const FinalAck&) = default;
|
|
};
|
|
// Final: the permanent outcome is durable (and in the outbox).
|
|
struct FinalOutcome {
|
|
persistence::BatchDecision decision{};
|
|
std::uint64_t commit_sequence{};
|
|
friend bool operator==(const FinalOutcome&, const FinalOutcome&) = default;
|
|
};
|
|
struct Dropped {
|
|
TickInput input{};
|
|
Unacknowledged reason{};
|
|
};
|
|
|
|
// Commit boundaries for fault injection (M7.7 kill tests).
|
|
enum class CrashPoint : std::uint8_t {
|
|
before_upload, // Batch sealed; nothing uploaded or sent.
|
|
before_transaction, // Payload and trace uploaded and verified; manifest not sent.
|
|
after_commit, // Commit confirmed; no acknowledgement or outcome emitted yet.
|
|
};
|
|
using Connect =
|
|
std::function<std::expected<std::unique_ptr<persistence::Postgres>, persistence::PgFailure>()>;
|
|
struct HostConfig {
|
|
persistence::RegionKey key{};
|
|
std::uint64_t frame{};
|
|
persistence::LeasePolicy lease{};
|
|
// Region quantum; must equal the committed checkpoint's tick_microseconds.
|
|
std::chrono::microseconds tick_quantum{50000};
|
|
std::chrono::milliseconds flush_interval{1000}; // Commit a non-empty batch this often.
|
|
std::size_t max_queued_inputs{64};
|
|
std::size_t max_queued_bytes{65536};
|
|
std::uint32_t max_catch_up_ticks{8}; // Per step; debt is never skipped.
|
|
persistence::Sha256Digest build{};
|
|
std::vector<persistence::Sha256Digest> retained_builds;
|
|
std::size_t restore_budget{8000000};
|
|
// Test-only: called at each commit boundary (e.g. to SIGKILL the process).
|
|
// Production configurations leave it empty.
|
|
std::function<void(CrashPoint)> crash_hook;
|
|
};
|
|
enum class HostConfigError : std::uint8_t { invalid };
|
|
|
|
// Scheduling time comes from the caller (monotonic, may be scaled in tests); lease
|
|
// deadlines always use LeaseClock (real, suspend-aware) and database time.
|
|
using SchedulerTime = std::chrono::steady_clock::time_point;
|
|
|
|
template <typename State> class RegionHost {
|
|
public:
|
|
[[nodiscard]] static std::expected<std::unique_ptr<RegionHost>, HostConfigError>
|
|
create(HostConfig config, Connect connect,
|
|
std::unique_ptr<persistence::FilesystemObjectStore> objects);
|
|
RegionHost(const RegionHost&) = delete;
|
|
RegionHost& operator=(const RegionHost&) = delete;
|
|
RegionHost(RegionHost&&) = delete;
|
|
RegionHost& operator=(RegionHost&&) = delete;
|
|
~RegionHost();
|
|
|
|
// Provisional admission only; nothing is final until a FinalAck.
|
|
[[nodiscard]] std::expected<void, AdmissionError> submit(const TickInput& input);
|
|
// Advances the state machine once: lease upkeep, recovery, resolution, commits
|
|
// and up to max_catch_up_ticks ticks. Blocks at most for bounded I/O.
|
|
const HostStatus& step(SchedulerTime now);
|
|
// Stops admission, drops the backlog, commits pending ticks, releases the lease.
|
|
void drain();
|
|
|
|
[[nodiscard]] std::vector<FinalAck> take_acks();
|
|
[[nodiscard]] std::vector<FinalOutcome> take_outcomes();
|
|
[[nodiscard]] std::vector<Dropped> take_dropped();
|
|
[[nodiscard]] const HostStatus& status() const noexcept {
|
|
return status_;
|
|
}
|
|
// Read-only, for operational reporting (free space against the reserve).
|
|
[[nodiscard]] const persistence::FilesystemObjectStore& objects() const noexcept {
|
|
return *objects_;
|
|
}
|
|
// Read-only view for diagnostics/tests; null unless recovered.
|
|
[[nodiscard]] const State* state() const noexcept {
|
|
return state_.get();
|
|
}
|
|
|
|
private:
|
|
struct Applied {
|
|
TickInput input;
|
|
std::uint64_t tick{};
|
|
};
|
|
RegionHost(HostConfig config, Connect connect,
|
|
std::unique_ptr<persistence::FilesystemObjectStore> objects);
|
|
bool database();
|
|
void acquire(SchedulerTime now);
|
|
void upkeep();
|
|
void simulate(SchedulerTime now);
|
|
void commit(SchedulerTime now);
|
|
void resolve(SchedulerTime now);
|
|
void confirmed(const persistence::PublishedBatch& batch, SchedulerTime now);
|
|
void lose_authority(Unacknowledged reason, HostPhase next);
|
|
void refresh(SchedulerTime now);
|
|
[[nodiscard]] persistence::Uuid next_commit_id() const;
|
|
|
|
HostConfig config_;
|
|
Connect connect_;
|
|
std::unique_ptr<persistence::FilesystemObjectStore> objects_;
|
|
std::unique_ptr<persistence::Postgres> db_;
|
|
persistence::LeaseAuthority authority_;
|
|
std::unique_ptr<State> state_;
|
|
std::optional<persistence::RegionBatch<State>> batch_;
|
|
std::optional<persistence::CommittedCheckpoint> committed_;
|
|
std::optional<persistence::PreparedCommit> prepared_;
|
|
std::deque<TickInput> queue_;
|
|
std::vector<Applied> applied_;
|
|
std::vector<FinalAck> acks_;
|
|
std::vector<FinalOutcome> outcomes_;
|
|
std::vector<Dropped> dropped_;
|
|
SchedulerTime origin_{}, batch_started_{};
|
|
std::uint64_t origin_tick_{};
|
|
bool draining_{};
|
|
HostStatus status_{};
|
|
};
|
|
|
|
extern template class RegionHost<processes::EnvironmentalCommunity>;
|
|
extern template class RegionHost<processes::LineageRegion>;
|
|
} // namespace evo::server
|