Files
aevolution/include/evo/persistence/checkpoint_store.hpp
T
dingoandClaude Opus 5.5 30411a83f9 Complete M7.7 recovery verification before readiness
ADR-0065: before any readiness, recovery now verifies the head's tail.
The trace object must exist with its manifest digest, its header must
bind profile, region, predecessor and range; the predecessor (schema v6
load_region_commit) must hash to the digest the head names; replaying
the trace from the predecessor's verified payload must reproduce every
report and the exact head bytes; and trace-derived decisions must equal
the head's durable decisions. Missing, corrupt or non-reproducing tails
quarantine (RecoveryError::tail); nothing older or empty is served.

Test-only crash seams (HostConfig::crash_hook, PublishHooks) and
evo-server --crash-at deliver real SIGKILL at before-upload,
before-transaction, after-commit and after-ack. The kill matrix,
a SIGSTOP/SIGCONT stale owner and damaged-head startups all behave as
the durability contract requires; successors verify before readiness.

Validated: PostgreSQL suites under GCC Release, clang-tidy and
ASan/UBSan; tail-skip and replay-equality mutations each caught. No
Docker/Gitea CI, production-lease kill test or deployment.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-28 22:31:40 -04:00

136 lines
7.7 KiB
C++

#pragma once
#include "evo/persistence/object_store.hpp"
#include "evo/persistence/region_batch.hpp"
#include "evo/persistence/region_checkpoint.hpp"
#include "evo/persistence/region_lease_store.hpp"
#include <chrono>
#include <expected>
#include <functional>
#include <memory>
#include <span>
#include <vector>
// Linux publication and recovery of region checkpoints (ADR-0061/0062/0063).
// Batch order: replay the recorded trace from the committed predecessor payload
// and require the candidate bytes -> derive decisions -> immutable payload and
// trace uploads with verified read-back -> one fenced transaction recording
// checkpoint, decisions and outbox. Nothing before the transaction has authority:
// a failure there leaves the previous head and at most unreferenced orphan objects.
// Permanent outcomes are final only when a receipt (publish or resolve) confirms them.
namespace evo::persistence {
enum class PublicationError : std::uint8_t {
candidate, // State, batch or manifest is not a valid successor; nothing uploaded.
predecessor, // Committed predecessor payload missing/corrupt/invalid: quarantine.
digest, // Digest provider failure; nothing uploaded.
upload, // Object write failed; no manifest was sent.
verify, // Read-back differs from the candidate; no manifest was sent.
rejected, // Database refused (see store); uploads are orphans.
unknown, // Outcome unknown (see store): stay paused; resolve `prepared`.
receipt, // Committed receipt disagrees with the local manifest: fail closed.
};
// The exact bytes sent to the database. After `unknown`, resolve or retry only these.
struct PreparedCommit {
RegionKey key{};
Uuid commit{};
std::vector<std::uint8_t> manifest;
std::vector<std::uint8_t> decisions;
};
struct PublicationFailure {
PublicationError kind{};
LeaseStoreFailure store{}; // Meaningful for rejected/unknown.
ObjectError object{}; // Meaningful for upload/verify/predecessor.
PreparedCommit prepared; // Present for rejected/unknown.
};
// Durably confirmed batch: its decisions are now final and may be acknowledged.
// Test/fault-injection seam: called after verified uploads, immediately before the
// fenced transaction is sent. Production callers pass nothing.
struct PublishHooks {
std::function<void()> before_transaction;
};
struct PublishedBatch {
CommittedCheckpoint committed;
std::vector<BatchDecision> decisions;
};
enum class RecoveryError : std::uint8_t {
store, // Database unavailable/failed (see store).
not_provisioned, // No genesis: recovery never creates one.
manifest, // Manifest bytes disagree with the committed digest/head.
unsupported, // Profile/versions/build not in this host's retained allowlist.
missing_payload, // Referenced object absent: quarantine, no older head.
corrupt_payload, // Length/digest mismatch: quarantine, no older head.
payload, // Verified bytes fail full state validation: quarantine.
history, // Durable decisions disagree with the restored lifecycle: quarantine.
tail, // Head batch evidence missing, corrupt or not reproducible: quarantine.
};
struct RecoveryFailure {
RecoveryError kind{};
LeaseStoreFailure store{};
};
struct RecoveredCheckpoint {
CommittedCheckpoint committed;
CheckpointManifest manifest;
std::unique_ptr<processes::EnvironmentalCommunity> state;
std::vector<CommittedDecision> decisions;
};
struct RecoveredLineageCheckpoint {
CommittedCheckpoint committed;
CheckpointManifest manifest;
std::unique_ptr<processes::LineageRegion> state;
std::vector<CommittedDecision> decisions;
};
// Owner/operator connection only: uploads a tick-0 state and records genesis.
[[nodiscard]] std::expected<CommittedCheckpoint, PublicationFailure> provision_environmental_region(
Postgres& owner, const FilesystemObjectStore& objects, RegionKey key, std::uint64_t frame,
const processes::EnvironmentalCommunity& opening, Uuid commit, const Sha256Digest& build,
std::size_t restore_budget, std::chrono::milliseconds timeout);
// Publishes a sealed batch that began at `predecessor`, under `lease` (which the
// caller has checked with LeaseAuthority::may_begin_commit). On success call
// batch.confirm(end_tick); only then are its decisions final.
[[nodiscard]] std::expected<PublishedBatch, PublicationFailure>
publish_environmental_batch(Postgres& db, const FilesystemObjectStore& objects,
const LeaseGrant& lease, const CommittedCheckpoint& predecessor,
const RegionBatch<processes::EnvironmentalCommunity>& batch,
Uuid commit, const Sha256Digest& build, std::size_t restore_budget,
std::chrono::milliseconds timeout, const PublishHooks& hooks = {});
// Loads and fully verifies the committed head before any readiness (ADR-0065):
// manifest digest and chain fields, retained build, payload length/SHA-256 and full
// restore; then the head's tail: its trace object binds to the committed
// predecessor, replaying it from the predecessor's verified payload reproduces the
// head bytes, and its derived decisions equal the durable ones; finally every
// durable decision agrees with the restored lifecycle. Any failure quarantines;
// nothing older or empty is ever substituted. Readiness also requires a fresh
// valid lease acquired before this call.
[[nodiscard]] std::expected<RecoveredCheckpoint, RecoveryFailure>
recover_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
std::span<const Sha256Digest> retained_builds,
std::size_t restore_budget, std::chrono::milliseconds timeout);
// environmental-lineage-v1 (ADR-0062): the same order and guarantees, with the
// LineageRegion container as payload. A region's profile never changes; each call
// refuses a predecessor/committed head of the other profile.
[[nodiscard]] std::expected<CommittedCheckpoint, PublicationFailure>
provision_lineage_region(Postgres& owner, const FilesystemObjectStore& objects, RegionKey key,
std::uint64_t frame, const processes::LineageRegion& opening, Uuid commit,
const Sha256Digest& build, std::size_t restore_budget,
std::chrono::milliseconds timeout);
[[nodiscard]] std::expected<PublishedBatch, PublicationFailure>
publish_lineage_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
const CommittedCheckpoint& predecessor,
const RegionBatch<processes::LineageRegion>& batch, Uuid commit,
const Sha256Digest& build, std::size_t restore_budget,
std::chrono::milliseconds timeout, const PublishHooks& hooks = {});
[[nodiscard]] std::expected<RecoveredLineageCheckpoint, RecoveryFailure>
recover_lineage_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
std::span<const Sha256Digest> retained_builds,
std::size_t restore_budget, std::chrono::milliseconds timeout);
// After an `unknown` publication: resolve exactly the prepared bytes. A committed
// result yields the final batch; absent may be retried with the same prepared bytes
// only while the lease is still valid; conflict fails closed.
[[nodiscard]] std::expected<ResolvedCommit, LeaseStoreFailure>
resolve_prepared(Postgres& db, const PreparedCommit& prepared, std::chrono::milliseconds timeout);
[[nodiscard]] std::expected<RegionHead, LeaseStoreFailure>
retry_prepared(Postgres& db, const PreparedCommit& prepared, std::chrono::milliseconds timeout);
} // namespace evo::persistence