Complete M7.7 recovery verification before readiness
ADR-0065: before any readiness, recovery now verifies the head's tail. The trace object must exist with its manifest digest, its header must bind profile, region, predecessor and range; the predecessor (schema v6 load_region_commit) must hash to the digest the head names; replaying the trace from the predecessor's verified payload must reproduce every report and the exact head bytes; and trace-derived decisions must equal the head's durable decisions. Missing, corrupt or non-reproducing tails quarantine (RecoveryError::tail); nothing older or empty is served. Test-only crash seams (HostConfig::crash_hook, PublishHooks) and evo-server --crash-at deliver real SIGKILL at before-upload, before-transaction, after-commit and after-ack. The kill matrix, a SIGSTOP/SIGCONT stale owner and damaged-head startups all behave as the durability contract requires; successors verify before readiness. Validated: PostgreSQL suites under GCC Release, clang-tidy and ASan/UBSan; tail-skip and replay-equality mutations each caught. No Docker/Gitea CI, production-lease kill test or deployment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -5,6 +5,7 @@
|
||||
#include "evo/persistence/region_lease_store.hpp"
|
||||
#include <chrono>
|
||||
#include <expected>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <span>
|
||||
#include <vector>
|
||||
@@ -41,6 +42,11 @@ struct PublicationFailure {
|
||||
PreparedCommit prepared; // Present for rejected/unknown.
|
||||
};
|
||||
// Durably confirmed batch: its decisions are now final and may be acknowledged.
|
||||
// Test/fault-injection seam: called after verified uploads, immediately before the
|
||||
// fenced transaction is sent. Production callers pass nothing.
|
||||
struct PublishHooks {
|
||||
std::function<void()> before_transaction;
|
||||
};
|
||||
struct PublishedBatch {
|
||||
CommittedCheckpoint committed;
|
||||
std::vector<BatchDecision> decisions;
|
||||
@@ -54,6 +60,7 @@ enum class RecoveryError : std::uint8_t {
|
||||
corrupt_payload, // Length/digest mismatch: quarantine, no older head.
|
||||
payload, // Verified bytes fail full state validation: quarantine.
|
||||
history, // Durable decisions disagree with the restored lifecycle: quarantine.
|
||||
tail, // Head batch evidence missing, corrupt or not reproducible: quarantine.
|
||||
};
|
||||
struct RecoveryFailure {
|
||||
RecoveryError kind{};
|
||||
@@ -85,9 +92,15 @@ publish_environmental_batch(Postgres& db, const FilesystemObjectStore& objects,
|
||||
const LeaseGrant& lease, const CommittedCheckpoint& predecessor,
|
||||
const RegionBatch<processes::EnvironmentalCommunity>& batch,
|
||||
Uuid commit, const Sha256Digest& build, std::size_t restore_budget,
|
||||
std::chrono::milliseconds timeout);
|
||||
// Loads and fully verifies the committed head and reconciles durable decisions.
|
||||
// Readiness additionally requires a fresh valid lease acquired before this call.
|
||||
std::chrono::milliseconds timeout, const PublishHooks& hooks = {});
|
||||
// Loads and fully verifies the committed head before any readiness (ADR-0065):
|
||||
// manifest digest and chain fields, retained build, payload length/SHA-256 and full
|
||||
// restore; then the head's tail: its trace object binds to the committed
|
||||
// predecessor, replaying it from the predecessor's verified payload reproduces the
|
||||
// head bytes, and its derived decisions equal the durable ones; finally every
|
||||
// durable decision agrees with the restored lifecycle. Any failure quarantines;
|
||||
// nothing older or empty is ever substituted. Readiness also requires a fresh
|
||||
// valid lease acquired before this call.
|
||||
[[nodiscard]] std::expected<RecoveredCheckpoint, RecoveryFailure>
|
||||
recover_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
std::span<const Sha256Digest> retained_builds,
|
||||
@@ -106,7 +119,7 @@ publish_lineage_batch(Postgres& db, const FilesystemObjectStore& objects, const
|
||||
const CommittedCheckpoint& predecessor,
|
||||
const RegionBatch<processes::LineageRegion>& batch, Uuid commit,
|
||||
const Sha256Digest& build, std::size_t restore_budget,
|
||||
std::chrono::milliseconds timeout);
|
||||
std::chrono::milliseconds timeout, const PublishHooks& hooks = {});
|
||||
[[nodiscard]] std::expected<RecoveredLineageCheckpoint, RecoveryFailure>
|
||||
recover_lineage_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
std::span<const Sha256Digest> retained_builds,
|
||||
|
||||
@@ -50,6 +50,11 @@ release_region_lease(Postgres& db, RegionKey key, const LeaseGrant& lease,
|
||||
load_region_head(Postgres& db, RegionKey key, std::chrono::milliseconds timeout);
|
||||
[[nodiscard]] std::expected<CommittedCheckpoint, LeaseStoreFailure>
|
||||
load_region_checkpoint(Postgres& db, RegionKey key, std::chrono::milliseconds timeout);
|
||||
// Any committed link by sequence (schema v6): its identity, digest, end tick and
|
||||
// exact manifest bytes. not_provisioned when the region has no such commit.
|
||||
[[nodiscard]] std::expected<CommittedCheckpoint, LeaseStoreFailure>
|
||||
load_region_commit(Postgres& db, RegionKey key, std::uint64_t sequence,
|
||||
std::chrono::milliseconds timeout);
|
||||
// Canonical manifest v2 bytes plus the decision block they digest (ADR-0063). The
|
||||
// database parses identity, lease and chain, locks the lease then the head, requires
|
||||
// the exact committed predecessor and records decisions and outbox rows in the same
|
||||
|
||||
@@ -75,6 +75,12 @@ struct HostStatus {
|
||||
std::uint64_t commits{}, store_failures{}, unknown_outcomes{}, fences{};
|
||||
};
|
||||
|
||||
// Commit boundaries for fault injection (M7.7 kill tests).
|
||||
enum class CrashPoint : std::uint8_t {
|
||||
before_upload, // Batch sealed; nothing uploaded or sent.
|
||||
before_transaction, // Payload and trace uploaded and verified; manifest not sent.
|
||||
after_commit, // Commit confirmed; no acknowledgement or outcome emitted yet.
|
||||
};
|
||||
using Connect =
|
||||
std::function<std::expected<std::unique_ptr<persistence::Postgres>, persistence::PgFailure>()>;
|
||||
struct HostConfig {
|
||||
@@ -90,6 +96,9 @@ struct HostConfig {
|
||||
persistence::Sha256Digest build{};
|
||||
std::vector<persistence::Sha256Digest> retained_builds;
|
||||
std::size_t restore_budget{8000000};
|
||||
// Test-only: called at each commit boundary (e.g. to SIGKILL the process).
|
||||
// Production configurations leave it empty.
|
||||
std::function<void(CrashPoint)> crash_hook;
|
||||
};
|
||||
enum class HostConfigError : std::uint8_t { invalid };
|
||||
|
||||
|
||||
Reference in New Issue
Block a user