Complete M7.7 recovery verification before readiness

ADR-0065: before any readiness, recovery now verifies the head's tail.
The trace object must exist with its manifest digest, its header must
bind profile, region, predecessor and range; the predecessor (schema v6
load_region_commit) must hash to the digest the head names; replaying
the trace from the predecessor's verified payload must reproduce every
report and the exact head bytes; and trace-derived decisions must equal
the head's durable decisions. Missing, corrupt or non-reproducing tails
quarantine (RecoveryError::tail); nothing older or empty is served.

Test-only crash seams (HostConfig::crash_hook, PublishHooks) and
evo-server --crash-at deliver real SIGKILL at before-upload,
before-transaction, after-commit and after-ack. The kill matrix,
a SIGSTOP/SIGCONT stale owner and damaged-head startups all behave as
the durability contract requires; successors verify before readiness.

Validated: PostgreSQL suites under GCC Release, clang-tidy and
ASan/UBSan; tail-skip and replay-equality mutations each caught. No
Docker/Gitea CI, production-lease kill test or deployment.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-28 22:31:40 -04:00
co-authored by Claude Opus 5.5
parent d6d0408aae
commit 30411a83f9
24 changed files with 990 additions and 284 deletions
+79 -5
View File
@@ -175,7 +175,8 @@ std::expected<PublishedBatch, PublicationFailure>
publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
const CommittedCheckpoint& predecessor,
const RegionBatch<typename Codec::State>& batch, Uuid commit,
const Sha256Digest& build, std::size_t budget, milliseconds timeout) {
const Sha256Digest& build, std::size_t budget, milliseconds timeout,
const PublishHooks& hooks) {
const auto previous = decode_manifest(predecessor.manifest);
if (!previous || previous->profile != Codec::profile || !batch.sealed() ||
batch.steps().empty() || batch.begin_tick() != previous->end_tick)
@@ -231,6 +232,8 @@ publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGra
return std::unexpected{stored.error()};
PreparedCommit prepared{previous->key, commit, *bytes, *decision_bytes};
if (hooks.before_transaction)
hooks.before_transaction();
auto published = publish_region_checkpoint(db, *bytes, *decision_bytes, timeout);
auto committed = confirm(published, std::move(*bytes), *manifest, std::move(prepared));
if (!committed)
@@ -238,6 +241,72 @@ publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGra
return PublishedBatch{std::move(*committed), decisions};
}
// Replays the head's recorded batch from its committed predecessor (ADR-0065).
template <typename Codec>
std::expected<void, RecoveryFailure>
verify_tail(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
const CheckpointManifest& m, std::span<const std::uint8_t> head_payload,
const typename Codec::State& head_state, std::span<const CommittedDecision> durable,
std::size_t budget, milliseconds timeout) {
const auto fail_with = [](RecoveryError kind) {
return std::unexpected{RecoveryFailure{kind, {}}};
};
if (m.sequence == 0)
return {}; // Genesis has no batch.
if (m.manifest_version != 2)
return fail_with(RecoveryError::unsupported);
auto trace_bytes = objects.get({m.batch.trace_sha256, m.batch.trace_bytes});
if (!trace_bytes)
return fail_with(trace_bytes.error() == ObjectError::io ? RecoveryError::store
: RecoveryError::tail);
auto trace = decode_trace(*trace_bytes);
const TraceHeader expected{static_cast<std::uint32_t>(Codec::profile),
key,
m.predecessor_commit,
m.predecessor_digest,
m.begin_tick,
m.end_tick};
if (!trace || trace->header != expected)
return fail_with(RecoveryError::tail);
auto predecessor = load_region_commit(db, key, m.sequence - 1, timeout);
if (!predecessor)
return predecessor.error().kind == LeaseStoreError::not_provisioned
? fail_with(RecoveryError::tail)
: std::unexpected{RecoveryFailure{RecoveryError::store, predecessor.error()}};
const auto digest = sha256(predecessor->manifest);
auto previous = decode_manifest(predecessor->manifest);
if (!digest || *digest != predecessor->head.digest ||
predecessor->head.digest != m.predecessor_digest ||
predecessor->head.commit != m.predecessor_commit || !previous ||
previous->end_tick != m.begin_tick)
return fail_with(RecoveryError::tail);
auto base = objects.get({previous->payload_sha256, previous->payload_bytes});
if (!base)
return fail_with(base.error() == ObjectError::io ? RecoveryError::store
: RecoveryError::tail);
auto replay = Codec::restore(*base, *previous, budget);
if (!replay)
return fail_with(RecoveryError::tail);
for (const auto& step : trace->steps) {
auto report = (*replay)->advance(step.report.tick, step.input);
if (!report || *report != step.report)
return fail_with(RecoveryError::tail);
}
const auto replayed = Codec::snapshot(**replay);
if (!replayed || !std::ranges::equal(*replayed, head_payload))
return fail_with(RecoveryError::tail);
// The head's durable outcomes are exactly those its trace derives.
const auto organisms = Codec::organisms(head_state);
const auto derived = derive_decisions(trace->steps, organisms.producer, organisms.consumer);
std::vector<BatchDecision> recorded;
for (const auto& d : durable)
if (d.commit_sequence == m.sequence)
recorded.push_back(d.decision);
if (derived != recorded || derived.size() != m.batch.decision_count)
return fail_with(RecoveryError::history);
return {};
}
template <typename Codec>
std::expected<typename Codec::Recovered, RecoveryFailure>
recover(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
@@ -284,6 +353,10 @@ recover(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
return store_failure(decisions.error());
if (!Codec::reconcile(**state, *decisions, head))
return fail_with(RecoveryError::history);
if (auto tail = verify_tail<Codec>(db, objects, key, *manifest, *payload, **state, *decisions,
budget, timeout);
!tail)
return std::unexpected{tail.error()};
return typename Codec::Recovered{std::move(*committed), *manifest, std::move(*state),
std::move(*decisions)};
}
@@ -301,9 +374,9 @@ publish_environmental_batch(Postgres& db, const FilesystemObjectStore& objects,
const LeaseGrant& lease, const CommittedCheckpoint& predecessor,
const RegionBatch<processes::EnvironmentalCommunity>& batch,
Uuid commit, const Sha256Digest& build, std::size_t restore_budget,
milliseconds timeout) {
milliseconds timeout, const PublishHooks& hooks) {
return publish_batch<EnvironmentalCodec>(db, objects, lease, predecessor, batch, commit, build,
restore_budget, timeout);
restore_budget, timeout, hooks);
}
std::expected<RecoveredCheckpoint, RecoveryFailure>
recover_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
@@ -324,9 +397,10 @@ std::expected<PublishedBatch, PublicationFailure>
publish_lineage_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
const CommittedCheckpoint& predecessor,
const RegionBatch<processes::LineageRegion>& batch, Uuid commit,
const Sha256Digest& build, std::size_t restore_budget, milliseconds timeout) {
const Sha256Digest& build, std::size_t restore_budget, milliseconds timeout,
const PublishHooks& hooks) {
return publish_batch<LineageCodec>(db, objects, lease, predecessor, batch, commit, build,
restore_budget, timeout);
restore_budget, timeout, hooks);
}
std::expected<RecoveredLineageCheckpoint, RecoveryFailure>
recover_lineage_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,