Complete M7.7 recovery verification before readiness
ADR-0065: before any readiness, recovery now verifies the head's tail. The trace object must exist with its manifest digest, its header must bind profile, region, predecessor and range; the predecessor (schema v6 load_region_commit) must hash to the digest the head names; replaying the trace from the predecessor's verified payload must reproduce every report and the exact head bytes; and trace-derived decisions must equal the head's durable decisions. Missing, corrupt or non-reproducing tails quarantine (RecoveryError::tail); nothing older or empty is served. Test-only crash seams (HostConfig::crash_hook, PublishHooks) and evo-server --crash-at deliver real SIGKILL at before-upload, before-transaction, after-commit and after-ack. The kill matrix, a SIGSTOP/SIGCONT stale owner and damaged-head startups all behave as the durability contract requires; successors verify before readiness. Validated: PostgreSQL suites under GCC Release, clang-tidy and ASan/UBSan; tail-skip and replay-equality mutations each caught. No Docker/Gitea CI, production-lease kill test or deployment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -175,7 +175,8 @@ std::expected<PublishedBatch, PublicationFailure>
|
||||
publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
|
||||
const CommittedCheckpoint& predecessor,
|
||||
const RegionBatch<typename Codec::State>& batch, Uuid commit,
|
||||
const Sha256Digest& build, std::size_t budget, milliseconds timeout) {
|
||||
const Sha256Digest& build, std::size_t budget, milliseconds timeout,
|
||||
const PublishHooks& hooks) {
|
||||
const auto previous = decode_manifest(predecessor.manifest);
|
||||
if (!previous || previous->profile != Codec::profile || !batch.sealed() ||
|
||||
batch.steps().empty() || batch.begin_tick() != previous->end_tick)
|
||||
@@ -231,6 +232,8 @@ publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGra
|
||||
return std::unexpected{stored.error()};
|
||||
|
||||
PreparedCommit prepared{previous->key, commit, *bytes, *decision_bytes};
|
||||
if (hooks.before_transaction)
|
||||
hooks.before_transaction();
|
||||
auto published = publish_region_checkpoint(db, *bytes, *decision_bytes, timeout);
|
||||
auto committed = confirm(published, std::move(*bytes), *manifest, std::move(prepared));
|
||||
if (!committed)
|
||||
@@ -238,6 +241,72 @@ publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGra
|
||||
return PublishedBatch{std::move(*committed), decisions};
|
||||
}
|
||||
|
||||
// Replays the head's recorded batch from its committed predecessor (ADR-0065).
|
||||
template <typename Codec>
|
||||
std::expected<void, RecoveryFailure>
|
||||
verify_tail(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
const CheckpointManifest& m, std::span<const std::uint8_t> head_payload,
|
||||
const typename Codec::State& head_state, std::span<const CommittedDecision> durable,
|
||||
std::size_t budget, milliseconds timeout) {
|
||||
const auto fail_with = [](RecoveryError kind) {
|
||||
return std::unexpected{RecoveryFailure{kind, {}}};
|
||||
};
|
||||
if (m.sequence == 0)
|
||||
return {}; // Genesis has no batch.
|
||||
if (m.manifest_version != 2)
|
||||
return fail_with(RecoveryError::unsupported);
|
||||
auto trace_bytes = objects.get({m.batch.trace_sha256, m.batch.trace_bytes});
|
||||
if (!trace_bytes)
|
||||
return fail_with(trace_bytes.error() == ObjectError::io ? RecoveryError::store
|
||||
: RecoveryError::tail);
|
||||
auto trace = decode_trace(*trace_bytes);
|
||||
const TraceHeader expected{static_cast<std::uint32_t>(Codec::profile),
|
||||
key,
|
||||
m.predecessor_commit,
|
||||
m.predecessor_digest,
|
||||
m.begin_tick,
|
||||
m.end_tick};
|
||||
if (!trace || trace->header != expected)
|
||||
return fail_with(RecoveryError::tail);
|
||||
auto predecessor = load_region_commit(db, key, m.sequence - 1, timeout);
|
||||
if (!predecessor)
|
||||
return predecessor.error().kind == LeaseStoreError::not_provisioned
|
||||
? fail_with(RecoveryError::tail)
|
||||
: std::unexpected{RecoveryFailure{RecoveryError::store, predecessor.error()}};
|
||||
const auto digest = sha256(predecessor->manifest);
|
||||
auto previous = decode_manifest(predecessor->manifest);
|
||||
if (!digest || *digest != predecessor->head.digest ||
|
||||
predecessor->head.digest != m.predecessor_digest ||
|
||||
predecessor->head.commit != m.predecessor_commit || !previous ||
|
||||
previous->end_tick != m.begin_tick)
|
||||
return fail_with(RecoveryError::tail);
|
||||
auto base = objects.get({previous->payload_sha256, previous->payload_bytes});
|
||||
if (!base)
|
||||
return fail_with(base.error() == ObjectError::io ? RecoveryError::store
|
||||
: RecoveryError::tail);
|
||||
auto replay = Codec::restore(*base, *previous, budget);
|
||||
if (!replay)
|
||||
return fail_with(RecoveryError::tail);
|
||||
for (const auto& step : trace->steps) {
|
||||
auto report = (*replay)->advance(step.report.tick, step.input);
|
||||
if (!report || *report != step.report)
|
||||
return fail_with(RecoveryError::tail);
|
||||
}
|
||||
const auto replayed = Codec::snapshot(**replay);
|
||||
if (!replayed || !std::ranges::equal(*replayed, head_payload))
|
||||
return fail_with(RecoveryError::tail);
|
||||
// The head's durable outcomes are exactly those its trace derives.
|
||||
const auto organisms = Codec::organisms(head_state);
|
||||
const auto derived = derive_decisions(trace->steps, organisms.producer, organisms.consumer);
|
||||
std::vector<BatchDecision> recorded;
|
||||
for (const auto& d : durable)
|
||||
if (d.commit_sequence == m.sequence)
|
||||
recorded.push_back(d.decision);
|
||||
if (derived != recorded || derived.size() != m.batch.decision_count)
|
||||
return fail_with(RecoveryError::history);
|
||||
return {};
|
||||
}
|
||||
|
||||
template <typename Codec>
|
||||
std::expected<typename Codec::Recovered, RecoveryFailure>
|
||||
recover(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
@@ -284,6 +353,10 @@ recover(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
return store_failure(decisions.error());
|
||||
if (!Codec::reconcile(**state, *decisions, head))
|
||||
return fail_with(RecoveryError::history);
|
||||
if (auto tail = verify_tail<Codec>(db, objects, key, *manifest, *payload, **state, *decisions,
|
||||
budget, timeout);
|
||||
!tail)
|
||||
return std::unexpected{tail.error()};
|
||||
return typename Codec::Recovered{std::move(*committed), *manifest, std::move(*state),
|
||||
std::move(*decisions)};
|
||||
}
|
||||
@@ -301,9 +374,9 @@ publish_environmental_batch(Postgres& db, const FilesystemObjectStore& objects,
|
||||
const LeaseGrant& lease, const CommittedCheckpoint& predecessor,
|
||||
const RegionBatch<processes::EnvironmentalCommunity>& batch,
|
||||
Uuid commit, const Sha256Digest& build, std::size_t restore_budget,
|
||||
milliseconds timeout) {
|
||||
milliseconds timeout, const PublishHooks& hooks) {
|
||||
return publish_batch<EnvironmentalCodec>(db, objects, lease, predecessor, batch, commit, build,
|
||||
restore_budget, timeout);
|
||||
restore_budget, timeout, hooks);
|
||||
}
|
||||
std::expected<RecoveredCheckpoint, RecoveryFailure>
|
||||
recover_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
@@ -324,9 +397,10 @@ std::expected<PublishedBatch, PublicationFailure>
|
||||
publish_lineage_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
|
||||
const CommittedCheckpoint& predecessor,
|
||||
const RegionBatch<processes::LineageRegion>& batch, Uuid commit,
|
||||
const Sha256Digest& build, std::size_t restore_budget, milliseconds timeout) {
|
||||
const Sha256Digest& build, std::size_t restore_budget, milliseconds timeout,
|
||||
const PublishHooks& hooks) {
|
||||
return publish_batch<LineageCodec>(db, objects, lease, predecessor, batch, commit, build,
|
||||
restore_budget, timeout);
|
||||
restore_budget, timeout, hooks);
|
||||
}
|
||||
std::expected<RecoveredLineageCheckpoint, RecoveryFailure>
|
||||
recover_lineage_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
|
||||
Reference in New Issue
Block a user