Complete M7.5 durable batch outcomes and commit resolution

ADR-0063: manifest v2 (360 bytes) binds a per-tick validation trace
object and a canonical decision block by SHA-256. Publication replays the
trace from the committed predecessor's verified payload and requires
every report and the candidate bytes, derives deaths deterministically,
uploads payload and trace with verified read-back, then sends manifest
and decisions to one fenced statement. RegionBatch seals at a terminal
transition or 64 ticks and refuses ticks until durable confirmation.

Schema v5 records the checkpoint, its decisions and outbox rows
atomically; a unique index keeps one terminal outcome per actor
(EVL08), triggers make commits/decisions immutable (EVL09), an exact
retry of a committed ID returns the original receipt, changed bytes
conflict, and read-only resolve_region_commit answers committed/absent/
conflict. Outbox pending/acknowledge give idempotent at-least-once
delivery. Recovery reconciles durable deaths with restored lifecycle
(and archive deaths for the lineage profile).

Validated: Windows CTest 58/58; Linux GCC Debug/Release, clang-tidy and
ASan/UBSan CTest 58/58 each; PostgreSQL suites Release, analysis, ASan
(lost response, committed-after-timeout, terminated orphan retry,
changed bytes, repeated/partial/reordered decisions, gap, outbox
redelivery, history quarantine); five mutations each caught. No
Docker/Gitea CI, process kill, broker or deployment.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-28 17:30:30 -04:00
co-authored by Claude Opus 5.5
parent a2c4543df2
commit dd7e8028b0
27 changed files with 2075 additions and 216 deletions
+180 -69
View File
@@ -8,7 +8,30 @@ namespace evo::persistence {
namespace {
using std::chrono::milliseconds;
// Per-profile payload operations; the publication and recovery paths are shared.
struct Organisms {
std::uint64_t producer{}, consumer{};
};
// Lifecycle agreement shared by both profiles: exactly one durable death per
// terminal actor, none for an active one, never after the committed tick.
bool lifecycle_agrees(const processes::EnvironmentalProgress& progress, Organisms organisms,
std::span<const CommittedDecision> decisions, const RegionHead& head) {
for (const auto actor : {Actor::producer, Actor::consumer}) {
const auto terminal =
actor == Actor::producer ? progress.producer_terminal : progress.consumer_terminal;
const auto count = std::ranges::count_if(
decisions, [&](const CommittedDecision& d) { return d.decision.actor == actor; });
if (count > 1 || (terminal == 1) != (count == 1))
return false;
}
return std::ranges::all_of(decisions, [&](const CommittedDecision& d) {
const auto organism =
d.decision.actor == Actor::producer ? organisms.producer : organisms.consumer;
return d.decision.tick >= 1 && d.decision.tick <= head.end_tick &&
d.commit_sequence <= head.sequence && d.decision.organism == organism;
});
}
// Per-profile payload operations; publication and recovery paths are shared.
struct EnvironmentalCodec {
using State = processes::EnvironmentalCommunity;
using Recovered = RecoveredCheckpoint;
@@ -22,10 +45,17 @@ struct EnvironmentalCodec {
static std::uint64_t quantum(const State& s) {
return s.tick_microseconds();
}
static Organisms organisms(const State&) {
return {};
}
static auto restore(std::span<const std::uint8_t> payload, const CheckpointManifest& m,
std::size_t budget) {
return restore_environmental_payload(payload, m, budget);
}
static bool reconcile(const State& s, std::span<const CommittedDecision> d,
const RegionHead& head) {
return lifecycle_agrees(s.progress(), organisms(s), d, head);
}
};
struct LineageCodec {
using State = processes::LineageRegion;
@@ -40,44 +70,61 @@ struct LineageCodec {
static std::uint64_t quantum(const State& s) {
return s.community().tick_microseconds();
}
static Organisms organisms(const State& s) {
return {s.binding().producer, s.binding().consumer};
}
static auto restore(std::span<const std::uint8_t> payload, const CheckpointManifest& m,
std::size_t budget) {
return restore_lineage_payload(payload, m, budget);
}
// Plus: each durable death is the archive's death, at the same tick.
static bool reconcile(const State& s, std::span<const CommittedDecision> d,
const RegionHead& head) {
if (!lifecycle_agrees(s.community().progress(), organisms(s), d, head))
return false;
return std::ranges::all_of(d, [&](const CommittedDecision& x) {
const auto record = s.archive().organism(x.decision.organism);
return record && !record->alive && record->died.value == x.decision.tick;
});
}
};
PublicationFailure fail(PublicationError kind) {
return {kind, {}, {}};
PublicationFailure fail(PublicationError kind, ObjectError object = {}) {
return {kind, {}, object, {}};
}
// Validates the exact payload bytes as recovery would, uploads and reads back.
// Returns the manifest bytes ready for the database.
template <typename Codec>
std::expected<std::vector<std::uint8_t>, PublicationFailure>
stage(const FilesystemObjectStore& objects, const std::vector<std::uint8_t>& payload,
const CheckpointManifest& manifest, std::size_t budget) {
auto bytes = encode_manifest(manifest);
if (!bytes || manifest.profile != Codec::profile || !Codec::restore(payload, manifest, budget))
return std::unexpected{fail(PublicationError::candidate)};
auto stored = objects.put(payload);
std::string hex_parameter(std::span<const std::uint8_t> bytes) {
constexpr char digits[] = "0123456789abcdef";
std::string out = "\\x";
for (const auto b : bytes) {
out.push_back(digits[b >> 4U]);
out.push_back(digits[b & 0x0FU]);
}
return out;
}
std::expected<void, PublicationFailure> put_verified(const FilesystemObjectStore& objects,
const std::vector<std::uint8_t>& bytes,
const VerifiedObject& expected) {
auto stored = objects.put(bytes);
if (!stored)
return std::unexpected{PublicationFailure{PublicationError::upload, {}, stored.error()}};
return std::unexpected{fail(PublicationError::upload, stored.error())};
auto back = objects.get(*stored);
if (!back || *stored != VerifiedObject{manifest.payload_sha256, manifest.payload_bytes} ||
!std::ranges::equal(*back, payload))
return std::unexpected{PublicationFailure{
PublicationError::verify, {}, back ? ObjectError::corrupt : back.error()}};
return std::move(*bytes);
if (!back || *stored != expected || !std::ranges::equal(*back, bytes))
return std::unexpected{
fail(PublicationError::verify, back ? ObjectError::corrupt : back.error())};
return {};
}
// The receipt must agree with the local manifest the database digested.
std::expected<CommittedCheckpoint, PublicationFailure>
confirm(const std::expected<RegionHead, LeaseStoreFailure>& published,
std::vector<std::uint8_t> manifest_bytes, const CheckpointManifest& manifest) {
std::vector<std::uint8_t> manifest_bytes, const CheckpointManifest& manifest,
PreparedCommit prepared) {
if (!published) {
const auto kind = published.error().kind == LeaseStoreError::unknown
? PublicationError::unknown
: PublicationError::rejected;
return std::unexpected{PublicationFailure{kind, published.error(), {}}};
return std::unexpected{
PublicationFailure{kind, published.error(), {}, std::move(prepared)}};
}
// The database digested the bytes independently; the receipt must agree.
const auto digest = sha256(manifest_bytes);
if (!digest || published->digest != *digest || published->sequence != manifest.sequence ||
published->commit != manifest.commit || published->end_tick != manifest.end_tick ||
@@ -99,67 +146,111 @@ provision(Postgres& owner, const FilesystemObjectStore& objects, RegionKey key,
return std::unexpected{fail(PublicationError::digest)};
auto manifest = genesis_manifest(key, frame, commit, Codec::quantum(opening), build,
payload->size(), *digest, Codec::profile);
if (!manifest)
auto bytes = manifest ? encode_manifest(*manifest)
: std::expected<std::vector<std::uint8_t>, CheckpointError>{
std::unexpected{CheckpointError::format}};
if (!bytes || !Codec::restore(*payload, *manifest, budget))
return std::unexpected{fail(PublicationError::candidate)};
auto bytes = stage<Codec>(objects, *payload, *manifest, budget);
if (!bytes)
return std::unexpected{bytes.error()};
constexpr char digits[] = "0123456789abcdef";
std::string parameter = "\\x";
for (const auto b : *bytes) {
parameter.push_back(digits[b >> 4U]);
parameter.push_back(digits[b & 0x0FU]);
}
const std::array<std::string, 1> p{parameter};
if (auto stored = put_verified(objects, *payload, {*digest, payload->size()}); !stored)
return std::unexpected{stored.error()};
const std::array<std::string, 1> p{hex_parameter(*bytes)};
auto done = owner.query("SELECT evolution.provision_region_checkpoint($1::bytea)", p, timeout);
if (!done) {
const bool unknown = done.error().kind == PgError::unknown;
return std::unexpected{PublicationFailure{
unknown ? PublicationError::unknown : PublicationError::rejected,
{unknown ? LeaseStoreError::unknown : LeaseStoreError::store, done.error()},
{},
{}}};
}
auto committed = load_region_checkpoint(owner, key, timeout);
if (!committed)
return std::unexpected{
PublicationFailure{PublicationError::unknown, committed.error(), {}}};
return confirm(committed->head, std::move(*bytes), *manifest);
PublicationFailure{PublicationError::unknown, committed.error(), {}, {}}};
return confirm(committed->head, std::move(*bytes), *manifest, {});
}
template <typename Codec>
std::expected<CommittedCheckpoint, PublicationFailure>
publish(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
const CommittedCheckpoint& predecessor, const typename Codec::State& state, Uuid commit,
const Sha256Digest& build, std::size_t budget, milliseconds timeout) {
std::expected<PublishedBatch, PublicationFailure>
publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
const CommittedCheckpoint& predecessor,
const RegionBatch<typename Codec::State>& batch, Uuid commit,
const Sha256Digest& build, std::size_t budget, milliseconds timeout) {
const auto previous = decode_manifest(predecessor.manifest);
auto payload = Codec::snapshot(state);
if (!previous || !payload || previous->profile != Codec::profile)
if (!previous || previous->profile != Codec::profile || !batch.sealed() ||
batch.steps().empty() || batch.begin_tick() != previous->end_tick)
return std::unexpected{fail(PublicationError::candidate)};
const auto digest = sha256(*payload);
if (!digest)
// Replay the recorded inputs from the committed predecessor's verified bytes.
auto base = objects.get({previous->payload_sha256, previous->payload_bytes});
if (!base)
return std::unexpected{fail(PublicationError::predecessor, base.error())};
auto replay = Codec::restore(*base, *previous, budget);
if (!replay)
return std::unexpected{fail(PublicationError::predecessor)};
for (const auto& step : batch.steps()) {
auto report = (*replay)->advance(step.report.tick, step.input);
if (!report || *report != step.report)
return std::unexpected{fail(PublicationError::candidate)};
}
auto payload = Codec::snapshot(batch.state());
const auto replayed = Codec::snapshot(**replay);
if (!payload || !replayed || *payload != *replayed)
return std::unexpected{fail(PublicationError::candidate)};
const auto organisms = Codec::organisms(batch.state());
const auto decisions = derive_decisions(batch.steps(), organisms.producer, organisms.consumer);
auto decision_bytes = encode_decisions(decisions, batch.begin_tick(), batch.end_tick());
const TraceHeader header{static_cast<std::uint32_t>(Codec::profile),
previous->key,
previous->commit,
predecessor.head.digest,
batch.begin_tick(),
batch.end_tick()};
auto trace = encode_trace(header, batch.steps());
if (!decision_bytes || !trace)
return std::unexpected{fail(PublicationError::candidate)};
const auto payload_digest = sha256(*payload);
const auto trace_digest = sha256(*trace);
const auto decisions_digest = sha256(*decision_bytes);
if (!payload_digest || !trace_digest || !decisions_digest)
return std::unexpected{fail(PublicationError::digest)};
auto manifest = successor_manifest(*previous, predecessor.head.digest, commit, lease,
Codec::tick(state), build, payload->size(), *digest);
if (!manifest)
const BatchDigests digests{trace->size(), *trace_digest,
static_cast<std::uint32_t>(decisions.size()),
decisions.empty() ? Sha256Digest{} : *decisions_digest};
auto manifest =
successor_manifest(*previous, predecessor.head.digest, commit, lease, batch.end_tick(),
build, payload->size(), *payload_digest, digests);
auto bytes = manifest ? encode_manifest(*manifest)
: std::expected<std::vector<std::uint8_t>, CheckpointError>{
std::unexpected{CheckpointError::format}};
if (!bytes || !Codec::restore(*payload, *manifest, budget))
return std::unexpected{fail(PublicationError::candidate)};
auto bytes = stage<Codec>(objects, *payload, *manifest, budget);
if (!bytes)
return std::unexpected{bytes.error()};
auto published = publish_region_checkpoint(db, *bytes, timeout);
return confirm(published, std::move(*bytes), *manifest);
if (auto stored = put_verified(objects, *payload, {*payload_digest, payload->size()}); !stored)
return std::unexpected{stored.error()};
if (auto stored = put_verified(objects, *trace, {*trace_digest, trace->size()}); !stored)
return std::unexpected{stored.error()};
PreparedCommit prepared{previous->key, commit, *bytes, *decision_bytes};
auto published = publish_region_checkpoint(db, *bytes, *decision_bytes, timeout);
auto committed = confirm(published, std::move(*bytes), *manifest, std::move(prepared));
if (!committed)
return std::unexpected{committed.error()};
return PublishedBatch{std::move(*committed), decisions};
}
template <typename Codec>
std::expected<typename Codec::Recovered, RecoveryFailure>
recover(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
std::span<const Sha256Digest> retained_builds, std::size_t budget, milliseconds timeout) {
const auto store_failure = [](const LeaseStoreFailure& f) {
return std::unexpected{RecoveryFailure{f.kind == LeaseStoreError::not_provisioned
? RecoveryError::not_provisioned
: RecoveryError::store,
f}};
};
auto committed = load_region_checkpoint(db, key, timeout);
if (!committed)
return std::unexpected{
RecoveryFailure{committed.error().kind == LeaseStoreError::not_provisioned
? RecoveryError::not_provisioned
: RecoveryError::store,
committed.error()}};
return store_failure(committed.error());
const auto fail_with = [](RecoveryError kind) {
return std::unexpected{RecoveryFailure{kind, {}}};
};
@@ -187,7 +278,14 @@ recover(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
auto state = Codec::restore(*payload, *manifest, budget);
if (!state)
return fail_with(RecoveryError::payload);
return typename Codec::Recovered{std::move(*committed), *manifest, std::move(*state)};
// Durable permanent outcomes must agree with the restored state they finalized.
auto decisions = load_region_decisions(db, key, timeout);
if (!decisions)
return store_failure(decisions.error());
if (!Codec::reconcile(**state, *decisions, head))
return fail_with(RecoveryError::history);
return typename Codec::Recovered{std::move(*committed), *manifest, std::move(*state),
std::move(*decisions)};
}
} // namespace
@@ -198,12 +296,14 @@ std::expected<CommittedCheckpoint, PublicationFailure> provision_environmental_r
return provision<EnvironmentalCodec>(owner, objects, key, frame, opening, commit, build,
restore_budget, timeout);
}
std::expected<CommittedCheckpoint, PublicationFailure> publish_environmental_checkpoint(
Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
const CommittedCheckpoint& predecessor, const processes::EnvironmentalCommunity& state,
Uuid commit, const Sha256Digest& build, std::size_t restore_budget, milliseconds timeout) {
return publish<EnvironmentalCodec>(db, objects, lease, predecessor, state, commit, build,
restore_budget, timeout);
std::expected<PublishedBatch, PublicationFailure>
publish_environmental_batch(Postgres& db, const FilesystemObjectStore& objects,
const LeaseGrant& lease, const CommittedCheckpoint& predecessor,
const RegionBatch<processes::EnvironmentalCommunity>& batch,
Uuid commit, const Sha256Digest& build, std::size_t restore_budget,
milliseconds timeout) {
return publish_batch<EnvironmentalCodec>(db, objects, lease, predecessor, batch, commit, build,
restore_budget, timeout);
}
std::expected<RecoveredCheckpoint, RecoveryFailure>
recover_environmental_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
@@ -220,12 +320,13 @@ provision_lineage_region(Postgres& owner, const FilesystemObjectStore& objects,
return provision<LineageCodec>(owner, objects, key, frame, opening, commit, build,
restore_budget, timeout);
}
std::expected<CommittedCheckpoint, PublicationFailure> publish_lineage_checkpoint(
Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
const CommittedCheckpoint& predecessor, const processes::LineageRegion& state, Uuid commit,
const Sha256Digest& build, std::size_t restore_budget, milliseconds timeout) {
return publish<LineageCodec>(db, objects, lease, predecessor, state, commit, build,
restore_budget, timeout);
std::expected<PublishedBatch, PublicationFailure>
publish_lineage_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGrant& lease,
const CommittedCheckpoint& predecessor,
const RegionBatch<processes::LineageRegion>& batch, Uuid commit,
const Sha256Digest& build, std::size_t restore_budget, milliseconds timeout) {
return publish_batch<LineageCodec>(db, objects, lease, predecessor, batch, commit, build,
restore_budget, timeout);
}
std::expected<RecoveredLineageCheckpoint, RecoveryFailure>
recover_lineage_checkpoint(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
@@ -233,4 +334,14 @@ recover_lineage_checkpoint(Postgres& db, const FilesystemObjectStore& objects, R
std::size_t restore_budget, milliseconds timeout) {
return recover<LineageCodec>(db, objects, key, retained_builds, restore_budget, timeout);
}
std::expected<ResolvedCommit, LeaseStoreFailure>
resolve_prepared(Postgres& db, const PreparedCommit& prepared, milliseconds timeout) {
return resolve_region_commit(db, prepared.key, prepared.commit, prepared.manifest,
prepared.decisions, timeout);
}
std::expected<RegionHead, LeaseStoreFailure>
retry_prepared(Postgres& db, const PreparedCommit& prepared, milliseconds timeout) {
return publish_region_checkpoint(db, prepared.manifest, prepared.decisions, timeout);
}
} // namespace evo::persistence