Complete M7.9a region clones and retained-history audit
Clone a verified committed head into an isolated world and database (ADR-0067): world re-identification with a proven byte round trip, a schema v7 clone genesis naming the source commit and digest with digested inherited outcomes, a read-only source session, an owner-only atomic target transaction and the evo-world-clone tool. audit_region_history re-verifies every retained link. Real-process tests cover trajectory equality after the cut, source isolation, killed clones and retention. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,278 @@
|
||||
#pragma once
|
||||
// Private to src/persistence: payload codecs, verified uploads, receipt checks, tail
|
||||
// verification and full recovery shared by publication (checkpoint_store.cpp) and
|
||||
// clone/audit (region_clone.cpp). Not an installed header.
|
||||
#include "evo/persistence/checkpoint_store.hpp"
|
||||
#include "evo/persistence/sha256.hpp"
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <string>
|
||||
|
||||
namespace evo::persistence::detail {
|
||||
using std::chrono::milliseconds;
|
||||
|
||||
struct Organisms {
|
||||
std::uint64_t producer{}, consumer{};
|
||||
};
|
||||
// Lifecycle agreement shared by both profiles: exactly one durable death per
|
||||
// terminal actor, none for an active one, never after the committed tick.
|
||||
inline bool lifecycle_agrees(const processes::EnvironmentalProgress& progress, Organisms organisms,
|
||||
std::span<const CommittedDecision> decisions, const RegionHead& head) {
|
||||
for (const auto actor : {Actor::producer, Actor::consumer}) {
|
||||
const auto terminal =
|
||||
actor == Actor::producer ? progress.producer_terminal : progress.consumer_terminal;
|
||||
const auto count = std::ranges::count_if(
|
||||
decisions, [&](const CommittedDecision& d) { return d.decision.actor == actor; });
|
||||
if (count > 1 || (terminal == 1) != (count == 1))
|
||||
return false;
|
||||
}
|
||||
return std::ranges::all_of(decisions, [&](const CommittedDecision& d) {
|
||||
const auto organism =
|
||||
d.decision.actor == Actor::producer ? organisms.producer : organisms.consumer;
|
||||
return d.decision.tick >= 1 && d.decision.tick <= head.end_tick &&
|
||||
d.commit_sequence <= head.sequence && d.decision.organism == organism;
|
||||
});
|
||||
}
|
||||
|
||||
// Per-profile payload operations; publication and recovery paths are shared.
|
||||
struct EnvironmentalCodec {
|
||||
using State = processes::EnvironmentalCommunity;
|
||||
using Recovered = RecoveredCheckpoint;
|
||||
static constexpr auto profile = CheckpointProfile::environmental_v1;
|
||||
static auto snapshot(const State& s) {
|
||||
return s.snapshot();
|
||||
}
|
||||
static std::uint64_t tick(const State& s) {
|
||||
return s.progress().tick;
|
||||
}
|
||||
static std::uint64_t quantum(const State& s) {
|
||||
return s.tick_microseconds();
|
||||
}
|
||||
static Organisms organisms(const State&) {
|
||||
return {};
|
||||
}
|
||||
static auto restore(std::span<const std::uint8_t> payload, const CheckpointManifest& m,
|
||||
std::size_t budget) {
|
||||
return restore_environmental_payload(payload, m, budget);
|
||||
}
|
||||
static bool reconcile(const State& s, std::span<const CommittedDecision> d,
|
||||
const RegionHead& head) {
|
||||
return lifecycle_agrees(s.progress(), organisms(s), d, head);
|
||||
}
|
||||
};
|
||||
struct LineageCodec {
|
||||
using State = processes::LineageRegion;
|
||||
using Recovered = RecoveredLineageCheckpoint;
|
||||
static constexpr auto profile = CheckpointProfile::environmental_lineage_v1;
|
||||
static auto snapshot(const State& s) {
|
||||
return s.snapshot();
|
||||
}
|
||||
static std::uint64_t tick(const State& s) {
|
||||
return s.community().progress().tick;
|
||||
}
|
||||
static std::uint64_t quantum(const State& s) {
|
||||
return s.community().tick_microseconds();
|
||||
}
|
||||
static Organisms organisms(const State& s) {
|
||||
return {s.binding().producer, s.binding().consumer};
|
||||
}
|
||||
static auto restore(std::span<const std::uint8_t> payload, const CheckpointManifest& m,
|
||||
std::size_t budget) {
|
||||
return restore_lineage_payload(payload, m, budget);
|
||||
}
|
||||
// Plus: each durable death is the archive's death, at the same tick.
|
||||
static bool reconcile(const State& s, std::span<const CommittedDecision> d,
|
||||
const RegionHead& head) {
|
||||
if (!lifecycle_agrees(s.community().progress(), organisms(s), d, head))
|
||||
return false;
|
||||
return std::ranges::all_of(d, [&](const CommittedDecision& x) {
|
||||
const auto record = s.archive().organism(x.decision.organism);
|
||||
return record && !record->alive && record->died.value == x.decision.tick;
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
inline PublicationFailure fail(PublicationError kind, ObjectError object = {}) {
|
||||
return {kind, {}, object, {}};
|
||||
}
|
||||
inline std::string hex_parameter(std::span<const std::uint8_t> bytes) {
|
||||
constexpr char digits[] = "0123456789abcdef";
|
||||
std::string out = "\\x";
|
||||
for (const auto b : bytes) {
|
||||
out.push_back(digits[b >> 4U]);
|
||||
out.push_back(digits[b & 0x0FU]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
inline std::expected<void, PublicationFailure> put_verified(const FilesystemObjectStore& objects,
|
||||
const std::vector<std::uint8_t>& bytes,
|
||||
const VerifiedObject& expected) {
|
||||
auto stored = objects.put(bytes);
|
||||
if (!stored)
|
||||
return std::unexpected{fail(PublicationError::upload, stored.error())};
|
||||
auto back = objects.get(*stored);
|
||||
if (!back || *stored != expected || !std::ranges::equal(*back, bytes))
|
||||
return std::unexpected{
|
||||
fail(PublicationError::verify, back ? ObjectError::corrupt : back.error())};
|
||||
return {};
|
||||
}
|
||||
// The receipt must agree with the local manifest the database digested.
|
||||
inline std::expected<CommittedCheckpoint, PublicationFailure>
|
||||
confirm(const std::expected<RegionHead, LeaseStoreFailure>& published,
|
||||
std::vector<std::uint8_t> manifest_bytes, const CheckpointManifest& manifest,
|
||||
PreparedCommit prepared) {
|
||||
if (!published) {
|
||||
const auto kind = published.error().kind == LeaseStoreError::unknown
|
||||
? PublicationError::unknown
|
||||
: PublicationError::rejected;
|
||||
return std::unexpected{
|
||||
PublicationFailure{kind, published.error(), {}, std::move(prepared)}};
|
||||
}
|
||||
const auto digest = sha256(manifest_bytes);
|
||||
if (!digest || published->digest != *digest || published->sequence != manifest.sequence ||
|
||||
published->commit != manifest.commit || published->end_tick != manifest.end_tick ||
|
||||
published->frame != manifest.frame)
|
||||
return std::unexpected{fail(PublicationError::receipt)};
|
||||
return CommittedCheckpoint{*published, std::move(manifest_bytes)};
|
||||
}
|
||||
|
||||
// Replays the head's recorded batch from its committed predecessor (ADR-0065).
|
||||
template <typename Codec>
|
||||
std::expected<void, RecoveryFailure>
|
||||
verify_tail(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
const CheckpointManifest& m, std::span<const std::uint8_t> head_payload,
|
||||
const typename Codec::State& head_state, std::span<const CommittedDecision> durable,
|
||||
std::size_t budget, milliseconds timeout) {
|
||||
const auto fail_with = [](RecoveryError kind) {
|
||||
return std::unexpected{RecoveryFailure{kind, {}}};
|
||||
};
|
||||
if (m.sequence == 0) {
|
||||
// Genesis has no batch. A clone genesis digests the outcomes it inherited
|
||||
// (ADR-0067); a provisioned genesis has none.
|
||||
std::vector<BatchDecision> inherited;
|
||||
for (const auto& d : durable)
|
||||
if (d.commit_sequence == 0)
|
||||
inherited.push_back(d.decision);
|
||||
bool agree = false;
|
||||
if (const auto block = encode_decisions(inherited, 0, m.end_tick); block) {
|
||||
const auto digest = sha256(*block);
|
||||
agree = inherited.empty() ? m.batch.decisions_sha256 == Sha256Digest{}
|
||||
: digest && *digest == m.batch.decisions_sha256;
|
||||
}
|
||||
if (!agree || inherited.size() != m.batch.decision_count)
|
||||
return fail_with(RecoveryError::history);
|
||||
return {};
|
||||
}
|
||||
if (m.manifest_version != 2)
|
||||
return fail_with(RecoveryError::unsupported);
|
||||
auto trace_bytes = objects.get({m.batch.trace_sha256, m.batch.trace_bytes});
|
||||
if (!trace_bytes)
|
||||
return fail_with(trace_bytes.error() == ObjectError::io ? RecoveryError::store
|
||||
: RecoveryError::tail);
|
||||
auto trace = decode_trace(*trace_bytes);
|
||||
const TraceHeader expected{static_cast<std::uint32_t>(Codec::profile),
|
||||
key,
|
||||
m.predecessor_commit,
|
||||
m.predecessor_digest,
|
||||
m.begin_tick,
|
||||
m.end_tick};
|
||||
if (!trace || trace->header != expected)
|
||||
return fail_with(RecoveryError::tail);
|
||||
auto predecessor = load_region_commit(db, key, m.sequence - 1, timeout);
|
||||
if (!predecessor)
|
||||
return predecessor.error().kind == LeaseStoreError::not_provisioned
|
||||
? fail_with(RecoveryError::tail)
|
||||
: std::unexpected{RecoveryFailure{RecoveryError::store, predecessor.error()}};
|
||||
const auto digest = sha256(predecessor->manifest);
|
||||
auto previous = decode_manifest(predecessor->manifest);
|
||||
if (!digest || *digest != predecessor->head.digest ||
|
||||
predecessor->head.digest != m.predecessor_digest ||
|
||||
predecessor->head.commit != m.predecessor_commit || !previous ||
|
||||
previous->end_tick != m.begin_tick)
|
||||
return fail_with(RecoveryError::tail);
|
||||
auto base = objects.get({previous->payload_sha256, previous->payload_bytes});
|
||||
if (!base)
|
||||
return fail_with(base.error() == ObjectError::io ? RecoveryError::store
|
||||
: RecoveryError::tail);
|
||||
auto replay = Codec::restore(*base, *previous, budget);
|
||||
if (!replay)
|
||||
return fail_with(RecoveryError::tail);
|
||||
for (const auto& step : trace->steps) {
|
||||
auto report = (*replay)->advance(step.report.tick, step.input);
|
||||
if (!report || *report != step.report)
|
||||
return fail_with(RecoveryError::tail);
|
||||
}
|
||||
const auto replayed = Codec::snapshot(**replay);
|
||||
if (!replayed || !std::ranges::equal(*replayed, head_payload))
|
||||
return fail_with(RecoveryError::tail);
|
||||
// The head's durable outcomes are exactly those its trace derives.
|
||||
const auto organisms = Codec::organisms(head_state);
|
||||
const auto derived = derive_decisions(trace->steps, organisms.producer, organisms.consumer);
|
||||
std::vector<BatchDecision> recorded;
|
||||
for (const auto& d : durable)
|
||||
if (d.commit_sequence == m.sequence)
|
||||
recorded.push_back(d.decision);
|
||||
if (derived != recorded || derived.size() != m.batch.decision_count)
|
||||
return fail_with(RecoveryError::history);
|
||||
return {};
|
||||
}
|
||||
|
||||
template <typename Codec>
|
||||
std::expected<typename Codec::Recovered, RecoveryFailure>
|
||||
recover(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
std::span<const Sha256Digest> retained_builds, std::size_t budget, milliseconds timeout) {
|
||||
const auto store_failure = [](const LeaseStoreFailure& f) {
|
||||
return std::unexpected{RecoveryFailure{f.kind == LeaseStoreError::not_provisioned
|
||||
? RecoveryError::not_provisioned
|
||||
: RecoveryError::store,
|
||||
f}};
|
||||
};
|
||||
auto committed = load_region_checkpoint(db, key, timeout);
|
||||
if (!committed)
|
||||
return store_failure(committed.error());
|
||||
const auto fail_with = [](RecoveryError kind) {
|
||||
return std::unexpected{RecoveryFailure{kind, {}}};
|
||||
};
|
||||
const auto digest = sha256(committed->manifest);
|
||||
auto manifest = decode_manifest(committed->manifest);
|
||||
const auto& head = committed->head;
|
||||
if (!digest || *digest != head.digest)
|
||||
return fail_with(RecoveryError::manifest);
|
||||
if (!manifest)
|
||||
return fail_with(manifest.error() == CheckpointError::unsupported
|
||||
? RecoveryError::unsupported
|
||||
: RecoveryError::manifest);
|
||||
if (manifest->key != key || manifest->frame != head.frame ||
|
||||
manifest->sequence != head.sequence || manifest->commit != head.commit ||
|
||||
manifest->end_tick != head.end_tick)
|
||||
return fail_with(RecoveryError::manifest);
|
||||
if (!supported(*manifest) || manifest->profile != Codec::profile ||
|
||||
std::ranges::find(retained_builds, manifest->build) == retained_builds.end())
|
||||
return fail_with(RecoveryError::unsupported);
|
||||
auto payload = objects.get({manifest->payload_sha256, manifest->payload_bytes});
|
||||
if (!payload)
|
||||
return fail_with(payload.error() == ObjectError::missing ? RecoveryError::missing_payload
|
||||
: payload.error() == ObjectError::corrupt ? RecoveryError::corrupt_payload
|
||||
: RecoveryError::store);
|
||||
auto state = Codec::restore(*payload, *manifest, budget);
|
||||
if (!state)
|
||||
return fail_with(RecoveryError::payload);
|
||||
// Durable permanent outcomes must agree with the restored state they finalized.
|
||||
auto decisions = load_region_decisions(db, key, timeout);
|
||||
if (!decisions)
|
||||
return store_failure(decisions.error());
|
||||
// Outcomes of a link committed after the head was read (possible only when the
|
||||
// reader holds no lease, e.g. a clone) belong to that newer, separately verified
|
||||
// link. Every other durable outcome must agree with this head.
|
||||
std::erase_if(*decisions,
|
||||
[&](const CommittedDecision& d) { return d.commit_sequence > head.sequence; });
|
||||
if (!Codec::reconcile(**state, *decisions, head))
|
||||
return fail_with(RecoveryError::history);
|
||||
if (auto tail = verify_tail<Codec>(db, objects, key, *manifest, *payload, **state, *decisions,
|
||||
budget, timeout);
|
||||
!tail)
|
||||
return std::unexpected{tail.error()};
|
||||
return typename Codec::Recovered{std::move(*committed), *manifest, std::move(*state),
|
||||
std::move(*decisions)};
|
||||
}
|
||||
} // namespace evo::persistence::detail
|
||||
@@ -1,138 +1,11 @@
|
||||
#include "evo/persistence/checkpoint_store.hpp"
|
||||
#include "evo/persistence/sha256.hpp"
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <string>
|
||||
#include "checkpoint_detail.hpp"
|
||||
|
||||
namespace evo::persistence {
|
||||
namespace {
|
||||
using namespace detail;
|
||||
using std::chrono::milliseconds;
|
||||
|
||||
struct Organisms {
|
||||
std::uint64_t producer{}, consumer{};
|
||||
};
|
||||
// Lifecycle agreement shared by both profiles: exactly one durable death per
|
||||
// terminal actor, none for an active one, never after the committed tick.
|
||||
bool lifecycle_agrees(const processes::EnvironmentalProgress& progress, Organisms organisms,
|
||||
std::span<const CommittedDecision> decisions, const RegionHead& head) {
|
||||
for (const auto actor : {Actor::producer, Actor::consumer}) {
|
||||
const auto terminal =
|
||||
actor == Actor::producer ? progress.producer_terminal : progress.consumer_terminal;
|
||||
const auto count = std::ranges::count_if(
|
||||
decisions, [&](const CommittedDecision& d) { return d.decision.actor == actor; });
|
||||
if (count > 1 || (terminal == 1) != (count == 1))
|
||||
return false;
|
||||
}
|
||||
return std::ranges::all_of(decisions, [&](const CommittedDecision& d) {
|
||||
const auto organism =
|
||||
d.decision.actor == Actor::producer ? organisms.producer : organisms.consumer;
|
||||
return d.decision.tick >= 1 && d.decision.tick <= head.end_tick &&
|
||||
d.commit_sequence <= head.sequence && d.decision.organism == organism;
|
||||
});
|
||||
}
|
||||
|
||||
// Per-profile payload operations; publication and recovery paths are shared.
|
||||
struct EnvironmentalCodec {
|
||||
using State = processes::EnvironmentalCommunity;
|
||||
using Recovered = RecoveredCheckpoint;
|
||||
static constexpr auto profile = CheckpointProfile::environmental_v1;
|
||||
static auto snapshot(const State& s) {
|
||||
return s.snapshot();
|
||||
}
|
||||
static std::uint64_t tick(const State& s) {
|
||||
return s.progress().tick;
|
||||
}
|
||||
static std::uint64_t quantum(const State& s) {
|
||||
return s.tick_microseconds();
|
||||
}
|
||||
static Organisms organisms(const State&) {
|
||||
return {};
|
||||
}
|
||||
static auto restore(std::span<const std::uint8_t> payload, const CheckpointManifest& m,
|
||||
std::size_t budget) {
|
||||
return restore_environmental_payload(payload, m, budget);
|
||||
}
|
||||
static bool reconcile(const State& s, std::span<const CommittedDecision> d,
|
||||
const RegionHead& head) {
|
||||
return lifecycle_agrees(s.progress(), organisms(s), d, head);
|
||||
}
|
||||
};
|
||||
struct LineageCodec {
|
||||
using State = processes::LineageRegion;
|
||||
using Recovered = RecoveredLineageCheckpoint;
|
||||
static constexpr auto profile = CheckpointProfile::environmental_lineage_v1;
|
||||
static auto snapshot(const State& s) {
|
||||
return s.snapshot();
|
||||
}
|
||||
static std::uint64_t tick(const State& s) {
|
||||
return s.community().progress().tick;
|
||||
}
|
||||
static std::uint64_t quantum(const State& s) {
|
||||
return s.community().tick_microseconds();
|
||||
}
|
||||
static Organisms organisms(const State& s) {
|
||||
return {s.binding().producer, s.binding().consumer};
|
||||
}
|
||||
static auto restore(std::span<const std::uint8_t> payload, const CheckpointManifest& m,
|
||||
std::size_t budget) {
|
||||
return restore_lineage_payload(payload, m, budget);
|
||||
}
|
||||
// Plus: each durable death is the archive's death, at the same tick.
|
||||
static bool reconcile(const State& s, std::span<const CommittedDecision> d,
|
||||
const RegionHead& head) {
|
||||
if (!lifecycle_agrees(s.community().progress(), organisms(s), d, head))
|
||||
return false;
|
||||
return std::ranges::all_of(d, [&](const CommittedDecision& x) {
|
||||
const auto record = s.archive().organism(x.decision.organism);
|
||||
return record && !record->alive && record->died.value == x.decision.tick;
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
PublicationFailure fail(PublicationError kind, ObjectError object = {}) {
|
||||
return {kind, {}, object, {}};
|
||||
}
|
||||
std::string hex_parameter(std::span<const std::uint8_t> bytes) {
|
||||
constexpr char digits[] = "0123456789abcdef";
|
||||
std::string out = "\\x";
|
||||
for (const auto b : bytes) {
|
||||
out.push_back(digits[b >> 4U]);
|
||||
out.push_back(digits[b & 0x0FU]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
std::expected<void, PublicationFailure> put_verified(const FilesystemObjectStore& objects,
|
||||
const std::vector<std::uint8_t>& bytes,
|
||||
const VerifiedObject& expected) {
|
||||
auto stored = objects.put(bytes);
|
||||
if (!stored)
|
||||
return std::unexpected{fail(PublicationError::upload, stored.error())};
|
||||
auto back = objects.get(*stored);
|
||||
if (!back || *stored != expected || !std::ranges::equal(*back, bytes))
|
||||
return std::unexpected{
|
||||
fail(PublicationError::verify, back ? ObjectError::corrupt : back.error())};
|
||||
return {};
|
||||
}
|
||||
// The receipt must agree with the local manifest the database digested.
|
||||
std::expected<CommittedCheckpoint, PublicationFailure>
|
||||
confirm(const std::expected<RegionHead, LeaseStoreFailure>& published,
|
||||
std::vector<std::uint8_t> manifest_bytes, const CheckpointManifest& manifest,
|
||||
PreparedCommit prepared) {
|
||||
if (!published) {
|
||||
const auto kind = published.error().kind == LeaseStoreError::unknown
|
||||
? PublicationError::unknown
|
||||
: PublicationError::rejected;
|
||||
return std::unexpected{
|
||||
PublicationFailure{kind, published.error(), {}, std::move(prepared)}};
|
||||
}
|
||||
const auto digest = sha256(manifest_bytes);
|
||||
if (!digest || published->digest != *digest || published->sequence != manifest.sequence ||
|
||||
published->commit != manifest.commit || published->end_tick != manifest.end_tick ||
|
||||
published->frame != manifest.frame)
|
||||
return std::unexpected{fail(PublicationError::receipt)};
|
||||
return CommittedCheckpoint{*published, std::move(manifest_bytes)};
|
||||
}
|
||||
|
||||
template <typename Codec>
|
||||
std::expected<CommittedCheckpoint, PublicationFailure>
|
||||
provision(Postgres& owner, const FilesystemObjectStore& objects, RegionKey key, std::uint64_t frame,
|
||||
@@ -241,125 +114,6 @@ publish_batch(Postgres& db, const FilesystemObjectStore& objects, const LeaseGra
|
||||
return PublishedBatch{std::move(*committed), decisions};
|
||||
}
|
||||
|
||||
// Replays the head's recorded batch from its committed predecessor (ADR-0065).
|
||||
template <typename Codec>
|
||||
std::expected<void, RecoveryFailure>
|
||||
verify_tail(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
const CheckpointManifest& m, std::span<const std::uint8_t> head_payload,
|
||||
const typename Codec::State& head_state, std::span<const CommittedDecision> durable,
|
||||
std::size_t budget, milliseconds timeout) {
|
||||
const auto fail_with = [](RecoveryError kind) {
|
||||
return std::unexpected{RecoveryFailure{kind, {}}};
|
||||
};
|
||||
if (m.sequence == 0)
|
||||
return {}; // Genesis has no batch.
|
||||
if (m.manifest_version != 2)
|
||||
return fail_with(RecoveryError::unsupported);
|
||||
auto trace_bytes = objects.get({m.batch.trace_sha256, m.batch.trace_bytes});
|
||||
if (!trace_bytes)
|
||||
return fail_with(trace_bytes.error() == ObjectError::io ? RecoveryError::store
|
||||
: RecoveryError::tail);
|
||||
auto trace = decode_trace(*trace_bytes);
|
||||
const TraceHeader expected{static_cast<std::uint32_t>(Codec::profile),
|
||||
key,
|
||||
m.predecessor_commit,
|
||||
m.predecessor_digest,
|
||||
m.begin_tick,
|
||||
m.end_tick};
|
||||
if (!trace || trace->header != expected)
|
||||
return fail_with(RecoveryError::tail);
|
||||
auto predecessor = load_region_commit(db, key, m.sequence - 1, timeout);
|
||||
if (!predecessor)
|
||||
return predecessor.error().kind == LeaseStoreError::not_provisioned
|
||||
? fail_with(RecoveryError::tail)
|
||||
: std::unexpected{RecoveryFailure{RecoveryError::store, predecessor.error()}};
|
||||
const auto digest = sha256(predecessor->manifest);
|
||||
auto previous = decode_manifest(predecessor->manifest);
|
||||
if (!digest || *digest != predecessor->head.digest ||
|
||||
predecessor->head.digest != m.predecessor_digest ||
|
||||
predecessor->head.commit != m.predecessor_commit || !previous ||
|
||||
previous->end_tick != m.begin_tick)
|
||||
return fail_with(RecoveryError::tail);
|
||||
auto base = objects.get({previous->payload_sha256, previous->payload_bytes});
|
||||
if (!base)
|
||||
return fail_with(base.error() == ObjectError::io ? RecoveryError::store
|
||||
: RecoveryError::tail);
|
||||
auto replay = Codec::restore(*base, *previous, budget);
|
||||
if (!replay)
|
||||
return fail_with(RecoveryError::tail);
|
||||
for (const auto& step : trace->steps) {
|
||||
auto report = (*replay)->advance(step.report.tick, step.input);
|
||||
if (!report || *report != step.report)
|
||||
return fail_with(RecoveryError::tail);
|
||||
}
|
||||
const auto replayed = Codec::snapshot(**replay);
|
||||
if (!replayed || !std::ranges::equal(*replayed, head_payload))
|
||||
return fail_with(RecoveryError::tail);
|
||||
// The head's durable outcomes are exactly those its trace derives.
|
||||
const auto organisms = Codec::organisms(head_state);
|
||||
const auto derived = derive_decisions(trace->steps, organisms.producer, organisms.consumer);
|
||||
std::vector<BatchDecision> recorded;
|
||||
for (const auto& d : durable)
|
||||
if (d.commit_sequence == m.sequence)
|
||||
recorded.push_back(d.decision);
|
||||
if (derived != recorded || derived.size() != m.batch.decision_count)
|
||||
return fail_with(RecoveryError::history);
|
||||
return {};
|
||||
}
|
||||
|
||||
template <typename Codec>
|
||||
std::expected<typename Codec::Recovered, RecoveryFailure>
|
||||
recover(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
std::span<const Sha256Digest> retained_builds, std::size_t budget, milliseconds timeout) {
|
||||
const auto store_failure = [](const LeaseStoreFailure& f) {
|
||||
return std::unexpected{RecoveryFailure{f.kind == LeaseStoreError::not_provisioned
|
||||
? RecoveryError::not_provisioned
|
||||
: RecoveryError::store,
|
||||
f}};
|
||||
};
|
||||
auto committed = load_region_checkpoint(db, key, timeout);
|
||||
if (!committed)
|
||||
return store_failure(committed.error());
|
||||
const auto fail_with = [](RecoveryError kind) {
|
||||
return std::unexpected{RecoveryFailure{kind, {}}};
|
||||
};
|
||||
const auto digest = sha256(committed->manifest);
|
||||
auto manifest = decode_manifest(committed->manifest);
|
||||
const auto& head = committed->head;
|
||||
if (!digest || *digest != head.digest)
|
||||
return fail_with(RecoveryError::manifest);
|
||||
if (!manifest)
|
||||
return fail_with(manifest.error() == CheckpointError::unsupported
|
||||
? RecoveryError::unsupported
|
||||
: RecoveryError::manifest);
|
||||
if (manifest->key != key || manifest->frame != head.frame ||
|
||||
manifest->sequence != head.sequence || manifest->commit != head.commit ||
|
||||
manifest->end_tick != head.end_tick)
|
||||
return fail_with(RecoveryError::manifest);
|
||||
if (!supported(*manifest) || manifest->profile != Codec::profile ||
|
||||
std::ranges::find(retained_builds, manifest->build) == retained_builds.end())
|
||||
return fail_with(RecoveryError::unsupported);
|
||||
auto payload = objects.get({manifest->payload_sha256, manifest->payload_bytes});
|
||||
if (!payload)
|
||||
return fail_with(payload.error() == ObjectError::missing ? RecoveryError::missing_payload
|
||||
: payload.error() == ObjectError::corrupt ? RecoveryError::corrupt_payload
|
||||
: RecoveryError::store);
|
||||
auto state = Codec::restore(*payload, *manifest, budget);
|
||||
if (!state)
|
||||
return fail_with(RecoveryError::payload);
|
||||
// Durable permanent outcomes must agree with the restored state they finalized.
|
||||
auto decisions = load_region_decisions(db, key, timeout);
|
||||
if (!decisions)
|
||||
return store_failure(decisions.error());
|
||||
if (!Codec::reconcile(**state, *decisions, head))
|
||||
return fail_with(RecoveryError::history);
|
||||
if (auto tail = verify_tail<Codec>(db, objects, key, *manifest, *payload, **state, *decisions,
|
||||
budget, timeout);
|
||||
!tail)
|
||||
return std::unexpected{tail.error()};
|
||||
return typename Codec::Recovered{std::move(*committed), *manifest, std::move(*state),
|
||||
std::move(*decisions)};
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::expected<CommittedCheckpoint, PublicationFailure> provision_environmental_region(
|
||||
|
||||
@@ -61,9 +61,18 @@ bool structural(const CheckpointManifest& m) noexcept {
|
||||
if (m.manifest_version != 1 && m.manifest_version != 2)
|
||||
return false;
|
||||
const bool no_batch = b == BatchDigests{};
|
||||
if (m.sequence == 0)
|
||||
return zero(m.predecessor_commit) && zero(m.predecessor_digest) && m.begin_tick == 0 &&
|
||||
m.end_tick == 0 && zero(m.holder) && m.epoch == 0 && no_batch;
|
||||
if (m.sequence == 0) {
|
||||
const bool genesis = zero(m.holder) && m.epoch == 0 && m.begin_tick == m.end_tick &&
|
||||
b.trace_bytes == 0 && zero(b.trace_sha256);
|
||||
if (zero(m.predecessor_commit) && zero(m.predecessor_digest))
|
||||
return genesis && m.end_tick == 0 && no_batch;
|
||||
// Clone genesis (ADR-0067, v2 only): the origin is the source commit and
|
||||
// manifest digest; the block digests inherited permanent outcomes.
|
||||
return genesis && m.manifest_version == 2 && !zero(m.predecessor_commit) &&
|
||||
!zero(m.predecessor_digest) && m.predecessor_commit != m.commit &&
|
||||
b.decision_count <= max_batch_decisions &&
|
||||
(b.decision_count == 0) == zero(b.decisions_sha256);
|
||||
}
|
||||
const bool link = !zero(m.predecessor_commit) && m.predecessor_commit != m.commit &&
|
||||
!zero(m.predecessor_digest) && !zero(m.holder) && m.epoch != 0 &&
|
||||
m.end_tick > m.begin_tick &&
|
||||
@@ -200,6 +209,33 @@ genesis_manifest(RegionKey key, std::uint64_t frame, Uuid commit, std::uint64_t
|
||||
return m;
|
||||
}
|
||||
|
||||
std::expected<CheckpointManifest, CheckpointError>
|
||||
clone_manifest(const CheckpointManifest& source, const Sha256Digest& source_digest,
|
||||
RegionKey target, Uuid commit, std::uint64_t payload_bytes,
|
||||
const Sha256Digest& payload_sha256, const BatchDigests& inherited) {
|
||||
if (!supported(source))
|
||||
return std::unexpected{CheckpointError::unsupported};
|
||||
if (!structural(source) || target.world == source.key.world || commit == source.commit ||
|
||||
inherited.trace_bytes != 0 || !zero(inherited.trace_sha256))
|
||||
return std::unexpected{CheckpointError::chain};
|
||||
CheckpointManifest m = source;
|
||||
m.manifest_version = 2;
|
||||
m.key = target;
|
||||
m.commit = commit;
|
||||
m.sequence = 0;
|
||||
m.predecessor_commit = source.commit;
|
||||
m.predecessor_digest = source_digest;
|
||||
m.begin_tick = source.end_tick;
|
||||
m.holder = {};
|
||||
m.epoch = 0;
|
||||
m.payload_bytes = payload_bytes;
|
||||
m.payload_sha256 = payload_sha256;
|
||||
m.batch = inherited;
|
||||
if (!structural(m))
|
||||
return std::unexpected{CheckpointError::format};
|
||||
return m;
|
||||
}
|
||||
|
||||
std::expected<CheckpointManifest, CheckpointError>
|
||||
successor_manifest(const CheckpointManifest& p, const Sha256Digest& predecessor_digest, Uuid commit,
|
||||
const LeaseGrant& lease, std::uint64_t end_tick, const Sha256Digest& build,
|
||||
|
||||
@@ -0,0 +1,269 @@
|
||||
#include "evo/persistence/region_clone.hpp"
|
||||
#include "checkpoint_detail.hpp"
|
||||
#include <iterator>
|
||||
|
||||
namespace evo::persistence {
|
||||
namespace {
|
||||
using namespace detail;
|
||||
using std::chrono::milliseconds;
|
||||
|
||||
std::unexpected<CloneFailure> clone_fail(CloneError kind, LeaseStoreFailure store = {}) {
|
||||
return std::unexpected{CloneFailure{kind, {}, store, {}}};
|
||||
}
|
||||
std::unexpected<CloneFailure> source_fail(RecoveryFailure recovery) {
|
||||
return std::unexpected{CloneFailure{CloneError::source, recovery, {}, {}}};
|
||||
}
|
||||
LeaseStoreFailure store_failure(const PgFailure& cause) {
|
||||
return {cause.kind == PgError::unknown ? LeaseStoreError::unknown : LeaseStoreError::store,
|
||||
cause};
|
||||
}
|
||||
|
||||
template <typename Codec>
|
||||
std::expected<ClonedRegion, CloneFailure>
|
||||
clone(Postgres& source, const FilesystemObjectStore& source_objects, RegionKey source_key,
|
||||
Postgres& target, const FilesystemObjectStore& target_objects, RegionKey target_key,
|
||||
Uuid commit, std::span<const Sha256Digest> builds, std::size_t budget, milliseconds timeout,
|
||||
const CloneHooks& hooks) {
|
||||
// The cut is the committed head, verified exactly as recovery verifies it.
|
||||
auto recovered = recover<Codec>(source, source_objects, source_key, builds, budget, timeout);
|
||||
if (!recovered)
|
||||
return source_fail(recovered.error());
|
||||
const auto& origin = recovered->manifest;
|
||||
const auto origin_digest = recovered->committed.head.digest;
|
||||
auto& state = *recovered->state;
|
||||
const auto original = Codec::snapshot(state);
|
||||
if (!original || !state.rebind_world(target_key.world))
|
||||
return clone_fail(CloneError::candidate);
|
||||
const auto payload = Codec::snapshot(state);
|
||||
std::vector<BatchDecision> inherited;
|
||||
inherited.reserve(recovered->decisions.size());
|
||||
for (const auto& d : recovered->decisions)
|
||||
inherited.push_back(d.decision);
|
||||
const auto block = encode_decisions(inherited, 0, origin.end_tick);
|
||||
if (!payload || !block)
|
||||
return clone_fail(CloneError::candidate);
|
||||
const auto payload_digest = sha256(*payload);
|
||||
const auto block_digest = sha256(*block);
|
||||
if (!payload_digest || !block_digest)
|
||||
return clone_fail(CloneError::candidate);
|
||||
const BatchDigests digests{0,
|
||||
{},
|
||||
static_cast<std::uint32_t>(inherited.size()),
|
||||
inherited.empty() ? Sha256Digest{} : *block_digest};
|
||||
const auto manifest = clone_manifest(origin, origin_digest, target_key, commit, payload->size(),
|
||||
*payload_digest, digests);
|
||||
auto bytes = manifest ? encode_manifest(*manifest)
|
||||
: std::expected<std::vector<std::uint8_t>, CheckpointError>{
|
||||
std::unexpected{CheckpointError::format}};
|
||||
if (!bytes)
|
||||
return clone_fail(CloneError::candidate);
|
||||
// The target bytes are a complete region of the target world, and rebinding them
|
||||
// back reproduces the verified source bytes exactly: only the identity moved.
|
||||
auto check = Codec::restore(*payload, *manifest, budget);
|
||||
if (!check || !(*check)->rebind_world(source_key.world))
|
||||
return clone_fail(CloneError::candidate);
|
||||
const auto back = Codec::snapshot(**check);
|
||||
if (!back || *back != *original)
|
||||
return clone_fail(CloneError::candidate);
|
||||
|
||||
if (auto stored = put_verified(target_objects, *payload, {*payload_digest, payload->size()});
|
||||
!stored)
|
||||
return std::unexpected{CloneFailure{stored.error().kind == PublicationError::upload
|
||||
? CloneError::upload
|
||||
: CloneError::verify,
|
||||
{},
|
||||
{},
|
||||
stored.error().object}};
|
||||
if (hooks.before_transaction)
|
||||
hooks.before_transaction();
|
||||
const std::array<std::string, 5> p{hex_parameter(*bytes), std::to_string(source_key.world),
|
||||
std::to_string(source_key.region),
|
||||
std::to_string(origin.sequence), hex_parameter(*block)};
|
||||
auto done = target.query("SELECT evolution.provision_region_clone($1::bytea, "
|
||||
"$2::evolution.u64, $3::evolution.u64, $4::evolution.u64, $5::bytea)",
|
||||
p, timeout);
|
||||
if (!done) {
|
||||
const auto failure = store_failure(done.error());
|
||||
return clone_fail(failure.kind == LeaseStoreError::unknown ? CloneError::unknown
|
||||
: CloneError::rejected,
|
||||
failure);
|
||||
}
|
||||
auto committed = load_region_checkpoint(target, target_key, timeout);
|
||||
if (!committed)
|
||||
return clone_fail(CloneError::unknown, committed.error());
|
||||
auto confirmed = confirm(committed->head, std::move(*bytes), *manifest, {});
|
||||
if (!confirmed || committed->manifest != confirmed->manifest)
|
||||
return clone_fail(CloneError::receipt);
|
||||
return ClonedRegion{
|
||||
std::move(*confirmed),
|
||||
CloneOrigin{source_key, origin.sequence, origin.end_tick, origin.commit, origin_digest},
|
||||
inherited.size()};
|
||||
}
|
||||
|
||||
std::unexpected<AuditFailure> audit_fail(AuditError kind, std::uint64_t sequence,
|
||||
LeaseStoreFailure store = {}) {
|
||||
return std::unexpected{AuditFailure{kind, sequence, store}};
|
||||
}
|
||||
AuditError from_recovery(RecoveryError kind) {
|
||||
switch (kind) {
|
||||
case RecoveryError::store:
|
||||
return AuditError::store;
|
||||
case RecoveryError::unsupported:
|
||||
return AuditError::unsupported;
|
||||
case RecoveryError::history:
|
||||
return AuditError::history;
|
||||
default:
|
||||
return AuditError::tail;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Codec>
|
||||
std::expected<HistoryAudit, AuditFailure>
|
||||
audit(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
std::span<const Sha256Digest> builds, std::size_t budget, std::uint64_t max_commits,
|
||||
milliseconds timeout, const CommittedCheckpoint& head) {
|
||||
const auto last = head.head.sequence;
|
||||
if (last >= max_commits)
|
||||
return audit_fail(AuditError::bounds, 0);
|
||||
auto loaded = load_region_decisions(db, key, timeout);
|
||||
if (!loaded)
|
||||
return audit_fail(AuditError::store, 0, loaded.error());
|
||||
// A live region may commit after `head` was read; judge only what it covers.
|
||||
std::vector<CommittedDecision> durable;
|
||||
std::ranges::copy_if(*loaded, std::back_inserter(durable),
|
||||
[&](const CommittedDecision& d) { return d.commit_sequence <= last; });
|
||||
|
||||
HistoryAudit out;
|
||||
out.profile = Codec::profile;
|
||||
CheckpointManifest previous;
|
||||
Sha256Digest previous_digest{};
|
||||
std::unique_ptr<typename Codec::State> state;
|
||||
for (std::uint64_t sequence = 0; sequence <= last; ++sequence) {
|
||||
auto link = sequence == last ? std::expected<CommittedCheckpoint, LeaseStoreFailure>{head}
|
||||
: load_region_commit(db, key, sequence, timeout);
|
||||
if (!link)
|
||||
return link.error().kind == LeaseStoreError::not_provisioned
|
||||
? audit_fail(AuditError::chain, sequence)
|
||||
: audit_fail(AuditError::store, sequence, link.error());
|
||||
const auto digest = sha256(link->manifest);
|
||||
auto m = decode_manifest(link->manifest);
|
||||
if (!digest || *digest != link->head.digest)
|
||||
return audit_fail(AuditError::manifest, sequence);
|
||||
if (!m)
|
||||
return audit_fail(m.error() == CheckpointError::unsupported ? AuditError::unsupported
|
||||
: AuditError::manifest,
|
||||
sequence);
|
||||
if (m->key != key || m->sequence != sequence || m->commit != link->head.commit ||
|
||||
m->end_tick != link->head.end_tick || m->frame != link->head.frame)
|
||||
return audit_fail(AuditError::manifest, sequence);
|
||||
if (!supported(*m) || m->profile != Codec::profile ||
|
||||
std::ranges::find(builds, m->build) == builds.end())
|
||||
return audit_fail(AuditError::unsupported, sequence);
|
||||
if (sequence == 0) {
|
||||
out.clone = m->predecessor_commit != Uuid{};
|
||||
out.origin_commit = m->predecessor_commit;
|
||||
out.origin_digest = m->predecessor_digest;
|
||||
} else if (m->predecessor_commit != previous.commit ||
|
||||
m->predecessor_digest != previous_digest || m->begin_tick != previous.end_tick) {
|
||||
return audit_fail(AuditError::chain, sequence);
|
||||
}
|
||||
auto payload = objects.get({m->payload_sha256, m->payload_bytes});
|
||||
if (!payload)
|
||||
return audit_fail(payload.error() == ObjectError::missing ? AuditError::missing_payload
|
||||
: payload.error() == ObjectError::corrupt
|
||||
? AuditError::corrupt_payload
|
||||
: AuditError::store,
|
||||
sequence);
|
||||
auto restored = Codec::restore(*payload, *m, budget);
|
||||
if (!restored)
|
||||
return audit_fail(AuditError::payload, sequence);
|
||||
if (auto tail = verify_tail<Codec>(db, objects, key, *m, *payload, **restored, durable,
|
||||
budget, timeout);
|
||||
!tail)
|
||||
return audit_fail(from_recovery(tail.error().kind), sequence, tail.error().store);
|
||||
previous = *m;
|
||||
previous_digest = *digest;
|
||||
state = std::move(*restored);
|
||||
}
|
||||
if (!state || !Codec::reconcile(*state, durable, head.head))
|
||||
return audit_fail(AuditError::history, last);
|
||||
out.commits = last + 1;
|
||||
out.end_tick = head.head.end_tick;
|
||||
out.outcomes = durable.size();
|
||||
return out;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::expected<ClonedRegion, CloneFailure>
|
||||
clone_region(Postgres& source, const FilesystemObjectStore& source_objects, RegionKey source_key,
|
||||
Postgres& target_owner, const FilesystemObjectStore& target_objects,
|
||||
RegionKey target_key, Uuid commit, std::span<const Sha256Digest> retained_builds,
|
||||
std::size_t restore_budget, milliseconds timeout, const CloneHooks& hooks) {
|
||||
if (source_key.world == 0 || source_key.region == 0 || target_key.world == 0 ||
|
||||
target_key.region == 0 || target_key.world == source_key.world)
|
||||
return clone_fail(CloneError::identity);
|
||||
// Every later transaction on the source session is read-only, including the
|
||||
// SECURITY DEFINER functions it calls: a clone cannot write to its source.
|
||||
if (auto set =
|
||||
source.query("SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY", {}, timeout);
|
||||
!set)
|
||||
return clone_fail(CloneError::session, store_failure(set.error()));
|
||||
auto read_only = source.query(
|
||||
"SELECT pg_catalog.current_setting('default_transaction_read_only')", {}, timeout);
|
||||
if (!read_only)
|
||||
return clone_fail(CloneError::session, store_failure(read_only.error()));
|
||||
if (read_only->size() != 1 || (*read_only)[0].size() != 1 || (*read_only)[0][0] != "on")
|
||||
return clone_fail(CloneError::session);
|
||||
auto head = load_region_checkpoint(source, source_key, timeout);
|
||||
if (!head)
|
||||
return source_fail({head.error().kind == LeaseStoreError::not_provisioned
|
||||
? RecoveryError::not_provisioned
|
||||
: RecoveryError::store,
|
||||
head.error()});
|
||||
const auto manifest = decode_manifest(head->manifest);
|
||||
if (!manifest)
|
||||
return source_fail({manifest.error() == CheckpointError::unsupported
|
||||
? RecoveryError::unsupported
|
||||
: RecoveryError::manifest,
|
||||
{}});
|
||||
switch (manifest->profile) {
|
||||
case CheckpointProfile::environmental_v1:
|
||||
return clone<EnvironmentalCodec>(source, source_objects, source_key, target_owner,
|
||||
target_objects, target_key, commit, retained_builds,
|
||||
restore_budget, timeout, hooks);
|
||||
case CheckpointProfile::environmental_lineage_v1:
|
||||
return clone<LineageCodec>(source, source_objects, source_key, target_owner, target_objects,
|
||||
target_key, commit, retained_builds, restore_budget, timeout,
|
||||
hooks);
|
||||
default:
|
||||
return source_fail({RecoveryError::unsupported, {}});
|
||||
}
|
||||
}
|
||||
|
||||
std::expected<HistoryAudit, AuditFailure>
|
||||
audit_region_history(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
std::span<const Sha256Digest> retained_builds, std::size_t restore_budget,
|
||||
std::uint64_t max_commits, milliseconds timeout) {
|
||||
auto head = load_region_checkpoint(db, key, timeout);
|
||||
if (!head)
|
||||
return audit_fail(head.error().kind == LeaseStoreError::not_provisioned
|
||||
? AuditError::not_provisioned
|
||||
: AuditError::store,
|
||||
0, head.error());
|
||||
const auto manifest = decode_manifest(head->manifest);
|
||||
if (!manifest)
|
||||
return audit_fail(manifest.error() == CheckpointError::unsupported ? AuditError::unsupported
|
||||
: AuditError::manifest,
|
||||
head->head.sequence);
|
||||
switch (manifest->profile) {
|
||||
case CheckpointProfile::environmental_v1:
|
||||
return audit<EnvironmentalCodec>(db, objects, key, retained_builds, restore_budget,
|
||||
max_commits, timeout, *head);
|
||||
case CheckpointProfile::environmental_lineage_v1:
|
||||
return audit<LineageCodec>(db, objects, key, retained_builds, restore_budget, max_commits,
|
||||
timeout, *head);
|
||||
default:
|
||||
return audit_fail(AuditError::unsupported, head->head.sequence);
|
||||
}
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
@@ -13,6 +13,13 @@ std::uint64_t portion(std::uint64_t x, std::uint64_t a) {
|
||||
return (x / 1000) * a + (x % 1000) * a / 1000;
|
||||
}
|
||||
} // namespace
|
||||
bool EnvironmentalCommunity::rebind_world(std::uint64_t world) noexcept {
|
||||
if (world == 0)
|
||||
return false;
|
||||
config_.world = world;
|
||||
resources_->rebind({world, config_.frame});
|
||||
return true;
|
||||
}
|
||||
bool EnvironmentalCommunity::configuration(const EnvironmentalConfig& c,
|
||||
const world::EnvironmentalTransfers& r) noexcept {
|
||||
if ((c.rules != 1 && c.rules != 2) ||
|
||||
|
||||
Reference in New Issue
Block a user