Drill-only --synthetic-until/--accelerate and evo-world-clone compare (compare_region_histories) prove a candidate build runs the same world on clones of one cut before any barrier. upgrade-check.sh rehearses a failed migration under load and the version barrier with real hosts; upgrade-barrier.sh retires an old build in dev/stage only once the new digest is Ready. Nodes: aevolution.io/stores pins every objects-volume pod, aevolution.io/compute is preference-only (Alertmanager first), checked by policy and scheduling-check.sh. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
359 lines
18 KiB
C++
359 lines
18 KiB
C++
#include "evo/persistence/region_clone.hpp"
|
|
#include "checkpoint_detail.hpp"
|
|
#include <algorithm>
|
|
#include <iterator>
|
|
#include <tuple>
|
|
|
|
namespace evo::persistence {
|
|
namespace {
|
|
using namespace detail;
|
|
using std::chrono::milliseconds;
|
|
|
|
std::unexpected<CloneFailure> clone_fail(CloneError kind, LeaseStoreFailure store = {}) {
|
|
return std::unexpected{CloneFailure{kind, {}, store, {}}};
|
|
}
|
|
std::unexpected<CloneFailure> source_fail(RecoveryFailure recovery) {
|
|
return std::unexpected{CloneFailure{CloneError::source, recovery, {}, {}}};
|
|
}
|
|
LeaseStoreFailure store_failure(const PgFailure& cause) {
|
|
return {cause.kind == PgError::unknown ? LeaseStoreError::unknown : LeaseStoreError::store,
|
|
cause};
|
|
}
|
|
|
|
template <typename Codec>
|
|
std::expected<ClonedRegion, CloneFailure>
|
|
clone(Postgres& source, const FilesystemObjectStore& source_objects, RegionKey source_key,
|
|
Postgres& target, const FilesystemObjectStore& target_objects, RegionKey target_key,
|
|
Uuid commit, std::span<const Sha256Digest> builds, std::size_t budget, milliseconds timeout,
|
|
const CloneHooks& hooks) {
|
|
// The cut is the committed head, verified exactly as recovery verifies it.
|
|
auto recovered = recover<Codec>(source, source_objects, source_key, builds, budget, timeout);
|
|
if (!recovered)
|
|
return source_fail(recovered.error());
|
|
const auto& origin = recovered->manifest;
|
|
const auto origin_digest = recovered->committed.head.digest;
|
|
auto& state = *recovered->state;
|
|
const auto original = Codec::snapshot(state);
|
|
if (!original || !state.rebind_world(target_key.world))
|
|
return clone_fail(CloneError::candidate);
|
|
const auto payload = Codec::snapshot(state);
|
|
std::vector<BatchDecision> inherited;
|
|
inherited.reserve(recovered->decisions.size());
|
|
for (const auto& d : recovered->decisions)
|
|
inherited.push_back(d.decision);
|
|
const auto block = encode_decisions(inherited, 0, origin.end_tick);
|
|
if (!payload || !block)
|
|
return clone_fail(CloneError::candidate);
|
|
const auto payload_digest = sha256(*payload);
|
|
const auto block_digest = sha256(*block);
|
|
if (!payload_digest || !block_digest)
|
|
return clone_fail(CloneError::candidate);
|
|
const BatchDigests digests{0,
|
|
{},
|
|
static_cast<std::uint32_t>(inherited.size()),
|
|
inherited.empty() ? Sha256Digest{} : *block_digest};
|
|
const auto manifest = clone_manifest(origin, origin_digest, target_key, commit, payload->size(),
|
|
*payload_digest, digests);
|
|
auto bytes = manifest ? encode_manifest(*manifest)
|
|
: std::expected<std::vector<std::uint8_t>, CheckpointError>{
|
|
std::unexpected{CheckpointError::format}};
|
|
if (!bytes)
|
|
return clone_fail(CloneError::candidate);
|
|
// The target bytes are a complete region of the target world, and rebinding them
|
|
// back reproduces the verified source bytes exactly: only the identity moved.
|
|
auto check = Codec::restore(*payload, *manifest, budget);
|
|
if (!check || !(*check)->rebind_world(source_key.world))
|
|
return clone_fail(CloneError::candidate);
|
|
const auto back = Codec::snapshot(**check);
|
|
if (!back || *back != *original)
|
|
return clone_fail(CloneError::candidate);
|
|
|
|
if (auto stored = put_verified(target_objects, *payload, {*payload_digest, payload->size()});
|
|
!stored)
|
|
return std::unexpected{CloneFailure{stored.error().kind == PublicationError::upload
|
|
? CloneError::upload
|
|
: CloneError::verify,
|
|
{},
|
|
{},
|
|
stored.error().object}};
|
|
if (hooks.before_transaction)
|
|
hooks.before_transaction();
|
|
const std::array<std::string, 5> p{hex_parameter(*bytes), std::to_string(source_key.world),
|
|
std::to_string(source_key.region),
|
|
std::to_string(origin.sequence), hex_parameter(*block)};
|
|
auto done = target.query("SELECT evolution.provision_region_clone($1::bytea, "
|
|
"$2::evolution.u64, $3::evolution.u64, $4::evolution.u64, $5::bytea)",
|
|
p, timeout);
|
|
if (!done) {
|
|
const auto failure = store_failure(done.error());
|
|
return clone_fail(failure.kind == LeaseStoreError::unknown ? CloneError::unknown
|
|
: CloneError::rejected,
|
|
failure);
|
|
}
|
|
auto committed = load_region_checkpoint(target, target_key, timeout);
|
|
if (!committed)
|
|
return clone_fail(CloneError::unknown, committed.error());
|
|
auto confirmed = confirm(committed->head, std::move(*bytes), *manifest, {});
|
|
if (!confirmed || committed->manifest != confirmed->manifest)
|
|
return clone_fail(CloneError::receipt);
|
|
return ClonedRegion{
|
|
std::move(*confirmed),
|
|
CloneOrigin{source_key, origin.sequence, origin.end_tick, origin.commit, origin_digest},
|
|
inherited.size()};
|
|
}
|
|
|
|
std::unexpected<AuditFailure> audit_fail(AuditError kind, std::uint64_t sequence,
|
|
LeaseStoreFailure store = {}) {
|
|
return std::unexpected{AuditFailure{kind, sequence, store}};
|
|
}
|
|
AuditError from_recovery(RecoveryError kind) {
|
|
switch (kind) {
|
|
case RecoveryError::store:
|
|
return AuditError::store;
|
|
case RecoveryError::unsupported:
|
|
return AuditError::unsupported;
|
|
case RecoveryError::history:
|
|
return AuditError::history;
|
|
default:
|
|
return AuditError::tail;
|
|
}
|
|
}
|
|
|
|
template <typename Codec>
|
|
std::expected<HistoryAudit, AuditFailure>
|
|
audit(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
|
std::span<const Sha256Digest> builds, std::size_t budget, std::uint64_t max_commits,
|
|
milliseconds timeout, const CommittedCheckpoint& head) {
|
|
const auto last = head.head.sequence;
|
|
if (last >= max_commits)
|
|
return audit_fail(AuditError::bounds, 0);
|
|
auto loaded = load_region_decisions(db, key, timeout);
|
|
if (!loaded)
|
|
return audit_fail(AuditError::store, 0, loaded.error());
|
|
// A live region may commit after `head` was read; judge only what it covers.
|
|
std::vector<CommittedDecision> durable;
|
|
std::ranges::copy_if(*loaded, std::back_inserter(durable),
|
|
[&](const CommittedDecision& d) { return d.commit_sequence <= last; });
|
|
|
|
HistoryAudit out;
|
|
out.profile = Codec::profile;
|
|
CheckpointManifest previous;
|
|
Sha256Digest previous_digest{};
|
|
std::unique_ptr<typename Codec::State> state;
|
|
for (std::uint64_t sequence = 0; sequence <= last; ++sequence) {
|
|
auto link = sequence == last ? std::expected<CommittedCheckpoint, LeaseStoreFailure>{head}
|
|
: load_region_commit(db, key, sequence, timeout);
|
|
if (!link)
|
|
return link.error().kind == LeaseStoreError::not_provisioned
|
|
? audit_fail(AuditError::chain, sequence)
|
|
: audit_fail(AuditError::store, sequence, link.error());
|
|
const auto digest = sha256(link->manifest);
|
|
auto m = decode_manifest(link->manifest);
|
|
if (!digest || *digest != link->head.digest)
|
|
return audit_fail(AuditError::manifest, sequence);
|
|
if (!m)
|
|
return audit_fail(m.error() == CheckpointError::unsupported ? AuditError::unsupported
|
|
: AuditError::manifest,
|
|
sequence);
|
|
if (m->key != key || m->sequence != sequence || m->commit != link->head.commit ||
|
|
m->end_tick != link->head.end_tick || m->frame != link->head.frame)
|
|
return audit_fail(AuditError::manifest, sequence);
|
|
if (!supported(*m) || m->profile != Codec::profile ||
|
|
std::ranges::find(builds, m->build) == builds.end())
|
|
return audit_fail(AuditError::unsupported, sequence);
|
|
if (sequence == 0) {
|
|
out.clone = m->predecessor_commit != Uuid{};
|
|
out.origin_commit = m->predecessor_commit;
|
|
out.origin_digest = m->predecessor_digest;
|
|
} else if (m->predecessor_commit != previous.commit ||
|
|
m->predecessor_digest != previous_digest || m->begin_tick != previous.end_tick) {
|
|
return audit_fail(AuditError::chain, sequence);
|
|
}
|
|
auto payload = objects.get({m->payload_sha256, m->payload_bytes});
|
|
if (!payload)
|
|
return audit_fail(payload.error() == ObjectError::missing ? AuditError::missing_payload
|
|
: payload.error() == ObjectError::corrupt
|
|
? AuditError::corrupt_payload
|
|
: AuditError::store,
|
|
sequence);
|
|
auto restored = Codec::restore(*payload, *m, budget);
|
|
if (!restored)
|
|
return audit_fail(AuditError::payload, sequence);
|
|
if (auto tail = verify_tail<Codec>(db, objects, key, *m, *payload, **restored, durable,
|
|
budget, timeout);
|
|
!tail)
|
|
return audit_fail(from_recovery(tail.error().kind), sequence, tail.error().store);
|
|
previous = *m;
|
|
previous_digest = *digest;
|
|
state = std::move(*restored);
|
|
}
|
|
if (!state || !Codec::reconcile(*state, durable, head.head))
|
|
return audit_fail(AuditError::history, last);
|
|
out.commits = last + 1;
|
|
out.end_tick = head.head.end_tick;
|
|
out.outcomes = durable.size();
|
|
return out;
|
|
}
|
|
} // namespace
|
|
|
|
std::expected<ClonedRegion, CloneFailure>
|
|
clone_region(Postgres& source, const FilesystemObjectStore& source_objects, RegionKey source_key,
|
|
Postgres& target_owner, const FilesystemObjectStore& target_objects,
|
|
RegionKey target_key, Uuid commit, std::span<const Sha256Digest> retained_builds,
|
|
std::size_t restore_budget, milliseconds timeout, const CloneHooks& hooks) {
|
|
if (source_key.world == 0 || source_key.region == 0 || target_key.world == 0 ||
|
|
target_key.region == 0 || target_key.world == source_key.world)
|
|
return clone_fail(CloneError::identity);
|
|
// Every later transaction on the source session is read-only, including the
|
|
// SECURITY DEFINER functions it calls: a clone cannot write to its source.
|
|
if (auto set =
|
|
source.query("SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY", {}, timeout);
|
|
!set)
|
|
return clone_fail(CloneError::session, store_failure(set.error()));
|
|
auto read_only = source.query(
|
|
"SELECT pg_catalog.current_setting('default_transaction_read_only')", {}, timeout);
|
|
if (!read_only)
|
|
return clone_fail(CloneError::session, store_failure(read_only.error()));
|
|
if (read_only->size() != 1 || (*read_only)[0].size() != 1 || (*read_only)[0][0] != "on")
|
|
return clone_fail(CloneError::session);
|
|
auto head = load_region_checkpoint(source, source_key, timeout);
|
|
if (!head)
|
|
return source_fail({head.error().kind == LeaseStoreError::not_provisioned
|
|
? RecoveryError::not_provisioned
|
|
: RecoveryError::store,
|
|
head.error()});
|
|
const auto manifest = decode_manifest(head->manifest);
|
|
if (!manifest)
|
|
return source_fail({manifest.error() == CheckpointError::unsupported
|
|
? RecoveryError::unsupported
|
|
: RecoveryError::manifest,
|
|
{}});
|
|
switch (manifest->profile) {
|
|
case CheckpointProfile::environmental_v1:
|
|
return clone<EnvironmentalCodec>(source, source_objects, source_key, target_owner,
|
|
target_objects, target_key, commit, retained_builds,
|
|
restore_budget, timeout, hooks);
|
|
case CheckpointProfile::environmental_lineage_v1:
|
|
return clone<LineageCodec>(source, source_objects, source_key, target_owner, target_objects,
|
|
target_key, commit, retained_builds, restore_budget, timeout,
|
|
hooks);
|
|
default:
|
|
return source_fail({RecoveryError::unsupported, {}});
|
|
}
|
|
}
|
|
|
|
std::expected<HistoryAudit, AuditFailure>
|
|
audit_region_history(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
|
std::span<const Sha256Digest> retained_builds, std::size_t restore_budget,
|
|
std::uint64_t max_commits, milliseconds timeout) {
|
|
auto head = load_region_checkpoint(db, key, timeout);
|
|
if (!head)
|
|
return audit_fail(head.error().kind == LeaseStoreError::not_provisioned
|
|
? AuditError::not_provisioned
|
|
: AuditError::store,
|
|
0, head.error());
|
|
const auto manifest = decode_manifest(head->manifest);
|
|
if (!manifest)
|
|
return audit_fail(manifest.error() == CheckpointError::unsupported ? AuditError::unsupported
|
|
: AuditError::manifest,
|
|
head->head.sequence);
|
|
switch (manifest->profile) {
|
|
case CheckpointProfile::environmental_v1:
|
|
return audit<EnvironmentalCodec>(db, objects, key, retained_builds, restore_budget,
|
|
max_commits, timeout, *head);
|
|
case CheckpointProfile::environmental_lineage_v1:
|
|
return audit<LineageCodec>(db, objects, key, retained_builds, restore_budget, max_commits,
|
|
timeout, *head);
|
|
default:
|
|
return audit_fail(AuditError::unsupported, head->head.sequence);
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
std::unexpected<CompareFailure> compare_fail(CompareError kind, AuditFailure audit = {},
|
|
char side = 0) {
|
|
return std::unexpected{CompareFailure{kind, audit, side}};
|
|
}
|
|
|
|
template <typename Codec>
|
|
std::expected<HistoryComparison, CompareFailure>
|
|
compare(Postgres& db_a, const FilesystemObjectStore& objects_a, RegionKey a, Postgres& db_b,
|
|
const FilesystemObjectStore& objects_b, RegionKey b, std::span<const Sha256Digest> builds,
|
|
std::size_t budget, milliseconds timeout, const HistoryAudit& audited_a,
|
|
const HistoryAudit& audited_b) {
|
|
auto ra = recover<Codec>(db_a, objects_a, a, builds, budget, timeout);
|
|
auto rb = recover<Codec>(db_b, objects_b, b, builds, budget, timeout);
|
|
if (!ra || !rb) {
|
|
const auto kind = !ra ? ra.error().kind : rb.error().kind;
|
|
return compare_fail(kind == RecoveryError::unsupported ? CompareError::versions
|
|
: CompareError::audit,
|
|
{}, !ra ? 'a' : 'b');
|
|
}
|
|
const auto& ma = ra->manifest;
|
|
const auto& mb = rb->manifest;
|
|
// The audited heads are the compared heads: a region that moved is not compared.
|
|
if (ma.end_tick != audited_a.end_tick || mb.end_tick != audited_b.end_tick ||
|
|
ma.end_tick != mb.end_tick)
|
|
return compare_fail(CompareError::ticks);
|
|
if (ma.frame != mb.frame || ma.tick_microseconds != mb.tick_microseconds ||
|
|
ma.versions != mb.versions || ma.history_profile != mb.history_profile ||
|
|
ma.rng_streams != mb.rng_streams)
|
|
return compare_fail(CompareError::versions);
|
|
const auto state_a = Codec::snapshot(*ra->state);
|
|
if (!state_a || !rb->state->rebind_world(a.world))
|
|
return compare_fail(CompareError::state);
|
|
const auto state_b = Codec::snapshot(*rb->state);
|
|
if (!state_b || *state_a != *state_b)
|
|
return compare_fail(CompareError::state);
|
|
const auto decisions = [](const auto& recovered) {
|
|
std::vector<BatchDecision> out;
|
|
out.reserve(recovered.decisions.size());
|
|
for (const auto& d : recovered.decisions)
|
|
out.push_back(d.decision);
|
|
std::ranges::sort(out, {}, [](const BatchDecision& d) {
|
|
return std::tuple{d.tick, static_cast<unsigned>(d.actor)};
|
|
});
|
|
return out;
|
|
};
|
|
const auto outcomes = decisions(*ra);
|
|
if (outcomes != decisions(*rb))
|
|
return compare_fail(CompareError::outcomes);
|
|
const auto digest = sha256(*state_a);
|
|
if (!digest)
|
|
return compare_fail(CompareError::state);
|
|
return HistoryComparison{ma.end_tick, outcomes.size(), audited_a.commits, audited_b.commits,
|
|
*digest};
|
|
}
|
|
} // namespace
|
|
|
|
std::expected<HistoryComparison, CompareFailure>
|
|
compare_region_histories(Postgres& db_a, const FilesystemObjectStore& objects_a, RegionKey a,
|
|
Postgres& db_b, const FilesystemObjectStore& objects_b, RegionKey b,
|
|
std::span<const Sha256Digest> retained_builds, std::size_t restore_budget,
|
|
std::uint64_t max_commits, milliseconds timeout) {
|
|
if (a.world == 0 || a.region == 0 || b.world == 0 || b.region == 0 || a.region != b.region ||
|
|
(a == b && &db_a == &db_b))
|
|
return compare_fail(CompareError::identity);
|
|
const auto audited_a = audit_region_history(db_a, objects_a, a, retained_builds, restore_budget,
|
|
max_commits, timeout);
|
|
if (!audited_a)
|
|
return compare_fail(CompareError::audit, audited_a.error(), 'a');
|
|
const auto audited_b = audit_region_history(db_b, objects_b, b, retained_builds, restore_budget,
|
|
max_commits, timeout);
|
|
if (!audited_b)
|
|
return compare_fail(CompareError::audit, audited_b.error(), 'b');
|
|
if (audited_a->profile != audited_b->profile)
|
|
return compare_fail(CompareError::versions);
|
|
switch (audited_a->profile) {
|
|
case CheckpointProfile::environmental_v1:
|
|
return compare<EnvironmentalCodec>(db_a, objects_a, a, db_b, objects_b, b, retained_builds,
|
|
restore_budget, timeout, *audited_a, *audited_b);
|
|
case CheckpointProfile::environmental_lineage_v1:
|
|
return compare<LineageCodec>(db_a, objects_a, a, db_b, objects_b, b, retained_builds,
|
|
restore_budget, timeout, *audited_a, *audited_b);
|
|
default:
|
|
return compare_fail(CompareError::versions);
|
|
}
|
|
}
|
|
} // namespace evo::persistence
|