feat(ops): M8.9 upgrade rehearsal on clones, barrier and node plan (ADR-0076)
Drill-only --synthetic-until/--accelerate and evo-world-clone compare (compare_region_histories) prove a candidate build runs the same world on clones of one cut before any barrier. upgrade-check.sh rehearses a failed migration under load and the version barrier with real hosts; upgrade-barrier.sh retires an old build in dev/stage only once the new digest is Ready. Nodes: aevolution.io/stores pins every objects-volume pod, aevolution.io/compute is preference-only (Alertmanager first), checked by policy and scheduling-check.sh. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
#include "evo/persistence/region_clone.hpp"
|
||||
#include "checkpoint_detail.hpp"
|
||||
#include <algorithm>
|
||||
#include <iterator>
|
||||
#include <tuple>
|
||||
|
||||
namespace evo::persistence {
|
||||
namespace {
|
||||
@@ -266,4 +268,91 @@ audit_region_history(Postgres& db, const FilesystemObjectStore& objects, RegionK
|
||||
return audit_fail(AuditError::unsupported, head->head.sequence);
|
||||
}
|
||||
}
|
||||
|
||||
namespace {
|
||||
std::unexpected<CompareFailure> compare_fail(CompareError kind, AuditFailure audit = {},
|
||||
char side = 0) {
|
||||
return std::unexpected{CompareFailure{kind, audit, side}};
|
||||
}
|
||||
|
||||
template <typename Codec>
|
||||
std::expected<HistoryComparison, CompareFailure>
|
||||
compare(Postgres& db_a, const FilesystemObjectStore& objects_a, RegionKey a, Postgres& db_b,
|
||||
const FilesystemObjectStore& objects_b, RegionKey b, std::span<const Sha256Digest> builds,
|
||||
std::size_t budget, milliseconds timeout, const HistoryAudit& audited_a,
|
||||
const HistoryAudit& audited_b) {
|
||||
auto ra = recover<Codec>(db_a, objects_a, a, builds, budget, timeout);
|
||||
auto rb = recover<Codec>(db_b, objects_b, b, builds, budget, timeout);
|
||||
if (!ra || !rb) {
|
||||
const auto kind = !ra ? ra.error().kind : rb.error().kind;
|
||||
return compare_fail(kind == RecoveryError::unsupported ? CompareError::versions
|
||||
: CompareError::audit,
|
||||
{}, !ra ? 'a' : 'b');
|
||||
}
|
||||
const auto& ma = ra->manifest;
|
||||
const auto& mb = rb->manifest;
|
||||
// The audited heads are the compared heads: a region that moved is not compared.
|
||||
if (ma.end_tick != audited_a.end_tick || mb.end_tick != audited_b.end_tick ||
|
||||
ma.end_tick != mb.end_tick)
|
||||
return compare_fail(CompareError::ticks);
|
||||
if (ma.frame != mb.frame || ma.tick_microseconds != mb.tick_microseconds ||
|
||||
ma.versions != mb.versions || ma.history_profile != mb.history_profile ||
|
||||
ma.rng_streams != mb.rng_streams)
|
||||
return compare_fail(CompareError::versions);
|
||||
const auto state_a = Codec::snapshot(*ra->state);
|
||||
if (!state_a || !rb->state->rebind_world(a.world))
|
||||
return compare_fail(CompareError::state);
|
||||
const auto state_b = Codec::snapshot(*rb->state);
|
||||
if (!state_b || *state_a != *state_b)
|
||||
return compare_fail(CompareError::state);
|
||||
const auto decisions = [](const auto& recovered) {
|
||||
std::vector<BatchDecision> out;
|
||||
out.reserve(recovered.decisions.size());
|
||||
for (const auto& d : recovered.decisions)
|
||||
out.push_back(d.decision);
|
||||
std::ranges::sort(out, {}, [](const BatchDecision& d) {
|
||||
return std::tuple{d.tick, static_cast<unsigned>(d.actor)};
|
||||
});
|
||||
return out;
|
||||
};
|
||||
const auto outcomes = decisions(*ra);
|
||||
if (outcomes != decisions(*rb))
|
||||
return compare_fail(CompareError::outcomes);
|
||||
const auto digest = sha256(*state_a);
|
||||
if (!digest)
|
||||
return compare_fail(CompareError::state);
|
||||
return HistoryComparison{ma.end_tick, outcomes.size(), audited_a.commits, audited_b.commits,
|
||||
*digest};
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::expected<HistoryComparison, CompareFailure>
|
||||
compare_region_histories(Postgres& db_a, const FilesystemObjectStore& objects_a, RegionKey a,
|
||||
Postgres& db_b, const FilesystemObjectStore& objects_b, RegionKey b,
|
||||
std::span<const Sha256Digest> retained_builds, std::size_t restore_budget,
|
||||
std::uint64_t max_commits, milliseconds timeout) {
|
||||
if (a.world == 0 || a.region == 0 || b.world == 0 || b.region == 0 || a.region != b.region ||
|
||||
(a == b && &db_a == &db_b))
|
||||
return compare_fail(CompareError::identity);
|
||||
const auto audited_a = audit_region_history(db_a, objects_a, a, retained_builds, restore_budget,
|
||||
max_commits, timeout);
|
||||
if (!audited_a)
|
||||
return compare_fail(CompareError::audit, audited_a.error(), 'a');
|
||||
const auto audited_b = audit_region_history(db_b, objects_b, b, retained_builds, restore_budget,
|
||||
max_commits, timeout);
|
||||
if (!audited_b)
|
||||
return compare_fail(CompareError::audit, audited_b.error(), 'b');
|
||||
if (audited_a->profile != audited_b->profile)
|
||||
return compare_fail(CompareError::versions);
|
||||
switch (audited_a->profile) {
|
||||
case CheckpointProfile::environmental_v1:
|
||||
return compare<EnvironmentalCodec>(db_a, objects_a, a, db_b, objects_b, b, retained_builds,
|
||||
restore_budget, timeout, *audited_a, *audited_b);
|
||||
case CheckpointProfile::environmental_lineage_v1:
|
||||
return compare<LineageCodec>(db_a, objects_a, a, db_b, objects_b, b, retained_builds,
|
||||
restore_budget, timeout, *audited_a, *audited_b);
|
||||
default:
|
||||
return compare_fail(CompareError::versions);
|
||||
}
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
|
||||
Reference in New Issue
Block a user