feat(ops): M8.9 upgrade rehearsal on clones, barrier and node plan (ADR-0076)

Drill-only --synthetic-until/--accelerate and evo-world-clone compare
(compare_region_histories) prove a candidate build runs the same world on
clones of one cut before any barrier. upgrade-check.sh rehearses a failed
migration under load and the version barrier with real hosts; upgrade-barrier.sh
retires an old build in dev/stage only once the new digest is Ready. Nodes:
aevolution.io/stores pins every objects-volume pod, aevolution.io/compute is
preference-only (Alertmanager first), checked by policy and scheduling-check.sh.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-30 17:51:17 -04:00
co-authored by Claude Opus 5.5
parent 645d16b27a
commit 5b482ad6a3
19 changed files with 1028 additions and 12 deletions
+89
View File
@@ -1,6 +1,8 @@
#include "evo/persistence/region_clone.hpp"
#include "checkpoint_detail.hpp"
#include <algorithm>
#include <iterator>
#include <tuple>
namespace evo::persistence {
namespace {
@@ -266,4 +268,91 @@ audit_region_history(Postgres& db, const FilesystemObjectStore& objects, RegionK
return audit_fail(AuditError::unsupported, head->head.sequence);
}
}
namespace {
std::unexpected<CompareFailure> compare_fail(CompareError kind, AuditFailure audit = {},
char side = 0) {
return std::unexpected{CompareFailure{kind, audit, side}};
}
template <typename Codec>
std::expected<HistoryComparison, CompareFailure>
compare(Postgres& db_a, const FilesystemObjectStore& objects_a, RegionKey a, Postgres& db_b,
const FilesystemObjectStore& objects_b, RegionKey b, std::span<const Sha256Digest> builds,
std::size_t budget, milliseconds timeout, const HistoryAudit& audited_a,
const HistoryAudit& audited_b) {
auto ra = recover<Codec>(db_a, objects_a, a, builds, budget, timeout);
auto rb = recover<Codec>(db_b, objects_b, b, builds, budget, timeout);
if (!ra || !rb) {
const auto kind = !ra ? ra.error().kind : rb.error().kind;
return compare_fail(kind == RecoveryError::unsupported ? CompareError::versions
: CompareError::audit,
{}, !ra ? 'a' : 'b');
}
const auto& ma = ra->manifest;
const auto& mb = rb->manifest;
// The audited heads are the compared heads: a region that moved is not compared.
if (ma.end_tick != audited_a.end_tick || mb.end_tick != audited_b.end_tick ||
ma.end_tick != mb.end_tick)
return compare_fail(CompareError::ticks);
if (ma.frame != mb.frame || ma.tick_microseconds != mb.tick_microseconds ||
ma.versions != mb.versions || ma.history_profile != mb.history_profile ||
ma.rng_streams != mb.rng_streams)
return compare_fail(CompareError::versions);
const auto state_a = Codec::snapshot(*ra->state);
if (!state_a || !rb->state->rebind_world(a.world))
return compare_fail(CompareError::state);
const auto state_b = Codec::snapshot(*rb->state);
if (!state_b || *state_a != *state_b)
return compare_fail(CompareError::state);
const auto decisions = [](const auto& recovered) {
std::vector<BatchDecision> out;
out.reserve(recovered.decisions.size());
for (const auto& d : recovered.decisions)
out.push_back(d.decision);
std::ranges::sort(out, {}, [](const BatchDecision& d) {
return std::tuple{d.tick, static_cast<unsigned>(d.actor)};
});
return out;
};
const auto outcomes = decisions(*ra);
if (outcomes != decisions(*rb))
return compare_fail(CompareError::outcomes);
const auto digest = sha256(*state_a);
if (!digest)
return compare_fail(CompareError::state);
return HistoryComparison{ma.end_tick, outcomes.size(), audited_a.commits, audited_b.commits,
*digest};
}
} // namespace
std::expected<HistoryComparison, CompareFailure>
compare_region_histories(Postgres& db_a, const FilesystemObjectStore& objects_a, RegionKey a,
Postgres& db_b, const FilesystemObjectStore& objects_b, RegionKey b,
std::span<const Sha256Digest> retained_builds, std::size_t restore_budget,
std::uint64_t max_commits, milliseconds timeout) {
if (a.world == 0 || a.region == 0 || b.world == 0 || b.region == 0 || a.region != b.region ||
(a == b && &db_a == &db_b))
return compare_fail(CompareError::identity);
const auto audited_a = audit_region_history(db_a, objects_a, a, retained_builds, restore_budget,
max_commits, timeout);
if (!audited_a)
return compare_fail(CompareError::audit, audited_a.error(), 'a');
const auto audited_b = audit_region_history(db_b, objects_b, b, retained_builds, restore_budget,
max_commits, timeout);
if (!audited_b)
return compare_fail(CompareError::audit, audited_b.error(), 'b');
if (audited_a->profile != audited_b->profile)
return compare_fail(CompareError::versions);
switch (audited_a->profile) {
case CheckpointProfile::environmental_v1:
return compare<EnvironmentalCodec>(db_a, objects_a, a, db_b, objects_b, b, retained_builds,
restore_budget, timeout, *audited_a, *audited_b);
case CheckpointProfile::environmental_lineage_v1:
return compare<LineageCodec>(db_a, objects_a, a, db_b, objects_b, b, retained_builds,
restore_budget, timeout, *audited_a, *audited_b);
default:
return compare_fail(CompareError::versions);
}
}
} // namespace evo::persistence