feat(ops): M8.9 upgrade rehearsal on clones, barrier and node plan (ADR-0076)

Drill-only --synthetic-until/--accelerate and evo-world-clone compare
(compare_region_histories) prove a candidate build runs the same world on
clones of one cut before any barrier. upgrade-check.sh rehearses a failed
migration under load and the version barrier with real hosts; upgrade-barrier.sh
retires an old build in dev/stage only once the new digest is Ready. Nodes:
aevolution.io/stores pins every objects-volume pod, aevolution.io/compute is
preference-only (Alertmanager first), checked by policy and scheduling-check.sh.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-30 17:51:17 -04:00
co-authored by Claude Opus 5.5
parent 645d16b27a
commit 5b482ad6a3
19 changed files with 1028 additions and 12 deletions
+26 -3
View File
@@ -28,6 +28,12 @@
// restarting from the committed tick after every recovery. Request IDs are
// "SYN1" + region + tick, so acknowledgements can be checked across owners.
//
// Upgrade rehearsal on clones only (M8.9, ADR-0076), both requiring
// --synthetic-inputs: --synthetic-until <tick> stops the driver after that tick, so
// the region commits exactly up to it and then idles (inputs are never invented);
// --accelerate <2..1000> runs the region's scheduling clock that many times faster
// than real time (lease deadlines stay real). Never in a deployed environment.
//
// --crash-at <point>:<n> (tests only, M7.7) SIGKILLs this process at the n-th
// occurrence of before-upload, before-transaction, after-commit or after-ack (right
// after a step's acknowledgements are written and flushed).
@@ -69,6 +75,8 @@ struct Options {
std::uint64_t crash_ordinal{};
std::string ops_listen;
bool synthetic{};
std::uint64_t synthetic_until{}; // 0: unbounded.
std::uint64_t accelerate{1};
std::uint64_t objects_reserve_mib{1024};
};
@@ -174,6 +182,10 @@ std::optional<Options> parse(int argc, char** argv) {
o.ops_listen = value;
else if (flag == "--synthetic-inputs" && value == "development-v1")
o.synthetic = true;
else if (flag == "--synthetic-until" && n && *n > 0)
o.synthetic_until = *n;
else if (flag == "--accelerate" && n && *n >= 2 && *n <= 1000)
o.accelerate = *n;
else if (flag == "--objects-reserve-mib" && n && *n <= 1048576)
o.objects_reserve_mib = *n;
else if (flag == "--crash-at") {
@@ -196,6 +208,9 @@ std::optional<Options> parse(int argc, char** argv) {
o.pg.port.empty() || o.pg.database.empty() || o.pg.user.empty() || o.objects.empty() ||
o.host.build == persistence::Sha256Digest{})
return std::nullopt;
// Rehearsal controls exist only for the synthetic driver (clones, drills).
if ((o.synthetic_until != 0 || o.accelerate != 1) && !o.synthetic)
return std::nullopt;
if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end())
o.host.retained_builds.push_back(o.host.build);
return o;
@@ -343,6 +358,12 @@ template <typename State> int serve(Options& o) {
}
return {404, "not found\n"};
};
// Scheduling time; --accelerate scales it from process start (clones only).
const auto started = std::chrono::steady_clock::now();
const auto scheduling_now = [&] {
const auto real = std::chrono::steady_clock::now();
return started + (real - started) * static_cast<std::int64_t>(o.accelerate);
};
LineReader reader;
bool input_open = !o.synthetic, ready = false;
// Synthetic driver state: the next tick to submit, reset at every recovery.
@@ -411,7 +432,8 @@ template <typename State> int serve(Options& o) {
synthetic_next = now_status.simulated_tick + 1; // Resume after the head.
synthetic_running = running;
while (running && synthetic_next &&
*synthetic_next <= host.status().simulated_tick + 64) {
*synthetic_next <= host.status().simulated_tick + 64 &&
(o.synthetic_until == 0 || *synthetic_next <= o.synthetic_until)) {
const server::TickInput in{synthetic_request(*synthetic_next),
processes::development_input(*synthetic_next)};
if (!host.submit(in))
@@ -419,7 +441,7 @@ template <typename State> int serve(Options& o) {
++*synthetic_next;
}
}
const auto& s = host.step(std::chrono::steady_clock::now());
const auto& s = host.step(scheduling_now());
max_queued = std::max(max_queued, s.queued);
// Checkpoint age restarts at recovery and at every confirmed commit.
if ((!recovered && s.phase == server::HostPhase::running) ||
@@ -492,7 +514,8 @@ int main(int argc, char** argv) {
"--objects DIR --build HEX64 [--retained HEX64]... [--pg-hostaddr A "
"--pg-passfile F --pg-ca F] [--flush-ms N] [--drain-ms N] "
"[--lease-profile default|fast] [--ready-file PATH] "
"[--ops-listen ADDRESS:PORT]\n",
"[--ops-listen ADDRESS:PORT] [--synthetic-inputs development-v1 "
"[--synthetic-until TICK] [--accelerate N]]\n",
stderr);
return 2;
}
+79 -6
View File
@@ -11,6 +11,8 @@
// [--reserve-mib N]
// evo-world-clone provision --world W --region R --opening development-v1 --pg-* ...
// --objects DIR --build HEX [--retain-build]
// evo-world-clone compare --world A --region R --other-world B --pg-* ...
// --objects DIR [--other-objects DIR] --retained HEX... [--max-commits N]
//
// --*pg-* is host, hostaddr, port, database, user, passfile, ca, or service (a libpq
// service file whose [evolution] section supplies everything but the password file). The source
@@ -40,8 +42,14 @@
// stage; World0's opening is a separate, reviewed decision.
// Output: provisioned|already-provisioned world=W region=R sequence=S tick=T
//
// compare (M8.9, ADR-0076) fully audits both regions of one database, then requires
// the same world: equal head tick, frame, quantum and versions, identical head state
// once B is rebound to A's world, and identical durable outcomes. Builds may differ.
// Output: compared tick=T outcomes=N commits_a=N commits_b=N state=HEX
// compare differs ticks|versions|state|outcomes (exit 6)
//
// Exit: 0 done, 2 usage, 3 source/history/database failure, 4 target/objects refused,
// 5 target unknown.
// 5 target unknown, 6 compared histories differ.
#include "evo/persistence/region_clone.hpp"
#include "evo/persistence/sha256.hpp"
#include "evo/processes/development_opening.hpp"
@@ -150,7 +158,7 @@ std::optional<Options> parse(int argc, char** argv) {
Options o;
o.command = argv[1];
if (o.command != "clone" && o.command != "audit" && o.command != "check-store" &&
o.command != "provision")
o.command != "provision" && o.command != "compare")
return std::nullopt;
const bool clone = o.command == "clone";
for (int i = 2; i < argc; ++i) {
@@ -174,13 +182,16 @@ std::optional<Options> parse(int argc, char** argv) {
o.source.key.world = *n;
else if (flag == (clone ? "--source-region" : "--region") && n)
o.source.key.region = *n;
else if (clone && flag == "--target-world" && n)
else if (((clone && flag == "--target-world") ||
(o.command == "compare" && flag == "--other-world")) &&
n)
o.target.key.world = *n;
else if (clone && flag == "--target-region" && n)
o.target.key.region = *n;
else if (flag == (clone ? "--source-objects" : "--objects"))
o.source.objects = value;
else if (clone && flag == "--target-objects")
else if ((clone && flag == "--target-objects") ||
(o.command == "compare" && flag == "--other-objects"))
o.target.objects = value;
else if (clone && flag == "--commit" && hex<16>(value))
o.commit = hex<16>(value);
@@ -188,7 +199,8 @@ std::optional<Options> parse(int argc, char** argv) {
o.audit_max = *n;
else if (clone && flag == "--crash-at" && value == "before-transaction")
o.crash = true;
else if (o.command == "audit" && flag == "--max-commits" && n && *n > 0)
else if ((o.command == "audit" || o.command == "compare") && flag == "--max-commits" && n &&
*n > 0)
o.max_commits = *n;
else if (o.command == "check-store" && flag == "--reserve-mib" && n && *n <= 1048576)
o.reserve_mib = *n;
@@ -203,6 +215,14 @@ std::optional<Options> parse(int argc, char** argv) {
}
if (clone && o.target.key.region == 0)
o.target.key.region = o.source.key.region;
if (o.command == "compare") {
if (o.target.key.world == 0)
return std::nullopt;
o.target.key.region = o.source.key.region;
o.target.pg = o.source.pg;
if (o.target.objects.empty())
o.target.objects = o.source.objects;
}
const bool needs_builds = o.command != "check-store" && o.command != "provision";
if (o.command == "provision" && (o.opening.empty() || !o.build))
return std::nullopt;
@@ -444,6 +464,56 @@ int audit(const Options& o) {
return 0;
}
const char* name(persistence::CompareError e) {
using enum persistence::CompareError;
switch (e) {
case identity:
return "identity";
case audit:
return "audit";
case ticks:
return "ticks";
case versions:
return "versions";
case state:
return "state";
case outcomes:
return "outcomes";
}
return "unknown";
}
int compare(const Options& o) {
auto db = connect(o.source);
auto objects_a = open_objects(o.source);
auto objects_b = open_objects(o.target);
if (!db || !objects_a || !objects_b) {
std::fputs("compare failed store\n", stderr);
return 3;
}
auto result = persistence::compare_region_histories(*db, *objects_a, o.source.key, *db,
*objects_b, o.target.key, o.retained,
restore_budget, o.max_commits, timeout);
if (!result) {
const auto& f = result.error();
using persistence::CompareError;
if (f.kind == CompareError::identity || f.kind == CompareError::audit) {
std::fprintf(stderr, "compare failed %s side=%c audit=%s sequence=%llu\n", name(f.kind),
f.side == 0 ? '-' : f.side, name(f.audit.kind),
static_cast<unsigned long long>(f.audit.sequence));
return f.kind == CompareError::identity ? 2 : 3;
}
std::printf("compare differs %s\n", name(f.kind));
return 6;
}
std::printf("compared tick=%llu outcomes=%llu commits_a=%llu commits_b=%llu state=%s\n",
static_cast<unsigned long long>(result->tick),
static_cast<unsigned long long>(result->outcomes),
static_cast<unsigned long long>(result->commits_a),
static_cast<unsigned long long>(result->commits_b), text(result->state).c_str());
return 0;
}
int clone(const Options& o) {
const auto& s = o.source.pg;
const auto& t = o.target.pg;
@@ -511,7 +581,8 @@ int clone(const Options& o) {
int main(int argc, char** argv) {
const auto options = parse(argc, argv);
if (!options) {
std::fputs("usage: evo-world-clone clone|audit ... (see apps/evo-world-clone/main.cpp)\n",
std::fputs("usage: evo-world-clone clone|audit|check-store|provision|compare ... "
"(see apps/evo-world-clone/main.cpp)\n",
stderr);
return 2;
}
@@ -519,5 +590,7 @@ int main(int argc, char** argv) {
return check_store(*options);
if (options->command == "provision")
return provision(*options);
if (options->command == "compare")
return compare(*options);
return options->command == "clone" ? clone(*options) : audit(*options);
}