feat(ops): M8.9 upgrade rehearsal on clones, barrier and node plan (ADR-0076)
Drill-only --synthetic-until/--accelerate and evo-world-clone compare (compare_region_histories) prove a candidate build runs the same world on clones of one cut before any barrier. upgrade-check.sh rehearses a failed migration under load and the version barrier with real hosts; upgrade-barrier.sh retires an old build in dev/stage only once the new digest is Ready. Nodes: aevolution.io/stores pins every objects-volume pod, aevolution.io/compute is preference-only (Alertmanager first), checked by policy and scheduling-check.sh. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -28,6 +28,12 @@
|
||||
// restarting from the committed tick after every recovery. Request IDs are
|
||||
// "SYN1" + region + tick, so acknowledgements can be checked across owners.
|
||||
//
|
||||
// Upgrade rehearsal on clones only (M8.9, ADR-0076), both requiring
|
||||
// --synthetic-inputs: --synthetic-until <tick> stops the driver after that tick, so
|
||||
// the region commits exactly up to it and then idles (inputs are never invented);
|
||||
// --accelerate <2..1000> runs the region's scheduling clock that many times faster
|
||||
// than real time (lease deadlines stay real). Never in a deployed environment.
|
||||
//
|
||||
// --crash-at <point>:<n> (tests only, M7.7) SIGKILLs this process at the n-th
|
||||
// occurrence of before-upload, before-transaction, after-commit or after-ack (right
|
||||
// after a step's acknowledgements are written and flushed).
|
||||
@@ -69,6 +75,8 @@ struct Options {
|
||||
std::uint64_t crash_ordinal{};
|
||||
std::string ops_listen;
|
||||
bool synthetic{};
|
||||
std::uint64_t synthetic_until{}; // 0: unbounded.
|
||||
std::uint64_t accelerate{1};
|
||||
std::uint64_t objects_reserve_mib{1024};
|
||||
};
|
||||
|
||||
@@ -174,6 +182,10 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
o.ops_listen = value;
|
||||
else if (flag == "--synthetic-inputs" && value == "development-v1")
|
||||
o.synthetic = true;
|
||||
else if (flag == "--synthetic-until" && n && *n > 0)
|
||||
o.synthetic_until = *n;
|
||||
else if (flag == "--accelerate" && n && *n >= 2 && *n <= 1000)
|
||||
o.accelerate = *n;
|
||||
else if (flag == "--objects-reserve-mib" && n && *n <= 1048576)
|
||||
o.objects_reserve_mib = *n;
|
||||
else if (flag == "--crash-at") {
|
||||
@@ -196,6 +208,9 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
o.pg.port.empty() || o.pg.database.empty() || o.pg.user.empty() || o.objects.empty() ||
|
||||
o.host.build == persistence::Sha256Digest{})
|
||||
return std::nullopt;
|
||||
// Rehearsal controls exist only for the synthetic driver (clones, drills).
|
||||
if ((o.synthetic_until != 0 || o.accelerate != 1) && !o.synthetic)
|
||||
return std::nullopt;
|
||||
if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end())
|
||||
o.host.retained_builds.push_back(o.host.build);
|
||||
return o;
|
||||
@@ -343,6 +358,12 @@ template <typename State> int serve(Options& o) {
|
||||
}
|
||||
return {404, "not found\n"};
|
||||
};
|
||||
// Scheduling time; --accelerate scales it from process start (clones only).
|
||||
const auto started = std::chrono::steady_clock::now();
|
||||
const auto scheduling_now = [&] {
|
||||
const auto real = std::chrono::steady_clock::now();
|
||||
return started + (real - started) * static_cast<std::int64_t>(o.accelerate);
|
||||
};
|
||||
LineReader reader;
|
||||
bool input_open = !o.synthetic, ready = false;
|
||||
// Synthetic driver state: the next tick to submit, reset at every recovery.
|
||||
@@ -411,7 +432,8 @@ template <typename State> int serve(Options& o) {
|
||||
synthetic_next = now_status.simulated_tick + 1; // Resume after the head.
|
||||
synthetic_running = running;
|
||||
while (running && synthetic_next &&
|
||||
*synthetic_next <= host.status().simulated_tick + 64) {
|
||||
*synthetic_next <= host.status().simulated_tick + 64 &&
|
||||
(o.synthetic_until == 0 || *synthetic_next <= o.synthetic_until)) {
|
||||
const server::TickInput in{synthetic_request(*synthetic_next),
|
||||
processes::development_input(*synthetic_next)};
|
||||
if (!host.submit(in))
|
||||
@@ -419,7 +441,7 @@ template <typename State> int serve(Options& o) {
|
||||
++*synthetic_next;
|
||||
}
|
||||
}
|
||||
const auto& s = host.step(std::chrono::steady_clock::now());
|
||||
const auto& s = host.step(scheduling_now());
|
||||
max_queued = std::max(max_queued, s.queued);
|
||||
// Checkpoint age restarts at recovery and at every confirmed commit.
|
||||
if ((!recovered && s.phase == server::HostPhase::running) ||
|
||||
@@ -492,7 +514,8 @@ int main(int argc, char** argv) {
|
||||
"--objects DIR --build HEX64 [--retained HEX64]... [--pg-hostaddr A "
|
||||
"--pg-passfile F --pg-ca F] [--flush-ms N] [--drain-ms N] "
|
||||
"[--lease-profile default|fast] [--ready-file PATH] "
|
||||
"[--ops-listen ADDRESS:PORT]\n",
|
||||
"[--ops-listen ADDRESS:PORT] [--synthetic-inputs development-v1 "
|
||||
"[--synthetic-until TICK] [--accelerate N]]\n",
|
||||
stderr);
|
||||
return 2;
|
||||
}
|
||||
|
||||
@@ -11,6 +11,8 @@
|
||||
// [--reserve-mib N]
|
||||
// evo-world-clone provision --world W --region R --opening development-v1 --pg-* ...
|
||||
// --objects DIR --build HEX [--retain-build]
|
||||
// evo-world-clone compare --world A --region R --other-world B --pg-* ...
|
||||
// --objects DIR [--other-objects DIR] --retained HEX... [--max-commits N]
|
||||
//
|
||||
// --*pg-* is host, hostaddr, port, database, user, passfile, ca, or service (a libpq
|
||||
// service file whose [evolution] section supplies everything but the password file). The source
|
||||
@@ -40,8 +42,14 @@
|
||||
// stage; World0's opening is a separate, reviewed decision.
|
||||
// Output: provisioned|already-provisioned world=W region=R sequence=S tick=T
|
||||
//
|
||||
// compare (M8.9, ADR-0076) fully audits both regions of one database, then requires
|
||||
// the same world: equal head tick, frame, quantum and versions, identical head state
|
||||
// once B is rebound to A's world, and identical durable outcomes. Builds may differ.
|
||||
// Output: compared tick=T outcomes=N commits_a=N commits_b=N state=HEX
|
||||
// compare differs ticks|versions|state|outcomes (exit 6)
|
||||
//
|
||||
// Exit: 0 done, 2 usage, 3 source/history/database failure, 4 target/objects refused,
|
||||
// 5 target unknown.
|
||||
// 5 target unknown, 6 compared histories differ.
|
||||
#include "evo/persistence/region_clone.hpp"
|
||||
#include "evo/persistence/sha256.hpp"
|
||||
#include "evo/processes/development_opening.hpp"
|
||||
@@ -150,7 +158,7 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
Options o;
|
||||
o.command = argv[1];
|
||||
if (o.command != "clone" && o.command != "audit" && o.command != "check-store" &&
|
||||
o.command != "provision")
|
||||
o.command != "provision" && o.command != "compare")
|
||||
return std::nullopt;
|
||||
const bool clone = o.command == "clone";
|
||||
for (int i = 2; i < argc; ++i) {
|
||||
@@ -174,13 +182,16 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
o.source.key.world = *n;
|
||||
else if (flag == (clone ? "--source-region" : "--region") && n)
|
||||
o.source.key.region = *n;
|
||||
else if (clone && flag == "--target-world" && n)
|
||||
else if (((clone && flag == "--target-world") ||
|
||||
(o.command == "compare" && flag == "--other-world")) &&
|
||||
n)
|
||||
o.target.key.world = *n;
|
||||
else if (clone && flag == "--target-region" && n)
|
||||
o.target.key.region = *n;
|
||||
else if (flag == (clone ? "--source-objects" : "--objects"))
|
||||
o.source.objects = value;
|
||||
else if (clone && flag == "--target-objects")
|
||||
else if ((clone && flag == "--target-objects") ||
|
||||
(o.command == "compare" && flag == "--other-objects"))
|
||||
o.target.objects = value;
|
||||
else if (clone && flag == "--commit" && hex<16>(value))
|
||||
o.commit = hex<16>(value);
|
||||
@@ -188,7 +199,8 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
o.audit_max = *n;
|
||||
else if (clone && flag == "--crash-at" && value == "before-transaction")
|
||||
o.crash = true;
|
||||
else if (o.command == "audit" && flag == "--max-commits" && n && *n > 0)
|
||||
else if ((o.command == "audit" || o.command == "compare") && flag == "--max-commits" && n &&
|
||||
*n > 0)
|
||||
o.max_commits = *n;
|
||||
else if (o.command == "check-store" && flag == "--reserve-mib" && n && *n <= 1048576)
|
||||
o.reserve_mib = *n;
|
||||
@@ -203,6 +215,14 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
}
|
||||
if (clone && o.target.key.region == 0)
|
||||
o.target.key.region = o.source.key.region;
|
||||
if (o.command == "compare") {
|
||||
if (o.target.key.world == 0)
|
||||
return std::nullopt;
|
||||
o.target.key.region = o.source.key.region;
|
||||
o.target.pg = o.source.pg;
|
||||
if (o.target.objects.empty())
|
||||
o.target.objects = o.source.objects;
|
||||
}
|
||||
const bool needs_builds = o.command != "check-store" && o.command != "provision";
|
||||
if (o.command == "provision" && (o.opening.empty() || !o.build))
|
||||
return std::nullopt;
|
||||
@@ -444,6 +464,56 @@ int audit(const Options& o) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const char* name(persistence::CompareError e) {
|
||||
using enum persistence::CompareError;
|
||||
switch (e) {
|
||||
case identity:
|
||||
return "identity";
|
||||
case audit:
|
||||
return "audit";
|
||||
case ticks:
|
||||
return "ticks";
|
||||
case versions:
|
||||
return "versions";
|
||||
case state:
|
||||
return "state";
|
||||
case outcomes:
|
||||
return "outcomes";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
int compare(const Options& o) {
|
||||
auto db = connect(o.source);
|
||||
auto objects_a = open_objects(o.source);
|
||||
auto objects_b = open_objects(o.target);
|
||||
if (!db || !objects_a || !objects_b) {
|
||||
std::fputs("compare failed store\n", stderr);
|
||||
return 3;
|
||||
}
|
||||
auto result = persistence::compare_region_histories(*db, *objects_a, o.source.key, *db,
|
||||
*objects_b, o.target.key, o.retained,
|
||||
restore_budget, o.max_commits, timeout);
|
||||
if (!result) {
|
||||
const auto& f = result.error();
|
||||
using persistence::CompareError;
|
||||
if (f.kind == CompareError::identity || f.kind == CompareError::audit) {
|
||||
std::fprintf(stderr, "compare failed %s side=%c audit=%s sequence=%llu\n", name(f.kind),
|
||||
f.side == 0 ? '-' : f.side, name(f.audit.kind),
|
||||
static_cast<unsigned long long>(f.audit.sequence));
|
||||
return f.kind == CompareError::identity ? 2 : 3;
|
||||
}
|
||||
std::printf("compare differs %s\n", name(f.kind));
|
||||
return 6;
|
||||
}
|
||||
std::printf("compared tick=%llu outcomes=%llu commits_a=%llu commits_b=%llu state=%s\n",
|
||||
static_cast<unsigned long long>(result->tick),
|
||||
static_cast<unsigned long long>(result->outcomes),
|
||||
static_cast<unsigned long long>(result->commits_a),
|
||||
static_cast<unsigned long long>(result->commits_b), text(result->state).c_str());
|
||||
return 0;
|
||||
}
|
||||
|
||||
int clone(const Options& o) {
|
||||
const auto& s = o.source.pg;
|
||||
const auto& t = o.target.pg;
|
||||
@@ -511,7 +581,8 @@ int clone(const Options& o) {
|
||||
int main(int argc, char** argv) {
|
||||
const auto options = parse(argc, argv);
|
||||
if (!options) {
|
||||
std::fputs("usage: evo-world-clone clone|audit ... (see apps/evo-world-clone/main.cpp)\n",
|
||||
std::fputs("usage: evo-world-clone clone|audit|check-store|provision|compare ... "
|
||||
"(see apps/evo-world-clone/main.cpp)\n",
|
||||
stderr);
|
||||
return 2;
|
||||
}
|
||||
@@ -519,5 +590,7 @@ int main(int argc, char** argv) {
|
||||
return check_store(*options);
|
||||
if (options->command == "provision")
|
||||
return provision(*options);
|
||||
if (options->command == "compare")
|
||||
return compare(*options);
|
||||
return options->command == "clone" ? clone(*options) : audit(*options);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user