feat(ops): M8.9 upgrade rehearsal on clones, barrier and node plan (ADR-0076)
Drill-only --synthetic-until/--accelerate and evo-world-clone compare (compare_region_histories) prove a candidate build runs the same world on clones of one cut before any barrier. upgrade-check.sh rehearses a failed migration under load and the version barrier with real hosts; upgrade-barrier.sh retires an old build in dev/stage only once the new digest is Ready. Nodes: aevolution.io/stores pins every objects-volume pod, aevolution.io/compute is preference-only (Alertmanager first), checked by policy and scheduling-check.sh. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -28,6 +28,12 @@
|
||||
// restarting from the committed tick after every recovery. Request IDs are
|
||||
// "SYN1" + region + tick, so acknowledgements can be checked across owners.
|
||||
//
|
||||
// Upgrade rehearsal on clones only (M8.9, ADR-0076), both requiring
|
||||
// --synthetic-inputs: --synthetic-until <tick> stops the driver after that tick, so
|
||||
// the region commits exactly up to it and then idles (inputs are never invented);
|
||||
// --accelerate <2..1000> runs the region's scheduling clock that many times faster
|
||||
// than real time (lease deadlines stay real). Never in a deployed environment.
|
||||
//
|
||||
// --crash-at <point>:<n> (tests only, M7.7) SIGKILLs this process at the n-th
|
||||
// occurrence of before-upload, before-transaction, after-commit or after-ack (right
|
||||
// after a step's acknowledgements are written and flushed).
|
||||
@@ -69,6 +75,8 @@ struct Options {
|
||||
std::uint64_t crash_ordinal{};
|
||||
std::string ops_listen;
|
||||
bool synthetic{};
|
||||
std::uint64_t synthetic_until{}; // 0: unbounded.
|
||||
std::uint64_t accelerate{1};
|
||||
std::uint64_t objects_reserve_mib{1024};
|
||||
};
|
||||
|
||||
@@ -174,6 +182,10 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
o.ops_listen = value;
|
||||
else if (flag == "--synthetic-inputs" && value == "development-v1")
|
||||
o.synthetic = true;
|
||||
else if (flag == "--synthetic-until" && n && *n > 0)
|
||||
o.synthetic_until = *n;
|
||||
else if (flag == "--accelerate" && n && *n >= 2 && *n <= 1000)
|
||||
o.accelerate = *n;
|
||||
else if (flag == "--objects-reserve-mib" && n && *n <= 1048576)
|
||||
o.objects_reserve_mib = *n;
|
||||
else if (flag == "--crash-at") {
|
||||
@@ -196,6 +208,9 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
o.pg.port.empty() || o.pg.database.empty() || o.pg.user.empty() || o.objects.empty() ||
|
||||
o.host.build == persistence::Sha256Digest{})
|
||||
return std::nullopt;
|
||||
// Rehearsal controls exist only for the synthetic driver (clones, drills).
|
||||
if ((o.synthetic_until != 0 || o.accelerate != 1) && !o.synthetic)
|
||||
return std::nullopt;
|
||||
if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end())
|
||||
o.host.retained_builds.push_back(o.host.build);
|
||||
return o;
|
||||
@@ -343,6 +358,12 @@ template <typename State> int serve(Options& o) {
|
||||
}
|
||||
return {404, "not found\n"};
|
||||
};
|
||||
// Scheduling time; --accelerate scales it from process start (clones only).
|
||||
const auto started = std::chrono::steady_clock::now();
|
||||
const auto scheduling_now = [&] {
|
||||
const auto real = std::chrono::steady_clock::now();
|
||||
return started + (real - started) * static_cast<std::int64_t>(o.accelerate);
|
||||
};
|
||||
LineReader reader;
|
||||
bool input_open = !o.synthetic, ready = false;
|
||||
// Synthetic driver state: the next tick to submit, reset at every recovery.
|
||||
@@ -411,7 +432,8 @@ template <typename State> int serve(Options& o) {
|
||||
synthetic_next = now_status.simulated_tick + 1; // Resume after the head.
|
||||
synthetic_running = running;
|
||||
while (running && synthetic_next &&
|
||||
*synthetic_next <= host.status().simulated_tick + 64) {
|
||||
*synthetic_next <= host.status().simulated_tick + 64 &&
|
||||
(o.synthetic_until == 0 || *synthetic_next <= o.synthetic_until)) {
|
||||
const server::TickInput in{synthetic_request(*synthetic_next),
|
||||
processes::development_input(*synthetic_next)};
|
||||
if (!host.submit(in))
|
||||
@@ -419,7 +441,7 @@ template <typename State> int serve(Options& o) {
|
||||
++*synthetic_next;
|
||||
}
|
||||
}
|
||||
const auto& s = host.step(std::chrono::steady_clock::now());
|
||||
const auto& s = host.step(scheduling_now());
|
||||
max_queued = std::max(max_queued, s.queued);
|
||||
// Checkpoint age restarts at recovery and at every confirmed commit.
|
||||
if ((!recovered && s.phase == server::HostPhase::running) ||
|
||||
@@ -492,7 +514,8 @@ int main(int argc, char** argv) {
|
||||
"--objects DIR --build HEX64 [--retained HEX64]... [--pg-hostaddr A "
|
||||
"--pg-passfile F --pg-ca F] [--flush-ms N] [--drain-ms N] "
|
||||
"[--lease-profile default|fast] [--ready-file PATH] "
|
||||
"[--ops-listen ADDRESS:PORT]\n",
|
||||
"[--ops-listen ADDRESS:PORT] [--synthetic-inputs development-v1 "
|
||||
"[--synthetic-until TICK] [--accelerate N]]\n",
|
||||
stderr);
|
||||
return 2;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user