feat(ops): M8.9 upgrade rehearsal on clones, barrier and node plan (ADR-0076)
Drill-only --synthetic-until/--accelerate and evo-world-clone compare (compare_region_histories) prove a candidate build runs the same world on clones of one cut before any barrier. upgrade-check.sh rehearses a failed migration under load and the version barrier with real hosts; upgrade-barrier.sh retires an old build in dev/stage only once the new digest is Ready. Nodes: aevolution.io/stores pins every objects-volume pod, aevolution.io/compute is preference-only (Alertmanager first), checked by policy and scheduling-check.sh. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+1
-1
@@ -128,7 +128,7 @@ if(EVO_BUILD_POSTGRES)
|
||||
tests/region_server_process_tests.cpp tests/region_recovery_tests.cpp
|
||||
tests/region_ops_tests.cpp tests/region_clone_tests.cpp
|
||||
tests/region_migration_tests.cpp tests/region_store_tests.cpp
|
||||
tests/region_load_tests.cpp)
|
||||
tests/region_load_tests.cpp tests/region_upgrade_tests.cpp)
|
||||
target_link_libraries(evo-postgres-tests PRIVATE evo::region_host Threads::Threads)
|
||||
add_dependencies(evo-postgres-tests evo-server evo-world-clone) # Spawned by process tests.
|
||||
evo_owned_target(evo-postgres-tests)
|
||||
|
||||
@@ -28,6 +28,12 @@
|
||||
// restarting from the committed tick after every recovery. Request IDs are
|
||||
// "SYN1" + region + tick, so acknowledgements can be checked across owners.
|
||||
//
|
||||
// Upgrade rehearsal on clones only (M8.9, ADR-0076), both requiring
|
||||
// --synthetic-inputs: --synthetic-until <tick> stops the driver after that tick, so
|
||||
// the region commits exactly up to it and then idles (inputs are never invented);
|
||||
// --accelerate <2..1000> runs the region's scheduling clock that many times faster
|
||||
// than real time (lease deadlines stay real). Never in a deployed environment.
|
||||
//
|
||||
// --crash-at <point>:<n> (tests only, M7.7) SIGKILLs this process at the n-th
|
||||
// occurrence of before-upload, before-transaction, after-commit or after-ack (right
|
||||
// after a step's acknowledgements are written and flushed).
|
||||
@@ -69,6 +75,8 @@ struct Options {
|
||||
std::uint64_t crash_ordinal{};
|
||||
std::string ops_listen;
|
||||
bool synthetic{};
|
||||
std::uint64_t synthetic_until{}; // 0: unbounded.
|
||||
std::uint64_t accelerate{1};
|
||||
std::uint64_t objects_reserve_mib{1024};
|
||||
};
|
||||
|
||||
@@ -174,6 +182,10 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
o.ops_listen = value;
|
||||
else if (flag == "--synthetic-inputs" && value == "development-v1")
|
||||
o.synthetic = true;
|
||||
else if (flag == "--synthetic-until" && n && *n > 0)
|
||||
o.synthetic_until = *n;
|
||||
else if (flag == "--accelerate" && n && *n >= 2 && *n <= 1000)
|
||||
o.accelerate = *n;
|
||||
else if (flag == "--objects-reserve-mib" && n && *n <= 1048576)
|
||||
o.objects_reserve_mib = *n;
|
||||
else if (flag == "--crash-at") {
|
||||
@@ -196,6 +208,9 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
o.pg.port.empty() || o.pg.database.empty() || o.pg.user.empty() || o.objects.empty() ||
|
||||
o.host.build == persistence::Sha256Digest{})
|
||||
return std::nullopt;
|
||||
// Rehearsal controls exist only for the synthetic driver (clones, drills).
|
||||
if ((o.synthetic_until != 0 || o.accelerate != 1) && !o.synthetic)
|
||||
return std::nullopt;
|
||||
if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end())
|
||||
o.host.retained_builds.push_back(o.host.build);
|
||||
return o;
|
||||
@@ -343,6 +358,12 @@ template <typename State> int serve(Options& o) {
|
||||
}
|
||||
return {404, "not found\n"};
|
||||
};
|
||||
// Scheduling time; --accelerate scales it from process start (clones only).
|
||||
const auto started = std::chrono::steady_clock::now();
|
||||
const auto scheduling_now = [&] {
|
||||
const auto real = std::chrono::steady_clock::now();
|
||||
return started + (real - started) * static_cast<std::int64_t>(o.accelerate);
|
||||
};
|
||||
LineReader reader;
|
||||
bool input_open = !o.synthetic, ready = false;
|
||||
// Synthetic driver state: the next tick to submit, reset at every recovery.
|
||||
@@ -411,7 +432,8 @@ template <typename State> int serve(Options& o) {
|
||||
synthetic_next = now_status.simulated_tick + 1; // Resume after the head.
|
||||
synthetic_running = running;
|
||||
while (running && synthetic_next &&
|
||||
*synthetic_next <= host.status().simulated_tick + 64) {
|
||||
*synthetic_next <= host.status().simulated_tick + 64 &&
|
||||
(o.synthetic_until == 0 || *synthetic_next <= o.synthetic_until)) {
|
||||
const server::TickInput in{synthetic_request(*synthetic_next),
|
||||
processes::development_input(*synthetic_next)};
|
||||
if (!host.submit(in))
|
||||
@@ -419,7 +441,7 @@ template <typename State> int serve(Options& o) {
|
||||
++*synthetic_next;
|
||||
}
|
||||
}
|
||||
const auto& s = host.step(std::chrono::steady_clock::now());
|
||||
const auto& s = host.step(scheduling_now());
|
||||
max_queued = std::max(max_queued, s.queued);
|
||||
// Checkpoint age restarts at recovery and at every confirmed commit.
|
||||
if ((!recovered && s.phase == server::HostPhase::running) ||
|
||||
@@ -492,7 +514,8 @@ int main(int argc, char** argv) {
|
||||
"--objects DIR --build HEX64 [--retained HEX64]... [--pg-hostaddr A "
|
||||
"--pg-passfile F --pg-ca F] [--flush-ms N] [--drain-ms N] "
|
||||
"[--lease-profile default|fast] [--ready-file PATH] "
|
||||
"[--ops-listen ADDRESS:PORT]\n",
|
||||
"[--ops-listen ADDRESS:PORT] [--synthetic-inputs development-v1 "
|
||||
"[--synthetic-until TICK] [--accelerate N]]\n",
|
||||
stderr);
|
||||
return 2;
|
||||
}
|
||||
|
||||
@@ -11,6 +11,8 @@
|
||||
// [--reserve-mib N]
|
||||
// evo-world-clone provision --world W --region R --opening development-v1 --pg-* ...
|
||||
// --objects DIR --build HEX [--retain-build]
|
||||
// evo-world-clone compare --world A --region R --other-world B --pg-* ...
|
||||
// --objects DIR [--other-objects DIR] --retained HEX... [--max-commits N]
|
||||
//
|
||||
// --*pg-* is host, hostaddr, port, database, user, passfile, ca, or service (a libpq
|
||||
// service file whose [evolution] section supplies everything but the password file). The source
|
||||
@@ -40,8 +42,14 @@
|
||||
// stage; World0's opening is a separate, reviewed decision.
|
||||
// Output: provisioned|already-provisioned world=W region=R sequence=S tick=T
|
||||
//
|
||||
// compare (M8.9, ADR-0076) fully audits both regions of one database, then requires
|
||||
// the same world: equal head tick, frame, quantum and versions, identical head state
|
||||
// once B is rebound to A's world, and identical durable outcomes. Builds may differ.
|
||||
// Output: compared tick=T outcomes=N commits_a=N commits_b=N state=HEX
|
||||
// compare differs ticks|versions|state|outcomes (exit 6)
|
||||
//
|
||||
// Exit: 0 done, 2 usage, 3 source/history/database failure, 4 target/objects refused,
|
||||
// 5 target unknown.
|
||||
// 5 target unknown, 6 compared histories differ.
|
||||
#include "evo/persistence/region_clone.hpp"
|
||||
#include "evo/persistence/sha256.hpp"
|
||||
#include "evo/processes/development_opening.hpp"
|
||||
@@ -150,7 +158,7 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
Options o;
|
||||
o.command = argv[1];
|
||||
if (o.command != "clone" && o.command != "audit" && o.command != "check-store" &&
|
||||
o.command != "provision")
|
||||
o.command != "provision" && o.command != "compare")
|
||||
return std::nullopt;
|
||||
const bool clone = o.command == "clone";
|
||||
for (int i = 2; i < argc; ++i) {
|
||||
@@ -174,13 +182,16 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
o.source.key.world = *n;
|
||||
else if (flag == (clone ? "--source-region" : "--region") && n)
|
||||
o.source.key.region = *n;
|
||||
else if (clone && flag == "--target-world" && n)
|
||||
else if (((clone && flag == "--target-world") ||
|
||||
(o.command == "compare" && flag == "--other-world")) &&
|
||||
n)
|
||||
o.target.key.world = *n;
|
||||
else if (clone && flag == "--target-region" && n)
|
||||
o.target.key.region = *n;
|
||||
else if (flag == (clone ? "--source-objects" : "--objects"))
|
||||
o.source.objects = value;
|
||||
else if (clone && flag == "--target-objects")
|
||||
else if ((clone && flag == "--target-objects") ||
|
||||
(o.command == "compare" && flag == "--other-objects"))
|
||||
o.target.objects = value;
|
||||
else if (clone && flag == "--commit" && hex<16>(value))
|
||||
o.commit = hex<16>(value);
|
||||
@@ -188,7 +199,8 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
o.audit_max = *n;
|
||||
else if (clone && flag == "--crash-at" && value == "before-transaction")
|
||||
o.crash = true;
|
||||
else if (o.command == "audit" && flag == "--max-commits" && n && *n > 0)
|
||||
else if ((o.command == "audit" || o.command == "compare") && flag == "--max-commits" && n &&
|
||||
*n > 0)
|
||||
o.max_commits = *n;
|
||||
else if (o.command == "check-store" && flag == "--reserve-mib" && n && *n <= 1048576)
|
||||
o.reserve_mib = *n;
|
||||
@@ -203,6 +215,14 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
}
|
||||
if (clone && o.target.key.region == 0)
|
||||
o.target.key.region = o.source.key.region;
|
||||
if (o.command == "compare") {
|
||||
if (o.target.key.world == 0)
|
||||
return std::nullopt;
|
||||
o.target.key.region = o.source.key.region;
|
||||
o.target.pg = o.source.pg;
|
||||
if (o.target.objects.empty())
|
||||
o.target.objects = o.source.objects;
|
||||
}
|
||||
const bool needs_builds = o.command != "check-store" && o.command != "provision";
|
||||
if (o.command == "provision" && (o.opening.empty() || !o.build))
|
||||
return std::nullopt;
|
||||
@@ -444,6 +464,56 @@ int audit(const Options& o) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const char* name(persistence::CompareError e) {
|
||||
using enum persistence::CompareError;
|
||||
switch (e) {
|
||||
case identity:
|
||||
return "identity";
|
||||
case audit:
|
||||
return "audit";
|
||||
case ticks:
|
||||
return "ticks";
|
||||
case versions:
|
||||
return "versions";
|
||||
case state:
|
||||
return "state";
|
||||
case outcomes:
|
||||
return "outcomes";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
int compare(const Options& o) {
|
||||
auto db = connect(o.source);
|
||||
auto objects_a = open_objects(o.source);
|
||||
auto objects_b = open_objects(o.target);
|
||||
if (!db || !objects_a || !objects_b) {
|
||||
std::fputs("compare failed store\n", stderr);
|
||||
return 3;
|
||||
}
|
||||
auto result = persistence::compare_region_histories(*db, *objects_a, o.source.key, *db,
|
||||
*objects_b, o.target.key, o.retained,
|
||||
restore_budget, o.max_commits, timeout);
|
||||
if (!result) {
|
||||
const auto& f = result.error();
|
||||
using persistence::CompareError;
|
||||
if (f.kind == CompareError::identity || f.kind == CompareError::audit) {
|
||||
std::fprintf(stderr, "compare failed %s side=%c audit=%s sequence=%llu\n", name(f.kind),
|
||||
f.side == 0 ? '-' : f.side, name(f.audit.kind),
|
||||
static_cast<unsigned long long>(f.audit.sequence));
|
||||
return f.kind == CompareError::identity ? 2 : 3;
|
||||
}
|
||||
std::printf("compare differs %s\n", name(f.kind));
|
||||
return 6;
|
||||
}
|
||||
std::printf("compared tick=%llu outcomes=%llu commits_a=%llu commits_b=%llu state=%s\n",
|
||||
static_cast<unsigned long long>(result->tick),
|
||||
static_cast<unsigned long long>(result->outcomes),
|
||||
static_cast<unsigned long long>(result->commits_a),
|
||||
static_cast<unsigned long long>(result->commits_b), text(result->state).c_str());
|
||||
return 0;
|
||||
}
|
||||
|
||||
int clone(const Options& o) {
|
||||
const auto& s = o.source.pg;
|
||||
const auto& t = o.target.pg;
|
||||
@@ -511,7 +581,8 @@ int clone(const Options& o) {
|
||||
int main(int argc, char** argv) {
|
||||
const auto options = parse(argc, argv);
|
||||
if (!options) {
|
||||
std::fputs("usage: evo-world-clone clone|audit ... (see apps/evo-world-clone/main.cpp)\n",
|
||||
std::fputs("usage: evo-world-clone clone|audit|check-store|provision|compare ... "
|
||||
"(see apps/evo-world-clone/main.cpp)\n",
|
||||
stderr);
|
||||
return 2;
|
||||
}
|
||||
@@ -519,5 +590,7 @@ int main(int argc, char** argv) {
|
||||
return check_store(*options);
|
||||
if (options->command == "provision")
|
||||
return provision(*options);
|
||||
if (options->command == "compare")
|
||||
return compare(*options);
|
||||
return options->command == "clone" ? clone(*options) : audit(*options);
|
||||
}
|
||||
|
||||
@@ -22,6 +22,15 @@ spec:
|
||||
spec:
|
||||
serviceAccountName: aevolution-alertmanager
|
||||
automountServiceAccountToken: false
|
||||
# Stateless: prefers nodes labelled for AEvolution compute (ADR-0076), and
|
||||
# schedules anywhere while none is labelled (the single-node plan).
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
preference:
|
||||
matchExpressions:
|
||||
- {key: aevolution.io/compute, operator: In, values: ["true"]}
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65534
|
||||
|
||||
@@ -829,3 +829,11 @@ durable failure domain exists, enforced by the deploy policy check; host-loss
|
||||
restore in place with barrier fencing of the superseded database rehearsed
|
||||
locally; promotion only of stage-pinned digests. **M8-dev/stage passed;
|
||||
M8-World0 launch not passed.**
|
||||
|
||||
M8.9 implemented and locally qualified (2026-09-30), under ADR-0076: accelerated
|
||||
clone runs of the current and candidate builds compared state-for-state and
|
||||
outcome-for-outcome (`evo-world-clone compare`); failed-migration recovery and the
|
||||
version barrier rehearsed with real hosts; `upgrade-barrier.sh` for dev/stage;
|
||||
stores/compute node labels with policy-checked placement and `scheduling-check.sh`.
|
||||
Rules changes stay refused until a converter ADR. In-cluster barrier and
|
||||
additional-node evidence wait on the maintainer.
|
||||
|
||||
@@ -1141,3 +1141,21 @@ ADR-0075:
|
||||
- `fence-database.sh` supersedes the old database through the upgrade barrier: a
|
||||
still-running old host quarantines, and no host can acquire it again.
|
||||
- `tools/ci/dr-check.sh` rehearses it in every PostgreSQL build.
|
||||
|
||||
## M8.9 upgrade and growth (2026-09-30)
|
||||
|
||||
ADR-0076:
|
||||
- **Compare before upgrading.** `evo-server --synthetic-until TICK --accelerate N`
|
||||
(drill-only, refused without the synthetic driver) runs clones to an exact tick
|
||||
faster than real time; `evo-world-clone compare` (library:
|
||||
`compare_region_histories`) audits both and requires the same world: same
|
||||
versions, identical rebound head state and identical outcomes. Builds may differ.
|
||||
- **Failed migration and barrier rehearsed** with real hosts
|
||||
(`tools/ci/upgrade-check.sh`): a failing migration leaves schema v9 and the host
|
||||
serving; the barrier fences the old build (exit 4), the new one resumes under a
|
||||
newer epoch, and the history across both builds audits.
|
||||
- **Dev/stage barrier:** `tools/ops/upgrade-barrier.sh <env> --retire HEX` retires an
|
||||
old build only when it is in `REGION_RETAINED` and the new digest is already Ready.
|
||||
- **Nodes:** `aevolution.io/stores` (one worker; every objects-volume pod selects it)
|
||||
and `aevolution.io/compute` (preferred only; Alertmanager first).
|
||||
`tools/ops/scheduling-check.sh <env>` verifies placement read-only.
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
# ADR-0076: Upgrade rehearsal on clones, version barrier in dev/stage, and node growth
|
||||
|
||||
## Status
|
||||
Accepted for M8.9, 2026-09-30. It builds on:
|
||||
- ADR-0067 (clones);
|
||||
- ADR-0068 (upgrade barrier);
|
||||
- ADR-0073 (synthetic driver);
|
||||
- ADR-0074/0075 (backup, restore, World0 gate).
|
||||
|
||||
## Context
|
||||
M8.9 requires:
|
||||
- clone, migrate, accelerate and compare before any world-wide rules change;
|
||||
- a node label and affinity plan that works on one node and admits added compute
|
||||
nodes.
|
||||
|
||||
The evidence asked for is:
|
||||
- a dev/stage version-barrier rehearsal;
|
||||
- recovery from a failed migration;
|
||||
- scheduling onto an additional node where hardware exists.
|
||||
|
||||
The named risks are mixed physical laws, and assuming that added CPU makes storage
|
||||
portable.
|
||||
|
||||
**No rules or format converter exists** (ADR-0068): a region never changes rules,
|
||||
versions, quantum or profile. The only upgrade this foundation can perform is a
|
||||
**build change with identical physical laws**, such as a bug fix, performance
|
||||
work or a refactor. The question the rehearsal must answer is: does the candidate
|
||||
build run the same world?
|
||||
|
||||
## Decision
|
||||
**1. Clone, accelerate, compare.** Two drill-only `evo-server` flags, each refused
|
||||
without `--synthetic-inputs`:
|
||||
- `--synthetic-until TICK` stops the deterministic driver after that tick. Inputs
|
||||
are never invented, so the region commits exactly up to TICK and idles.
|
||||
- `--accelerate N` (2–1000) runs the scheduling clock N times faster than real
|
||||
time. Lease deadlines stay real.
|
||||
|
||||
`evo-world-clone compare --world A --other-world B ...` calls the new library
|
||||
function `compare_region_histories`:
|
||||
- It fully audits both histories.
|
||||
- It then requires equal head tick, frame, quantum, versions and history profile.
|
||||
- It requires identical head states once B is rebound to A's world.
|
||||
- It requires identical durable outcomes: tick, actor, outcome and organism,
|
||||
including those inherited at clone genesis.
|
||||
- Builds may differ. Exit 6 means "different world".
|
||||
|
||||
The procedure:
|
||||
1. Clone the live world's cut twice into an isolated database.
|
||||
2. Run the current build on one clone and the candidate on the other, with the same
|
||||
inputs, accelerated, to the same tick.
|
||||
3. `compare` must pass. Only then may the candidate proceed.
|
||||
|
||||
A rules change cannot pass this procedure. It stays refused until a converter ADR
|
||||
defines its own comparison: ecological distributions and approval
|
||||
(PERSISTENCE_ARCHITECTURE.md). Mixed physical laws are therefore impossible by
|
||||
construction: a barrier admits only a build that `compare` proved runs the same
|
||||
world.
|
||||
|
||||
**2. Migration failure.** A migration runs in one transaction under the migration
|
||||
lock, with a 5 s lock timeout and a 10 s statement timeout (ADR-0068). A failure
|
||||
leaves the previous schema exactly.
|
||||
- While the migration holds its locks, a publication may wait. Its outcome is then
|
||||
resolved (ADR-0063), or its lease lapses and the host re-acquires under a newer
|
||||
epoch.
|
||||
- In a cluster, the migration Job is a Sync hook at wave −1. When it fails, the
|
||||
sync stops before wave 1, so the Deployment keeps the old image and nothing new
|
||||
starts.
|
||||
- Recovery means fixing the migration and syncing again. No database restore is
|
||||
involved: a restore after new writes would itself be a rewind (ADR-0075).
|
||||
|
||||
**3. Barrier in dev/stage.** The ADR-0068 order has these steps:
|
||||
1. Migrate.
|
||||
2. Retain the new build. The provision hook does this.
|
||||
3. Deploy the new build with the old build listed in `REGION_RETAINED`. The
|
||||
Recreate strategy drains the old pod first.
|
||||
4. Run `tools/ops/upgrade-barrier.sh <env> --retire OLD`.
|
||||
|
||||
The barrier tool refuses the retirement unless:
|
||||
- the build is not the one Git deploys;
|
||||
- it is listed in `REGION_RETAINED`;
|
||||
- a Ready pod already runs the Git-pinned digest.
|
||||
|
||||
It then runs `upgrade_barrier` in a short Pod with the migrator login, using the
|
||||
`migrate` component's egress, and measures the not-ready gap while the new host
|
||||
re-acquires. A rollback to a retired image quarantines. That is intended.
|
||||
|
||||
**4. Nodes.**
|
||||
|
||||
| Label | Meaning | Rule (policy-checked) |
|
||||
|---|---|---|
|
||||
| `aevolution.io/stores=true` | Holds the node-local durable volumes. Exactly one worker (today rke2-worker-1). | Every pod that mounts `aevolution-objects` must select it. Region hosts, provisioning, store checks and backups stay there. |
|
||||
| `aevolution.io/compute=true` | Admitted for stateless AEvolution work. | Only ever *preferred*, never required, so everything still schedules on one node or with no labelled node. Alertmanager is the first user. |
|
||||
|
||||
Adding a node means labelling it `aevolution.io/compute=true`, then running
|
||||
`tools/ops/scheduling-check.sh <env>`. Only stateless pods move. A region moves to
|
||||
another node only as a data operation: a restore or a clone onto that node's
|
||||
storage (ADR-0074/0075). Rescheduling a region does not move it, because its
|
||||
objects volume is pinned to the stores node by the local-path PV's node affinity.
|
||||
The policy check rejects:
|
||||
- a pod that mounts the objects volume without the stores selector;
|
||||
- a required compute affinity;
|
||||
- rehearsal flags in a deployed workload.
|
||||
|
||||
## Evidence
|
||||
Every PostgreSQL build runs two tests.
|
||||
|
||||
`region_upgrade_tests` (real processes):
|
||||
- A source world is run past its death to tick 30 and cloned four times.
|
||||
- Build b1 and candidate build b2 each run to tick 150 at 20× acceleration: 120
|
||||
ticks, 6 s of region time, in under 4 s of wall time.
|
||||
- `compare` passes, with one outcome.
|
||||
- A clone fed a temperature-shifted input stream from the same cut differs in
|
||||
`state`.
|
||||
- A clone stopped at tick 100 differs in `ticks`.
|
||||
- Comparing a world with itself, or using the rehearsal flags without the driver,
|
||||
is a usage error.
|
||||
|
||||
`tools/ci/upgrade-check.sh`:
|
||||
- A migration that takes the migration lock, adds a column to `region_commits`,
|
||||
holds its locks for 3 s and then fails runs while b1 serves. Afterwards the
|
||||
schema is still v9, the column does not exist, b1 goes on acknowledging, and the
|
||||
real migrate is a no-op.
|
||||
- b2 is retained and started, and waits without becoming ready.
|
||||
- `upgrade_barrier(b1)` fences one lease. b1 quarantines (exit 4). b2 resumes at a
|
||||
newer epoch, at or past every b1 acknowledgement, and continues with the next
|
||||
tick.
|
||||
- A late b1 host quarantines at once.
|
||||
- The history, now written by both builds, passes the full audit.
|
||||
|
||||
In the cluster, still to be run by the maintainer:
|
||||
- `upgrade-barrier.sh` on dev, retiring a build listed in `REGION_RETAINED`;
|
||||
- `scheduling-check.sh` on dev and stage, before and after labelling a second
|
||||
worker `aevolution.io/compute=true`.
|
||||
|
||||
## Consequences
|
||||
- Upgrades that keep the physical laws are qualified and operable in dev and stage.
|
||||
Rules changes remain impossible until a converter ADR exists.
|
||||
- Mutation tests show the compare check is not vacuous: removing the state
|
||||
comparison, the stop tick or the acceleration each fails the tests.
|
||||
- Region hosts stay on the stores node. Horizontal growth applies only to
|
||||
stateless work until object storage is network-attached. The storage rework
|
||||
(backlog) may revisit this.
|
||||
|
||||
## Alternatives rejected
|
||||
- **Comparing live regions at arbitrary commits.** Commit boundaries depend on flush
|
||||
timing. A shared stop tick makes the comparison exact.
|
||||
- **Required node affinity for compute.** It would break the single-node plan and
|
||||
strand pods while no node carries the label.
|
||||
- **Moving a region by relabelling nodes.** The volume stays behind. That is
|
||||
exactly the named risk.
|
||||
- **Retiring the old build automatically on every sync.** Retirement is
|
||||
irreversible. It stays a deliberate step, guarded by the checks above.
|
||||
@@ -86,3 +86,4 @@ design. No unadopted candidate is a link dependency of the proof.
|
||||
- [ADR-0073: Synthetic load and fault drills](0073-fault-drills.md) - accepted for M8.6 drill-only deterministic input driver, kill/full/empty/db-outage drill with RTO and zero-lost-acknowledgement checks.
|
||||
- [ADR-0074: World backup bundles and verified restore](0074-world-backup-and-restore.md) - accepted for M8.7 snapshot-consistent online bundles (dump + immutable objects + manifest), fail-closed audited restore on independent storage, isolated rebuild through clones; destination, encryption, schedule and WAL archiving pending.
|
||||
- [ADR-0075: World0 acknowledgment gate, host-loss recovery and promotion](0075-world0-acknowledgment-gate.md) - accepted for M8.8 ratified dev/stage RPO/RTO, World0 launch blocked (policy-enforced) until acknowledgements survive host loss in an independent failure domain, explicit-rewind in-place restore, barrier-based fencing of a superseded database, promotion only of stage-pinned digests.
|
||||
- [ADR-0076: Upgrade rehearsal on clones, version barrier in dev/stage, and node growth](0076-upgrade-and-growth.md) - accepted for M8.9 accelerated clone runs compared state-for-state before a build change, failed-migration recovery, guarded in-cluster barrier, stores/compute node labels with policy-checked placement.
|
||||
|
||||
@@ -2992,3 +2992,45 @@ Fixes (this segment):
|
||||
- any in-cluster host-loss restore or fence;
|
||||
- a second failure domain, which does not exist;
|
||||
- any World0 deployment or promotion, which is blocked by design.
|
||||
|
||||
## M8.9 upgrade rehearsal and node plan - 2026-09-30
|
||||
|
||||
- `region_upgrade_tests` (real evo-server and evo-world-clone processes) passed in
|
||||
the Release, analysis and ASan/UBSan PostgreSQL builds:
|
||||
- Current build b1 and candidate b2 ran on clones of one cut, at 20×
|
||||
acceleration, from tick 30 to tick 150: 120 ticks, 6 s of region time. Wall
|
||||
times were 306/307 ms (Release), 322/323 ms (analysis) and 299/299 ms (ASan).
|
||||
- `compare` passed: `compared tick=150 outcomes=1`.
|
||||
- A temperature-shifted input stream gave `compare differs state` (exit 6).
|
||||
- A clone stopped at tick 100 gave `compare differs ticks` (exit 6).
|
||||
- Comparing a world with itself, and `--accelerate` or `--synthetic-until`
|
||||
without the driver, were usage errors (exit 2).
|
||||
- `tools/ci/upgrade-check.sh` passed in all three builds:
|
||||
- A failing migration (migration lock, `ALTER TABLE region_commits`, locks held
|
||||
3 s, then an error) left schema v9 and no new column. b1 went on
|
||||
acknowledging, and the real migrate was a no-op.
|
||||
- b2 waited while b1 held the lease. `upgrade_barrier(b1)` fenced one lease, and
|
||||
b1 exited 4.
|
||||
- b2 resumed at `epoch=2 committed=135`, at b1's last acknowledgement (135), and
|
||||
continued with tick 136. The ASan run: 137.
|
||||
- A late b1 host exited 4 without becoming ready.
|
||||
- The history audit across both builds passed: 24 commits to tick 167.
|
||||
- Mutations, each caught:
|
||||
- `compare` without the state comparison: upgrade test line 201 (the diverged
|
||||
world passed);
|
||||
- `--synthetic-until` ignored: line 165;
|
||||
- `--accelerate` as a no-op: line 184 (over 4 s).
|
||||
- The WSL `checks` set: 59/59 in debug, release, analysis and ASan. clang-tidy
|
||||
(analysis builds) and the format check were clean.
|
||||
- Deploy policy: 67 objects, 0 failures. Negatives, each rejected: an
|
||||
objects-volume pod without the stores selector; a required compute affinity;
|
||||
`--accelerate` in the region Deployment. `upgrade-barrier.sh` and
|
||||
`scheduling-check.sh` refuse World0 before kubectl.
|
||||
- Found during validation: genesis seed 0xF1 collided with a checkpoint-store test
|
||||
(fixture provisioning failed), and was replaced by the unused 0xD7. clang-tidy
|
||||
flagged duplicated option branches, which were merged.
|
||||
- Not executed:
|
||||
- `upgrade-barrier.sh` or `scheduling-check.sh` against dev or stage;
|
||||
- scheduling onto a second labelled node;
|
||||
- the Windows MSVC build (no Windows target changed; the server and PostgreSQL
|
||||
code are Linux-only).
|
||||
|
||||
@@ -232,3 +232,20 @@ rebuilt as isolated world ...` counts as restore evidence. The restore uses fres
|
||||
roles on a throwaway cluster; nothing reaches the source database. A restored copy
|
||||
keeps the source world identity and is never served; restoring in place after host
|
||||
loss is M8.8.
|
||||
|
||||
## Upgrades and nodes (M8.9, ADR-0076)
|
||||
|
||||
**Build upgrade (same rules).**
|
||||
1. Rehearse on clones first: clone the cut twice into an isolated database, run the
|
||||
current and candidate builds with `--synthetic-inputs development-v1
|
||||
--synthetic-until T --accelerate 20`, then `evo-world-clone compare` must exit 0.
|
||||
2. Pin the new server digest in `deploy/kubernetes/region/<env>` (image and
|
||||
`REGION_BUILD`), add the old digest to `REGION_RETAINED`, and sync. The migrate
|
||||
hook (wave -1) stops the sync on failure, leaving the old host running.
|
||||
3. When the new pod is Ready: `tools/ops/upgrade-barrier.sh <env> --retire OLD`.
|
||||
Keep OLD in `REGION_RETAINED`: its history stays recoverable.
|
||||
|
||||
**Adding a node.** Label it `aevolution.io/compute=true` and run
|
||||
`tools/ops/scheduling-check.sh <env>`. Only stateless pods (Alertmanager today) may
|
||||
move there. Never move `aevolution.io/stores` to rebalance: the objects volume stays
|
||||
on its node; a region moves only by restore or clone onto the new node's storage.
|
||||
|
||||
@@ -101,4 +101,34 @@ struct HistoryAudit {
|
||||
audit_region_history(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
|
||||
std::span<const Sha256Digest> retained_builds, std::size_t restore_budget,
|
||||
std::uint64_t max_commits, std::chrono::milliseconds timeout);
|
||||
|
||||
// M8.9 upgrade comparison (ADR-0076). Before a build replaces another at a barrier,
|
||||
// both run the same inputs on clones of one cut (accelerated), and their histories
|
||||
// must be the same world: equal head tick, frame, quantum, versions and profile;
|
||||
// identical head states once B is rebound to A's world; identical durable outcomes
|
||||
// (tick, actor, outcome, organism), including those inherited at clone genesis.
|
||||
// Builds may differ; nothing else may. Both histories are fully audited first.
|
||||
// Read-only. A rules or format change cannot pass (no converter exists, ADR-0068).
|
||||
enum class CompareError : std::uint8_t {
|
||||
identity, // Same key, a zero ID, or different region IDs (only worlds rebind).
|
||||
audit, // Either history failed its audit (see audit and side).
|
||||
ticks, // Heads at different ticks: nothing compared.
|
||||
versions, // Frame, quantum, versions or profile differ.
|
||||
state, // Head states differ after rebinding B to A's world.
|
||||
outcomes, // Durable outcome histories differ.
|
||||
};
|
||||
struct CompareFailure {
|
||||
CompareError kind{};
|
||||
AuditFailure audit{}; // Meaningful for audit.
|
||||
char side{}; // 'a' or 'b' for audit.
|
||||
};
|
||||
struct HistoryComparison {
|
||||
std::uint64_t tick{}, outcomes{}, commits_a{}, commits_b{};
|
||||
Sha256Digest state{}; // SHA-256 of the identical head state, in A's world.
|
||||
};
|
||||
[[nodiscard]] std::expected<HistoryComparison, CompareFailure>
|
||||
compare_region_histories(Postgres& db_a, const FilesystemObjectStore& objects_a, RegionKey a,
|
||||
Postgres& db_b, const FilesystemObjectStore& objects_b, RegionKey b,
|
||||
std::span<const Sha256Digest> retained_builds, std::size_t restore_budget,
|
||||
std::uint64_t max_commits, std::chrono::milliseconds timeout);
|
||||
} // namespace evo::persistence
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
#include "evo/persistence/region_clone.hpp"
|
||||
#include "checkpoint_detail.hpp"
|
||||
#include <algorithm>
|
||||
#include <iterator>
|
||||
#include <tuple>
|
||||
|
||||
namespace evo::persistence {
|
||||
namespace {
|
||||
@@ -266,4 +268,91 @@ audit_region_history(Postgres& db, const FilesystemObjectStore& objects, RegionK
|
||||
return audit_fail(AuditError::unsupported, head->head.sequence);
|
||||
}
|
||||
}
|
||||
|
||||
namespace {
|
||||
std::unexpected<CompareFailure> compare_fail(CompareError kind, AuditFailure audit = {},
|
||||
char side = 0) {
|
||||
return std::unexpected{CompareFailure{kind, audit, side}};
|
||||
}
|
||||
|
||||
template <typename Codec>
|
||||
std::expected<HistoryComparison, CompareFailure>
|
||||
compare(Postgres& db_a, const FilesystemObjectStore& objects_a, RegionKey a, Postgres& db_b,
|
||||
const FilesystemObjectStore& objects_b, RegionKey b, std::span<const Sha256Digest> builds,
|
||||
std::size_t budget, milliseconds timeout, const HistoryAudit& audited_a,
|
||||
const HistoryAudit& audited_b) {
|
||||
auto ra = recover<Codec>(db_a, objects_a, a, builds, budget, timeout);
|
||||
auto rb = recover<Codec>(db_b, objects_b, b, builds, budget, timeout);
|
||||
if (!ra || !rb) {
|
||||
const auto kind = !ra ? ra.error().kind : rb.error().kind;
|
||||
return compare_fail(kind == RecoveryError::unsupported ? CompareError::versions
|
||||
: CompareError::audit,
|
||||
{}, !ra ? 'a' : 'b');
|
||||
}
|
||||
const auto& ma = ra->manifest;
|
||||
const auto& mb = rb->manifest;
|
||||
// The audited heads are the compared heads: a region that moved is not compared.
|
||||
if (ma.end_tick != audited_a.end_tick || mb.end_tick != audited_b.end_tick ||
|
||||
ma.end_tick != mb.end_tick)
|
||||
return compare_fail(CompareError::ticks);
|
||||
if (ma.frame != mb.frame || ma.tick_microseconds != mb.tick_microseconds ||
|
||||
ma.versions != mb.versions || ma.history_profile != mb.history_profile ||
|
||||
ma.rng_streams != mb.rng_streams)
|
||||
return compare_fail(CompareError::versions);
|
||||
const auto state_a = Codec::snapshot(*ra->state);
|
||||
if (!state_a || !rb->state->rebind_world(a.world))
|
||||
return compare_fail(CompareError::state);
|
||||
const auto state_b = Codec::snapshot(*rb->state);
|
||||
if (!state_b || *state_a != *state_b)
|
||||
return compare_fail(CompareError::state);
|
||||
const auto decisions = [](const auto& recovered) {
|
||||
std::vector<BatchDecision> out;
|
||||
out.reserve(recovered.decisions.size());
|
||||
for (const auto& d : recovered.decisions)
|
||||
out.push_back(d.decision);
|
||||
std::ranges::sort(out, {}, [](const BatchDecision& d) {
|
||||
return std::tuple{d.tick, static_cast<unsigned>(d.actor)};
|
||||
});
|
||||
return out;
|
||||
};
|
||||
const auto outcomes = decisions(*ra);
|
||||
if (outcomes != decisions(*rb))
|
||||
return compare_fail(CompareError::outcomes);
|
||||
const auto digest = sha256(*state_a);
|
||||
if (!digest)
|
||||
return compare_fail(CompareError::state);
|
||||
return HistoryComparison{ma.end_tick, outcomes.size(), audited_a.commits, audited_b.commits,
|
||||
*digest};
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::expected<HistoryComparison, CompareFailure>
|
||||
compare_region_histories(Postgres& db_a, const FilesystemObjectStore& objects_a, RegionKey a,
|
||||
Postgres& db_b, const FilesystemObjectStore& objects_b, RegionKey b,
|
||||
std::span<const Sha256Digest> retained_builds, std::size_t restore_budget,
|
||||
std::uint64_t max_commits, milliseconds timeout) {
|
||||
if (a.world == 0 || a.region == 0 || b.world == 0 || b.region == 0 || a.region != b.region ||
|
||||
(a == b && &db_a == &db_b))
|
||||
return compare_fail(CompareError::identity);
|
||||
const auto audited_a = audit_region_history(db_a, objects_a, a, retained_builds, restore_budget,
|
||||
max_commits, timeout);
|
||||
if (!audited_a)
|
||||
return compare_fail(CompareError::audit, audited_a.error(), 'a');
|
||||
const auto audited_b = audit_region_history(db_b, objects_b, b, retained_builds, restore_budget,
|
||||
max_commits, timeout);
|
||||
if (!audited_b)
|
||||
return compare_fail(CompareError::audit, audited_b.error(), 'b');
|
||||
if (audited_a->profile != audited_b->profile)
|
||||
return compare_fail(CompareError::versions);
|
||||
switch (audited_a->profile) {
|
||||
case CheckpointProfile::environmental_v1:
|
||||
return compare<EnvironmentalCodec>(db_a, objects_a, a, db_b, objects_b, b, retained_builds,
|
||||
restore_budget, timeout, *audited_a, *audited_b);
|
||||
case CheckpointProfile::environmental_lineage_v1:
|
||||
return compare<LineageCodec>(db_a, objects_a, a, db_b, objects_b, b, retained_builds,
|
||||
restore_budget, timeout, *audited_a, *audited_b);
|
||||
default:
|
||||
return compare_fail(CompareError::versions);
|
||||
}
|
||||
}
|
||||
} // namespace evo::persistence
|
||||
|
||||
@@ -13,6 +13,7 @@ void run_region_clone_tests();
|
||||
void run_region_migration_tests();
|
||||
void run_region_store_tests();
|
||||
void run_region_load_tests();
|
||||
void run_region_upgrade_tests();
|
||||
namespace {
|
||||
using namespace evo::persistence;
|
||||
using namespace std::chrono_literals;
|
||||
@@ -129,6 +130,8 @@ int main() {
|
||||
std::puts("region store qualification integration passed");
|
||||
run_region_load_tests();
|
||||
std::puts("region synthetic load integration passed");
|
||||
run_region_upgrade_tests();
|
||||
std::puts("region upgrade clone comparison integration passed");
|
||||
run_region_migration_tests();
|
||||
std::puts("region migration, barrier and qualification integration passed");
|
||||
return 0;
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
// M8.9 upgrade rehearsal on clones (ADR-0076) with real evo-server and evo-world-clone
|
||||
// processes: one cut is cloned into the isolated clone database several times; the
|
||||
// current build (b1) and a candidate build (b2) each run the same deterministic inputs
|
||||
// on their own clone, accelerated, to the same tick; `compare` must find the same
|
||||
// world. A clone fed a different input stream, or stopped at another tick, must be
|
||||
// reported as different. Run by tools/ci/postgres-check.sh.
|
||||
#include "evo/processes/development_opening.hpp"
|
||||
#include "fixtures/server_process.hpp"
|
||||
#include <algorithm>
|
||||
#include <cstdio>
|
||||
#include <source_location>
|
||||
|
||||
void run_region_upgrade_tests();
|
||||
|
||||
namespace {
|
||||
using namespace evo::test::process;
|
||||
using evo::persistence::RegionKey;
|
||||
void check(bool value, std::source_location at = std::source_location::current()) {
|
||||
if (!value)
|
||||
throw std::runtime_error("region upgrade test line " + std::to_string(at.line()));
|
||||
}
|
||||
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
||||
std::string b1_hex() {
|
||||
std::string out;
|
||||
for (int i = 0; i < 32; ++i)
|
||||
out += "b1";
|
||||
return out;
|
||||
}
|
||||
std::string b2_hex() {
|
||||
std::string out;
|
||||
for (int i = 0; i < 32; ++i)
|
||||
out += "b2";
|
||||
return out;
|
||||
}
|
||||
constexpr std::uint64_t region = 95;
|
||||
|
||||
ServerRegion clone_root(std::uint64_t world) {
|
||||
ServerRegion r{{world, region}, fs::path(env("PGHOST")) / ("upgrade-" + std::to_string(world))};
|
||||
fs::remove_all(r.root);
|
||||
fs::create_directory(r.root);
|
||||
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
|
||||
return r;
|
||||
}
|
||||
std::vector<std::string> pg(const std::string& prefix, const std::string& database,
|
||||
const std::string& user) {
|
||||
return {prefix + "host", env("PGHOST"), prefix + "port", env("PGPORT"),
|
||||
prefix + "database", database, prefix + "user", user};
|
||||
}
|
||||
int run(const std::vector<std::string>& args, std::vector<std::string>* lines = nullptr) {
|
||||
Child tool{args};
|
||||
const auto code = tool.wait_exit(60s);
|
||||
if (lines != nullptr)
|
||||
*lines = tool.lines();
|
||||
return code;
|
||||
}
|
||||
void clone_to(const ServerRegion& source, const ServerRegion& target) {
|
||||
std::vector<std::string> args{env("EVO_WORLD_CLONE_BIN"), "clone",
|
||||
"--source-world", std::to_string(source.key.world),
|
||||
"--source-region", std::to_string(region),
|
||||
"--target-world", std::to_string(target.key.world)};
|
||||
for (auto& a : pg("--source-pg-", env("PGDATABASE"), env("EVO_PG_RUNTIME_USER")))
|
||||
args.push_back(a);
|
||||
for (auto& a : pg("--target-pg-", env("EVO_PG_CLONE_DATABASE"), env("PGUSER")))
|
||||
args.push_back(a);
|
||||
for (const auto& a :
|
||||
std::vector<std::string>{"--source-objects", source.root.string(), "--target-objects",
|
||||
target.root.string(), "--retained", b1_hex()})
|
||||
args.push_back(a);
|
||||
CHECK(run(args) == 0);
|
||||
}
|
||||
// evo-server on a clone of the clone database with the given build.
|
||||
std::vector<std::string> clone_server(const ServerRegion& r, const std::string& build,
|
||||
const std::vector<std::string>& extra) {
|
||||
std::vector<std::string> args{env("EVO_SERVER_BIN"),
|
||||
"--profile",
|
||||
"lineage",
|
||||
"--world",
|
||||
std::to_string(r.key.world),
|
||||
"--region",
|
||||
std::to_string(region),
|
||||
"--frame",
|
||||
"9"};
|
||||
for (auto& a : pg("--pg-", env("EVO_PG_CLONE_DATABASE"), env("EVO_PG_CLONE_RUNTIME_USER")))
|
||||
args.push_back(a);
|
||||
for (const auto& a :
|
||||
std::vector<std::string>{"--objects", r.root.string(), "--build", build, "--retained",
|
||||
b1_hex(), "--flush-ms", "1000", "--lease-profile", "fast"})
|
||||
args.push_back(a);
|
||||
args.insert(args.end(), extra.begin(), extra.end());
|
||||
return args;
|
||||
}
|
||||
bool acked(const Child& c, std::uint64_t tick) {
|
||||
for (const auto& line : c.lines()) {
|
||||
unsigned long long t = 0, s = 0;
|
||||
char request[40];
|
||||
if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &t, &s) == 3 && t >= tick)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
// Runs a synthetic, accelerated owner to `until` and drains; returns wall ms from
|
||||
// readiness to the acknowledgement of `until`.
|
||||
long long accelerated(const ServerRegion& r, const std::string& build, std::uint64_t until) {
|
||||
Child server{clone_server(r, build,
|
||||
{"--synthetic-inputs", "development-v1", "--synthetic-until",
|
||||
std::to_string(until), "--accelerate", "20"})};
|
||||
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
||||
const auto start = Clock::now();
|
||||
CHECK(server.wait_for([&](const std::string&) { return acked(server, until); }, 30s));
|
||||
const auto ms =
|
||||
std::chrono::duration_cast<std::chrono::milliseconds>(Clock::now() - start).count();
|
||||
CHECK(!acked(server, until + 1)); // The driver stops at `until`.
|
||||
server.signal(SIGTERM);
|
||||
CHECK(server.wait_exit(20s) == 0);
|
||||
return ms;
|
||||
}
|
||||
// The development inputs, with the temperature shifted: a different input stream.
|
||||
std::string shifted_lines(std::uint64_t first, std::uint64_t last) {
|
||||
std::string out;
|
||||
for (auto tick = first; tick <= last; ++tick) {
|
||||
const auto in = evo::processes::development_input(tick);
|
||||
std::string line = "input " + request_hex(tick, 0x5A) + " " +
|
||||
std::to_string(in.temperature_mK + 1000) + " " +
|
||||
std::to_string(in.source_enabled);
|
||||
for (const auto b : in.bite)
|
||||
line += " " + std::to_string(b);
|
||||
for (const auto t : in.terminal)
|
||||
line += " " + std::to_string(t);
|
||||
out += line + "\n";
|
||||
}
|
||||
return out;
|
||||
}
|
||||
int compare(std::uint64_t a, const ServerRegion& ra, std::uint64_t b, const ServerRegion& rb,
|
||||
std::vector<std::string>& lines) {
|
||||
std::vector<std::string> args{env("EVO_WORLD_CLONE_BIN"),
|
||||
"compare",
|
||||
"--world",
|
||||
std::to_string(a),
|
||||
"--region",
|
||||
std::to_string(region),
|
||||
"--other-world",
|
||||
std::to_string(b)};
|
||||
for (auto& x : pg("--pg-", env("EVO_PG_CLONE_DATABASE"), env("PGUSER")))
|
||||
args.push_back(x);
|
||||
for (const auto& x :
|
||||
std::vector<std::string>{"--objects", ra.root.string(), "--other-objects",
|
||||
rb.root.string(), "--retained", b1_hex(), "--retained", b2_hex()})
|
||||
args.push_back(x);
|
||||
return run(args, &lines);
|
||||
}
|
||||
bool printed(const std::vector<std::string>& lines, std::string_view prefix) {
|
||||
return std::ranges::any_of(lines, [&](const std::string& l) { return l.starts_with(prefix); });
|
||||
}
|
||||
|
||||
void candidate_build_matches_on_clones() {
|
||||
// The live source: provisioned and run past the tick-20 death, then stopped.
|
||||
const auto source = provision(region, 0xD7);
|
||||
{
|
||||
Child owner{server_args(
|
||||
source, {"--synthetic-inputs", "development-v1", "--synthetic-until", "30"})};
|
||||
CHECK(owner.wait_for([&](const std::string&) { return acked(owner, 30); }, 30s));
|
||||
owner.signal(SIGTERM);
|
||||
CHECK(owner.wait_exit(20s) == 0);
|
||||
}
|
||||
CHECK(head_of(source.key).tick == 30);
|
||||
auto clone_owner = evo::persistence::Postgres::connect(
|
||||
{env("PGHOST"), {}, env("PGPORT"), env("EVO_PG_CLONE_DATABASE"), env("PGUSER"), {}, {}},
|
||||
5s);
|
||||
CHECK(clone_owner);
|
||||
CHECK((*clone_owner)
|
||||
->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
||||
"(decode(repeat('b1', 32), 'hex')), (decode(repeat('b2', 32), 'hex')) "
|
||||
"ON CONFLICT DO NOTHING",
|
||||
{}, 5s));
|
||||
const auto reference = clone_root(951), candidate = clone_root(952), diverged = clone_root(953),
|
||||
shorter = clone_root(954);
|
||||
for (const auto* r : {&reference, &candidate, &diverged, &shorter})
|
||||
clone_to(source, *r);
|
||||
|
||||
// Current and candidate builds, the same inputs, accelerated x20 to tick 150:
|
||||
// 120 ticks are 6 s of region time.
|
||||
const auto reference_ms = accelerated(reference, b1_hex(), 150);
|
||||
const auto candidate_ms = accelerated(candidate, b2_hex(), 150);
|
||||
CHECK(reference_ms < 4000 && candidate_ms < 4000);
|
||||
std::printf("qualification upgrade_compare ticks=120 accelerate=20 reference_ms=%lld "
|
||||
"candidate_ms=%lld realtime_ms=6000\n",
|
||||
reference_ms, candidate_ms);
|
||||
std::vector<std::string> lines;
|
||||
CHECK(compare(951, reference, 952, candidate, lines) == 0);
|
||||
CHECK(printed(lines, "compared tick=150 outcomes=1 "));
|
||||
|
||||
// A different input stream from the same cut is a different world.
|
||||
{
|
||||
Child owner{clone_server(diverged, b2_hex(), {})};
|
||||
CHECK(owner.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
||||
owner.send(shifted_lines(31, 150));
|
||||
CHECK(owner.wait_for([&](const std::string&) { return acked(owner, 150); }, 30s));
|
||||
owner.signal(SIGTERM);
|
||||
CHECK(owner.wait_exit(20s) == 0);
|
||||
}
|
||||
CHECK(compare(951, reference, 953, diverged, lines) == 6);
|
||||
CHECK(printed(lines, "compare differs state"));
|
||||
// Heads at different ticks are not compared.
|
||||
static_cast<void>(accelerated(shorter, b1_hex(), 100));
|
||||
CHECK(compare(951, reference, 954, shorter, lines) == 6);
|
||||
CHECK(printed(lines, "compare differs ticks"));
|
||||
// A world is not compared with itself, and rehearsal flags need the driver.
|
||||
CHECK(compare(951, reference, 951, reference, lines) == 2);
|
||||
CHECK(run(clone_server(reference, b1_hex(), {"--accelerate", "20"})) == 2);
|
||||
CHECK(run(clone_server(reference, b1_hex(), {"--synthetic-until", "10"})) == 2);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void run_region_upgrade_tests() {
|
||||
candidate_build_matches_on_clones();
|
||||
}
|
||||
@@ -18,7 +18,10 @@ environment-isolation or World0-protection invariant is broken:
|
||||
- the cleanup tool refuses World0 before invoking kubectl;
|
||||
- World0 launch stays blocked (ADR-0075): no workload targets World0 and its
|
||||
Application keeps an unresolved path; any image a World0 overlay pins must be
|
||||
the digest already pinned in stage (promotion, never a fresh build).
|
||||
the digest already pinned in stage (promotion, never a fresh build);
|
||||
- scheduling (ADR-0076): every pod that mounts the node-local objects volume
|
||||
selects the store node; aevolution.io/compute is only ever preferred, never
|
||||
required; upgrade-rehearsal flags never appear in a deployed workload.
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
@@ -281,6 +284,28 @@ for directory, _, files in os.walk(os.path.join(ROOT, 'deploy', 'kubernetes')):
|
||||
check((image.get('name'), image.get('digest')) in staged,
|
||||
f'{rel}: {image.get("name")} must be promoted from {stage}, not pinned independently')
|
||||
|
||||
# M8.9 scheduling (ADR-0076). Node-local volumes pin their pods: added CPU does not
|
||||
# make the objects volume portable. Compute is a preference so one node still works.
|
||||
for path, doc in objects:
|
||||
if doc['kind'] not in workload_kinds or path.endswith('region-values.yaml'):
|
||||
continue
|
||||
spec = doc.get('spec', {})
|
||||
pod = spec.get('jobTemplate', {}).get('spec', {}).get('template', {}).get('spec') \
|
||||
or spec.get('template', {}).get('spec') or spec
|
||||
claims = [v.get('persistentVolumeClaim', {}).get('claimName') for v in pod.get('volumes', [])]
|
||||
if 'aevolution-objects' in claims:
|
||||
check(pod.get('nodeSelector', {}).get('aevolution.io/stores') == 'true',
|
||||
f'{path}: a pod mounting aevolution-objects must select aevolution.io/stores')
|
||||
required = (pod.get('affinity', {}).get('nodeAffinity', {})
|
||||
.get('requiredDuringSchedulingIgnoredDuringExecution', {}))
|
||||
check('aevolution.io/compute' not in yaml.safe_dump(required)
|
||||
and 'aevolution.io/compute' not in pod.get('nodeSelector', {}),
|
||||
f'{path}: aevolution.io/compute may only be preferred, never required')
|
||||
for container in pod.get('containers', []):
|
||||
text = ' '.join(container.get('args', []) + container.get('command', []))
|
||||
check('--accelerate' not in text and '--synthetic-until' not in text,
|
||||
f'{path}: upgrade-rehearsal flags never run in a deployed workload')
|
||||
|
||||
# The platform bundle: every rendered object is protected from prune and deletion.
|
||||
bundle = manifests.get(platform + '/cluster/kustomization.yaml', [{}])[0]
|
||||
check(bundle.get('commonAnnotations', {}).get('argocd.argoproj.io/sync-options') == 'Prune=false,Delete=false',
|
||||
@@ -334,7 +359,8 @@ for path, doc in objects:
|
||||
with tempfile.TemporaryDirectory() as empty:
|
||||
env = dict(os.environ, PATH=empty + os.pathsep + os.path.dirname(sys.executable))
|
||||
for tool in ('tools/ops/cleanup-environment.sh', 'tools/ops/region-drill.sh', 'tools/ops/fault-drill.sh',
|
||||
'tools/backup/cluster-backup.sh'):
|
||||
'tools/backup/cluster-backup.sh', 'tools/ops/scheduling-check.sh',
|
||||
'tools/ops/upgrade-barrier.sh'):
|
||||
for bad in ('world0', 'evolution-world0', 'World0', 'prod', ''):
|
||||
result = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, tool), bad],
|
||||
capture_output=True, text=True, env=env)
|
||||
|
||||
@@ -34,6 +34,8 @@ EVO_WORLD_CLONE_BIN="$(dirname "$1")/evo-world-clone"
|
||||
bash "$root/tools/ci/backup-check.sh"
|
||||
# M8.8: host-loss rehearsal: in-place restore, fenced old environment, measured rewind.
|
||||
bash "$root/tools/ci/dr-check.sh"
|
||||
# M8.9: failed migration under load and the version barrier with real hosts.
|
||||
bash "$root/tools/ci/upgrade-check.sh"
|
||||
v1_sha="$(sha256sum "$root/migrations/0001_region_bootstrap.sql")"
|
||||
v1_sha="${v1_sha%% *}"
|
||||
# Migration failure must not publish a partial schema.
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env bash
|
||||
# M8.9 version-barrier rehearsal (ADR-0076). Run by postgres-check.sh inside its
|
||||
# throwaway cluster (PG* set; EVO_SERVER_BIN, EVO_WORLD_CLONE_BIN, runtime roles
|
||||
# exported). Real evo-server processes on world 500:
|
||||
# 1. Failed migration: a migration that takes the migration lock, alters a history
|
||||
# table, holds its locks for 3 s and then fails runs while build b1 serves.
|
||||
# Nothing it did remains (schema still v9, no new column); the host keeps its
|
||||
# authority, and every acknowledgement stays durable.
|
||||
# 2. Barrier (ADR-0068 operator order): build b2 is retained, and a b2 host is started
|
||||
# while b1 still holds the lease (it waits, never ready). upgrade_barrier(b1) fences
|
||||
# b1, which quarantines (exit 4); b2 resumes at the committed head under a newer
|
||||
# epoch. A b1 host started afterwards quarantines at once, and the history across
|
||||
# both builds passes the full audit.
|
||||
set -euo pipefail
|
||||
root="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
work="$(mktemp -d /tmp/evo-upgrade-check.XXXXXXXX)"
|
||||
chmod 700 "$work"
|
||||
pids=()
|
||||
trap 'for p in "${pids[@]}"; do kill -9 "$p" 2> /dev/null || true; done; rm -rf -- "$work"' EXIT
|
||||
fail() { echo "upgrade rehearsal FAILED: $*" >&2; exit 1; }
|
||||
b1=b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1
|
||||
b2=b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2
|
||||
createdb evo_upgrade
|
||||
export PGDATABASE=evo_upgrade
|
||||
bash "$root/tools/migrate-postgres.sh" > /dev/null
|
||||
q() { psql -XAt -v ON_ERROR_STOP=1 "$@"; }
|
||||
mkdir -m 700 "$work/objects"
|
||||
pg=(--pg-host "$PGHOST" --pg-port "$PGPORT" --pg-database evo_upgrade)
|
||||
"$EVO_WORLD_CLONE_BIN" provision --world 500 --region 1 --opening development-v1 "${pg[@]}" \
|
||||
--pg-user "$PGUSER" --objects "$work/objects" --build "$b1" --retain-build true > /dev/null
|
||||
serve() { # serve <build> <log> [retained...]
|
||||
local build="$1" log="$2"; shift 2
|
||||
local retained=()
|
||||
for r in "$@"; do retained+=(--retained "$r"); done
|
||||
"$EVO_SERVER_BIN" --profile lineage --world 500 --region 1 --frame 9 "${pg[@]}" \
|
||||
--pg-user "$EVO_PG_RUNTIME_USER" --objects "$work/objects" --build "$build" "${retained[@]}" \
|
||||
--flush-ms 200 --lease-profile fast --synthetic-inputs development-v1 < /dev/null > "$log" 2>&1 &
|
||||
pids+=($!)
|
||||
server_pid=$!
|
||||
}
|
||||
acked_through() { awk -v t="$2" '$1 == "ack" && $3 >= t {f = 1} END {exit !f}' "$1"; }
|
||||
max_ack() { awk '$1 == "ack" && $3 > m {m = $3} END {print m + 0}' "$1"; }
|
||||
wait_ack() { for _ in $(seq 1 300); do acked_through "$1" "$2" && return 0; sleep 0.1; done; return 1; }
|
||||
head_tick() { q -c 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 500'; }
|
||||
|
||||
serve "$b1" "$work/b1.log"; b1_pid=$server_pid
|
||||
wait_ack "$work/b1.log" 30 || fail "b1 acknowledged nothing"
|
||||
|
||||
# 1. A failing migration while b1 serves.
|
||||
before="$(max_ack "$work/b1.log")"
|
||||
code=0
|
||||
q -v ON_ERROR_STOP=1 > "$work/migration.log" 2>&1 <<'SQL' || code=$?
|
||||
BEGIN;
|
||||
SET LOCAL lock_timeout = '5s';
|
||||
SELECT pg_catalog.pg_advisory_xact_lock(71420001);
|
||||
ALTER TABLE evolution.region_commits ADD COLUMN rehearsal_column integer;
|
||||
SELECT pg_catalog.pg_sleep(3);
|
||||
SELECT 1 / 0; -- The migration fails after holding its locks.
|
||||
COMMIT;
|
||||
SQL
|
||||
[ "$code" != 0 ] && grep -q 'division by zero' "$work/migration.log" || fail "the rehearsal migration did not fail"
|
||||
[ "$(q -c "SELECT count(*) FROM information_schema.columns WHERE table_name = 'region_commits' AND column_name = 'rehearsal_column'")" = 0 ] ||
|
||||
fail "the failed migration left a column"
|
||||
[ "$(q -c 'SELECT max(version) FROM evolution.schema_migrations')" = 9 ] || fail "schema version moved"
|
||||
wait_ack "$work/b1.log" $((before + 40)) || fail "b1 did not resume after the failed migration"
|
||||
kill -0 "$b1_pid" 2> /dev/null || fail "b1 exited during the failed migration"
|
||||
bash "$root/tools/migrate-postgres.sh" > /dev/null || fail "migrate after the failure"
|
||||
|
||||
# 2. The barrier: retain b2, start its host, then retire b1.
|
||||
q -c "INSERT INTO evolution.retained_builds(build_identity) VALUES (decode('$b2', 'hex'))" > /dev/null
|
||||
serve "$b2" "$work/b2.log" "$b1"; b2_pid=$server_pid
|
||||
sleep 2
|
||||
! grep -q '^ready 1' "$work/b2.log" || fail "b2 became ready while b1 held the lease"
|
||||
[ "$(q -c "SELECT evolution.upgrade_barrier(decode('$b1', 'hex'))")" = 1 ] || fail "barrier fenced no lease"
|
||||
code=0; wait "$b1_pid" || code=$?
|
||||
[ "$code" = 4 ] || { tail "$work/b1.log" >&2; fail "b1 exited $code after the barrier, not 4"; }
|
||||
b1_final="$(max_ack "$work/b1.log")"
|
||||
wait_ack "$work/b2.log" $((b1_final + 30)) || { tail "$work/b2.log" >&2; fail "b2 did not take over"; }
|
||||
resumed="$(sed -n 's/^phase running epoch=\([0-9]*\) committed=\([0-9]*\) .*/\1 \2/p' "$work/b2.log" | head -1)"
|
||||
read -r epoch committed <<< "$resumed"
|
||||
[ "$epoch" -ge 2 ] && [ "$committed" -ge "$b1_final" ] ||
|
||||
fail "b2 resumed at epoch=$epoch committed=$committed; b1 acknowledged $b1_final"
|
||||
[ "$(awk '$1 == "ack" {print $3; exit}' "$work/b2.log")" = $((committed + 1)) ] || fail "b2 skipped or repeated ticks"
|
||||
serve "$b1" "$work/late.log"; late_pid=$server_pid
|
||||
code=0; wait "$late_pid" || code=$?
|
||||
[ "$code" = 4 ] && ! grep -q '^ready 1' "$work/late.log" || fail "a late b1 host exited $code or served"
|
||||
kill -TERM "$b2_pid"; code=0; wait "$b2_pid" || code=$?
|
||||
[ "$code" = 0 ] || fail "b2 exited $code"
|
||||
audit="$("$EVO_WORLD_CLONE_BIN" audit --world 500 --region 1 "${pg[@]}" --pg-user "$PGUSER" \
|
||||
--objects "$work/objects" --retained "$b1" --retained "$b2")" || fail "audit across builds"
|
||||
[[ "$audit" == *" tick=$(head_tick) "* ]] || fail "audit: $audit"
|
||||
builds="$(q -c 'SELECT count(DISTINCT build_identity) FROM evolution.region_commits WHERE world_id = 500')"
|
||||
[ "$builds" = 2 ] || fail "history holds $builds builds, expected 2"
|
||||
echo "upgrade rehearsal passed: failed migration held locks 3 s, b1 acknowledged $b1_final," \
|
||||
"b2 resumed epoch=$epoch committed=$committed, head $(head_tick) ($audit)"
|
||||
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env bash
|
||||
# M8.9 read-only placement check (ADR-0076) for a disposable environment.
|
||||
#
|
||||
# tools/ops/scheduling-check.sh <dev|stage> [--context NAME]
|
||||
#
|
||||
# Verifies the node plan against the live cluster, changing nothing:
|
||||
# - exactly one Ready worker carries aevolution.io/stores=true, and it is not a
|
||||
# control-plane node;
|
||||
# - the environment's aevolution-objects volume is pinned to that node (local
|
||||
# volumes cannot follow a region to another node);
|
||||
# - the evo-region pod runs there;
|
||||
# - nodes labelled aevolution.io/compute=true are listed, with whether they can
|
||||
# take work (Ready, schedulable). None is valid: compute is only a preference.
|
||||
# Adding a node: label it aevolution.io/compute=true and re-run. Only stateless pods
|
||||
# move to it; a region moves between nodes only through a backup restore or a clone
|
||||
# onto that node's storage (ADR-0074/0075), never by rescheduling.
|
||||
set -euo pipefail
|
||||
usage() { echo "usage: $0 <dev|stage> [--context NAME]" >&2; exit 2; }
|
||||
[ $# -ge 1 ] || usage
|
||||
env="$1"; shift
|
||||
case "$env" in
|
||||
dev|stage) ;;
|
||||
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
|
||||
exit 2 ;;
|
||||
esac
|
||||
k=(kubectl --request-timeout=30s)
|
||||
[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2")
|
||||
ns="evolution-$env"
|
||||
failures=0
|
||||
ok() { echo " ok $*"; }
|
||||
bad() { echo " FAIL $*"; failures=$((failures + 1)); }
|
||||
ready='{range .status.conditions[?(@.type=="Ready")]}{.status}{end}'
|
||||
|
||||
echo "stores node"
|
||||
stores="$("${k[@]}" get nodes -l aevolution.io/stores=true -o name | sed 's#^node/##')"
|
||||
if [ "$(grep -c . <<< "$stores")" = 1 ]; then ok "one stores node: $stores"; else bad "stores nodes: '${stores//$'\n'/ }' (need exactly one)"; fi
|
||||
store_node="$(head -1 <<< "$stores")"
|
||||
if [ -n "$store_node" ]; then
|
||||
[ "$("${k[@]}" get node "$store_node" -o jsonpath="$ready")" = True ] && ok "$store_node Ready" || bad "$store_node not Ready"
|
||||
"${k[@]}" get node "$store_node" -o jsonpath='{.metadata.labels}' | grep -q 'node-role.kubernetes.io/control-plane' &&
|
||||
bad "$store_node is a control-plane node" || ok "$store_node is a worker"
|
||||
fi
|
||||
|
||||
echo "objects volume ($ns)"
|
||||
pv="$("${k[@]}" get pvc -n "$ns" aevolution-objects -o jsonpath='{.spec.volumeName}' 2> /dev/null || true)"
|
||||
if [ -z "$pv" ]; then
|
||||
bad "aevolution-objects is not bound"
|
||||
else
|
||||
pinned="$("${k[@]}" get pv "$pv" -o jsonpath='{.spec.nodeAffinity.required.nodeSelectorTerms[*].matchExpressions[*].values[*]}')"
|
||||
[ "$pinned" = "$store_node" ] && ok "$pv pinned to $pinned" || bad "$pv pinned to '$pinned', not the stores node '$store_node'"
|
||||
fi
|
||||
|
||||
echo "region host"
|
||||
placed="$("${k[@]}" get pods -n "$ns" -l app.kubernetes.io/name=evo-region -o jsonpath='{range .items[*]}{.metadata.name}={.spec.nodeName}{"\n"}{end}')"
|
||||
if [ -z "$placed" ]; then bad "no evo-region pod"; fi
|
||||
while IFS='=' read -r pod node; do
|
||||
[ -n "$pod" ] || continue
|
||||
[ "$node" = "$store_node" ] && ok "$pod on $node" || bad "$pod on '$node', not the stores node"
|
||||
done <<< "$placed"
|
||||
|
||||
echo "compute nodes (preferred for stateless AEvolution pods)"
|
||||
compute="$("${k[@]}" get nodes -l aevolution.io/compute=true -o name | sed 's#^node/##')"
|
||||
if [ -z "$compute" ]; then
|
||||
echo " none labelled: stateless pods schedule anywhere (single-node plan)"
|
||||
else
|
||||
while read -r node; do
|
||||
state="$("${k[@]}" get node "$node" -o jsonpath="$ready")"
|
||||
cordoned="$("${k[@]}" get node "$node" -o jsonpath='{.spec.unschedulable}')"
|
||||
echo " $node Ready=$state schedulable=$([ "$cordoned" = true ] && echo no || echo yes)"
|
||||
done <<< "$compute"
|
||||
fi
|
||||
placed_am="$("${k[@]}" get pods -n aevolution-monitoring -l app.kubernetes.io/name=alertmanager -o jsonpath='{.items[0].spec.nodeName}' 2> /dev/null || true)"
|
||||
[ -n "$placed_am" ] && echo " alertmanager runs on $placed_am"
|
||||
echo
|
||||
[ "$failures" = 0 ] && echo "PASS scheduling plan in $ns" || { echo "FAIL $failures check(s) in $ns"; exit 1; }
|
||||
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env bash
|
||||
# M8.9 version barrier in a disposable environment (ADR-0068, ADR-0076).
|
||||
#
|
||||
# tools/ops/upgrade-barrier.sh <dev|stage> --retire HEX64 [--context NAME]
|
||||
#
|
||||
# The last step of the upgrade order: migrate -> retain the new build (the provision
|
||||
# hook does) -> deploy it with the old build in REGION_RETAINED -> retire the old
|
||||
# build here. Refused unless:
|
||||
# - HEX64 is not the build Git currently deploys (deploy/kubernetes/region/<env>);
|
||||
# - HEX64 is listed in REGION_RETAINED (its history must stay recoverable);
|
||||
# - the evo-region pod is Ready and runs exactly the Git-pinned image, so the new
|
||||
# build already serves.
|
||||
# Then one short Pod (the postgres-tools image pinned for <env>, migrator login) runs
|
||||
# evolution.upgrade_barrier(HEX64): the old build can never acquire, renew or commit
|
||||
# again, and every lease is fenced. The new host re-acquires within one lease. The
|
||||
# script waits for Ready again and prints the time without a ready host.
|
||||
# Retirement is irreversible here. World0 is refused before kubectl runs.
|
||||
set -euo pipefail
|
||||
usage() { echo "usage: $0 <dev|stage> --retire HEX64 [--context NAME]" >&2; exit 2; }
|
||||
[ $# -ge 1 ] || usage
|
||||
env="$1"; shift
|
||||
case "$env" in
|
||||
dev|stage) ;;
|
||||
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
|
||||
exit 2 ;;
|
||||
esac
|
||||
retire="" k=(kubectl --request-timeout=30s)
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--retire) retire="${2:-}"; shift 2 ;;
|
||||
--context) k+=(--context "${2:?}"); shift 2 ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
done
|
||||
[[ "$retire" =~ ^[0-9a-f]{64}$ ]] || usage
|
||||
root="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
ns="evolution-$env"
|
||||
values="$root/deploy/kubernetes/region/$env/region-values.yaml"
|
||||
current="$(sed -n 's/.*name: REGION_BUILD, value: \([0-9a-f]\{64\}\)}.*/\1/p' "$values" | sort -u)"
|
||||
retained="$(sed -n 's/.*name: REGION_RETAINED, value: "\([0-9a-f ]*\)"}.*/\1/p' "$values" | head -1)"
|
||||
[ "$(wc -l <<< "$current")" = 1 ] || { echo "cannot read one REGION_BUILD from $values" >&2; exit 2; }
|
||||
[ "$retire" != "$current" ] || { echo "refused: $retire is the build Git deploys" >&2; exit 2; }
|
||||
grep -qw "$retire" <<< "$retained" || { echo "refused: $retire is not in REGION_RETAINED" >&2; exit 2; }
|
||||
image="$(awk '/name: aevolution-postgres-tools/ {f = 1} f && /newName:/ {n = $2} f && /digest:/ {print n "@" $2; exit}' \
|
||||
"$root/deploy/kubernetes/stores/$env/kustomization.yaml")"
|
||||
[[ "$image" =~ @sha256:[0-9a-f]{64}$ ]] || { echo "no pinned postgres-tools image for $env" >&2; exit 2; }
|
||||
say() { echo "$(date -u +%H:%M:%S) $*"; }
|
||||
ready_pod() {
|
||||
"${k[@]}" get pods -n "$ns" -l app.kubernetes.io/name=evo-region -o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{range .status.conditions}}{{if and (eq .type "Ready") (eq .status "True")}}{{$.metadata.name}}{{end}}{{end}}{{end}}{{end}}' 2> /dev/null || true
|
||||
}
|
||||
serving() { # The Ready pod's running image digest.
|
||||
local pod; pod="$(ready_pod)"
|
||||
[ -n "$pod" ] && "${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.containerStatuses[0].imageID}'
|
||||
}
|
||||
[[ "$(serving)" == *"@sha256:$current" ]] ||
|
||||
{ echo "refused: no Ready evo-region pod runs sha256:$current yet (deploy the new build first)" >&2; exit 3; }
|
||||
|
||||
pod="aevolution-barrier-$(date -u +%Y%m%d%H%M%S)"
|
||||
cleanup() { "${k[@]}" delete pod -n "$ns" "$pod" --wait=false > /dev/null 2>&1 || true; }
|
||||
trap cleanup EXIT
|
||||
say "retiring $retire in $ns (serving sha256:$current)"
|
||||
"${k[@]}" apply -n "$ns" -f - > /dev/null <<YAML
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: $pod
|
||||
labels:
|
||||
app.kubernetes.io/part-of: aevolution
|
||||
app.kubernetes.io/component: migrate
|
||||
spec:
|
||||
serviceAccountName: aevolution-migrator
|
||||
automountServiceAccountToken: false
|
||||
restartPolicy: Never
|
||||
activeDeadlineSeconds: 120
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
containers:
|
||||
- name: barrier
|
||||
image: $image
|
||||
command: ["/bin/bash", "-c"]
|
||||
args:
|
||||
- |
|
||||
set -eu
|
||||
umask 077
|
||||
cp /run/postgres/pgpass /tmp/pgpass
|
||||
export PGPASSFILE=/tmp/pgpass
|
||||
psql -X --no-password -At -v ON_ERROR_STOP=1 \
|
||||
-c "SELECT 'fenced ' || evolution.upgrade_barrier(decode('$retire', 'hex'))" \
|
||||
-c "SELECT 'retired ' || encode(build_identity, 'hex') FROM evolution.retained_builds WHERE retired_at IS NOT NULL ORDER BY 1"
|
||||
env:
|
||||
- {name: PGSERVICE, value: evolution}
|
||||
- {name: PGSERVICEFILE, value: /run/postgres/pg_service.conf}
|
||||
resources:
|
||||
requests: {cpu: 50m, memory: 64Mi}
|
||||
limits: {memory: 128Mi}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities: {drop: [ALL]}
|
||||
volumeMounts:
|
||||
- {name: connection, mountPath: /run/postgres, readOnly: true}
|
||||
- {name: scratch, mountPath: /tmp}
|
||||
volumes:
|
||||
- name: connection
|
||||
secret:
|
||||
secretName: evolution-postgres-migrator
|
||||
defaultMode: 0440
|
||||
items:
|
||||
- {key: pg_service.conf, path: pg_service.conf}
|
||||
- {key: pgpass, path: pgpass}
|
||||
- {key: ca.crt, path: ca.crt}
|
||||
- name: scratch
|
||||
emptyDir: {medium: Memory, sizeLimit: 1Mi}
|
||||
YAML
|
||||
start="$(date +%s%3N)"
|
||||
for _ in $(seq 1 120); do
|
||||
phase="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}' 2> /dev/null || true)"
|
||||
[ "$phase" = Succeeded ] || [ "$phase" = Failed ] && break
|
||||
sleep 1
|
||||
done
|
||||
"${k[@]}" logs -n "$ns" "$pod" 2>&1 | sed 's/^/ /'
|
||||
[ "$phase" = Succeeded ] || { say "FAIL barrier pod $phase"; exit 3; }
|
||||
not_ready_since=""
|
||||
for _ in $(seq 1 600); do
|
||||
if [ -n "$(ready_pod)" ]; then
|
||||
[ -z "$not_ready_since" ] && [ $(( $(date +%s%3N) - start )) -lt 3000 ] && { sleep 0.5; continue; }
|
||||
break
|
||||
fi
|
||||
[ -n "$not_ready_since" ] || not_ready_since="$(date +%s%3N)"
|
||||
sleep 0.5
|
||||
done
|
||||
[[ "$(serving)" == *"@sha256:$current" ]] || { say "FAIL no Ready pod serves sha256:$current after the barrier"; exit 3; }
|
||||
gap=0; [ -n "$not_ready_since" ] && gap=$(( $(date +%s%3N) - not_ready_since ))
|
||||
say "PASS $retire retired in $ns; not-ready gap ${gap} ms; sha256:$current serving"
|
||||
Reference in New Issue
Block a user