feat(ops): M8.9 upgrade rehearsal on clones, barrier and node plan (ADR-0076)

Drill-only --synthetic-until/--accelerate and evo-world-clone compare
(compare_region_histories) prove a candidate build runs the same world on
clones of one cut before any barrier. upgrade-check.sh rehearses a failed
migration under load and the version barrier with real hosts; upgrade-barrier.sh
retires an old build in dev/stage only once the new digest is Ready. Nodes:
aevolution.io/stores pins every objects-volume pod, aevolution.io/compute is
preference-only (Alertmanager first), checked by policy and scheduling-check.sh.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-30 17:51:17 -04:00
co-authored by Claude Opus 5.5
parent 645d16b27a
commit 5b482ad6a3
19 changed files with 1028 additions and 12 deletions
+1 -1
View File
@@ -128,7 +128,7 @@ if(EVO_BUILD_POSTGRES)
tests/region_server_process_tests.cpp tests/region_recovery_tests.cpp
tests/region_ops_tests.cpp tests/region_clone_tests.cpp
tests/region_migration_tests.cpp tests/region_store_tests.cpp
tests/region_load_tests.cpp)
tests/region_load_tests.cpp tests/region_upgrade_tests.cpp)
target_link_libraries(evo-postgres-tests PRIVATE evo::region_host Threads::Threads)
add_dependencies(evo-postgres-tests evo-server evo-world-clone) # Spawned by process tests.
evo_owned_target(evo-postgres-tests)
+26 -3
View File
@@ -28,6 +28,12 @@
// restarting from the committed tick after every recovery. Request IDs are
// "SYN1" + region + tick, so acknowledgements can be checked across owners.
//
// Upgrade rehearsal on clones only (M8.9, ADR-0076), both requiring
// --synthetic-inputs: --synthetic-until <tick> stops the driver after that tick, so
// the region commits exactly up to it and then idles (inputs are never invented);
// --accelerate <2..1000> runs the region's scheduling clock that many times faster
// than real time (lease deadlines stay real). Never in a deployed environment.
//
// --crash-at <point>:<n> (tests only, M7.7) SIGKILLs this process at the n-th
// occurrence of before-upload, before-transaction, after-commit or after-ack (right
// after a step's acknowledgements are written and flushed).
@@ -69,6 +75,8 @@ struct Options {
std::uint64_t crash_ordinal{};
std::string ops_listen;
bool synthetic{};
std::uint64_t synthetic_until{}; // 0: unbounded.
std::uint64_t accelerate{1};
std::uint64_t objects_reserve_mib{1024};
};
@@ -174,6 +182,10 @@ std::optional<Options> parse(int argc, char** argv) {
o.ops_listen = value;
else if (flag == "--synthetic-inputs" && value == "development-v1")
o.synthetic = true;
else if (flag == "--synthetic-until" && n && *n > 0)
o.synthetic_until = *n;
else if (flag == "--accelerate" && n && *n >= 2 && *n <= 1000)
o.accelerate = *n;
else if (flag == "--objects-reserve-mib" && n && *n <= 1048576)
o.objects_reserve_mib = *n;
else if (flag == "--crash-at") {
@@ -196,6 +208,9 @@ std::optional<Options> parse(int argc, char** argv) {
o.pg.port.empty() || o.pg.database.empty() || o.pg.user.empty() || o.objects.empty() ||
o.host.build == persistence::Sha256Digest{})
return std::nullopt;
// Rehearsal controls exist only for the synthetic driver (clones, drills).
if ((o.synthetic_until != 0 || o.accelerate != 1) && !o.synthetic)
return std::nullopt;
if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end())
o.host.retained_builds.push_back(o.host.build);
return o;
@@ -343,6 +358,12 @@ template <typename State> int serve(Options& o) {
}
return {404, "not found\n"};
};
// Scheduling time; --accelerate scales it from process start (clones only).
const auto started = std::chrono::steady_clock::now();
const auto scheduling_now = [&] {
const auto real = std::chrono::steady_clock::now();
return started + (real - started) * static_cast<std::int64_t>(o.accelerate);
};
LineReader reader;
bool input_open = !o.synthetic, ready = false;
// Synthetic driver state: the next tick to submit, reset at every recovery.
@@ -411,7 +432,8 @@ template <typename State> int serve(Options& o) {
synthetic_next = now_status.simulated_tick + 1; // Resume after the head.
synthetic_running = running;
while (running && synthetic_next &&
*synthetic_next <= host.status().simulated_tick + 64) {
*synthetic_next <= host.status().simulated_tick + 64 &&
(o.synthetic_until == 0 || *synthetic_next <= o.synthetic_until)) {
const server::TickInput in{synthetic_request(*synthetic_next),
processes::development_input(*synthetic_next)};
if (!host.submit(in))
@@ -419,7 +441,7 @@ template <typename State> int serve(Options& o) {
++*synthetic_next;
}
}
const auto& s = host.step(std::chrono::steady_clock::now());
const auto& s = host.step(scheduling_now());
max_queued = std::max(max_queued, s.queued);
// Checkpoint age restarts at recovery and at every confirmed commit.
if ((!recovered && s.phase == server::HostPhase::running) ||
@@ -492,7 +514,8 @@ int main(int argc, char** argv) {
"--objects DIR --build HEX64 [--retained HEX64]... [--pg-hostaddr A "
"--pg-passfile F --pg-ca F] [--flush-ms N] [--drain-ms N] "
"[--lease-profile default|fast] [--ready-file PATH] "
"[--ops-listen ADDRESS:PORT]\n",
"[--ops-listen ADDRESS:PORT] [--synthetic-inputs development-v1 "
"[--synthetic-until TICK] [--accelerate N]]\n",
stderr);
return 2;
}
+79 -6
View File
@@ -11,6 +11,8 @@
// [--reserve-mib N]
// evo-world-clone provision --world W --region R --opening development-v1 --pg-* ...
// --objects DIR --build HEX [--retain-build]
// evo-world-clone compare --world A --region R --other-world B --pg-* ...
// --objects DIR [--other-objects DIR] --retained HEX... [--max-commits N]
//
// --*pg-* is host, hostaddr, port, database, user, passfile, ca, or service (a libpq
// service file whose [evolution] section supplies everything but the password file). The source
@@ -40,8 +42,14 @@
// stage; World0's opening is a separate, reviewed decision.
// Output: provisioned|already-provisioned world=W region=R sequence=S tick=T
//
// compare (M8.9, ADR-0076) fully audits both regions of one database, then requires
// the same world: equal head tick, frame, quantum and versions, identical head state
// once B is rebound to A's world, and identical durable outcomes. Builds may differ.
// Output: compared tick=T outcomes=N commits_a=N commits_b=N state=HEX
// compare differs ticks|versions|state|outcomes (exit 6)
//
// Exit: 0 done, 2 usage, 3 source/history/database failure, 4 target/objects refused,
// 5 target unknown.
// 5 target unknown, 6 compared histories differ.
#include "evo/persistence/region_clone.hpp"
#include "evo/persistence/sha256.hpp"
#include "evo/processes/development_opening.hpp"
@@ -150,7 +158,7 @@ std::optional<Options> parse(int argc, char** argv) {
Options o;
o.command = argv[1];
if (o.command != "clone" && o.command != "audit" && o.command != "check-store" &&
o.command != "provision")
o.command != "provision" && o.command != "compare")
return std::nullopt;
const bool clone = o.command == "clone";
for (int i = 2; i < argc; ++i) {
@@ -174,13 +182,16 @@ std::optional<Options> parse(int argc, char** argv) {
o.source.key.world = *n;
else if (flag == (clone ? "--source-region" : "--region") && n)
o.source.key.region = *n;
else if (clone && flag == "--target-world" && n)
else if (((clone && flag == "--target-world") ||
(o.command == "compare" && flag == "--other-world")) &&
n)
o.target.key.world = *n;
else if (clone && flag == "--target-region" && n)
o.target.key.region = *n;
else if (flag == (clone ? "--source-objects" : "--objects"))
o.source.objects = value;
else if (clone && flag == "--target-objects")
else if ((clone && flag == "--target-objects") ||
(o.command == "compare" && flag == "--other-objects"))
o.target.objects = value;
else if (clone && flag == "--commit" && hex<16>(value))
o.commit = hex<16>(value);
@@ -188,7 +199,8 @@ std::optional<Options> parse(int argc, char** argv) {
o.audit_max = *n;
else if (clone && flag == "--crash-at" && value == "before-transaction")
o.crash = true;
else if (o.command == "audit" && flag == "--max-commits" && n && *n > 0)
else if ((o.command == "audit" || o.command == "compare") && flag == "--max-commits" && n &&
*n > 0)
o.max_commits = *n;
else if (o.command == "check-store" && flag == "--reserve-mib" && n && *n <= 1048576)
o.reserve_mib = *n;
@@ -203,6 +215,14 @@ std::optional<Options> parse(int argc, char** argv) {
}
if (clone && o.target.key.region == 0)
o.target.key.region = o.source.key.region;
if (o.command == "compare") {
if (o.target.key.world == 0)
return std::nullopt;
o.target.key.region = o.source.key.region;
o.target.pg = o.source.pg;
if (o.target.objects.empty())
o.target.objects = o.source.objects;
}
const bool needs_builds = o.command != "check-store" && o.command != "provision";
if (o.command == "provision" && (o.opening.empty() || !o.build))
return std::nullopt;
@@ -444,6 +464,56 @@ int audit(const Options& o) {
return 0;
}
const char* name(persistence::CompareError e) {
using enum persistence::CompareError;
switch (e) {
case identity:
return "identity";
case audit:
return "audit";
case ticks:
return "ticks";
case versions:
return "versions";
case state:
return "state";
case outcomes:
return "outcomes";
}
return "unknown";
}
int compare(const Options& o) {
auto db = connect(o.source);
auto objects_a = open_objects(o.source);
auto objects_b = open_objects(o.target);
if (!db || !objects_a || !objects_b) {
std::fputs("compare failed store\n", stderr);
return 3;
}
auto result = persistence::compare_region_histories(*db, *objects_a, o.source.key, *db,
*objects_b, o.target.key, o.retained,
restore_budget, o.max_commits, timeout);
if (!result) {
const auto& f = result.error();
using persistence::CompareError;
if (f.kind == CompareError::identity || f.kind == CompareError::audit) {
std::fprintf(stderr, "compare failed %s side=%c audit=%s sequence=%llu\n", name(f.kind),
f.side == 0 ? '-' : f.side, name(f.audit.kind),
static_cast<unsigned long long>(f.audit.sequence));
return f.kind == CompareError::identity ? 2 : 3;
}
std::printf("compare differs %s\n", name(f.kind));
return 6;
}
std::printf("compared tick=%llu outcomes=%llu commits_a=%llu commits_b=%llu state=%s\n",
static_cast<unsigned long long>(result->tick),
static_cast<unsigned long long>(result->outcomes),
static_cast<unsigned long long>(result->commits_a),
static_cast<unsigned long long>(result->commits_b), text(result->state).c_str());
return 0;
}
int clone(const Options& o) {
const auto& s = o.source.pg;
const auto& t = o.target.pg;
@@ -511,7 +581,8 @@ int clone(const Options& o) {
int main(int argc, char** argv) {
const auto options = parse(argc, argv);
if (!options) {
std::fputs("usage: evo-world-clone clone|audit ... (see apps/evo-world-clone/main.cpp)\n",
std::fputs("usage: evo-world-clone clone|audit|check-store|provision|compare ... "
"(see apps/evo-world-clone/main.cpp)\n",
stderr);
return 2;
}
@@ -519,5 +590,7 @@ int main(int argc, char** argv) {
return check_store(*options);
if (options->command == "provision")
return provision(*options);
if (options->command == "compare")
return compare(*options);
return options->command == "clone" ? clone(*options) : audit(*options);
}
@@ -22,6 +22,15 @@ spec:
spec:
serviceAccountName: aevolution-alertmanager
automountServiceAccountToken: false
# Stateless: prefers nodes labelled for AEvolution compute (ADR-0076), and
# schedules anywhere while none is labelled (the single-node plan).
affinity:
nodeAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- {key: aevolution.io/compute, operator: In, values: ["true"]}
securityContext:
runAsNonRoot: true
runAsUser: 65534
+8
View File
@@ -829,3 +829,11 @@ durable failure domain exists, enforced by the deploy policy check; host-loss
restore in place with barrier fencing of the superseded database rehearsed
locally; promotion only of stage-pinned digests. **M8-dev/stage passed;
M8-World0 launch not passed.**
M8.9 implemented and locally qualified (2026-09-30), under ADR-0076: accelerated
clone runs of the current and candidate builds compared state-for-state and
outcome-for-outcome (`evo-world-clone compare`); failed-migration recovery and the
version barrier rehearsed with real hosts; `upgrade-barrier.sh` for dev/stage;
stores/compute node labels with policy-checked placement and `scheduling-check.sh`.
Rules changes stay refused until a converter ADR. In-cluster barrier and
additional-node evidence wait on the maintainer.
+18
View File
@@ -1141,3 +1141,21 @@ ADR-0075:
- `fence-database.sh` supersedes the old database through the upgrade barrier: a
still-running old host quarantines, and no host can acquire it again.
- `tools/ci/dr-check.sh` rehearses it in every PostgreSQL build.
## M8.9 upgrade and growth (2026-09-30)
ADR-0076:
- **Compare before upgrading.** `evo-server --synthetic-until TICK --accelerate N`
(drill-only, refused without the synthetic driver) runs clones to an exact tick
faster than real time; `evo-world-clone compare` (library:
`compare_region_histories`) audits both and requires the same world: same
versions, identical rebound head state and identical outcomes. Builds may differ.
- **Failed migration and barrier rehearsed** with real hosts
(`tools/ci/upgrade-check.sh`): a failing migration leaves schema v9 and the host
serving; the barrier fences the old build (exit 4), the new one resumes under a
newer epoch, and the history across both builds audits.
- **Dev/stage barrier:** `tools/ops/upgrade-barrier.sh <env> --retire HEX` retires an
old build only when it is in `REGION_RETAINED` and the new digest is already Ready.
- **Nodes:** `aevolution.io/stores` (one worker; every objects-volume pod selects it)
and `aevolution.io/compute` (preferred only; Alertmanager first).
`tools/ops/scheduling-check.sh <env>` verifies placement read-only.
+152
View File
@@ -0,0 +1,152 @@
# ADR-0076: Upgrade rehearsal on clones, version barrier in dev/stage, and node growth
## Status
Accepted for M8.9, 2026-09-30. It builds on:
- ADR-0067 (clones);
- ADR-0068 (upgrade barrier);
- ADR-0073 (synthetic driver);
- ADR-0074/0075 (backup, restore, World0 gate).
## Context
M8.9 requires:
- clone, migrate, accelerate and compare before any world-wide rules change;
- a node label and affinity plan that works on one node and admits added compute
nodes.
The evidence asked for is:
- a dev/stage version-barrier rehearsal;
- recovery from a failed migration;
- scheduling onto an additional node where hardware exists.
The named risks are mixed physical laws, and assuming that added CPU makes storage
portable.
**No rules or format converter exists** (ADR-0068): a region never changes rules,
versions, quantum or profile. The only upgrade this foundation can perform is a
**build change with identical physical laws**, such as a bug fix, performance
work or a refactor. The question the rehearsal must answer is: does the candidate
build run the same world?
## Decision
**1. Clone, accelerate, compare.** Two drill-only `evo-server` flags, each refused
without `--synthetic-inputs`:
- `--synthetic-until TICK` stops the deterministic driver after that tick. Inputs
are never invented, so the region commits exactly up to TICK and idles.
- `--accelerate N` (2–1000) runs the scheduling clock N times faster than real
time. Lease deadlines stay real.
`evo-world-clone compare --world A --other-world B ...` calls the new library
function `compare_region_histories`:
- It fully audits both histories.
- It then requires equal head tick, frame, quantum, versions and history profile.
- It requires identical head states once B is rebound to A's world.
- It requires identical durable outcomes: tick, actor, outcome and organism,
including those inherited at clone genesis.
- Builds may differ. Exit 6 means "different world".
The procedure:
1. Clone the live world's cut twice into an isolated database.
2. Run the current build on one clone and the candidate on the other, with the same
inputs, accelerated, to the same tick.
3. `compare` must pass. Only then may the candidate proceed.
A rules change cannot pass this procedure. It stays refused until a converter ADR
defines its own comparison: ecological distributions and approval
(PERSISTENCE_ARCHITECTURE.md). Mixed physical laws are therefore impossible by
construction: a barrier admits only a build that `compare` proved runs the same
world.
**2. Migration failure.** A migration runs in one transaction under the migration
lock, with a 5 s lock timeout and a 10 s statement timeout (ADR-0068). A failure
leaves the previous schema exactly.
- While the migration holds its locks, a publication may wait. Its outcome is then
resolved (ADR-0063), or its lease lapses and the host re-acquires under a newer
epoch.
- In a cluster, the migration Job is a Sync hook at wave −1. When it fails, the
sync stops before wave 1, so the Deployment keeps the old image and nothing new
starts.
- Recovery means fixing the migration and syncing again. No database restore is
involved: a restore after new writes would itself be a rewind (ADR-0075).
**3. Barrier in dev/stage.** The ADR-0068 order has these steps:
1. Migrate.
2. Retain the new build. The provision hook does this.
3. Deploy the new build with the old build listed in `REGION_RETAINED`. The
Recreate strategy drains the old pod first.
4. Run `tools/ops/upgrade-barrier.sh <env> --retire OLD`.
The barrier tool refuses the retirement unless:
- the build is not the one Git deploys;
- it is listed in `REGION_RETAINED`;
- a Ready pod already runs the Git-pinned digest.
It then runs `upgrade_barrier` in a short Pod with the migrator login, using the
`migrate` component's egress, and measures the not-ready gap while the new host
re-acquires. A rollback to a retired image quarantines. That is intended.
**4. Nodes.**
| Label | Meaning | Rule (policy-checked) |
|---|---|---|
| `aevolution.io/stores=true` | Holds the node-local durable volumes. Exactly one worker (today rke2-worker-1). | Every pod that mounts `aevolution-objects` must select it. Region hosts, provisioning, store checks and backups stay there. |
| `aevolution.io/compute=true` | Admitted for stateless AEvolution work. | Only ever *preferred*, never required, so everything still schedules on one node or with no labelled node. Alertmanager is the first user. |
Adding a node means labelling it `aevolution.io/compute=true`, then running
`tools/ops/scheduling-check.sh <env>`. Only stateless pods move. A region moves to
another node only as a data operation: a restore or a clone onto that node's
storage (ADR-0074/0075). Rescheduling a region does not move it, because its
objects volume is pinned to the stores node by the local-path PV's node affinity.
The policy check rejects:
- a pod that mounts the objects volume without the stores selector;
- a required compute affinity;
- rehearsal flags in a deployed workload.
## Evidence
Every PostgreSQL build runs two tests.
`region_upgrade_tests` (real processes):
- A source world is run past its death to tick 30 and cloned four times.
- Build b1 and candidate build b2 each run to tick 150 at 20× acceleration: 120
ticks, 6 s of region time, in under 4 s of wall time.
- `compare` passes, with one outcome.
- A clone fed a temperature-shifted input stream from the same cut differs in
`state`.
- A clone stopped at tick 100 differs in `ticks`.
- Comparing a world with itself, or using the rehearsal flags without the driver,
is a usage error.
`tools/ci/upgrade-check.sh`:
- A migration that takes the migration lock, adds a column to `region_commits`,
holds its locks for 3 s and then fails runs while b1 serves. Afterwards the
schema is still v9, the column does not exist, b1 goes on acknowledging, and the
real migrate is a no-op.
- b2 is retained and started, and waits without becoming ready.
- `upgrade_barrier(b1)` fences one lease. b1 quarantines (exit 4). b2 resumes at a
newer epoch, at or past every b1 acknowledgement, and continues with the next
tick.
- A late b1 host quarantines at once.
- The history, now written by both builds, passes the full audit.
In the cluster, still to be run by the maintainer:
- `upgrade-barrier.sh` on dev, retiring a build listed in `REGION_RETAINED`;
- `scheduling-check.sh` on dev and stage, before and after labelling a second
worker `aevolution.io/compute=true`.
## Consequences
- Upgrades that keep the physical laws are qualified and operable in dev and stage.
Rules changes remain impossible until a converter ADR exists.
- Mutation tests show the compare check is not vacuous: removing the state
comparison, the stop tick or the acceleration each fails the tests.
- Region hosts stay on the stores node. Horizontal growth applies only to
stateless work until object storage is network-attached. The storage rework
(backlog) may revisit this.
## Alternatives rejected
- **Comparing live regions at arbitrary commits.** Commit boundaries depend on flush
timing. A shared stop tick makes the comparison exact.
- **Required node affinity for compute.** It would break the single-node plan and
strand pods while no node carries the label.
- **Moving a region by relabelling nodes.** The volume stays behind. That is
exactly the named risk.
- **Retiring the old build automatically on every sync.** Retirement is
irreversible. It stays a deliberate step, guarded by the checks above.
+1
View File
@@ -86,3 +86,4 @@ design. No unadopted candidate is a link dependency of the proof.
- [ADR-0073: Synthetic load and fault drills](0073-fault-drills.md) - accepted for M8.6 drill-only deterministic input driver, kill/full/empty/db-outage drill with RTO and zero-lost-acknowledgement checks.
- [ADR-0074: World backup bundles and verified restore](0074-world-backup-and-restore.md) - accepted for M8.7 snapshot-consistent online bundles (dump + immutable objects + manifest), fail-closed audited restore on independent storage, isolated rebuild through clones; destination, encryption, schedule and WAL archiving pending.
- [ADR-0075: World0 acknowledgment gate, host-loss recovery and promotion](0075-world0-acknowledgment-gate.md) - accepted for M8.8 ratified dev/stage RPO/RTO, World0 launch blocked (policy-enforced) until acknowledgements survive host loss in an independent failure domain, explicit-rewind in-place restore, barrier-based fencing of a superseded database, promotion only of stage-pinned digests.
- [ADR-0076: Upgrade rehearsal on clones, version barrier in dev/stage, and node growth](0076-upgrade-and-growth.md) - accepted for M8.9 accelerated clone runs compared state-for-state before a build change, failed-migration recovery, guarded in-cluster barrier, stores/compute node labels with policy-checked placement.
+42
View File
@@ -2992,3 +2992,45 @@ Fixes (this segment):
- any in-cluster host-loss restore or fence;
- a second failure domain, which does not exist;
- any World0 deployment or promotion, which is blocked by design.
## M8.9 upgrade rehearsal and node plan - 2026-09-30
- `region_upgrade_tests` (real evo-server and evo-world-clone processes) passed in
the Release, analysis and ASan/UBSan PostgreSQL builds:
- Current build b1 and candidate b2 ran on clones of one cut, at 20×
acceleration, from tick 30 to tick 150: 120 ticks, 6 s of region time. Wall
times were 306/307 ms (Release), 322/323 ms (analysis) and 299/299 ms (ASan).
- `compare` passed: `compared tick=150 outcomes=1`.
- A temperature-shifted input stream gave `compare differs state` (exit 6).
- A clone stopped at tick 100 gave `compare differs ticks` (exit 6).
- Comparing a world with itself, and `--accelerate` or `--synthetic-until`
without the driver, were usage errors (exit 2).
- `tools/ci/upgrade-check.sh` passed in all three builds:
- A failing migration (migration lock, `ALTER TABLE region_commits`, locks held
3 s, then an error) left schema v9 and no new column. b1 went on
acknowledging, and the real migrate was a no-op.
- b2 waited while b1 held the lease. `upgrade_barrier(b1)` fenced one lease, and
b1 exited 4.
- b2 resumed at `epoch=2 committed=135`, at b1's last acknowledgement (135), and
continued with tick 136. The ASan run: 137.
- A late b1 host exited 4 without becoming ready.
- The history audit across both builds passed: 24 commits to tick 167.
- Mutations, each caught:
- `compare` without the state comparison: upgrade test line 201 (the diverged
world passed);
- `--synthetic-until` ignored: line 165;
- `--accelerate` as a no-op: line 184 (over 4 s).
- The WSL `checks` set: 59/59 in debug, release, analysis and ASan. clang-tidy
(analysis builds) and the format check were clean.
- Deploy policy: 67 objects, 0 failures. Negatives, each rejected: an
objects-volume pod without the stores selector; a required compute affinity;
`--accelerate` in the region Deployment. `upgrade-barrier.sh` and
`scheduling-check.sh` refuse World0 before kubectl.
- Found during validation: genesis seed 0xF1 collided with a checkpoint-store test
(fixture provisioning failed), and was replaced by the unused 0xD7. clang-tidy
flagged duplicated option branches, which were merged.
- Not executed:
- `upgrade-barrier.sh` or `scheduling-check.sh` against dev or stage;
- scheduling onto a second labelled node;
- the Windows MSVC build (no Windows target changed; the server and PostgreSQL
code are Linux-only).
+17
View File
@@ -232,3 +232,20 @@ rebuilt as isolated world ...` counts as restore evidence. The restore uses fres
roles on a throwaway cluster; nothing reaches the source database. A restored copy
keeps the source world identity and is never served; restoring in place after host
loss is M8.8.
## Upgrades and nodes (M8.9, ADR-0076)
**Build upgrade (same rules).**
1. Rehearse on clones first: clone the cut twice into an isolated database, run the
current and candidate builds with `--synthetic-inputs development-v1
--synthetic-until T --accelerate 20`, then `evo-world-clone compare` must exit 0.
2. Pin the new server digest in `deploy/kubernetes/region/<env>` (image and
`REGION_BUILD`), add the old digest to `REGION_RETAINED`, and sync. The migrate
hook (wave -1) stops the sync on failure, leaving the old host running.
3. When the new pod is Ready: `tools/ops/upgrade-barrier.sh <env> --retire OLD`.
Keep OLD in `REGION_RETAINED`: its history stays recoverable.
**Adding a node.** Label it `aevolution.io/compute=true` and run
`tools/ops/scheduling-check.sh <env>`. Only stateless pods (Alertmanager today) may
move there. Never move `aevolution.io/stores` to rebalance: the objects volume stays
on its node; a region moves only by restore or clone onto the new node's storage.
+30
View File
@@ -101,4 +101,34 @@ struct HistoryAudit {
audit_region_history(Postgres& db, const FilesystemObjectStore& objects, RegionKey key,
std::span<const Sha256Digest> retained_builds, std::size_t restore_budget,
std::uint64_t max_commits, std::chrono::milliseconds timeout);
// M8.9 upgrade comparison (ADR-0076). Before a build replaces another at a barrier,
// both run the same inputs on clones of one cut (accelerated), and their histories
// must be the same world: equal head tick, frame, quantum, versions and profile;
// identical head states once B is rebound to A's world; identical durable outcomes
// (tick, actor, outcome, organism), including those inherited at clone genesis.
// Builds may differ; nothing else may. Both histories are fully audited first.
// Read-only. A rules or format change cannot pass (no converter exists, ADR-0068).
enum class CompareError : std::uint8_t {
identity, // Same key, a zero ID, or different region IDs (only worlds rebind).
audit, // Either history failed its audit (see audit and side).
ticks, // Heads at different ticks: nothing compared.
versions, // Frame, quantum, versions or profile differ.
state, // Head states differ after rebinding B to A's world.
outcomes, // Durable outcome histories differ.
};
struct CompareFailure {
CompareError kind{};
AuditFailure audit{}; // Meaningful for audit.
char side{}; // 'a' or 'b' for audit.
};
struct HistoryComparison {
std::uint64_t tick{}, outcomes{}, commits_a{}, commits_b{};
Sha256Digest state{}; // SHA-256 of the identical head state, in A's world.
};
[[nodiscard]] std::expected<HistoryComparison, CompareFailure>
compare_region_histories(Postgres& db_a, const FilesystemObjectStore& objects_a, RegionKey a,
Postgres& db_b, const FilesystemObjectStore& objects_b, RegionKey b,
std::span<const Sha256Digest> retained_builds, std::size_t restore_budget,
std::uint64_t max_commits, std::chrono::milliseconds timeout);
} // namespace evo::persistence
+89
View File
@@ -1,6 +1,8 @@
#include "evo/persistence/region_clone.hpp"
#include "checkpoint_detail.hpp"
#include <algorithm>
#include <iterator>
#include <tuple>
namespace evo::persistence {
namespace {
@@ -266,4 +268,91 @@ audit_region_history(Postgres& db, const FilesystemObjectStore& objects, RegionK
return audit_fail(AuditError::unsupported, head->head.sequence);
}
}
namespace {
std::unexpected<CompareFailure> compare_fail(CompareError kind, AuditFailure audit = {},
char side = 0) {
return std::unexpected{CompareFailure{kind, audit, side}};
}
template <typename Codec>
std::expected<HistoryComparison, CompareFailure>
compare(Postgres& db_a, const FilesystemObjectStore& objects_a, RegionKey a, Postgres& db_b,
const FilesystemObjectStore& objects_b, RegionKey b, std::span<const Sha256Digest> builds,
std::size_t budget, milliseconds timeout, const HistoryAudit& audited_a,
const HistoryAudit& audited_b) {
auto ra = recover<Codec>(db_a, objects_a, a, builds, budget, timeout);
auto rb = recover<Codec>(db_b, objects_b, b, builds, budget, timeout);
if (!ra || !rb) {
const auto kind = !ra ? ra.error().kind : rb.error().kind;
return compare_fail(kind == RecoveryError::unsupported ? CompareError::versions
: CompareError::audit,
{}, !ra ? 'a' : 'b');
}
const auto& ma = ra->manifest;
const auto& mb = rb->manifest;
// The audited heads are the compared heads: a region that moved is not compared.
if (ma.end_tick != audited_a.end_tick || mb.end_tick != audited_b.end_tick ||
ma.end_tick != mb.end_tick)
return compare_fail(CompareError::ticks);
if (ma.frame != mb.frame || ma.tick_microseconds != mb.tick_microseconds ||
ma.versions != mb.versions || ma.history_profile != mb.history_profile ||
ma.rng_streams != mb.rng_streams)
return compare_fail(CompareError::versions);
const auto state_a = Codec::snapshot(*ra->state);
if (!state_a || !rb->state->rebind_world(a.world))
return compare_fail(CompareError::state);
const auto state_b = Codec::snapshot(*rb->state);
if (!state_b || *state_a != *state_b)
return compare_fail(CompareError::state);
const auto decisions = [](const auto& recovered) {
std::vector<BatchDecision> out;
out.reserve(recovered.decisions.size());
for (const auto& d : recovered.decisions)
out.push_back(d.decision);
std::ranges::sort(out, {}, [](const BatchDecision& d) {
return std::tuple{d.tick, static_cast<unsigned>(d.actor)};
});
return out;
};
const auto outcomes = decisions(*ra);
if (outcomes != decisions(*rb))
return compare_fail(CompareError::outcomes);
const auto digest = sha256(*state_a);
if (!digest)
return compare_fail(CompareError::state);
return HistoryComparison{ma.end_tick, outcomes.size(), audited_a.commits, audited_b.commits,
*digest};
}
} // namespace
std::expected<HistoryComparison, CompareFailure>
compare_region_histories(Postgres& db_a, const FilesystemObjectStore& objects_a, RegionKey a,
Postgres& db_b, const FilesystemObjectStore& objects_b, RegionKey b,
std::span<const Sha256Digest> retained_builds, std::size_t restore_budget,
std::uint64_t max_commits, milliseconds timeout) {
if (a.world == 0 || a.region == 0 || b.world == 0 || b.region == 0 || a.region != b.region ||
(a == b && &db_a == &db_b))
return compare_fail(CompareError::identity);
const auto audited_a = audit_region_history(db_a, objects_a, a, retained_builds, restore_budget,
max_commits, timeout);
if (!audited_a)
return compare_fail(CompareError::audit, audited_a.error(), 'a');
const auto audited_b = audit_region_history(db_b, objects_b, b, retained_builds, restore_budget,
max_commits, timeout);
if (!audited_b)
return compare_fail(CompareError::audit, audited_b.error(), 'b');
if (audited_a->profile != audited_b->profile)
return compare_fail(CompareError::versions);
switch (audited_a->profile) {
case CheckpointProfile::environmental_v1:
return compare<EnvironmentalCodec>(db_a, objects_a, a, db_b, objects_b, b, retained_builds,
restore_budget, timeout, *audited_a, *audited_b);
case CheckpointProfile::environmental_lineage_v1:
return compare<LineageCodec>(db_a, objects_a, a, db_b, objects_b, b, retained_builds,
restore_budget, timeout, *audited_a, *audited_b);
default:
return compare_fail(CompareError::versions);
}
}
} // namespace evo::persistence
+3
View File
@@ -13,6 +13,7 @@ void run_region_clone_tests();
void run_region_migration_tests();
void run_region_store_tests();
void run_region_load_tests();
void run_region_upgrade_tests();
namespace {
using namespace evo::persistence;
using namespace std::chrono_literals;
@@ -129,6 +130,8 @@ int main() {
std::puts("region store qualification integration passed");
run_region_load_tests();
std::puts("region synthetic load integration passed");
run_region_upgrade_tests();
std::puts("region upgrade clone comparison integration passed");
run_region_migration_tests();
std::puts("region migration, barrier and qualification integration passed");
return 0;
+216
View File
@@ -0,0 +1,216 @@
// M8.9 upgrade rehearsal on clones (ADR-0076) with real evo-server and evo-world-clone
// processes: one cut is cloned into the isolated clone database several times; the
// current build (b1) and a candidate build (b2) each run the same deterministic inputs
// on their own clone, accelerated, to the same tick; `compare` must find the same
// world. A clone fed a different input stream, or stopped at another tick, must be
// reported as different. Run by tools/ci/postgres-check.sh.
#include "evo/processes/development_opening.hpp"
#include "fixtures/server_process.hpp"
#include <algorithm>
#include <cstdio>
#include <source_location>
void run_region_upgrade_tests();
namespace {
using namespace evo::test::process;
using evo::persistence::RegionKey;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region upgrade test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
std::string b1_hex() {
std::string out;
for (int i = 0; i < 32; ++i)
out += "b1";
return out;
}
std::string b2_hex() {
std::string out;
for (int i = 0; i < 32; ++i)
out += "b2";
return out;
}
constexpr std::uint64_t region = 95;
ServerRegion clone_root(std::uint64_t world) {
ServerRegion r{{world, region}, fs::path(env("PGHOST")) / ("upgrade-" + std::to_string(world))};
fs::remove_all(r.root);
fs::create_directory(r.root);
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
return r;
}
std::vector<std::string> pg(const std::string& prefix, const std::string& database,
const std::string& user) {
return {prefix + "host", env("PGHOST"), prefix + "port", env("PGPORT"),
prefix + "database", database, prefix + "user", user};
}
int run(const std::vector<std::string>& args, std::vector<std::string>* lines = nullptr) {
Child tool{args};
const auto code = tool.wait_exit(60s);
if (lines != nullptr)
*lines = tool.lines();
return code;
}
void clone_to(const ServerRegion& source, const ServerRegion& target) {
std::vector<std::string> args{env("EVO_WORLD_CLONE_BIN"), "clone",
"--source-world", std::to_string(source.key.world),
"--source-region", std::to_string(region),
"--target-world", std::to_string(target.key.world)};
for (auto& a : pg("--source-pg-", env("PGDATABASE"), env("EVO_PG_RUNTIME_USER")))
args.push_back(a);
for (auto& a : pg("--target-pg-", env("EVO_PG_CLONE_DATABASE"), env("PGUSER")))
args.push_back(a);
for (const auto& a :
std::vector<std::string>{"--source-objects", source.root.string(), "--target-objects",
target.root.string(), "--retained", b1_hex()})
args.push_back(a);
CHECK(run(args) == 0);
}
// evo-server on a clone of the clone database with the given build.
std::vector<std::string> clone_server(const ServerRegion& r, const std::string& build,
const std::vector<std::string>& extra) {
std::vector<std::string> args{env("EVO_SERVER_BIN"),
"--profile",
"lineage",
"--world",
std::to_string(r.key.world),
"--region",
std::to_string(region),
"--frame",
"9"};
for (auto& a : pg("--pg-", env("EVO_PG_CLONE_DATABASE"), env("EVO_PG_CLONE_RUNTIME_USER")))
args.push_back(a);
for (const auto& a :
std::vector<std::string>{"--objects", r.root.string(), "--build", build, "--retained",
b1_hex(), "--flush-ms", "1000", "--lease-profile", "fast"})
args.push_back(a);
args.insert(args.end(), extra.begin(), extra.end());
return args;
}
bool acked(const Child& c, std::uint64_t tick) {
for (const auto& line : c.lines()) {
unsigned long long t = 0, s = 0;
char request[40];
if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &t, &s) == 3 && t >= tick)
return true;
}
return false;
}
// Runs a synthetic, accelerated owner to `until` and drains; returns wall ms from
// readiness to the acknowledgement of `until`.
long long accelerated(const ServerRegion& r, const std::string& build, std::uint64_t until) {
Child server{clone_server(r, build,
{"--synthetic-inputs", "development-v1", "--synthetic-until",
std::to_string(until), "--accelerate", "20"})};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
const auto start = Clock::now();
CHECK(server.wait_for([&](const std::string&) { return acked(server, until); }, 30s));
const auto ms =
std::chrono::duration_cast<std::chrono::milliseconds>(Clock::now() - start).count();
CHECK(!acked(server, until + 1)); // The driver stops at `until`.
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 0);
return ms;
}
// The development inputs, with the temperature shifted: a different input stream.
std::string shifted_lines(std::uint64_t first, std::uint64_t last) {
std::string out;
for (auto tick = first; tick <= last; ++tick) {
const auto in = evo::processes::development_input(tick);
std::string line = "input " + request_hex(tick, 0x5A) + " " +
std::to_string(in.temperature_mK + 1000) + " " +
std::to_string(in.source_enabled);
for (const auto b : in.bite)
line += " " + std::to_string(b);
for (const auto t : in.terminal)
line += " " + std::to_string(t);
out += line + "\n";
}
return out;
}
int compare(std::uint64_t a, const ServerRegion& ra, std::uint64_t b, const ServerRegion& rb,
std::vector<std::string>& lines) {
std::vector<std::string> args{env("EVO_WORLD_CLONE_BIN"),
"compare",
"--world",
std::to_string(a),
"--region",
std::to_string(region),
"--other-world",
std::to_string(b)};
for (auto& x : pg("--pg-", env("EVO_PG_CLONE_DATABASE"), env("PGUSER")))
args.push_back(x);
for (const auto& x :
std::vector<std::string>{"--objects", ra.root.string(), "--other-objects",
rb.root.string(), "--retained", b1_hex(), "--retained", b2_hex()})
args.push_back(x);
return run(args, &lines);
}
bool printed(const std::vector<std::string>& lines, std::string_view prefix) {
return std::ranges::any_of(lines, [&](const std::string& l) { return l.starts_with(prefix); });
}
void candidate_build_matches_on_clones() {
// The live source: provisioned and run past the tick-20 death, then stopped.
const auto source = provision(region, 0xD7);
{
Child owner{server_args(
source, {"--synthetic-inputs", "development-v1", "--synthetic-until", "30"})};
CHECK(owner.wait_for([&](const std::string&) { return acked(owner, 30); }, 30s));
owner.signal(SIGTERM);
CHECK(owner.wait_exit(20s) == 0);
}
CHECK(head_of(source.key).tick == 30);
auto clone_owner = evo::persistence::Postgres::connect(
{env("PGHOST"), {}, env("PGPORT"), env("EVO_PG_CLONE_DATABASE"), env("PGUSER"), {}, {}},
5s);
CHECK(clone_owner);
CHECK((*clone_owner)
->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('b1', 32), 'hex')), (decode(repeat('b2', 32), 'hex')) "
"ON CONFLICT DO NOTHING",
{}, 5s));
const auto reference = clone_root(951), candidate = clone_root(952), diverged = clone_root(953),
shorter = clone_root(954);
for (const auto* r : {&reference, &candidate, &diverged, &shorter})
clone_to(source, *r);
// Current and candidate builds, the same inputs, accelerated x20 to tick 150:
// 120 ticks are 6 s of region time.
const auto reference_ms = accelerated(reference, b1_hex(), 150);
const auto candidate_ms = accelerated(candidate, b2_hex(), 150);
CHECK(reference_ms < 4000 && candidate_ms < 4000);
std::printf("qualification upgrade_compare ticks=120 accelerate=20 reference_ms=%lld "
"candidate_ms=%lld realtime_ms=6000\n",
reference_ms, candidate_ms);
std::vector<std::string> lines;
CHECK(compare(951, reference, 952, candidate, lines) == 0);
CHECK(printed(lines, "compared tick=150 outcomes=1 "));
// A different input stream from the same cut is a different world.
{
Child owner{clone_server(diverged, b2_hex(), {})};
CHECK(owner.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
owner.send(shifted_lines(31, 150));
CHECK(owner.wait_for([&](const std::string&) { return acked(owner, 150); }, 30s));
owner.signal(SIGTERM);
CHECK(owner.wait_exit(20s) == 0);
}
CHECK(compare(951, reference, 953, diverged, lines) == 6);
CHECK(printed(lines, "compare differs state"));
// Heads at different ticks are not compared.
static_cast<void>(accelerated(shorter, b1_hex(), 100));
CHECK(compare(951, reference, 954, shorter, lines) == 6);
CHECK(printed(lines, "compare differs ticks"));
// A world is not compared with itself, and rehearsal flags need the driver.
CHECK(compare(951, reference, 951, reference, lines) == 2);
CHECK(run(clone_server(reference, b1_hex(), {"--accelerate", "20"})) == 2);
CHECK(run(clone_server(reference, b1_hex(), {"--synthetic-until", "10"})) == 2);
}
} // namespace
void run_region_upgrade_tests() {
candidate_build_matches_on_clones();
}
+28 -2
View File
@@ -18,7 +18,10 @@ environment-isolation or World0-protection invariant is broken:
- the cleanup tool refuses World0 before invoking kubectl;
- World0 launch stays blocked (ADR-0075): no workload targets World0 and its
Application keeps an unresolved path; any image a World0 overlay pins must be
the digest already pinned in stage (promotion, never a fresh build).
the digest already pinned in stage (promotion, never a fresh build);
- scheduling (ADR-0076): every pod that mounts the node-local objects volume
selects the store node; aevolution.io/compute is only ever preferred, never
required; upgrade-rehearsal flags never appear in a deployed workload.
"""
import os
import shutil
@@ -281,6 +284,28 @@ for directory, _, files in os.walk(os.path.join(ROOT, 'deploy', 'kubernetes')):
check((image.get('name'), image.get('digest')) in staged,
f'{rel}: {image.get("name")} must be promoted from {stage}, not pinned independently')
# M8.9 scheduling (ADR-0076). Node-local volumes pin their pods: added CPU does not
# make the objects volume portable. Compute is a preference so one node still works.
for path, doc in objects:
if doc['kind'] not in workload_kinds or path.endswith('region-values.yaml'):
continue
spec = doc.get('spec', {})
pod = spec.get('jobTemplate', {}).get('spec', {}).get('template', {}).get('spec') \
or spec.get('template', {}).get('spec') or spec
claims = [v.get('persistentVolumeClaim', {}).get('claimName') for v in pod.get('volumes', [])]
if 'aevolution-objects' in claims:
check(pod.get('nodeSelector', {}).get('aevolution.io/stores') == 'true',
f'{path}: a pod mounting aevolution-objects must select aevolution.io/stores')
required = (pod.get('affinity', {}).get('nodeAffinity', {})
.get('requiredDuringSchedulingIgnoredDuringExecution', {}))
check('aevolution.io/compute' not in yaml.safe_dump(required)
and 'aevolution.io/compute' not in pod.get('nodeSelector', {}),
f'{path}: aevolution.io/compute may only be preferred, never required')
for container in pod.get('containers', []):
text = ' '.join(container.get('args', []) + container.get('command', []))
check('--accelerate' not in text and '--synthetic-until' not in text,
f'{path}: upgrade-rehearsal flags never run in a deployed workload')
# The platform bundle: every rendered object is protected from prune and deletion.
bundle = manifests.get(platform + '/cluster/kustomization.yaml', [{}])[0]
check(bundle.get('commonAnnotations', {}).get('argocd.argoproj.io/sync-options') == 'Prune=false,Delete=false',
@@ -334,7 +359,8 @@ for path, doc in objects:
with tempfile.TemporaryDirectory() as empty:
env = dict(os.environ, PATH=empty + os.pathsep + os.path.dirname(sys.executable))
for tool in ('tools/ops/cleanup-environment.sh', 'tools/ops/region-drill.sh', 'tools/ops/fault-drill.sh',
'tools/backup/cluster-backup.sh'):
'tools/backup/cluster-backup.sh', 'tools/ops/scheduling-check.sh',
'tools/ops/upgrade-barrier.sh'):
for bad in ('world0', 'evolution-world0', 'World0', 'prod', ''):
result = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, tool), bad],
capture_output=True, text=True, env=env)
+2
View File
@@ -34,6 +34,8 @@ EVO_WORLD_CLONE_BIN="$(dirname "$1")/evo-world-clone"
bash "$root/tools/ci/backup-check.sh"
# M8.8: host-loss rehearsal: in-place restore, fenced old environment, measured rewind.
bash "$root/tools/ci/dr-check.sh"
# M8.9: failed migration under load and the version barrier with real hosts.
bash "$root/tools/ci/upgrade-check.sh"
v1_sha="$(sha256sum "$root/migrations/0001_region_bootstrap.sql")"
v1_sha="${v1_sha%% *}"
# Migration failure must not publish a partial schema.
+95
View File
@@ -0,0 +1,95 @@
#!/usr/bin/env bash
# M8.9 version-barrier rehearsal (ADR-0076). Run by postgres-check.sh inside its
# throwaway cluster (PG* set; EVO_SERVER_BIN, EVO_WORLD_CLONE_BIN, runtime roles
# exported). Real evo-server processes on world 500:
# 1. Failed migration: a migration that takes the migration lock, alters a history
# table, holds its locks for 3 s and then fails runs while build b1 serves.
# Nothing it did remains (schema still v9, no new column); the host keeps its
# authority, and every acknowledgement stays durable.
# 2. Barrier (ADR-0068 operator order): build b2 is retained, and a b2 host is started
# while b1 still holds the lease (it waits, never ready). upgrade_barrier(b1) fences
# b1, which quarantines (exit 4); b2 resumes at the committed head under a newer
# epoch. A b1 host started afterwards quarantines at once, and the history across
# both builds passes the full audit.
set -euo pipefail
root="$(cd "$(dirname "$0")/../.." && pwd)"
work="$(mktemp -d /tmp/evo-upgrade-check.XXXXXXXX)"
chmod 700 "$work"
pids=()
trap 'for p in "${pids[@]}"; do kill -9 "$p" 2> /dev/null || true; done; rm -rf -- "$work"' EXIT
fail() { echo "upgrade rehearsal FAILED: $*" >&2; exit 1; }
b1=b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1
b2=b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2
createdb evo_upgrade
export PGDATABASE=evo_upgrade
bash "$root/tools/migrate-postgres.sh" > /dev/null
q() { psql -XAt -v ON_ERROR_STOP=1 "$@"; }
mkdir -m 700 "$work/objects"
pg=(--pg-host "$PGHOST" --pg-port "$PGPORT" --pg-database evo_upgrade)
"$EVO_WORLD_CLONE_BIN" provision --world 500 --region 1 --opening development-v1 "${pg[@]}" \
--pg-user "$PGUSER" --objects "$work/objects" --build "$b1" --retain-build true > /dev/null
serve() { # serve <build> <log> [retained...]
local build="$1" log="$2"; shift 2
local retained=()
for r in "$@"; do retained+=(--retained "$r"); done
"$EVO_SERVER_BIN" --profile lineage --world 500 --region 1 --frame 9 "${pg[@]}" \
--pg-user "$EVO_PG_RUNTIME_USER" --objects "$work/objects" --build "$build" "${retained[@]}" \
--flush-ms 200 --lease-profile fast --synthetic-inputs development-v1 < /dev/null > "$log" 2>&1 &
pids+=($!)
server_pid=$!
}
acked_through() { awk -v t="$2" '$1 == "ack" && $3 >= t {f = 1} END {exit !f}' "$1"; }
max_ack() { awk '$1 == "ack" && $3 > m {m = $3} END {print m + 0}' "$1"; }
wait_ack() { for _ in $(seq 1 300); do acked_through "$1" "$2" && return 0; sleep 0.1; done; return 1; }
head_tick() { q -c 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 500'; }
serve "$b1" "$work/b1.log"; b1_pid=$server_pid
wait_ack "$work/b1.log" 30 || fail "b1 acknowledged nothing"
# 1. A failing migration while b1 serves.
before="$(max_ack "$work/b1.log")"
code=0
q -v ON_ERROR_STOP=1 > "$work/migration.log" 2>&1 <<'SQL' || code=$?
BEGIN;
SET LOCAL lock_timeout = '5s';
SELECT pg_catalog.pg_advisory_xact_lock(71420001);
ALTER TABLE evolution.region_commits ADD COLUMN rehearsal_column integer;
SELECT pg_catalog.pg_sleep(3);
SELECT 1 / 0; -- The migration fails after holding its locks.
COMMIT;
SQL
[ "$code" != 0 ] && grep -q 'division by zero' "$work/migration.log" || fail "the rehearsal migration did not fail"
[ "$(q -c "SELECT count(*) FROM information_schema.columns WHERE table_name = 'region_commits' AND column_name = 'rehearsal_column'")" = 0 ] ||
fail "the failed migration left a column"
[ "$(q -c 'SELECT max(version) FROM evolution.schema_migrations')" = 9 ] || fail "schema version moved"
wait_ack "$work/b1.log" $((before + 40)) || fail "b1 did not resume after the failed migration"
kill -0 "$b1_pid" 2> /dev/null || fail "b1 exited during the failed migration"
bash "$root/tools/migrate-postgres.sh" > /dev/null || fail "migrate after the failure"
# 2. The barrier: retain b2, start its host, then retire b1.
q -c "INSERT INTO evolution.retained_builds(build_identity) VALUES (decode('$b2', 'hex'))" > /dev/null
serve "$b2" "$work/b2.log" "$b1"; b2_pid=$server_pid
sleep 2
! grep -q '^ready 1' "$work/b2.log" || fail "b2 became ready while b1 held the lease"
[ "$(q -c "SELECT evolution.upgrade_barrier(decode('$b1', 'hex'))")" = 1 ] || fail "barrier fenced no lease"
code=0; wait "$b1_pid" || code=$?
[ "$code" = 4 ] || { tail "$work/b1.log" >&2; fail "b1 exited $code after the barrier, not 4"; }
b1_final="$(max_ack "$work/b1.log")"
wait_ack "$work/b2.log" $((b1_final + 30)) || { tail "$work/b2.log" >&2; fail "b2 did not take over"; }
resumed="$(sed -n 's/^phase running epoch=\([0-9]*\) committed=\([0-9]*\) .*/\1 \2/p' "$work/b2.log" | head -1)"
read -r epoch committed <<< "$resumed"
[ "$epoch" -ge 2 ] && [ "$committed" -ge "$b1_final" ] ||
fail "b2 resumed at epoch=$epoch committed=$committed; b1 acknowledged $b1_final"
[ "$(awk '$1 == "ack" {print $3; exit}' "$work/b2.log")" = $((committed + 1)) ] || fail "b2 skipped or repeated ticks"
serve "$b1" "$work/late.log"; late_pid=$server_pid
code=0; wait "$late_pid" || code=$?
[ "$code" = 4 ] && ! grep -q '^ready 1' "$work/late.log" || fail "a late b1 host exited $code or served"
kill -TERM "$b2_pid"; code=0; wait "$b2_pid" || code=$?
[ "$code" = 0 ] || fail "b2 exited $code"
audit="$("$EVO_WORLD_CLONE_BIN" audit --world 500 --region 1 "${pg[@]}" --pg-user "$PGUSER" \
--objects "$work/objects" --retained "$b1" --retained "$b2")" || fail "audit across builds"
[[ "$audit" == *" tick=$(head_tick) "* ]] || fail "audit: $audit"
builds="$(q -c 'SELECT count(DISTINCT build_identity) FROM evolution.region_commits WHERE world_id = 500')"
[ "$builds" = 2 ] || fail "history holds $builds builds, expected 2"
echo "upgrade rehearsal passed: failed migration held locks 3 s, b1 acknowledged $b1_final," \
"b2 resumed epoch=$epoch committed=$committed, head $(head_tick) ($audit)"
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env bash
# M8.9 read-only placement check (ADR-0076) for a disposable environment.
#
# tools/ops/scheduling-check.sh <dev|stage> [--context NAME]
#
# Verifies the node plan against the live cluster, changing nothing:
# - exactly one Ready worker carries aevolution.io/stores=true, and it is not a
# control-plane node;
# - the environment's aevolution-objects volume is pinned to that node (local
# volumes cannot follow a region to another node);
# - the evo-region pod runs there;
# - nodes labelled aevolution.io/compute=true are listed, with whether they can
# take work (Ready, schedulable). None is valid: compute is only a preference.
# Adding a node: label it aevolution.io/compute=true and re-run. Only stateless pods
# move to it; a region moves between nodes only through a backup restore or a clone
# onto that node's storage (ADR-0074/0075), never by rescheduling.
set -euo pipefail
usage() { echo "usage: $0 <dev|stage> [--context NAME]" >&2; exit 2; }
[ $# -ge 1 ] || usage
env="$1"; shift
case "$env" in
dev|stage) ;;
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
exit 2 ;;
esac
k=(kubectl --request-timeout=30s)
[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2")
ns="evolution-$env"
failures=0
ok() { echo " ok $*"; }
bad() { echo " FAIL $*"; failures=$((failures + 1)); }
ready='{range .status.conditions[?(@.type=="Ready")]}{.status}{end}'
echo "stores node"
stores="$("${k[@]}" get nodes -l aevolution.io/stores=true -o name | sed 's#^node/##')"
if [ "$(grep -c . <<< "$stores")" = 1 ]; then ok "one stores node: $stores"; else bad "stores nodes: '${stores//$'\n'/ }' (need exactly one)"; fi
store_node="$(head -1 <<< "$stores")"
if [ -n "$store_node" ]; then
[ "$("${k[@]}" get node "$store_node" -o jsonpath="$ready")" = True ] && ok "$store_node Ready" || bad "$store_node not Ready"
"${k[@]}" get node "$store_node" -o jsonpath='{.metadata.labels}' | grep -q 'node-role.kubernetes.io/control-plane' &&
bad "$store_node is a control-plane node" || ok "$store_node is a worker"
fi
echo "objects volume ($ns)"
pv="$("${k[@]}" get pvc -n "$ns" aevolution-objects -o jsonpath='{.spec.volumeName}' 2> /dev/null || true)"
if [ -z "$pv" ]; then
bad "aevolution-objects is not bound"
else
pinned="$("${k[@]}" get pv "$pv" -o jsonpath='{.spec.nodeAffinity.required.nodeSelectorTerms[*].matchExpressions[*].values[*]}')"
[ "$pinned" = "$store_node" ] && ok "$pv pinned to $pinned" || bad "$pv pinned to '$pinned', not the stores node '$store_node'"
fi
echo "region host"
placed="$("${k[@]}" get pods -n "$ns" -l app.kubernetes.io/name=evo-region -o jsonpath='{range .items[*]}{.metadata.name}={.spec.nodeName}{"\n"}{end}')"
if [ -z "$placed" ]; then bad "no evo-region pod"; fi
while IFS='=' read -r pod node; do
[ -n "$pod" ] || continue
[ "$node" = "$store_node" ] && ok "$pod on $node" || bad "$pod on '$node', not the stores node"
done <<< "$placed"
echo "compute nodes (preferred for stateless AEvolution pods)"
compute="$("${k[@]}" get nodes -l aevolution.io/compute=true -o name | sed 's#^node/##')"
if [ -z "$compute" ]; then
echo " none labelled: stateless pods schedule anywhere (single-node plan)"
else
while read -r node; do
state="$("${k[@]}" get node "$node" -o jsonpath="$ready")"
cordoned="$("${k[@]}" get node "$node" -o jsonpath='{.spec.unschedulable}')"
echo " $node Ready=$state schedulable=$([ "$cordoned" = true ] && echo no || echo yes)"
done <<< "$compute"
fi
placed_am="$("${k[@]}" get pods -n aevolution-monitoring -l app.kubernetes.io/name=alertmanager -o jsonpath='{.items[0].spec.nodeName}' 2> /dev/null || true)"
[ -n "$placed_am" ] && echo " alertmanager runs on $placed_am"
echo
[ "$failures" = 0 ] && echo "PASS scheduling plan in $ns" || { echo "FAIL $failures check(s) in $ns"; exit 1; }
+137
View File
@@ -0,0 +1,137 @@
#!/usr/bin/env bash
# M8.9 version barrier in a disposable environment (ADR-0068, ADR-0076).
#
# tools/ops/upgrade-barrier.sh <dev|stage> --retire HEX64 [--context NAME]
#
# The last step of the upgrade order: migrate -> retain the new build (the provision
# hook does) -> deploy it with the old build in REGION_RETAINED -> retire the old
# build here. Refused unless:
# - HEX64 is not the build Git currently deploys (deploy/kubernetes/region/<env>);
# - HEX64 is listed in REGION_RETAINED (its history must stay recoverable);
# - the evo-region pod is Ready and runs exactly the Git-pinned image, so the new
# build already serves.
# Then one short Pod (the postgres-tools image pinned for <env>, migrator login) runs
# evolution.upgrade_barrier(HEX64): the old build can never acquire, renew or commit
# again, and every lease is fenced. The new host re-acquires within one lease. The
# script waits for Ready again and prints the time without a ready host.
# Retirement is irreversible here. World0 is refused before kubectl runs.
set -euo pipefail
usage() { echo "usage: $0 <dev|stage> --retire HEX64 [--context NAME]" >&2; exit 2; }
[ $# -ge 1 ] || usage
env="$1"; shift
case "$env" in
dev|stage) ;;
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
exit 2 ;;
esac
retire="" k=(kubectl --request-timeout=30s)
while [ $# -gt 0 ]; do
case "$1" in
--retire) retire="${2:-}"; shift 2 ;;
--context) k+=(--context "${2:?}"); shift 2 ;;
*) usage ;;
esac
done
[[ "$retire" =~ ^[0-9a-f]{64}$ ]] || usage
root="$(cd "$(dirname "$0")/../.." && pwd)"
ns="evolution-$env"
values="$root/deploy/kubernetes/region/$env/region-values.yaml"
current="$(sed -n 's/.*name: REGION_BUILD, value: \([0-9a-f]\{64\}\)}.*/\1/p' "$values" | sort -u)"
retained="$(sed -n 's/.*name: REGION_RETAINED, value: "\([0-9a-f ]*\)"}.*/\1/p' "$values" | head -1)"
[ "$(wc -l <<< "$current")" = 1 ] || { echo "cannot read one REGION_BUILD from $values" >&2; exit 2; }
[ "$retire" != "$current" ] || { echo "refused: $retire is the build Git deploys" >&2; exit 2; }
grep -qw "$retire" <<< "$retained" || { echo "refused: $retire is not in REGION_RETAINED" >&2; exit 2; }
image="$(awk '/name: aevolution-postgres-tools/ {f = 1} f && /newName:/ {n = $2} f && /digest:/ {print n "@" $2; exit}' \
"$root/deploy/kubernetes/stores/$env/kustomization.yaml")"
[[ "$image" =~ @sha256:[0-9a-f]{64}$ ]] || { echo "no pinned postgres-tools image for $env" >&2; exit 2; }
say() { echo "$(date -u +%H:%M:%S) $*"; }
ready_pod() {
"${k[@]}" get pods -n "$ns" -l app.kubernetes.io/name=evo-region -o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{range .status.conditions}}{{if and (eq .type "Ready") (eq .status "True")}}{{$.metadata.name}}{{end}}{{end}}{{end}}{{end}}' 2> /dev/null || true
}
serving() { # The Ready pod's running image digest.
local pod; pod="$(ready_pod)"
[ -n "$pod" ] && "${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.containerStatuses[0].imageID}'
}
[[ "$(serving)" == *"@sha256:$current" ]] ||
{ echo "refused: no Ready evo-region pod runs sha256:$current yet (deploy the new build first)" >&2; exit 3; }
pod="aevolution-barrier-$(date -u +%Y%m%d%H%M%S)"
cleanup() { "${k[@]}" delete pod -n "$ns" "$pod" --wait=false > /dev/null 2>&1 || true; }
trap cleanup EXIT
say "retiring $retire in $ns (serving sha256:$current)"
"${k[@]}" apply -n "$ns" -f - > /dev/null <<YAML
apiVersion: v1
kind: Pod
metadata:
name: $pod
labels:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: migrate
spec:
serviceAccountName: aevolution-migrator
automountServiceAccountToken: false
restartPolicy: Never
activeDeadlineSeconds: 120
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
seccompProfile: {type: RuntimeDefault}
containers:
- name: barrier
image: $image
command: ["/bin/bash", "-c"]
args:
- |
set -eu
umask 077
cp /run/postgres/pgpass /tmp/pgpass
export PGPASSFILE=/tmp/pgpass
psql -X --no-password -At -v ON_ERROR_STOP=1 \
-c "SELECT 'fenced ' || evolution.upgrade_barrier(decode('$retire', 'hex'))" \
-c "SELECT 'retired ' || encode(build_identity, 'hex') FROM evolution.retained_builds WHERE retired_at IS NOT NULL ORDER BY 1"
env:
- {name: PGSERVICE, value: evolution}
- {name: PGSERVICEFILE, value: /run/postgres/pg_service.conf}
resources:
requests: {cpu: 50m, memory: 64Mi}
limits: {memory: 128Mi}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
volumeMounts:
- {name: connection, mountPath: /run/postgres, readOnly: true}
- {name: scratch, mountPath: /tmp}
volumes:
- name: connection
secret:
secretName: evolution-postgres-migrator
defaultMode: 0440
items:
- {key: pg_service.conf, path: pg_service.conf}
- {key: pgpass, path: pgpass}
- {key: ca.crt, path: ca.crt}
- name: scratch
emptyDir: {medium: Memory, sizeLimit: 1Mi}
YAML
start="$(date +%s%3N)"
for _ in $(seq 1 120); do
phase="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}' 2> /dev/null || true)"
[ "$phase" = Succeeded ] || [ "$phase" = Failed ] && break
sleep 1
done
"${k[@]}" logs -n "$ns" "$pod" 2>&1 | sed 's/^/ /'
[ "$phase" = Succeeded ] || { say "FAIL barrier pod $phase"; exit 3; }
not_ready_since=""
for _ in $(seq 1 600); do
if [ -n "$(ready_pod)" ]; then
[ -z "$not_ready_since" ] && [ $(( $(date +%s%3N) - start )) -lt 3000 ] && { sleep 0.5; continue; }
break
fi
[ -n "$not_ready_since" ] || not_ready_since="$(date +%s%3N)"
sleep 0.5
done
[[ "$(serving)" == *"@sha256:$current" ]] || { say "FAIL no Ready pod serves sha256:$current after the barrier"; exit 3; }
gap=0; [ -n "$not_ready_since" ] && gap=$(( $(date +%s%3N) - not_ready_since ))
say "PASS $retire retired in $ns; not-ready gap ${gap} ms; sha256:$current serving"