diff --git a/CMakeLists.txt b/CMakeLists.txt index 6077f44..2cb5fc7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -128,7 +128,7 @@ if(EVO_BUILD_POSTGRES) tests/region_server_process_tests.cpp tests/region_recovery_tests.cpp tests/region_ops_tests.cpp tests/region_clone_tests.cpp tests/region_migration_tests.cpp tests/region_store_tests.cpp - tests/region_load_tests.cpp) + tests/region_load_tests.cpp tests/region_upgrade_tests.cpp) target_link_libraries(evo-postgres-tests PRIVATE evo::region_host Threads::Threads) add_dependencies(evo-postgres-tests evo-server evo-world-clone) # Spawned by process tests. evo_owned_target(evo-postgres-tests) diff --git a/apps/evo-server/main.cpp b/apps/evo-server/main.cpp index db402df..7533879 100644 --- a/apps/evo-server/main.cpp +++ b/apps/evo-server/main.cpp @@ -28,6 +28,12 @@ // restarting from the committed tick after every recovery. Request IDs are // "SYN1" + region + tick, so acknowledgements can be checked across owners. // +// Upgrade rehearsal on clones only (M8.9, ADR-0076), both requiring +// --synthetic-inputs: --synthetic-until stops the driver after that tick, so +// the region commits exactly up to it and then idles (inputs are never invented); +// --accelerate <2..1000> runs the region's scheduling clock that many times faster +// than real time (lease deadlines stay real). Never in a deployed environment. +// // --crash-at : (tests only, M7.7) SIGKILLs this process at the n-th // occurrence of before-upload, before-transaction, after-commit or after-ack (right // after a step's acknowledgements are written and flushed). @@ -69,6 +75,8 @@ struct Options { std::uint64_t crash_ordinal{}; std::string ops_listen; bool synthetic{}; + std::uint64_t synthetic_until{}; // 0: unbounded. + std::uint64_t accelerate{1}; std::uint64_t objects_reserve_mib{1024}; }; @@ -174,6 +182,10 @@ std::optional parse(int argc, char** argv) { o.ops_listen = value; else if (flag == "--synthetic-inputs" && value == "development-v1") o.synthetic = true; + else if (flag == "--synthetic-until" && n && *n > 0) + o.synthetic_until = *n; + else if (flag == "--accelerate" && n && *n >= 2 && *n <= 1000) + o.accelerate = *n; else if (flag == "--objects-reserve-mib" && n && *n <= 1048576) o.objects_reserve_mib = *n; else if (flag == "--crash-at") { @@ -196,6 +208,9 @@ std::optional parse(int argc, char** argv) { o.pg.port.empty() || o.pg.database.empty() || o.pg.user.empty() || o.objects.empty() || o.host.build == persistence::Sha256Digest{}) return std::nullopt; + // Rehearsal controls exist only for the synthetic driver (clones, drills). + if ((o.synthetic_until != 0 || o.accelerate != 1) && !o.synthetic) + return std::nullopt; if (std::ranges::find(o.host.retained_builds, o.host.build) == o.host.retained_builds.end()) o.host.retained_builds.push_back(o.host.build); return o; @@ -343,6 +358,12 @@ template int serve(Options& o) { } return {404, "not found\n"}; }; + // Scheduling time; --accelerate scales it from process start (clones only). + const auto started = std::chrono::steady_clock::now(); + const auto scheduling_now = [&] { + const auto real = std::chrono::steady_clock::now(); + return started + (real - started) * static_cast(o.accelerate); + }; LineReader reader; bool input_open = !o.synthetic, ready = false; // Synthetic driver state: the next tick to submit, reset at every recovery. @@ -411,7 +432,8 @@ template int serve(Options& o) { synthetic_next = now_status.simulated_tick + 1; // Resume after the head. synthetic_running = running; while (running && synthetic_next && - *synthetic_next <= host.status().simulated_tick + 64) { + *synthetic_next <= host.status().simulated_tick + 64 && + (o.synthetic_until == 0 || *synthetic_next <= o.synthetic_until)) { const server::TickInput in{synthetic_request(*synthetic_next), processes::development_input(*synthetic_next)}; if (!host.submit(in)) @@ -419,7 +441,7 @@ template int serve(Options& o) { ++*synthetic_next; } } - const auto& s = host.step(std::chrono::steady_clock::now()); + const auto& s = host.step(scheduling_now()); max_queued = std::max(max_queued, s.queued); // Checkpoint age restarts at recovery and at every confirmed commit. if ((!recovered && s.phase == server::HostPhase::running) || @@ -492,7 +514,8 @@ int main(int argc, char** argv) { "--objects DIR --build HEX64 [--retained HEX64]... [--pg-hostaddr A " "--pg-passfile F --pg-ca F] [--flush-ms N] [--drain-ms N] " "[--lease-profile default|fast] [--ready-file PATH] " - "[--ops-listen ADDRESS:PORT]\n", + "[--ops-listen ADDRESS:PORT] [--synthetic-inputs development-v1 " + "[--synthetic-until TICK] [--accelerate N]]\n", stderr); return 2; } diff --git a/apps/evo-world-clone/main.cpp b/apps/evo-world-clone/main.cpp index 28a0dad..8424da9 100644 --- a/apps/evo-world-clone/main.cpp +++ b/apps/evo-world-clone/main.cpp @@ -11,6 +11,8 @@ // [--reserve-mib N] // evo-world-clone provision --world W --region R --opening development-v1 --pg-* ... // --objects DIR --build HEX [--retain-build] +// evo-world-clone compare --world A --region R --other-world B --pg-* ... +// --objects DIR [--other-objects DIR] --retained HEX... [--max-commits N] // // --*pg-* is host, hostaddr, port, database, user, passfile, ca, or service (a libpq // service file whose [evolution] section supplies everything but the password file). The source @@ -40,8 +42,14 @@ // stage; World0's opening is a separate, reviewed decision. // Output: provisioned|already-provisioned world=W region=R sequence=S tick=T // +// compare (M8.9, ADR-0076) fully audits both regions of one database, then requires +// the same world: equal head tick, frame, quantum and versions, identical head state +// once B is rebound to A's world, and identical durable outcomes. Builds may differ. +// Output: compared tick=T outcomes=N commits_a=N commits_b=N state=HEX +// compare differs ticks|versions|state|outcomes (exit 6) +// // Exit: 0 done, 2 usage, 3 source/history/database failure, 4 target/objects refused, -// 5 target unknown. +// 5 target unknown, 6 compared histories differ. #include "evo/persistence/region_clone.hpp" #include "evo/persistence/sha256.hpp" #include "evo/processes/development_opening.hpp" @@ -150,7 +158,7 @@ std::optional parse(int argc, char** argv) { Options o; o.command = argv[1]; if (o.command != "clone" && o.command != "audit" && o.command != "check-store" && - o.command != "provision") + o.command != "provision" && o.command != "compare") return std::nullopt; const bool clone = o.command == "clone"; for (int i = 2; i < argc; ++i) { @@ -174,13 +182,16 @@ std::optional parse(int argc, char** argv) { o.source.key.world = *n; else if (flag == (clone ? "--source-region" : "--region") && n) o.source.key.region = *n; - else if (clone && flag == "--target-world" && n) + else if (((clone && flag == "--target-world") || + (o.command == "compare" && flag == "--other-world")) && + n) o.target.key.world = *n; else if (clone && flag == "--target-region" && n) o.target.key.region = *n; else if (flag == (clone ? "--source-objects" : "--objects")) o.source.objects = value; - else if (clone && flag == "--target-objects") + else if ((clone && flag == "--target-objects") || + (o.command == "compare" && flag == "--other-objects")) o.target.objects = value; else if (clone && flag == "--commit" && hex<16>(value)) o.commit = hex<16>(value); @@ -188,7 +199,8 @@ std::optional parse(int argc, char** argv) { o.audit_max = *n; else if (clone && flag == "--crash-at" && value == "before-transaction") o.crash = true; - else if (o.command == "audit" && flag == "--max-commits" && n && *n > 0) + else if ((o.command == "audit" || o.command == "compare") && flag == "--max-commits" && n && + *n > 0) o.max_commits = *n; else if (o.command == "check-store" && flag == "--reserve-mib" && n && *n <= 1048576) o.reserve_mib = *n; @@ -203,6 +215,14 @@ std::optional parse(int argc, char** argv) { } if (clone && o.target.key.region == 0) o.target.key.region = o.source.key.region; + if (o.command == "compare") { + if (o.target.key.world == 0) + return std::nullopt; + o.target.key.region = o.source.key.region; + o.target.pg = o.source.pg; + if (o.target.objects.empty()) + o.target.objects = o.source.objects; + } const bool needs_builds = o.command != "check-store" && o.command != "provision"; if (o.command == "provision" && (o.opening.empty() || !o.build)) return std::nullopt; @@ -444,6 +464,56 @@ int audit(const Options& o) { return 0; } +const char* name(persistence::CompareError e) { + using enum persistence::CompareError; + switch (e) { + case identity: + return "identity"; + case audit: + return "audit"; + case ticks: + return "ticks"; + case versions: + return "versions"; + case state: + return "state"; + case outcomes: + return "outcomes"; + } + return "unknown"; +} + +int compare(const Options& o) { + auto db = connect(o.source); + auto objects_a = open_objects(o.source); + auto objects_b = open_objects(o.target); + if (!db || !objects_a || !objects_b) { + std::fputs("compare failed store\n", stderr); + return 3; + } + auto result = persistence::compare_region_histories(*db, *objects_a, o.source.key, *db, + *objects_b, o.target.key, o.retained, + restore_budget, o.max_commits, timeout); + if (!result) { + const auto& f = result.error(); + using persistence::CompareError; + if (f.kind == CompareError::identity || f.kind == CompareError::audit) { + std::fprintf(stderr, "compare failed %s side=%c audit=%s sequence=%llu\n", name(f.kind), + f.side == 0 ? '-' : f.side, name(f.audit.kind), + static_cast(f.audit.sequence)); + return f.kind == CompareError::identity ? 2 : 3; + } + std::printf("compare differs %s\n", name(f.kind)); + return 6; + } + std::printf("compared tick=%llu outcomes=%llu commits_a=%llu commits_b=%llu state=%s\n", + static_cast(result->tick), + static_cast(result->outcomes), + static_cast(result->commits_a), + static_cast(result->commits_b), text(result->state).c_str()); + return 0; +} + int clone(const Options& o) { const auto& s = o.source.pg; const auto& t = o.target.pg; @@ -511,7 +581,8 @@ int clone(const Options& o) { int main(int argc, char** argv) { const auto options = parse(argc, argv); if (!options) { - std::fputs("usage: evo-world-clone clone|audit ... (see apps/evo-world-clone/main.cpp)\n", + std::fputs("usage: evo-world-clone clone|audit|check-store|provision|compare ... " + "(see apps/evo-world-clone/main.cpp)\n", stderr); return 2; } @@ -519,5 +590,7 @@ int main(int argc, char** argv) { return check_store(*options); if (options->command == "provision") return provision(*options); + if (options->command == "compare") + return compare(*options); return options->command == "clone" ? clone(*options) : audit(*options); } diff --git a/deploy/kubernetes/monitoring/alertmanager.yaml b/deploy/kubernetes/monitoring/alertmanager.yaml index 03892e8..68d2c66 100644 --- a/deploy/kubernetes/monitoring/alertmanager.yaml +++ b/deploy/kubernetes/monitoring/alertmanager.yaml @@ -22,6 +22,15 @@ spec: spec: serviceAccountName: aevolution-alertmanager automountServiceAccountToken: false + # Stateless: prefers nodes labelled for AEvolution compute (ADR-0076), and + # schedules anywhere while none is labelled (the single-node plan). + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - {key: aevolution.io/compute, operator: In, values: ["true"]} securityContext: runAsNonRoot: true runAsUser: 65534 diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index b66c0a4..35ee4fd 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -829,3 +829,11 @@ durable failure domain exists, enforced by the deploy policy check; host-loss restore in place with barrier fencing of the superseded database rehearsed locally; promotion only of stage-pinned digests. **M8-dev/stage passed; M8-World0 launch not passed.** + +M8.9 implemented and locally qualified (2026-09-30), under ADR-0076: accelerated +clone runs of the current and candidate builds compared state-for-state and +outcome-for-outcome (`evo-world-clone compare`); failed-migration recovery and the +version barrier rehearsed with real hosts; `upgrade-barrier.sh` for dev/stage; +stores/compute node labels with policy-checked placement and `scheduling-check.sh`. +Rules changes stay refused until a converter ADR. In-cluster barrier and +additional-node evidence wait on the maintainer. diff --git a/docs/STATUS.md b/docs/STATUS.md index 5225f03..c46fc47 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1141,3 +1141,21 @@ ADR-0075: - `fence-database.sh` supersedes the old database through the upgrade barrier: a still-running old host quarantines, and no host can acquire it again. - `tools/ci/dr-check.sh` rehearses it in every PostgreSQL build. + +## M8.9 upgrade and growth (2026-09-30) + +ADR-0076: +- **Compare before upgrading.** `evo-server --synthetic-until TICK --accelerate N` + (drill-only, refused without the synthetic driver) runs clones to an exact tick + faster than real time; `evo-world-clone compare` (library: + `compare_region_histories`) audits both and requires the same world: same + versions, identical rebound head state and identical outcomes. Builds may differ. +- **Failed migration and barrier rehearsed** with real hosts + (`tools/ci/upgrade-check.sh`): a failing migration leaves schema v9 and the host + serving; the barrier fences the old build (exit 4), the new one resumes under a + newer epoch, and the history across both builds audits. +- **Dev/stage barrier:** `tools/ops/upgrade-barrier.sh --retire HEX` retires an + old build only when it is in `REGION_RETAINED` and the new digest is already Ready. +- **Nodes:** `aevolution.io/stores` (one worker; every objects-volume pod selects it) + and `aevolution.io/compute` (preferred only; Alertmanager first). + `tools/ops/scheduling-check.sh ` verifies placement read-only. diff --git a/docs/adr/0076-upgrade-and-growth.md b/docs/adr/0076-upgrade-and-growth.md new file mode 100644 index 0000000..3f67310 --- /dev/null +++ b/docs/adr/0076-upgrade-and-growth.md @@ -0,0 +1,152 @@ +# ADR-0076: Upgrade rehearsal on clones, version barrier in dev/stage, and node growth + +## Status +Accepted for M8.9, 2026-09-30. It builds on: +- ADR-0067 (clones); +- ADR-0068 (upgrade barrier); +- ADR-0073 (synthetic driver); +- ADR-0074/0075 (backup, restore, World0 gate). + +## Context +M8.9 requires: +- clone, migrate, accelerate and compare before any world-wide rules change; +- a node label and affinity plan that works on one node and admits added compute + nodes. + +The evidence asked for is: +- a dev/stage version-barrier rehearsal; +- recovery from a failed migration; +- scheduling onto an additional node where hardware exists. + +The named risks are mixed physical laws, and assuming that added CPU makes storage +portable. + +**No rules or format converter exists** (ADR-0068): a region never changes rules, +versions, quantum or profile. The only upgrade this foundation can perform is a +**build change with identical physical laws**, such as a bug fix, performance +work or a refactor. The question the rehearsal must answer is: does the candidate +build run the same world? + +## Decision +**1. Clone, accelerate, compare.** Two drill-only `evo-server` flags, each refused +without `--synthetic-inputs`: +- `--synthetic-until TICK` stops the deterministic driver after that tick. Inputs + are never invented, so the region commits exactly up to TICK and idles. +- `--accelerate N` (2–1000) runs the scheduling clock N times faster than real + time. Lease deadlines stay real. + +`evo-world-clone compare --world A --other-world B ...` calls the new library +function `compare_region_histories`: +- It fully audits both histories. +- It then requires equal head tick, frame, quantum, versions and history profile. +- It requires identical head states once B is rebound to A's world. +- It requires identical durable outcomes: tick, actor, outcome and organism, + including those inherited at clone genesis. +- Builds may differ. Exit 6 means "different world". + +The procedure: +1. Clone the live world's cut twice into an isolated database. +2. Run the current build on one clone and the candidate on the other, with the same + inputs, accelerated, to the same tick. +3. `compare` must pass. Only then may the candidate proceed. + +A rules change cannot pass this procedure. It stays refused until a converter ADR +defines its own comparison: ecological distributions and approval +(PERSISTENCE_ARCHITECTURE.md). Mixed physical laws are therefore impossible by +construction: a barrier admits only a build that `compare` proved runs the same +world. + +**2. Migration failure.** A migration runs in one transaction under the migration +lock, with a 5 s lock timeout and a 10 s statement timeout (ADR-0068). A failure +leaves the previous schema exactly. +- While the migration holds its locks, a publication may wait. Its outcome is then + resolved (ADR-0063), or its lease lapses and the host re-acquires under a newer + epoch. +- In a cluster, the migration Job is a Sync hook at wave −1. When it fails, the + sync stops before wave 1, so the Deployment keeps the old image and nothing new + starts. +- Recovery means fixing the migration and syncing again. No database restore is + involved: a restore after new writes would itself be a rewind (ADR-0075). + +**3. Barrier in dev/stage.** The ADR-0068 order has these steps: +1. Migrate. +2. Retain the new build. The provision hook does this. +3. Deploy the new build with the old build listed in `REGION_RETAINED`. The + Recreate strategy drains the old pod first. +4. Run `tools/ops/upgrade-barrier.sh --retire OLD`. + +The barrier tool refuses the retirement unless: +- the build is not the one Git deploys; +- it is listed in `REGION_RETAINED`; +- a Ready pod already runs the Git-pinned digest. + +It then runs `upgrade_barrier` in a short Pod with the migrator login, using the +`migrate` component's egress, and measures the not-ready gap while the new host +re-acquires. A rollback to a retired image quarantines. That is intended. + +**4. Nodes.** + +| Label | Meaning | Rule (policy-checked) | +|---|---|---| +| `aevolution.io/stores=true` | Holds the node-local durable volumes. Exactly one worker (today rke2-worker-1). | Every pod that mounts `aevolution-objects` must select it. Region hosts, provisioning, store checks and backups stay there. | +| `aevolution.io/compute=true` | Admitted for stateless AEvolution work. | Only ever *preferred*, never required, so everything still schedules on one node or with no labelled node. Alertmanager is the first user. | + +Adding a node means labelling it `aevolution.io/compute=true`, then running +`tools/ops/scheduling-check.sh `. Only stateless pods move. A region moves to +another node only as a data operation: a restore or a clone onto that node's +storage (ADR-0074/0075). Rescheduling a region does not move it, because its +objects volume is pinned to the stores node by the local-path PV's node affinity. +The policy check rejects: +- a pod that mounts the objects volume without the stores selector; +- a required compute affinity; +- rehearsal flags in a deployed workload. + +## Evidence +Every PostgreSQL build runs two tests. + +`region_upgrade_tests` (real processes): +- A source world is run past its death to tick 30 and cloned four times. +- Build b1 and candidate build b2 each run to tick 150 at 20× acceleration: 120 + ticks, 6 s of region time, in under 4 s of wall time. +- `compare` passes, with one outcome. +- A clone fed a temperature-shifted input stream from the same cut differs in + `state`. +- A clone stopped at tick 100 differs in `ticks`. +- Comparing a world with itself, or using the rehearsal flags without the driver, + is a usage error. + +`tools/ci/upgrade-check.sh`: +- A migration that takes the migration lock, adds a column to `region_commits`, + holds its locks for 3 s and then fails runs while b1 serves. Afterwards the + schema is still v9, the column does not exist, b1 goes on acknowledging, and the + real migrate is a no-op. +- b2 is retained and started, and waits without becoming ready. +- `upgrade_barrier(b1)` fences one lease. b1 quarantines (exit 4). b2 resumes at a + newer epoch, at or past every b1 acknowledgement, and continues with the next + tick. +- A late b1 host quarantines at once. +- The history, now written by both builds, passes the full audit. + +In the cluster, still to be run by the maintainer: +- `upgrade-barrier.sh` on dev, retiring a build listed in `REGION_RETAINED`; +- `scheduling-check.sh` on dev and stage, before and after labelling a second + worker `aevolution.io/compute=true`. + +## Consequences +- Upgrades that keep the physical laws are qualified and operable in dev and stage. + Rules changes remain impossible until a converter ADR exists. +- Mutation tests show the compare check is not vacuous: removing the state + comparison, the stop tick or the acceleration each fails the tests. +- Region hosts stay on the stores node. Horizontal growth applies only to + stateless work until object storage is network-attached. The storage rework + (backlog) may revisit this. + +## Alternatives rejected +- **Comparing live regions at arbitrary commits.** Commit boundaries depend on flush + timing. A shared stop tick makes the comparison exact. +- **Required node affinity for compute.** It would break the single-node plan and + strand pods while no node carries the label. +- **Moving a region by relabelling nodes.** The volume stays behind. That is + exactly the named risk. +- **Retiring the old build automatically on every sync.** Retirement is + irreversible. It stays a deliberate step, guarded by the checks above. diff --git a/docs/adr/README.md b/docs/adr/README.md index bc95bf3..83c2372 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -86,3 +86,4 @@ design. No unadopted candidate is a link dependency of the proof. - [ADR-0073: Synthetic load and fault drills](0073-fault-drills.md) - accepted for M8.6 drill-only deterministic input driver, kill/full/empty/db-outage drill with RTO and zero-lost-acknowledgement checks. - [ADR-0074: World backup bundles and verified restore](0074-world-backup-and-restore.md) - accepted for M8.7 snapshot-consistent online bundles (dump + immutable objects + manifest), fail-closed audited restore on independent storage, isolated rebuild through clones; destination, encryption, schedule and WAL archiving pending. - [ADR-0075: World0 acknowledgment gate, host-loss recovery and promotion](0075-world0-acknowledgment-gate.md) - accepted for M8.8 ratified dev/stage RPO/RTO, World0 launch blocked (policy-enforced) until acknowledgements survive host loss in an independent failure domain, explicit-rewind in-place restore, barrier-based fencing of a superseded database, promotion only of stage-pinned digests. +- [ADR-0076: Upgrade rehearsal on clones, version barrier in dev/stage, and node growth](0076-upgrade-and-growth.md) - accepted for M8.9 accelerated clone runs compared state-for-state before a build change, failed-migration recovery, guarded in-cluster barrier, stores/compute node labels with policy-checked placement. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index db38053..51bfaac 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -2992,3 +2992,45 @@ Fixes (this segment): - any in-cluster host-loss restore or fence; - a second failure domain, which does not exist; - any World0 deployment or promotion, which is blocked by design. + +## M8.9 upgrade rehearsal and node plan - 2026-09-30 + +- `region_upgrade_tests` (real evo-server and evo-world-clone processes) passed in + the Release, analysis and ASan/UBSan PostgreSQL builds: + - Current build b1 and candidate b2 ran on clones of one cut, at 20× + acceleration, from tick 30 to tick 150: 120 ticks, 6 s of region time. Wall + times were 306/307 ms (Release), 322/323 ms (analysis) and 299/299 ms (ASan). + - `compare` passed: `compared tick=150 outcomes=1`. + - A temperature-shifted input stream gave `compare differs state` (exit 6). + - A clone stopped at tick 100 gave `compare differs ticks` (exit 6). + - Comparing a world with itself, and `--accelerate` or `--synthetic-until` + without the driver, were usage errors (exit 2). +- `tools/ci/upgrade-check.sh` passed in all three builds: + - A failing migration (migration lock, `ALTER TABLE region_commits`, locks held + 3 s, then an error) left schema v9 and no new column. b1 went on + acknowledging, and the real migrate was a no-op. + - b2 waited while b1 held the lease. `upgrade_barrier(b1)` fenced one lease, and + b1 exited 4. + - b2 resumed at `epoch=2 committed=135`, at b1's last acknowledgement (135), and + continued with tick 136. The ASan run: 137. + - A late b1 host exited 4 without becoming ready. + - The history audit across both builds passed: 24 commits to tick 167. +- Mutations, each caught: + - `compare` without the state comparison: upgrade test line 201 (the diverged + world passed); + - `--synthetic-until` ignored: line 165; + - `--accelerate` as a no-op: line 184 (over 4 s). +- The WSL `checks` set: 59/59 in debug, release, analysis and ASan. clang-tidy + (analysis builds) and the format check were clean. +- Deploy policy: 67 objects, 0 failures. Negatives, each rejected: an + objects-volume pod without the stores selector; a required compute affinity; + `--accelerate` in the region Deployment. `upgrade-barrier.sh` and + `scheduling-check.sh` refuse World0 before kubectl. +- Found during validation: genesis seed 0xF1 collided with a checkpoint-store test + (fixture provisioning failed), and was replaced by the unused 0xD7. clang-tidy + flagged duplicated option branches, which were merged. +- Not executed: + - `upgrade-barrier.sh` or `scheduling-check.sh` against dev or stage; + - scheduling onto a second labelled node; + - the Windows MSVC build (no Windows target changed; the server and PostgreSQL + code are Linux-only). diff --git a/docs/infrastructure/ENVIRONMENTS.md b/docs/infrastructure/ENVIRONMENTS.md index 9a5a64d..361d01a 100644 --- a/docs/infrastructure/ENVIRONMENTS.md +++ b/docs/infrastructure/ENVIRONMENTS.md @@ -232,3 +232,20 @@ rebuilt as isolated world ...` counts as restore evidence. The restore uses fres roles on a throwaway cluster; nothing reaches the source database. A restored copy keeps the source world identity and is never served; restoring in place after host loss is M8.8. + +## Upgrades and nodes (M8.9, ADR-0076) + +**Build upgrade (same rules).** +1. Rehearse on clones first: clone the cut twice into an isolated database, run the + current and candidate builds with `--synthetic-inputs development-v1 + --synthetic-until T --accelerate 20`, then `evo-world-clone compare` must exit 0. +2. Pin the new server digest in `deploy/kubernetes/region/` (image and + `REGION_BUILD`), add the old digest to `REGION_RETAINED`, and sync. The migrate + hook (wave -1) stops the sync on failure, leaving the old host running. +3. When the new pod is Ready: `tools/ops/upgrade-barrier.sh --retire OLD`. + Keep OLD in `REGION_RETAINED`: its history stays recoverable. + +**Adding a node.** Label it `aevolution.io/compute=true` and run +`tools/ops/scheduling-check.sh `. Only stateless pods (Alertmanager today) may +move there. Never move `aevolution.io/stores` to rebalance: the objects volume stays +on its node; a region moves only by restore or clone onto the new node's storage. diff --git a/include/evo/persistence/region_clone.hpp b/include/evo/persistence/region_clone.hpp index 40c12c3..57e0040 100644 --- a/include/evo/persistence/region_clone.hpp +++ b/include/evo/persistence/region_clone.hpp @@ -101,4 +101,34 @@ struct HistoryAudit { audit_region_history(Postgres& db, const FilesystemObjectStore& objects, RegionKey key, std::span retained_builds, std::size_t restore_budget, std::uint64_t max_commits, std::chrono::milliseconds timeout); + +// M8.9 upgrade comparison (ADR-0076). Before a build replaces another at a barrier, +// both run the same inputs on clones of one cut (accelerated), and their histories +// must be the same world: equal head tick, frame, quantum, versions and profile; +// identical head states once B is rebound to A's world; identical durable outcomes +// (tick, actor, outcome, organism), including those inherited at clone genesis. +// Builds may differ; nothing else may. Both histories are fully audited first. +// Read-only. A rules or format change cannot pass (no converter exists, ADR-0068). +enum class CompareError : std::uint8_t { + identity, // Same key, a zero ID, or different region IDs (only worlds rebind). + audit, // Either history failed its audit (see audit and side). + ticks, // Heads at different ticks: nothing compared. + versions, // Frame, quantum, versions or profile differ. + state, // Head states differ after rebinding B to A's world. + outcomes, // Durable outcome histories differ. +}; +struct CompareFailure { + CompareError kind{}; + AuditFailure audit{}; // Meaningful for audit. + char side{}; // 'a' or 'b' for audit. +}; +struct HistoryComparison { + std::uint64_t tick{}, outcomes{}, commits_a{}, commits_b{}; + Sha256Digest state{}; // SHA-256 of the identical head state, in A's world. +}; +[[nodiscard]] std::expected +compare_region_histories(Postgres& db_a, const FilesystemObjectStore& objects_a, RegionKey a, + Postgres& db_b, const FilesystemObjectStore& objects_b, RegionKey b, + std::span retained_builds, std::size_t restore_budget, + std::uint64_t max_commits, std::chrono::milliseconds timeout); } // namespace evo::persistence diff --git a/src/persistence/region_clone.cpp b/src/persistence/region_clone.cpp index 7ce0500..19953a2 100644 --- a/src/persistence/region_clone.cpp +++ b/src/persistence/region_clone.cpp @@ -1,6 +1,8 @@ #include "evo/persistence/region_clone.hpp" #include "checkpoint_detail.hpp" +#include #include +#include namespace evo::persistence { namespace { @@ -266,4 +268,91 @@ audit_region_history(Postgres& db, const FilesystemObjectStore& objects, RegionK return audit_fail(AuditError::unsupported, head->head.sequence); } } + +namespace { +std::unexpected compare_fail(CompareError kind, AuditFailure audit = {}, + char side = 0) { + return std::unexpected{CompareFailure{kind, audit, side}}; +} + +template +std::expected +compare(Postgres& db_a, const FilesystemObjectStore& objects_a, RegionKey a, Postgres& db_b, + const FilesystemObjectStore& objects_b, RegionKey b, std::span builds, + std::size_t budget, milliseconds timeout, const HistoryAudit& audited_a, + const HistoryAudit& audited_b) { + auto ra = recover(db_a, objects_a, a, builds, budget, timeout); + auto rb = recover(db_b, objects_b, b, builds, budget, timeout); + if (!ra || !rb) { + const auto kind = !ra ? ra.error().kind : rb.error().kind; + return compare_fail(kind == RecoveryError::unsupported ? CompareError::versions + : CompareError::audit, + {}, !ra ? 'a' : 'b'); + } + const auto& ma = ra->manifest; + const auto& mb = rb->manifest; + // The audited heads are the compared heads: a region that moved is not compared. + if (ma.end_tick != audited_a.end_tick || mb.end_tick != audited_b.end_tick || + ma.end_tick != mb.end_tick) + return compare_fail(CompareError::ticks); + if (ma.frame != mb.frame || ma.tick_microseconds != mb.tick_microseconds || + ma.versions != mb.versions || ma.history_profile != mb.history_profile || + ma.rng_streams != mb.rng_streams) + return compare_fail(CompareError::versions); + const auto state_a = Codec::snapshot(*ra->state); + if (!state_a || !rb->state->rebind_world(a.world)) + return compare_fail(CompareError::state); + const auto state_b = Codec::snapshot(*rb->state); + if (!state_b || *state_a != *state_b) + return compare_fail(CompareError::state); + const auto decisions = [](const auto& recovered) { + std::vector out; + out.reserve(recovered.decisions.size()); + for (const auto& d : recovered.decisions) + out.push_back(d.decision); + std::ranges::sort(out, {}, [](const BatchDecision& d) { + return std::tuple{d.tick, static_cast(d.actor)}; + }); + return out; + }; + const auto outcomes = decisions(*ra); + if (outcomes != decisions(*rb)) + return compare_fail(CompareError::outcomes); + const auto digest = sha256(*state_a); + if (!digest) + return compare_fail(CompareError::state); + return HistoryComparison{ma.end_tick, outcomes.size(), audited_a.commits, audited_b.commits, + *digest}; +} +} // namespace + +std::expected +compare_region_histories(Postgres& db_a, const FilesystemObjectStore& objects_a, RegionKey a, + Postgres& db_b, const FilesystemObjectStore& objects_b, RegionKey b, + std::span retained_builds, std::size_t restore_budget, + std::uint64_t max_commits, milliseconds timeout) { + if (a.world == 0 || a.region == 0 || b.world == 0 || b.region == 0 || a.region != b.region || + (a == b && &db_a == &db_b)) + return compare_fail(CompareError::identity); + const auto audited_a = audit_region_history(db_a, objects_a, a, retained_builds, restore_budget, + max_commits, timeout); + if (!audited_a) + return compare_fail(CompareError::audit, audited_a.error(), 'a'); + const auto audited_b = audit_region_history(db_b, objects_b, b, retained_builds, restore_budget, + max_commits, timeout); + if (!audited_b) + return compare_fail(CompareError::audit, audited_b.error(), 'b'); + if (audited_a->profile != audited_b->profile) + return compare_fail(CompareError::versions); + switch (audited_a->profile) { + case CheckpointProfile::environmental_v1: + return compare(db_a, objects_a, a, db_b, objects_b, b, retained_builds, + restore_budget, timeout, *audited_a, *audited_b); + case CheckpointProfile::environmental_lineage_v1: + return compare(db_a, objects_a, a, db_b, objects_b, b, retained_builds, + restore_budget, timeout, *audited_a, *audited_b); + default: + return compare_fail(CompareError::versions); + } +} } // namespace evo::persistence diff --git a/tests/postgres_tests.cpp b/tests/postgres_tests.cpp index 57e7c3f..d5c8ccf 100644 --- a/tests/postgres_tests.cpp +++ b/tests/postgres_tests.cpp @@ -13,6 +13,7 @@ void run_region_clone_tests(); void run_region_migration_tests(); void run_region_store_tests(); void run_region_load_tests(); +void run_region_upgrade_tests(); namespace { using namespace evo::persistence; using namespace std::chrono_literals; @@ -129,6 +130,8 @@ int main() { std::puts("region store qualification integration passed"); run_region_load_tests(); std::puts("region synthetic load integration passed"); + run_region_upgrade_tests(); + std::puts("region upgrade clone comparison integration passed"); run_region_migration_tests(); std::puts("region migration, barrier and qualification integration passed"); return 0; diff --git a/tests/region_upgrade_tests.cpp b/tests/region_upgrade_tests.cpp new file mode 100644 index 0000000..4515ef1 --- /dev/null +++ b/tests/region_upgrade_tests.cpp @@ -0,0 +1,216 @@ +// M8.9 upgrade rehearsal on clones (ADR-0076) with real evo-server and evo-world-clone +// processes: one cut is cloned into the isolated clone database several times; the +// current build (b1) and a candidate build (b2) each run the same deterministic inputs +// on their own clone, accelerated, to the same tick; `compare` must find the same +// world. A clone fed a different input stream, or stopped at another tick, must be +// reported as different. Run by tools/ci/postgres-check.sh. +#include "evo/processes/development_opening.hpp" +#include "fixtures/server_process.hpp" +#include +#include +#include + +void run_region_upgrade_tests(); + +namespace { +using namespace evo::test::process; +using evo::persistence::RegionKey; +void check(bool value, std::source_location at = std::source_location::current()) { + if (!value) + throw std::runtime_error("region upgrade test line " + std::to_string(at.line())); +} +#define CHECK(...) check(static_cast(__VA_ARGS__)) +std::string b1_hex() { + std::string out; + for (int i = 0; i < 32; ++i) + out += "b1"; + return out; +} +std::string b2_hex() { + std::string out; + for (int i = 0; i < 32; ++i) + out += "b2"; + return out; +} +constexpr std::uint64_t region = 95; + +ServerRegion clone_root(std::uint64_t world) { + ServerRegion r{{world, region}, fs::path(env("PGHOST")) / ("upgrade-" + std::to_string(world))}; + fs::remove_all(r.root); + fs::create_directory(r.root); + fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace); + return r; +} +std::vector pg(const std::string& prefix, const std::string& database, + const std::string& user) { + return {prefix + "host", env("PGHOST"), prefix + "port", env("PGPORT"), + prefix + "database", database, prefix + "user", user}; +} +int run(const std::vector& args, std::vector* lines = nullptr) { + Child tool{args}; + const auto code = tool.wait_exit(60s); + if (lines != nullptr) + *lines = tool.lines(); + return code; +} +void clone_to(const ServerRegion& source, const ServerRegion& target) { + std::vector args{env("EVO_WORLD_CLONE_BIN"), "clone", + "--source-world", std::to_string(source.key.world), + "--source-region", std::to_string(region), + "--target-world", std::to_string(target.key.world)}; + for (auto& a : pg("--source-pg-", env("PGDATABASE"), env("EVO_PG_RUNTIME_USER"))) + args.push_back(a); + for (auto& a : pg("--target-pg-", env("EVO_PG_CLONE_DATABASE"), env("PGUSER"))) + args.push_back(a); + for (const auto& a : + std::vector{"--source-objects", source.root.string(), "--target-objects", + target.root.string(), "--retained", b1_hex()}) + args.push_back(a); + CHECK(run(args) == 0); +} +// evo-server on a clone of the clone database with the given build. +std::vector clone_server(const ServerRegion& r, const std::string& build, + const std::vector& extra) { + std::vector args{env("EVO_SERVER_BIN"), + "--profile", + "lineage", + "--world", + std::to_string(r.key.world), + "--region", + std::to_string(region), + "--frame", + "9"}; + for (auto& a : pg("--pg-", env("EVO_PG_CLONE_DATABASE"), env("EVO_PG_CLONE_RUNTIME_USER"))) + args.push_back(a); + for (const auto& a : + std::vector{"--objects", r.root.string(), "--build", build, "--retained", + b1_hex(), "--flush-ms", "1000", "--lease-profile", "fast"}) + args.push_back(a); + args.insert(args.end(), extra.begin(), extra.end()); + return args; +} +bool acked(const Child& c, std::uint64_t tick) { + for (const auto& line : c.lines()) { + unsigned long long t = 0, s = 0; + char request[40]; + if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &t, &s) == 3 && t >= tick) + return true; + } + return false; +} +// Runs a synthetic, accelerated owner to `until` and drains; returns wall ms from +// readiness to the acknowledgement of `until`. +long long accelerated(const ServerRegion& r, const std::string& build, std::uint64_t until) { + Child server{clone_server(r, build, + {"--synthetic-inputs", "development-v1", "--synthetic-until", + std::to_string(until), "--accelerate", "20"})}; + CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); + const auto start = Clock::now(); + CHECK(server.wait_for([&](const std::string&) { return acked(server, until); }, 30s)); + const auto ms = + std::chrono::duration_cast(Clock::now() - start).count(); + CHECK(!acked(server, until + 1)); // The driver stops at `until`. + server.signal(SIGTERM); + CHECK(server.wait_exit(20s) == 0); + return ms; +} +// The development inputs, with the temperature shifted: a different input stream. +std::string shifted_lines(std::uint64_t first, std::uint64_t last) { + std::string out; + for (auto tick = first; tick <= last; ++tick) { + const auto in = evo::processes::development_input(tick); + std::string line = "input " + request_hex(tick, 0x5A) + " " + + std::to_string(in.temperature_mK + 1000) + " " + + std::to_string(in.source_enabled); + for (const auto b : in.bite) + line += " " + std::to_string(b); + for (const auto t : in.terminal) + line += " " + std::to_string(t); + out += line + "\n"; + } + return out; +} +int compare(std::uint64_t a, const ServerRegion& ra, std::uint64_t b, const ServerRegion& rb, + std::vector& lines) { + std::vector args{env("EVO_WORLD_CLONE_BIN"), + "compare", + "--world", + std::to_string(a), + "--region", + std::to_string(region), + "--other-world", + std::to_string(b)}; + for (auto& x : pg("--pg-", env("EVO_PG_CLONE_DATABASE"), env("PGUSER"))) + args.push_back(x); + for (const auto& x : + std::vector{"--objects", ra.root.string(), "--other-objects", + rb.root.string(), "--retained", b1_hex(), "--retained", b2_hex()}) + args.push_back(x); + return run(args, &lines); +} +bool printed(const std::vector& lines, std::string_view prefix) { + return std::ranges::any_of(lines, [&](const std::string& l) { return l.starts_with(prefix); }); +} + +void candidate_build_matches_on_clones() { + // The live source: provisioned and run past the tick-20 death, then stopped. + const auto source = provision(region, 0xD7); + { + Child owner{server_args( + source, {"--synthetic-inputs", "development-v1", "--synthetic-until", "30"})}; + CHECK(owner.wait_for([&](const std::string&) { return acked(owner, 30); }, 30s)); + owner.signal(SIGTERM); + CHECK(owner.wait_exit(20s) == 0); + } + CHECK(head_of(source.key).tick == 30); + auto clone_owner = evo::persistence::Postgres::connect( + {env("PGHOST"), {}, env("PGPORT"), env("EVO_PG_CLONE_DATABASE"), env("PGUSER"), {}, {}}, + 5s); + CHECK(clone_owner); + CHECK((*clone_owner) + ->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " + "(decode(repeat('b1', 32), 'hex')), (decode(repeat('b2', 32), 'hex')) " + "ON CONFLICT DO NOTHING", + {}, 5s)); + const auto reference = clone_root(951), candidate = clone_root(952), diverged = clone_root(953), + shorter = clone_root(954); + for (const auto* r : {&reference, &candidate, &diverged, &shorter}) + clone_to(source, *r); + + // Current and candidate builds, the same inputs, accelerated x20 to tick 150: + // 120 ticks are 6 s of region time. + const auto reference_ms = accelerated(reference, b1_hex(), 150); + const auto candidate_ms = accelerated(candidate, b2_hex(), 150); + CHECK(reference_ms < 4000 && candidate_ms < 4000); + std::printf("qualification upgrade_compare ticks=120 accelerate=20 reference_ms=%lld " + "candidate_ms=%lld realtime_ms=6000\n", + reference_ms, candidate_ms); + std::vector lines; + CHECK(compare(951, reference, 952, candidate, lines) == 0); + CHECK(printed(lines, "compared tick=150 outcomes=1 ")); + + // A different input stream from the same cut is a different world. + { + Child owner{clone_server(diverged, b2_hex(), {})}; + CHECK(owner.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); + owner.send(shifted_lines(31, 150)); + CHECK(owner.wait_for([&](const std::string&) { return acked(owner, 150); }, 30s)); + owner.signal(SIGTERM); + CHECK(owner.wait_exit(20s) == 0); + } + CHECK(compare(951, reference, 953, diverged, lines) == 6); + CHECK(printed(lines, "compare differs state")); + // Heads at different ticks are not compared. + static_cast(accelerated(shorter, b1_hex(), 100)); + CHECK(compare(951, reference, 954, shorter, lines) == 6); + CHECK(printed(lines, "compare differs ticks")); + // A world is not compared with itself, and rehearsal flags need the driver. + CHECK(compare(951, reference, 951, reference, lines) == 2); + CHECK(run(clone_server(reference, b1_hex(), {"--accelerate", "20"})) == 2); + CHECK(run(clone_server(reference, b1_hex(), {"--synthetic-until", "10"})) == 2); +} +} // namespace + +void run_region_upgrade_tests() { + candidate_build_matches_on_clones(); +} diff --git a/tools/ci/deploy-policy-check.py b/tools/ci/deploy-policy-check.py index d3f27fd..e3ed1a8 100644 --- a/tools/ci/deploy-policy-check.py +++ b/tools/ci/deploy-policy-check.py @@ -18,7 +18,10 @@ environment-isolation or World0-protection invariant is broken: - the cleanup tool refuses World0 before invoking kubectl; - World0 launch stays blocked (ADR-0075): no workload targets World0 and its Application keeps an unresolved path; any image a World0 overlay pins must be - the digest already pinned in stage (promotion, never a fresh build). + the digest already pinned in stage (promotion, never a fresh build); + - scheduling (ADR-0076): every pod that mounts the node-local objects volume + selects the store node; aevolution.io/compute is only ever preferred, never + required; upgrade-rehearsal flags never appear in a deployed workload. """ import os import shutil @@ -281,6 +284,28 @@ for directory, _, files in os.walk(os.path.join(ROOT, 'deploy', 'kubernetes')): check((image.get('name'), image.get('digest')) in staged, f'{rel}: {image.get("name")} must be promoted from {stage}, not pinned independently') +# M8.9 scheduling (ADR-0076). Node-local volumes pin their pods: added CPU does not +# make the objects volume portable. Compute is a preference so one node still works. +for path, doc in objects: + if doc['kind'] not in workload_kinds or path.endswith('region-values.yaml'): + continue + spec = doc.get('spec', {}) + pod = spec.get('jobTemplate', {}).get('spec', {}).get('template', {}).get('spec') \ + or spec.get('template', {}).get('spec') or spec + claims = [v.get('persistentVolumeClaim', {}).get('claimName') for v in pod.get('volumes', [])] + if 'aevolution-objects' in claims: + check(pod.get('nodeSelector', {}).get('aevolution.io/stores') == 'true', + f'{path}: a pod mounting aevolution-objects must select aevolution.io/stores') + required = (pod.get('affinity', {}).get('nodeAffinity', {}) + .get('requiredDuringSchedulingIgnoredDuringExecution', {})) + check('aevolution.io/compute' not in yaml.safe_dump(required) + and 'aevolution.io/compute' not in pod.get('nodeSelector', {}), + f'{path}: aevolution.io/compute may only be preferred, never required') + for container in pod.get('containers', []): + text = ' '.join(container.get('args', []) + container.get('command', [])) + check('--accelerate' not in text and '--synthetic-until' not in text, + f'{path}: upgrade-rehearsal flags never run in a deployed workload') + # The platform bundle: every rendered object is protected from prune and deletion. bundle = manifests.get(platform + '/cluster/kustomization.yaml', [{}])[0] check(bundle.get('commonAnnotations', {}).get('argocd.argoproj.io/sync-options') == 'Prune=false,Delete=false', @@ -334,7 +359,8 @@ for path, doc in objects: with tempfile.TemporaryDirectory() as empty: env = dict(os.environ, PATH=empty + os.pathsep + os.path.dirname(sys.executable)) for tool in ('tools/ops/cleanup-environment.sh', 'tools/ops/region-drill.sh', 'tools/ops/fault-drill.sh', - 'tools/backup/cluster-backup.sh'): + 'tools/backup/cluster-backup.sh', 'tools/ops/scheduling-check.sh', + 'tools/ops/upgrade-barrier.sh'): for bad in ('world0', 'evolution-world0', 'World0', 'prod', ''): result = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, tool), bad], capture_output=True, text=True, env=env) diff --git a/tools/ci/postgres-check.sh b/tools/ci/postgres-check.sh index eaa0211..c0defba 100644 --- a/tools/ci/postgres-check.sh +++ b/tools/ci/postgres-check.sh @@ -34,6 +34,8 @@ EVO_WORLD_CLONE_BIN="$(dirname "$1")/evo-world-clone" bash "$root/tools/ci/backup-check.sh" # M8.8: host-loss rehearsal: in-place restore, fenced old environment, measured rewind. bash "$root/tools/ci/dr-check.sh" +# M8.9: failed migration under load and the version barrier with real hosts. +bash "$root/tools/ci/upgrade-check.sh" v1_sha="$(sha256sum "$root/migrations/0001_region_bootstrap.sql")" v1_sha="${v1_sha%% *}" # Migration failure must not publish a partial schema. diff --git a/tools/ci/upgrade-check.sh b/tools/ci/upgrade-check.sh new file mode 100644 index 0000000..5ad514a --- /dev/null +++ b/tools/ci/upgrade-check.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# M8.9 version-barrier rehearsal (ADR-0076). Run by postgres-check.sh inside its +# throwaway cluster (PG* set; EVO_SERVER_BIN, EVO_WORLD_CLONE_BIN, runtime roles +# exported). Real evo-server processes on world 500: +# 1. Failed migration: a migration that takes the migration lock, alters a history +# table, holds its locks for 3 s and then fails runs while build b1 serves. +# Nothing it did remains (schema still v9, no new column); the host keeps its +# authority, and every acknowledgement stays durable. +# 2. Barrier (ADR-0068 operator order): build b2 is retained, and a b2 host is started +# while b1 still holds the lease (it waits, never ready). upgrade_barrier(b1) fences +# b1, which quarantines (exit 4); b2 resumes at the committed head under a newer +# epoch. A b1 host started afterwards quarantines at once, and the history across +# both builds passes the full audit. +set -euo pipefail +root="$(cd "$(dirname "$0")/../.." && pwd)" +work="$(mktemp -d /tmp/evo-upgrade-check.XXXXXXXX)" +chmod 700 "$work" +pids=() +trap 'for p in "${pids[@]}"; do kill -9 "$p" 2> /dev/null || true; done; rm -rf -- "$work"' EXIT +fail() { echo "upgrade rehearsal FAILED: $*" >&2; exit 1; } +b1=b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1 +b2=b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2 +createdb evo_upgrade +export PGDATABASE=evo_upgrade +bash "$root/tools/migrate-postgres.sh" > /dev/null +q() { psql -XAt -v ON_ERROR_STOP=1 "$@"; } +mkdir -m 700 "$work/objects" +pg=(--pg-host "$PGHOST" --pg-port "$PGPORT" --pg-database evo_upgrade) +"$EVO_WORLD_CLONE_BIN" provision --world 500 --region 1 --opening development-v1 "${pg[@]}" \ + --pg-user "$PGUSER" --objects "$work/objects" --build "$b1" --retain-build true > /dev/null +serve() { # serve [retained...] + local build="$1" log="$2"; shift 2 + local retained=() + for r in "$@"; do retained+=(--retained "$r"); done + "$EVO_SERVER_BIN" --profile lineage --world 500 --region 1 --frame 9 "${pg[@]}" \ + --pg-user "$EVO_PG_RUNTIME_USER" --objects "$work/objects" --build "$build" "${retained[@]}" \ + --flush-ms 200 --lease-profile fast --synthetic-inputs development-v1 < /dev/null > "$log" 2>&1 & + pids+=($!) + server_pid=$! +} +acked_through() { awk -v t="$2" '$1 == "ack" && $3 >= t {f = 1} END {exit !f}' "$1"; } +max_ack() { awk '$1 == "ack" && $3 > m {m = $3} END {print m + 0}' "$1"; } +wait_ack() { for _ in $(seq 1 300); do acked_through "$1" "$2" && return 0; sleep 0.1; done; return 1; } +head_tick() { q -c 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 500'; } + +serve "$b1" "$work/b1.log"; b1_pid=$server_pid +wait_ack "$work/b1.log" 30 || fail "b1 acknowledged nothing" + +# 1. A failing migration while b1 serves. +before="$(max_ack "$work/b1.log")" +code=0 +q -v ON_ERROR_STOP=1 > "$work/migration.log" 2>&1 <<'SQL' || code=$? +BEGIN; +SET LOCAL lock_timeout = '5s'; +SELECT pg_catalog.pg_advisory_xact_lock(71420001); +ALTER TABLE evolution.region_commits ADD COLUMN rehearsal_column integer; +SELECT pg_catalog.pg_sleep(3); +SELECT 1 / 0; -- The migration fails after holding its locks. +COMMIT; +SQL +[ "$code" != 0 ] && grep -q 'division by zero' "$work/migration.log" || fail "the rehearsal migration did not fail" +[ "$(q -c "SELECT count(*) FROM information_schema.columns WHERE table_name = 'region_commits' AND column_name = 'rehearsal_column'")" = 0 ] || + fail "the failed migration left a column" +[ "$(q -c 'SELECT max(version) FROM evolution.schema_migrations')" = 9 ] || fail "schema version moved" +wait_ack "$work/b1.log" $((before + 40)) || fail "b1 did not resume after the failed migration" +kill -0 "$b1_pid" 2> /dev/null || fail "b1 exited during the failed migration" +bash "$root/tools/migrate-postgres.sh" > /dev/null || fail "migrate after the failure" + +# 2. The barrier: retain b2, start its host, then retire b1. +q -c "INSERT INTO evolution.retained_builds(build_identity) VALUES (decode('$b2', 'hex'))" > /dev/null +serve "$b2" "$work/b2.log" "$b1"; b2_pid=$server_pid +sleep 2 +! grep -q '^ready 1' "$work/b2.log" || fail "b2 became ready while b1 held the lease" +[ "$(q -c "SELECT evolution.upgrade_barrier(decode('$b1', 'hex'))")" = 1 ] || fail "barrier fenced no lease" +code=0; wait "$b1_pid" || code=$? +[ "$code" = 4 ] || { tail "$work/b1.log" >&2; fail "b1 exited $code after the barrier, not 4"; } +b1_final="$(max_ack "$work/b1.log")" +wait_ack "$work/b2.log" $((b1_final + 30)) || { tail "$work/b2.log" >&2; fail "b2 did not take over"; } +resumed="$(sed -n 's/^phase running epoch=\([0-9]*\) committed=\([0-9]*\) .*/\1 \2/p' "$work/b2.log" | head -1)" +read -r epoch committed <<< "$resumed" +[ "$epoch" -ge 2 ] && [ "$committed" -ge "$b1_final" ] || + fail "b2 resumed at epoch=$epoch committed=$committed; b1 acknowledged $b1_final" +[ "$(awk '$1 == "ack" {print $3; exit}' "$work/b2.log")" = $((committed + 1)) ] || fail "b2 skipped or repeated ticks" +serve "$b1" "$work/late.log"; late_pid=$server_pid +code=0; wait "$late_pid" || code=$? +[ "$code" = 4 ] && ! grep -q '^ready 1' "$work/late.log" || fail "a late b1 host exited $code or served" +kill -TERM "$b2_pid"; code=0; wait "$b2_pid" || code=$? +[ "$code" = 0 ] || fail "b2 exited $code" +audit="$("$EVO_WORLD_CLONE_BIN" audit --world 500 --region 1 "${pg[@]}" --pg-user "$PGUSER" \ + --objects "$work/objects" --retained "$b1" --retained "$b2")" || fail "audit across builds" +[[ "$audit" == *" tick=$(head_tick) "* ]] || fail "audit: $audit" +builds="$(q -c 'SELECT count(DISTINCT build_identity) FROM evolution.region_commits WHERE world_id = 500')" +[ "$builds" = 2 ] || fail "history holds $builds builds, expected 2" +echo "upgrade rehearsal passed: failed migration held locks 3 s, b1 acknowledged $b1_final," \ + "b2 resumed epoch=$epoch committed=$committed, head $(head_tick) ($audit)" diff --git a/tools/ops/scheduling-check.sh b/tools/ops/scheduling-check.sh new file mode 100644 index 0000000..4939d9e --- /dev/null +++ b/tools/ops/scheduling-check.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# M8.9 read-only placement check (ADR-0076) for a disposable environment. +# +# tools/ops/scheduling-check.sh [--context NAME] +# +# Verifies the node plan against the live cluster, changing nothing: +# - exactly one Ready worker carries aevolution.io/stores=true, and it is not a +# control-plane node; +# - the environment's aevolution-objects volume is pinned to that node (local +# volumes cannot follow a region to another node); +# - the evo-region pod runs there; +# - nodes labelled aevolution.io/compute=true are listed, with whether they can +# take work (Ready, schedulable). None is valid: compute is only a preference. +# Adding a node: label it aevolution.io/compute=true and re-run. Only stateless pods +# move to it; a region moves between nodes only through a backup restore or a clone +# onto that node's storage (ADR-0074/0075), never by rescheduling. +set -euo pipefail +usage() { echo "usage: $0 [--context NAME]" >&2; exit 2; } +[ $# -ge 1 ] || usage +env="$1"; shift +case "$env" in + dev|stage) ;; + *) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2 + exit 2 ;; +esac +k=(kubectl --request-timeout=30s) +[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2") +ns="evolution-$env" +failures=0 +ok() { echo " ok $*"; } +bad() { echo " FAIL $*"; failures=$((failures + 1)); } +ready='{range .status.conditions[?(@.type=="Ready")]}{.status}{end}' + +echo "stores node" +stores="$("${k[@]}" get nodes -l aevolution.io/stores=true -o name | sed 's#^node/##')" +if [ "$(grep -c . <<< "$stores")" = 1 ]; then ok "one stores node: $stores"; else bad "stores nodes: '${stores//$'\n'/ }' (need exactly one)"; fi +store_node="$(head -1 <<< "$stores")" +if [ -n "$store_node" ]; then + [ "$("${k[@]}" get node "$store_node" -o jsonpath="$ready")" = True ] && ok "$store_node Ready" || bad "$store_node not Ready" + "${k[@]}" get node "$store_node" -o jsonpath='{.metadata.labels}' | grep -q 'node-role.kubernetes.io/control-plane' && + bad "$store_node is a control-plane node" || ok "$store_node is a worker" +fi + +echo "objects volume ($ns)" +pv="$("${k[@]}" get pvc -n "$ns" aevolution-objects -o jsonpath='{.spec.volumeName}' 2> /dev/null || true)" +if [ -z "$pv" ]; then + bad "aevolution-objects is not bound" +else + pinned="$("${k[@]}" get pv "$pv" -o jsonpath='{.spec.nodeAffinity.required.nodeSelectorTerms[*].matchExpressions[*].values[*]}')" + [ "$pinned" = "$store_node" ] && ok "$pv pinned to $pinned" || bad "$pv pinned to '$pinned', not the stores node '$store_node'" +fi + +echo "region host" +placed="$("${k[@]}" get pods -n "$ns" -l app.kubernetes.io/name=evo-region -o jsonpath='{range .items[*]}{.metadata.name}={.spec.nodeName}{"\n"}{end}')" +if [ -z "$placed" ]; then bad "no evo-region pod"; fi +while IFS='=' read -r pod node; do + [ -n "$pod" ] || continue + [ "$node" = "$store_node" ] && ok "$pod on $node" || bad "$pod on '$node', not the stores node" +done <<< "$placed" + +echo "compute nodes (preferred for stateless AEvolution pods)" +compute="$("${k[@]}" get nodes -l aevolution.io/compute=true -o name | sed 's#^node/##')" +if [ -z "$compute" ]; then + echo " none labelled: stateless pods schedule anywhere (single-node plan)" +else + while read -r node; do + state="$("${k[@]}" get node "$node" -o jsonpath="$ready")" + cordoned="$("${k[@]}" get node "$node" -o jsonpath='{.spec.unschedulable}')" + echo " $node Ready=$state schedulable=$([ "$cordoned" = true ] && echo no || echo yes)" + done <<< "$compute" +fi +placed_am="$("${k[@]}" get pods -n aevolution-monitoring -l app.kubernetes.io/name=alertmanager -o jsonpath='{.items[0].spec.nodeName}' 2> /dev/null || true)" +[ -n "$placed_am" ] && echo " alertmanager runs on $placed_am" +echo +[ "$failures" = 0 ] && echo "PASS scheduling plan in $ns" || { echo "FAIL $failures check(s) in $ns"; exit 1; } diff --git a/tools/ops/upgrade-barrier.sh b/tools/ops/upgrade-barrier.sh new file mode 100644 index 0000000..03e5c1b --- /dev/null +++ b/tools/ops/upgrade-barrier.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +# M8.9 version barrier in a disposable environment (ADR-0068, ADR-0076). +# +# tools/ops/upgrade-barrier.sh --retire HEX64 [--context NAME] +# +# The last step of the upgrade order: migrate -> retain the new build (the provision +# hook does) -> deploy it with the old build in REGION_RETAINED -> retire the old +# build here. Refused unless: +# - HEX64 is not the build Git currently deploys (deploy/kubernetes/region/); +# - HEX64 is listed in REGION_RETAINED (its history must stay recoverable); +# - the evo-region pod is Ready and runs exactly the Git-pinned image, so the new +# build already serves. +# Then one short Pod (the postgres-tools image pinned for , migrator login) runs +# evolution.upgrade_barrier(HEX64): the old build can never acquire, renew or commit +# again, and every lease is fenced. The new host re-acquires within one lease. The +# script waits for Ready again and prints the time without a ready host. +# Retirement is irreversible here. World0 is refused before kubectl runs. +set -euo pipefail +usage() { echo "usage: $0 --retire HEX64 [--context NAME]" >&2; exit 2; } +[ $# -ge 1 ] || usage +env="$1"; shift +case "$env" in + dev|stage) ;; + *) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2 + exit 2 ;; +esac +retire="" k=(kubectl --request-timeout=30s) +while [ $# -gt 0 ]; do + case "$1" in + --retire) retire="${2:-}"; shift 2 ;; + --context) k+=(--context "${2:?}"); shift 2 ;; + *) usage ;; + esac +done +[[ "$retire" =~ ^[0-9a-f]{64}$ ]] || usage +root="$(cd "$(dirname "$0")/../.." && pwd)" +ns="evolution-$env" +values="$root/deploy/kubernetes/region/$env/region-values.yaml" +current="$(sed -n 's/.*name: REGION_BUILD, value: \([0-9a-f]\{64\}\)}.*/\1/p' "$values" | sort -u)" +retained="$(sed -n 's/.*name: REGION_RETAINED, value: "\([0-9a-f ]*\)"}.*/\1/p' "$values" | head -1)" +[ "$(wc -l <<< "$current")" = 1 ] || { echo "cannot read one REGION_BUILD from $values" >&2; exit 2; } +[ "$retire" != "$current" ] || { echo "refused: $retire is the build Git deploys" >&2; exit 2; } +grep -qw "$retire" <<< "$retained" || { echo "refused: $retire is not in REGION_RETAINED" >&2; exit 2; } +image="$(awk '/name: aevolution-postgres-tools/ {f = 1} f && /newName:/ {n = $2} f && /digest:/ {print n "@" $2; exit}' \ + "$root/deploy/kubernetes/stores/$env/kustomization.yaml")" +[[ "$image" =~ @sha256:[0-9a-f]{64}$ ]] || { echo "no pinned postgres-tools image for $env" >&2; exit 2; } +say() { echo "$(date -u +%H:%M:%S) $*"; } +ready_pod() { + "${k[@]}" get pods -n "$ns" -l app.kubernetes.io/name=evo-region -o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{range .status.conditions}}{{if and (eq .type "Ready") (eq .status "True")}}{{$.metadata.name}}{{end}}{{end}}{{end}}{{end}}' 2> /dev/null || true +} +serving() { # The Ready pod's running image digest. + local pod; pod="$(ready_pod)" + [ -n "$pod" ] && "${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.containerStatuses[0].imageID}' +} +[[ "$(serving)" == *"@sha256:$current" ]] || + { echo "refused: no Ready evo-region pod runs sha256:$current yet (deploy the new build first)" >&2; exit 3; } + +pod="aevolution-barrier-$(date -u +%Y%m%d%H%M%S)" +cleanup() { "${k[@]}" delete pod -n "$ns" "$pod" --wait=false > /dev/null 2>&1 || true; } +trap cleanup EXIT +say "retiring $retire in $ns (serving sha256:$current)" +"${k[@]}" apply -n "$ns" -f - > /dev/null < /dev/null || true)" + [ "$phase" = Succeeded ] || [ "$phase" = Failed ] && break + sleep 1 +done +"${k[@]}" logs -n "$ns" "$pod" 2>&1 | sed 's/^/ /' +[ "$phase" = Succeeded ] || { say "FAIL barrier pod $phase"; exit 3; } +not_ready_since="" +for _ in $(seq 1 600); do + if [ -n "$(ready_pod)" ]; then + [ -z "$not_ready_since" ] && [ $(( $(date +%s%3N) - start )) -lt 3000 ] && { sleep 0.5; continue; } + break + fi + [ -n "$not_ready_since" ] || not_ready_since="$(date +%s%3N)" + sleep 0.5 +done +[[ "$(serving)" == *"@sha256:$current" ]] || { say "FAIL no Ready pod serves sha256:$current after the barrier"; exit 3; } +gap=0; [ -n "$not_ready_since" ] && gap=$(( $(date +%s%3N) - not_ready_since )) +say "PASS $retire retired in $ns; not-ready gap ${gap} ms; sha256:$current serving"