From 3f08a1d221ed561577c6fd65c0149e775581d483 Mon Sep 17 00:00:00 2001 From: dingo Date: Wed, 30 Sep 2026 10:52:01 -0400 Subject: [PATCH] Add M8.6 synthetic load driver and fault drill A drill-only evo-server --synthetic-inputs development-v1 driver feeds deterministic inputs from the committed head after every recovery, with request IDs that match acknowledgements across owners (ADR-0073); it is off in Git and the policy check keeps it off. fault-drill.sh loads dev or stage and runs repeated forced kills (RTO, zero lost acknowledgements), a full object volume (nothing acknowledged), an empty volume (quarantine, never a fresh world) and an operator-driven database outage (stale epoch never resumes), restoring the Git configuration on exit. Process tests prove contiguous acknowledgement across a SIGKILL and exact resumption after the committed head. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 3 +- apps/evo-server/main.cpp | 38 +++- deploy/kubernetes/region/base/deployment.yaml | 10 +- docs/ROADMAP.md | 7 + docs/STATUS.md | 22 +++ docs/adr/0073-fault-drills.md | 87 +++++++++ docs/adr/README.md | 1 + docs/engineering/VALIDATION.md | 22 +++ docs/infrastructure/ENVIRONMENTS.md | 22 +++ include/evo/processes/development_opening.hpp | 4 + src/processes/development_opening.cpp | 11 ++ tests/lineage_region_tests.cpp | 2 + tests/postgres_tests.cpp | 3 + tests/region_load_tests.cpp | 104 +++++++++++ tools/ci/deploy-policy-check.py | 9 +- tools/ops/fault-drill.sh | 173 ++++++++++++++++++ 16 files changed, 513 insertions(+), 5 deletions(-) create mode 100644 docs/adr/0073-fault-drills.md create mode 100644 tests/region_load_tests.cpp create mode 100644 tools/ops/fault-drill.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 2502fce..6077f44 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -127,7 +127,8 @@ if(EVO_BUILD_POSTGRES) tests/region_checkpoint_store_tests.cpp tests/region_host_tests.cpp tests/region_server_process_tests.cpp tests/region_recovery_tests.cpp tests/region_ops_tests.cpp tests/region_clone_tests.cpp - tests/region_migration_tests.cpp tests/region_store_tests.cpp) + tests/region_migration_tests.cpp tests/region_store_tests.cpp + tests/region_load_tests.cpp) target_link_libraries(evo-postgres-tests PRIVATE evo::region_host Threads::Threads) add_dependencies(evo-postgres-tests evo-server evo-world-clone) # Spawned by process tests. evo_owned_target(evo-postgres-tests) diff --git a/apps/evo-server/main.cpp b/apps/evo-server/main.cpp index 2c5a261..db402df 100644 --- a/apps/evo-server/main.cpp +++ b/apps/evo-server/main.cpp @@ -22,9 +22,16 @@ // nothing about authority), /readyz (200 only with current authority) and /metrics // (Prometheus text). It prints "ops listening " once bound. // +// --synthetic-inputs development-v1 (dev/stage drills only, M8.6, ADR-0073) replaces +// stdin with a deterministic driver: whenever the host is running it keeps the +// backlog fed with development_input(t) for the ticks after the committed head, +// restarting from the committed tick after every recovery. Request IDs are +// "SYN1" + region + tick, so acknowledgements can be checked across owners. +// // --crash-at : (tests only, M7.7) SIGKILLs this process at the n-th // occurrence of before-upload, before-transaction, after-commit or after-ack (right // after a step's acknowledgements are written and flushed). +#include "evo/processes/development_opening.hpp" #include "evo/server/ops_http.hpp" #include "evo/server/region_host.hpp" @@ -61,6 +68,7 @@ struct Options { std::string crash_point; std::uint64_t crash_ordinal{}; std::string ops_listen; + bool synthetic{}; std::uint64_t objects_reserve_mib{1024}; }; @@ -164,6 +172,8 @@ std::optional parse(int argc, char** argv) { o.host.lease = {}; else if (flag == "--ops-listen") o.ops_listen = value; + else if (flag == "--synthetic-inputs" && value == "development-v1") + o.synthetic = true; else if (flag == "--objects-reserve-mib" && n && *n <= 1048576) o.objects_reserve_mib = *n; else if (flag == "--crash-at") { @@ -334,7 +344,18 @@ template int serve(Options& o) { return {404, "not found\n"}; }; LineReader reader; - bool input_open = true, ready = false; + bool input_open = !o.synthetic, ready = false; + // Synthetic driver state: the next tick to submit, reset at every recovery. + std::optional synthetic_next; + bool synthetic_running = false; + const auto synthetic_request = [&](std::uint64_t tick) { + persistence::Uuid id{'S', 'Y', 'N', '1'}; + for (unsigned i = 0; i < 4; ++i) + id[4 + i] = static_cast((o.host.key.region >> (8U * (3U - i))) & 0xFFU); + for (unsigned i = 0; i < 8; ++i) + id[8 + i] = static_cast((tick >> (8U * (7U - i))) & 0xFFU); + return id; + }; std::optional pending; std::optional phase; std::size_t max_queued = 0; @@ -383,6 +404,21 @@ template int serve(Options& o) { admission_name(admitted.error())); pending.reset(); } + if (o.synthetic && !drain_deadline) { + const auto& now_status = host.status(); + const bool running = now_status.phase == server::HostPhase::running && now_status.ready; + if (running && !synthetic_running) + synthetic_next = now_status.simulated_tick + 1; // Resume after the head. + synthetic_running = running; + while (running && synthetic_next && + *synthetic_next <= host.status().simulated_tick + 64) { + const server::TickInput in{synthetic_request(*synthetic_next), + processes::development_input(*synthetic_next)}; + if (!host.submit(in)) + break; // Full or no longer ready: continue next iteration. + ++*synthetic_next; + } + } const auto& s = host.step(std::chrono::steady_clock::now()); max_queued = std::max(max_queued, s.queued); // Checkpoint age restarts at recovery and at every confirmed commit. diff --git a/deploy/kubernetes/region/base/deployment.yaml b/deploy/kubernetes/region/base/deployment.yaml index 370bd6a..b31d897 100644 --- a/deploy/kubernetes/region/base/deployment.yaml +++ b/deploy/kubernetes/region/base/deployment.yaml @@ -55,10 +55,13 @@ spec: # Builds that wrote this region's history stay recoverable (ADR-0068). retained=() for build in $REGION_RETAINED; do retained+=(--retained "$build"); done - exec /usr/local/bin/evo-server --profile lineage "${retained[@]}" \ + # Synthetic load is set only by tools/ops/fault-drill.sh (ADR-0073). + inputs=() + if [ -n "$REGION_INPUTS" ]; then inputs=(--synthetic-inputs "$REGION_INPUTS"); fi + exec /usr/local/bin/evo-server --profile lineage "${retained[@]}" "${inputs[@]}" \ --world "$REGION_WORLD" --region "$REGION_ID" --frame "$REGION_FRAME" \ --pg-service /run/postgres/pg_service.conf --pg-passfile /tmp/pgpass \ - --objects /objects --objects-reserve-mib 1024 \ + --objects /objects --objects-reserve-mib "$REGION_RESERVE_MIB" \ --build "$REGION_BUILD" --lease-profile default \ --flush-ms 1000 --drain-ms 20000 --ops-listen 0.0.0.0:9100 env: @@ -68,6 +71,9 @@ spec: - {name: REGION_BUILD, value: set-by-overlay} # Earlier builds whose commits this region may still hold, space-separated. - {name: REGION_RETAINED, value: ""} + # Drill-only overrides; Git keeps no synthetic load and the 1024 MiB reserve. + - {name: REGION_INPUTS, value: ""} + - {name: REGION_RESERVE_MIB, value: "1024"} ports: - {name: ops, containerPort: 9100, protocol: TCP} readinessProbe: diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index f9da858..61c5ad8 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -643,3 +643,10 @@ M8.5 implemented and locally qualified (2026-09-30), under ADR-0072: - the database outbox kept (no broker); - bounded logs, and a cardinality and cost report. Alert delivery waits on the maintainer's choice of receiver. + +M8.6 implemented and locally qualified (2026-09-30), under ADR-0073: +- a drill-only synthetic input driver; +- `fault-drill.sh`, covering repeated forced kills, a full volume, an empty volume + and an operator-driven database outage; +- RTO and zero-lost-acknowledgement checks. +The in-cluster drill results wait on the maintainer. diff --git a/docs/STATUS.md b/docs/STATUS.md index e8dbe7b..d7e508a 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1083,3 +1083,25 @@ rotation. Monitoring costs about 1,300 series, roughly 11 MB a day. Next: sync the monitoring app, confirm the targets, and run the `EvoRegionDown` injection. M8.4's drill results are still to be recorded. + +## M8.6 fault drills (2026-09-30) + +ADR-0073 adds a drill-only synthetic driver, `evo-server --synthetic-inputs +development-v1`: +- it feeds deterministic inputs from the committed head after every recovery, + including the tick-20 death; +- its request IDs are `SYN1‖region‖tick`; +- it is off in Git, and the policy check enforces that. + +`tools/ops/fault-drill.sh` puts dev or stage under load and runs these scenarios: +- **kills:** three forced kills, measuring RTO and checking that no acknowledgement + is lost; +- **full:** a full volume, which must acknowledge nothing; +- **empty:** an empty volume, which must quarantine and never start a fresh world; +- **db-outage:** an operator-blocked database, after which a stale epoch never + resumes. + +It restores the Git configuration on exit. + +Earlier (M8.4 follow-up): `REGION_RETAINED` keeps earlier builds recoverable +across image upgrades. diff --git a/docs/adr/0073-fault-drills.md b/docs/adr/0073-fault-drills.md new file mode 100644 index 0000000..d8c75e4 --- /dev/null +++ b/docs/adr/0073-fault-drills.md @@ -0,0 +1,87 @@ +# ADR-0073: Synthetic load and fault drills in dev and stage + +## Status +Accepted for M8.6, 2026-09-30. It builds on: +- ADR-0064/0065 (host and recovery); +- ADR-0070 (the object reserve); +- ADR-0071 (the Deployment); +- ADR-0072 (alerts). + +## Context +M8.6 requires: +- dev/stage pod kills to restore the committed world automatically; +- a stale owner that can never resume; +- unavailable state that never initializes a fresh world. + +The evidence asked for covers: +- repeated pod kills and a stale process; +- a database or object-store outage, a partition, and a full disk; +- measured RTO, and zero lost acknowledgements. + +The named risk is a happy-path pod restart mistaken for disaster recovery. + +A region in the cluster receives no inputs: there is no gateway, and a pod's stdin +cannot be fed. It therefore never commits past genesis, and an idle region cannot +show lost acknowledgements. + +## Decision +**Synthetic load for drills.** `evo-server --synthetic-inputs development-v1` +replaces stdin with a deterministic driver. +- Whenever the host is running with authority, it keeps the bounded backlog fed + with `development_input(t)` (the qualification fixture's schedule, including + the consumer's death at tick 20). +- It continues from the committed head, and restarts from it after every + recovery. +- Request IDs are `SYN1 ‖ region ‖ tick`, so acknowledgements can be matched + across owners. + +Like `--crash-at`, it is a test driver: a real region's inputs come from its +gateway and are never invented. In the Deployment it is off (`REGION_INPUTS=""`), +and the policy check forbids enabling it in Git. `tools/ops/fault-drill.sh` turns +it on for the drill and restores the Git values on exit, including on failure. The +driver keeps the region committing continuously, and history is never deleted (no +GC), so drills are short. + +**Fault drill** (`tools/ops/fault-drill.sh [scenario]`; World0 is +refused before anything touches the cluster). It streams each pod's log for +acknowledgement accounting, and reads the recovered commit from the successor's +`phase running epoch=E committed=C` line. + +| Scenario | Fault | Required result | +|---|---|---| +| kills | Three forced pod deletions under load | Measured time to Ready (RTO). Every tick the dead pod acknowledged is at or below the committed head its successor recovers, and the epoch increases | +| full | `REGION_RESERVE_MIB` beyond any disk | The host recovers and simulates, but acknowledges nothing (the store refuses new objects). Restoring it resumes from the committed head with nothing lost | +| empty | The object volume swapped for an empty one | The host quarantines and never becomes Ready (missing payload), and never provisions a fresh world. Restoring it recovers a head at or past every earlier acknowledgement | +| db-outage | Operator blocks the database longer than the lease | The host stops being Ready and acknowledging. When the block lifts, it re-acquires under a newer epoch (the stale epoch never resumes) with nothing lost | + +The database outage is an operator step, for example dropping port 5432 from the +store node on the database VM. A NetworkPolicy change cannot sever an established +connection under Calico (existing flows are accepted by conntrack), so it would +not partition a live session. + +**RTO.** The forced-deletion RTO is expected to be lease-bound: about 20 s under the +production lease (M7.9 measured 19.6 s locally; the M8.4 drill measured 19.3 s in +dev). The 60 s target remains a proposal until M8.8 ratifies it. + +**Code evidence.** `region_load_tests` runs `evo-server` under synthetic load on real +PostgreSQL: +- it acknowledges contiguously past the tick-20 death with deterministic IDs; +- after a SIGKILL, no acknowledgement exceeds the committed head; +- the successor resumes at exactly committed + 1, with no repeated death; +- stdin is ignored in synthetic mode. + +## Consequences +- Every drill leaves permanent history in dev or stage: about 20 ticks a second, + with a commit about every second while loaded. Keep drills to minutes. +- Pod-kill evidence is worker-loss evidence only. Host loss and restoring from + backup are M8.7/M8.8. This is the distinction the named risk asks for. +- Argo shows the dev or stage application OutOfSync during a drill, and Synced + again once the script restores the Git values. + +## Alternatives rejected +- **A network input listener (gateway-lite) now.** Premature design of the + gateway's contract. +- **A synthetic load sidecar.** A pod's stdin cannot be fed from another container + without a shared pipe that the image does not have. +- **Partitioning with NetworkPolicy.** It does not cut established connections + under Calico. diff --git a/docs/adr/README.md b/docs/adr/README.md index 95b7cfd..7705435 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -83,3 +83,4 @@ design. No unadopted candidate is a link dependency of the proof. - [ADR-0070: Durable stores for dev and stage](0070-dev-stage-stores.md) - accepted for M8.3 external PostgreSQL per environment, Retain-class node-local object volumes, free-space reserve and in-cluster store qualification. - [ADR-0071: Region host deployment in dev and stage](0071-region-host-deployment.md) - accepted for M8.4 single-replica Recreate host, digest-as-build identity, idempotent provisioning hook and the region drill; no director yet. - [ADR-0072: Observability, alerting and background delivery](0072-observability.md) - accepted for M8.5 pinned AEvolution Prometheus/Alertmanager, 16 tested alert rules, bounded logs, database outbox over any broker, cardinality/cost report. +- [ADR-0073: Synthetic load and fault drills](0073-fault-drills.md) - accepted for M8.6 drill-only deterministic input driver, kill/full/empty/db-outage drill with RTO and zero-lost-acknowledgement checks. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index cd46425..093fffa 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -2860,3 +2860,25 @@ admission policy and the stage and World0 AppProjects, then ran - Image upgrade to `edb39f…`: completed with `REGION_RETAINED=4af72099…`, so the existing dev and stage heads, written by the earlier build, stay recoverable after the upgrade. + +## M8.6 synthetic load and fault drills - 2026-09-30 + +- `region_load_tests` (real PostgreSQL and real `evo-server`, fast lease): + - A synthetic owner acknowledged contiguously from tick 1 past tick 45, with + deterministic `SYN1` IDs. The consumer's death was emitted as an outcome. + - After a SIGKILL mid-stream, every acknowledgement was at or below the + committed head, and there was exactly one durable decision. + - The successor started at epoch 2 and acknowledged exactly committed + 1 onward, + with no repeated death. + - With synthetic inputs enabled, a line typed on stdin was ignored. +- `lineage-region-contracts`: `development_input(t)` equals the fixture's + `region_input(t, true)` for ticks 1–200. +- Mutation: resuming the driver at committed + 2 was caught by the contiguity check. +- Windows MSVC CTest 59/59. WSL Linux `checks` set: 59/59 in each configuration. + PostgreSQL suites passed in Release, analysis and ASan/UBSan. The format check + passed. +- `deploy-policy-check.py`: 67 objects, 0 failures. New rules: `REGION_INPUTS` + must be empty in Git, and `fault-drill.sh` must refuse World0 before kubectl. +- `fault-drill.sh` passed `bash -n`. It refuses `world0` and unknown scenarios + (exit 2) without calling kubectl. +- Not executed: the in-cluster drill (kills, full, empty, db-outage). diff --git a/docs/infrastructure/ENVIRONMENTS.md b/docs/infrastructure/ENVIRONMENTS.md index ae44bef..cc77a50 100644 --- a/docs/infrastructure/ENVIRONMENTS.md +++ b/docs/infrastructure/ENVIRONMENTS.md @@ -160,6 +160,28 @@ curl -s localhost:9090/api/v1/alerts # Watchdog Record the timestamps. The rule tests in CI already prove the thresholds. +## Fault drills (M8.6, ADR-0073) + +`bash tools/ops/fault-drill.sh dev all` enables synthetic load, then runs three +forced kills, a full volume and an empty volume. It restores the Git settings on +exit. + +For the database outage, run `bash tools/ops/fault-drill.sh dev db-outage` and +follow its prompt. On `pg-forge-01`, block the store node for at least 30 s, then +lift the block: +```sh +sudo iptables -I INPUT -p tcp --dport 5432 -s 172.10.1.135 -j DROP # rke2-worker-1 +sleep 40 +sudo iptables -D INPUT -p tcp --dport 5432 -s 172.10.1.135 -j DROP +``` +Record each `kill`, `full`, `empty` and `db-outage` line in VALIDATION.md. During +the drill, Prometheus should show: +- `EvoRegionObjectsFull` (full); +- `EvoRegionQuarantined` or `EvoRegionNotReady` (empty); +- `EvoRegionFenced` (db-outage). + +Keep drills short: synthetic load leaves permanent history. + ## Break-glass How membership is granted is not decided yet; see docs/BACKLOG.md. Until it is, diff --git a/include/evo/processes/development_opening.hpp b/include/evo/processes/development_opening.hpp index 1962efd..c62ada2 100644 --- a/include/evo/processes/development_opening.hpp +++ b/include/evo/processes/development_opening.hpp @@ -15,4 +15,8 @@ enum class DevelopmentOpeningError : std::uint8_t { identity, build }; // Built for `world` in frame development_opening_frame; the world must be nonzero. [[nodiscard]] std::expected, DevelopmentOpeningError> development_opening(std::uint64_t world); +// M8.6 synthetic load: the fixture's deterministic input for `tick` (source off every +// 7th tick, a bite at tick 16, the consumer's death at tick 20). Test/drill driver +// only; a real region's inputs come from its gateway and are never invented. +[[nodiscard]] EnvironmentalInput development_input(std::uint64_t tick) noexcept; } // namespace evo::processes diff --git a/src/processes/development_opening.cpp b/src/processes/development_opening.cpp index 7a9cc80..0a2e077 100644 --- a/src/processes/development_opening.cpp +++ b/src/processes/development_opening.cpp @@ -73,6 +73,17 @@ std::optional genome(std::uint64_t id, std::uint64_t parent) { } } // namespace +EnvironmentalInput development_input(std::uint64_t tick) noexcept { + EnvironmentalInput in; + in.temperature_mK = 290000; + in.source_enabled = tick % 7 == 0 ? 0 : 1; + if (tick == 16) + in.bite = {2, 2, 2, 2, 1}; + if (tick == 20) + in.terminal[1] = 1; + return in; +} + std::expected, DevelopmentOpeningError> development_opening(std::uint64_t world) { if (world == 0) diff --git a/tests/lineage_region_tests.cpp b/tests/lineage_region_tests.cpp index 137b82b..99a137f 100644 --- a/tests/lineage_region_tests.cpp +++ b/tests/lineage_region_tests.cpp @@ -176,6 +176,8 @@ void development_opening_matches_the_fixture() { CHECK((*elsewhere)->rebind_world(region_identity.world)); CHECK((*elsewhere)->snapshot() == lineage_region()->snapshot()); CHECK(!development_opening(0)); + for (std::uint64_t tick = 1; tick <= 200; ++tick) + CHECK(development_input(tick) == evo::test::region_input(tick, true)); CHECK(development_opening_frame == region_identity.frame); } } // namespace diff --git a/tests/postgres_tests.cpp b/tests/postgres_tests.cpp index 7aadef3..57e7c3f 100644 --- a/tests/postgres_tests.cpp +++ b/tests/postgres_tests.cpp @@ -12,6 +12,7 @@ void run_region_ops_tests(); void run_region_clone_tests(); void run_region_migration_tests(); void run_region_store_tests(); +void run_region_load_tests(); namespace { using namespace evo::persistence; using namespace std::chrono_literals; @@ -126,6 +127,8 @@ int main() { std::puts("region clone and history integration passed"); run_region_store_tests(); std::puts("region store qualification integration passed"); + run_region_load_tests(); + std::puts("region synthetic load integration passed"); run_region_migration_tests(); std::puts("region migration, barrier and qualification integration passed"); return 0; diff --git a/tests/region_load_tests.cpp b/tests/region_load_tests.cpp new file mode 100644 index 0000000..48cea68 --- /dev/null +++ b/tests/region_load_tests.cpp @@ -0,0 +1,104 @@ +// M8.6 synthetic load (ADR-0073) on real evo-server processes: the deterministic +// driver keeps a region busy across a death and a SIGKILL, every acknowledgement is +// durable, and the next owner resumes exactly after the committed head with no gap, +// repeat or lost acknowledgement. Run by tools/ci/postgres-check.sh. +#include "fixtures/server_process.hpp" +#include +#include +#include + +void run_region_load_tests(); + +namespace { +using namespace evo::test::process; +void check(bool value, std::source_location at = std::source_location::current()) { + if (!value) + throw std::runtime_error("region load test line " + std::to_string(at.line())); +} +#define CHECK(...) check(static_cast(__VA_ARGS__)) + +struct Ack { + std::string request; + std::uint64_t tick{}, sequence{}; +}; +std::vector ack_lines(const Child& c) { + std::vector out; + for (const auto& line : c.lines()) { + char request[40]; + unsigned long long tick = 0, sequence = 0; + if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &tick, &sequence) == 3) + out.push_back({request, tick, sequence}); + } + return out; +} +std::string synthetic_request(std::uint64_t region, std::uint64_t tick) { + char out[33]; + std::snprintf(out, sizeof out, "53594e31%08llx%016llx", + static_cast(region & 0xFFFFFFFFU), + static_cast(tick)); + return out; +} +bool acked_through(const Child& c, std::uint64_t tick) { + const auto acks = ack_lines(c); + return std::ranges::any_of(acks, [&](const Ack& a) { return a.tick >= tick; }); +} +// Acks are contiguous from `first`, in tick order, each with its deterministic ID. +void contiguous_from(const std::vector& acks, std::uint64_t first, std::uint64_t region) { + CHECK(!acks.empty() && acks.front().tick == first); + for (std::size_t i = 0; i < acks.size(); ++i) { + CHECK(acks[i].tick == first + i); + CHECK(acks[i].request == synthetic_request(region, acks[i].tick)); + } +} + +void synthetic_load_survives_a_kill() { + const auto r = provision(87, 0xE6); + const auto args = server_args(r, {"--synthetic-inputs", "development-v1"}); + std::uint64_t committed = 0; + { + Child first{args}; + CHECK(first.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); + CHECK(first.wait_for([&](const std::string&) { return acked_through(first, 45); }, 30s)); + CHECK(first.printed("outcome 20 1 11 ")); // The consumer's death, made durable. + first.signal(SIGKILL); // Mid-stream: inputs queued, a batch open. + CHECK(first.wait_exit(20s) == 128 + SIGKILL); + const auto acks = ack_lines(first); + contiguous_from(acks, 1, r.key.region); + const auto head = head_of(r.key); + committed = head.tick; + CHECK(acks.back().tick <= committed); // Nothing acknowledged was lost. + CHECK(decisions_of(r.key) == 1); + } + // The next owner waits out the lease, then resumes right after the head. + Child second{args}; + CHECK(second.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); + CHECK(second.printed("phase running epoch=2 committed=" + std::to_string(committed) + " ")); + CHECK(second.wait_for([&](const std::string&) { return acked_through(second, committed + 30); }, + 30s)); + second.signal(SIGTERM); + CHECK(second.wait_exit(20s) == 0); + const auto acks = ack_lines(second); + contiguous_from(acks, committed + 1, r.key.region); + const auto head = head_of(r.key); + CHECK(acks.back().tick <= head.tick && head.released); + CHECK(decisions_of(r.key) == 1); // The death is never repeated. +} + +// Stdin is not read in synthetic mode: typed inputs cannot interleave with the driver. +void synthetic_mode_ignores_stdin() { + const auto r = provision(88, 0xE5); + Child server{server_args(r, {"--synthetic-inputs", "development-v1"})}; + CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); + server.send(input_line(1, false, 0x77)); + CHECK(server.wait_for([&](const std::string&) { return acked_through(server, 10); }, 30s)); + for (const auto& a : ack_lines(server)) + CHECK(a.request.starts_with("53594e31")); + server.signal(SIGTERM); + CHECK(server.wait_exit(20s) == 0); +} +} // namespace + +void run_region_load_tests() { + synthetic_load_survives_a_kill(); + synthetic_mode_ignores_stdin(); +} diff --git a/tools/ci/deploy-policy-check.py b/tools/ci/deploy-policy-check.py index 606e074..a0cc9d1 100644 --- a/tools/ci/deploy-policy-check.py +++ b/tools/ci/deploy-policy-check.py @@ -217,6 +217,13 @@ for path, doc in objects: drain = [int(x) for x in __import__('re').findall(r'--drain-ms (\d+)', script)] check(len(drain) == 1 and pod.get('terminationGracePeriodSeconds', 30) * 1000 > drain[0], f'{path}: the termination grace period must exceed --drain-ms') +# M8.6 (ADR-0073): synthetic load is a drill-time override, never committed. +for path, doc in objects: + if path.startswith('deploy/kubernetes/region/') and doc['kind'] == 'Deployment': + for container in doc['spec']['template']['spec'].get('containers', []): + for variable in container.get('env', []): + if variable.get('name') == 'REGION_INPUTS': + check(variable.get('value') == '', f'{path}: synthetic inputs are never enabled in Git') for env in ('dev', 'stage'): overlay = f'deploy/kubernetes/region/{env}/' images = manifests.get(overlay + 'kustomization.yaml', [{}])[0].get('images', []) @@ -298,7 +305,7 @@ for path, doc in objects: # The cleanup tool refuses World0 before kubectl runs (no kubectl on PATH here). with tempfile.TemporaryDirectory() as empty: env = dict(os.environ, PATH=empty + os.pathsep + os.path.dirname(sys.executable)) - for tool in ('tools/ops/cleanup-environment.sh', 'tools/ops/region-drill.sh'): + for tool in ('tools/ops/cleanup-environment.sh', 'tools/ops/region-drill.sh', 'tools/ops/fault-drill.sh'): for bad in ('world0', 'evolution-world0', 'World0', 'prod', ''): result = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, tool), bad], capture_output=True, text=True, env=env) diff --git a/tools/ops/fault-drill.sh b/tools/ops/fault-drill.sh new file mode 100644 index 0000000..7cee41c --- /dev/null +++ b/tools/ops/fault-drill.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +# M8.6 fault drill (ADR-0073) for a disposable environment only. +# +# tools/ops/fault-drill.sh [kills|full|empty|db-outage|all] [--context ] +# +# Puts the region under synthetic load (REGION_INPUTS=development-v1), then runs: +# kills three forced pod deletions under load: time to Ready (RTO) and zero +# lost acknowledgements (every tick a dying pod acknowledged is at or +# below the committed head its successor recovers); +# full the object volume "fills" (REGION_RESERVE_MIB beyond any disk): the +# host keeps simulating but acknowledges nothing; restoring the reserve +# resumes from the committed head with nothing lost; +# empty the object volume is replaced by an empty one: the host must +# quarantine, never become ready and never start a fresh world; +# restoring the volume recovers the same committed head; +# db-outage interactive: the operator blocks the database for longer than the +# lease (e.g. drop port 5432 from the store node on the database VM); +# the host must stop being ready and acknowledge nothing, then after +# the block is lifted re-acquire under a newer epoch with nothing lost. +# `all` runs kills, full and empty (db-outage needs an operator). Every change is +# made with `kubectl set env` / `kubectl patch` on the evo-region Deployment and +# reverted on exit, including on failure, back to the Git values (so Argo returns to +# Synced); World0 is refused before anything touches the cluster. +set -euo pipefail +usage() { echo "usage: $0 [kills|full|empty|db-outage|all] [--context ]" >&2; exit 2; } +[ $# -ge 1 ] || usage +env="$1"; shift +case "$env" in + dev|stage) ;; + *) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2 + exit 2 ;; +esac +scenario="${1:-all}" +case "$scenario" in kills|full|empty|db-outage|all) shift || true ;; --context) scenario=all ;; *) usage ;; esac +k=(kubectl --request-timeout=30s) +[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2") +ns="evolution-$env" +selector=app.kubernetes.io/name=evo-region +work="$(mktemp -d)" +log_pids=() + +now_ms() { date +%s%3N; } +say() { echo "$(date -u +%H:%M:%S) $*"; } +fail() { say "FAIL $*"; exit 1; } +current_pods() { "${k[@]}" get pods -n "$ns" -l "$selector" \ + -o jsonpath='{range .items[?(!@.metadata.deletionTimestamp)]}{.metadata.name}{"\n"}{end}'; } +ready_pod() { # Waits up to $2 seconds (default 120) for a Ready pod other than $1. + local old="$1" limit="${2:-120}" + for _ in $(seq 1 $((limit * 5))); do + for p in $(current_pods); do + [ "$p" = "$old" ] && continue + [ "$("${k[@]}" get pod -n "$ns" "$p" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null)" = True ] && + { echo "$p"; return 0; } + done + sleep 0.2 + done + return 1 +} +metric() { # metric : from the pod's own /metrics (bash /dev/tcp). + "${k[@]}" exec -n "$ns" "$1" -- /bin/bash -c \ + 'exec 3<>/dev/tcp/127.0.0.1/9100; printf "GET /metrics HTTP/1.1\r\nHost: drill\r\n\r\n" >&3; cat <&3' 2>/dev/null | + awk -v s="$2" '$1 == s {print $2}' +} +capture() { # Streams a pod's log to $work/.log for acknowledgement accounting. + "${k[@]}" logs -n "$ns" -f "$1" > "$work/$1.log" 2>/dev/null & + log_pids+=($!) +} +max_ack() { awk '$1 == "ack" {if ($3 > m) m = $3} END {print m + 0}' "$work/$1.log"; } +recovered_at() { # committed tick from the pod's "phase running epoch=E committed=C" line. + sed -n 's/^phase running epoch=\([0-9]*\) committed=\([0-9]*\) .*/\1 \2/p' "$work/$1.log" | tail -1 +} +wait_for_line() { # wait_for_line + for _ in $(seq 1 $(($3 * 5))); do grep -qE "$2" "$work/$1.log" && return 0; sleep 0.2; done + return 1 +} +set_env() { "${k[@]}" set env -n "$ns" deployment/evo-region "$@" > /dev/null; } +restore() { + set +e + say "restoring the Git configuration of evo-region" + set_env REGION_INPUTS= REGION_RESERVE_MIB=1024 + "${k[@]}" get deployment -n "$ns" evo-region -o jsonpath='{.spec.template.spec.volumes[0].emptyDir}' | + grep -q . && "${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \ + '[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","persistentVolumeClaim":{"claimName":"aevolution-objects"}}}]' > /dev/null + for pid in "${log_pids[@]}"; do kill "$pid" 2>/dev/null; done + ready_pod "" 120 > /dev/null && say "restored: evo-region Ready" || say "WARN evo-region not Ready after restore" + say "logs kept in $work" +} +trap restore EXIT + +POD="" +loaded_pod() { # Sets POD to a Ready pod other than $1 once acknowledgements flow. + POD="$(ready_pod "${1:-}" 120)" || fail "no Ready evo-region pod" + capture "$POD" + wait_for_line "$POD" '^ack ' 60 || fail "$POD acknowledges nothing under load" +} + +say "enabling synthetic load in $ns" +before="$(current_pods | head -1)" +set_env REGION_INPUTS=development-v1 +loaded_pod "$before"; pod="$POD" +say "loaded pod=$pod ($(recovered_at "$pod"))" + +if [ "$scenario" = kills ] || [ "$scenario" = all ]; then + for round in 1 2 3; do + sleep 10 + acked="$(max_ack "$pod")" + start="$(now_ms)" + "${k[@]}" delete pod -n "$ns" "$pod" --grace-period=0 --force > /dev/null 2>&1 + loaded_pod "$pod"; next="$POD" + rto=$(( $(now_ms) - start )) + read -r epoch committed <<< "$(recovered_at "$next")" + say "kill round=$round rto_ms=$rto dead_acked_max=$acked recovered_committed=$committed epoch=$epoch pod=$next" + [ "$acked" -le "$committed" ] || fail "round $round lost acknowledgements ($acked > $committed)" + pod="$next" + done +fi + +if [ "$scenario" = full ] || [ "$scenario" = all ]; then + sleep 5 + acked="$(max_ack "$pod")" + set_env REGION_RESERVE_MIB=1048576 + full_pod="$(ready_pod "$pod" 120)" || fail "full: host did not recover (recovery needs no space)" + capture "$full_pod" + sleep 30 + new_acks="$(awk '$1 == "ack"' "$work/$full_pod.log" | wc -l)" + available="$(metric "$full_pod" evo_region_objects_available_bytes)" + reserve="$(metric "$full_pod" evo_region_objects_reserve_bytes)" + failures="$(metric "$full_pod" evo_region_store_failures_total)" + say "full acks_while_full=$new_acks available=$available reserve=$reserve store_failures=$failures" + [ "$new_acks" -eq 0 ] || fail "full: acknowledged while the volume was full" + set_env REGION_RESERVE_MIB=1024 + loaded_pod "$full_pod"; pod="$POD" + read -r epoch committed <<< "$(recovered_at "$pod")" + say "full restored committed=$committed (acked before=$acked) epoch=$epoch pod=$pod" + [ "$acked" -le "$committed" ] || fail "full: lost acknowledgements" +fi + +if [ "$scenario" = empty ] || [ "$scenario" = all ]; then + sleep 5 + acked="$(max_ack "$pod")" + "${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \ + '[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","emptyDir":{"sizeLimit":"64Mi"}}}]' > /dev/null + sleep 45 + if ready_pod "$pod" 1 > /dev/null; then fail "empty: a host became Ready on an empty volume"; fi + empty_pod="$(current_pods | grep -v "^$pod$" | head -1)" + quarantined="$("${k[@]}" logs -n "$ns" "$empty_pod" --previous 2>/dev/null | grep -c '^phase quarantined' || true)" + say "empty pod=$empty_pod ready=never quarantined_lines=$quarantined" + "${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \ + '[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","persistentVolumeClaim":{"claimName":"aevolution-objects"}}}]' > /dev/null + loaded_pod "$empty_pod"; pod="$POD" + read -r epoch committed <<< "$(recovered_at "$pod")" + say "empty restored committed=$committed (acked before=$acked) epoch=$epoch pod=$pod" + [ "$acked" -le "$committed" ] || fail "empty: the world was reset or acknowledgements lost" +fi + +if [ "$scenario" = db-outage ]; then + sleep 5 + epoch0="$(metric "$pod" evo_region_lease_epoch)" + say "block the database from the store node for at least 30 s now (lease 20 s), then lift it." + say "waiting for the host to lose readiness..." + for _ in $(seq 1 600); do [ "$(metric "$pod" evo_region_ready)" = 0 ] && break; sleep 0.5; done + [ "$(metric "$pod" evo_region_ready)" = 0 ] || fail "db-outage: host stayed ready" + lost="$(now_ms)" + acked="$(max_ack "$pod")" + say "host not ready; acknowledgements stopped at tick $acked. Waiting for recovery after the block is lifted..." + for _ in $(seq 1 1200); do [ "$(metric "$pod" evo_region_ready)" = 1 ] && break; sleep 0.5; done + [ "$(metric "$pod" evo_region_ready)" = 1 ] || fail "db-outage: host did not recover within 10 minutes" + epoch1="$(metric "$pod" evo_region_lease_epoch)" + committed="$(metric "$pod" evo_region_committed_tick)" + say "db-outage unavailable_ms=$(( $(now_ms) - lost )) epoch=$epoch0->$epoch1 committed=$committed acked_before=$acked fences=$(metric "$pod" evo_region_fences_total)" + [ "$epoch1" -gt "$epoch0" ] && [ "$acked" -le "$committed" ] || fail "db-outage: stale authority resumed or acknowledgements lost" +fi +say "PASS fault drill ($scenario) in $ns"