Add M8.6 synthetic load driver and fault drill

A drill-only evo-server --synthetic-inputs development-v1 driver feeds
deterministic inputs from the committed head after every recovery, with
request IDs that match acknowledgements across owners (ADR-0073); it is
off in Git and the policy check keeps it off. fault-drill.sh loads dev
or stage and runs repeated forced kills (RTO, zero lost
acknowledgements), a full object volume (nothing acknowledged), an
empty volume (quarantine, never a fresh world) and an operator-driven
database outage (stale epoch never resumes), restoring the Git
configuration on exit. Process tests prove contiguous acknowledgement
across a SIGKILL and exact resumption after the committed head.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-30 10:52:01 -04:00
co-authored by Claude Opus 5.5
parent 4ef0fe8167
commit 3f08a1d221
16 changed files with 513 additions and 5 deletions
+2 -1
View File
@@ -127,7 +127,8 @@ if(EVO_BUILD_POSTGRES)
tests/region_checkpoint_store_tests.cpp tests/region_host_tests.cpp
tests/region_server_process_tests.cpp tests/region_recovery_tests.cpp
tests/region_ops_tests.cpp tests/region_clone_tests.cpp
tests/region_migration_tests.cpp tests/region_store_tests.cpp)
tests/region_migration_tests.cpp tests/region_store_tests.cpp
tests/region_load_tests.cpp)
target_link_libraries(evo-postgres-tests PRIVATE evo::region_host Threads::Threads)
add_dependencies(evo-postgres-tests evo-server evo-world-clone) # Spawned by process tests.
evo_owned_target(evo-postgres-tests)
+37 -1
View File
@@ -22,9 +22,16 @@
// nothing about authority), /readyz (200 only with current authority) and /metrics
// (Prometheus text). It prints "ops listening <port>" once bound.
//
// --synthetic-inputs development-v1 (dev/stage drills only, M8.6, ADR-0073) replaces
// stdin with a deterministic driver: whenever the host is running it keeps the
// backlog fed with development_input(t) for the ticks after the committed head,
// restarting from the committed tick after every recovery. Request IDs are
// "SYN1" + region + tick, so acknowledgements can be checked across owners.
//
// --crash-at <point>:<n> (tests only, M7.7) SIGKILLs this process at the n-th
// occurrence of before-upload, before-transaction, after-commit or after-ack (right
// after a step's acknowledgements are written and flushed).
#include "evo/processes/development_opening.hpp"
#include "evo/server/ops_http.hpp"
#include "evo/server/region_host.hpp"
@@ -61,6 +68,7 @@ struct Options {
std::string crash_point;
std::uint64_t crash_ordinal{};
std::string ops_listen;
bool synthetic{};
std::uint64_t objects_reserve_mib{1024};
};
@@ -164,6 +172,8 @@ std::optional<Options> parse(int argc, char** argv) {
o.host.lease = {};
else if (flag == "--ops-listen")
o.ops_listen = value;
else if (flag == "--synthetic-inputs" && value == "development-v1")
o.synthetic = true;
else if (flag == "--objects-reserve-mib" && n && *n <= 1048576)
o.objects_reserve_mib = *n;
else if (flag == "--crash-at") {
@@ -334,7 +344,18 @@ template <typename State> int serve(Options& o) {
return {404, "not found\n"};
};
LineReader reader;
bool input_open = true, ready = false;
bool input_open = !o.synthetic, ready = false;
// Synthetic driver state: the next tick to submit, reset at every recovery.
std::optional<std::uint64_t> synthetic_next;
bool synthetic_running = false;
const auto synthetic_request = [&](std::uint64_t tick) {
persistence::Uuid id{'S', 'Y', 'N', '1'};
for (unsigned i = 0; i < 4; ++i)
id[4 + i] = static_cast<std::uint8_t>((o.host.key.region >> (8U * (3U - i))) & 0xFFU);
for (unsigned i = 0; i < 8; ++i)
id[8 + i] = static_cast<std::uint8_t>((tick >> (8U * (7U - i))) & 0xFFU);
return id;
};
std::optional<std::string> pending;
std::optional<server::HostPhase> phase;
std::size_t max_queued = 0;
@@ -383,6 +404,21 @@ template <typename State> int serve(Options& o) {
admission_name(admitted.error()));
pending.reset();
}
if (o.synthetic && !drain_deadline) {
const auto& now_status = host.status();
const bool running = now_status.phase == server::HostPhase::running && now_status.ready;
if (running && !synthetic_running)
synthetic_next = now_status.simulated_tick + 1; // Resume after the head.
synthetic_running = running;
while (running && synthetic_next &&
*synthetic_next <= host.status().simulated_tick + 64) {
const server::TickInput in{synthetic_request(*synthetic_next),
processes::development_input(*synthetic_next)};
if (!host.submit(in))
break; // Full or no longer ready: continue next iteration.
++*synthetic_next;
}
}
const auto& s = host.step(std::chrono::steady_clock::now());
max_queued = std::max(max_queued, s.queued);
// Checkpoint age restarts at recovery and at every confirmed commit.
@@ -55,10 +55,13 @@ spec:
# Builds that wrote this region's history stay recoverable (ADR-0068).
retained=()
for build in $REGION_RETAINED; do retained+=(--retained "$build"); done
exec /usr/local/bin/evo-server --profile lineage "${retained[@]}" \
# Synthetic load is set only by tools/ops/fault-drill.sh (ADR-0073).
inputs=()
if [ -n "$REGION_INPUTS" ]; then inputs=(--synthetic-inputs "$REGION_INPUTS"); fi
exec /usr/local/bin/evo-server --profile lineage "${retained[@]}" "${inputs[@]}" \
--world "$REGION_WORLD" --region "$REGION_ID" --frame "$REGION_FRAME" \
--pg-service /run/postgres/pg_service.conf --pg-passfile /tmp/pgpass \
--objects /objects --objects-reserve-mib 1024 \
--objects /objects --objects-reserve-mib "$REGION_RESERVE_MIB" \
--build "$REGION_BUILD" --lease-profile default \
--flush-ms 1000 --drain-ms 20000 --ops-listen 0.0.0.0:9100
env:
@@ -68,6 +71,9 @@ spec:
- {name: REGION_BUILD, value: set-by-overlay}
# Earlier builds whose commits this region may still hold, space-separated.
- {name: REGION_RETAINED, value: ""}
# Drill-only overrides; Git keeps no synthetic load and the 1024 MiB reserve.
- {name: REGION_INPUTS, value: ""}
- {name: REGION_RESERVE_MIB, value: "1024"}
ports:
- {name: ops, containerPort: 9100, protocol: TCP}
readinessProbe:
+7
View File
@@ -643,3 +643,10 @@ M8.5 implemented and locally qualified (2026-09-30), under ADR-0072:
- the database outbox kept (no broker);
- bounded logs, and a cardinality and cost report.
Alert delivery waits on the maintainer's choice of receiver.
M8.6 implemented and locally qualified (2026-09-30), under ADR-0073:
- a drill-only synthetic input driver;
- `fault-drill.sh`, covering repeated forced kills, a full volume, an empty volume
and an operator-driven database outage;
- RTO and zero-lost-acknowledgement checks.
The in-cluster drill results wait on the maintainer.
+22
View File
@@ -1083,3 +1083,25 @@ rotation. Monitoring costs about 1,300 series, roughly 11 MB a day.
Next: sync the monitoring app, confirm the targets, and run the `EvoRegionDown`
injection. M8.4's drill results are still to be recorded.
## M8.6 fault drills (2026-09-30)
ADR-0073 adds a drill-only synthetic driver, `evo-server --synthetic-inputs
development-v1`:
- it feeds deterministic inputs from the committed head after every recovery,
including the tick-20 death;
- its request IDs are `SYN1‖region‖tick`;
- it is off in Git, and the policy check enforces that.
`tools/ops/fault-drill.sh` puts dev or stage under load and runs these scenarios:
- **kills:** three forced kills, measuring RTO and checking that no acknowledgement
is lost;
- **full:** a full volume, which must acknowledge nothing;
- **empty:** an empty volume, which must quarantine and never start a fresh world;
- **db-outage:** an operator-blocked database, after which a stale epoch never
resumes.
It restores the Git configuration on exit.
Earlier (M8.4 follow-up): `REGION_RETAINED` keeps earlier builds recoverable
across image upgrades.
+87
View File
@@ -0,0 +1,87 @@
# ADR-0073: Synthetic load and fault drills in dev and stage
## Status
Accepted for M8.6, 2026-09-30. It builds on:
- ADR-0064/0065 (host and recovery);
- ADR-0070 (the object reserve);
- ADR-0071 (the Deployment);
- ADR-0072 (alerts).
## Context
M8.6 requires:
- dev/stage pod kills to restore the committed world automatically;
- a stale owner that can never resume;
- unavailable state that never initializes a fresh world.
The evidence asked for covers:
- repeated pod kills and a stale process;
- a database or object-store outage, a partition, and a full disk;
- measured RTO, and zero lost acknowledgements.
The named risk is a happy-path pod restart mistaken for disaster recovery.
A region in the cluster receives no inputs: there is no gateway, and a pod's stdin
cannot be fed. It therefore never commits past genesis, and an idle region cannot
show lost acknowledgements.
## Decision
**Synthetic load for drills.** `evo-server --synthetic-inputs development-v1`
replaces stdin with a deterministic driver.
- Whenever the host is running with authority, it keeps the bounded backlog fed
with `development_input(t)` (the qualification fixture's schedule, including
the consumer's death at tick 20).
- It continues from the committed head, and restarts from it after every
recovery.
- Request IDs are `SYN1 ‖ region ‖ tick`, so acknowledgements can be matched
across owners.
Like `--crash-at`, it is a test driver: a real region's inputs come from its
gateway and are never invented. In the Deployment it is off (`REGION_INPUTS=""`),
and the policy check forbids enabling it in Git. `tools/ops/fault-drill.sh` turns
it on for the drill and restores the Git values on exit, including on failure. The
driver keeps the region committing continuously, and history is never deleted (no
GC), so drills are short.
**Fault drill** (`tools/ops/fault-drill.sh <dev|stage> [scenario]`; World0 is
refused before anything touches the cluster). It streams each pod's log for
acknowledgement accounting, and reads the recovered commit from the successor's
`phase running epoch=E committed=C` line.
| Scenario | Fault | Required result |
|---|---|---|
| kills | Three forced pod deletions under load | Measured time to Ready (RTO). Every tick the dead pod acknowledged is at or below the committed head its successor recovers, and the epoch increases |
| full | `REGION_RESERVE_MIB` beyond any disk | The host recovers and simulates, but acknowledges nothing (the store refuses new objects). Restoring it resumes from the committed head with nothing lost |
| empty | The object volume swapped for an empty one | The host quarantines and never becomes Ready (missing payload), and never provisions a fresh world. Restoring it recovers a head at or past every earlier acknowledgement |
| db-outage | Operator blocks the database longer than the lease | The host stops being Ready and acknowledging. When the block lifts, it re-acquires under a newer epoch (the stale epoch never resumes) with nothing lost |
The database outage is an operator step, for example dropping port 5432 from the
store node on the database VM. A NetworkPolicy change cannot sever an established
connection under Calico (existing flows are accepted by conntrack), so it would
not partition a live session.
**RTO.** The forced-deletion RTO is expected to be lease-bound: about 20 s under the
production lease (M7.9 measured 19.6 s locally; the M8.4 drill measured 19.3 s in
dev). The 60 s target remains a proposal until M8.8 ratifies it.
**Code evidence.** `region_load_tests` runs `evo-server` under synthetic load on real
PostgreSQL:
- it acknowledges contiguously past the tick-20 death with deterministic IDs;
- after a SIGKILL, no acknowledgement exceeds the committed head;
- the successor resumes at exactly committed + 1, with no repeated death;
- stdin is ignored in synthetic mode.
## Consequences
- Every drill leaves permanent history in dev or stage: about 20 ticks a second,
with a commit about every second while loaded. Keep drills to minutes.
- Pod-kill evidence is worker-loss evidence only. Host loss and restoring from
backup are M8.7/M8.8. This is the distinction the named risk asks for.
- Argo shows the dev or stage application OutOfSync during a drill, and Synced
again once the script restores the Git values.
## Alternatives rejected
- **A network input listener (gateway-lite) now.** Premature design of the
gateway's contract.
- **A synthetic load sidecar.** A pod's stdin cannot be fed from another container
without a shared pipe that the image does not have.
- **Partitioning with NetworkPolicy.** It does not cut established connections
under Calico.
+1
View File
@@ -83,3 +83,4 @@ design. No unadopted candidate is a link dependency of the proof.
- [ADR-0070: Durable stores for dev and stage](0070-dev-stage-stores.md) - accepted for M8.3 external PostgreSQL per environment, Retain-class node-local object volumes, free-space reserve and in-cluster store qualification.
- [ADR-0071: Region host deployment in dev and stage](0071-region-host-deployment.md) - accepted for M8.4 single-replica Recreate host, digest-as-build identity, idempotent provisioning hook and the region drill; no director yet.
- [ADR-0072: Observability, alerting and background delivery](0072-observability.md) - accepted for M8.5 pinned AEvolution Prometheus/Alertmanager, 16 tested alert rules, bounded logs, database outbox over any broker, cardinality/cost report.
- [ADR-0073: Synthetic load and fault drills](0073-fault-drills.md) - accepted for M8.6 drill-only deterministic input driver, kill/full/empty/db-outage drill with RTO and zero-lost-acknowledgement checks.
+22
View File
@@ -2860,3 +2860,25 @@ admission policy and the stage and World0 AppProjects, then ran
- Image upgrade to `edb39f…`: completed with `REGION_RETAINED=4af72099…`, so the
existing dev and stage heads, written by the earlier build, stay recoverable
after the upgrade.
## M8.6 synthetic load and fault drills - 2026-09-30
- `region_load_tests` (real PostgreSQL and real `evo-server`, fast lease):
- A synthetic owner acknowledged contiguously from tick 1 past tick 45, with
deterministic `SYN1` IDs. The consumer's death was emitted as an outcome.
- After a SIGKILL mid-stream, every acknowledgement was at or below the
committed head, and there was exactly one durable decision.
- The successor started at epoch 2 and acknowledged exactly committed + 1 onward,
with no repeated death.
- With synthetic inputs enabled, a line typed on stdin was ignored.
- `lineage-region-contracts`: `development_input(t)` equals the fixture's
`region_input(t, true)` for ticks 1–200.
- Mutation: resuming the driver at committed + 2 was caught by the contiguity check.
- Windows MSVC CTest 59/59. WSL Linux `checks` set: 59/59 in each configuration.
PostgreSQL suites passed in Release, analysis and ASan/UBSan. The format check
passed.
- `deploy-policy-check.py`: 67 objects, 0 failures. New rules: `REGION_INPUTS`
must be empty in Git, and `fault-drill.sh` must refuse World0 before kubectl.
- `fault-drill.sh` passed `bash -n`. It refuses `world0` and unknown scenarios
(exit 2) without calling kubectl.
- Not executed: the in-cluster drill (kills, full, empty, db-outage).
+22
View File
@@ -160,6 +160,28 @@ curl -s localhost:9090/api/v1/alerts # Watchdog
Record the timestamps. The rule tests in CI already prove the thresholds.
## Fault drills (M8.6, ADR-0073)
`bash tools/ops/fault-drill.sh dev all` enables synthetic load, then runs three
forced kills, a full volume and an empty volume. It restores the Git settings on
exit.
For the database outage, run `bash tools/ops/fault-drill.sh dev db-outage` and
follow its prompt. On `pg-forge-01`, block the store node for at least 30 s, then
lift the block:
```sh
sudo iptables -I INPUT -p tcp --dport 5432 -s 172.10.1.135 -j DROP # rke2-worker-1
sleep 40
sudo iptables -D INPUT -p tcp --dport 5432 -s 172.10.1.135 -j DROP
```
Record each `kill`, `full`, `empty` and `db-outage` line in VALIDATION.md. During
the drill, Prometheus should show:
- `EvoRegionObjectsFull` (full);
- `EvoRegionQuarantined` or `EvoRegionNotReady` (empty);
- `EvoRegionFenced` (db-outage).
Keep drills short: synthetic load leaves permanent history.
## Break-glass
How membership is granted is not decided yet; see docs/BACKLOG.md. Until it is,
@@ -15,4 +15,8 @@ enum class DevelopmentOpeningError : std::uint8_t { identity, build };
// Built for `world` in frame development_opening_frame; the world must be nonzero.
[[nodiscard]] std::expected<std::unique_ptr<LineageRegion>, DevelopmentOpeningError>
development_opening(std::uint64_t world);
// M8.6 synthetic load: the fixture's deterministic input for `tick` (source off every
// 7th tick, a bite at tick 16, the consumer's death at tick 20). Test/drill driver
// only; a real region's inputs come from its gateway and are never invented.
[[nodiscard]] EnvironmentalInput development_input(std::uint64_t tick) noexcept;
} // namespace evo::processes
+11
View File
@@ -73,6 +73,17 @@ std::optional<biology::Genome> genome(std::uint64_t id, std::uint64_t parent) {
}
} // namespace
EnvironmentalInput development_input(std::uint64_t tick) noexcept {
EnvironmentalInput in;
in.temperature_mK = 290000;
in.source_enabled = tick % 7 == 0 ? 0 : 1;
if (tick == 16)
in.bite = {2, 2, 2, 2, 1};
if (tick == 20)
in.terminal[1] = 1;
return in;
}
std::expected<std::unique_ptr<LineageRegion>, DevelopmentOpeningError>
development_opening(std::uint64_t world) {
if (world == 0)
+2
View File
@@ -176,6 +176,8 @@ void development_opening_matches_the_fixture() {
CHECK((*elsewhere)->rebind_world(region_identity.world));
CHECK((*elsewhere)->snapshot() == lineage_region()->snapshot());
CHECK(!development_opening(0));
for (std::uint64_t tick = 1; tick <= 200; ++tick)
CHECK(development_input(tick) == evo::test::region_input(tick, true));
CHECK(development_opening_frame == region_identity.frame);
}
} // namespace
+3
View File
@@ -12,6 +12,7 @@ void run_region_ops_tests();
void run_region_clone_tests();
void run_region_migration_tests();
void run_region_store_tests();
void run_region_load_tests();
namespace {
using namespace evo::persistence;
using namespace std::chrono_literals;
@@ -126,6 +127,8 @@ int main() {
std::puts("region clone and history integration passed");
run_region_store_tests();
std::puts("region store qualification integration passed");
run_region_load_tests();
std::puts("region synthetic load integration passed");
run_region_migration_tests();
std::puts("region migration, barrier and qualification integration passed");
return 0;
+104
View File
@@ -0,0 +1,104 @@
// M8.6 synthetic load (ADR-0073) on real evo-server processes: the deterministic
// driver keeps a region busy across a death and a SIGKILL, every acknowledgement is
// durable, and the next owner resumes exactly after the committed head with no gap,
// repeat or lost acknowledgement. Run by tools/ci/postgres-check.sh.
#include "fixtures/server_process.hpp"
#include <algorithm>
#include <cstdio>
#include <source_location>
void run_region_load_tests();
namespace {
using namespace evo::test::process;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region load test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
struct Ack {
std::string request;
std::uint64_t tick{}, sequence{};
};
std::vector<Ack> ack_lines(const Child& c) {
std::vector<Ack> out;
for (const auto& line : c.lines()) {
char request[40];
unsigned long long tick = 0, sequence = 0;
if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &tick, &sequence) == 3)
out.push_back({request, tick, sequence});
}
return out;
}
std::string synthetic_request(std::uint64_t region, std::uint64_t tick) {
char out[33];
std::snprintf(out, sizeof out, "53594e31%08llx%016llx",
static_cast<unsigned long long>(region & 0xFFFFFFFFU),
static_cast<unsigned long long>(tick));
return out;
}
bool acked_through(const Child& c, std::uint64_t tick) {
const auto acks = ack_lines(c);
return std::ranges::any_of(acks, [&](const Ack& a) { return a.tick >= tick; });
}
// Acks are contiguous from `first`, in tick order, each with its deterministic ID.
void contiguous_from(const std::vector<Ack>& acks, std::uint64_t first, std::uint64_t region) {
CHECK(!acks.empty() && acks.front().tick == first);
for (std::size_t i = 0; i < acks.size(); ++i) {
CHECK(acks[i].tick == first + i);
CHECK(acks[i].request == synthetic_request(region, acks[i].tick));
}
}
void synthetic_load_survives_a_kill() {
const auto r = provision(87, 0xE6);
const auto args = server_args(r, {"--synthetic-inputs", "development-v1"});
std::uint64_t committed = 0;
{
Child first{args};
CHECK(first.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
CHECK(first.wait_for([&](const std::string&) { return acked_through(first, 45); }, 30s));
CHECK(first.printed("outcome 20 1 11 ")); // The consumer's death, made durable.
first.signal(SIGKILL); // Mid-stream: inputs queued, a batch open.
CHECK(first.wait_exit(20s) == 128 + SIGKILL);
const auto acks = ack_lines(first);
contiguous_from(acks, 1, r.key.region);
const auto head = head_of(r.key);
committed = head.tick;
CHECK(acks.back().tick <= committed); // Nothing acknowledged was lost.
CHECK(decisions_of(r.key) == 1);
}
// The next owner waits out the lease, then resumes right after the head.
Child second{args};
CHECK(second.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
CHECK(second.printed("phase running epoch=2 committed=" + std::to_string(committed) + " "));
CHECK(second.wait_for([&](const std::string&) { return acked_through(second, committed + 30); },
30s));
second.signal(SIGTERM);
CHECK(second.wait_exit(20s) == 0);
const auto acks = ack_lines(second);
contiguous_from(acks, committed + 1, r.key.region);
const auto head = head_of(r.key);
CHECK(acks.back().tick <= head.tick && head.released);
CHECK(decisions_of(r.key) == 1); // The death is never repeated.
}
// Stdin is not read in synthetic mode: typed inputs cannot interleave with the driver.
void synthetic_mode_ignores_stdin() {
const auto r = provision(88, 0xE5);
Child server{server_args(r, {"--synthetic-inputs", "development-v1"})};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
server.send(input_line(1, false, 0x77));
CHECK(server.wait_for([&](const std::string&) { return acked_through(server, 10); }, 30s));
for (const auto& a : ack_lines(server))
CHECK(a.request.starts_with("53594e31"));
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 0);
}
} // namespace
void run_region_load_tests() {
synthetic_load_survives_a_kill();
synthetic_mode_ignores_stdin();
}
+8 -1
View File
@@ -217,6 +217,13 @@ for path, doc in objects:
drain = [int(x) for x in __import__('re').findall(r'--drain-ms (\d+)', script)]
check(len(drain) == 1 and pod.get('terminationGracePeriodSeconds', 30) * 1000 > drain[0],
f'{path}: the termination grace period must exceed --drain-ms')
# M8.6 (ADR-0073): synthetic load is a drill-time override, never committed.
for path, doc in objects:
if path.startswith('deploy/kubernetes/region/') and doc['kind'] == 'Deployment':
for container in doc['spec']['template']['spec'].get('containers', []):
for variable in container.get('env', []):
if variable.get('name') == 'REGION_INPUTS':
check(variable.get('value') == '', f'{path}: synthetic inputs are never enabled in Git')
for env in ('dev', 'stage'):
overlay = f'deploy/kubernetes/region/{env}/'
images = manifests.get(overlay + 'kustomization.yaml', [{}])[0].get('images', [])
@@ -298,7 +305,7 @@ for path, doc in objects:
# The cleanup tool refuses World0 before kubectl runs (no kubectl on PATH here).
with tempfile.TemporaryDirectory() as empty:
env = dict(os.environ, PATH=empty + os.pathsep + os.path.dirname(sys.executable))
for tool in ('tools/ops/cleanup-environment.sh', 'tools/ops/region-drill.sh'):
for tool in ('tools/ops/cleanup-environment.sh', 'tools/ops/region-drill.sh', 'tools/ops/fault-drill.sh'):
for bad in ('world0', 'evolution-world0', 'World0', 'prod', ''):
result = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, tool), bad],
capture_output=True, text=True, env=env)
+173
View File
@@ -0,0 +1,173 @@
#!/usr/bin/env bash
# M8.6 fault drill (ADR-0073) for a disposable environment only.
#
# tools/ops/fault-drill.sh <dev|stage> [kills|full|empty|db-outage|all] [--context <name>]
#
# Puts the region under synthetic load (REGION_INPUTS=development-v1), then runs:
# kills three forced pod deletions under load: time to Ready (RTO) and zero
# lost acknowledgements (every tick a dying pod acknowledged is at or
# below the committed head its successor recovers);
# full the object volume "fills" (REGION_RESERVE_MIB beyond any disk): the
# host keeps simulating but acknowledges nothing; restoring the reserve
# resumes from the committed head with nothing lost;
# empty the object volume is replaced by an empty one: the host must
# quarantine, never become ready and never start a fresh world;
# restoring the volume recovers the same committed head;
# db-outage interactive: the operator blocks the database for longer than the
# lease (e.g. drop port 5432 from the store node on the database VM);
# the host must stop being ready and acknowledge nothing, then after
# the block is lifted re-acquire under a newer epoch with nothing lost.
# `all` runs kills, full and empty (db-outage needs an operator). Every change is
# made with `kubectl set env` / `kubectl patch` on the evo-region Deployment and
# reverted on exit, including on failure, back to the Git values (so Argo returns to
# Synced); World0 is refused before anything touches the cluster.
set -euo pipefail
usage() { echo "usage: $0 <dev|stage> [kills|full|empty|db-outage|all] [--context <name>]" >&2; exit 2; }
[ $# -ge 1 ] || usage
env="$1"; shift
case "$env" in
dev|stage) ;;
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
exit 2 ;;
esac
scenario="${1:-all}"
case "$scenario" in kills|full|empty|db-outage|all) shift || true ;; --context) scenario=all ;; *) usage ;; esac
k=(kubectl --request-timeout=30s)
[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2")
ns="evolution-$env"
selector=app.kubernetes.io/name=evo-region
work="$(mktemp -d)"
log_pids=()
now_ms() { date +%s%3N; }
say() { echo "$(date -u +%H:%M:%S) $*"; }
fail() { say "FAIL $*"; exit 1; }
current_pods() { "${k[@]}" get pods -n "$ns" -l "$selector" \
-o jsonpath='{range .items[?(!@.metadata.deletionTimestamp)]}{.metadata.name}{"\n"}{end}'; }
ready_pod() { # Waits up to $2 seconds (default 120) for a Ready pod other than $1.
local old="$1" limit="${2:-120}"
for _ in $(seq 1 $((limit * 5))); do
for p in $(current_pods); do
[ "$p" = "$old" ] && continue
[ "$("${k[@]}" get pod -n "$ns" "$p" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null)" = True ] &&
{ echo "$p"; return 0; }
done
sleep 0.2
done
return 1
}
metric() { # metric <pod> <series>: from the pod's own /metrics (bash /dev/tcp).
"${k[@]}" exec -n "$ns" "$1" -- /bin/bash -c \
'exec 3<>/dev/tcp/127.0.0.1/9100; printf "GET /metrics HTTP/1.1\r\nHost: drill\r\n\r\n" >&3; cat <&3' 2>/dev/null |
awk -v s="$2" '$1 == s {print $2}'
}
capture() { # Streams a pod's log to $work/<pod>.log for acknowledgement accounting.
"${k[@]}" logs -n "$ns" -f "$1" > "$work/$1.log" 2>/dev/null &
log_pids+=($!)
}
max_ack() { awk '$1 == "ack" {if ($3 > m) m = $3} END {print m + 0}' "$work/$1.log"; }
recovered_at() { # committed tick from the pod's "phase running epoch=E committed=C" line.
sed -n 's/^phase running epoch=\([0-9]*\) committed=\([0-9]*\) .*/\1 \2/p' "$work/$1.log" | tail -1
}
wait_for_line() { # wait_for_line <pod> <regex> <seconds>
for _ in $(seq 1 $(($3 * 5))); do grep -qE "$2" "$work/$1.log" && return 0; sleep 0.2; done
return 1
}
set_env() { "${k[@]}" set env -n "$ns" deployment/evo-region "$@" > /dev/null; }
restore() {
set +e
say "restoring the Git configuration of evo-region"
set_env REGION_INPUTS= REGION_RESERVE_MIB=1024
"${k[@]}" get deployment -n "$ns" evo-region -o jsonpath='{.spec.template.spec.volumes[0].emptyDir}' |
grep -q . && "${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \
'[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","persistentVolumeClaim":{"claimName":"aevolution-objects"}}}]' > /dev/null
for pid in "${log_pids[@]}"; do kill "$pid" 2>/dev/null; done
ready_pod "" 120 > /dev/null && say "restored: evo-region Ready" || say "WARN evo-region not Ready after restore"
say "logs kept in $work"
}
trap restore EXIT
POD=""
loaded_pod() { # Sets POD to a Ready pod other than $1 once acknowledgements flow.
POD="$(ready_pod "${1:-}" 120)" || fail "no Ready evo-region pod"
capture "$POD"
wait_for_line "$POD" '^ack ' 60 || fail "$POD acknowledges nothing under load"
}
say "enabling synthetic load in $ns"
before="$(current_pods | head -1)"
set_env REGION_INPUTS=development-v1
loaded_pod "$before"; pod="$POD"
say "loaded pod=$pod ($(recovered_at "$pod"))"
if [ "$scenario" = kills ] || [ "$scenario" = all ]; then
for round in 1 2 3; do
sleep 10
acked="$(max_ack "$pod")"
start="$(now_ms)"
"${k[@]}" delete pod -n "$ns" "$pod" --grace-period=0 --force > /dev/null 2>&1
loaded_pod "$pod"; next="$POD"
rto=$(( $(now_ms) - start ))
read -r epoch committed <<< "$(recovered_at "$next")"
say "kill round=$round rto_ms=$rto dead_acked_max=$acked recovered_committed=$committed epoch=$epoch pod=$next"
[ "$acked" -le "$committed" ] || fail "round $round lost acknowledgements ($acked > $committed)"
pod="$next"
done
fi
if [ "$scenario" = full ] || [ "$scenario" = all ]; then
sleep 5
acked="$(max_ack "$pod")"
set_env REGION_RESERVE_MIB=1048576
full_pod="$(ready_pod "$pod" 120)" || fail "full: host did not recover (recovery needs no space)"
capture "$full_pod"
sleep 30
new_acks="$(awk '$1 == "ack"' "$work/$full_pod.log" | wc -l)"
available="$(metric "$full_pod" evo_region_objects_available_bytes)"
reserve="$(metric "$full_pod" evo_region_objects_reserve_bytes)"
failures="$(metric "$full_pod" evo_region_store_failures_total)"
say "full acks_while_full=$new_acks available=$available reserve=$reserve store_failures=$failures"
[ "$new_acks" -eq 0 ] || fail "full: acknowledged while the volume was full"
set_env REGION_RESERVE_MIB=1024
loaded_pod "$full_pod"; pod="$POD"
read -r epoch committed <<< "$(recovered_at "$pod")"
say "full restored committed=$committed (acked before=$acked) epoch=$epoch pod=$pod"
[ "$acked" -le "$committed" ] || fail "full: lost acknowledgements"
fi
if [ "$scenario" = empty ] || [ "$scenario" = all ]; then
sleep 5
acked="$(max_ack "$pod")"
"${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \
'[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","emptyDir":{"sizeLimit":"64Mi"}}}]' > /dev/null
sleep 45
if ready_pod "$pod" 1 > /dev/null; then fail "empty: a host became Ready on an empty volume"; fi
empty_pod="$(current_pods | grep -v "^$pod$" | head -1)"
quarantined="$("${k[@]}" logs -n "$ns" "$empty_pod" --previous 2>/dev/null | grep -c '^phase quarantined' || true)"
say "empty pod=$empty_pod ready=never quarantined_lines=$quarantined"
"${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \
'[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","persistentVolumeClaim":{"claimName":"aevolution-objects"}}}]' > /dev/null
loaded_pod "$empty_pod"; pod="$POD"
read -r epoch committed <<< "$(recovered_at "$pod")"
say "empty restored committed=$committed (acked before=$acked) epoch=$epoch pod=$pod"
[ "$acked" -le "$committed" ] || fail "empty: the world was reset or acknowledgements lost"
fi
if [ "$scenario" = db-outage ]; then
sleep 5
epoch0="$(metric "$pod" evo_region_lease_epoch)"
say "block the database from the store node for at least 30 s now (lease 20 s), then lift it."
say "waiting for the host to lose readiness..."
for _ in $(seq 1 600); do [ "$(metric "$pod" evo_region_ready)" = 0 ] && break; sleep 0.5; done
[ "$(metric "$pod" evo_region_ready)" = 0 ] || fail "db-outage: host stayed ready"
lost="$(now_ms)"
acked="$(max_ack "$pod")"
say "host not ready; acknowledgements stopped at tick $acked. Waiting for recovery after the block is lifted..."
for _ in $(seq 1 1200); do [ "$(metric "$pod" evo_region_ready)" = 1 ] && break; sleep 0.5; done
[ "$(metric "$pod" evo_region_ready)" = 1 ] || fail "db-outage: host did not recover within 10 minutes"
epoch1="$(metric "$pod" evo_region_lease_epoch)"
committed="$(metric "$pod" evo_region_committed_tick)"
say "db-outage unavailable_ms=$(( $(now_ms) - lost )) epoch=$epoch0->$epoch1 committed=$committed acked_before=$acked fences=$(metric "$pod" evo_region_fences_total)"
[ "$epoch1" -gt "$epoch0" ] && [ "$acked" -le "$committed" ] || fail "db-outage: stale authority resumed or acknowledgements lost"
fi
say "PASS fault drill ($scenario) in $ns"