Add M8.6 synthetic load driver and fault drill
A drill-only evo-server --synthetic-inputs development-v1 driver feeds deterministic inputs from the committed head after every recovery, with request IDs that match acknowledgements across owners (ADR-0073); it is off in Git and the policy check keeps it off. fault-drill.sh loads dev or stage and runs repeated forced kills (RTO, zero lost acknowledgements), a full object volume (nothing acknowledged), an empty volume (quarantine, never a fresh world) and an operator-driven database outage (stale epoch never resumes), restoring the Git configuration on exit. Process tests prove contiguous acknowledgement across a SIGKILL and exact resumption after the committed head. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+2
-1
@@ -127,7 +127,8 @@ if(EVO_BUILD_POSTGRES)
|
||||
tests/region_checkpoint_store_tests.cpp tests/region_host_tests.cpp
|
||||
tests/region_server_process_tests.cpp tests/region_recovery_tests.cpp
|
||||
tests/region_ops_tests.cpp tests/region_clone_tests.cpp
|
||||
tests/region_migration_tests.cpp tests/region_store_tests.cpp)
|
||||
tests/region_migration_tests.cpp tests/region_store_tests.cpp
|
||||
tests/region_load_tests.cpp)
|
||||
target_link_libraries(evo-postgres-tests PRIVATE evo::region_host Threads::Threads)
|
||||
add_dependencies(evo-postgres-tests evo-server evo-world-clone) # Spawned by process tests.
|
||||
evo_owned_target(evo-postgres-tests)
|
||||
|
||||
@@ -22,9 +22,16 @@
|
||||
// nothing about authority), /readyz (200 only with current authority) and /metrics
|
||||
// (Prometheus text). It prints "ops listening <port>" once bound.
|
||||
//
|
||||
// --synthetic-inputs development-v1 (dev/stage drills only, M8.6, ADR-0073) replaces
|
||||
// stdin with a deterministic driver: whenever the host is running it keeps the
|
||||
// backlog fed with development_input(t) for the ticks after the committed head,
|
||||
// restarting from the committed tick after every recovery. Request IDs are
|
||||
// "SYN1" + region + tick, so acknowledgements can be checked across owners.
|
||||
//
|
||||
// --crash-at <point>:<n> (tests only, M7.7) SIGKILLs this process at the n-th
|
||||
// occurrence of before-upload, before-transaction, after-commit or after-ack (right
|
||||
// after a step's acknowledgements are written and flushed).
|
||||
#include "evo/processes/development_opening.hpp"
|
||||
#include "evo/server/ops_http.hpp"
|
||||
#include "evo/server/region_host.hpp"
|
||||
|
||||
@@ -61,6 +68,7 @@ struct Options {
|
||||
std::string crash_point;
|
||||
std::uint64_t crash_ordinal{};
|
||||
std::string ops_listen;
|
||||
bool synthetic{};
|
||||
std::uint64_t objects_reserve_mib{1024};
|
||||
};
|
||||
|
||||
@@ -164,6 +172,8 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
o.host.lease = {};
|
||||
else if (flag == "--ops-listen")
|
||||
o.ops_listen = value;
|
||||
else if (flag == "--synthetic-inputs" && value == "development-v1")
|
||||
o.synthetic = true;
|
||||
else if (flag == "--objects-reserve-mib" && n && *n <= 1048576)
|
||||
o.objects_reserve_mib = *n;
|
||||
else if (flag == "--crash-at") {
|
||||
@@ -334,7 +344,18 @@ template <typename State> int serve(Options& o) {
|
||||
return {404, "not found\n"};
|
||||
};
|
||||
LineReader reader;
|
||||
bool input_open = true, ready = false;
|
||||
bool input_open = !o.synthetic, ready = false;
|
||||
// Synthetic driver state: the next tick to submit, reset at every recovery.
|
||||
std::optional<std::uint64_t> synthetic_next;
|
||||
bool synthetic_running = false;
|
||||
const auto synthetic_request = [&](std::uint64_t tick) {
|
||||
persistence::Uuid id{'S', 'Y', 'N', '1'};
|
||||
for (unsigned i = 0; i < 4; ++i)
|
||||
id[4 + i] = static_cast<std::uint8_t>((o.host.key.region >> (8U * (3U - i))) & 0xFFU);
|
||||
for (unsigned i = 0; i < 8; ++i)
|
||||
id[8 + i] = static_cast<std::uint8_t>((tick >> (8U * (7U - i))) & 0xFFU);
|
||||
return id;
|
||||
};
|
||||
std::optional<std::string> pending;
|
||||
std::optional<server::HostPhase> phase;
|
||||
std::size_t max_queued = 0;
|
||||
@@ -383,6 +404,21 @@ template <typename State> int serve(Options& o) {
|
||||
admission_name(admitted.error()));
|
||||
pending.reset();
|
||||
}
|
||||
if (o.synthetic && !drain_deadline) {
|
||||
const auto& now_status = host.status();
|
||||
const bool running = now_status.phase == server::HostPhase::running && now_status.ready;
|
||||
if (running && !synthetic_running)
|
||||
synthetic_next = now_status.simulated_tick + 1; // Resume after the head.
|
||||
synthetic_running = running;
|
||||
while (running && synthetic_next &&
|
||||
*synthetic_next <= host.status().simulated_tick + 64) {
|
||||
const server::TickInput in{synthetic_request(*synthetic_next),
|
||||
processes::development_input(*synthetic_next)};
|
||||
if (!host.submit(in))
|
||||
break; // Full or no longer ready: continue next iteration.
|
||||
++*synthetic_next;
|
||||
}
|
||||
}
|
||||
const auto& s = host.step(std::chrono::steady_clock::now());
|
||||
max_queued = std::max(max_queued, s.queued);
|
||||
// Checkpoint age restarts at recovery and at every confirmed commit.
|
||||
|
||||
@@ -55,10 +55,13 @@ spec:
|
||||
# Builds that wrote this region's history stay recoverable (ADR-0068).
|
||||
retained=()
|
||||
for build in $REGION_RETAINED; do retained+=(--retained "$build"); done
|
||||
exec /usr/local/bin/evo-server --profile lineage "${retained[@]}" \
|
||||
# Synthetic load is set only by tools/ops/fault-drill.sh (ADR-0073).
|
||||
inputs=()
|
||||
if [ -n "$REGION_INPUTS" ]; then inputs=(--synthetic-inputs "$REGION_INPUTS"); fi
|
||||
exec /usr/local/bin/evo-server --profile lineage "${retained[@]}" "${inputs[@]}" \
|
||||
--world "$REGION_WORLD" --region "$REGION_ID" --frame "$REGION_FRAME" \
|
||||
--pg-service /run/postgres/pg_service.conf --pg-passfile /tmp/pgpass \
|
||||
--objects /objects --objects-reserve-mib 1024 \
|
||||
--objects /objects --objects-reserve-mib "$REGION_RESERVE_MIB" \
|
||||
--build "$REGION_BUILD" --lease-profile default \
|
||||
--flush-ms 1000 --drain-ms 20000 --ops-listen 0.0.0.0:9100
|
||||
env:
|
||||
@@ -68,6 +71,9 @@ spec:
|
||||
- {name: REGION_BUILD, value: set-by-overlay}
|
||||
# Earlier builds whose commits this region may still hold, space-separated.
|
||||
- {name: REGION_RETAINED, value: ""}
|
||||
# Drill-only overrides; Git keeps no synthetic load and the 1024 MiB reserve.
|
||||
- {name: REGION_INPUTS, value: ""}
|
||||
- {name: REGION_RESERVE_MIB, value: "1024"}
|
||||
ports:
|
||||
- {name: ops, containerPort: 9100, protocol: TCP}
|
||||
readinessProbe:
|
||||
|
||||
@@ -643,3 +643,10 @@ M8.5 implemented and locally qualified (2026-09-30), under ADR-0072:
|
||||
- the database outbox kept (no broker);
|
||||
- bounded logs, and a cardinality and cost report.
|
||||
Alert delivery waits on the maintainer's choice of receiver.
|
||||
|
||||
M8.6 implemented and locally qualified (2026-09-30), under ADR-0073:
|
||||
- a drill-only synthetic input driver;
|
||||
- `fault-drill.sh`, covering repeated forced kills, a full volume, an empty volume
|
||||
and an operator-driven database outage;
|
||||
- RTO and zero-lost-acknowledgement checks.
|
||||
The in-cluster drill results wait on the maintainer.
|
||||
|
||||
@@ -1083,3 +1083,25 @@ rotation. Monitoring costs about 1,300 series, roughly 11 MB a day.
|
||||
|
||||
Next: sync the monitoring app, confirm the targets, and run the `EvoRegionDown`
|
||||
injection. M8.4's drill results are still to be recorded.
|
||||
|
||||
## M8.6 fault drills (2026-09-30)
|
||||
|
||||
ADR-0073 adds a drill-only synthetic driver, `evo-server --synthetic-inputs
|
||||
development-v1`:
|
||||
- it feeds deterministic inputs from the committed head after every recovery,
|
||||
including the tick-20 death;
|
||||
- its request IDs are `SYN1‖region‖tick`;
|
||||
- it is off in Git, and the policy check enforces that.
|
||||
|
||||
`tools/ops/fault-drill.sh` puts dev or stage under load and runs these scenarios:
|
||||
- **kills:** three forced kills, measuring RTO and checking that no acknowledgement
|
||||
is lost;
|
||||
- **full:** a full volume, which must acknowledge nothing;
|
||||
- **empty:** an empty volume, which must quarantine and never start a fresh world;
|
||||
- **db-outage:** an operator-blocked database, after which a stale epoch never
|
||||
resumes.
|
||||
|
||||
It restores the Git configuration on exit.
|
||||
|
||||
Earlier (M8.4 follow-up): `REGION_RETAINED` keeps earlier builds recoverable
|
||||
across image upgrades.
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
# ADR-0073: Synthetic load and fault drills in dev and stage
|
||||
|
||||
## Status
|
||||
Accepted for M8.6, 2026-09-30. It builds on:
|
||||
- ADR-0064/0065 (host and recovery);
|
||||
- ADR-0070 (the object reserve);
|
||||
- ADR-0071 (the Deployment);
|
||||
- ADR-0072 (alerts).
|
||||
|
||||
## Context
|
||||
M8.6 requires:
|
||||
- dev/stage pod kills to restore the committed world automatically;
|
||||
- a stale owner that can never resume;
|
||||
- unavailable state that never initializes a fresh world.
|
||||
|
||||
The evidence asked for covers:
|
||||
- repeated pod kills and a stale process;
|
||||
- a database or object-store outage, a partition, and a full disk;
|
||||
- measured RTO, and zero lost acknowledgements.
|
||||
|
||||
The named risk is a happy-path pod restart mistaken for disaster recovery.
|
||||
|
||||
A region in the cluster receives no inputs: there is no gateway, and a pod's stdin
|
||||
cannot be fed. It therefore never commits past genesis, and an idle region cannot
|
||||
show lost acknowledgements.
|
||||
|
||||
## Decision
|
||||
**Synthetic load for drills.** `evo-server --synthetic-inputs development-v1`
|
||||
replaces stdin with a deterministic driver.
|
||||
- Whenever the host is running with authority, it keeps the bounded backlog fed
|
||||
with `development_input(t)` (the qualification fixture's schedule, including
|
||||
the consumer's death at tick 20).
|
||||
- It continues from the committed head, and restarts from it after every
|
||||
recovery.
|
||||
- Request IDs are `SYN1 ‖ region ‖ tick`, so acknowledgements can be matched
|
||||
across owners.
|
||||
|
||||
Like `--crash-at`, it is a test driver: a real region's inputs come from its
|
||||
gateway and are never invented. In the Deployment it is off (`REGION_INPUTS=""`),
|
||||
and the policy check forbids enabling it in Git. `tools/ops/fault-drill.sh` turns
|
||||
it on for the drill and restores the Git values on exit, including on failure. The
|
||||
driver keeps the region committing continuously, and history is never deleted (no
|
||||
GC), so drills are short.
|
||||
|
||||
**Fault drill** (`tools/ops/fault-drill.sh <dev|stage> [scenario]`; World0 is
|
||||
refused before anything touches the cluster). It streams each pod's log for
|
||||
acknowledgement accounting, and reads the recovered commit from the successor's
|
||||
`phase running epoch=E committed=C` line.
|
||||
|
||||
| Scenario | Fault | Required result |
|
||||
|---|---|---|
|
||||
| kills | Three forced pod deletions under load | Measured time to Ready (RTO). Every tick the dead pod acknowledged is at or below the committed head its successor recovers, and the epoch increases |
|
||||
| full | `REGION_RESERVE_MIB` beyond any disk | The host recovers and simulates, but acknowledges nothing (the store refuses new objects). Restoring it resumes from the committed head with nothing lost |
|
||||
| empty | The object volume swapped for an empty one | The host quarantines and never becomes Ready (missing payload), and never provisions a fresh world. Restoring it recovers a head at or past every earlier acknowledgement |
|
||||
| db-outage | Operator blocks the database longer than the lease | The host stops being Ready and acknowledging. When the block lifts, it re-acquires under a newer epoch (the stale epoch never resumes) with nothing lost |
|
||||
|
||||
The database outage is an operator step, for example dropping port 5432 from the
|
||||
store node on the database VM. A NetworkPolicy change cannot sever an established
|
||||
connection under Calico (existing flows are accepted by conntrack), so it would
|
||||
not partition a live session.
|
||||
|
||||
**RTO.** The forced-deletion RTO is expected to be lease-bound: about 20 s under the
|
||||
production lease (M7.9 measured 19.6 s locally; the M8.4 drill measured 19.3 s in
|
||||
dev). The 60 s target remains a proposal until M8.8 ratifies it.
|
||||
|
||||
**Code evidence.** `region_load_tests` runs `evo-server` under synthetic load on real
|
||||
PostgreSQL:
|
||||
- it acknowledges contiguously past the tick-20 death with deterministic IDs;
|
||||
- after a SIGKILL, no acknowledgement exceeds the committed head;
|
||||
- the successor resumes at exactly committed + 1, with no repeated death;
|
||||
- stdin is ignored in synthetic mode.
|
||||
|
||||
## Consequences
|
||||
- Every drill leaves permanent history in dev or stage: about 20 ticks a second,
|
||||
with a commit about every second while loaded. Keep drills to minutes.
|
||||
- Pod-kill evidence is worker-loss evidence only. Host loss and restoring from
|
||||
backup are M8.7/M8.8. This is the distinction the named risk asks for.
|
||||
- Argo shows the dev or stage application OutOfSync during a drill, and Synced
|
||||
again once the script restores the Git values.
|
||||
|
||||
## Alternatives rejected
|
||||
- **A network input listener (gateway-lite) now.** Premature design of the
|
||||
gateway's contract.
|
||||
- **A synthetic load sidecar.** A pod's stdin cannot be fed from another container
|
||||
without a shared pipe that the image does not have.
|
||||
- **Partitioning with NetworkPolicy.** It does not cut established connections
|
||||
under Calico.
|
||||
@@ -83,3 +83,4 @@ design. No unadopted candidate is a link dependency of the proof.
|
||||
- [ADR-0070: Durable stores for dev and stage](0070-dev-stage-stores.md) - accepted for M8.3 external PostgreSQL per environment, Retain-class node-local object volumes, free-space reserve and in-cluster store qualification.
|
||||
- [ADR-0071: Region host deployment in dev and stage](0071-region-host-deployment.md) - accepted for M8.4 single-replica Recreate host, digest-as-build identity, idempotent provisioning hook and the region drill; no director yet.
|
||||
- [ADR-0072: Observability, alerting and background delivery](0072-observability.md) - accepted for M8.5 pinned AEvolution Prometheus/Alertmanager, 16 tested alert rules, bounded logs, database outbox over any broker, cardinality/cost report.
|
||||
- [ADR-0073: Synthetic load and fault drills](0073-fault-drills.md) - accepted for M8.6 drill-only deterministic input driver, kill/full/empty/db-outage drill with RTO and zero-lost-acknowledgement checks.
|
||||
|
||||
@@ -2860,3 +2860,25 @@ admission policy and the stage and World0 AppProjects, then ran
|
||||
- Image upgrade to `edb39f…`: completed with `REGION_RETAINED=4af72099…`, so the
|
||||
existing dev and stage heads, written by the earlier build, stay recoverable
|
||||
after the upgrade.
|
||||
|
||||
## M8.6 synthetic load and fault drills - 2026-09-30
|
||||
|
||||
- `region_load_tests` (real PostgreSQL and real `evo-server`, fast lease):
|
||||
- A synthetic owner acknowledged contiguously from tick 1 past tick 45, with
|
||||
deterministic `SYN1` IDs. The consumer's death was emitted as an outcome.
|
||||
- After a SIGKILL mid-stream, every acknowledgement was at or below the
|
||||
committed head, and there was exactly one durable decision.
|
||||
- The successor started at epoch 2 and acknowledged exactly committed + 1 onward,
|
||||
with no repeated death.
|
||||
- With synthetic inputs enabled, a line typed on stdin was ignored.
|
||||
- `lineage-region-contracts`: `development_input(t)` equals the fixture's
|
||||
`region_input(t, true)` for ticks 1–200.
|
||||
- Mutation: resuming the driver at committed + 2 was caught by the contiguity check.
|
||||
- Windows MSVC CTest 59/59. WSL Linux `checks` set: 59/59 in each configuration.
|
||||
PostgreSQL suites passed in Release, analysis and ASan/UBSan. The format check
|
||||
passed.
|
||||
- `deploy-policy-check.py`: 67 objects, 0 failures. New rules: `REGION_INPUTS`
|
||||
must be empty in Git, and `fault-drill.sh` must refuse World0 before kubectl.
|
||||
- `fault-drill.sh` passed `bash -n`. It refuses `world0` and unknown scenarios
|
||||
(exit 2) without calling kubectl.
|
||||
- Not executed: the in-cluster drill (kills, full, empty, db-outage).
|
||||
|
||||
@@ -160,6 +160,28 @@ curl -s localhost:9090/api/v1/alerts # Watchdog
|
||||
|
||||
Record the timestamps. The rule tests in CI already prove the thresholds.
|
||||
|
||||
## Fault drills (M8.6, ADR-0073)
|
||||
|
||||
`bash tools/ops/fault-drill.sh dev all` enables synthetic load, then runs three
|
||||
forced kills, a full volume and an empty volume. It restores the Git settings on
|
||||
exit.
|
||||
|
||||
For the database outage, run `bash tools/ops/fault-drill.sh dev db-outage` and
|
||||
follow its prompt. On `pg-forge-01`, block the store node for at least 30 s, then
|
||||
lift the block:
|
||||
```sh
|
||||
sudo iptables -I INPUT -p tcp --dport 5432 -s 172.10.1.135 -j DROP # rke2-worker-1
|
||||
sleep 40
|
||||
sudo iptables -D INPUT -p tcp --dport 5432 -s 172.10.1.135 -j DROP
|
||||
```
|
||||
Record each `kill`, `full`, `empty` and `db-outage` line in VALIDATION.md. During
|
||||
the drill, Prometheus should show:
|
||||
- `EvoRegionObjectsFull` (full);
|
||||
- `EvoRegionQuarantined` or `EvoRegionNotReady` (empty);
|
||||
- `EvoRegionFenced` (db-outage).
|
||||
|
||||
Keep drills short: synthetic load leaves permanent history.
|
||||
|
||||
## Break-glass
|
||||
|
||||
How membership is granted is not decided yet; see docs/BACKLOG.md. Until it is,
|
||||
|
||||
@@ -15,4 +15,8 @@ enum class DevelopmentOpeningError : std::uint8_t { identity, build };
|
||||
// Built for `world` in frame development_opening_frame; the world must be nonzero.
|
||||
[[nodiscard]] std::expected<std::unique_ptr<LineageRegion>, DevelopmentOpeningError>
|
||||
development_opening(std::uint64_t world);
|
||||
// M8.6 synthetic load: the fixture's deterministic input for `tick` (source off every
|
||||
// 7th tick, a bite at tick 16, the consumer's death at tick 20). Test/drill driver
|
||||
// only; a real region's inputs come from its gateway and are never invented.
|
||||
[[nodiscard]] EnvironmentalInput development_input(std::uint64_t tick) noexcept;
|
||||
} // namespace evo::processes
|
||||
|
||||
@@ -73,6 +73,17 @@ std::optional<biology::Genome> genome(std::uint64_t id, std::uint64_t parent) {
|
||||
}
|
||||
} // namespace
|
||||
|
||||
EnvironmentalInput development_input(std::uint64_t tick) noexcept {
|
||||
EnvironmentalInput in;
|
||||
in.temperature_mK = 290000;
|
||||
in.source_enabled = tick % 7 == 0 ? 0 : 1;
|
||||
if (tick == 16)
|
||||
in.bite = {2, 2, 2, 2, 1};
|
||||
if (tick == 20)
|
||||
in.terminal[1] = 1;
|
||||
return in;
|
||||
}
|
||||
|
||||
std::expected<std::unique_ptr<LineageRegion>, DevelopmentOpeningError>
|
||||
development_opening(std::uint64_t world) {
|
||||
if (world == 0)
|
||||
|
||||
@@ -176,6 +176,8 @@ void development_opening_matches_the_fixture() {
|
||||
CHECK((*elsewhere)->rebind_world(region_identity.world));
|
||||
CHECK((*elsewhere)->snapshot() == lineage_region()->snapshot());
|
||||
CHECK(!development_opening(0));
|
||||
for (std::uint64_t tick = 1; tick <= 200; ++tick)
|
||||
CHECK(development_input(tick) == evo::test::region_input(tick, true));
|
||||
CHECK(development_opening_frame == region_identity.frame);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
@@ -12,6 +12,7 @@ void run_region_ops_tests();
|
||||
void run_region_clone_tests();
|
||||
void run_region_migration_tests();
|
||||
void run_region_store_tests();
|
||||
void run_region_load_tests();
|
||||
namespace {
|
||||
using namespace evo::persistence;
|
||||
using namespace std::chrono_literals;
|
||||
@@ -126,6 +127,8 @@ int main() {
|
||||
std::puts("region clone and history integration passed");
|
||||
run_region_store_tests();
|
||||
std::puts("region store qualification integration passed");
|
||||
run_region_load_tests();
|
||||
std::puts("region synthetic load integration passed");
|
||||
run_region_migration_tests();
|
||||
std::puts("region migration, barrier and qualification integration passed");
|
||||
return 0;
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
// M8.6 synthetic load (ADR-0073) on real evo-server processes: the deterministic
|
||||
// driver keeps a region busy across a death and a SIGKILL, every acknowledgement is
|
||||
// durable, and the next owner resumes exactly after the committed head with no gap,
|
||||
// repeat or lost acknowledgement. Run by tools/ci/postgres-check.sh.
|
||||
#include "fixtures/server_process.hpp"
|
||||
#include <algorithm>
|
||||
#include <cstdio>
|
||||
#include <source_location>
|
||||
|
||||
void run_region_load_tests();
|
||||
|
||||
namespace {
|
||||
using namespace evo::test::process;
|
||||
void check(bool value, std::source_location at = std::source_location::current()) {
|
||||
if (!value)
|
||||
throw std::runtime_error("region load test line " + std::to_string(at.line()));
|
||||
}
|
||||
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
||||
|
||||
struct Ack {
|
||||
std::string request;
|
||||
std::uint64_t tick{}, sequence{};
|
||||
};
|
||||
std::vector<Ack> ack_lines(const Child& c) {
|
||||
std::vector<Ack> out;
|
||||
for (const auto& line : c.lines()) {
|
||||
char request[40];
|
||||
unsigned long long tick = 0, sequence = 0;
|
||||
if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &tick, &sequence) == 3)
|
||||
out.push_back({request, tick, sequence});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
std::string synthetic_request(std::uint64_t region, std::uint64_t tick) {
|
||||
char out[33];
|
||||
std::snprintf(out, sizeof out, "53594e31%08llx%016llx",
|
||||
static_cast<unsigned long long>(region & 0xFFFFFFFFU),
|
||||
static_cast<unsigned long long>(tick));
|
||||
return out;
|
||||
}
|
||||
bool acked_through(const Child& c, std::uint64_t tick) {
|
||||
const auto acks = ack_lines(c);
|
||||
return std::ranges::any_of(acks, [&](const Ack& a) { return a.tick >= tick; });
|
||||
}
|
||||
// Acks are contiguous from `first`, in tick order, each with its deterministic ID.
|
||||
void contiguous_from(const std::vector<Ack>& acks, std::uint64_t first, std::uint64_t region) {
|
||||
CHECK(!acks.empty() && acks.front().tick == first);
|
||||
for (std::size_t i = 0; i < acks.size(); ++i) {
|
||||
CHECK(acks[i].tick == first + i);
|
||||
CHECK(acks[i].request == synthetic_request(region, acks[i].tick));
|
||||
}
|
||||
}
|
||||
|
||||
void synthetic_load_survives_a_kill() {
|
||||
const auto r = provision(87, 0xE6);
|
||||
const auto args = server_args(r, {"--synthetic-inputs", "development-v1"});
|
||||
std::uint64_t committed = 0;
|
||||
{
|
||||
Child first{args};
|
||||
CHECK(first.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
||||
CHECK(first.wait_for([&](const std::string&) { return acked_through(first, 45); }, 30s));
|
||||
CHECK(first.printed("outcome 20 1 11 ")); // The consumer's death, made durable.
|
||||
first.signal(SIGKILL); // Mid-stream: inputs queued, a batch open.
|
||||
CHECK(first.wait_exit(20s) == 128 + SIGKILL);
|
||||
const auto acks = ack_lines(first);
|
||||
contiguous_from(acks, 1, r.key.region);
|
||||
const auto head = head_of(r.key);
|
||||
committed = head.tick;
|
||||
CHECK(acks.back().tick <= committed); // Nothing acknowledged was lost.
|
||||
CHECK(decisions_of(r.key) == 1);
|
||||
}
|
||||
// The next owner waits out the lease, then resumes right after the head.
|
||||
Child second{args};
|
||||
CHECK(second.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
||||
CHECK(second.printed("phase running epoch=2 committed=" + std::to_string(committed) + " "));
|
||||
CHECK(second.wait_for([&](const std::string&) { return acked_through(second, committed + 30); },
|
||||
30s));
|
||||
second.signal(SIGTERM);
|
||||
CHECK(second.wait_exit(20s) == 0);
|
||||
const auto acks = ack_lines(second);
|
||||
contiguous_from(acks, committed + 1, r.key.region);
|
||||
const auto head = head_of(r.key);
|
||||
CHECK(acks.back().tick <= head.tick && head.released);
|
||||
CHECK(decisions_of(r.key) == 1); // The death is never repeated.
|
||||
}
|
||||
|
||||
// Stdin is not read in synthetic mode: typed inputs cannot interleave with the driver.
|
||||
void synthetic_mode_ignores_stdin() {
|
||||
const auto r = provision(88, 0xE5);
|
||||
Child server{server_args(r, {"--synthetic-inputs", "development-v1"})};
|
||||
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
||||
server.send(input_line(1, false, 0x77));
|
||||
CHECK(server.wait_for([&](const std::string&) { return acked_through(server, 10); }, 30s));
|
||||
for (const auto& a : ack_lines(server))
|
||||
CHECK(a.request.starts_with("53594e31"));
|
||||
server.signal(SIGTERM);
|
||||
CHECK(server.wait_exit(20s) == 0);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void run_region_load_tests() {
|
||||
synthetic_load_survives_a_kill();
|
||||
synthetic_mode_ignores_stdin();
|
||||
}
|
||||
@@ -217,6 +217,13 @@ for path, doc in objects:
|
||||
drain = [int(x) for x in __import__('re').findall(r'--drain-ms (\d+)', script)]
|
||||
check(len(drain) == 1 and pod.get('terminationGracePeriodSeconds', 30) * 1000 > drain[0],
|
||||
f'{path}: the termination grace period must exceed --drain-ms')
|
||||
# M8.6 (ADR-0073): synthetic load is a drill-time override, never committed.
|
||||
for path, doc in objects:
|
||||
if path.startswith('deploy/kubernetes/region/') and doc['kind'] == 'Deployment':
|
||||
for container in doc['spec']['template']['spec'].get('containers', []):
|
||||
for variable in container.get('env', []):
|
||||
if variable.get('name') == 'REGION_INPUTS':
|
||||
check(variable.get('value') == '', f'{path}: synthetic inputs are never enabled in Git')
|
||||
for env in ('dev', 'stage'):
|
||||
overlay = f'deploy/kubernetes/region/{env}/'
|
||||
images = manifests.get(overlay + 'kustomization.yaml', [{}])[0].get('images', [])
|
||||
@@ -298,7 +305,7 @@ for path, doc in objects:
|
||||
# The cleanup tool refuses World0 before kubectl runs (no kubectl on PATH here).
|
||||
with tempfile.TemporaryDirectory() as empty:
|
||||
env = dict(os.environ, PATH=empty + os.pathsep + os.path.dirname(sys.executable))
|
||||
for tool in ('tools/ops/cleanup-environment.sh', 'tools/ops/region-drill.sh'):
|
||||
for tool in ('tools/ops/cleanup-environment.sh', 'tools/ops/region-drill.sh', 'tools/ops/fault-drill.sh'):
|
||||
for bad in ('world0', 'evolution-world0', 'World0', 'prod', ''):
|
||||
result = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, tool), bad],
|
||||
capture_output=True, text=True, env=env)
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env bash
|
||||
# M8.6 fault drill (ADR-0073) for a disposable environment only.
|
||||
#
|
||||
# tools/ops/fault-drill.sh <dev|stage> [kills|full|empty|db-outage|all] [--context <name>]
|
||||
#
|
||||
# Puts the region under synthetic load (REGION_INPUTS=development-v1), then runs:
|
||||
# kills three forced pod deletions under load: time to Ready (RTO) and zero
|
||||
# lost acknowledgements (every tick a dying pod acknowledged is at or
|
||||
# below the committed head its successor recovers);
|
||||
# full the object volume "fills" (REGION_RESERVE_MIB beyond any disk): the
|
||||
# host keeps simulating but acknowledges nothing; restoring the reserve
|
||||
# resumes from the committed head with nothing lost;
|
||||
# empty the object volume is replaced by an empty one: the host must
|
||||
# quarantine, never become ready and never start a fresh world;
|
||||
# restoring the volume recovers the same committed head;
|
||||
# db-outage interactive: the operator blocks the database for longer than the
|
||||
# lease (e.g. drop port 5432 from the store node on the database VM);
|
||||
# the host must stop being ready and acknowledge nothing, then after
|
||||
# the block is lifted re-acquire under a newer epoch with nothing lost.
|
||||
# `all` runs kills, full and empty (db-outage needs an operator). Every change is
|
||||
# made with `kubectl set env` / `kubectl patch` on the evo-region Deployment and
|
||||
# reverted on exit, including on failure, back to the Git values (so Argo returns to
|
||||
# Synced); World0 is refused before anything touches the cluster.
|
||||
set -euo pipefail
|
||||
usage() { echo "usage: $0 <dev|stage> [kills|full|empty|db-outage|all] [--context <name>]" >&2; exit 2; }
|
||||
[ $# -ge 1 ] || usage
|
||||
env="$1"; shift
|
||||
case "$env" in
|
||||
dev|stage) ;;
|
||||
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
|
||||
exit 2 ;;
|
||||
esac
|
||||
scenario="${1:-all}"
|
||||
case "$scenario" in kills|full|empty|db-outage|all) shift || true ;; --context) scenario=all ;; *) usage ;; esac
|
||||
k=(kubectl --request-timeout=30s)
|
||||
[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2")
|
||||
ns="evolution-$env"
|
||||
selector=app.kubernetes.io/name=evo-region
|
||||
work="$(mktemp -d)"
|
||||
log_pids=()
|
||||
|
||||
now_ms() { date +%s%3N; }
|
||||
say() { echo "$(date -u +%H:%M:%S) $*"; }
|
||||
fail() { say "FAIL $*"; exit 1; }
|
||||
current_pods() { "${k[@]}" get pods -n "$ns" -l "$selector" \
|
||||
-o jsonpath='{range .items[?(!@.metadata.deletionTimestamp)]}{.metadata.name}{"\n"}{end}'; }
|
||||
ready_pod() { # Waits up to $2 seconds (default 120) for a Ready pod other than $1.
|
||||
local old="$1" limit="${2:-120}"
|
||||
for _ in $(seq 1 $((limit * 5))); do
|
||||
for p in $(current_pods); do
|
||||
[ "$p" = "$old" ] && continue
|
||||
[ "$("${k[@]}" get pod -n "$ns" "$p" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null)" = True ] &&
|
||||
{ echo "$p"; return 0; }
|
||||
done
|
||||
sleep 0.2
|
||||
done
|
||||
return 1
|
||||
}
|
||||
metric() { # metric <pod> <series>: from the pod's own /metrics (bash /dev/tcp).
|
||||
"${k[@]}" exec -n "$ns" "$1" -- /bin/bash -c \
|
||||
'exec 3<>/dev/tcp/127.0.0.1/9100; printf "GET /metrics HTTP/1.1\r\nHost: drill\r\n\r\n" >&3; cat <&3' 2>/dev/null |
|
||||
awk -v s="$2" '$1 == s {print $2}'
|
||||
}
|
||||
capture() { # Streams a pod's log to $work/<pod>.log for acknowledgement accounting.
|
||||
"${k[@]}" logs -n "$ns" -f "$1" > "$work/$1.log" 2>/dev/null &
|
||||
log_pids+=($!)
|
||||
}
|
||||
max_ack() { awk '$1 == "ack" {if ($3 > m) m = $3} END {print m + 0}' "$work/$1.log"; }
|
||||
recovered_at() { # committed tick from the pod's "phase running epoch=E committed=C" line.
|
||||
sed -n 's/^phase running epoch=\([0-9]*\) committed=\([0-9]*\) .*/\1 \2/p' "$work/$1.log" | tail -1
|
||||
}
|
||||
wait_for_line() { # wait_for_line <pod> <regex> <seconds>
|
||||
for _ in $(seq 1 $(($3 * 5))); do grep -qE "$2" "$work/$1.log" && return 0; sleep 0.2; done
|
||||
return 1
|
||||
}
|
||||
set_env() { "${k[@]}" set env -n "$ns" deployment/evo-region "$@" > /dev/null; }
|
||||
restore() {
|
||||
set +e
|
||||
say "restoring the Git configuration of evo-region"
|
||||
set_env REGION_INPUTS= REGION_RESERVE_MIB=1024
|
||||
"${k[@]}" get deployment -n "$ns" evo-region -o jsonpath='{.spec.template.spec.volumes[0].emptyDir}' |
|
||||
grep -q . && "${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \
|
||||
'[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","persistentVolumeClaim":{"claimName":"aevolution-objects"}}}]' > /dev/null
|
||||
for pid in "${log_pids[@]}"; do kill "$pid" 2>/dev/null; done
|
||||
ready_pod "" 120 > /dev/null && say "restored: evo-region Ready" || say "WARN evo-region not Ready after restore"
|
||||
say "logs kept in $work"
|
||||
}
|
||||
trap restore EXIT
|
||||
|
||||
POD=""
|
||||
loaded_pod() { # Sets POD to a Ready pod other than $1 once acknowledgements flow.
|
||||
POD="$(ready_pod "${1:-}" 120)" || fail "no Ready evo-region pod"
|
||||
capture "$POD"
|
||||
wait_for_line "$POD" '^ack ' 60 || fail "$POD acknowledges nothing under load"
|
||||
}
|
||||
|
||||
say "enabling synthetic load in $ns"
|
||||
before="$(current_pods | head -1)"
|
||||
set_env REGION_INPUTS=development-v1
|
||||
loaded_pod "$before"; pod="$POD"
|
||||
say "loaded pod=$pod ($(recovered_at "$pod"))"
|
||||
|
||||
if [ "$scenario" = kills ] || [ "$scenario" = all ]; then
|
||||
for round in 1 2 3; do
|
||||
sleep 10
|
||||
acked="$(max_ack "$pod")"
|
||||
start="$(now_ms)"
|
||||
"${k[@]}" delete pod -n "$ns" "$pod" --grace-period=0 --force > /dev/null 2>&1
|
||||
loaded_pod "$pod"; next="$POD"
|
||||
rto=$(( $(now_ms) - start ))
|
||||
read -r epoch committed <<< "$(recovered_at "$next")"
|
||||
say "kill round=$round rto_ms=$rto dead_acked_max=$acked recovered_committed=$committed epoch=$epoch pod=$next"
|
||||
[ "$acked" -le "$committed" ] || fail "round $round lost acknowledgements ($acked > $committed)"
|
||||
pod="$next"
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "$scenario" = full ] || [ "$scenario" = all ]; then
|
||||
sleep 5
|
||||
acked="$(max_ack "$pod")"
|
||||
set_env REGION_RESERVE_MIB=1048576
|
||||
full_pod="$(ready_pod "$pod" 120)" || fail "full: host did not recover (recovery needs no space)"
|
||||
capture "$full_pod"
|
||||
sleep 30
|
||||
new_acks="$(awk '$1 == "ack"' "$work/$full_pod.log" | wc -l)"
|
||||
available="$(metric "$full_pod" evo_region_objects_available_bytes)"
|
||||
reserve="$(metric "$full_pod" evo_region_objects_reserve_bytes)"
|
||||
failures="$(metric "$full_pod" evo_region_store_failures_total)"
|
||||
say "full acks_while_full=$new_acks available=$available reserve=$reserve store_failures=$failures"
|
||||
[ "$new_acks" -eq 0 ] || fail "full: acknowledged while the volume was full"
|
||||
set_env REGION_RESERVE_MIB=1024
|
||||
loaded_pod "$full_pod"; pod="$POD"
|
||||
read -r epoch committed <<< "$(recovered_at "$pod")"
|
||||
say "full restored committed=$committed (acked before=$acked) epoch=$epoch pod=$pod"
|
||||
[ "$acked" -le "$committed" ] || fail "full: lost acknowledgements"
|
||||
fi
|
||||
|
||||
if [ "$scenario" = empty ] || [ "$scenario" = all ]; then
|
||||
sleep 5
|
||||
acked="$(max_ack "$pod")"
|
||||
"${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \
|
||||
'[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","emptyDir":{"sizeLimit":"64Mi"}}}]' > /dev/null
|
||||
sleep 45
|
||||
if ready_pod "$pod" 1 > /dev/null; then fail "empty: a host became Ready on an empty volume"; fi
|
||||
empty_pod="$(current_pods | grep -v "^$pod$" | head -1)"
|
||||
quarantined="$("${k[@]}" logs -n "$ns" "$empty_pod" --previous 2>/dev/null | grep -c '^phase quarantined' || true)"
|
||||
say "empty pod=$empty_pod ready=never quarantined_lines=$quarantined"
|
||||
"${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \
|
||||
'[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","persistentVolumeClaim":{"claimName":"aevolution-objects"}}}]' > /dev/null
|
||||
loaded_pod "$empty_pod"; pod="$POD"
|
||||
read -r epoch committed <<< "$(recovered_at "$pod")"
|
||||
say "empty restored committed=$committed (acked before=$acked) epoch=$epoch pod=$pod"
|
||||
[ "$acked" -le "$committed" ] || fail "empty: the world was reset or acknowledgements lost"
|
||||
fi
|
||||
|
||||
if [ "$scenario" = db-outage ]; then
|
||||
sleep 5
|
||||
epoch0="$(metric "$pod" evo_region_lease_epoch)"
|
||||
say "block the database from the store node for at least 30 s now (lease 20 s), then lift it."
|
||||
say "waiting for the host to lose readiness..."
|
||||
for _ in $(seq 1 600); do [ "$(metric "$pod" evo_region_ready)" = 0 ] && break; sleep 0.5; done
|
||||
[ "$(metric "$pod" evo_region_ready)" = 0 ] || fail "db-outage: host stayed ready"
|
||||
lost="$(now_ms)"
|
||||
acked="$(max_ack "$pod")"
|
||||
say "host not ready; acknowledgements stopped at tick $acked. Waiting for recovery after the block is lifted..."
|
||||
for _ in $(seq 1 1200); do [ "$(metric "$pod" evo_region_ready)" = 1 ] && break; sleep 0.5; done
|
||||
[ "$(metric "$pod" evo_region_ready)" = 1 ] || fail "db-outage: host did not recover within 10 minutes"
|
||||
epoch1="$(metric "$pod" evo_region_lease_epoch)"
|
||||
committed="$(metric "$pod" evo_region_committed_tick)"
|
||||
say "db-outage unavailable_ms=$(( $(now_ms) - lost )) epoch=$epoch0->$epoch1 committed=$committed acked_before=$acked fences=$(metric "$pod" evo_region_fences_total)"
|
||||
[ "$epoch1" -gt "$epoch0" ] && [ "$acked" -le "$committed" ] || fail "db-outage: stale authority resumed or acknowledgements lost"
|
||||
fi
|
||||
say "PASS fault drill ($scenario) in $ns"
|
||||
Reference in New Issue
Block a user