Add M8.6 synthetic load driver and fault drill

A drill-only evo-server --synthetic-inputs development-v1 driver feeds
deterministic inputs from the committed head after every recovery, with
request IDs that match acknowledgements across owners (ADR-0073); it is
off in Git and the policy check keeps it off. fault-drill.sh loads dev
or stage and runs repeated forced kills (RTO, zero lost
acknowledgements), a full object volume (nothing acknowledged), an
empty volume (quarantine, never a fresh world) and an operator-driven
database outage (stale epoch never resumes), restoring the Git
configuration on exit. Process tests prove contiguous acknowledgement
across a SIGKILL and exact resumption after the committed head.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-30 10:52:01 -04:00
co-authored by Claude Opus 5.5
parent 4ef0fe8167
commit 3f08a1d221
16 changed files with 513 additions and 5 deletions
+37 -1
View File
@@ -22,9 +22,16 @@
// nothing about authority), /readyz (200 only with current authority) and /metrics
// (Prometheus text). It prints "ops listening <port>" once bound.
//
// --synthetic-inputs development-v1 (dev/stage drills only, M8.6, ADR-0073) replaces
// stdin with a deterministic driver: whenever the host is running it keeps the
// backlog fed with development_input(t) for the ticks after the committed head,
// restarting from the committed tick after every recovery. Request IDs are
// "SYN1" + region + tick, so acknowledgements can be checked across owners.
//
// --crash-at <point>:<n> (tests only, M7.7) SIGKILLs this process at the n-th
// occurrence of before-upload, before-transaction, after-commit or after-ack (right
// after a step's acknowledgements are written and flushed).
#include "evo/processes/development_opening.hpp"
#include "evo/server/ops_http.hpp"
#include "evo/server/region_host.hpp"
@@ -61,6 +68,7 @@ struct Options {
std::string crash_point;
std::uint64_t crash_ordinal{};
std::string ops_listen;
bool synthetic{};
std::uint64_t objects_reserve_mib{1024};
};
@@ -164,6 +172,8 @@ std::optional<Options> parse(int argc, char** argv) {
o.host.lease = {};
else if (flag == "--ops-listen")
o.ops_listen = value;
else if (flag == "--synthetic-inputs" && value == "development-v1")
o.synthetic = true;
else if (flag == "--objects-reserve-mib" && n && *n <= 1048576)
o.objects_reserve_mib = *n;
else if (flag == "--crash-at") {
@@ -334,7 +344,18 @@ template <typename State> int serve(Options& o) {
return {404, "not found\n"};
};
LineReader reader;
bool input_open = true, ready = false;
bool input_open = !o.synthetic, ready = false;
// Synthetic driver state: the next tick to submit, reset at every recovery.
std::optional<std::uint64_t> synthetic_next;
bool synthetic_running = false;
const auto synthetic_request = [&](std::uint64_t tick) {
persistence::Uuid id{'S', 'Y', 'N', '1'};
for (unsigned i = 0; i < 4; ++i)
id[4 + i] = static_cast<std::uint8_t>((o.host.key.region >> (8U * (3U - i))) & 0xFFU);
for (unsigned i = 0; i < 8; ++i)
id[8 + i] = static_cast<std::uint8_t>((tick >> (8U * (7U - i))) & 0xFFU);
return id;
};
std::optional<std::string> pending;
std::optional<server::HostPhase> phase;
std::size_t max_queued = 0;
@@ -383,6 +404,21 @@ template <typename State> int serve(Options& o) {
admission_name(admitted.error()));
pending.reset();
}
if (o.synthetic && !drain_deadline) {
const auto& now_status = host.status();
const bool running = now_status.phase == server::HostPhase::running && now_status.ready;
if (running && !synthetic_running)
synthetic_next = now_status.simulated_tick + 1; // Resume after the head.
synthetic_running = running;
while (running && synthetic_next &&
*synthetic_next <= host.status().simulated_tick + 64) {
const server::TickInput in{synthetic_request(*synthetic_next),
processes::development_input(*synthetic_next)};
if (!host.submit(in))
break; // Full or no longer ready: continue next iteration.
++*synthetic_next;
}
}
const auto& s = host.step(std::chrono::steady_clock::now());
max_queued = std::max(max_queued, s.queued);
// Checkpoint age restarts at recovery and at every confirmed commit.