Add M8.6 synthetic load driver and fault drill
A drill-only evo-server --synthetic-inputs development-v1 driver feeds deterministic inputs from the committed head after every recovery, with request IDs that match acknowledgements across owners (ADR-0073); it is off in Git and the policy check keeps it off. fault-drill.sh loads dev or stage and runs repeated forced kills (RTO, zero lost acknowledgements), a full object volume (nothing acknowledged), an empty volume (quarantine, never a fresh world) and an operator-driven database outage (stale epoch never resumes), restoring the Git configuration on exit. Process tests prove contiguous acknowledgement across a SIGKILL and exact resumption after the committed head. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -22,9 +22,16 @@
|
||||
// nothing about authority), /readyz (200 only with current authority) and /metrics
|
||||
// (Prometheus text). It prints "ops listening <port>" once bound.
|
||||
//
|
||||
// --synthetic-inputs development-v1 (dev/stage drills only, M8.6, ADR-0073) replaces
|
||||
// stdin with a deterministic driver: whenever the host is running it keeps the
|
||||
// backlog fed with development_input(t) for the ticks after the committed head,
|
||||
// restarting from the committed tick after every recovery. Request IDs are
|
||||
// "SYN1" + region + tick, so acknowledgements can be checked across owners.
|
||||
//
|
||||
// --crash-at <point>:<n> (tests only, M7.7) SIGKILLs this process at the n-th
|
||||
// occurrence of before-upload, before-transaction, after-commit or after-ack (right
|
||||
// after a step's acknowledgements are written and flushed).
|
||||
#include "evo/processes/development_opening.hpp"
|
||||
#include "evo/server/ops_http.hpp"
|
||||
#include "evo/server/region_host.hpp"
|
||||
|
||||
@@ -61,6 +68,7 @@ struct Options {
|
||||
std::string crash_point;
|
||||
std::uint64_t crash_ordinal{};
|
||||
std::string ops_listen;
|
||||
bool synthetic{};
|
||||
std::uint64_t objects_reserve_mib{1024};
|
||||
};
|
||||
|
||||
@@ -164,6 +172,8 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
o.host.lease = {};
|
||||
else if (flag == "--ops-listen")
|
||||
o.ops_listen = value;
|
||||
else if (flag == "--synthetic-inputs" && value == "development-v1")
|
||||
o.synthetic = true;
|
||||
else if (flag == "--objects-reserve-mib" && n && *n <= 1048576)
|
||||
o.objects_reserve_mib = *n;
|
||||
else if (flag == "--crash-at") {
|
||||
@@ -334,7 +344,18 @@ template <typename State> int serve(Options& o) {
|
||||
return {404, "not found\n"};
|
||||
};
|
||||
LineReader reader;
|
||||
bool input_open = true, ready = false;
|
||||
bool input_open = !o.synthetic, ready = false;
|
||||
// Synthetic driver state: the next tick to submit, reset at every recovery.
|
||||
std::optional<std::uint64_t> synthetic_next;
|
||||
bool synthetic_running = false;
|
||||
const auto synthetic_request = [&](std::uint64_t tick) {
|
||||
persistence::Uuid id{'S', 'Y', 'N', '1'};
|
||||
for (unsigned i = 0; i < 4; ++i)
|
||||
id[4 + i] = static_cast<std::uint8_t>((o.host.key.region >> (8U * (3U - i))) & 0xFFU);
|
||||
for (unsigned i = 0; i < 8; ++i)
|
||||
id[8 + i] = static_cast<std::uint8_t>((tick >> (8U * (7U - i))) & 0xFFU);
|
||||
return id;
|
||||
};
|
||||
std::optional<std::string> pending;
|
||||
std::optional<server::HostPhase> phase;
|
||||
std::size_t max_queued = 0;
|
||||
@@ -383,6 +404,21 @@ template <typename State> int serve(Options& o) {
|
||||
admission_name(admitted.error()));
|
||||
pending.reset();
|
||||
}
|
||||
if (o.synthetic && !drain_deadline) {
|
||||
const auto& now_status = host.status();
|
||||
const bool running = now_status.phase == server::HostPhase::running && now_status.ready;
|
||||
if (running && !synthetic_running)
|
||||
synthetic_next = now_status.simulated_tick + 1; // Resume after the head.
|
||||
synthetic_running = running;
|
||||
while (running && synthetic_next &&
|
||||
*synthetic_next <= host.status().simulated_tick + 64) {
|
||||
const server::TickInput in{synthetic_request(*synthetic_next),
|
||||
processes::development_input(*synthetic_next)};
|
||||
if (!host.submit(in))
|
||||
break; // Full or no longer ready: continue next iteration.
|
||||
++*synthetic_next;
|
||||
}
|
||||
}
|
||||
const auto& s = host.step(std::chrono::steady_clock::now());
|
||||
max_queued = std::max(max_queued, s.queued);
|
||||
// Checkpoint age restarts at recovery and at every confirmed commit.
|
||||
|
||||
Reference in New Issue
Block a user