From 4a900b0bd316f918a926691e273ca9a4fc18a92c Mon Sep 17 00:00:00 2001 From: dingo Date: Wed, 30 Sep 2026 15:12:58 -0400 Subject: [PATCH] fix(server): back off commit retries after store failures; sharpen fault drill The first dev drill showed a full volume driving ~140 refused publications a second. Commit retries now back off 100 ms doubling to a 2 s cap, reset on the next confirmed commit. fault-drill.sh reports the empty-volume pod's exit code, restarts and logs, and prints node-specific iptables rules for db-outage. Co-Authored-By: Claude Opus 5.5 --- docs/adr/0073-fault-drills.md | 5 +++++ docs/engineering/VALIDATION.md | 34 ++++++++++++++++++++++++++++++ include/evo/server/region_host.hpp | 4 ++++ src/server/region_host.cpp | 20 ++++++++++++++++-- tests/region_host_tests.cpp | 23 ++++++++++++++------ tools/ops/fault-drill.sh | 22 ++++++++++++++++--- 6 files changed, 97 insertions(+), 11 deletions(-) diff --git a/docs/adr/0073-fault-drills.md b/docs/adr/0073-fault-drills.md index d8c75e4..c3acefb 100644 --- a/docs/adr/0073-fault-drills.md +++ b/docs/adr/0073-fault-drills.md @@ -77,6 +77,11 @@ PostgreSQL: backup are M8.7/M8.8. This is the distinction the named risk asks for. - Argo shows the dev or stage application OutOfSync during a drill, and Synced again once the script restores the Git values. +- The first dev drill found that a full volume made the host retry its refused + publication on every step (about 140 a second). Commit retries after a store + failure now back off from 100 ms, doubling to a 2 s cap, and reset on the next + confirmed commit. The cap is well inside the lease, so recovery after an outage + stays prompt. ## Alternatives rejected - **A network input listener (gateway-lite) now.** Premature design of the diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 093fffa..7b0363a 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -2882,3 +2882,37 @@ admission policy and the stage and World0 AppProjects, then ran - `fault-drill.sh` passed `bash -n`. It refuses `world0` and unknown scenarios (exit 2) without calling kubectl. - Not executed: the in-cluster drill (kills, full, empty, db-outage). + +### M8.6 first in-cluster drill (dev, run by the maintainer) - 2026-09-30 + +`fault-drill.sh dev all`, then `fault-drill.sh dev db-outage`: +- **kills: PASS.** RTO 20 147, 19 777 and 20 222 ms (lease-bound, as ADR-0073 + expects). The dead pods' highest acknowledgements (209, 419, 629) equalled the + heads their successors recovered: no acknowledgement was lost. +- **full: PASS, with a finding.** No acknowledgement while full, and recovery from + the committed head afterwards. But `store_failures` reached 4 247 in about 30 s: + the host retried a refused publication on every step. +- **empty: PASS, with an evidence gap.** No pod became Ready, and restoring the + volume recovered committed 894, past the last acknowledgement (888). The script + found no `phase quarantined` line, because it read only the previous container's + log. +- **db-outage: FAIL, not yet diagnosed.** The host stayed Ready for the whole + 10-minute window. Nothing shows that the database was actually unreachable from + the pod's node. The prompt did not name the node or the rule to apply, so the + block may have targeted another address. + +Fixes (this segment): +- `RegionHost` backs off commit retries after a store failure: 100 ms, doubling to + a 2 s cap, and reset by the next confirmed commit. `region_host_tests` now + expects exactly 2 attempts in the first 200 ms and 5 in 2 s of a store outage, + and a commit within 23 steps after the store heals. Mutation: removing the + backoff gate was caught. +- `fault-drill.sh` empty: reports the restart count and last exit code (4 means + quarantined), keeps the current and previous logs, and prints their tails. +- `fault-drill.sh` db-outage: prints the pod's node, the node's InternalIP, and + the exact `iptables` insert and delete rules to run on the database VM. +- Checks run: the format check; WSL `checks` 59/59 in debug, release, analysis + and ASan; the PostgreSQL suites in analysis and ASan/UBSan; the mutation. Windows + MSVC CTest 59/59 (the server is not part of the Windows build). + `fault-drill.sh` passed `bash -n`. +- Pending: a new image, then re-running the full, empty and db-outage scenarios. diff --git a/include/evo/server/region_host.hpp b/include/evo/server/region_host.hpp index bed1305..f01c385 100644 --- a/include/evo/server/region_host.hpp +++ b/include/evo/server/region_host.hpp @@ -149,6 +149,10 @@ template class RegionHost { std::vector outcomes_; std::vector dropped_; SchedulerTime origin_{}, batch_started_{}; + // Store failures back off: a failed commit is retried after retry_delay_, which + // doubles to a cap and resets on the next confirmed commit. + SchedulerTime retry_at_{}; + std::chrono::milliseconds retry_delay_{}; std::uint64_t origin_tick_{}; bool draining_{}; HostStatus status_{}; diff --git a/src/server/region_host.cpp b/src/server/region_host.cpp index bce43ed..1135ac3 100644 --- a/src/server/region_host.cpp +++ b/src/server/region_host.cpp @@ -9,6 +9,11 @@ using persistence::LeaseClock; using persistence::LeaseStoreError; using persistence::PublicationError; using persistence::RecoveryError; +// Commit retries after a store failure: 100 ms doubling to 2 s (well inside a +// lease, so recovery after an outage is prompt; bounded so a full volume or an +// unreachable store costs at most one upload attempt every 2 s). +constexpr std::chrono::milliseconds commit_retry_initial{100}; +constexpr std::chrono::milliseconds commit_retry_limit{2000}; // Per-profile calls into the persistence layer. template struct Ops; @@ -245,6 +250,8 @@ void RegionHost::confirmed(const persistence::PublishedBatch& published, outcomes_.push_back({d, head.sequence}); prepared_.reset(); batch_started_ = now; + retry_at_ = {}; + retry_delay_ = {}; ++status_.commits; status_.phase = draining_ ? HostPhase::draining : HostPhase::running; } @@ -431,10 +438,19 @@ template const HostStatus& RegionHost::step(SchedulerTim if (!batch_) break; // Running always has a batch; defensive. const bool flush = - !batch_->steps().empty() && + !batch_->steps().empty() && now >= retry_at_ && (batch_->sealed() || draining_ || now - batch_started_ >= config_.flush_interval); - if (flush) + if (flush) { + const auto failures = status_.store_failures; commit(now); // May lose authority and reset the batch. + if (status_.store_failures != failures) { + // Every retry re-uploads the batch: back off instead of spinning. + retry_delay_ = retry_delay_ == std::chrono::milliseconds{} + ? commit_retry_initial + : std::min(retry_delay_ * 2, commit_retry_limit); + retry_at_ = now + retry_delay_; + } + } if (draining_ && status_.phase == HostPhase::draining && batch_ && batch_->steps().empty()) { const auto& grant = authority_.grant(); diff --git a/tests/region_host_tests.cpp b/tests/region_host_tests.cpp index a3a2e15..84cef78 100644 --- a/tests/region_host_tests.cpp +++ b/tests/region_host_tests.cpp @@ -185,8 +185,9 @@ void starvation_accrues_bounded_debt() { CHECK(s.queued == 10); } -// Object store outage: the sealed batch waits, nothing is acknowledged, and the -// region resumes once storage is durable again. +// Object store outage: the sealed batch waits, nothing is acknowledged, retries +// back off (100 ms doubling, not one upload per step), and the region resumes once +// storage is durable again. void storage_outage_pauses_the_region() { const auto r = provisioned(3, 0x41); ObjectStoreFaults faults; @@ -202,12 +203,22 @@ void storage_outage_pauses_the_region() { const auto& s = h->step(t); CHECK(s.simulated_tick == 20 && s.committed_tick == 0); // Paused at the death. } - CHECK(h->status().store_failures >= 5 && h->take_acks().empty() && h->take_outcomes().empty()); + // Attempts at +0 and +100 ms; the next waits until +300 ms. + CHECK(h->status().store_failures == 2 && h->take_acks().empty() && h->take_outcomes().empty()); CHECK(h->status().tick_debt == 4); + for (int i = 5; i < 40; ++i) { // Two seconds of 50 ms steps: +300, +700, +1500 only. + t += 50ms; + CHECK(h->step(t).committed_tick == 0); + } + CHECK(h->status().store_failures == 5 && h->take_acks().empty()); faults.fail_directory_syncs = 0; - t += 50ms; - auto s = h->step(t); - CHECK(s.committed_tick == 20 && h->take_acks().size() == 20 && h->take_outcomes().size() == 1); + int waited = 0; // The next attempt (+3100 ms, delay capped at 2 s) succeeds. + auto s = step_until(*h, t, 50ms, [&](const HostStatus& x) { + ++waited; + return x.committed_tick == 20; + }); + CHECK(waited <= 23 && s.store_failures == 5); + CHECK(h->take_acks().size() == 20 && h->take_outcomes().size() == 1); s = h->step(t); // Catch-up after the outage is bounded but continues. CHECK(s.simulated_tick > 20 && s.simulated_tick <= 28); } diff --git a/tools/ops/fault-drill.sh b/tools/ops/fault-drill.sh index 48cf3b4..99d51b3 100644 --- a/tools/ops/fault-drill.sh +++ b/tools/ops/fault-drill.sh @@ -144,8 +144,18 @@ if [ "$scenario" = empty ] || [ "$scenario" = all ]; then sleep 45 if ready_pod "$pod" 1 > /dev/null; then fail "empty: a host became Ready on an empty volume"; fi empty_pod="$(current_pods | grep -v "^$pod$" | head -1)" - quarantined="$("${k[@]}" logs -n "$ns" "$empty_pod" --previous 2>/dev/null | grep -c '^phase quarantined' || true)" - say "empty pod=$empty_pod ready=never quarantined_lines=$quarantined" + [ -n "$empty_pod" ] || fail "empty: no replacement pod was created" + # Evidence: the container's exits (4 = quarantined) and both of its logs. + "${k[@]}" logs -n "$ns" "$empty_pod" > "$work/$empty_pod.current.log" 2>&1 || true + "${k[@]}" logs -n "$ns" "$empty_pod" --previous > "$work/$empty_pod.previous.log" 2>&1 || true + restarts="$("${k[@]}" get pod -n "$ns" "$empty_pod" -o jsonpath='{.status.containerStatuses[0].restartCount}')" + exit_code="$("${k[@]}" get pod -n "$ns" "$empty_pod" -o jsonpath='{.status.containerStatuses[0].lastState.terminated.exitCode}')" + quarantined="$(cat "$work/$empty_pod.current.log" "$work/$empty_pod.previous.log" | grep -c '^phase quarantined' || true)" + say "empty pod=$empty_pod ready=never restarts=${restarts:-0} last_exit=${exit_code:-none} quarantined_lines=$quarantined" + say "empty last log lines:"; tail -n 5 "$work/$empty_pod.previous.log" "$work/$empty_pod.current.log" | sed 's/^/ /' + if [ "${exit_code:-}" != 4 ] && [ "$quarantined" -eq 0 ]; then + say "WARN empty: no quarantine evidence (exit 4 or a 'phase quarantined' line); see the logs above" + fi "${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \ '[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","persistentVolumeClaim":{"claimName":"aevolution-objects"}}}]' > /dev/null loaded_pod "$empty_pod"; pod="$POD" @@ -157,7 +167,13 @@ fi if [ "$scenario" = db-outage ]; then sleep 5 epoch0="$(metric "$pod" evo_region_lease_epoch)" - say "block the database from the store node for at least 30 s now (lease 20 s), then lift it." + node="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.spec.nodeName}')" + node_ip="$("${k[@]}" get node "$node" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}')" + # The pod reaches the database from its node's address (egress is SNATed off-cluster). + say "pod $pod runs on $node ($node_ip). On the database VM, block it for at least 30 s (lease 20 s):" + say " sudo iptables -I INPUT -s $node_ip -p tcp --dport 5432 -j DROP" + say " then lift it with:" + say " sudo iptables -D INPUT -s $node_ip -p tcp --dport 5432 -j DROP" say "waiting for the host to lose readiness..." for _ in $(seq 1 600); do [ "$(metric "$pod" evo_region_ready)" = 0 ] && break; sleep 0.5; done [ "$(metric "$pod" evo_region_ready)" = 0 ] || fail "db-outage: host stayed ready"