fix(server): back off commit retries after store failures; sharpen fault drill

The first dev drill showed a full volume driving ~140 refused publications a
second. Commit retries now back off 100 ms doubling to a 2 s cap, reset on the
next confirmed commit. fault-drill.sh reports the empty-volume pod's exit code,
restarts and logs, and prints node-specific iptables rules for db-outage.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-30 15:12:58 -04:00
co-authored by Claude Opus 5.5
parent 6b5da397e8
commit 4a900b0bd3
6 changed files with 97 additions and 11 deletions
+18 -2
View File
@@ -9,6 +9,11 @@ using persistence::LeaseClock;
using persistence::LeaseStoreError;
using persistence::PublicationError;
using persistence::RecoveryError;
// Commit retries after a store failure: 100 ms doubling to 2 s (well inside a
// lease, so recovery after an outage is prompt; bounded so a full volume or an
// unreachable store costs at most one upload attempt every 2 s).
constexpr std::chrono::milliseconds commit_retry_initial{100};
constexpr std::chrono::milliseconds commit_retry_limit{2000};
// Per-profile calls into the persistence layer.
template <typename State> struct Ops;
@@ -245,6 +250,8 @@ void RegionHost<State>::confirmed(const persistence::PublishedBatch& published,
outcomes_.push_back({d, head.sequence});
prepared_.reset();
batch_started_ = now;
retry_at_ = {};
retry_delay_ = {};
++status_.commits;
status_.phase = draining_ ? HostPhase::draining : HostPhase::running;
}
@@ -431,10 +438,19 @@ template <typename State> const HostStatus& RegionHost<State>::step(SchedulerTim
if (!batch_)
break; // Running always has a batch; defensive.
const bool flush =
!batch_->steps().empty() &&
!batch_->steps().empty() && now >= retry_at_ &&
(batch_->sealed() || draining_ || now - batch_started_ >= config_.flush_interval);
if (flush)
if (flush) {
const auto failures = status_.store_failures;
commit(now); // May lose authority and reset the batch.
if (status_.store_failures != failures) {
// Every retry re-uploads the batch: back off instead of spinning.
retry_delay_ = retry_delay_ == std::chrono::milliseconds{}
? commit_retry_initial
: std::min(retry_delay_ * 2, commit_retry_limit);
retry_at_ = now + retry_delay_;
}
}
if (draining_ && status_.phase == HostPhase::draining && batch_ &&
batch_->steps().empty()) {
const auto& grant = authority_.grant();