fix(server): back off commit retries after store failures; sharpen fault drill
The first dev drill showed a full volume driving ~140 refused publications a second. Commit retries now back off 100 ms doubling to a 2 s cap, reset on the next confirmed commit. fault-drill.sh reports the empty-volume pod's exit code, restarts and logs, and prints node-specific iptables rules for db-outage. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -9,6 +9,11 @@ using persistence::LeaseClock;
|
||||
using persistence::LeaseStoreError;
|
||||
using persistence::PublicationError;
|
||||
using persistence::RecoveryError;
|
||||
// Commit retries after a store failure: 100 ms doubling to 2 s (well inside a
|
||||
// lease, so recovery after an outage is prompt; bounded so a full volume or an
|
||||
// unreachable store costs at most one upload attempt every 2 s).
|
||||
constexpr std::chrono::milliseconds commit_retry_initial{100};
|
||||
constexpr std::chrono::milliseconds commit_retry_limit{2000};
|
||||
|
||||
// Per-profile calls into the persistence layer.
|
||||
template <typename State> struct Ops;
|
||||
@@ -245,6 +250,8 @@ void RegionHost<State>::confirmed(const persistence::PublishedBatch& published,
|
||||
outcomes_.push_back({d, head.sequence});
|
||||
prepared_.reset();
|
||||
batch_started_ = now;
|
||||
retry_at_ = {};
|
||||
retry_delay_ = {};
|
||||
++status_.commits;
|
||||
status_.phase = draining_ ? HostPhase::draining : HostPhase::running;
|
||||
}
|
||||
@@ -431,10 +438,19 @@ template <typename State> const HostStatus& RegionHost<State>::step(SchedulerTim
|
||||
if (!batch_)
|
||||
break; // Running always has a batch; defensive.
|
||||
const bool flush =
|
||||
!batch_->steps().empty() &&
|
||||
!batch_->steps().empty() && now >= retry_at_ &&
|
||||
(batch_->sealed() || draining_ || now - batch_started_ >= config_.flush_interval);
|
||||
if (flush)
|
||||
if (flush) {
|
||||
const auto failures = status_.store_failures;
|
||||
commit(now); // May lose authority and reset the batch.
|
||||
if (status_.store_failures != failures) {
|
||||
// Every retry re-uploads the batch: back off instead of spinning.
|
||||
retry_delay_ = retry_delay_ == std::chrono::milliseconds{}
|
||||
? commit_retry_initial
|
||||
: std::min(retry_delay_ * 2, commit_retry_limit);
|
||||
retry_at_ = now + retry_delay_;
|
||||
}
|
||||
}
|
||||
if (draining_ && status_.phase == HostPhase::draining && batch_ &&
|
||||
batch_->steps().empty()) {
|
||||
const auto& grant = authority_.grant();
|
||||
|
||||
Reference in New Issue
Block a user