# M8.5 region host alerts (ADR-0072). Thresholds follow the measured behaviour: # the production lease is 20 s renewed every 5 s (15-20 s remaining when healthy); # batches are at most 64 ticks with a 1 s flush; an idle region (no inputs, no # gateway yet) always accrues tick debt, so debt and staleness only alert while # inputs are waiting. Unit tests: evo-region.rules.test.yml (promtool). groups: - name: evo-region-authority rules: - alert: EvoRegionDown expr: up{job="evo-region"} == 0 for: 2m labels: {severity: critical} annotations: summary: "Region {{ $labels.environment }}/{{ $labels.world }}/{{ $labels.region }} is not scrapeable" description: "No operations endpoint answers. The process or its pod is gone; a forced deletion recovers in about 20 s, so 2 minutes is a real outage." - alert: EvoRegionNotReady expr: evo_region_ready == 0 for: 2m labels: {severity: critical} annotations: summary: "Region {{ $labels.environment }} has no authoritative host" description: "The process is alive but holds no lease or has not verified its head. Check the phase metric and the host log." - alert: EvoRegionQuarantined expr: evo_region_phase{phase="quarantined"} == 1 for: 1m labels: {severity: critical} annotations: summary: "Region {{ $labels.environment }} is quarantined" description: "Committed state or history failed verification. The region stays unavailable; never re-provision it. Audit with evo-world-clone audit." - alert: EvoRegionLeaseMargin expr: evo_region_ready == 1 and evo_region_lease_remaining_milliseconds < 10000 for: 1m labels: {severity: warning} annotations: summary: "Region {{ $labels.environment }} lease is not being renewed" description: "Healthy renewals keep 15-20 s of lease. Under 10 s for a minute means renewals are failing; the host will fence itself." - alert: EvoRegionFenced expr: increase(evo_region_fences_total[10m]) > 0 labels: {severity: warning} annotations: summary: "Region {{ $labels.environment }} host lost authority" description: "A host was fenced within the last 10 minutes. Expected once after a forced deletion's predecessor resumes; repeated fences point to database or clock trouble." - name: evo-region-progress rules: - alert: EvoRegionStoreFailures expr: increase(evo_region_store_failures_total[5m]) > 3 labels: {severity: warning} annotations: summary: "Region {{ $labels.environment }} storage or database failures" description: "Publication or database calls are failing; progress pauses and nothing is acknowledged until they succeed." - alert: EvoRegionUnknownOutcomes expr: increase(evo_region_unknown_outcomes_total[15m]) > 0 labels: {severity: warning} annotations: summary: "Region {{ $labels.environment }} had commits with unknown outcomes" description: "Commits timed out or lost their connection and were resolved by exact bytes. Occasional is safe; frequent indicates database latency near the 5 s commit deadline." - alert: EvoRegionCommitLag expr: evo_region_commit_lag_ticks > 256 for: 2m labels: {severity: warning} annotations: summary: "Region {{ $labels.environment }} is not committing what it simulates" description: "More than four full batches are simulated but not durable; acknowledgements are waiting." - alert: EvoRegionTickDebt expr: evo_region_tick_debt > 8 and evo_region_queued_inputs > 0 for: 2m labels: {severity: warning} annotations: summary: "Region {{ $labels.environment }} is behind real time with inputs waiting" description: "Due ticks are not being simulated although inputs are queued (catch-up is 8 ticks per step)." - alert: EvoRegionCheckpointStale expr: evo_region_checkpoint_age_milliseconds > 300000 and (evo_region_queued_inputs > 0 or evo_region_commit_lag_ticks > 0) for: 1m labels: {severity: warning} annotations: summary: "Region {{ $labels.environment }} has not committed for 5 minutes with work pending" description: "An idle region does not commit; this fires only while inputs or simulated ticks wait for durability." - alert: EvoRegionQueueSaturated expr: evo_region_queued_inputs >= 0.9 * evo_region_queue_capacity_inputs for: 2m labels: {severity: warning} annotations: summary: "Region {{ $labels.environment }} input backlog near its bound" description: "New inputs are being backpressured or rejected." - alert: EvoRegionInputsDropped expr: increase(evo_region_dropped_total{reason!="refused"}[10m]) > 0 labels: {severity: warning} annotations: summary: "Region {{ $labels.environment }} accepted inputs that will never be acknowledged" description: "Inputs were drained, fenced or quarantined ({{ $labels.reason }}); clients must resubmit to the current owner." - name: evo-region-storage rules: - alert: EvoRegionObjectsLow expr: evo_region_objects_available_bytes < 2 * evo_region_objects_reserve_bytes for: 5m labels: {severity: warning} annotations: summary: "Region {{ $labels.environment }} object volume is below twice its reserve" description: "local-path volumes do not enforce their size; the node's disk is running out." - alert: EvoRegionObjectsFull expr: evo_region_objects_available_bytes < evo_region_objects_reserve_bytes for: 1m labels: {severity: critical} annotations: summary: "Region {{ $labels.environment }} object volume is below its reserve" description: "New checkpoints are refused; the region cannot commit or acknowledge until space is freed." - name: aevolution-monitoring rules: - alert: Watchdog expr: vector(1) labels: {severity: none} annotations: summary: "Always firing: proves the alert pipeline end to end" description: "Route to a heartbeat receiver once one exists; its absence then means monitoring is down." - alert: AlertmanagerDown expr: up{job="alertmanager"} == 0 for: 5m labels: {severity: critical} annotations: summary: "Alertmanager is not scrapeable" description: "Alerts are evaluated but cannot be routed."