# M8.5 alert injection tests (ADR-0072): promtool test rules. Each alert is shown # firing on an injected series and staying silent on the healthy or idle case. rule_files: - ../config/evo-region.rules.yml evaluation_interval: 15s tests: # Healthy idle region: ready, renewing, idle (debt accrues with no inputs). - interval: 15s input_series: - series: 'up{job="evo-region",environment="dev"}' values: '1x40' - series: 'evo_region_ready{environment="dev"}' values: '1x40' - series: 'evo_region_phase{environment="dev",phase="running"}' values: '1x40' - series: 'evo_region_phase{environment="dev",phase="quarantined"}' values: '0x40' - series: 'evo_region_lease_remaining_milliseconds{environment="dev"}' values: '18000x40' - series: 'evo_region_tick_debt{environment="dev"}' values: '0+20x40' - series: 'evo_region_queued_inputs{environment="dev"}' values: '0x40' - series: 'evo_region_queue_capacity_inputs{environment="dev"}' values: '64x40' - series: 'evo_region_checkpoint_age_milliseconds{environment="dev"}' values: '0+15000x40' - series: 'evo_region_commit_lag_ticks{environment="dev"}' values: '0x40' - series: 'evo_region_fences_total{environment="dev"}' values: '0x40' - series: 'evo_region_store_failures_total{environment="dev"}' values: '0x40' - series: 'evo_region_objects_available_bytes{environment="dev"}' values: '40000000000x40' - series: 'evo_region_objects_reserve_bytes{environment="dev"}' values: '1073741824x40' - series: 'up{job="alertmanager"}' values: '1x40' alert_rule_test: - {eval_time: 10m, alertname: EvoRegionDown, exp_alerts: []} - {eval_time: 10m, alertname: EvoRegionNotReady, exp_alerts: []} - {eval_time: 10m, alertname: EvoRegionQuarantined, exp_alerts: []} - {eval_time: 10m, alertname: EvoRegionLeaseMargin, exp_alerts: []} - {eval_time: 10m, alertname: EvoRegionFenced, exp_alerts: []} - {eval_time: 10m, alertname: EvoRegionStoreFailures, exp_alerts: []} - {eval_time: 10m, alertname: EvoRegionTickDebt, exp_alerts: []} - {eval_time: 10m, alertname: EvoRegionCheckpointStale, exp_alerts: []} - {eval_time: 10m, alertname: EvoRegionQueueSaturated, exp_alerts: []} - {eval_time: 10m, alertname: EvoRegionObjectsLow, exp_alerts: []} - {eval_time: 10m, alertname: EvoRegionObjectsFull, exp_alerts: []} - {eval_time: 10m, alertname: AlertmanagerDown, exp_alerts: []} - eval_time: 1m alertname: Watchdog exp_alerts: - exp_labels: {severity: none} exp_annotations: summary: "Always firing: proves the alert pipeline end to end" description: "Route to a heartbeat receiver once one exists; its absence then means monitoring is down." # Outage: the region disappears, then comes back not ready and quarantined. - interval: 15s input_series: - series: 'up{job="evo-region",environment="dev",world="100",region="1"}' values: '0x20' - series: 'evo_region_ready{environment="stage"}' values: '0x20' - series: 'evo_region_phase{environment="stage",phase="quarantined"}' values: '1x20' alert_rule_test: - eval_time: 3m alertname: EvoRegionDown exp_alerts: - exp_labels: {severity: critical, job: evo-region, environment: dev, world: "100", region: "1"} exp_annotations: summary: "Region dev/100/1 is not scrapeable" description: "No operations endpoint answers. The process or its pod is gone; a forced deletion recovers in about 20 s, so 2 minutes is a real outage." - eval_time: 1m alertname: EvoRegionDown exp_alerts: [] - eval_time: 3m alertname: EvoRegionNotReady exp_alerts: - exp_labels: {severity: critical, environment: stage} exp_annotations: summary: "Region stage has no authoritative host" description: "The process is alive but holds no lease or has not verified its head. Check the phase metric and the host log." - eval_time: 2m alertname: EvoRegionQuarantined exp_alerts: - exp_labels: {severity: critical, environment: stage, phase: quarantined} exp_annotations: summary: "Region stage is quarantined" description: "Committed state or history failed verification. The region stays unavailable; never re-provision it. Audit with evo-world-clone audit." # A 20 s lease-bound forced-deletion recovery must not page anyone. - interval: 15s input_series: - series: 'evo_region_ready{environment="dev"}' values: '1 1 0 0 1 1 1 1 1 1' alert_rule_test: - {eval_time: 45s, alertname: EvoRegionNotReady, exp_alerts: []} # During the dip. - {eval_time: 2m, alertname: EvoRegionNotReady, exp_alerts: []} # Lease, fences, storage and database failures. - interval: 15s input_series: - series: 'evo_region_ready{environment="dev"}' values: '1x20' - series: 'evo_region_lease_remaining_milliseconds{environment="dev"}' values: '6000x20' - series: 'evo_region_fences_total{environment="dev"}' values: '0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1' - series: 'evo_region_store_failures_total{environment="dev"}' values: '0+1x20' - series: 'evo_region_unknown_outcomes_total{environment="dev"}' values: '0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1' alert_rule_test: - eval_time: 2m alertname: EvoRegionLeaseMargin exp_alerts: - exp_labels: {severity: warning, environment: dev} exp_annotations: summary: "Region dev lease is not being renewed" description: "Healthy renewals keep 15-20 s of lease. Under 10 s for a minute means renewals are failing; the host will fence itself." - eval_time: 2m alertname: EvoRegionFenced exp_alerts: - exp_labels: {severity: warning, environment: dev} exp_annotations: summary: "Region dev host lost authority" description: "A host was fenced within the last 10 minutes. Expected once after a forced deletion's predecessor resumes; repeated fences point to database or clock trouble." - eval_time: 3m alertname: EvoRegionStoreFailures exp_alerts: - exp_labels: {severity: warning, environment: dev} exp_annotations: summary: "Region dev storage or database failures" description: "Publication or database calls are failing; progress pauses and nothing is acknowledged until they succeed." - eval_time: 2m alertname: EvoRegionUnknownOutcomes exp_alerts: - exp_labels: {severity: warning, environment: dev} exp_annotations: summary: "Region dev had commits with unknown outcomes" description: "Commits timed out or lost their connection and were resolved by exact bytes. Occasional is safe; frequent indicates database latency near the 5 s commit deadline." # Work waiting: debt, staleness, lag, a saturated queue and dropped inputs. - interval: 15s input_series: - series: 'evo_region_tick_debt{environment="dev"}' values: '20x20' - series: 'evo_region_queued_inputs{environment="dev"}' values: '60x20' - series: 'evo_region_queue_capacity_inputs{environment="dev"}' values: '64x20' - series: 'evo_region_checkpoint_age_milliseconds{environment="dev"}' values: '400000+15000x20' - series: 'evo_region_commit_lag_ticks{environment="dev"}' values: '300x20' - series: 'evo_region_dropped_total{environment="dev",reason="fenced"}' values: '0 0 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3' - series: 'evo_region_dropped_total{environment="dev",reason="refused"}' values: '0+1x20' alert_rule_test: - eval_time: 3m alertname: EvoRegionTickDebt exp_alerts: - exp_labels: {severity: warning, environment: dev} exp_annotations: summary: "Region dev is behind real time with inputs waiting" description: "Due ticks are not being simulated although inputs are queued (catch-up is 8 ticks per step)." - eval_time: 2m alertname: EvoRegionCheckpointStale exp_alerts: - exp_labels: {severity: warning, environment: dev} exp_annotations: summary: "Region dev has not committed for 5 minutes with work pending" description: "An idle region does not commit; this fires only while inputs or simulated ticks wait for durability." - eval_time: 3m alertname: EvoRegionCommitLag exp_alerts: - exp_labels: {severity: warning, environment: dev} exp_annotations: summary: "Region dev is not committing what it simulates" description: "More than four full batches are simulated but not durable; acknowledgements are waiting." - eval_time: 3m alertname: EvoRegionQueueSaturated exp_alerts: - exp_labels: {severity: warning, environment: dev} exp_annotations: summary: "Region dev input backlog near its bound" description: "New inputs are being backpressured or rejected." - eval_time: 2m alertname: EvoRegionInputsDropped exp_alerts: - exp_labels: {severity: warning, environment: dev, reason: fenced} exp_annotations: summary: "Region dev accepted inputs that will never be acknowledged" description: "Inputs were drained, fenced or quarantined (fenced); clients must resubmit to the current owner." # Object volume filling up, then below its reserve. - interval: 15s input_series: - series: 'evo_region_objects_available_bytes{environment="dev"}' values: '1500000000x30' - series: 'evo_region_objects_available_bytes{environment="stage"}' values: '900000000x30' # Below the reserve but above half of it. - series: 'evo_region_objects_reserve_bytes{environment="dev"}' values: '1073741824x30' - series: 'evo_region_objects_reserve_bytes{environment="stage"}' values: '1073741824x30' - series: 'up{job="alertmanager"}' values: '0x30' alert_rule_test: - eval_time: 6m alertname: EvoRegionObjectsLow exp_alerts: - exp_labels: {severity: warning, environment: dev} exp_annotations: summary: "Region dev object volume is below twice its reserve" description: "local-path volumes do not enforce their size; the node's disk is running out." - exp_labels: {severity: warning, environment: stage} exp_annotations: summary: "Region stage object volume is below twice its reserve" description: "local-path volumes do not enforce their size; the node's disk is running out." - eval_time: 2m alertname: EvoRegionObjectsFull exp_alerts: - exp_labels: {severity: critical, environment: stage} exp_annotations: summary: "Region stage object volume is below its reserve" description: "New checkpoints are refused; the region cannot commit or acknowledge until space is freed." - eval_time: 6m alertname: AlertmanagerDown exp_alerts: - exp_labels: {severity: critical, job: alertmanager} exp_annotations: summary: "Alertmanager is not scrapeable" description: "Alerts are evaluated but cannot be routed."