Files
dingoandClaude Opus 5.5 b4493b3d0f Add M8.5 observability: pinned Prometheus, tested alerts, capacity metrics
An AEvolution-only Prometheus 3.13.3 and Alertmanager 0.34.1, pinned by
digest and delivered by Argo (ADR-0072), scrape each region's operations
Service through static targets (no API access, no RBAC); the region
namespaces admit port 9100 from Prometheus only. Sixteen alert rules
cover authority, lease, commits, backlog and object capacity and are
proven by promtool injection tests in CI with tools copied from the same
pinned images. Two new gauges expose object volume free space and the
reserve. Alerts route to a null receiver until a destination is chosen;
no broker is adopted and the database outbox stays.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 06:16:36 -04:00

241 lines
12 KiB
YAML

# M8.5 alert injection tests (ADR-0072): promtool test rules. Each alert is shown
# firing on an injected series and staying silent on the healthy or idle case.
rule_files:
- ../config/evo-region.rules.yml
evaluation_interval: 15s
tests:
# Healthy idle region: ready, renewing, idle (debt accrues with no inputs).
- interval: 15s
input_series:
- series: 'up{job="evo-region",environment="dev"}'
values: '1x40'
- series: 'evo_region_ready{environment="dev"}'
values: '1x40'
- series: 'evo_region_phase{environment="dev",phase="running"}'
values: '1x40'
- series: 'evo_region_phase{environment="dev",phase="quarantined"}'
values: '0x40'
- series: 'evo_region_lease_remaining_milliseconds{environment="dev"}'
values: '18000x40'
- series: 'evo_region_tick_debt{environment="dev"}'
values: '0+20x40'
- series: 'evo_region_queued_inputs{environment="dev"}'
values: '0x40'
- series: 'evo_region_queue_capacity_inputs{environment="dev"}'
values: '64x40'
- series: 'evo_region_checkpoint_age_milliseconds{environment="dev"}'
values: '0+15000x40'
- series: 'evo_region_commit_lag_ticks{environment="dev"}'
values: '0x40'
- series: 'evo_region_fences_total{environment="dev"}'
values: '0x40'
- series: 'evo_region_store_failures_total{environment="dev"}'
values: '0x40'
- series: 'evo_region_objects_available_bytes{environment="dev"}'
values: '40000000000x40'
- series: 'evo_region_objects_reserve_bytes{environment="dev"}'
values: '1073741824x40'
- series: 'up{job="alertmanager"}'
values: '1x40'
alert_rule_test:
- {eval_time: 10m, alertname: EvoRegionDown, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionNotReady, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionQuarantined, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionLeaseMargin, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionFenced, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionStoreFailures, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionTickDebt, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionCheckpointStale, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionQueueSaturated, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionObjectsLow, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionObjectsFull, exp_alerts: []}
- {eval_time: 10m, alertname: AlertmanagerDown, exp_alerts: []}
- eval_time: 1m
alertname: Watchdog
exp_alerts:
- exp_labels: {severity: none}
exp_annotations:
summary: "Always firing: proves the alert pipeline end to end"
description: "Route to a heartbeat receiver once one exists; its absence then means monitoring is down."
# Outage: the region disappears, then comes back not ready and quarantined.
- interval: 15s
input_series:
- series: 'up{job="evo-region",environment="dev",world="100",region="1"}'
values: '0x20'
- series: 'evo_region_ready{environment="stage"}'
values: '0x20'
- series: 'evo_region_phase{environment="stage",phase="quarantined"}'
values: '1x20'
alert_rule_test:
- eval_time: 3m
alertname: EvoRegionDown
exp_alerts:
- exp_labels: {severity: critical, job: evo-region, environment: dev, world: "100", region: "1"}
exp_annotations:
summary: "Region dev/100/1 is not scrapeable"
description: "No operations endpoint answers. The process or its pod is gone; a forced deletion recovers in about 20 s, so 2 minutes is a real outage."
- eval_time: 1m
alertname: EvoRegionDown
exp_alerts: []
- eval_time: 3m
alertname: EvoRegionNotReady
exp_alerts:
- exp_labels: {severity: critical, environment: stage}
exp_annotations:
summary: "Region stage has no authoritative host"
description: "The process is alive but holds no lease or has not verified its head. Check the phase metric and the host log."
- eval_time: 2m
alertname: EvoRegionQuarantined
exp_alerts:
- exp_labels: {severity: critical, environment: stage, phase: quarantined}
exp_annotations:
summary: "Region stage is quarantined"
description: "Committed state or history failed verification. The region stays unavailable; never re-provision it. Audit with evo-world-clone audit."
# A 20 s lease-bound forced-deletion recovery must not page anyone.
- interval: 15s
input_series:
- series: 'evo_region_ready{environment="dev"}'
values: '1 1 0 0 1 1 1 1 1 1'
alert_rule_test:
- {eval_time: 45s, alertname: EvoRegionNotReady, exp_alerts: []} # During the dip.
- {eval_time: 2m, alertname: EvoRegionNotReady, exp_alerts: []}
# Lease, fences, storage and database failures.
- interval: 15s
input_series:
- series: 'evo_region_ready{environment="dev"}'
values: '1x20'
- series: 'evo_region_lease_remaining_milliseconds{environment="dev"}'
values: '6000x20'
- series: 'evo_region_fences_total{environment="dev"}'
values: '0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1'
- series: 'evo_region_store_failures_total{environment="dev"}'
values: '0+1x20'
- series: 'evo_region_unknown_outcomes_total{environment="dev"}'
values: '0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1'
alert_rule_test:
- eval_time: 2m
alertname: EvoRegionLeaseMargin
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev lease is not being renewed"
description: "Healthy renewals keep 15-20 s of lease. Under 10 s for a minute means renewals are failing; the host will fence itself."
- eval_time: 2m
alertname: EvoRegionFenced
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev host lost authority"
description: "A host was fenced within the last 10 minutes. Expected once after a forced deletion's predecessor resumes; repeated fences point to database or clock trouble."
- eval_time: 3m
alertname: EvoRegionStoreFailures
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev storage or database failures"
description: "Publication or database calls are failing; progress pauses and nothing is acknowledged until they succeed."
- eval_time: 2m
alertname: EvoRegionUnknownOutcomes
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev had commits with unknown outcomes"
description: "Commits timed out or lost their connection and were resolved by exact bytes. Occasional is safe; frequent indicates database latency near the 5 s commit deadline."
# Work waiting: debt, staleness, lag, a saturated queue and dropped inputs.
- interval: 15s
input_series:
- series: 'evo_region_tick_debt{environment="dev"}'
values: '20x20'
- series: 'evo_region_queued_inputs{environment="dev"}'
values: '60x20'
- series: 'evo_region_queue_capacity_inputs{environment="dev"}'
values: '64x20'
- series: 'evo_region_checkpoint_age_milliseconds{environment="dev"}'
values: '400000+15000x20'
- series: 'evo_region_commit_lag_ticks{environment="dev"}'
values: '300x20'
- series: 'evo_region_dropped_total{environment="dev",reason="fenced"}'
values: '0 0 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3'
- series: 'evo_region_dropped_total{environment="dev",reason="refused"}'
values: '0+1x20'
alert_rule_test:
- eval_time: 3m
alertname: EvoRegionTickDebt
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev is behind real time with inputs waiting"
description: "Due ticks are not being simulated although inputs are queued (catch-up is 8 ticks per step)."
- eval_time: 2m
alertname: EvoRegionCheckpointStale
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev has not committed for 5 minutes with work pending"
description: "An idle region does not commit; this fires only while inputs or simulated ticks wait for durability."
- eval_time: 3m
alertname: EvoRegionCommitLag
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev is not committing what it simulates"
description: "More than four full batches are simulated but not durable; acknowledgements are waiting."
- eval_time: 3m
alertname: EvoRegionQueueSaturated
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev input backlog near its bound"
description: "New inputs are being backpressured or rejected."
- eval_time: 2m
alertname: EvoRegionInputsDropped
exp_alerts:
- exp_labels: {severity: warning, environment: dev, reason: fenced}
exp_annotations:
summary: "Region dev accepted inputs that will never be acknowledged"
description: "Inputs were drained, fenced or quarantined (fenced); clients must resubmit to the current owner."
# Object volume filling up, then below its reserve.
- interval: 15s
input_series:
- series: 'evo_region_objects_available_bytes{environment="dev"}'
values: '1500000000x30'
- series: 'evo_region_objects_available_bytes{environment="stage"}'
values: '900000000x30' # Below the reserve but above half of it.
- series: 'evo_region_objects_reserve_bytes{environment="dev"}'
values: '1073741824x30'
- series: 'evo_region_objects_reserve_bytes{environment="stage"}'
values: '1073741824x30'
- series: 'up{job="alertmanager"}'
values: '0x30'
alert_rule_test:
- eval_time: 6m
alertname: EvoRegionObjectsLow
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev object volume is below twice its reserve"
description: "local-path volumes do not enforce their size; the node's disk is running out."
- exp_labels: {severity: warning, environment: stage}
exp_annotations:
summary: "Region stage object volume is below twice its reserve"
description: "local-path volumes do not enforce their size; the node's disk is running out."
- eval_time: 2m
alertname: EvoRegionObjectsFull
exp_alerts:
- exp_labels: {severity: critical, environment: stage}
exp_annotations:
summary: "Region stage object volume is below its reserve"
description: "New checkpoints are refused; the region cannot commit or acknowledge until space is freed."
- eval_time: 6m
alertname: AlertmanagerDown
exp_alerts:
- exp_labels: {severity: critical, job: alertmanager}
exp_annotations:
summary: "Alertmanager is not scrapeable"
description: "Alerts are evaluated but cannot be routed."