Add M8.5 observability: pinned Prometheus, tested alerts, capacity metrics

An AEvolution-only Prometheus 3.13.3 and Alertmanager 0.34.1, pinned by
digest and delivered by Argo (ADR-0072), scrape each region's operations
Service through static targets (no API access, no RBAC); the region
namespaces admit port 9100 from Prometheus only. Sixteen alert rules
cover authority, lease, commits, backlog and object capacity and are
proven by promtool injection tests in CI with tools copied from the same
pinned images. Two new gauges expose object volume free space and the
reserve. Alerts route to a null receiver until a destination is chosen;
no broker is adopted and the database outbox stays.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-30 06:16:36 -04:00
co-authored by Claude Opus 5.5
parent 2c79d4748c
commit b4493b3d0f
33 changed files with 1068 additions and 8 deletions
+19
View File
@@ -0,0 +1,19 @@
# M8.5 monitoring application (ADR-0072). Manual sync like the other AEvolution
# workloads; it holds no world data.
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: aevolution-monitoring
namespace: argocd
spec:
project: aevolution-monitoring
source:
repoURL: https://git.aeternalisgames.com/dingo/AEvolution.git
targetRevision: main
path: deploy/kubernetes/monitoring
destination:
server: https://kubernetes.default.svc
namespace: aevolution-monitoring
syncPolicy:
syncOptions:
- FailOnSharedResource=true
+23
View File
@@ -0,0 +1,23 @@
# M8.5 monitoring project (ADR-0072). Apply by hand once:
# kubectl apply -f deploy/argocd/monitoring/
apiVersion: argoproj.io/v1alpha1
kind: AppProject
metadata:
name: aevolution-monitoring
namespace: argocd
spec:
description: AEvolution metrics and alerting (aevolution-monitoring only)
sourceRepos:
- https://git.aeternalisgames.com/dingo/AEvolution.git
destinations:
- server: https://kubernetes.default.svc
namespace: aevolution-monitoring
clusterResourceWhitelist: []
namespaceResourceWhitelist:
- {group: "", kind: Service}
- {group: "", kind: ConfigMap}
- {group: "", kind: PersistentVolumeClaim}
- {group: apps, kind: Deployment}
- {group: networking.k8s.io, kind: NetworkPolicy}
orphanedResources:
warn: true
+2
View File
@@ -18,6 +18,8 @@ spec:
namespace: evolution-stage
- server: https://kubernetes.default.svc
namespace: evolution-world0
- server: https://kubernetes.default.svc
namespace: aevolution-monitoring
clusterResourceWhitelist:
- {group: "", kind: Namespace}
- {group: storage.k8s.io, kind: StorageClass}
+5
View File
@@ -57,6 +57,11 @@ COPY deploy/ deploy/
COPY tools/ci/deploy-policy-check.py tools/ci/
COPY tools/ops/ tools/ops/
RUN python3 tools/ci/deploy-policy-check.py
# M8.5: monitoring configuration and alert tests with the pinned images' own tools.
COPY --from=quay.io/prometheus/prometheus@sha256:6976aa8a60fec930796ce5772b8d12da7a318a5daa8d40d69c5c7819a05eeed7 /bin/promtool /usr/local/bin/promtool
COPY --from=quay.io/prometheus/alertmanager@sha256:e9733bafb1bdef9b00e25a21f8f99dc26a22224bf16641ad754d1649f4c3357a /bin/amtool /usr/local/bin/amtool
COPY tools/ci/monitoring-check.sh tools/ci/
RUN bash tools/ci/monitoring-check.sh
FROM ci-toolchain AS postgres-build
RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -0,0 +1,90 @@
# M8.5 Alertmanager (ADR-0072), pinned by digest. Every alert routes to the null
# receiver until the maintainer chooses a destination (docs/BACKLOG.md); alerts are
# still visible in its API and UI. Silences live in memory (emptyDir).
apiVersion: apps/v1
kind: Deployment
metadata:
name: alertmanager
labels:
app.kubernetes.io/name: alertmanager
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app.kubernetes.io/name: alertmanager
template:
metadata:
labels:
app.kubernetes.io/name: alertmanager
app.kubernetes.io/part-of: aevolution
spec:
serviceAccountName: aevolution-alertmanager
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 65534
runAsGroup: 65534
fsGroup: 65534
seccompProfile: {type: RuntimeDefault}
containers:
- name: alertmanager
image: quay.io/prometheus/alertmanager:v0.34.1@sha256:e9733bafb1bdef9b00e25a21f8f99dc26a22224bf16641ad754d1649f4c3357a
args:
- --config.file=/etc/alertmanager/alertmanager.yml
- --storage.path=/alertmanager
- --web.listen-address=:9093
- --cluster.listen-address= # Single instance: no gossip port.
ports:
- {name: web, containerPort: 9093}
readinessProbe:
httpGet: {path: /-/ready, port: web}
periodSeconds: 10
livenessProbe:
httpGet: {path: /-/healthy, port: web}
periodSeconds: 20
resources:
requests: {cpu: 50m, memory: 64Mi}
limits: {memory: 128Mi}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
volumeMounts:
- {name: config, mountPath: /etc/alertmanager, readOnly: true}
- {name: data, mountPath: /alertmanager}
volumes:
- name: config
configMap:
name: alertmanager-config
- name: data
emptyDir: {sizeLimit: 64Mi}
---
apiVersion: v1
kind: Service
metadata:
name: alertmanager
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: alertmanager
ports:
- {name: web, port: 9093, targetPort: web}
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: alertmanager
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: alertmanager
policyTypes: [Ingress]
ingress:
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: prometheus
ports:
- {protocol: TCP, port: 9093}
@@ -0,0 +1,17 @@
# M8.5 Alertmanager routing (ADR-0072). No destination is chosen yet
# (docs/BACKLOG.md): every alert routes to the null receiver, which keeps it visible
# in Alertmanager's API and UI without delivering it. Adding a receiver later is a
# reviewed change here plus a Secret for its credentials, never a credential in Git.
route:
receiver: "null"
group_by: [alertname, environment]
group_wait: 30s
group_interval: 5m
repeat_interval: 4h
receivers:
- name: "null"
inhibit_rules:
# A region that is down cannot also meaningfully be not ready or stale.
- source_matchers: [alertname = EvoRegionDown]
target_matchers: [alertname =~ "EvoRegion(NotReady|LeaseMargin|CheckpointStale)"]
equal: [environment]
@@ -0,0 +1,121 @@
# M8.5 region host alerts (ADR-0072). Thresholds follow the measured behaviour:
# the production lease is 20 s renewed every 5 s (15-20 s remaining when healthy);
# batches are at most 64 ticks with a 1 s flush; an idle region (no inputs, no
# gateway yet) always accrues tick debt, so debt and staleness only alert while
# inputs are waiting. Unit tests: evo-region.rules.test.yml (promtool).
groups:
- name: evo-region-authority
rules:
- alert: EvoRegionDown
expr: up{job="evo-region"} == 0
for: 2m
labels: {severity: critical}
annotations:
summary: "Region {{ $labels.environment }}/{{ $labels.world }}/{{ $labels.region }} is not scrapeable"
description: "No operations endpoint answers. The process or its pod is gone; a forced deletion recovers in about 20 s, so 2 minutes is a real outage."
- alert: EvoRegionNotReady
expr: evo_region_ready == 0
for: 2m
labels: {severity: critical}
annotations:
summary: "Region {{ $labels.environment }} has no authoritative host"
description: "The process is alive but holds no lease or has not verified its head. Check the phase metric and the host log."
- alert: EvoRegionQuarantined
expr: evo_region_phase{phase="quarantined"} == 1
for: 1m
labels: {severity: critical}
annotations:
summary: "Region {{ $labels.environment }} is quarantined"
description: "Committed state or history failed verification. The region stays unavailable; never re-provision it. Audit with evo-world-clone audit."
- alert: EvoRegionLeaseMargin
expr: evo_region_ready == 1 and evo_region_lease_remaining_milliseconds < 10000
for: 1m
labels: {severity: warning}
annotations:
summary: "Region {{ $labels.environment }} lease is not being renewed"
description: "Healthy renewals keep 15-20 s of lease. Under 10 s for a minute means renewals are failing; the host will fence itself."
- alert: EvoRegionFenced
expr: increase(evo_region_fences_total[10m]) > 0
labels: {severity: warning}
annotations:
summary: "Region {{ $labels.environment }} host lost authority"
description: "A host was fenced within the last 10 minutes. Expected once after a forced deletion's predecessor resumes; repeated fences point to database or clock trouble."
- name: evo-region-progress
rules:
- alert: EvoRegionStoreFailures
expr: increase(evo_region_store_failures_total[5m]) > 3
labels: {severity: warning}
annotations:
summary: "Region {{ $labels.environment }} storage or database failures"
description: "Publication or database calls are failing; progress pauses and nothing is acknowledged until they succeed."
- alert: EvoRegionUnknownOutcomes
expr: increase(evo_region_unknown_outcomes_total[15m]) > 0
labels: {severity: warning}
annotations:
summary: "Region {{ $labels.environment }} had commits with unknown outcomes"
description: "Commits timed out or lost their connection and were resolved by exact bytes. Occasional is safe; frequent indicates database latency near the 5 s commit deadline."
- alert: EvoRegionCommitLag
expr: evo_region_commit_lag_ticks > 256
for: 2m
labels: {severity: warning}
annotations:
summary: "Region {{ $labels.environment }} is not committing what it simulates"
description: "More than four full batches are simulated but not durable; acknowledgements are waiting."
- alert: EvoRegionTickDebt
expr: evo_region_tick_debt > 8 and evo_region_queued_inputs > 0
for: 2m
labels: {severity: warning}
annotations:
summary: "Region {{ $labels.environment }} is behind real time with inputs waiting"
description: "Due ticks are not being simulated although inputs are queued (catch-up is 8 ticks per step)."
- alert: EvoRegionCheckpointStale
expr: evo_region_checkpoint_age_milliseconds > 300000 and (evo_region_queued_inputs > 0 or evo_region_commit_lag_ticks > 0)
for: 1m
labels: {severity: warning}
annotations:
summary: "Region {{ $labels.environment }} has not committed for 5 minutes with work pending"
description: "An idle region does not commit; this fires only while inputs or simulated ticks wait for durability."
- alert: EvoRegionQueueSaturated
expr: evo_region_queued_inputs >= 0.9 * evo_region_queue_capacity_inputs
for: 2m
labels: {severity: warning}
annotations:
summary: "Region {{ $labels.environment }} input backlog near its bound"
description: "New inputs are being backpressured or rejected."
- alert: EvoRegionInputsDropped
expr: increase(evo_region_dropped_total{reason!="refused"}[10m]) > 0
labels: {severity: warning}
annotations:
summary: "Region {{ $labels.environment }} accepted inputs that will never be acknowledged"
description: "Inputs were drained, fenced or quarantined ({{ $labels.reason }}); clients must resubmit to the current owner."
- name: evo-region-storage
rules:
- alert: EvoRegionObjectsLow
expr: evo_region_objects_available_bytes < 2 * evo_region_objects_reserve_bytes
for: 5m
labels: {severity: warning}
annotations:
summary: "Region {{ $labels.environment }} object volume is below twice its reserve"
description: "local-path volumes do not enforce their size; the node's disk is running out."
- alert: EvoRegionObjectsFull
expr: evo_region_objects_available_bytes < evo_region_objects_reserve_bytes
for: 1m
labels: {severity: critical}
annotations:
summary: "Region {{ $labels.environment }} object volume is below its reserve"
description: "New checkpoints are refused; the region cannot commit or acknowledge until space is freed."
- name: aevolution-monitoring
rules:
- alert: Watchdog
expr: vector(1)
labels: {severity: none}
annotations:
summary: "Always firing: proves the alert pipeline end to end"
description: "Route to a heartbeat receiver once one exists; its absence then means monitoring is down."
- alert: AlertmanagerDown
expr: up{job="alertmanager"} == 0
for: 5m
labels: {severity: critical}
annotations:
summary: "Alertmanager is not scrapeable"
description: "Alerts are evaluated but cannot be routed."
@@ -0,0 +1,33 @@
# M8.5 Prometheus configuration (ADR-0072). Static targets: the region operations
# Service of each disposable environment. The scrape target carries the region's
# identity as labels (ADR-0066): the exposition itself has no world/region labels.
global:
scrape_interval: 15s
scrape_timeout: 5s
evaluation_interval: 15s
rule_files:
- /etc/prometheus/evo-region.rules.yml
alerting:
alertmanagers:
- static_configs:
- targets: [alertmanager:9093]
scrape_configs:
- job_name: evo-region
metrics_path: /metrics
# The exposition is at most 8 KiB (ADR-0066); anything larger is refused.
body_size_limit: 64KB
sample_limit: 64
static_configs:
- targets: [evo-region-ops.evolution-dev.svc:9100]
labels: {environment: dev, world: "100", region: "1"}
- targets: [evo-region-ops.evolution-stage.svc:9100]
labels: {environment: stage, world: "200", region: "1"}
- job_name: prometheus
static_configs:
- targets: [localhost:9090]
- job_name: alertmanager
static_configs:
- targets: [alertmanager:9093]
@@ -0,0 +1,20 @@
# M8.5 monitoring workload (ADR-0072), synced by the aevolution-monitoring Argo
# application. Configuration and rules live in config/ and are unit-tested in CI
# with promtool (tools/ci/monitoring-check.sh).
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: aevolution-monitoring
resources:
- prometheus.yaml
- alertmanager.yaml
configMapGenerator:
- name: prometheus-config
files:
- config/prometheus.yml
- config/evo-region.rules.yml
- name: alertmanager-config
files:
- config/alertmanager.yml
labels:
- pairs:
app.kubernetes.io/part-of: aevolution
@@ -0,0 +1,115 @@
# M8.5 Prometheus (ADR-0072), pinned by digest. Static scrape targets (no API access,
# no RBAC); 15 s scrape and evaluation; retention bounded by time and size.
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: prometheus-data
spec:
accessModes: [ReadWriteOnce]
storageClassName: local-path # Metrics are not world data; Delete reclaim is fine.
resources:
requests:
storage: 5Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: prometheus
labels:
app.kubernetes.io/name: prometheus
spec:
replicas: 1
strategy:
type: Recreate # One TSDB writer on an RWO volume.
selector:
matchLabels:
app.kubernetes.io/name: prometheus
template:
metadata:
labels:
app.kubernetes.io/name: prometheus
app.kubernetes.io/part-of: aevolution
spec:
serviceAccountName: aevolution-prometheus
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 65534
runAsGroup: 65534
fsGroup: 65534
seccompProfile: {type: RuntimeDefault}
containers:
- name: prometheus
image: quay.io/prometheus/prometheus:v3.13.3@sha256:6976aa8a60fec930796ce5772b8d12da7a318a5daa8d40d69c5c7819a05eeed7
args:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=15d
- --storage.tsdb.retention.size=3GB
- --web.listen-address=:9090
ports:
- {name: web, containerPort: 9090}
readinessProbe:
httpGet: {path: /-/ready, port: web}
periodSeconds: 10
livenessProbe:
httpGet: {path: /-/healthy, port: web}
periodSeconds: 20
failureThreshold: 3
resources:
requests: {cpu: 100m, memory: 256Mi}
limits: {memory: 1Gi}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
volumeMounts:
- {name: config, mountPath: /etc/prometheus, readOnly: true}
- {name: data, mountPath: /prometheus}
volumes:
- name: config
configMap:
name: prometheus-config
- name: data
persistentVolumeClaim:
claimName: prometheus-data
---
apiVersion: v1
kind: Service
metadata:
name: prometheus
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: prometheus
ports:
- {name: web, port: 9090, targetPort: web}
---
# Prometheus reaches the region operations ports, Alertmanager and itself only.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: prometheus
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: prometheus
policyTypes: [Egress]
egress:
- to:
- namespaceSelector:
matchExpressions:
- key: kubernetes.io/metadata.name
operator: In
values: [evolution-dev, evolution-stage]
podSelector:
matchLabels:
app.kubernetes.io/name: evo-region
ports:
- {protocol: TCP, port: 9100}
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: alertmanager
ports:
- {protocol: TCP, port: 9093}
@@ -0,0 +1,240 @@
# M8.5 alert injection tests (ADR-0072): promtool test rules. Each alert is shown
# firing on an injected series and staying silent on the healthy or idle case.
rule_files:
- ../config/evo-region.rules.yml
evaluation_interval: 15s
tests:
# Healthy idle region: ready, renewing, idle (debt accrues with no inputs).
- interval: 15s
input_series:
- series: 'up{job="evo-region",environment="dev"}'
values: '1x40'
- series: 'evo_region_ready{environment="dev"}'
values: '1x40'
- series: 'evo_region_phase{environment="dev",phase="running"}'
values: '1x40'
- series: 'evo_region_phase{environment="dev",phase="quarantined"}'
values: '0x40'
- series: 'evo_region_lease_remaining_milliseconds{environment="dev"}'
values: '18000x40'
- series: 'evo_region_tick_debt{environment="dev"}'
values: '0+20x40'
- series: 'evo_region_queued_inputs{environment="dev"}'
values: '0x40'
- series: 'evo_region_queue_capacity_inputs{environment="dev"}'
values: '64x40'
- series: 'evo_region_checkpoint_age_milliseconds{environment="dev"}'
values: '0+15000x40'
- series: 'evo_region_commit_lag_ticks{environment="dev"}'
values: '0x40'
- series: 'evo_region_fences_total{environment="dev"}'
values: '0x40'
- series: 'evo_region_store_failures_total{environment="dev"}'
values: '0x40'
- series: 'evo_region_objects_available_bytes{environment="dev"}'
values: '40000000000x40'
- series: 'evo_region_objects_reserve_bytes{environment="dev"}'
values: '1073741824x40'
- series: 'up{job="alertmanager"}'
values: '1x40'
alert_rule_test:
- {eval_time: 10m, alertname: EvoRegionDown, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionNotReady, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionQuarantined, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionLeaseMargin, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionFenced, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionStoreFailures, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionTickDebt, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionCheckpointStale, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionQueueSaturated, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionObjectsLow, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionObjectsFull, exp_alerts: []}
- {eval_time: 10m, alertname: AlertmanagerDown, exp_alerts: []}
- eval_time: 1m
alertname: Watchdog
exp_alerts:
- exp_labels: {severity: none}
exp_annotations:
summary: "Always firing: proves the alert pipeline end to end"
description: "Route to a heartbeat receiver once one exists; its absence then means monitoring is down."
# Outage: the region disappears, then comes back not ready and quarantined.
- interval: 15s
input_series:
- series: 'up{job="evo-region",environment="dev",world="100",region="1"}'
values: '0x20'
- series: 'evo_region_ready{environment="stage"}'
values: '0x20'
- series: 'evo_region_phase{environment="stage",phase="quarantined"}'
values: '1x20'
alert_rule_test:
- eval_time: 3m
alertname: EvoRegionDown
exp_alerts:
- exp_labels: {severity: critical, job: evo-region, environment: dev, world: "100", region: "1"}
exp_annotations:
summary: "Region dev/100/1 is not scrapeable"
description: "No operations endpoint answers. The process or its pod is gone; a forced deletion recovers in about 20 s, so 2 minutes is a real outage."
- eval_time: 1m
alertname: EvoRegionDown
exp_alerts: []
- eval_time: 3m
alertname: EvoRegionNotReady
exp_alerts:
- exp_labels: {severity: critical, environment: stage}
exp_annotations:
summary: "Region stage has no authoritative host"
description: "The process is alive but holds no lease or has not verified its head. Check the phase metric and the host log."
- eval_time: 2m
alertname: EvoRegionQuarantined
exp_alerts:
- exp_labels: {severity: critical, environment: stage, phase: quarantined}
exp_annotations:
summary: "Region stage is quarantined"
description: "Committed state or history failed verification. The region stays unavailable; never re-provision it. Audit with evo-world-clone audit."
# A 20 s lease-bound forced-deletion recovery must not page anyone.
- interval: 15s
input_series:
- series: 'evo_region_ready{environment="dev"}'
values: '1 1 0 0 1 1 1 1 1 1'
alert_rule_test:
- {eval_time: 45s, alertname: EvoRegionNotReady, exp_alerts: []} # During the dip.
- {eval_time: 2m, alertname: EvoRegionNotReady, exp_alerts: []}
# Lease, fences, storage and database failures.
- interval: 15s
input_series:
- series: 'evo_region_ready{environment="dev"}'
values: '1x20'
- series: 'evo_region_lease_remaining_milliseconds{environment="dev"}'
values: '6000x20'
- series: 'evo_region_fences_total{environment="dev"}'
values: '0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1'
- series: 'evo_region_store_failures_total{environment="dev"}'
values: '0+1x20'
- series: 'evo_region_unknown_outcomes_total{environment="dev"}'
values: '0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1'
alert_rule_test:
- eval_time: 2m
alertname: EvoRegionLeaseMargin
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev lease is not being renewed"
description: "Healthy renewals keep 15-20 s of lease. Under 10 s for a minute means renewals are failing; the host will fence itself."
- eval_time: 2m
alertname: EvoRegionFenced
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev host lost authority"
description: "A host was fenced within the last 10 minutes. Expected once after a forced deletion's predecessor resumes; repeated fences point to database or clock trouble."
- eval_time: 3m
alertname: EvoRegionStoreFailures
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev storage or database failures"
description: "Publication or database calls are failing; progress pauses and nothing is acknowledged until they succeed."
- eval_time: 2m
alertname: EvoRegionUnknownOutcomes
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev had commits with unknown outcomes"
description: "Commits timed out or lost their connection and were resolved by exact bytes. Occasional is safe; frequent indicates database latency near the 5 s commit deadline."
# Work waiting: debt, staleness, lag, a saturated queue and dropped inputs.
- interval: 15s
input_series:
- series: 'evo_region_tick_debt{environment="dev"}'
values: '20x20'
- series: 'evo_region_queued_inputs{environment="dev"}'
values: '60x20'
- series: 'evo_region_queue_capacity_inputs{environment="dev"}'
values: '64x20'
- series: 'evo_region_checkpoint_age_milliseconds{environment="dev"}'
values: '400000+15000x20'
- series: 'evo_region_commit_lag_ticks{environment="dev"}'
values: '300x20'
- series: 'evo_region_dropped_total{environment="dev",reason="fenced"}'
values: '0 0 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3'
- series: 'evo_region_dropped_total{environment="dev",reason="refused"}'
values: '0+1x20'
alert_rule_test:
- eval_time: 3m
alertname: EvoRegionTickDebt
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev is behind real time with inputs waiting"
description: "Due ticks are not being simulated although inputs are queued (catch-up is 8 ticks per step)."
- eval_time: 2m
alertname: EvoRegionCheckpointStale
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev has not committed for 5 minutes with work pending"
description: "An idle region does not commit; this fires only while inputs or simulated ticks wait for durability."
- eval_time: 3m
alertname: EvoRegionCommitLag
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev is not committing what it simulates"
description: "More than four full batches are simulated but not durable; acknowledgements are waiting."
- eval_time: 3m
alertname: EvoRegionQueueSaturated
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev input backlog near its bound"
description: "New inputs are being backpressured or rejected."
- eval_time: 2m
alertname: EvoRegionInputsDropped
exp_alerts:
- exp_labels: {severity: warning, environment: dev, reason: fenced}
exp_annotations:
summary: "Region dev accepted inputs that will never be acknowledged"
description: "Inputs were drained, fenced or quarantined (fenced); clients must resubmit to the current owner."
# Object volume filling up, then below its reserve.
- interval: 15s
input_series:
- series: 'evo_region_objects_available_bytes{environment="dev"}'
values: '1500000000x30'
- series: 'evo_region_objects_available_bytes{environment="stage"}'
values: '900000000x30' # Below the reserve but above half of it.
- series: 'evo_region_objects_reserve_bytes{environment="dev"}'
values: '1073741824x30'
- series: 'evo_region_objects_reserve_bytes{environment="stage"}'
values: '1073741824x30'
- series: 'up{job="alertmanager"}'
values: '0x30'
alert_rule_test:
- eval_time: 6m
alertname: EvoRegionObjectsLow
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev object volume is below twice its reserve"
description: "local-path volumes do not enforce their size; the node's disk is running out."
- exp_labels: {severity: warning, environment: stage}
exp_annotations:
summary: "Region stage object volume is below twice its reserve"
description: "local-path volumes do not enforce their size; the node's disk is running out."
- eval_time: 2m
alertname: EvoRegionObjectsFull
exp_alerts:
- exp_labels: {severity: critical, environment: stage}
exp_annotations:
summary: "Region stage object volume is below its reserve"
description: "New checkpoints are refused; the region cannot commit or acknowledge until space is freed."
- eval_time: 6m
alertname: AlertmanagerDown
exp_alerts:
- exp_labels: {severity: critical, job: alertmanager}
exp_annotations:
summary: "Alertmanager is not scrapeable"
description: "Alerts are evaluated but cannot be routed."
@@ -12,5 +12,6 @@ resources:
- ../dev
- ../stage
- ../world0
- ../monitoring
commonAnnotations:
argocd.argoproj.io/sync-options: Prune=false,Delete=false
@@ -0,0 +1,65 @@
apiVersion: v1
kind: ResourceQuota
metadata:
name: aevolution-quota
spec:
hard:
requests.cpu: "1"
requests.memory: 1Gi
limits.memory: 2Gi
pods: "6"
persistentvolumeclaims: "2"
requests.storage: 10Gi
services.loadbalancers: "0"
services.nodeports: "0"
---
apiVersion: v1
kind: LimitRange
metadata:
name: aevolution-defaults
spec:
limits:
- type: Container
defaultRequest: {cpu: 50m, memory: 64Mi}
default: {memory: 256Mi}
max: {memory: 2Gi}
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: aevolution-default-deny
spec:
podSelector: {}
policyTypes: [Ingress, Egress]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: aevolution-allow-dns
spec:
podSelector: {}
policyTypes: [Egress]
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
podSelector:
matchLabels:
k8s-app: kube-dns
ports:
- {protocol: UDP, port: 53}
- {protocol: TCP, port: 53}
---
# No Kubernetes API access: scrape targets are static, so no token and no RBAC.
apiVersion: v1
kind: ServiceAccount
metadata:
name: aevolution-prometheus
automountServiceAccountToken: false
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: aevolution-alertmanager
automountServiceAccountToken: false
@@ -0,0 +1,13 @@
# M8.5 monitoring namespace (ADR-0072): guardrails and identities owned by the
# platform app, like every AEvolution namespace; Prometheus and Alertmanager
# themselves are the aevolution-monitoring workload app.
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: aevolution-monitoring
resources:
- namespace.yaml
- guardrails.yaml
labels:
- pairs:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/managed-by: aevolution-platform
@@ -0,0 +1,10 @@
# Metrics and alerting for the AEvolution environments. Holds no world data.
apiVersion: v1
kind: Namespace
metadata:
name: aevolution-monitoring
labels:
aevolution.io/environment: monitoring
pod-security.kubernetes.io/enforce: restricted
pod-security.kubernetes.io/enforce-version: v1.32
pod-security.kubernetes.io/warn: restricted
@@ -8,6 +8,7 @@ resources:
- provision-job.yaml
- deployment.yaml
- service.yaml
- scrape-policy.yaml
labels:
- pairs:
app.kubernetes.io/part-of: aevolution
@@ -0,0 +1,21 @@
# M8.5: the region's operations port accepts connections only from the AEvolution
# Prometheus. Kubelet probes come from the node and are not affected (Calico).
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: evo-region-scrape
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: evo-region
policyTypes: [Ingress]
ingress:
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: aevolution-monitoring
podSelector:
matchLabels:
app.kubernetes.io/name: prometheus
ports:
- {protocol: TCP, port: 9100}