Add M8.5 observability: pinned Prometheus, tested alerts, capacity metrics
An AEvolution-only Prometheus 3.13.3 and Alertmanager 0.34.1, pinned by digest and delivered by Argo (ADR-0072), scrape each region's operations Service through static targets (no API access, no RBAC); the region namespaces admit port 9100 from Prometheus only. Sixteen alert rules cover authority, lease, commits, backlog and object capacity and are proven by promtool injection tests in CI with tools copied from the same pinned images. Two new gauges expose object volume free space and the reserve. Alerts route to a null receiver until a destination is chosen; no broker is adopted and the database outbox stays. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,19 @@
|
||||
# M8.5 monitoring application (ADR-0072). Manual sync like the other AEvolution
|
||||
# workloads; it holds no world data.
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: aevolution-monitoring
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: aevolution-monitoring
|
||||
source:
|
||||
repoURL: https://git.aeternalisgames.com/dingo/AEvolution.git
|
||||
targetRevision: main
|
||||
path: deploy/kubernetes/monitoring
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: aevolution-monitoring
|
||||
syncPolicy:
|
||||
syncOptions:
|
||||
- FailOnSharedResource=true
|
||||
@@ -0,0 +1,23 @@
|
||||
# M8.5 monitoring project (ADR-0072). Apply by hand once:
|
||||
# kubectl apply -f deploy/argocd/monitoring/
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: AppProject
|
||||
metadata:
|
||||
name: aevolution-monitoring
|
||||
namespace: argocd
|
||||
spec:
|
||||
description: AEvolution metrics and alerting (aevolution-monitoring only)
|
||||
sourceRepos:
|
||||
- https://git.aeternalisgames.com/dingo/AEvolution.git
|
||||
destinations:
|
||||
- server: https://kubernetes.default.svc
|
||||
namespace: aevolution-monitoring
|
||||
clusterResourceWhitelist: []
|
||||
namespaceResourceWhitelist:
|
||||
- {group: "", kind: Service}
|
||||
- {group: "", kind: ConfigMap}
|
||||
- {group: "", kind: PersistentVolumeClaim}
|
||||
- {group: apps, kind: Deployment}
|
||||
- {group: networking.k8s.io, kind: NetworkPolicy}
|
||||
orphanedResources:
|
||||
warn: true
|
||||
@@ -18,6 +18,8 @@ spec:
|
||||
namespace: evolution-stage
|
||||
- server: https://kubernetes.default.svc
|
||||
namespace: evolution-world0
|
||||
- server: https://kubernetes.default.svc
|
||||
namespace: aevolution-monitoring
|
||||
clusterResourceWhitelist:
|
||||
- {group: "", kind: Namespace}
|
||||
- {group: storage.k8s.io, kind: StorageClass}
|
||||
|
||||
@@ -57,6 +57,11 @@ COPY deploy/ deploy/
|
||||
COPY tools/ci/deploy-policy-check.py tools/ci/
|
||||
COPY tools/ops/ tools/ops/
|
||||
RUN python3 tools/ci/deploy-policy-check.py
|
||||
# M8.5: monitoring configuration and alert tests with the pinned images' own tools.
|
||||
COPY --from=quay.io/prometheus/prometheus@sha256:6976aa8a60fec930796ce5772b8d12da7a318a5daa8d40d69c5c7819a05eeed7 /bin/promtool /usr/local/bin/promtool
|
||||
COPY --from=quay.io/prometheus/alertmanager@sha256:e9733bafb1bdef9b00e25a21f8f99dc26a22224bf16641ad754d1649f4c3357a /bin/amtool /usr/local/bin/amtool
|
||||
COPY tools/ci/monitoring-check.sh tools/ci/
|
||||
RUN bash tools/ci/monitoring-check.sh
|
||||
|
||||
FROM ci-toolchain AS postgres-build
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
# M8.5 Alertmanager (ADR-0072), pinned by digest. Every alert routes to the null
|
||||
# receiver until the maintainer chooses a destination (docs/BACKLOG.md); alerts are
|
||||
# still visible in its API and UI. Silences live in memory (emptyDir).
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: alertmanager
|
||||
labels:
|
||||
app.kubernetes.io/name: alertmanager
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: alertmanager
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: alertmanager
|
||||
app.kubernetes.io/part-of: aevolution
|
||||
spec:
|
||||
serviceAccountName: aevolution-alertmanager
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65534
|
||||
runAsGroup: 65534
|
||||
fsGroup: 65534
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
containers:
|
||||
- name: alertmanager
|
||||
image: quay.io/prometheus/alertmanager:v0.34.1@sha256:e9733bafb1bdef9b00e25a21f8f99dc26a22224bf16641ad754d1649f4c3357a
|
||||
args:
|
||||
- --config.file=/etc/alertmanager/alertmanager.yml
|
||||
- --storage.path=/alertmanager
|
||||
- --web.listen-address=:9093
|
||||
- --cluster.listen-address= # Single instance: no gossip port.
|
||||
ports:
|
||||
- {name: web, containerPort: 9093}
|
||||
readinessProbe:
|
||||
httpGet: {path: /-/ready, port: web}
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet: {path: /-/healthy, port: web}
|
||||
periodSeconds: 20
|
||||
resources:
|
||||
requests: {cpu: 50m, memory: 64Mi}
|
||||
limits: {memory: 128Mi}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities: {drop: [ALL]}
|
||||
volumeMounts:
|
||||
- {name: config, mountPath: /etc/alertmanager, readOnly: true}
|
||||
- {name: data, mountPath: /alertmanager}
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: alertmanager-config
|
||||
- name: data
|
||||
emptyDir: {sizeLimit: 64Mi}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: alertmanager
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app.kubernetes.io/name: alertmanager
|
||||
ports:
|
||||
- {name: web, port: 9093, targetPort: web}
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: alertmanager
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: alertmanager
|
||||
policyTypes: [Ingress]
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
ports:
|
||||
- {protocol: TCP, port: 9093}
|
||||
@@ -0,0 +1,17 @@
|
||||
# M8.5 Alertmanager routing (ADR-0072). No destination is chosen yet
|
||||
# (docs/BACKLOG.md): every alert routes to the null receiver, which keeps it visible
|
||||
# in Alertmanager's API and UI without delivering it. Adding a receiver later is a
|
||||
# reviewed change here plus a Secret for its credentials, never a credential in Git.
|
||||
route:
|
||||
receiver: "null"
|
||||
group_by: [alertname, environment]
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 4h
|
||||
receivers:
|
||||
- name: "null"
|
||||
inhibit_rules:
|
||||
# A region that is down cannot also meaningfully be not ready or stale.
|
||||
- source_matchers: [alertname = EvoRegionDown]
|
||||
target_matchers: [alertname =~ "EvoRegion(NotReady|LeaseMargin|CheckpointStale)"]
|
||||
equal: [environment]
|
||||
@@ -0,0 +1,121 @@
|
||||
# M8.5 region host alerts (ADR-0072). Thresholds follow the measured behaviour:
|
||||
# the production lease is 20 s renewed every 5 s (15-20 s remaining when healthy);
|
||||
# batches are at most 64 ticks with a 1 s flush; an idle region (no inputs, no
|
||||
# gateway yet) always accrues tick debt, so debt and staleness only alert while
|
||||
# inputs are waiting. Unit tests: evo-region.rules.test.yml (promtool).
|
||||
groups:
|
||||
- name: evo-region-authority
|
||||
rules:
|
||||
- alert: EvoRegionDown
|
||||
expr: up{job="evo-region"} == 0
|
||||
for: 2m
|
||||
labels: {severity: critical}
|
||||
annotations:
|
||||
summary: "Region {{ $labels.environment }}/{{ $labels.world }}/{{ $labels.region }} is not scrapeable"
|
||||
description: "No operations endpoint answers. The process or its pod is gone; a forced deletion recovers in about 20 s, so 2 minutes is a real outage."
|
||||
- alert: EvoRegionNotReady
|
||||
expr: evo_region_ready == 0
|
||||
for: 2m
|
||||
labels: {severity: critical}
|
||||
annotations:
|
||||
summary: "Region {{ $labels.environment }} has no authoritative host"
|
||||
description: "The process is alive but holds no lease or has not verified its head. Check the phase metric and the host log."
|
||||
- alert: EvoRegionQuarantined
|
||||
expr: evo_region_phase{phase="quarantined"} == 1
|
||||
for: 1m
|
||||
labels: {severity: critical}
|
||||
annotations:
|
||||
summary: "Region {{ $labels.environment }} is quarantined"
|
||||
description: "Committed state or history failed verification. The region stays unavailable; never re-provision it. Audit with evo-world-clone audit."
|
||||
- alert: EvoRegionLeaseMargin
|
||||
expr: evo_region_ready == 1 and evo_region_lease_remaining_milliseconds < 10000
|
||||
for: 1m
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: "Region {{ $labels.environment }} lease is not being renewed"
|
||||
description: "Healthy renewals keep 15-20 s of lease. Under 10 s for a minute means renewals are failing; the host will fence itself."
|
||||
- alert: EvoRegionFenced
|
||||
expr: increase(evo_region_fences_total[10m]) > 0
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: "Region {{ $labels.environment }} host lost authority"
|
||||
description: "A host was fenced within the last 10 minutes. Expected once after a forced deletion's predecessor resumes; repeated fences point to database or clock trouble."
|
||||
- name: evo-region-progress
|
||||
rules:
|
||||
- alert: EvoRegionStoreFailures
|
||||
expr: increase(evo_region_store_failures_total[5m]) > 3
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: "Region {{ $labels.environment }} storage or database failures"
|
||||
description: "Publication or database calls are failing; progress pauses and nothing is acknowledged until they succeed."
|
||||
- alert: EvoRegionUnknownOutcomes
|
||||
expr: increase(evo_region_unknown_outcomes_total[15m]) > 0
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: "Region {{ $labels.environment }} had commits with unknown outcomes"
|
||||
description: "Commits timed out or lost their connection and were resolved by exact bytes. Occasional is safe; frequent indicates database latency near the 5 s commit deadline."
|
||||
- alert: EvoRegionCommitLag
|
||||
expr: evo_region_commit_lag_ticks > 256
|
||||
for: 2m
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: "Region {{ $labels.environment }} is not committing what it simulates"
|
||||
description: "More than four full batches are simulated but not durable; acknowledgements are waiting."
|
||||
- alert: EvoRegionTickDebt
|
||||
expr: evo_region_tick_debt > 8 and evo_region_queued_inputs > 0
|
||||
for: 2m
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: "Region {{ $labels.environment }} is behind real time with inputs waiting"
|
||||
description: "Due ticks are not being simulated although inputs are queued (catch-up is 8 ticks per step)."
|
||||
- alert: EvoRegionCheckpointStale
|
||||
expr: evo_region_checkpoint_age_milliseconds > 300000 and (evo_region_queued_inputs > 0 or evo_region_commit_lag_ticks > 0)
|
||||
for: 1m
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: "Region {{ $labels.environment }} has not committed for 5 minutes with work pending"
|
||||
description: "An idle region does not commit; this fires only while inputs or simulated ticks wait for durability."
|
||||
- alert: EvoRegionQueueSaturated
|
||||
expr: evo_region_queued_inputs >= 0.9 * evo_region_queue_capacity_inputs
|
||||
for: 2m
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: "Region {{ $labels.environment }} input backlog near its bound"
|
||||
description: "New inputs are being backpressured or rejected."
|
||||
- alert: EvoRegionInputsDropped
|
||||
expr: increase(evo_region_dropped_total{reason!="refused"}[10m]) > 0
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: "Region {{ $labels.environment }} accepted inputs that will never be acknowledged"
|
||||
description: "Inputs were drained, fenced or quarantined ({{ $labels.reason }}); clients must resubmit to the current owner."
|
||||
- name: evo-region-storage
|
||||
rules:
|
||||
- alert: EvoRegionObjectsLow
|
||||
expr: evo_region_objects_available_bytes < 2 * evo_region_objects_reserve_bytes
|
||||
for: 5m
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: "Region {{ $labels.environment }} object volume is below twice its reserve"
|
||||
description: "local-path volumes do not enforce their size; the node's disk is running out."
|
||||
- alert: EvoRegionObjectsFull
|
||||
expr: evo_region_objects_available_bytes < evo_region_objects_reserve_bytes
|
||||
for: 1m
|
||||
labels: {severity: critical}
|
||||
annotations:
|
||||
summary: "Region {{ $labels.environment }} object volume is below its reserve"
|
||||
description: "New checkpoints are refused; the region cannot commit or acknowledge until space is freed."
|
||||
- name: aevolution-monitoring
|
||||
rules:
|
||||
- alert: Watchdog
|
||||
expr: vector(1)
|
||||
labels: {severity: none}
|
||||
annotations:
|
||||
summary: "Always firing: proves the alert pipeline end to end"
|
||||
description: "Route to a heartbeat receiver once one exists; its absence then means monitoring is down."
|
||||
- alert: AlertmanagerDown
|
||||
expr: up{job="alertmanager"} == 0
|
||||
for: 5m
|
||||
labels: {severity: critical}
|
||||
annotations:
|
||||
summary: "Alertmanager is not scrapeable"
|
||||
description: "Alerts are evaluated but cannot be routed."
|
||||
@@ -0,0 +1,33 @@
|
||||
# M8.5 Prometheus configuration (ADR-0072). Static targets: the region operations
|
||||
# Service of each disposable environment. The scrape target carries the region's
|
||||
# identity as labels (ADR-0066): the exposition itself has no world/region labels.
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
scrape_timeout: 5s
|
||||
evaluation_interval: 15s
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/evo-region.rules.yml
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets: [alertmanager:9093]
|
||||
|
||||
scrape_configs:
|
||||
- job_name: evo-region
|
||||
metrics_path: /metrics
|
||||
# The exposition is at most 8 KiB (ADR-0066); anything larger is refused.
|
||||
body_size_limit: 64KB
|
||||
sample_limit: 64
|
||||
static_configs:
|
||||
- targets: [evo-region-ops.evolution-dev.svc:9100]
|
||||
labels: {environment: dev, world: "100", region: "1"}
|
||||
- targets: [evo-region-ops.evolution-stage.svc:9100]
|
||||
labels: {environment: stage, world: "200", region: "1"}
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: [localhost:9090]
|
||||
- job_name: alertmanager
|
||||
static_configs:
|
||||
- targets: [alertmanager:9093]
|
||||
@@ -0,0 +1,20 @@
|
||||
# M8.5 monitoring workload (ADR-0072), synced by the aevolution-monitoring Argo
|
||||
# application. Configuration and rules live in config/ and are unit-tested in CI
|
||||
# with promtool (tools/ci/monitoring-check.sh).
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: aevolution-monitoring
|
||||
resources:
|
||||
- prometheus.yaml
|
||||
- alertmanager.yaml
|
||||
configMapGenerator:
|
||||
- name: prometheus-config
|
||||
files:
|
||||
- config/prometheus.yml
|
||||
- config/evo-region.rules.yml
|
||||
- name: alertmanager-config
|
||||
files:
|
||||
- config/alertmanager.yml
|
||||
labels:
|
||||
- pairs:
|
||||
app.kubernetes.io/part-of: aevolution
|
||||
@@ -0,0 +1,115 @@
|
||||
# M8.5 Prometheus (ADR-0072), pinned by digest. Static scrape targets (no API access,
|
||||
# no RBAC); 15 s scrape and evaluation; retention bounded by time and size.
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: prometheus-data
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: local-path # Metrics are not world data; Delete reclaim is fine.
|
||||
resources:
|
||||
requests:
|
||||
storage: 5Gi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: prometheus
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate # One TSDB writer on an RWO volume.
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
app.kubernetes.io/part-of: aevolution
|
||||
spec:
|
||||
serviceAccountName: aevolution-prometheus
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65534
|
||||
runAsGroup: 65534
|
||||
fsGroup: 65534
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
containers:
|
||||
- name: prometheus
|
||||
image: quay.io/prometheus/prometheus:v3.13.3@sha256:6976aa8a60fec930796ce5772b8d12da7a318a5daa8d40d69c5c7819a05eeed7
|
||||
args:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=15d
|
||||
- --storage.tsdb.retention.size=3GB
|
||||
- --web.listen-address=:9090
|
||||
ports:
|
||||
- {name: web, containerPort: 9090}
|
||||
readinessProbe:
|
||||
httpGet: {path: /-/ready, port: web}
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet: {path: /-/healthy, port: web}
|
||||
periodSeconds: 20
|
||||
failureThreshold: 3
|
||||
resources:
|
||||
requests: {cpu: 100m, memory: 256Mi}
|
||||
limits: {memory: 1Gi}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities: {drop: [ALL]}
|
||||
volumeMounts:
|
||||
- {name: config, mountPath: /etc/prometheus, readOnly: true}
|
||||
- {name: data, mountPath: /prometheus}
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: prometheus-config
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: prometheus-data
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: prometheus
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app.kubernetes.io/name: prometheus
|
||||
ports:
|
||||
- {name: web, port: 9090, targetPort: web}
|
||||
---
|
||||
# Prometheus reaches the region operations ports, Alertmanager and itself only.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: prometheus
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
policyTypes: [Egress]
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchExpressions:
|
||||
- key: kubernetes.io/metadata.name
|
||||
operator: In
|
||||
values: [evolution-dev, evolution-stage]
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: evo-region
|
||||
ports:
|
||||
- {protocol: TCP, port: 9100}
|
||||
- to:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: alertmanager
|
||||
ports:
|
||||
- {protocol: TCP, port: 9093}
|
||||
@@ -0,0 +1,240 @@
|
||||
# M8.5 alert injection tests (ADR-0072): promtool test rules. Each alert is shown
|
||||
# firing on an injected series and staying silent on the healthy or idle case.
|
||||
rule_files:
|
||||
- ../config/evo-region.rules.yml
|
||||
evaluation_interval: 15s
|
||||
tests:
|
||||
# Healthy idle region: ready, renewing, idle (debt accrues with no inputs).
|
||||
- interval: 15s
|
||||
input_series:
|
||||
- series: 'up{job="evo-region",environment="dev"}'
|
||||
values: '1x40'
|
||||
- series: 'evo_region_ready{environment="dev"}'
|
||||
values: '1x40'
|
||||
- series: 'evo_region_phase{environment="dev",phase="running"}'
|
||||
values: '1x40'
|
||||
- series: 'evo_region_phase{environment="dev",phase="quarantined"}'
|
||||
values: '0x40'
|
||||
- series: 'evo_region_lease_remaining_milliseconds{environment="dev"}'
|
||||
values: '18000x40'
|
||||
- series: 'evo_region_tick_debt{environment="dev"}'
|
||||
values: '0+20x40'
|
||||
- series: 'evo_region_queued_inputs{environment="dev"}'
|
||||
values: '0x40'
|
||||
- series: 'evo_region_queue_capacity_inputs{environment="dev"}'
|
||||
values: '64x40'
|
||||
- series: 'evo_region_checkpoint_age_milliseconds{environment="dev"}'
|
||||
values: '0+15000x40'
|
||||
- series: 'evo_region_commit_lag_ticks{environment="dev"}'
|
||||
values: '0x40'
|
||||
- series: 'evo_region_fences_total{environment="dev"}'
|
||||
values: '0x40'
|
||||
- series: 'evo_region_store_failures_total{environment="dev"}'
|
||||
values: '0x40'
|
||||
- series: 'evo_region_objects_available_bytes{environment="dev"}'
|
||||
values: '40000000000x40'
|
||||
- series: 'evo_region_objects_reserve_bytes{environment="dev"}'
|
||||
values: '1073741824x40'
|
||||
- series: 'up{job="alertmanager"}'
|
||||
values: '1x40'
|
||||
alert_rule_test:
|
||||
- {eval_time: 10m, alertname: EvoRegionDown, exp_alerts: []}
|
||||
- {eval_time: 10m, alertname: EvoRegionNotReady, exp_alerts: []}
|
||||
- {eval_time: 10m, alertname: EvoRegionQuarantined, exp_alerts: []}
|
||||
- {eval_time: 10m, alertname: EvoRegionLeaseMargin, exp_alerts: []}
|
||||
- {eval_time: 10m, alertname: EvoRegionFenced, exp_alerts: []}
|
||||
- {eval_time: 10m, alertname: EvoRegionStoreFailures, exp_alerts: []}
|
||||
- {eval_time: 10m, alertname: EvoRegionTickDebt, exp_alerts: []}
|
||||
- {eval_time: 10m, alertname: EvoRegionCheckpointStale, exp_alerts: []}
|
||||
- {eval_time: 10m, alertname: EvoRegionQueueSaturated, exp_alerts: []}
|
||||
- {eval_time: 10m, alertname: EvoRegionObjectsLow, exp_alerts: []}
|
||||
- {eval_time: 10m, alertname: EvoRegionObjectsFull, exp_alerts: []}
|
||||
- {eval_time: 10m, alertname: AlertmanagerDown, exp_alerts: []}
|
||||
- eval_time: 1m
|
||||
alertname: Watchdog
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: none}
|
||||
exp_annotations:
|
||||
summary: "Always firing: proves the alert pipeline end to end"
|
||||
description: "Route to a heartbeat receiver once one exists; its absence then means monitoring is down."
|
||||
|
||||
# Outage: the region disappears, then comes back not ready and quarantined.
|
||||
- interval: 15s
|
||||
input_series:
|
||||
- series: 'up{job="evo-region",environment="dev",world="100",region="1"}'
|
||||
values: '0x20'
|
||||
- series: 'evo_region_ready{environment="stage"}'
|
||||
values: '0x20'
|
||||
- series: 'evo_region_phase{environment="stage",phase="quarantined"}'
|
||||
values: '1x20'
|
||||
alert_rule_test:
|
||||
- eval_time: 3m
|
||||
alertname: EvoRegionDown
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: critical, job: evo-region, environment: dev, world: "100", region: "1"}
|
||||
exp_annotations:
|
||||
summary: "Region dev/100/1 is not scrapeable"
|
||||
description: "No operations endpoint answers. The process or its pod is gone; a forced deletion recovers in about 20 s, so 2 minutes is a real outage."
|
||||
- eval_time: 1m
|
||||
alertname: EvoRegionDown
|
||||
exp_alerts: []
|
||||
- eval_time: 3m
|
||||
alertname: EvoRegionNotReady
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: critical, environment: stage}
|
||||
exp_annotations:
|
||||
summary: "Region stage has no authoritative host"
|
||||
description: "The process is alive but holds no lease or has not verified its head. Check the phase metric and the host log."
|
||||
- eval_time: 2m
|
||||
alertname: EvoRegionQuarantined
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: critical, environment: stage, phase: quarantined}
|
||||
exp_annotations:
|
||||
summary: "Region stage is quarantined"
|
||||
description: "Committed state or history failed verification. The region stays unavailable; never re-provision it. Audit with evo-world-clone audit."
|
||||
|
||||
# A 20 s lease-bound forced-deletion recovery must not page anyone.
|
||||
- interval: 15s
|
||||
input_series:
|
||||
- series: 'evo_region_ready{environment="dev"}'
|
||||
values: '1 1 0 0 1 1 1 1 1 1'
|
||||
alert_rule_test:
|
||||
- {eval_time: 45s, alertname: EvoRegionNotReady, exp_alerts: []} # During the dip.
|
||||
- {eval_time: 2m, alertname: EvoRegionNotReady, exp_alerts: []}
|
||||
|
||||
# Lease, fences, storage and database failures.
|
||||
- interval: 15s
|
||||
input_series:
|
||||
- series: 'evo_region_ready{environment="dev"}'
|
||||
values: '1x20'
|
||||
- series: 'evo_region_lease_remaining_milliseconds{environment="dev"}'
|
||||
values: '6000x20'
|
||||
- series: 'evo_region_fences_total{environment="dev"}'
|
||||
values: '0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1'
|
||||
- series: 'evo_region_store_failures_total{environment="dev"}'
|
||||
values: '0+1x20'
|
||||
- series: 'evo_region_unknown_outcomes_total{environment="dev"}'
|
||||
values: '0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1'
|
||||
alert_rule_test:
|
||||
- eval_time: 2m
|
||||
alertname: EvoRegionLeaseMargin
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning, environment: dev}
|
||||
exp_annotations:
|
||||
summary: "Region dev lease is not being renewed"
|
||||
description: "Healthy renewals keep 15-20 s of lease. Under 10 s for a minute means renewals are failing; the host will fence itself."
|
||||
- eval_time: 2m
|
||||
alertname: EvoRegionFenced
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning, environment: dev}
|
||||
exp_annotations:
|
||||
summary: "Region dev host lost authority"
|
||||
description: "A host was fenced within the last 10 minutes. Expected once after a forced deletion's predecessor resumes; repeated fences point to database or clock trouble."
|
||||
- eval_time: 3m
|
||||
alertname: EvoRegionStoreFailures
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning, environment: dev}
|
||||
exp_annotations:
|
||||
summary: "Region dev storage or database failures"
|
||||
description: "Publication or database calls are failing; progress pauses and nothing is acknowledged until they succeed."
|
||||
- eval_time: 2m
|
||||
alertname: EvoRegionUnknownOutcomes
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning, environment: dev}
|
||||
exp_annotations:
|
||||
summary: "Region dev had commits with unknown outcomes"
|
||||
description: "Commits timed out or lost their connection and were resolved by exact bytes. Occasional is safe; frequent indicates database latency near the 5 s commit deadline."
|
||||
|
||||
# Work waiting: debt, staleness, lag, a saturated queue and dropped inputs.
|
||||
- interval: 15s
|
||||
input_series:
|
||||
- series: 'evo_region_tick_debt{environment="dev"}'
|
||||
values: '20x20'
|
||||
- series: 'evo_region_queued_inputs{environment="dev"}'
|
||||
values: '60x20'
|
||||
- series: 'evo_region_queue_capacity_inputs{environment="dev"}'
|
||||
values: '64x20'
|
||||
- series: 'evo_region_checkpoint_age_milliseconds{environment="dev"}'
|
||||
values: '400000+15000x20'
|
||||
- series: 'evo_region_commit_lag_ticks{environment="dev"}'
|
||||
values: '300x20'
|
||||
- series: 'evo_region_dropped_total{environment="dev",reason="fenced"}'
|
||||
values: '0 0 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3'
|
||||
- series: 'evo_region_dropped_total{environment="dev",reason="refused"}'
|
||||
values: '0+1x20'
|
||||
alert_rule_test:
|
||||
- eval_time: 3m
|
||||
alertname: EvoRegionTickDebt
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning, environment: dev}
|
||||
exp_annotations:
|
||||
summary: "Region dev is behind real time with inputs waiting"
|
||||
description: "Due ticks are not being simulated although inputs are queued (catch-up is 8 ticks per step)."
|
||||
- eval_time: 2m
|
||||
alertname: EvoRegionCheckpointStale
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning, environment: dev}
|
||||
exp_annotations:
|
||||
summary: "Region dev has not committed for 5 minutes with work pending"
|
||||
description: "An idle region does not commit; this fires only while inputs or simulated ticks wait for durability."
|
||||
- eval_time: 3m
|
||||
alertname: EvoRegionCommitLag
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning, environment: dev}
|
||||
exp_annotations:
|
||||
summary: "Region dev is not committing what it simulates"
|
||||
description: "More than four full batches are simulated but not durable; acknowledgements are waiting."
|
||||
- eval_time: 3m
|
||||
alertname: EvoRegionQueueSaturated
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning, environment: dev}
|
||||
exp_annotations:
|
||||
summary: "Region dev input backlog near its bound"
|
||||
description: "New inputs are being backpressured or rejected."
|
||||
- eval_time: 2m
|
||||
alertname: EvoRegionInputsDropped
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning, environment: dev, reason: fenced}
|
||||
exp_annotations:
|
||||
summary: "Region dev accepted inputs that will never be acknowledged"
|
||||
description: "Inputs were drained, fenced or quarantined (fenced); clients must resubmit to the current owner."
|
||||
|
||||
# Object volume filling up, then below its reserve.
|
||||
- interval: 15s
|
||||
input_series:
|
||||
- series: 'evo_region_objects_available_bytes{environment="dev"}'
|
||||
values: '1500000000x30'
|
||||
- series: 'evo_region_objects_available_bytes{environment="stage"}'
|
||||
values: '900000000x30' # Below the reserve but above half of it.
|
||||
- series: 'evo_region_objects_reserve_bytes{environment="dev"}'
|
||||
values: '1073741824x30'
|
||||
- series: 'evo_region_objects_reserve_bytes{environment="stage"}'
|
||||
values: '1073741824x30'
|
||||
- series: 'up{job="alertmanager"}'
|
||||
values: '0x30'
|
||||
alert_rule_test:
|
||||
- eval_time: 6m
|
||||
alertname: EvoRegionObjectsLow
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: warning, environment: dev}
|
||||
exp_annotations:
|
||||
summary: "Region dev object volume is below twice its reserve"
|
||||
description: "local-path volumes do not enforce their size; the node's disk is running out."
|
||||
- exp_labels: {severity: warning, environment: stage}
|
||||
exp_annotations:
|
||||
summary: "Region stage object volume is below twice its reserve"
|
||||
description: "local-path volumes do not enforce their size; the node's disk is running out."
|
||||
- eval_time: 2m
|
||||
alertname: EvoRegionObjectsFull
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: critical, environment: stage}
|
||||
exp_annotations:
|
||||
summary: "Region stage object volume is below its reserve"
|
||||
description: "New checkpoints are refused; the region cannot commit or acknowledge until space is freed."
|
||||
- eval_time: 6m
|
||||
alertname: AlertmanagerDown
|
||||
exp_alerts:
|
||||
- exp_labels: {severity: critical, job: alertmanager}
|
||||
exp_annotations:
|
||||
summary: "Alertmanager is not scrapeable"
|
||||
description: "Alerts are evaluated but cannot be routed."
|
||||
@@ -12,5 +12,6 @@ resources:
|
||||
- ../dev
|
||||
- ../stage
|
||||
- ../world0
|
||||
- ../monitoring
|
||||
commonAnnotations:
|
||||
argocd.argoproj.io/sync-options: Prune=false,Delete=false
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
apiVersion: v1
|
||||
kind: ResourceQuota
|
||||
metadata:
|
||||
name: aevolution-quota
|
||||
spec:
|
||||
hard:
|
||||
requests.cpu: "1"
|
||||
requests.memory: 1Gi
|
||||
limits.memory: 2Gi
|
||||
pods: "6"
|
||||
persistentvolumeclaims: "2"
|
||||
requests.storage: 10Gi
|
||||
services.loadbalancers: "0"
|
||||
services.nodeports: "0"
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: LimitRange
|
||||
metadata:
|
||||
name: aevolution-defaults
|
||||
spec:
|
||||
limits:
|
||||
- type: Container
|
||||
defaultRequest: {cpu: 50m, memory: 64Mi}
|
||||
default: {memory: 256Mi}
|
||||
max: {memory: 2Gi}
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: aevolution-default-deny
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes: [Ingress, Egress]
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: aevolution-allow-dns
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes: [Egress]
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- {protocol: UDP, port: 53}
|
||||
- {protocol: TCP, port: 53}
|
||||
---
|
||||
# No Kubernetes API access: scrape targets are static, so no token and no RBAC.
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: aevolution-prometheus
|
||||
automountServiceAccountToken: false
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: aevolution-alertmanager
|
||||
automountServiceAccountToken: false
|
||||
@@ -0,0 +1,13 @@
|
||||
# M8.5 monitoring namespace (ADR-0072): guardrails and identities owned by the
|
||||
# platform app, like every AEvolution namespace; Prometheus and Alertmanager
|
||||
# themselves are the aevolution-monitoring workload app.
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: aevolution-monitoring
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- guardrails.yaml
|
||||
labels:
|
||||
- pairs:
|
||||
app.kubernetes.io/part-of: aevolution
|
||||
app.kubernetes.io/managed-by: aevolution-platform
|
||||
@@ -0,0 +1,10 @@
|
||||
# Metrics and alerting for the AEvolution environments. Holds no world data.
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: aevolution-monitoring
|
||||
labels:
|
||||
aevolution.io/environment: monitoring
|
||||
pod-security.kubernetes.io/enforce: restricted
|
||||
pod-security.kubernetes.io/enforce-version: v1.32
|
||||
pod-security.kubernetes.io/warn: restricted
|
||||
@@ -8,6 +8,7 @@ resources:
|
||||
- provision-job.yaml
|
||||
- deployment.yaml
|
||||
- service.yaml
|
||||
- scrape-policy.yaml
|
||||
labels:
|
||||
- pairs:
|
||||
app.kubernetes.io/part-of: aevolution
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# M8.5: the region's operations port accepts connections only from the AEvolution
|
||||
# Prometheus. Kubelet probes come from the node and are not affected (Calico).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: evo-region-scrape
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: evo-region
|
||||
policyTypes: [Ingress]
|
||||
ingress:
|
||||
- from:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: aevolution-monitoring
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
ports:
|
||||
- {protocol: TCP, port: 9100}
|
||||
Reference in New Issue
Block a user