Add M8.5 observability: pinned Prometheus, tested alerts, capacity metrics

An AEvolution-only Prometheus 3.13.3 and Alertmanager 0.34.1, pinned by
digest and delivered by Argo (ADR-0072), scrape each region's operations
Service through static targets (no API access, no RBAC); the region
namespaces admit port 9100 from Prometheus only. Sixteen alert rules
cover authority, lease, commits, backlog and object capacity and are
proven by promtool injection tests in CI with tools copied from the same
pinned images. Two new gauges expose object volume free space and the
reserve. Alerts route to a null receiver until a destination is chosen;
no broker is adopted and the database outbox stays.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-30 06:16:36 -04:00
co-authored by Claude Opus 5.5
parent 2c79d4748c
commit b4493b3d0f
33 changed files with 1068 additions and 8 deletions
+2
View File
@@ -310,6 +310,7 @@ template <typename State> int serve(Options& o) {
server::OpsCounters counters;
counters.queue_capacity_inputs = o.host.max_queued_inputs;
counters.queue_capacity_bytes = o.host.max_queued_bytes;
counters.objects_reserve_bytes = o.objects_reserve_mib * 1024 * 1024;
std::uint64_t last_sequence = 0;
auto last_checkpoint = std::chrono::steady_clock::now();
bool recovered = false;
@@ -326,6 +327,7 @@ template <typename State> int serve(Options& o) {
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - last_checkpoint)
.count());
counters.objects_available_bytes = host.objects().available_bytes().value_or(0);
return {200, server::render_metrics(s, counters),
"text/plain; version=0.0.4; charset=utf-8"};
}
+19
View File
@@ -0,0 +1,19 @@
# M8.5 monitoring application (ADR-0072). Manual sync like the other AEvolution
# workloads; it holds no world data.
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: aevolution-monitoring
namespace: argocd
spec:
project: aevolution-monitoring
source:
repoURL: https://git.aeternalisgames.com/dingo/AEvolution.git
targetRevision: main
path: deploy/kubernetes/monitoring
destination:
server: https://kubernetes.default.svc
namespace: aevolution-monitoring
syncPolicy:
syncOptions:
- FailOnSharedResource=true
+23
View File
@@ -0,0 +1,23 @@
# M8.5 monitoring project (ADR-0072). Apply by hand once:
# kubectl apply -f deploy/argocd/monitoring/
apiVersion: argoproj.io/v1alpha1
kind: AppProject
metadata:
name: aevolution-monitoring
namespace: argocd
spec:
description: AEvolution metrics and alerting (aevolution-monitoring only)
sourceRepos:
- https://git.aeternalisgames.com/dingo/AEvolution.git
destinations:
- server: https://kubernetes.default.svc
namespace: aevolution-monitoring
clusterResourceWhitelist: []
namespaceResourceWhitelist:
- {group: "", kind: Service}
- {group: "", kind: ConfigMap}
- {group: "", kind: PersistentVolumeClaim}
- {group: apps, kind: Deployment}
- {group: networking.k8s.io, kind: NetworkPolicy}
orphanedResources:
warn: true
+2
View File
@@ -18,6 +18,8 @@ spec:
namespace: evolution-stage
- server: https://kubernetes.default.svc
namespace: evolution-world0
- server: https://kubernetes.default.svc
namespace: aevolution-monitoring
clusterResourceWhitelist:
- {group: "", kind: Namespace}
- {group: storage.k8s.io, kind: StorageClass}
+5
View File
@@ -57,6 +57,11 @@ COPY deploy/ deploy/
COPY tools/ci/deploy-policy-check.py tools/ci/
COPY tools/ops/ tools/ops/
RUN python3 tools/ci/deploy-policy-check.py
# M8.5: monitoring configuration and alert tests with the pinned images' own tools.
COPY --from=quay.io/prometheus/prometheus@sha256:6976aa8a60fec930796ce5772b8d12da7a318a5daa8d40d69c5c7819a05eeed7 /bin/promtool /usr/local/bin/promtool
COPY --from=quay.io/prometheus/alertmanager@sha256:e9733bafb1bdef9b00e25a21f8f99dc26a22224bf16641ad754d1649f4c3357a /bin/amtool /usr/local/bin/amtool
COPY tools/ci/monitoring-check.sh tools/ci/
RUN bash tools/ci/monitoring-check.sh
FROM ci-toolchain AS postgres-build
RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -0,0 +1,90 @@
# M8.5 Alertmanager (ADR-0072), pinned by digest. Every alert routes to the null
# receiver until the maintainer chooses a destination (docs/BACKLOG.md); alerts are
# still visible in its API and UI. Silences live in memory (emptyDir).
apiVersion: apps/v1
kind: Deployment
metadata:
name: alertmanager
labels:
app.kubernetes.io/name: alertmanager
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app.kubernetes.io/name: alertmanager
template:
metadata:
labels:
app.kubernetes.io/name: alertmanager
app.kubernetes.io/part-of: aevolution
spec:
serviceAccountName: aevolution-alertmanager
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 65534
runAsGroup: 65534
fsGroup: 65534
seccompProfile: {type: RuntimeDefault}
containers:
- name: alertmanager
image: quay.io/prometheus/alertmanager:v0.34.1@sha256:e9733bafb1bdef9b00e25a21f8f99dc26a22224bf16641ad754d1649f4c3357a
args:
- --config.file=/etc/alertmanager/alertmanager.yml
- --storage.path=/alertmanager
- --web.listen-address=:9093
- --cluster.listen-address= # Single instance: no gossip port.
ports:
- {name: web, containerPort: 9093}
readinessProbe:
httpGet: {path: /-/ready, port: web}
periodSeconds: 10
livenessProbe:
httpGet: {path: /-/healthy, port: web}
periodSeconds: 20
resources:
requests: {cpu: 50m, memory: 64Mi}
limits: {memory: 128Mi}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
volumeMounts:
- {name: config, mountPath: /etc/alertmanager, readOnly: true}
- {name: data, mountPath: /alertmanager}
volumes:
- name: config
configMap:
name: alertmanager-config
- name: data
emptyDir: {sizeLimit: 64Mi}
---
apiVersion: v1
kind: Service
metadata:
name: alertmanager
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: alertmanager
ports:
- {name: web, port: 9093, targetPort: web}
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: alertmanager
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: alertmanager
policyTypes: [Ingress]
ingress:
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: prometheus
ports:
- {protocol: TCP, port: 9093}
@@ -0,0 +1,17 @@
# M8.5 Alertmanager routing (ADR-0072). No destination is chosen yet
# (docs/BACKLOG.md): every alert routes to the null receiver, which keeps it visible
# in Alertmanager's API and UI without delivering it. Adding a receiver later is a
# reviewed change here plus a Secret for its credentials, never a credential in Git.
route:
receiver: "null"
group_by: [alertname, environment]
group_wait: 30s
group_interval: 5m
repeat_interval: 4h
receivers:
- name: "null"
inhibit_rules:
# A region that is down cannot also meaningfully be not ready or stale.
- source_matchers: [alertname = EvoRegionDown]
target_matchers: [alertname =~ "EvoRegion(NotReady|LeaseMargin|CheckpointStale)"]
equal: [environment]
@@ -0,0 +1,121 @@
# M8.5 region host alerts (ADR-0072). Thresholds follow the measured behaviour:
# the production lease is 20 s renewed every 5 s (15-20 s remaining when healthy);
# batches are at most 64 ticks with a 1 s flush; an idle region (no inputs, no
# gateway yet) always accrues tick debt, so debt and staleness only alert while
# inputs are waiting. Unit tests: evo-region.rules.test.yml (promtool).
groups:
- name: evo-region-authority
rules:
- alert: EvoRegionDown
expr: up{job="evo-region"} == 0
for: 2m
labels: {severity: critical}
annotations:
summary: "Region {{ $labels.environment }}/{{ $labels.world }}/{{ $labels.region }} is not scrapeable"
description: "No operations endpoint answers. The process or its pod is gone; a forced deletion recovers in about 20 s, so 2 minutes is a real outage."
- alert: EvoRegionNotReady
expr: evo_region_ready == 0
for: 2m
labels: {severity: critical}
annotations:
summary: "Region {{ $labels.environment }} has no authoritative host"
description: "The process is alive but holds no lease or has not verified its head. Check the phase metric and the host log."
- alert: EvoRegionQuarantined
expr: evo_region_phase{phase="quarantined"} == 1
for: 1m
labels: {severity: critical}
annotations:
summary: "Region {{ $labels.environment }} is quarantined"
description: "Committed state or history failed verification. The region stays unavailable; never re-provision it. Audit with evo-world-clone audit."
- alert: EvoRegionLeaseMargin
expr: evo_region_ready == 1 and evo_region_lease_remaining_milliseconds < 10000
for: 1m
labels: {severity: warning}
annotations:
summary: "Region {{ $labels.environment }} lease is not being renewed"
description: "Healthy renewals keep 15-20 s of lease. Under 10 s for a minute means renewals are failing; the host will fence itself."
- alert: EvoRegionFenced
expr: increase(evo_region_fences_total[10m]) > 0
labels: {severity: warning}
annotations:
summary: "Region {{ $labels.environment }} host lost authority"
description: "A host was fenced within the last 10 minutes. Expected once after a forced deletion's predecessor resumes; repeated fences point to database or clock trouble."
- name: evo-region-progress
rules:
- alert: EvoRegionStoreFailures
expr: increase(evo_region_store_failures_total[5m]) > 3
labels: {severity: warning}
annotations:
summary: "Region {{ $labels.environment }} storage or database failures"
description: "Publication or database calls are failing; progress pauses and nothing is acknowledged until they succeed."
- alert: EvoRegionUnknownOutcomes
expr: increase(evo_region_unknown_outcomes_total[15m]) > 0
labels: {severity: warning}
annotations:
summary: "Region {{ $labels.environment }} had commits with unknown outcomes"
description: "Commits timed out or lost their connection and were resolved by exact bytes. Occasional is safe; frequent indicates database latency near the 5 s commit deadline."
- alert: EvoRegionCommitLag
expr: evo_region_commit_lag_ticks > 256
for: 2m
labels: {severity: warning}
annotations:
summary: "Region {{ $labels.environment }} is not committing what it simulates"
description: "More than four full batches are simulated but not durable; acknowledgements are waiting."
- alert: EvoRegionTickDebt
expr: evo_region_tick_debt > 8 and evo_region_queued_inputs > 0
for: 2m
labels: {severity: warning}
annotations:
summary: "Region {{ $labels.environment }} is behind real time with inputs waiting"
description: "Due ticks are not being simulated although inputs are queued (catch-up is 8 ticks per step)."
- alert: EvoRegionCheckpointStale
expr: evo_region_checkpoint_age_milliseconds > 300000 and (evo_region_queued_inputs > 0 or evo_region_commit_lag_ticks > 0)
for: 1m
labels: {severity: warning}
annotations:
summary: "Region {{ $labels.environment }} has not committed for 5 minutes with work pending"
description: "An idle region does not commit; this fires only while inputs or simulated ticks wait for durability."
- alert: EvoRegionQueueSaturated
expr: evo_region_queued_inputs >= 0.9 * evo_region_queue_capacity_inputs
for: 2m
labels: {severity: warning}
annotations:
summary: "Region {{ $labels.environment }} input backlog near its bound"
description: "New inputs are being backpressured or rejected."
- alert: EvoRegionInputsDropped
expr: increase(evo_region_dropped_total{reason!="refused"}[10m]) > 0
labels: {severity: warning}
annotations:
summary: "Region {{ $labels.environment }} accepted inputs that will never be acknowledged"
description: "Inputs were drained, fenced or quarantined ({{ $labels.reason }}); clients must resubmit to the current owner."
- name: evo-region-storage
rules:
- alert: EvoRegionObjectsLow
expr: evo_region_objects_available_bytes < 2 * evo_region_objects_reserve_bytes
for: 5m
labels: {severity: warning}
annotations:
summary: "Region {{ $labels.environment }} object volume is below twice its reserve"
description: "local-path volumes do not enforce their size; the node's disk is running out."
- alert: EvoRegionObjectsFull
expr: evo_region_objects_available_bytes < evo_region_objects_reserve_bytes
for: 1m
labels: {severity: critical}
annotations:
summary: "Region {{ $labels.environment }} object volume is below its reserve"
description: "New checkpoints are refused; the region cannot commit or acknowledge until space is freed."
- name: aevolution-monitoring
rules:
- alert: Watchdog
expr: vector(1)
labels: {severity: none}
annotations:
summary: "Always firing: proves the alert pipeline end to end"
description: "Route to a heartbeat receiver once one exists; its absence then means monitoring is down."
- alert: AlertmanagerDown
expr: up{job="alertmanager"} == 0
for: 5m
labels: {severity: critical}
annotations:
summary: "Alertmanager is not scrapeable"
description: "Alerts are evaluated but cannot be routed."
@@ -0,0 +1,33 @@
# M8.5 Prometheus configuration (ADR-0072). Static targets: the region operations
# Service of each disposable environment. The scrape target carries the region's
# identity as labels (ADR-0066): the exposition itself has no world/region labels.
global:
scrape_interval: 15s
scrape_timeout: 5s
evaluation_interval: 15s
rule_files:
- /etc/prometheus/evo-region.rules.yml
alerting:
alertmanagers:
- static_configs:
- targets: [alertmanager:9093]
scrape_configs:
- job_name: evo-region
metrics_path: /metrics
# The exposition is at most 8 KiB (ADR-0066); anything larger is refused.
body_size_limit: 64KB
sample_limit: 64
static_configs:
- targets: [evo-region-ops.evolution-dev.svc:9100]
labels: {environment: dev, world: "100", region: "1"}
- targets: [evo-region-ops.evolution-stage.svc:9100]
labels: {environment: stage, world: "200", region: "1"}
- job_name: prometheus
static_configs:
- targets: [localhost:9090]
- job_name: alertmanager
static_configs:
- targets: [alertmanager:9093]
@@ -0,0 +1,20 @@
# M8.5 monitoring workload (ADR-0072), synced by the aevolution-monitoring Argo
# application. Configuration and rules live in config/ and are unit-tested in CI
# with promtool (tools/ci/monitoring-check.sh).
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: aevolution-monitoring
resources:
- prometheus.yaml
- alertmanager.yaml
configMapGenerator:
- name: prometheus-config
files:
- config/prometheus.yml
- config/evo-region.rules.yml
- name: alertmanager-config
files:
- config/alertmanager.yml
labels:
- pairs:
app.kubernetes.io/part-of: aevolution
@@ -0,0 +1,115 @@
# M8.5 Prometheus (ADR-0072), pinned by digest. Static scrape targets (no API access,
# no RBAC); 15 s scrape and evaluation; retention bounded by time and size.
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: prometheus-data
spec:
accessModes: [ReadWriteOnce]
storageClassName: local-path # Metrics are not world data; Delete reclaim is fine.
resources:
requests:
storage: 5Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: prometheus
labels:
app.kubernetes.io/name: prometheus
spec:
replicas: 1
strategy:
type: Recreate # One TSDB writer on an RWO volume.
selector:
matchLabels:
app.kubernetes.io/name: prometheus
template:
metadata:
labels:
app.kubernetes.io/name: prometheus
app.kubernetes.io/part-of: aevolution
spec:
serviceAccountName: aevolution-prometheus
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 65534
runAsGroup: 65534
fsGroup: 65534
seccompProfile: {type: RuntimeDefault}
containers:
- name: prometheus
image: quay.io/prometheus/prometheus:v3.13.3@sha256:6976aa8a60fec930796ce5772b8d12da7a318a5daa8d40d69c5c7819a05eeed7
args:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=15d
- --storage.tsdb.retention.size=3GB
- --web.listen-address=:9090
ports:
- {name: web, containerPort: 9090}
readinessProbe:
httpGet: {path: /-/ready, port: web}
periodSeconds: 10
livenessProbe:
httpGet: {path: /-/healthy, port: web}
periodSeconds: 20
failureThreshold: 3
resources:
requests: {cpu: 100m, memory: 256Mi}
limits: {memory: 1Gi}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
volumeMounts:
- {name: config, mountPath: /etc/prometheus, readOnly: true}
- {name: data, mountPath: /prometheus}
volumes:
- name: config
configMap:
name: prometheus-config
- name: data
persistentVolumeClaim:
claimName: prometheus-data
---
apiVersion: v1
kind: Service
metadata:
name: prometheus
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: prometheus
ports:
- {name: web, port: 9090, targetPort: web}
---
# Prometheus reaches the region operations ports, Alertmanager and itself only.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: prometheus
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: prometheus
policyTypes: [Egress]
egress:
- to:
- namespaceSelector:
matchExpressions:
- key: kubernetes.io/metadata.name
operator: In
values: [evolution-dev, evolution-stage]
podSelector:
matchLabels:
app.kubernetes.io/name: evo-region
ports:
- {protocol: TCP, port: 9100}
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: alertmanager
ports:
- {protocol: TCP, port: 9093}
@@ -0,0 +1,240 @@
# M8.5 alert injection tests (ADR-0072): promtool test rules. Each alert is shown
# firing on an injected series and staying silent on the healthy or idle case.
rule_files:
- ../config/evo-region.rules.yml
evaluation_interval: 15s
tests:
# Healthy idle region: ready, renewing, idle (debt accrues with no inputs).
- interval: 15s
input_series:
- series: 'up{job="evo-region",environment="dev"}'
values: '1x40'
- series: 'evo_region_ready{environment="dev"}'
values: '1x40'
- series: 'evo_region_phase{environment="dev",phase="running"}'
values: '1x40'
- series: 'evo_region_phase{environment="dev",phase="quarantined"}'
values: '0x40'
- series: 'evo_region_lease_remaining_milliseconds{environment="dev"}'
values: '18000x40'
- series: 'evo_region_tick_debt{environment="dev"}'
values: '0+20x40'
- series: 'evo_region_queued_inputs{environment="dev"}'
values: '0x40'
- series: 'evo_region_queue_capacity_inputs{environment="dev"}'
values: '64x40'
- series: 'evo_region_checkpoint_age_milliseconds{environment="dev"}'
values: '0+15000x40'
- series: 'evo_region_commit_lag_ticks{environment="dev"}'
values: '0x40'
- series: 'evo_region_fences_total{environment="dev"}'
values: '0x40'
- series: 'evo_region_store_failures_total{environment="dev"}'
values: '0x40'
- series: 'evo_region_objects_available_bytes{environment="dev"}'
values: '40000000000x40'
- series: 'evo_region_objects_reserve_bytes{environment="dev"}'
values: '1073741824x40'
- series: 'up{job="alertmanager"}'
values: '1x40'
alert_rule_test:
- {eval_time: 10m, alertname: EvoRegionDown, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionNotReady, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionQuarantined, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionLeaseMargin, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionFenced, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionStoreFailures, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionTickDebt, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionCheckpointStale, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionQueueSaturated, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionObjectsLow, exp_alerts: []}
- {eval_time: 10m, alertname: EvoRegionObjectsFull, exp_alerts: []}
- {eval_time: 10m, alertname: AlertmanagerDown, exp_alerts: []}
- eval_time: 1m
alertname: Watchdog
exp_alerts:
- exp_labels: {severity: none}
exp_annotations:
summary: "Always firing: proves the alert pipeline end to end"
description: "Route to a heartbeat receiver once one exists; its absence then means monitoring is down."
# Outage: the region disappears, then comes back not ready and quarantined.
- interval: 15s
input_series:
- series: 'up{job="evo-region",environment="dev",world="100",region="1"}'
values: '0x20'
- series: 'evo_region_ready{environment="stage"}'
values: '0x20'
- series: 'evo_region_phase{environment="stage",phase="quarantined"}'
values: '1x20'
alert_rule_test:
- eval_time: 3m
alertname: EvoRegionDown
exp_alerts:
- exp_labels: {severity: critical, job: evo-region, environment: dev, world: "100", region: "1"}
exp_annotations:
summary: "Region dev/100/1 is not scrapeable"
description: "No operations endpoint answers. The process or its pod is gone; a forced deletion recovers in about 20 s, so 2 minutes is a real outage."
- eval_time: 1m
alertname: EvoRegionDown
exp_alerts: []
- eval_time: 3m
alertname: EvoRegionNotReady
exp_alerts:
- exp_labels: {severity: critical, environment: stage}
exp_annotations:
summary: "Region stage has no authoritative host"
description: "The process is alive but holds no lease or has not verified its head. Check the phase metric and the host log."
- eval_time: 2m
alertname: EvoRegionQuarantined
exp_alerts:
- exp_labels: {severity: critical, environment: stage, phase: quarantined}
exp_annotations:
summary: "Region stage is quarantined"
description: "Committed state or history failed verification. The region stays unavailable; never re-provision it. Audit with evo-world-clone audit."
# A 20 s lease-bound forced-deletion recovery must not page anyone.
- interval: 15s
input_series:
- series: 'evo_region_ready{environment="dev"}'
values: '1 1 0 0 1 1 1 1 1 1'
alert_rule_test:
- {eval_time: 45s, alertname: EvoRegionNotReady, exp_alerts: []} # During the dip.
- {eval_time: 2m, alertname: EvoRegionNotReady, exp_alerts: []}
# Lease, fences, storage and database failures.
- interval: 15s
input_series:
- series: 'evo_region_ready{environment="dev"}'
values: '1x20'
- series: 'evo_region_lease_remaining_milliseconds{environment="dev"}'
values: '6000x20'
- series: 'evo_region_fences_total{environment="dev"}'
values: '0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1'
- series: 'evo_region_store_failures_total{environment="dev"}'
values: '0+1x20'
- series: 'evo_region_unknown_outcomes_total{environment="dev"}'
values: '0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1'
alert_rule_test:
- eval_time: 2m
alertname: EvoRegionLeaseMargin
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev lease is not being renewed"
description: "Healthy renewals keep 15-20 s of lease. Under 10 s for a minute means renewals are failing; the host will fence itself."
- eval_time: 2m
alertname: EvoRegionFenced
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev host lost authority"
description: "A host was fenced within the last 10 minutes. Expected once after a forced deletion's predecessor resumes; repeated fences point to database or clock trouble."
- eval_time: 3m
alertname: EvoRegionStoreFailures
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev storage or database failures"
description: "Publication or database calls are failing; progress pauses and nothing is acknowledged until they succeed."
- eval_time: 2m
alertname: EvoRegionUnknownOutcomes
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev had commits with unknown outcomes"
description: "Commits timed out or lost their connection and were resolved by exact bytes. Occasional is safe; frequent indicates database latency near the 5 s commit deadline."
# Work waiting: debt, staleness, lag, a saturated queue and dropped inputs.
- interval: 15s
input_series:
- series: 'evo_region_tick_debt{environment="dev"}'
values: '20x20'
- series: 'evo_region_queued_inputs{environment="dev"}'
values: '60x20'
- series: 'evo_region_queue_capacity_inputs{environment="dev"}'
values: '64x20'
- series: 'evo_region_checkpoint_age_milliseconds{environment="dev"}'
values: '400000+15000x20'
- series: 'evo_region_commit_lag_ticks{environment="dev"}'
values: '300x20'
- series: 'evo_region_dropped_total{environment="dev",reason="fenced"}'
values: '0 0 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3'
- series: 'evo_region_dropped_total{environment="dev",reason="refused"}'
values: '0+1x20'
alert_rule_test:
- eval_time: 3m
alertname: EvoRegionTickDebt
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev is behind real time with inputs waiting"
description: "Due ticks are not being simulated although inputs are queued (catch-up is 8 ticks per step)."
- eval_time: 2m
alertname: EvoRegionCheckpointStale
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev has not committed for 5 minutes with work pending"
description: "An idle region does not commit; this fires only while inputs or simulated ticks wait for durability."
- eval_time: 3m
alertname: EvoRegionCommitLag
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev is not committing what it simulates"
description: "More than four full batches are simulated but not durable; acknowledgements are waiting."
- eval_time: 3m
alertname: EvoRegionQueueSaturated
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev input backlog near its bound"
description: "New inputs are being backpressured or rejected."
- eval_time: 2m
alertname: EvoRegionInputsDropped
exp_alerts:
- exp_labels: {severity: warning, environment: dev, reason: fenced}
exp_annotations:
summary: "Region dev accepted inputs that will never be acknowledged"
description: "Inputs were drained, fenced or quarantined (fenced); clients must resubmit to the current owner."
# Object volume filling up, then below its reserve.
- interval: 15s
input_series:
- series: 'evo_region_objects_available_bytes{environment="dev"}'
values: '1500000000x30'
- series: 'evo_region_objects_available_bytes{environment="stage"}'
values: '900000000x30' # Below the reserve but above half of it.
- series: 'evo_region_objects_reserve_bytes{environment="dev"}'
values: '1073741824x30'
- series: 'evo_region_objects_reserve_bytes{environment="stage"}'
values: '1073741824x30'
- series: 'up{job="alertmanager"}'
values: '0x30'
alert_rule_test:
- eval_time: 6m
alertname: EvoRegionObjectsLow
exp_alerts:
- exp_labels: {severity: warning, environment: dev}
exp_annotations:
summary: "Region dev object volume is below twice its reserve"
description: "local-path volumes do not enforce their size; the node's disk is running out."
- exp_labels: {severity: warning, environment: stage}
exp_annotations:
summary: "Region stage object volume is below twice its reserve"
description: "local-path volumes do not enforce their size; the node's disk is running out."
- eval_time: 2m
alertname: EvoRegionObjectsFull
exp_alerts:
- exp_labels: {severity: critical, environment: stage}
exp_annotations:
summary: "Region stage object volume is below its reserve"
description: "New checkpoints are refused; the region cannot commit or acknowledge until space is freed."
- eval_time: 6m
alertname: AlertmanagerDown
exp_alerts:
- exp_labels: {severity: critical, job: alertmanager}
exp_annotations:
summary: "Alertmanager is not scrapeable"
description: "Alerts are evaluated but cannot be routed."
@@ -12,5 +12,6 @@ resources:
- ../dev
- ../stage
- ../world0
- ../monitoring
commonAnnotations:
argocd.argoproj.io/sync-options: Prune=false,Delete=false
@@ -0,0 +1,65 @@
apiVersion: v1
kind: ResourceQuota
metadata:
name: aevolution-quota
spec:
hard:
requests.cpu: "1"
requests.memory: 1Gi
limits.memory: 2Gi
pods: "6"
persistentvolumeclaims: "2"
requests.storage: 10Gi
services.loadbalancers: "0"
services.nodeports: "0"
---
apiVersion: v1
kind: LimitRange
metadata:
name: aevolution-defaults
spec:
limits:
- type: Container
defaultRequest: {cpu: 50m, memory: 64Mi}
default: {memory: 256Mi}
max: {memory: 2Gi}
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: aevolution-default-deny
spec:
podSelector: {}
policyTypes: [Ingress, Egress]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: aevolution-allow-dns
spec:
podSelector: {}
policyTypes: [Egress]
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
podSelector:
matchLabels:
k8s-app: kube-dns
ports:
- {protocol: UDP, port: 53}
- {protocol: TCP, port: 53}
---
# No Kubernetes API access: scrape targets are static, so no token and no RBAC.
apiVersion: v1
kind: ServiceAccount
metadata:
name: aevolution-prometheus
automountServiceAccountToken: false
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: aevolution-alertmanager
automountServiceAccountToken: false
@@ -0,0 +1,13 @@
# M8.5 monitoring namespace (ADR-0072): guardrails and identities owned by the
# platform app, like every AEvolution namespace; Prometheus and Alertmanager
# themselves are the aevolution-monitoring workload app.
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: aevolution-monitoring
resources:
- namespace.yaml
- guardrails.yaml
labels:
- pairs:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/managed-by: aevolution-platform
@@ -0,0 +1,10 @@
# Metrics and alerting for the AEvolution environments. Holds no world data.
apiVersion: v1
kind: Namespace
metadata:
name: aevolution-monitoring
labels:
aevolution.io/environment: monitoring
pod-security.kubernetes.io/enforce: restricted
pod-security.kubernetes.io/enforce-version: v1.32
pod-security.kubernetes.io/warn: restricted
@@ -8,6 +8,7 @@ resources:
- provision-job.yaml
- deployment.yaml
- service.yaml
- scrape-policy.yaml
labels:
- pairs:
app.kubernetes.io/part-of: aevolution
@@ -0,0 +1,21 @@
# M8.5: the region's operations port accepts connections only from the AEvolution
# Prometheus. Kubelet probes come from the node and are not affected (Calico).
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: evo-region-scrape
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: evo-region
policyTypes: [Ingress]
ingress:
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: aevolution-monitoring
podSelector:
matchLabels:
app.kubernetes.io/name: prometheus
ports:
- {protocol: TCP, port: 9100}
+2
View File
@@ -10,6 +10,8 @@ names it. Remove an item only when it is done, and record where it was done.
| **Backup destination for dev/stage stores.** There are no backups yet (maintainer, 2026-09-29). The plan is to build backup tooling for the AEvolution PostgreSQL VM and each `aevolution-objects` volume up to a pluggable destination, then add the destination. It must be off the host for anything beyond disposable environments. | M8.3, 2026-09-29 | The maintainer will supply a destination later | M8.7 backup and restore; any World0 data |
| **Secret manager for Argo-delivered configuration.** Secrets are still created by hand (ADR-0069 amendment). Choose how they reach the cluster through Git or Argo: for example Vault with the Agent Injector (as in the maintainer's other environment), or sealed or external secrets. Until then, rotate by hand. | M8.3, 2026-09-29 | The maintainer wants everything through Argo; a secret manager is not chosen | Stage and World0 credentials; M8.4 |
| **Store Job values.** Supply the published registry digests for `aevolution-postgres-tools` and `aevolution-server` (from each CI Publish step's `Digest:` line), and the NOLOGIN runtime group role for dev (`aevolution`) and for stage (`aevolution_stage`). Then include `stores/<env>` in each environment's path. | M8.3, 2026-09-29 | Values pending | M8.3 in-cluster evidence |
| **Alert destination.** Alertmanager routes every alert to a null receiver (ADR-0072). Choose where alerts go (Discord, email, a webhook), and optionally a heartbeat receiver for the always-firing Watchdog. Receiver credentials go in a Secret. | M8.5, 2026-09-30 | The maintainer will decide later | Any claim that alerts reach a person; M8.6 alert evidence; World0 |
| **Kubelet log rotation check.** Confirm the RKE2 nodes' `container-log-max-size` and `container-log-max-files` (default 10Mi × 5), which bound region host logs (ADR-0072). | M8.5, 2026-09-30 | Needs node access | M8.6 |
| **Storage after the RAID/disk rework.** Re-run `deploy/kubernetes/inventory/disk-probe-local-path.yaml` (or a new class), and update INVENTORY.md. | M8.1, 2026-09-29 | Maintainer is reorganizing disks | M8.3's final storage decision |
## Accepted risks (maintainer decisions)
+8
View File
@@ -635,3 +635,11 @@ M8.4 implemented and locally qualified (2026-09-29), under ADR-0071:
- `region-drill.sh` for rollout and forced-deletion evidence.
The in-cluster sync and drill wait on the maintainer. No director yet: assignment
is static and lives in Git.
M8.5 implemented and locally qualified (2026-09-30), under ADR-0072:
- AEvolution-only Prometheus 3.13.3 and Alertmanager 0.34.1, pinned by digest;
- 16 alert rules, tested by injection with promtool in CI;
- object volume capacity metrics;
- the database outbox kept (no broker);
- bounded logs, and a cardinality and cost report.
Alert delivery waits on the maintainer's choice of receiver.
+19
View File
@@ -1064,3 +1064,22 @@ the policy check enforces this. Dev is world 100 and stage world 200, both regio
Next: apply `deploy/argocd/dev/project.yaml`, sync, and run
`tools/ops/region-drill.sh dev`.
## M8.5 observability (2026-09-30)
ADR-0072 adds an AEvolution-only monitoring stack in `aevolution-monitoring`:
- **Components:** Prometheus 3.13.3 and Alertmanager 0.34.1, pinned by digest and
delivered through Argo.
- **Scraping:** static targets, so no Kubernetes API access. The region's port
9100 is reachable only from Prometheus.
- **Alerts:** 16 rules covering authority, lease, commits, backlog and object
capacity, plus a Watchdog. Each is proven firing and silent by promtool
injection tests in CI. They route to a null receiver until a destination is
chosen.
- **Metrics:** two new capacity gauges (32 fixed series).
No broker is adopted: the database outbox stays. Logs are bounded by kubelet
rotation. Monitoring costs about 1,300 series, roughly 11 MB a day.
Next: sync the monitoring app, confirm the targets, and run the `EvoRegionDown`
injection. M8.4's drill results are still to be recorded.
+7
View File
@@ -111,3 +111,10 @@ Tests check the per-call budget, including at the connection cap, and that an
accepted probe is still answered under replenished pressure. A real server keeps
its lease and readiness while a client opens and closes connections continuously
through 60 ticks.
## Amendment: object volume capacity (2026-09-30, M8.5)
Two gauges were added, `evo_region_objects_available_bytes` and
`evo_region_objects_reserve_bytes`, so that ADR-0070's free-space reserve can be
alerted on before new objects are refused (ADR-0072). The fixed exposition is now
32 series. It still has only the `phase` and `reason` labels, and stays under the
8 KiB bound.
+125
View File
@@ -0,0 +1,125 @@
# ADR-0072: Observability, alerting and background delivery for dev and stage
## Status
Accepted for M8.5, 2026-09-30. It implements the M8 part of ADR-0015
(observability), and builds on:
- ADR-0066 (the operations endpoints);
- ADR-0070 (the object reserve);
- ADR-0071 (the region host).
The maintainer chose an AEvolution-only Prometheus, and left the alert destination
for later.
## Context
M8.5 asks for:
- tick, lease, commit, storage and backup alerts, and bounded logs;
- an evaluation of the database outbox before any optional JetStream, with no
mandatory Redis or Kafka;
- evidence: alert injection, and a telemetry cardinality and cost report.
The named risks are broker redelivery changing authority, and logs filling a disk.
The cluster has no monitoring stack beyond metrics-server (INVENTORY.md). Each
region exposes 32 fixed series. The only labels are a 6-value phase and a 4-value
drop reason.
## Decision
**Stack.** Namespace `aevolution-monitoring`, with platform-owned guardrails, and
a manually synced Argo application `aevolution-monitoring`.
- **Components.** One Prometheus and one Alertmanager:
- `quay.io/prometheus/prometheus:v3.13.3`, pinned by digest `6976aa8a…`;
- `quay.io/prometheus/alertmanager:v0.34.1`, pinned by digest `e9733baf…`.
Both are Apache-2.0, and Prometheus is a CNCF graduated project. 3.13 is a
maintained line: 3.13.3 of 2026-09-07 carries current security fixes.
- **Prometheus configuration:**
- Static scrape targets: the `evo-region-ops` Service in dev and stage, labelled
with `environment`, `world` and `region`. No service discovery, so no token
and no RBAC.
- 15 s scrape and evaluation, a 64-sample and 64 KB cap per scrape, and
retention of 15 days or 3 GB.
- A 5 Gi `local-path` volume. Metrics are not world data, so Delete reclaim is
acceptable.
- **Network.** The region namespaces admit port 9100 only from Prometheus.
Prometheus reaches only the region operations ports and Alertmanager;
Alertmanager accepts only Prometheus. Pods run as nobody, with a read-only root
filesystem and no capabilities.
- **Routing.** Alertmanager routes everything to a **null** receiver until a
destination is chosen (docs/BACKLOG.md). Alerts remain visible in its API and
UI.
**Metrics added.** `evo_region_objects_available_bytes` and
`evo_region_objects_reserve_bytes` make volume capacity alertable before new
objects are refused. The fixed exposition is now 32 series.
**Alerts** (`config/evo-region.rules.yml`, 16 rules). Thresholds come from
measured behaviour:
| Area | Alerts |
|---|---|
| Authority | Down (2 min), NotReady (2 min; a forced-deletion recovery takes about 20 s and does not fire), Quarantined, LeaseMargin (below 10 s while ready; healthy is 15–20 s), Fenced |
| Progress | StoreFailures, UnknownOutcomes, CommitLag (over 256 ticks), TickDebt and CheckpointStale (only while inputs or lag wait, because an idle region always accrues debt), QueueSaturated, InputsDropped (not counting `refused`) |
| Storage | ObjectsLow (below 2× the reserve), ObjectsFull (below the reserve) |
| Pipeline | Watchdog (always firing, for a later heartbeat receiver), AlertmanagerDown |
**Backup alerts** wait for backups (M8.7). There is nothing to alert on until a
backup destination exists; see docs/BACKLOG.md.
**Alert injection.** `promtool test rules` checks each alert firing on injected
series and staying silent in the healthy, idle and 20 s recovery cases. The CI
`deploy` stage runs it with the `promtool` and `amtool` copied from the same
pinned images, together with syntax checks of the Prometheus and Alertmanager
configurations.
**Logs.** Container logs are rotated by the kubelet. The defaults are 10 Mi per
file and 5 files per container, 50 Mi at most per container. The actual RKE2
setting has not been verified on the nodes yet. At most:
- `evo-server` writes one ack line of about 60 bytes per tick, about 100 MB a day
at 20 Hz, so the rotation keeps the most recent 12 hours;
- Prometheus and Alertmanager log little.
Nothing writes logs to a volume. Durable history lives in PostgreSQL and the
object store, never in logs.
**Background delivery.** The database outbox stays the delivery mechanism, and no
broker is adopted:
- `region_outbox` is written in the same transaction as each permanent outcome,
delivery is idempotent (`acknowledge_region_outbox`), and delivery never
mutates simulation state (ADR-0063).
- There is no consumer of outcomes yet: no gateway, client or projection. A relay
worker would deliver to nothing, so it is deferred until a consumer exists.
- JetStream is not adopted. There is no demonstrated delivery workload, and a
broker would add a second durability layer to operate and back up. It could
also never hold authority, since its redelivery could only re-announce an
already-final outcome. Redis and Kafka are not needed.
**Cardinality and cost** (per 15 s scrape):
| Source | Series |
|---|---|
| Each region | 32, plus about 5 per-target scrape series |
| Two regions (dev, stage) | about 75 |
| Prometheus self-scrape | about 1,000 (typical) |
| Alertmanager | about 200 |
| **Total** | **about 1,300** |
That is about 87 samples per second. At about 1.5 bytes per compressed sample,
that is about 11 MB a day, or about 170 MB over 15 days, well inside the 3 GB
retention cap and the 5 Gi volume. Each added region adds about 37 series. The
request is 100m CPU and 256 Mi, with a 1 Gi memory limit. Alertmanager requests
50m and 64 Mi, with a 128 Mi limit.
## Consequences
- Alerts are evaluated and visible, but not delivered until a receiver is chosen.
The Watchdog is ready for a dead-man's-switch receiver.
- Changing a threshold is a Git change that must keep the rule tests passing.
- The kubelet log rotation values should be confirmed on the nodes.
- World0 monitoring (M8.8) reuses the rules with its own target.
## Alternatives rejected
- **kube-prometheus-stack or the Prometheus Operator.** Cluster-wide CRDs and
RBAC for a single project; not the maintainer's choice.
- **Kubernetes service discovery.** It needs a token and cluster read RBAC for no
gain with two static targets.
- **JetStream, Kafka or Redis for outcomes.** There is no workload yet, and each
would add another stateful system.
+1
View File
@@ -82,3 +82,4 @@ design. No unadopted candidate is a link dependency of the proof.
- [ADR-0069: Environment isolation and World0 protection](0069-environment-isolation.md) - accepted for M8.2 per-environment guardrails, native World0 admission protection, scoped Argo projects and verification.
- [ADR-0070: Durable stores for dev and stage](0070-dev-stage-stores.md) - accepted for M8.3 external PostgreSQL per environment, Retain-class node-local object volumes, free-space reserve and in-cluster store qualification.
- [ADR-0071: Region host deployment in dev and stage](0071-region-host-deployment.md) - accepted for M8.4 single-replica Recreate host, digest-as-build identity, idempotent provisioning hook and the region drill; no director yet.
- [ADR-0072: Observability, alerting and background delivery](0072-observability.md) - accepted for M8.5 pinned AEvolution Prometheus/Alertmanager, 16 tested alert rules, bounded logs, database outbox over any broker, cardinality/cost report.
+33
View File
@@ -2811,3 +2811,36 @@ admission policy and the stage and World0 AppProjects, then ran
- `region-drill.sh` passed `bash -n`, and refuses World0 before kubectl (also
enforced by the policy check).
- Not executed: the in-cluster region sync and the drill.
## M8.5 observability - 2026-09-30
- `promtool` 3.13.3 and `amtool` 0.34.1 came from the official release tarballs,
with SHA-256 verified against the published sums: `b349c732…` and `265b9d1e…`.
- The Prometheus configuration syntax is valid, and all 16 rules are valid.
- The alert injection tests passed, as did the Alertmanager `check-config`.
- Rule mutations: five were tried, and three were caught at first.
- **NotReady firing after 15 s:** missed, because the forced-deletion test only
evaluated after recovery. The test now also checks during the dip.
- **ObjectsFull weakened to half the reserve:** missed, because the injected
free space was below both thresholds. The test now uses 900 MB.
- After these fixes all five were caught: the lease threshold, NotReady too
eager, debt counted without inputs, a weakened ObjectsFull, and `refused`
drops counted.
- Windows MSVC CTest 59/59, including the `ops-contracts` update to 32 series.
- WSL Linux `checks` set: 59/59 in each configuration. The format check passed.
- PostgreSQL suites: Release, analysis and ASan/UBSan passed. The ops process
test checks the reserve gauge (1 GiB) and that free space is nonzero.
- `deploy-policy-check.py`: 67 objects, 0 failures. New rules:
- monitoring images must be digest-pinned;
- monitoring pods mount no token;
- no RBAC objects are allowed;
- the monitoring project is scoped.
- Kustomize renders the platform bundle (four namespaces), the monitoring app
(2 Deployments, 2 ConfigMaps, 2 Services, 2 policies, 1 claim), and dev and
stage with the scrape policy.
- The CI `ci-deploy` stage now copies `promtool` and `amtool` from the pinned
images and runs `tools/ci/monitoring-check.sh`. It has not run locally, because
Docker is not running.
- Not executed: anything in the cluster (monitoring sync, target health, the
`EvoRegionDown` injection), alert delivery (no receiver), and the node log
rotation check.
+21
View File
@@ -139,6 +139,27 @@ record:
- the first pod's `phase running epoch=... committed=...` line;
- that `/readyz` stays 503 while a replacement waits out the lease.
## Monitoring (M8.5, ADR-0072)
Bootstrap once: `kubectl apply -f deploy/argocd/monitoring/`, then sync
`aevolution-platform` (it adds the `aevolution-monitoring` namespace and its
guardrails) and `aevolution-monitoring`. Re-sync dev and stage, which adds the
`evo-region-scrape` policy.
```sh
kubectl -n aevolution-monitoring port-forward svc/prometheus 9090 &
curl -s localhost:9090/api/v1/targets | grep -o '"health":"[a-z]*"' # both regions up
curl -s localhost:9090/api/v1/alerts # Watchdog firing only
```
**Injection in the cluster** (dev only):
- `kubectl -n evolution-dev scale deploy/evo-region --replicas=0` makes
`EvoRegionDown` pending, then firing after 2 minutes, in Alertmanager (null
receiver).
- Scale back to 1 and it resolves.
Record the timestamps. The rule tests in CI already prove the thresholds.
## Break-glass
How membership is granted is not decided yet; see docs/BACKLOG.md. Until it is,
+3
View File
@@ -44,6 +44,9 @@ struct OpsCounters {
std::uint64_t checkpoint_age_ms{}; // Since the last confirmed commit or recovery.
std::uint64_t acks{}, outcomes{};
std::array<std::uint64_t, unacknowledged_count> dropped{}; // Indexed by Unacknowledged.
// M8.5: object volume free space and the store's reserve (ADR-0070), so capacity
// is alerted on before new objects are refused.
std::uint64_t objects_available_bytes{}, objects_reserve_bytes{};
};
[[nodiscard]] const char* phase_name(HostPhase phase) noexcept;
+4
View File
@@ -107,6 +107,10 @@ template <typename State> class RegionHost {
[[nodiscard]] const HostStatus& status() const noexcept {
return status_;
}
// Read-only, for operational reporting (free space against the reserve).
[[nodiscard]] const persistence::FilesystemObjectStore& objects() const noexcept {
return *objects_;
}
// Read-only view for diagnostics/tests; null unless recovered.
[[nodiscard]] const State* state() const noexcept {
return state_.get();
+4
View File
@@ -81,6 +81,10 @@ std::string render_metrics(const HostStatus& s, const OpsCounters& c) {
c.queue_capacity_inputs);
metric(out, "evo_region_queue_capacity_bytes", "gauge", "Input backlog byte bound",
c.queue_capacity_bytes);
metric(out, "evo_region_objects_available_bytes", "gauge",
"Free bytes on the object volume available to the host", c.objects_available_bytes);
metric(out, "evo_region_objects_reserve_bytes", "gauge",
"Free-space reserve below which new objects are refused", c.objects_reserve_bytes);
metric(out, "evo_region_commits_total", "counter", "Confirmed commits", s.commits);
metric(out, "evo_region_store_failures_total", "counter",
"Storage or database failures that paused progress", s.store_failures);
+3 -3
View File
@@ -82,8 +82,8 @@ void exposition_is_bounded_and_well_formed() {
CHECK(text.find("evo_region_commit_lag_ticks 9\n") != std::string::npos);
CHECK(text.find("evo_region_lease_remaining_milliseconds 1234\n") != std::string::npos);
CHECK(text.find("evo_region_dropped_total{reason=\"fenced\"} 5\n") != std::string::npos);
// Series count is fixed: 6 phases + 4 reasons + 20 unlabelled families.
CHECK(samples.size() == 6 + 4 + 20);
// Series count is fixed: 6 phases + 4 reasons + 22 unlabelled families.
CHECK(samples.size() == 6 + 4 + 22);
// Worst-case values keep the size bounded and never underflow the lag.
HostStatus big;
@@ -93,7 +93,7 @@ void exposition_is_bounded_and_well_formed() {
big.max_tick_debt = big.commits = big.store_failures = big.unknown_outcomes = most;
big.fences = big.lease_remaining_ms = most;
big.simulated_tick = 1; // Lag saturates at zero rather than wrapping.
OpsCounters bigc{most, most, most, most, most, {most, most, most, most}};
OpsCounters bigc{most, most, most, most, most, {most, most, most, most}, most, most};
const auto large = render_metrics(big, bigc);
CHECK(large.size() <= max_metrics_bytes);
CHECK(large.find("evo_region_commit_lag_ticks 0\n") != std::string::npos);
+3
View File
@@ -97,6 +97,9 @@ void probes_follow_authority_and_stay_bounded() {
CHECK(metric(port, "evo_region_lease_remaining_milliseconds") > 0);
CHECK(metric(port, "evo_region_queue_capacity_inputs") == 64 &&
metric(port, "evo_region_queue_capacity_bytes") == 65536);
// M8.5: the default reserve is 1024 MiB, and the volume reports real free space.
CHECK(metric(port, "evo_region_objects_reserve_bytes") == std::uint64_t{1024} * 1024 * 1024 &&
metric(port, "evo_region_objects_available_bytes") > 0);
auto page = get(port, "/metrics");
CHECK(page.body.find("evo_region_phase{phase=\"running\"} 1\n") != std::string::npos);
server.send(input_lines(1, 30, false));
+22 -5
View File
@@ -121,8 +121,8 @@ for path, doc in objects:
if doc['kind'] == 'AppProject' and name == 'aevolution-platform':
spec = doc['spec']
namespaces = sorted(d.get('namespace') for d in spec.get('destinations', []))
check(namespaces == ['evolution-dev', 'evolution-stage', 'evolution-world0'],
f'{path}: the platform project targets exactly the three AEvolution namespaces')
check(namespaces == ['aevolution-monitoring', 'evolution-dev', 'evolution-stage', 'evolution-world0'],
f'{path}: the platform project targets exactly the AEvolution namespaces')
cluster = sorted((w.get('group'), w.get('kind')) for w in spec.get('clusterResourceWhitelist', []))
check(cluster == [('', 'Namespace'),
('admissionregistration.k8s.io', 'ValidatingAdmissionPolicy'),
@@ -131,6 +131,11 @@ for path, doc in objects:
f'{path}: the platform project may own only its four cluster kinds')
kinds = {w.get('kind') for w in spec.get('namespaceResourceWhitelist', [])}
check(not kinds & {'Secret', 'Role', 'RoleBinding', '*'}, f'{path}: platform whitelists a forbidden kind')
elif doc['kind'] == 'AppProject' and name == 'aevolution-monitoring':
spec = doc['spec']
check([d.get('namespace') for d in spec.get('destinations', [])] == ['aevolution-monitoring']
and spec.get('clusterResourceWhitelist') == [],
f'{path}: monitoring targets only its namespace, with no cluster kinds')
elif doc['kind'] == 'AppProject' and name.startswith('aevolution-'):
env = name[len('aevolution-'):]
spec = doc['spec']
@@ -161,7 +166,7 @@ for path, doc in objects:
check(automated is None, f'{path}: {name} must not auto-sync')
check('resources-finalizer.argocd.argoproj.io' not in doc['metadata'].get('finalizers', []),
f'{path}: {name} must not cascade-delete')
if project.startswith('aevolution-') and project != 'aevolution-platform':
if project.startswith('aevolution-') and project not in ('aevolution-platform', 'aevolution-monitoring'):
check(destination == 'evolution-' + project[len('aevolution-'):],
f'{path}: {name} leaves its project environment')
check(destination != 'evolution-world0' or name in ('aevolution-world0', 'aevolution-platform'),
@@ -222,12 +227,24 @@ for env in ('dev', 'stage'):
check(len(server) == 1 and len(builds) == 2 and all('sha256:' + b == server[0] for b in builds),
f'{overlay}: the build identity must equal the server image digest')
# M8.5 monitoring (ADR-0072): third-party images are pinned by digest, and Prometheus
# needs no Kubernetes API access.
for path, doc in objects:
if path.startswith('deploy/kubernetes/monitoring/') and doc['kind'] == 'Deployment':
pod = doc['spec']['template']['spec']
for container in pod['containers']:
check('@sha256:' in container.get('image', ''), f'{path}: {container["name"]} must be pinned by digest')
check(pod.get('automountServiceAccountToken') is False, f'{path}: monitoring pods mount no token')
if path.startswith('deploy/') and doc['kind'] in ('ClusterRole', 'ClusterRoleBinding', 'Role', 'RoleBinding'):
check(False, f'{path}: AEvolution grants no Kubernetes RBAC')
# The platform bundle: every rendered object is protected from prune and deletion.
bundle = manifests.get(platform + '/cluster/kustomization.yaml', [{}])[0]
check(bundle.get('commonAnnotations', {}).get('argocd.argoproj.io/sync-options') == 'Prune=false,Delete=false',
'the platform bundle must mark every object Prune=false,Delete=false')
check(sorted(bundle.get('resources', [])) == ['../admission', '../dev', '../stage', '../storage', '../world0'],
'the platform bundle must include admission, storage and the three environments')
check(sorted(bundle.get('resources', [])) == ['../admission', '../dev', '../monitoring', '../stage', '../storage',
'../world0'],
'the platform bundle must include admission, storage, the environments and monitoring')
# Paths a real (non-example) Application syncs must be complete: no REPLACE markers
+15
View File
@@ -0,0 +1,15 @@
#!/usr/bin/env bash
# M8.5 monitoring configuration checks (ADR-0072): Prometheus configuration syntax,
# alert rule validity, the alert-injection unit tests, and the Alertmanager routing.
# Needs promtool and amtool on PATH, taken from the same pinned images that run in
# the cluster (the ci-deploy Docker stage copies them). No cluster access.
set -euo pipefail
root="$(cd "$(dirname "$0")/../.." && pwd)"
config="$root/deploy/kubernetes/monitoring/config"
promtool --version | head -1
amtool --version 2>&1 | head -1
promtool check config --syntax-only "$config/prometheus.yml"
promtool check rules "$config/evo-region.rules.yml"
(cd "$root/deploy/kubernetes/monitoring/tests" && promtool test rules evo-region.rules.test.yml)
amtool check-config "$config/alertmanager.yml"
echo "monitoring configuration, rules and alert tests passed"