diff --git a/apps/evo-server/main.cpp b/apps/evo-server/main.cpp index cff6a50..2c5a261 100644 --- a/apps/evo-server/main.cpp +++ b/apps/evo-server/main.cpp @@ -310,6 +310,7 @@ template int serve(Options& o) { server::OpsCounters counters; counters.queue_capacity_inputs = o.host.max_queued_inputs; counters.queue_capacity_bytes = o.host.max_queued_bytes; + counters.objects_reserve_bytes = o.objects_reserve_mib * 1024 * 1024; std::uint64_t last_sequence = 0; auto last_checkpoint = std::chrono::steady_clock::now(); bool recovered = false; @@ -326,6 +327,7 @@ template int serve(Options& o) { static_cast(std::chrono::duration_cast( std::chrono::steady_clock::now() - last_checkpoint) .count()); + counters.objects_available_bytes = host.objects().available_bytes().value_or(0); return {200, server::render_metrics(s, counters), "text/plain; version=0.0.4; charset=utf-8"}; } diff --git a/deploy/argocd/monitoring/application.yaml b/deploy/argocd/monitoring/application.yaml new file mode 100644 index 0000000..c42c084 --- /dev/null +++ b/deploy/argocd/monitoring/application.yaml @@ -0,0 +1,19 @@ +# M8.5 monitoring application (ADR-0072). Manual sync like the other AEvolution +# workloads; it holds no world data. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: aevolution-monitoring + namespace: argocd +spec: + project: aevolution-monitoring + source: + repoURL: https://git.aeternalisgames.com/dingo/AEvolution.git + targetRevision: main + path: deploy/kubernetes/monitoring + destination: + server: https://kubernetes.default.svc + namespace: aevolution-monitoring + syncPolicy: + syncOptions: + - FailOnSharedResource=true diff --git a/deploy/argocd/monitoring/project.yaml b/deploy/argocd/monitoring/project.yaml new file mode 100644 index 0000000..396c079 --- /dev/null +++ b/deploy/argocd/monitoring/project.yaml @@ -0,0 +1,23 @@ +# M8.5 monitoring project (ADR-0072). Apply by hand once: +# kubectl apply -f deploy/argocd/monitoring/ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: aevolution-monitoring + namespace: argocd +spec: + description: AEvolution metrics and alerting (aevolution-monitoring only) + sourceRepos: + - https://git.aeternalisgames.com/dingo/AEvolution.git + destinations: + - server: https://kubernetes.default.svc + namespace: aevolution-monitoring + clusterResourceWhitelist: [] + namespaceResourceWhitelist: + - {group: "", kind: Service} + - {group: "", kind: ConfigMap} + - {group: "", kind: PersistentVolumeClaim} + - {group: apps, kind: Deployment} + - {group: networking.k8s.io, kind: NetworkPolicy} + orphanedResources: + warn: true diff --git a/deploy/argocd/platform/project.yaml b/deploy/argocd/platform/project.yaml index ea23ca6..911be85 100644 --- a/deploy/argocd/platform/project.yaml +++ b/deploy/argocd/platform/project.yaml @@ -18,6 +18,8 @@ spec: namespace: evolution-stage - server: https://kubernetes.default.svc namespace: evolution-world0 + - server: https://kubernetes.default.svc + namespace: aevolution-monitoring clusterResourceWhitelist: - {group: "", kind: Namespace} - {group: storage.k8s.io, kind: StorageClass} diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index efbf777..7ad688c 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -57,6 +57,11 @@ COPY deploy/ deploy/ COPY tools/ci/deploy-policy-check.py tools/ci/ COPY tools/ops/ tools/ops/ RUN python3 tools/ci/deploy-policy-check.py +# M8.5: monitoring configuration and alert tests with the pinned images' own tools. +COPY --from=quay.io/prometheus/prometheus@sha256:6976aa8a60fec930796ce5772b8d12da7a318a5daa8d40d69c5c7819a05eeed7 /bin/promtool /usr/local/bin/promtool +COPY --from=quay.io/prometheus/alertmanager@sha256:e9733bafb1bdef9b00e25a21f8f99dc26a22224bf16641ad754d1649f4c3357a /bin/amtool /usr/local/bin/amtool +COPY tools/ci/monitoring-check.sh tools/ci/ +RUN bash tools/ci/monitoring-check.sh FROM ci-toolchain AS postgres-build RUN apt-get update && apt-get install -y --no-install-recommends \ diff --git a/deploy/kubernetes/monitoring/alertmanager.yaml b/deploy/kubernetes/monitoring/alertmanager.yaml new file mode 100644 index 0000000..03892e8 --- /dev/null +++ b/deploy/kubernetes/monitoring/alertmanager.yaml @@ -0,0 +1,90 @@ +# M8.5 Alertmanager (ADR-0072), pinned by digest. Every alert routes to the null +# receiver until the maintainer chooses a destination (docs/BACKLOG.md); alerts are +# still visible in its API and UI. Silences live in memory (emptyDir). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: alertmanager + labels: + app.kubernetes.io/name: alertmanager +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: alertmanager + template: + metadata: + labels: + app.kubernetes.io/name: alertmanager + app.kubernetes.io/part-of: aevolution + spec: + serviceAccountName: aevolution-alertmanager + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + fsGroup: 65534 + seccompProfile: {type: RuntimeDefault} + containers: + - name: alertmanager + image: quay.io/prometheus/alertmanager:v0.34.1@sha256:e9733bafb1bdef9b00e25a21f8f99dc26a22224bf16641ad754d1649f4c3357a + args: + - --config.file=/etc/alertmanager/alertmanager.yml + - --storage.path=/alertmanager + - --web.listen-address=:9093 + - --cluster.listen-address= # Single instance: no gossip port. + ports: + - {name: web, containerPort: 9093} + readinessProbe: + httpGet: {path: /-/ready, port: web} + periodSeconds: 10 + livenessProbe: + httpGet: {path: /-/healthy, port: web} + periodSeconds: 20 + resources: + requests: {cpu: 50m, memory: 64Mi} + limits: {memory: 128Mi} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + volumeMounts: + - {name: config, mountPath: /etc/alertmanager, readOnly: true} + - {name: data, mountPath: /alertmanager} + volumes: + - name: config + configMap: + name: alertmanager-config + - name: data + emptyDir: {sizeLimit: 64Mi} +--- +apiVersion: v1 +kind: Service +metadata: + name: alertmanager +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: alertmanager + ports: + - {name: web, port: 9093, targetPort: web} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: alertmanager +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: alertmanager + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + ports: + - {protocol: TCP, port: 9093} diff --git a/deploy/kubernetes/monitoring/config/alertmanager.yml b/deploy/kubernetes/monitoring/config/alertmanager.yml new file mode 100644 index 0000000..26fcf47 --- /dev/null +++ b/deploy/kubernetes/monitoring/config/alertmanager.yml @@ -0,0 +1,17 @@ +# M8.5 Alertmanager routing (ADR-0072). No destination is chosen yet +# (docs/BACKLOG.md): every alert routes to the null receiver, which keeps it visible +# in Alertmanager's API and UI without delivering it. Adding a receiver later is a +# reviewed change here plus a Secret for its credentials, never a credential in Git. +route: + receiver: "null" + group_by: [alertname, environment] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h +receivers: + - name: "null" +inhibit_rules: + # A region that is down cannot also meaningfully be not ready or stale. + - source_matchers: [alertname = EvoRegionDown] + target_matchers: [alertname =~ "EvoRegion(NotReady|LeaseMargin|CheckpointStale)"] + equal: [environment] diff --git a/deploy/kubernetes/monitoring/config/evo-region.rules.yml b/deploy/kubernetes/monitoring/config/evo-region.rules.yml new file mode 100644 index 0000000..95a0ac5 --- /dev/null +++ b/deploy/kubernetes/monitoring/config/evo-region.rules.yml @@ -0,0 +1,121 @@ +# M8.5 region host alerts (ADR-0072). Thresholds follow the measured behaviour: +# the production lease is 20 s renewed every 5 s (15-20 s remaining when healthy); +# batches are at most 64 ticks with a 1 s flush; an idle region (no inputs, no +# gateway yet) always accrues tick debt, so debt and staleness only alert while +# inputs are waiting. Unit tests: evo-region.rules.test.yml (promtool). +groups: + - name: evo-region-authority + rules: + - alert: EvoRegionDown + expr: up{job="evo-region"} == 0 + for: 2m + labels: {severity: critical} + annotations: + summary: "Region {{ $labels.environment }}/{{ $labels.world }}/{{ $labels.region }} is not scrapeable" + description: "No operations endpoint answers. The process or its pod is gone; a forced deletion recovers in about 20 s, so 2 minutes is a real outage." + - alert: EvoRegionNotReady + expr: evo_region_ready == 0 + for: 2m + labels: {severity: critical} + annotations: + summary: "Region {{ $labels.environment }} has no authoritative host" + description: "The process is alive but holds no lease or has not verified its head. Check the phase metric and the host log." + - alert: EvoRegionQuarantined + expr: evo_region_phase{phase="quarantined"} == 1 + for: 1m + labels: {severity: critical} + annotations: + summary: "Region {{ $labels.environment }} is quarantined" + description: "Committed state or history failed verification. The region stays unavailable; never re-provision it. Audit with evo-world-clone audit." + - alert: EvoRegionLeaseMargin + expr: evo_region_ready == 1 and evo_region_lease_remaining_milliseconds < 10000 + for: 1m + labels: {severity: warning} + annotations: + summary: "Region {{ $labels.environment }} lease is not being renewed" + description: "Healthy renewals keep 15-20 s of lease. Under 10 s for a minute means renewals are failing; the host will fence itself." + - alert: EvoRegionFenced + expr: increase(evo_region_fences_total[10m]) > 0 + labels: {severity: warning} + annotations: + summary: "Region {{ $labels.environment }} host lost authority" + description: "A host was fenced within the last 10 minutes. Expected once after a forced deletion's predecessor resumes; repeated fences point to database or clock trouble." + - name: evo-region-progress + rules: + - alert: EvoRegionStoreFailures + expr: increase(evo_region_store_failures_total[5m]) > 3 + labels: {severity: warning} + annotations: + summary: "Region {{ $labels.environment }} storage or database failures" + description: "Publication or database calls are failing; progress pauses and nothing is acknowledged until they succeed." + - alert: EvoRegionUnknownOutcomes + expr: increase(evo_region_unknown_outcomes_total[15m]) > 0 + labels: {severity: warning} + annotations: + summary: "Region {{ $labels.environment }} had commits with unknown outcomes" + description: "Commits timed out or lost their connection and were resolved by exact bytes. Occasional is safe; frequent indicates database latency near the 5 s commit deadline." + - alert: EvoRegionCommitLag + expr: evo_region_commit_lag_ticks > 256 + for: 2m + labels: {severity: warning} + annotations: + summary: "Region {{ $labels.environment }} is not committing what it simulates" + description: "More than four full batches are simulated but not durable; acknowledgements are waiting." + - alert: EvoRegionTickDebt + expr: evo_region_tick_debt > 8 and evo_region_queued_inputs > 0 + for: 2m + labels: {severity: warning} + annotations: + summary: "Region {{ $labels.environment }} is behind real time with inputs waiting" + description: "Due ticks are not being simulated although inputs are queued (catch-up is 8 ticks per step)." + - alert: EvoRegionCheckpointStale + expr: evo_region_checkpoint_age_milliseconds > 300000 and (evo_region_queued_inputs > 0 or evo_region_commit_lag_ticks > 0) + for: 1m + labels: {severity: warning} + annotations: + summary: "Region {{ $labels.environment }} has not committed for 5 minutes with work pending" + description: "An idle region does not commit; this fires only while inputs or simulated ticks wait for durability." + - alert: EvoRegionQueueSaturated + expr: evo_region_queued_inputs >= 0.9 * evo_region_queue_capacity_inputs + for: 2m + labels: {severity: warning} + annotations: + summary: "Region {{ $labels.environment }} input backlog near its bound" + description: "New inputs are being backpressured or rejected." + - alert: EvoRegionInputsDropped + expr: increase(evo_region_dropped_total{reason!="refused"}[10m]) > 0 + labels: {severity: warning} + annotations: + summary: "Region {{ $labels.environment }} accepted inputs that will never be acknowledged" + description: "Inputs were drained, fenced or quarantined ({{ $labels.reason }}); clients must resubmit to the current owner." + - name: evo-region-storage + rules: + - alert: EvoRegionObjectsLow + expr: evo_region_objects_available_bytes < 2 * evo_region_objects_reserve_bytes + for: 5m + labels: {severity: warning} + annotations: + summary: "Region {{ $labels.environment }} object volume is below twice its reserve" + description: "local-path volumes do not enforce their size; the node's disk is running out." + - alert: EvoRegionObjectsFull + expr: evo_region_objects_available_bytes < evo_region_objects_reserve_bytes + for: 1m + labels: {severity: critical} + annotations: + summary: "Region {{ $labels.environment }} object volume is below its reserve" + description: "New checkpoints are refused; the region cannot commit or acknowledge until space is freed." + - name: aevolution-monitoring + rules: + - alert: Watchdog + expr: vector(1) + labels: {severity: none} + annotations: + summary: "Always firing: proves the alert pipeline end to end" + description: "Route to a heartbeat receiver once one exists; its absence then means monitoring is down." + - alert: AlertmanagerDown + expr: up{job="alertmanager"} == 0 + for: 5m + labels: {severity: critical} + annotations: + summary: "Alertmanager is not scrapeable" + description: "Alerts are evaluated but cannot be routed." diff --git a/deploy/kubernetes/monitoring/config/prometheus.yml b/deploy/kubernetes/monitoring/config/prometheus.yml new file mode 100644 index 0000000..92c18cc --- /dev/null +++ b/deploy/kubernetes/monitoring/config/prometheus.yml @@ -0,0 +1,33 @@ +# M8.5 Prometheus configuration (ADR-0072). Static targets: the region operations +# Service of each disposable environment. The scrape target carries the region's +# identity as labels (ADR-0066): the exposition itself has no world/region labels. +global: + scrape_interval: 15s + scrape_timeout: 5s + evaluation_interval: 15s + +rule_files: + - /etc/prometheus/evo-region.rules.yml + +alerting: + alertmanagers: + - static_configs: + - targets: [alertmanager:9093] + +scrape_configs: + - job_name: evo-region + metrics_path: /metrics + # The exposition is at most 8 KiB (ADR-0066); anything larger is refused. + body_size_limit: 64KB + sample_limit: 64 + static_configs: + - targets: [evo-region-ops.evolution-dev.svc:9100] + labels: {environment: dev, world: "100", region: "1"} + - targets: [evo-region-ops.evolution-stage.svc:9100] + labels: {environment: stage, world: "200", region: "1"} + - job_name: prometheus + static_configs: + - targets: [localhost:9090] + - job_name: alertmanager + static_configs: + - targets: [alertmanager:9093] diff --git a/deploy/kubernetes/monitoring/kustomization.yaml b/deploy/kubernetes/monitoring/kustomization.yaml new file mode 100644 index 0000000..754ab32 --- /dev/null +++ b/deploy/kubernetes/monitoring/kustomization.yaml @@ -0,0 +1,20 @@ +# M8.5 monitoring workload (ADR-0072), synced by the aevolution-monitoring Argo +# application. Configuration and rules live in config/ and are unit-tested in CI +# with promtool (tools/ci/monitoring-check.sh). +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: aevolution-monitoring +resources: + - prometheus.yaml + - alertmanager.yaml +configMapGenerator: + - name: prometheus-config + files: + - config/prometheus.yml + - config/evo-region.rules.yml + - name: alertmanager-config + files: + - config/alertmanager.yml +labels: + - pairs: + app.kubernetes.io/part-of: aevolution diff --git a/deploy/kubernetes/monitoring/prometheus.yaml b/deploy/kubernetes/monitoring/prometheus.yaml new file mode 100644 index 0000000..266d7d3 --- /dev/null +++ b/deploy/kubernetes/monitoring/prometheus.yaml @@ -0,0 +1,115 @@ +# M8.5 Prometheus (ADR-0072), pinned by digest. Static scrape targets (no API access, +# no RBAC); 15 s scrape and evaluation; retention bounded by time and size. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: prometheus-data +spec: + accessModes: [ReadWriteOnce] + storageClassName: local-path # Metrics are not world data; Delete reclaim is fine. + resources: + requests: + storage: 5Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + labels: + app.kubernetes.io/name: prometheus +spec: + replicas: 1 + strategy: + type: Recreate # One TSDB writer on an RWO volume. + selector: + matchLabels: + app.kubernetes.io/name: prometheus + template: + metadata: + labels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/part-of: aevolution + spec: + serviceAccountName: aevolution-prometheus + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + fsGroup: 65534 + seccompProfile: {type: RuntimeDefault} + containers: + - name: prometheus + image: quay.io/prometheus/prometheus:v3.13.3@sha256:6976aa8a60fec930796ce5772b8d12da7a318a5daa8d40d69c5c7819a05eeed7 + args: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=15d + - --storage.tsdb.retention.size=3GB + - --web.listen-address=:9090 + ports: + - {name: web, containerPort: 9090} + readinessProbe: + httpGet: {path: /-/ready, port: web} + periodSeconds: 10 + livenessProbe: + httpGet: {path: /-/healthy, port: web} + periodSeconds: 20 + failureThreshold: 3 + resources: + requests: {cpu: 100m, memory: 256Mi} + limits: {memory: 1Gi} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + volumeMounts: + - {name: config, mountPath: /etc/prometheus, readOnly: true} + - {name: data, mountPath: /prometheus} + volumes: + - name: config + configMap: + name: prometheus-config + - name: data + persistentVolumeClaim: + claimName: prometheus-data +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: prometheus + ports: + - {name: web, port: 9090, targetPort: web} +--- +# Prometheus reaches the region operations ports, Alertmanager and itself only. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: prometheus +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + policyTypes: [Egress] + egress: + - to: + - namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: [evolution-dev, evolution-stage] + podSelector: + matchLabels: + app.kubernetes.io/name: evo-region + ports: + - {protocol: TCP, port: 9100} + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: alertmanager + ports: + - {protocol: TCP, port: 9093} diff --git a/deploy/kubernetes/monitoring/tests/evo-region.rules.test.yml b/deploy/kubernetes/monitoring/tests/evo-region.rules.test.yml new file mode 100644 index 0000000..e17849a --- /dev/null +++ b/deploy/kubernetes/monitoring/tests/evo-region.rules.test.yml @@ -0,0 +1,240 @@ +# M8.5 alert injection tests (ADR-0072): promtool test rules. Each alert is shown +# firing on an injected series and staying silent on the healthy or idle case. +rule_files: + - ../config/evo-region.rules.yml +evaluation_interval: 15s +tests: + # Healthy idle region: ready, renewing, idle (debt accrues with no inputs). + - interval: 15s + input_series: + - series: 'up{job="evo-region",environment="dev"}' + values: '1x40' + - series: 'evo_region_ready{environment="dev"}' + values: '1x40' + - series: 'evo_region_phase{environment="dev",phase="running"}' + values: '1x40' + - series: 'evo_region_phase{environment="dev",phase="quarantined"}' + values: '0x40' + - series: 'evo_region_lease_remaining_milliseconds{environment="dev"}' + values: '18000x40' + - series: 'evo_region_tick_debt{environment="dev"}' + values: '0+20x40' + - series: 'evo_region_queued_inputs{environment="dev"}' + values: '0x40' + - series: 'evo_region_queue_capacity_inputs{environment="dev"}' + values: '64x40' + - series: 'evo_region_checkpoint_age_milliseconds{environment="dev"}' + values: '0+15000x40' + - series: 'evo_region_commit_lag_ticks{environment="dev"}' + values: '0x40' + - series: 'evo_region_fences_total{environment="dev"}' + values: '0x40' + - series: 'evo_region_store_failures_total{environment="dev"}' + values: '0x40' + - series: 'evo_region_objects_available_bytes{environment="dev"}' + values: '40000000000x40' + - series: 'evo_region_objects_reserve_bytes{environment="dev"}' + values: '1073741824x40' + - series: 'up{job="alertmanager"}' + values: '1x40' + alert_rule_test: + - {eval_time: 10m, alertname: EvoRegionDown, exp_alerts: []} + - {eval_time: 10m, alertname: EvoRegionNotReady, exp_alerts: []} + - {eval_time: 10m, alertname: EvoRegionQuarantined, exp_alerts: []} + - {eval_time: 10m, alertname: EvoRegionLeaseMargin, exp_alerts: []} + - {eval_time: 10m, alertname: EvoRegionFenced, exp_alerts: []} + - {eval_time: 10m, alertname: EvoRegionStoreFailures, exp_alerts: []} + - {eval_time: 10m, alertname: EvoRegionTickDebt, exp_alerts: []} + - {eval_time: 10m, alertname: EvoRegionCheckpointStale, exp_alerts: []} + - {eval_time: 10m, alertname: EvoRegionQueueSaturated, exp_alerts: []} + - {eval_time: 10m, alertname: EvoRegionObjectsLow, exp_alerts: []} + - {eval_time: 10m, alertname: EvoRegionObjectsFull, exp_alerts: []} + - {eval_time: 10m, alertname: AlertmanagerDown, exp_alerts: []} + - eval_time: 1m + alertname: Watchdog + exp_alerts: + - exp_labels: {severity: none} + exp_annotations: + summary: "Always firing: proves the alert pipeline end to end" + description: "Route to a heartbeat receiver once one exists; its absence then means monitoring is down." + + # Outage: the region disappears, then comes back not ready and quarantined. + - interval: 15s + input_series: + - series: 'up{job="evo-region",environment="dev",world="100",region="1"}' + values: '0x20' + - series: 'evo_region_ready{environment="stage"}' + values: '0x20' + - series: 'evo_region_phase{environment="stage",phase="quarantined"}' + values: '1x20' + alert_rule_test: + - eval_time: 3m + alertname: EvoRegionDown + exp_alerts: + - exp_labels: {severity: critical, job: evo-region, environment: dev, world: "100", region: "1"} + exp_annotations: + summary: "Region dev/100/1 is not scrapeable" + description: "No operations endpoint answers. The process or its pod is gone; a forced deletion recovers in about 20 s, so 2 minutes is a real outage." + - eval_time: 1m + alertname: EvoRegionDown + exp_alerts: [] + - eval_time: 3m + alertname: EvoRegionNotReady + exp_alerts: + - exp_labels: {severity: critical, environment: stage} + exp_annotations: + summary: "Region stage has no authoritative host" + description: "The process is alive but holds no lease or has not verified its head. Check the phase metric and the host log." + - eval_time: 2m + alertname: EvoRegionQuarantined + exp_alerts: + - exp_labels: {severity: critical, environment: stage, phase: quarantined} + exp_annotations: + summary: "Region stage is quarantined" + description: "Committed state or history failed verification. The region stays unavailable; never re-provision it. Audit with evo-world-clone audit." + + # A 20 s lease-bound forced-deletion recovery must not page anyone. + - interval: 15s + input_series: + - series: 'evo_region_ready{environment="dev"}' + values: '1 1 0 0 1 1 1 1 1 1' + alert_rule_test: + - {eval_time: 45s, alertname: EvoRegionNotReady, exp_alerts: []} # During the dip. + - {eval_time: 2m, alertname: EvoRegionNotReady, exp_alerts: []} + + # Lease, fences, storage and database failures. + - interval: 15s + input_series: + - series: 'evo_region_ready{environment="dev"}' + values: '1x20' + - series: 'evo_region_lease_remaining_milliseconds{environment="dev"}' + values: '6000x20' + - series: 'evo_region_fences_total{environment="dev"}' + values: '0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1' + - series: 'evo_region_store_failures_total{environment="dev"}' + values: '0+1x20' + - series: 'evo_region_unknown_outcomes_total{environment="dev"}' + values: '0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1' + alert_rule_test: + - eval_time: 2m + alertname: EvoRegionLeaseMargin + exp_alerts: + - exp_labels: {severity: warning, environment: dev} + exp_annotations: + summary: "Region dev lease is not being renewed" + description: "Healthy renewals keep 15-20 s of lease. Under 10 s for a minute means renewals are failing; the host will fence itself." + - eval_time: 2m + alertname: EvoRegionFenced + exp_alerts: + - exp_labels: {severity: warning, environment: dev} + exp_annotations: + summary: "Region dev host lost authority" + description: "A host was fenced within the last 10 minutes. Expected once after a forced deletion's predecessor resumes; repeated fences point to database or clock trouble." + - eval_time: 3m + alertname: EvoRegionStoreFailures + exp_alerts: + - exp_labels: {severity: warning, environment: dev} + exp_annotations: + summary: "Region dev storage or database failures" + description: "Publication or database calls are failing; progress pauses and nothing is acknowledged until they succeed." + - eval_time: 2m + alertname: EvoRegionUnknownOutcomes + exp_alerts: + - exp_labels: {severity: warning, environment: dev} + exp_annotations: + summary: "Region dev had commits with unknown outcomes" + description: "Commits timed out or lost their connection and were resolved by exact bytes. Occasional is safe; frequent indicates database latency near the 5 s commit deadline." + + # Work waiting: debt, staleness, lag, a saturated queue and dropped inputs. + - interval: 15s + input_series: + - series: 'evo_region_tick_debt{environment="dev"}' + values: '20x20' + - series: 'evo_region_queued_inputs{environment="dev"}' + values: '60x20' + - series: 'evo_region_queue_capacity_inputs{environment="dev"}' + values: '64x20' + - series: 'evo_region_checkpoint_age_milliseconds{environment="dev"}' + values: '400000+15000x20' + - series: 'evo_region_commit_lag_ticks{environment="dev"}' + values: '300x20' + - series: 'evo_region_dropped_total{environment="dev",reason="fenced"}' + values: '0 0 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3' + - series: 'evo_region_dropped_total{environment="dev",reason="refused"}' + values: '0+1x20' + alert_rule_test: + - eval_time: 3m + alertname: EvoRegionTickDebt + exp_alerts: + - exp_labels: {severity: warning, environment: dev} + exp_annotations: + summary: "Region dev is behind real time with inputs waiting" + description: "Due ticks are not being simulated although inputs are queued (catch-up is 8 ticks per step)." + - eval_time: 2m + alertname: EvoRegionCheckpointStale + exp_alerts: + - exp_labels: {severity: warning, environment: dev} + exp_annotations: + summary: "Region dev has not committed for 5 minutes with work pending" + description: "An idle region does not commit; this fires only while inputs or simulated ticks wait for durability." + - eval_time: 3m + alertname: EvoRegionCommitLag + exp_alerts: + - exp_labels: {severity: warning, environment: dev} + exp_annotations: + summary: "Region dev is not committing what it simulates" + description: "More than four full batches are simulated but not durable; acknowledgements are waiting." + - eval_time: 3m + alertname: EvoRegionQueueSaturated + exp_alerts: + - exp_labels: {severity: warning, environment: dev} + exp_annotations: + summary: "Region dev input backlog near its bound" + description: "New inputs are being backpressured or rejected." + - eval_time: 2m + alertname: EvoRegionInputsDropped + exp_alerts: + - exp_labels: {severity: warning, environment: dev, reason: fenced} + exp_annotations: + summary: "Region dev accepted inputs that will never be acknowledged" + description: "Inputs were drained, fenced or quarantined (fenced); clients must resubmit to the current owner." + + # Object volume filling up, then below its reserve. + - interval: 15s + input_series: + - series: 'evo_region_objects_available_bytes{environment="dev"}' + values: '1500000000x30' + - series: 'evo_region_objects_available_bytes{environment="stage"}' + values: '900000000x30' # Below the reserve but above half of it. + - series: 'evo_region_objects_reserve_bytes{environment="dev"}' + values: '1073741824x30' + - series: 'evo_region_objects_reserve_bytes{environment="stage"}' + values: '1073741824x30' + - series: 'up{job="alertmanager"}' + values: '0x30' + alert_rule_test: + - eval_time: 6m + alertname: EvoRegionObjectsLow + exp_alerts: + - exp_labels: {severity: warning, environment: dev} + exp_annotations: + summary: "Region dev object volume is below twice its reserve" + description: "local-path volumes do not enforce their size; the node's disk is running out." + - exp_labels: {severity: warning, environment: stage} + exp_annotations: + summary: "Region stage object volume is below twice its reserve" + description: "local-path volumes do not enforce their size; the node's disk is running out." + - eval_time: 2m + alertname: EvoRegionObjectsFull + exp_alerts: + - exp_labels: {severity: critical, environment: stage} + exp_annotations: + summary: "Region stage object volume is below its reserve" + description: "New checkpoints are refused; the region cannot commit or acknowledge until space is freed." + - eval_time: 6m + alertname: AlertmanagerDown + exp_alerts: + - exp_labels: {severity: critical, job: alertmanager} + exp_annotations: + summary: "Alertmanager is not scrapeable" + description: "Alerts are evaluated but cannot be routed." diff --git a/deploy/kubernetes/platform/cluster/kustomization.yaml b/deploy/kubernetes/platform/cluster/kustomization.yaml index faed6a2..8ba827d 100644 --- a/deploy/kubernetes/platform/cluster/kustomization.yaml +++ b/deploy/kubernetes/platform/cluster/kustomization.yaml @@ -12,5 +12,6 @@ resources: - ../dev - ../stage - ../world0 + - ../monitoring commonAnnotations: argocd.argoproj.io/sync-options: Prune=false,Delete=false diff --git a/deploy/kubernetes/platform/monitoring/guardrails.yaml b/deploy/kubernetes/platform/monitoring/guardrails.yaml new file mode 100644 index 0000000..e254e20 --- /dev/null +++ b/deploy/kubernetes/platform/monitoring/guardrails.yaml @@ -0,0 +1,65 @@ +apiVersion: v1 +kind: ResourceQuota +metadata: + name: aevolution-quota +spec: + hard: + requests.cpu: "1" + requests.memory: 1Gi + limits.memory: 2Gi + pods: "6" + persistentvolumeclaims: "2" + requests.storage: 10Gi + services.loadbalancers: "0" + services.nodeports: "0" +--- +apiVersion: v1 +kind: LimitRange +metadata: + name: aevolution-defaults +spec: + limits: + - type: Container + defaultRequest: {cpu: 50m, memory: 64Mi} + default: {memory: 256Mi} + max: {memory: 2Gi} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: aevolution-default-deny +spec: + podSelector: {} + policyTypes: [Ingress, Egress] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: aevolution-allow-dns +spec: + podSelector: {} + policyTypes: [Egress] + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - {protocol: UDP, port: 53} + - {protocol: TCP, port: 53} +--- +# No Kubernetes API access: scrape targets are static, so no token and no RBAC. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: aevolution-prometheus +automountServiceAccountToken: false +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: aevolution-alertmanager +automountServiceAccountToken: false diff --git a/deploy/kubernetes/platform/monitoring/kustomization.yaml b/deploy/kubernetes/platform/monitoring/kustomization.yaml new file mode 100644 index 0000000..e5e72f3 --- /dev/null +++ b/deploy/kubernetes/platform/monitoring/kustomization.yaml @@ -0,0 +1,13 @@ +# M8.5 monitoring namespace (ADR-0072): guardrails and identities owned by the +# platform app, like every AEvolution namespace; Prometheus and Alertmanager +# themselves are the aevolution-monitoring workload app. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: aevolution-monitoring +resources: + - namespace.yaml + - guardrails.yaml +labels: + - pairs: + app.kubernetes.io/part-of: aevolution + app.kubernetes.io/managed-by: aevolution-platform diff --git a/deploy/kubernetes/platform/monitoring/namespace.yaml b/deploy/kubernetes/platform/monitoring/namespace.yaml new file mode 100644 index 0000000..30b1a79 --- /dev/null +++ b/deploy/kubernetes/platform/monitoring/namespace.yaml @@ -0,0 +1,10 @@ +# Metrics and alerting for the AEvolution environments. Holds no world data. +apiVersion: v1 +kind: Namespace +metadata: + name: aevolution-monitoring + labels: + aevolution.io/environment: monitoring + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/enforce-version: v1.32 + pod-security.kubernetes.io/warn: restricted diff --git a/deploy/kubernetes/region/base/kustomization.yaml b/deploy/kubernetes/region/base/kustomization.yaml index 393abb5..3b2a21c 100644 --- a/deploy/kubernetes/region/base/kustomization.yaml +++ b/deploy/kubernetes/region/base/kustomization.yaml @@ -8,6 +8,7 @@ resources: - provision-job.yaml - deployment.yaml - service.yaml + - scrape-policy.yaml labels: - pairs: app.kubernetes.io/part-of: aevolution diff --git a/deploy/kubernetes/region/base/scrape-policy.yaml b/deploy/kubernetes/region/base/scrape-policy.yaml new file mode 100644 index 0000000..9afab4c --- /dev/null +++ b/deploy/kubernetes/region/base/scrape-policy.yaml @@ -0,0 +1,21 @@ +# M8.5: the region's operations port accepts connections only from the AEvolution +# Prometheus. Kubelet probes come from the node and are not affected (Calico). +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: evo-region-scrape +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: evo-region + policyTypes: [Ingress] + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: aevolution-monitoring + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + ports: + - {protocol: TCP, port: 9100} diff --git a/docs/BACKLOG.md b/docs/BACKLOG.md index 7f528ed..57fbbaa 100644 --- a/docs/BACKLOG.md +++ b/docs/BACKLOG.md @@ -10,6 +10,8 @@ names it. Remove an item only when it is done, and record where it was done. | **Backup destination for dev/stage stores.** There are no backups yet (maintainer, 2026-09-29). The plan is to build backup tooling for the AEvolution PostgreSQL VM and each `aevolution-objects` volume up to a pluggable destination, then add the destination. It must be off the host for anything beyond disposable environments. | M8.3, 2026-09-29 | The maintainer will supply a destination later | M8.7 backup and restore; any World0 data | | **Secret manager for Argo-delivered configuration.** Secrets are still created by hand (ADR-0069 amendment). Choose how they reach the cluster through Git or Argo: for example Vault with the Agent Injector (as in the maintainer's other environment), or sealed or external secrets. Until then, rotate by hand. | M8.3, 2026-09-29 | The maintainer wants everything through Argo; a secret manager is not chosen | Stage and World0 credentials; M8.4 | | **Store Job values.** Supply the published registry digests for `aevolution-postgres-tools` and `aevolution-server` (from each CI Publish step's `Digest:` line), and the NOLOGIN runtime group role for dev (`aevolution`) and for stage (`aevolution_stage`). Then include `stores/` in each environment's path. | M8.3, 2026-09-29 | Values pending | M8.3 in-cluster evidence | +| **Alert destination.** Alertmanager routes every alert to a null receiver (ADR-0072). Choose where alerts go (Discord, email, a webhook), and optionally a heartbeat receiver for the always-firing Watchdog. Receiver credentials go in a Secret. | M8.5, 2026-09-30 | The maintainer will decide later | Any claim that alerts reach a person; M8.6 alert evidence; World0 | +| **Kubelet log rotation check.** Confirm the RKE2 nodes' `container-log-max-size` and `container-log-max-files` (default 10Mi × 5), which bound region host logs (ADR-0072). | M8.5, 2026-09-30 | Needs node access | M8.6 | | **Storage after the RAID/disk rework.** Re-run `deploy/kubernetes/inventory/disk-probe-local-path.yaml` (or a new class), and update INVENTORY.md. | M8.1, 2026-09-29 | Maintainer is reorganizing disks | M8.3's final storage decision | ## Accepted risks (maintainer decisions) diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index f5d424a..f9da858 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -635,3 +635,11 @@ M8.4 implemented and locally qualified (2026-09-29), under ADR-0071: - `region-drill.sh` for rollout and forced-deletion evidence. The in-cluster sync and drill wait on the maintainer. No director yet: assignment is static and lives in Git. + +M8.5 implemented and locally qualified (2026-09-30), under ADR-0072: +- AEvolution-only Prometheus 3.13.3 and Alertmanager 0.34.1, pinned by digest; +- 16 alert rules, tested by injection with promtool in CI; +- object volume capacity metrics; +- the database outbox kept (no broker); +- bounded logs, and a cardinality and cost report. +Alert delivery waits on the maintainer's choice of receiver. diff --git a/docs/STATUS.md b/docs/STATUS.md index a0f8dc5..e8dbe7b 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1064,3 +1064,22 @@ the policy check enforces this. Dev is world 100 and stage world 200, both regio Next: apply `deploy/argocd/dev/project.yaml`, sync, and run `tools/ops/region-drill.sh dev`. + +## M8.5 observability (2026-09-30) + +ADR-0072 adds an AEvolution-only monitoring stack in `aevolution-monitoring`: +- **Components:** Prometheus 3.13.3 and Alertmanager 0.34.1, pinned by digest and + delivered through Argo. +- **Scraping:** static targets, so no Kubernetes API access. The region's port + 9100 is reachable only from Prometheus. +- **Alerts:** 16 rules covering authority, lease, commits, backlog and object + capacity, plus a Watchdog. Each is proven firing and silent by promtool + injection tests in CI. They route to a null receiver until a destination is + chosen. +- **Metrics:** two new capacity gauges (32 fixed series). + +No broker is adopted: the database outbox stays. Logs are bounded by kubelet +rotation. Monitoring costs about 1,300 series, roughly 11 MB a day. + +Next: sync the monitoring app, confirm the targets, and run the `EvoRegionDown` +injection. M8.4's drill results are still to be recorded. diff --git a/docs/adr/0066-operations-endpoints.md b/docs/adr/0066-operations-endpoints.md index 882a9ed..fd1fe5e 100644 --- a/docs/adr/0066-operations-endpoints.md +++ b/docs/adr/0066-operations-endpoints.md @@ -111,3 +111,10 @@ Tests check the per-call budget, including at the connection cap, and that an accepted probe is still answered under replenished pressure. A real server keeps its lease and readiness while a client opens and closes connections continuously through 60 ticks. + +## Amendment: object volume capacity (2026-09-30, M8.5) +Two gauges were added, `evo_region_objects_available_bytes` and +`evo_region_objects_reserve_bytes`, so that ADR-0070's free-space reserve can be +alerted on before new objects are refused (ADR-0072). The fixed exposition is now +32 series. It still has only the `phase` and `reason` labels, and stays under the +8 KiB bound. diff --git a/docs/adr/0072-observability.md b/docs/adr/0072-observability.md new file mode 100644 index 0000000..cc7cefc --- /dev/null +++ b/docs/adr/0072-observability.md @@ -0,0 +1,125 @@ +# ADR-0072: Observability, alerting and background delivery for dev and stage + +## Status +Accepted for M8.5, 2026-09-30. It implements the M8 part of ADR-0015 +(observability), and builds on: +- ADR-0066 (the operations endpoints); +- ADR-0070 (the object reserve); +- ADR-0071 (the region host). + +The maintainer chose an AEvolution-only Prometheus, and left the alert destination +for later. + +## Context +M8.5 asks for: +- tick, lease, commit, storage and backup alerts, and bounded logs; +- an evaluation of the database outbox before any optional JetStream, with no + mandatory Redis or Kafka; +- evidence: alert injection, and a telemetry cardinality and cost report. + +The named risks are broker redelivery changing authority, and logs filling a disk. + +The cluster has no monitoring stack beyond metrics-server (INVENTORY.md). Each +region exposes 32 fixed series. The only labels are a 6-value phase and a 4-value +drop reason. + +## Decision +**Stack.** Namespace `aevolution-monitoring`, with platform-owned guardrails, and +a manually synced Argo application `aevolution-monitoring`. +- **Components.** One Prometheus and one Alertmanager: + - `quay.io/prometheus/prometheus:v3.13.3`, pinned by digest `6976aa8a…`; + - `quay.io/prometheus/alertmanager:v0.34.1`, pinned by digest `e9733baf…`. + + Both are Apache-2.0, and Prometheus is a CNCF graduated project. 3.13 is a + maintained line: 3.13.3 of 2026-09-07 carries current security fixes. +- **Prometheus configuration:** + - Static scrape targets: the `evo-region-ops` Service in dev and stage, labelled + with `environment`, `world` and `region`. No service discovery, so no token + and no RBAC. + - 15 s scrape and evaluation, a 64-sample and 64 KB cap per scrape, and + retention of 15 days or 3 GB. + - A 5 Gi `local-path` volume. Metrics are not world data, so Delete reclaim is + acceptable. +- **Network.** The region namespaces admit port 9100 only from Prometheus. + Prometheus reaches only the region operations ports and Alertmanager; + Alertmanager accepts only Prometheus. Pods run as nobody, with a read-only root + filesystem and no capabilities. +- **Routing.** Alertmanager routes everything to a **null** receiver until a + destination is chosen (docs/BACKLOG.md). Alerts remain visible in its API and + UI. + +**Metrics added.** `evo_region_objects_available_bytes` and +`evo_region_objects_reserve_bytes` make volume capacity alertable before new +objects are refused. The fixed exposition is now 32 series. + +**Alerts** (`config/evo-region.rules.yml`, 16 rules). Thresholds come from +measured behaviour: + +| Area | Alerts | +|---|---| +| Authority | Down (2 min), NotReady (2 min; a forced-deletion recovery takes about 20 s and does not fire), Quarantined, LeaseMargin (below 10 s while ready; healthy is 15–20 s), Fenced | +| Progress | StoreFailures, UnknownOutcomes, CommitLag (over 256 ticks), TickDebt and CheckpointStale (only while inputs or lag wait, because an idle region always accrues debt), QueueSaturated, InputsDropped (not counting `refused`) | +| Storage | ObjectsLow (below 2× the reserve), ObjectsFull (below the reserve) | +| Pipeline | Watchdog (always firing, for a later heartbeat receiver), AlertmanagerDown | + +**Backup alerts** wait for backups (M8.7). There is nothing to alert on until a +backup destination exists; see docs/BACKLOG.md. + +**Alert injection.** `promtool test rules` checks each alert firing on injected +series and staying silent in the healthy, idle and 20 s recovery cases. The CI +`deploy` stage runs it with the `promtool` and `amtool` copied from the same +pinned images, together with syntax checks of the Prometheus and Alertmanager +configurations. + +**Logs.** Container logs are rotated by the kubelet. The defaults are 10 Mi per +file and 5 files per container, 50 Mi at most per container. The actual RKE2 +setting has not been verified on the nodes yet. At most: +- `evo-server` writes one ack line of about 60 bytes per tick, about 100 MB a day + at 20 Hz, so the rotation keeps the most recent 12 hours; +- Prometheus and Alertmanager log little. + +Nothing writes logs to a volume. Durable history lives in PostgreSQL and the +object store, never in logs. + +**Background delivery.** The database outbox stays the delivery mechanism, and no +broker is adopted: +- `region_outbox` is written in the same transaction as each permanent outcome, + delivery is idempotent (`acknowledge_region_outbox`), and delivery never + mutates simulation state (ADR-0063). +- There is no consumer of outcomes yet: no gateway, client or projection. A relay + worker would deliver to nothing, so it is deferred until a consumer exists. +- JetStream is not adopted. There is no demonstrated delivery workload, and a + broker would add a second durability layer to operate and back up. It could + also never hold authority, since its redelivery could only re-announce an + already-final outcome. Redis and Kafka are not needed. + +**Cardinality and cost** (per 15 s scrape): + +| Source | Series | +|---|---| +| Each region | 32, plus about 5 per-target scrape series | +| Two regions (dev, stage) | about 75 | +| Prometheus self-scrape | about 1,000 (typical) | +| Alertmanager | about 200 | +| **Total** | **about 1,300** | + +That is about 87 samples per second. At about 1.5 bytes per compressed sample, +that is about 11 MB a day, or about 170 MB over 15 days, well inside the 3 GB +retention cap and the 5 Gi volume. Each added region adds about 37 series. The +request is 100m CPU and 256 Mi, with a 1 Gi memory limit. Alertmanager requests +50m and 64 Mi, with a 128 Mi limit. + +## Consequences +- Alerts are evaluated and visible, but not delivered until a receiver is chosen. + The Watchdog is ready for a dead-man's-switch receiver. +- Changing a threshold is a Git change that must keep the rule tests passing. +- The kubelet log rotation values should be confirmed on the nodes. +- World0 monitoring (M8.8) reuses the rules with its own target. + +## Alternatives rejected +- **kube-prometheus-stack or the Prometheus Operator.** Cluster-wide CRDs and + RBAC for a single project; not the maintainer's choice. +- **Kubernetes service discovery.** It needs a token and cluster read RBAC for no + gain with two static targets. +- **JetStream, Kafka or Redis for outcomes.** There is no workload yet, and each + would add another stateful system. diff --git a/docs/adr/README.md b/docs/adr/README.md index 1a1f370..95b7cfd 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -82,3 +82,4 @@ design. No unadopted candidate is a link dependency of the proof. - [ADR-0069: Environment isolation and World0 protection](0069-environment-isolation.md) - accepted for M8.2 per-environment guardrails, native World0 admission protection, scoped Argo projects and verification. - [ADR-0070: Durable stores for dev and stage](0070-dev-stage-stores.md) - accepted for M8.3 external PostgreSQL per environment, Retain-class node-local object volumes, free-space reserve and in-cluster store qualification. - [ADR-0071: Region host deployment in dev and stage](0071-region-host-deployment.md) - accepted for M8.4 single-replica Recreate host, digest-as-build identity, idempotent provisioning hook and the region drill; no director yet. +- [ADR-0072: Observability, alerting and background delivery](0072-observability.md) - accepted for M8.5 pinned AEvolution Prometheus/Alertmanager, 16 tested alert rules, bounded logs, database outbox over any broker, cardinality/cost report. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index ab7cc7d..b458ac9 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -2811,3 +2811,36 @@ admission policy and the stage and World0 AppProjects, then ran - `region-drill.sh` passed `bash -n`, and refuses World0 before kubectl (also enforced by the policy check). - Not executed: the in-cluster region sync and the drill. + +## M8.5 observability - 2026-09-30 + +- `promtool` 3.13.3 and `amtool` 0.34.1 came from the official release tarballs, + with SHA-256 verified against the published sums: `b349c732…` and `265b9d1e…`. + - The Prometheus configuration syntax is valid, and all 16 rules are valid. + - The alert injection tests passed, as did the Alertmanager `check-config`. +- Rule mutations: five were tried, and three were caught at first. + - **NotReady firing after 15 s:** missed, because the forced-deletion test only + evaluated after recovery. The test now also checks during the dip. + - **ObjectsFull weakened to half the reserve:** missed, because the injected + free space was below both thresholds. The test now uses 900 MB. + - After these fixes all five were caught: the lease threshold, NotReady too + eager, debt counted without inputs, a weakened ObjectsFull, and `refused` + drops counted. +- Windows MSVC CTest 59/59, including the `ops-contracts` update to 32 series. +- WSL Linux `checks` set: 59/59 in each configuration. The format check passed. +- PostgreSQL suites: Release, analysis and ASan/UBSan passed. The ops process + test checks the reserve gauge (1 GiB) and that free space is nonzero. +- `deploy-policy-check.py`: 67 objects, 0 failures. New rules: + - monitoring images must be digest-pinned; + - monitoring pods mount no token; + - no RBAC objects are allowed; + - the monitoring project is scoped. +- Kustomize renders the platform bundle (four namespaces), the monitoring app + (2 Deployments, 2 ConfigMaps, 2 Services, 2 policies, 1 claim), and dev and + stage with the scrape policy. +- The CI `ci-deploy` stage now copies `promtool` and `amtool` from the pinned + images and runs `tools/ci/monitoring-check.sh`. It has not run locally, because + Docker is not running. +- Not executed: anything in the cluster (monitoring sync, target health, the + `EvoRegionDown` injection), alert delivery (no receiver), and the node log + rotation check. diff --git a/docs/infrastructure/ENVIRONMENTS.md b/docs/infrastructure/ENVIRONMENTS.md index 34a4fad..ae44bef 100644 --- a/docs/infrastructure/ENVIRONMENTS.md +++ b/docs/infrastructure/ENVIRONMENTS.md @@ -139,6 +139,27 @@ record: - the first pod's `phase running epoch=... committed=...` line; - that `/readyz` stays 503 while a replacement waits out the lease. +## Monitoring (M8.5, ADR-0072) + +Bootstrap once: `kubectl apply -f deploy/argocd/monitoring/`, then sync +`aevolution-platform` (it adds the `aevolution-monitoring` namespace and its +guardrails) and `aevolution-monitoring`. Re-sync dev and stage, which adds the +`evo-region-scrape` policy. + +```sh +kubectl -n aevolution-monitoring port-forward svc/prometheus 9090 & +curl -s localhost:9090/api/v1/targets | grep -o '"health":"[a-z]*"' # both regions up +curl -s localhost:9090/api/v1/alerts # Watchdog firing only +``` + +**Injection in the cluster** (dev only): +- `kubectl -n evolution-dev scale deploy/evo-region --replicas=0` makes + `EvoRegionDown` pending, then firing after 2 minutes, in Alertmanager (null + receiver). +- Scale back to 1 and it resolves. + +Record the timestamps. The rule tests in CI already prove the thresholds. + ## Break-glass How membership is granted is not decided yet; see docs/BACKLOG.md. Until it is, diff --git a/include/evo/server/ops.hpp b/include/evo/server/ops.hpp index 866b7dd..d2f7755 100644 --- a/include/evo/server/ops.hpp +++ b/include/evo/server/ops.hpp @@ -44,6 +44,9 @@ struct OpsCounters { std::uint64_t checkpoint_age_ms{}; // Since the last confirmed commit or recovery. std::uint64_t acks{}, outcomes{}; std::array dropped{}; // Indexed by Unacknowledged. + // M8.5: object volume free space and the store's reserve (ADR-0070), so capacity + // is alerted on before new objects are refused. + std::uint64_t objects_available_bytes{}, objects_reserve_bytes{}; }; [[nodiscard]] const char* phase_name(HostPhase phase) noexcept; diff --git a/include/evo/server/region_host.hpp b/include/evo/server/region_host.hpp index 3f6d6da..bed1305 100644 --- a/include/evo/server/region_host.hpp +++ b/include/evo/server/region_host.hpp @@ -107,6 +107,10 @@ template class RegionHost { [[nodiscard]] const HostStatus& status() const noexcept { return status_; } + // Read-only, for operational reporting (free space against the reserve). + [[nodiscard]] const persistence::FilesystemObjectStore& objects() const noexcept { + return *objects_; + } // Read-only view for diagnostics/tests; null unless recovered. [[nodiscard]] const State* state() const noexcept { return state_.get(); diff --git a/src/server/ops.cpp b/src/server/ops.cpp index 3d9507c..2a6e063 100644 --- a/src/server/ops.cpp +++ b/src/server/ops.cpp @@ -81,6 +81,10 @@ std::string render_metrics(const HostStatus& s, const OpsCounters& c) { c.queue_capacity_inputs); metric(out, "evo_region_queue_capacity_bytes", "gauge", "Input backlog byte bound", c.queue_capacity_bytes); + metric(out, "evo_region_objects_available_bytes", "gauge", + "Free bytes on the object volume available to the host", c.objects_available_bytes); + metric(out, "evo_region_objects_reserve_bytes", "gauge", + "Free-space reserve below which new objects are refused", c.objects_reserve_bytes); metric(out, "evo_region_commits_total", "counter", "Confirmed commits", s.commits); metric(out, "evo_region_store_failures_total", "counter", "Storage or database failures that paused progress", s.store_failures); diff --git a/tests/ops_tests.cpp b/tests/ops_tests.cpp index 5a26077..accb4e0 100644 --- a/tests/ops_tests.cpp +++ b/tests/ops_tests.cpp @@ -82,8 +82,8 @@ void exposition_is_bounded_and_well_formed() { CHECK(text.find("evo_region_commit_lag_ticks 9\n") != std::string::npos); CHECK(text.find("evo_region_lease_remaining_milliseconds 1234\n") != std::string::npos); CHECK(text.find("evo_region_dropped_total{reason=\"fenced\"} 5\n") != std::string::npos); - // Series count is fixed: 6 phases + 4 reasons + 20 unlabelled families. - CHECK(samples.size() == 6 + 4 + 20); + // Series count is fixed: 6 phases + 4 reasons + 22 unlabelled families. + CHECK(samples.size() == 6 + 4 + 22); // Worst-case values keep the size bounded and never underflow the lag. HostStatus big; @@ -93,7 +93,7 @@ void exposition_is_bounded_and_well_formed() { big.max_tick_debt = big.commits = big.store_failures = big.unknown_outcomes = most; big.fences = big.lease_remaining_ms = most; big.simulated_tick = 1; // Lag saturates at zero rather than wrapping. - OpsCounters bigc{most, most, most, most, most, {most, most, most, most}}; + OpsCounters bigc{most, most, most, most, most, {most, most, most, most}, most, most}; const auto large = render_metrics(big, bigc); CHECK(large.size() <= max_metrics_bytes); CHECK(large.find("evo_region_commit_lag_ticks 0\n") != std::string::npos); diff --git a/tests/region_ops_tests.cpp b/tests/region_ops_tests.cpp index 7d8b5a2..ad024c8 100644 --- a/tests/region_ops_tests.cpp +++ b/tests/region_ops_tests.cpp @@ -97,6 +97,9 @@ void probes_follow_authority_and_stay_bounded() { CHECK(metric(port, "evo_region_lease_remaining_milliseconds") > 0); CHECK(metric(port, "evo_region_queue_capacity_inputs") == 64 && metric(port, "evo_region_queue_capacity_bytes") == 65536); + // M8.5: the default reserve is 1024 MiB, and the volume reports real free space. + CHECK(metric(port, "evo_region_objects_reserve_bytes") == std::uint64_t{1024} * 1024 * 1024 && + metric(port, "evo_region_objects_available_bytes") > 0); auto page = get(port, "/metrics"); CHECK(page.body.find("evo_region_phase{phase=\"running\"} 1\n") != std::string::npos); server.send(input_lines(1, 30, false)); diff --git a/tools/ci/deploy-policy-check.py b/tools/ci/deploy-policy-check.py index 34dde18..2930d52 100644 --- a/tools/ci/deploy-policy-check.py +++ b/tools/ci/deploy-policy-check.py @@ -121,8 +121,8 @@ for path, doc in objects: if doc['kind'] == 'AppProject' and name == 'aevolution-platform': spec = doc['spec'] namespaces = sorted(d.get('namespace') for d in spec.get('destinations', [])) - check(namespaces == ['evolution-dev', 'evolution-stage', 'evolution-world0'], - f'{path}: the platform project targets exactly the three AEvolution namespaces') + check(namespaces == ['aevolution-monitoring', 'evolution-dev', 'evolution-stage', 'evolution-world0'], + f'{path}: the platform project targets exactly the AEvolution namespaces') cluster = sorted((w.get('group'), w.get('kind')) for w in spec.get('clusterResourceWhitelist', [])) check(cluster == [('', 'Namespace'), ('admissionregistration.k8s.io', 'ValidatingAdmissionPolicy'), @@ -131,6 +131,11 @@ for path, doc in objects: f'{path}: the platform project may own only its four cluster kinds') kinds = {w.get('kind') for w in spec.get('namespaceResourceWhitelist', [])} check(not kinds & {'Secret', 'Role', 'RoleBinding', '*'}, f'{path}: platform whitelists a forbidden kind') + elif doc['kind'] == 'AppProject' and name == 'aevolution-monitoring': + spec = doc['spec'] + check([d.get('namespace') for d in spec.get('destinations', [])] == ['aevolution-monitoring'] + and spec.get('clusterResourceWhitelist') == [], + f'{path}: monitoring targets only its namespace, with no cluster kinds') elif doc['kind'] == 'AppProject' and name.startswith('aevolution-'): env = name[len('aevolution-'):] spec = doc['spec'] @@ -161,7 +166,7 @@ for path, doc in objects: check(automated is None, f'{path}: {name} must not auto-sync') check('resources-finalizer.argocd.argoproj.io' not in doc['metadata'].get('finalizers', []), f'{path}: {name} must not cascade-delete') - if project.startswith('aevolution-') and project != 'aevolution-platform': + if project.startswith('aevolution-') and project not in ('aevolution-platform', 'aevolution-monitoring'): check(destination == 'evolution-' + project[len('aevolution-'):], f'{path}: {name} leaves its project environment') check(destination != 'evolution-world0' or name in ('aevolution-world0', 'aevolution-platform'), @@ -222,12 +227,24 @@ for env in ('dev', 'stage'): check(len(server) == 1 and len(builds) == 2 and all('sha256:' + b == server[0] for b in builds), f'{overlay}: the build identity must equal the server image digest') +# M8.5 monitoring (ADR-0072): third-party images are pinned by digest, and Prometheus +# needs no Kubernetes API access. +for path, doc in objects: + if path.startswith('deploy/kubernetes/monitoring/') and doc['kind'] == 'Deployment': + pod = doc['spec']['template']['spec'] + for container in pod['containers']: + check('@sha256:' in container.get('image', ''), f'{path}: {container["name"]} must be pinned by digest') + check(pod.get('automountServiceAccountToken') is False, f'{path}: monitoring pods mount no token') + if path.startswith('deploy/') and doc['kind'] in ('ClusterRole', 'ClusterRoleBinding', 'Role', 'RoleBinding'): + check(False, f'{path}: AEvolution grants no Kubernetes RBAC') + # The platform bundle: every rendered object is protected from prune and deletion. bundle = manifests.get(platform + '/cluster/kustomization.yaml', [{}])[0] check(bundle.get('commonAnnotations', {}).get('argocd.argoproj.io/sync-options') == 'Prune=false,Delete=false', 'the platform bundle must mark every object Prune=false,Delete=false') -check(sorted(bundle.get('resources', [])) == ['../admission', '../dev', '../stage', '../storage', '../world0'], - 'the platform bundle must include admission, storage and the three environments') +check(sorted(bundle.get('resources', [])) == ['../admission', '../dev', '../monitoring', '../stage', '../storage', + '../world0'], + 'the platform bundle must include admission, storage, the environments and monitoring') # Paths a real (non-example) Application syncs must be complete: no REPLACE markers diff --git a/tools/ci/monitoring-check.sh b/tools/ci/monitoring-check.sh new file mode 100644 index 0000000..9fdfce0 --- /dev/null +++ b/tools/ci/monitoring-check.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# M8.5 monitoring configuration checks (ADR-0072): Prometheus configuration syntax, +# alert rule validity, the alert-injection unit tests, and the Alertmanager routing. +# Needs promtool and amtool on PATH, taken from the same pinned images that run in +# the cluster (the ci-deploy Docker stage copies them). No cluster access. +set -euo pipefail +root="$(cd "$(dirname "$0")/../.." && pwd)" +config="$root/deploy/kubernetes/monitoring/config" +promtool --version | head -1 +amtool --version 2>&1 | head -1 +promtool check config --syntax-only "$config/prometheus.yml" +promtool check rules "$config/evo-region.rules.yml" +(cd "$root/deploy/kubernetes/monitoring/tests" && promtool test rules evo-region.rules.test.yml) +amtool check-config "$config/alertmanager.yml" +echo "monitoring configuration, rules and alert tests passed"