Files
aevolution/tools/ci/deploy-policy-check.py
T
dingoandClaude Opus 5.5 b4493b3d0f Add M8.5 observability: pinned Prometheus, tested alerts, capacity metrics
An AEvolution-only Prometheus 3.13.3 and Alertmanager 0.34.1, pinned by
digest and delivered by Argo (ADR-0072), scrape each region's operations
Service through static targets (no API access, no RBAC); the region
namespaces admit port 9100 from Prometheus only. Sixteen alert rules
cover authority, lease, commits, backlog and object capacity and are
proven by promtool injection tests in CI with tools copied from the same
pinned images. Two new gauges expose object volume free space and the
reserve. Alerts route to a null receiver until a destination is chosen;
no broker is adopted and the database outbox stays.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 06:16:36 -04:00

305 lines
18 KiB
Python

#!/usr/bin/env python3
"""M8.2 offline deployment-policy check (ADR-0069). Needs Python 3 and PyYAML only.
Parses every manifest under deploy/ (including .yaml.example files) and fails if an
environment-isolation or World0-protection invariant is broken:
- no Secret objects in Git;
- every ServiceAccount disables token automounting;
- each platform overlay: labelled restricted namespace, quota, limit range,
default-deny and DNS-only policies, three service accounts; World0 alone is
labelled protected;
- the World0 admission policy denies what ADR-0069 lists, fails closed and is
bound with Deny;
- every AEvolution AppProject targets exactly its own namespace with no cluster
kinds; World0 blocks automated sync; no Application auto-syncs or carries the
cascade-delete finalizer, and each stays in its own environment;
- the live Argo path (deploy/kubernetes/dev) holds no Namespace, Secret, quota or
RBAC objects;
- the cleanup tool refuses World0 before invoking kubectl.
"""
import os
import shutil
import subprocess
import sys
import tempfile
import yaml
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
failures = []
def check(condition, message):
if not condition:
failures.append(message)
def documents(path):
with open(path, encoding='utf-8') as f:
return [d for d in yaml.safe_load_all(f) if isinstance(d, dict)]
manifests = {}
for base, _, files in os.walk(os.path.join(ROOT, 'deploy')):
for name in files:
if name.endswith(('.yaml', '.yml', '.yaml.example')):
path = os.path.join(base, name)
manifests[os.path.relpath(path, ROOT).replace(os.sep, '/')] = documents(path)
objects = [(p, d) for p, ds in manifests.items() for d in ds
if 'kind' in d and d['kind'] != 'Kustomization']
# Secrets and service-account tokens.
for path, doc in objects:
check(doc['kind'] != 'Secret', f'{path}: Secret objects never belong in Git')
if doc['kind'] == 'ServiceAccount':
check(doc.get('automountServiceAccountToken') is False,
f'{path}: ServiceAccount {doc["metadata"]["name"]} must not automount a token')
# Platform overlays.
platform = 'deploy/kubernetes/platform'
base = {d['kind'] + '/' + d['metadata']['name']: d
for p, d in objects if p.startswith(platform + '/base/')}
for key in ('ResourceQuota/aevolution-quota', 'LimitRange/aevolution-defaults',
'NetworkPolicy/aevolution-default-deny', 'NetworkPolicy/aevolution-allow-dns',
'ServiceAccount/aevolution-migrator', 'ServiceAccount/aevolution-region',
'ServiceAccount/aevolution-backup'):
check(key in base, f'{platform}/base: missing {key}')
deny = base.get('NetworkPolicy/aevolution-default-deny', {}).get('spec', {})
check(deny.get('podSelector') == {} and sorted(deny.get('policyTypes', [])) == ['Egress', 'Ingress']
and not deny.get('ingress') and not deny.get('egress'),
'default-deny must select every pod and allow nothing')
dns = base.get('NetworkPolicy/aevolution-allow-dns', {}).get('spec', {})
dns_ports = [p.get('port') for rule in dns.get('egress', []) for p in rule.get('ports', [])]
check(dns.get('policyTypes') == ['Egress'] and set(dns_ports) == {53} and len(dns.get('egress', [])) == 1,
'allow-dns must allow only egress to port 53')
quota = base.get('ResourceQuota/aevolution-quota', {}).get('spec', {}).get('hard', {})
check(quota.get('services.loadbalancers') == '0' and quota.get('services.nodeports') == '0',
'quota must forbid LoadBalancer and NodePort services')
for env in ('dev', 'stage', 'world0'):
k = f'{platform}/{env}/kustomization.yaml'
ns_path = f'{platform}/{env}/namespace.yaml'
check(k in manifests and ns_path in manifests, f'{platform}/{env}: overlay missing')
if k not in manifests or ns_path not in manifests:
continue
kustomization = manifests[k][0]
check(kustomization.get('namespace') == f'evolution-{env}' and '../base' in kustomization.get('resources', []),
f'{k}: must set namespace evolution-{env} and include ../base')
labels = manifests[ns_path][0]['metadata'].get('labels', {})
check(manifests[ns_path][0]['metadata']['name'] == f'evolution-{env}', f'{ns_path}: wrong name')
check(labels.get('aevolution.io/environment') == env, f'{ns_path}: environment label')
check(labels.get('pod-security.kubernetes.io/enforce') == 'restricted', f'{ns_path}: restricted pod security')
check((labels.get('aevolution.io/protected') == 'true') == (env == 'world0'),
f'{ns_path}: only world0 is protected')
# World0 admission policy.
policy = [d for p, d in objects if d['kind'] == 'ValidatingAdmissionPolicy'
and d['metadata']['name'] == 'aevolution-world0-protection']
binding = [d for p, d in objects if d['kind'] == 'ValidatingAdmissionPolicyBinding'
and d['metadata']['name'] == 'aevolution-world0-protection']
check(len(policy) == 1 and len(binding) == 1, 'world0 admission policy and binding must exist once')
if policy and binding:
spec = policy[0]['spec']
check(spec.get('failurePolicy') == 'Fail', 'world0 policy must fail closed')
check(binding[0]['spec'].get('validationActions') == ['Deny'] and 'paramRef' not in binding[0]['spec']
and 'matchResources' not in binding[0]['spec'], 'world0 binding must Deny everywhere')
covered = {(r, op) for rule in spec['matchConstraints']['resourceRules']
for r in rule['resources'] for op in rule['operations']}
for resource in ('namespaces', 'persistentvolumes', 'persistentvolumeclaims', 'secrets',
'serviceaccounts', 'applications', 'appprojects'):
check((resource, 'DELETE') in covered, f'world0 policy must cover DELETE {resource}')
for resource in ('namespaces', 'persistentvolumes'):
check((resource, 'UPDATE') in covered, f'world0 policy must cover UPDATE {resource}')
text = yaml.safe_dump(spec)
for needle in ("aevolution:world0-break-glass", "evolution-world0", "aevolution-world0",
"aevolution-platform",
"aevolution.io/protected", "Retain"):
check(needle in text, f'world0 policy must reference {needle}')
# Argo CD projects and applications.
for path, doc in objects:
name = doc['metadata'].get('name', '')
if doc['kind'] == 'AppProject' and name == 'aevolution-platform':
spec = doc['spec']
namespaces = sorted(d.get('namespace') for d in spec.get('destinations', []))
check(namespaces == ['aevolution-monitoring', 'evolution-dev', 'evolution-stage', 'evolution-world0'],
f'{path}: the platform project targets exactly the AEvolution namespaces')
cluster = sorted((w.get('group'), w.get('kind')) for w in spec.get('clusterResourceWhitelist', []))
check(cluster == [('', 'Namespace'),
('admissionregistration.k8s.io', 'ValidatingAdmissionPolicy'),
('admissionregistration.k8s.io', 'ValidatingAdmissionPolicyBinding'),
('storage.k8s.io', 'StorageClass')],
f'{path}: the platform project may own only its four cluster kinds')
kinds = {w.get('kind') for w in spec.get('namespaceResourceWhitelist', [])}
check(not kinds & {'Secret', 'Role', 'RoleBinding', '*'}, f'{path}: platform whitelists a forbidden kind')
elif doc['kind'] == 'AppProject' and name == 'aevolution-monitoring':
spec = doc['spec']
check([d.get('namespace') for d in spec.get('destinations', [])] == ['aevolution-monitoring']
and spec.get('clusterResourceWhitelist') == [],
f'{path}: monitoring targets only its namespace, with no cluster kinds')
elif doc['kind'] == 'AppProject' and name.startswith('aevolution-'):
env = name[len('aevolution-'):]
spec = doc['spec']
namespaces = [d.get('namespace') for d in spec.get('destinations', [])]
check(namespaces == [f'evolution-{env}'], f'{path}: {name} must target only evolution-{env}')
check(spec.get('clusterResourceWhitelist') == [], f'{path}: {name} must allow no cluster kinds')
kinds = {w.get('kind') for w in spec.get('namespaceResourceWhitelist', [])}
check(not kinds & {'Secret', 'Namespace', 'ResourceQuota', 'LimitRange', 'Role', 'RoleBinding', '*'},
f'{path}: {name} whitelists a forbidden kind')
if env == 'world0':
windows = spec.get('syncWindows', [])
check(any(w.get('kind') == 'deny' and w.get('manualSync') is True and w.get('duration') == '24h'
and w.get('schedule') == '* * * * *' for w in windows),
f'{path}: world0 must deny automated sync at all times')
check('ServiceAccount' not in kinds, f'{path}: world0 service accounts are platform-managed')
if doc['kind'] == 'Application':
spec = doc['spec']
project = spec.get('project', '')
destination = spec.get('destination', {}).get('namespace', '')
automated = (spec.get('syncPolicy') or {}).get('automated')
if name == 'aevolution-platform':
# Self-heal only: never prune, so no Git change can delete durable objects.
check(automated is not None and automated.get('prune') is False,
f'{path}: the platform app self-heals and never prunes')
check(spec.get('source', {}).get('path') == 'deploy/kubernetes/platform/cluster',
f'{path}: the platform app syncs deploy/kubernetes/platform/cluster')
else:
check(automated is None, f'{path}: {name} must not auto-sync')
check('resources-finalizer.argocd.argoproj.io' not in doc['metadata'].get('finalizers', []),
f'{path}: {name} must not cascade-delete')
if project.startswith('aevolution-') and project not in ('aevolution-platform', 'aevolution-monitoring'):
check(destination == 'evolution-' + project[len('aevolution-'):],
f'{path}: {name} leaves its project environment')
check(destination != 'evolution-world0' or name in ('aevolution-world0', 'aevolution-platform'),
f'{path}: only aevolution-world0 may target evolution-world0')
# M8.3 storage (ADR-0070): durable volumes retain their data; database passwords are
# only ever mounted as files, never placed in the environment.
classes = {d['metadata']['name']: d for p, d in objects if d['kind'] == 'StorageClass'}
retain = classes.get('aevolution-local-retain')
check(retain is not None and retain.get('reclaimPolicy') == 'Retain'
and retain.get('volumeBindingMode') == 'WaitForFirstConsumer'
and 'storageclass.kubernetes.io/is-default-class' not in retain['metadata'].get('annotations', {}),
'aevolution-local-retain must be a non-default Retain class that binds on first consumer')
for path, doc in objects:
if doc['kind'] == 'PersistentVolumeClaim' and path.startswith(platform + '/'):
check(doc['spec'].get('storageClassName') == 'aevolution-local-retain',
f'{path}: platform volumes must use aevolution-local-retain')
if doc['kind'] in ('Job', 'Deployment', 'StatefulSet', 'Pod'):
pod = doc['spec'].get('template', {}).get('spec', doc['spec'])
for container in pod.get('containers', []) + pod.get('initContainers', []):
check(not any('secretRef' in source for source in container.get('envFrom', [])),
f'{path}: never import a whole Secret into the environment')
for variable in container.get('env', []):
ref = variable.get('valueFrom', {}).get('secretKeyRef', {})
check(ref.get('key') not in ('PGPASSWORD', 'POSTGRES_PASSWORD', 'PGUSER', 'POSTGRES_USER'),
f'{path}: {variable.get("name")} must not come from {ref.get("key")}')
if pod.get('volumes') and 'store' in doc['metadata'].get('name', ''):
check(doc['metadata'].get('namespace') != 'evolution-world0',
f'{path}: store Jobs never target World0')
# M8.4 region host (ADR-0071): one owner at a time, readiness means authority.
for path, doc in objects:
# Overlay patches (region-values.yaml) are partial objects; the base is complete.
if (doc['kind'] == 'Deployment' and doc['metadata'].get('name') == 'evo-region'
and not path.endswith('region-values.yaml')):
spec = doc['spec']
pod = spec['template']['spec']
container = pod['containers'][0]
check(spec.get('replicas') == 1 and spec.get('strategy', {}).get('type') == 'Recreate',
f'{path}: the region host runs one replica with the Recreate strategy')
check(container.get('readinessProbe', {}).get('httpGet', {}).get('path') == '/readyz'
and container.get('livenessProbe', {}).get('httpGet', {}).get('path') == '/livez',
f'{path}: readiness must be /readyz and liveness /livez')
limits = container.get('resources', {}).get('limits', {})
check('cpu' not in limits and 'memory' in limits,
f'{path}: memory limit required, no CPU limit (throttling delays renewals)')
script = ' '.join(container.get('args', []))
drain = [int(x) for x in __import__('re').findall(r'--drain-ms (\d+)', script)]
check(len(drain) == 1 and pod.get('terminationGracePeriodSeconds', 30) * 1000 > drain[0],
f'{path}: the termination grace period must exceed --drain-ms')
for env in ('dev', 'stage'):
overlay = f'deploy/kubernetes/region/{env}/'
images = manifests.get(overlay + 'kustomization.yaml', [{}])[0].get('images', [])
server = [i.get('digest', '') for i in images if i.get('name') == 'aevolution-server']
builds = [v.get('value') for d in manifests.get(overlay + 'region-values.yaml', [])
for c in d['spec']['template']['spec']['containers'] for v in c.get('env', [])
if v.get('name') == 'REGION_BUILD']
check(len(server) == 1 and len(builds) == 2 and all('sha256:' + b == server[0] for b in builds),
f'{overlay}: the build identity must equal the server image digest')
# M8.5 monitoring (ADR-0072): third-party images are pinned by digest, and Prometheus
# needs no Kubernetes API access.
for path, doc in objects:
if path.startswith('deploy/kubernetes/monitoring/') and doc['kind'] == 'Deployment':
pod = doc['spec']['template']['spec']
for container in pod['containers']:
check('@sha256:' in container.get('image', ''), f'{path}: {container["name"]} must be pinned by digest')
check(pod.get('automountServiceAccountToken') is False, f'{path}: monitoring pods mount no token')
if path.startswith('deploy/') and doc['kind'] in ('ClusterRole', 'ClusterRoleBinding', 'Role', 'RoleBinding'):
check(False, f'{path}: AEvolution grants no Kubernetes RBAC')
# The platform bundle: every rendered object is protected from prune and deletion.
bundle = manifests.get(platform + '/cluster/kustomization.yaml', [{}])[0]
check(bundle.get('commonAnnotations', {}).get('argocd.argoproj.io/sync-options') == 'Prune=false,Delete=false',
'the platform bundle must mark every object Prune=false,Delete=false')
check(sorted(bundle.get('resources', [])) == ['../admission', '../dev', '../monitoring', '../stage', '../storage',
'../world0'],
'the platform bundle must include admission, storage, the environments and monitoring')
# Paths a real (non-example) Application syncs must be complete: no REPLACE markers
# anywhere they render, following kustomization resources.
def kustomize_files(directory, seen):
directory = os.path.normpath(directory)
if directory in seen:
return []
seen.add(directory)
found = []
k = os.path.join(directory, 'kustomization.yaml')
if not os.path.isfile(k):
return found
found.append(k)
spec = documents(k)[0]
for entry in spec.get('resources', []) + spec.get('components', []):
target = os.path.normpath(os.path.join(directory, entry))
if os.path.isdir(target):
found += kustomize_files(target, seen)
else:
found.append(target)
return found
for path, doc in objects:
# Examples count too once their path is real (the live dev app is only an example
# here); an example whose path is itself still a marker is not yet deployable.
source_path = doc.get('spec', {}).get('source', {}).get('path', '')
if doc['kind'] == 'Application' and 'REPLACE' not in source_path:
source = os.path.join(ROOT, source_path)
for f in kustomize_files(source, set()):
with open(f, encoding='utf-8') as handle:
check('REPLACE' not in handle.read(),
f'{os.path.relpath(f, ROOT)}: unresolved REPLACE marker in a synced path')
# The live Argo path.
for path, doc in objects:
if path.startswith('deploy/kubernetes/dev/'):
check(doc['kind'] not in {'Namespace', 'Secret', 'ResourceQuota', 'LimitRange', 'Role',
'RoleBinding', 'ClusterRole', 'ClusterRoleBinding'},
f'{path}: {doc["kind"]} must not live in the Argo workload path')
# The cleanup tool refuses World0 before kubectl runs (no kubectl on PATH here).
with tempfile.TemporaryDirectory() as empty:
env = dict(os.environ, PATH=empty + os.pathsep + os.path.dirname(sys.executable))
for tool in ('tools/ops/cleanup-environment.sh', 'tools/ops/region-drill.sh'):
for bad in ('world0', 'evolution-world0', 'World0', 'prod', ''):
result = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, tool), bad],
capture_output=True, text=True, env=env)
check(result.returncode == 2 and ('refused' in result.stderr or 'usage' in result.stderr)
and 'kubectl' not in result.stderr,
f'{tool} must refuse {bad!r} before kubectl (rc={result.returncode})')
for message in failures:
print('FAIL', message)
print(f'deploy policy: {len(objects)} objects in {len(manifests)} files, {len(failures)} failures')
sys.exit(1 if failures else 0)