#!/usr/bin/env python3 """M8.2 offline deployment-policy check (ADR-0069). Needs Python 3 and PyYAML only. Parses every manifest under deploy/ (including .yaml.example files) and fails if an environment-isolation or World0-protection invariant is broken: - no Secret objects in Git; - every ServiceAccount disables token automounting; - each platform overlay: labelled restricted namespace, quota, limit range, default-deny and DNS-only policies, three service accounts; World0 alone is labelled protected; - the World0 admission policy denies what ADR-0069 lists, fails closed and is bound with Deny; - every AEvolution AppProject targets exactly its own namespace with no cluster kinds; World0 blocks automated sync; no Application auto-syncs or carries the cascade-delete finalizer, and each stays in its own environment; - the live Argo path (deploy/kubernetes/dev) holds no Namespace, Secret, quota or RBAC objects; - the cleanup tool refuses World0 before invoking kubectl; - World0 launch stays blocked (ADR-0075): no workload targets World0 and its Application keeps an unresolved path; any image a World0 overlay pins must be the digest already pinned in stage (promotion, never a fresh build); - scheduling (ADR-0076): every pod that mounts the node-local objects volume selects the store node; aevolution.io/compute is only ever preferred, never required; upgrade-rehearsal flags never appear in a deployed workload; - the Pods the backup and barrier scripts generate (rendered with --print-manifest) keep the database-tool security contract: non-root 10001 with fsGroup 10001 so the 0440 Secret files are readable, no token, read-only root, no escalation. """ import os import shutil import subprocess import sys import tempfile import yaml ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) failures = [] def check(condition, message): if not condition: failures.append(message) def documents(path): with open(path, encoding='utf-8') as f: return [d for d in yaml.safe_load_all(f) if isinstance(d, dict)] manifests = {} for base, _, files in os.walk(os.path.join(ROOT, 'deploy')): for name in files: if name.endswith(('.yaml', '.yml', '.yaml.example')): path = os.path.join(base, name) manifests[os.path.relpath(path, ROOT).replace(os.sep, '/')] = documents(path) objects = [(p, d) for p, ds in manifests.items() for d in ds if 'kind' in d and d['kind'] != 'Kustomization'] # Secrets and service-account tokens. for path, doc in objects: check(doc['kind'] != 'Secret', f'{path}: Secret objects never belong in Git') if doc['kind'] == 'ServiceAccount': check(doc.get('automountServiceAccountToken') is False, f'{path}: ServiceAccount {doc["metadata"]["name"]} must not automount a token') # Platform overlays. platform = 'deploy/kubernetes/platform' base = {d['kind'] + '/' + d['metadata']['name']: d for p, d in objects if p.startswith(platform + '/base/')} for key in ('ResourceQuota/aevolution-quota', 'LimitRange/aevolution-defaults', 'NetworkPolicy/aevolution-default-deny', 'NetworkPolicy/aevolution-allow-dns', 'ServiceAccount/aevolution-migrator', 'ServiceAccount/aevolution-region', 'ServiceAccount/aevolution-backup'): check(key in base, f'{platform}/base: missing {key}') deny = base.get('NetworkPolicy/aevolution-default-deny', {}).get('spec', {}) check(deny.get('podSelector') == {} and sorted(deny.get('policyTypes', [])) == ['Egress', 'Ingress'] and not deny.get('ingress') and not deny.get('egress'), 'default-deny must select every pod and allow nothing') dns = base.get('NetworkPolicy/aevolution-allow-dns', {}).get('spec', {}) dns_ports = [p.get('port') for rule in dns.get('egress', []) for p in rule.get('ports', [])] check(dns.get('policyTypes') == ['Egress'] and set(dns_ports) == {53} and len(dns.get('egress', [])) == 1, 'allow-dns must allow only egress to port 53') quota = base.get('ResourceQuota/aevolution-quota', {}).get('spec', {}).get('hard', {}) check(quota.get('services.loadbalancers') == '0' and quota.get('services.nodeports') == '0', 'quota must forbid LoadBalancer and NodePort services') for env in ('dev', 'stage', 'world0'): k = f'{platform}/{env}/kustomization.yaml' ns_path = f'{platform}/{env}/namespace.yaml' check(k in manifests and ns_path in manifests, f'{platform}/{env}: overlay missing') if k not in manifests or ns_path not in manifests: continue kustomization = manifests[k][0] check(kustomization.get('namespace') == f'evolution-{env}' and '../base' in kustomization.get('resources', []), f'{k}: must set namespace evolution-{env} and include ../base') labels = manifests[ns_path][0]['metadata'].get('labels', {}) check(manifests[ns_path][0]['metadata']['name'] == f'evolution-{env}', f'{ns_path}: wrong name') check(labels.get('aevolution.io/environment') == env, f'{ns_path}: environment label') check(labels.get('pod-security.kubernetes.io/enforce') == 'restricted', f'{ns_path}: restricted pod security') check((labels.get('aevolution.io/protected') == 'true') == (env == 'world0'), f'{ns_path}: only world0 is protected') # World0 admission policy. policy = [d for p, d in objects if d['kind'] == 'ValidatingAdmissionPolicy' and d['metadata']['name'] == 'aevolution-world0-protection'] binding = [d for p, d in objects if d['kind'] == 'ValidatingAdmissionPolicyBinding' and d['metadata']['name'] == 'aevolution-world0-protection'] check(len(policy) == 1 and len(binding) == 1, 'world0 admission policy and binding must exist once') if policy and binding: spec = policy[0]['spec'] check(spec.get('failurePolicy') == 'Fail', 'world0 policy must fail closed') check(binding[0]['spec'].get('validationActions') == ['Deny'] and 'paramRef' not in binding[0]['spec'] and 'matchResources' not in binding[0]['spec'], 'world0 binding must Deny everywhere') covered = {(r, op) for rule in spec['matchConstraints']['resourceRules'] for r in rule['resources'] for op in rule['operations']} for resource in ('namespaces', 'persistentvolumes', 'persistentvolumeclaims', 'secrets', 'serviceaccounts', 'applications', 'appprojects'): check((resource, 'DELETE') in covered, f'world0 policy must cover DELETE {resource}') for resource in ('namespaces', 'persistentvolumes'): check((resource, 'UPDATE') in covered, f'world0 policy must cover UPDATE {resource}') text = yaml.safe_dump(spec) for needle in ("aevolution:world0-break-glass", "evolution-world0", "aevolution-world0", "aevolution-platform", "aevolution.io/protected", "Retain", "has(object.spec.claimRef.namespace)", "object.spec.claimRef.name == oldObject.spec.claimRef.name"): check(needle in text, f'world0 policy must reference {needle}') # Argo CD projects and applications. for path, doc in objects: name = doc['metadata'].get('name', '') if doc['kind'] == 'AppProject' and name == 'aevolution-platform': spec = doc['spec'] namespaces = sorted(d.get('namespace') for d in spec.get('destinations', [])) check(namespaces == ['aevolution-monitoring', 'evolution-dev', 'evolution-stage', 'evolution-world0'], f'{path}: the platform project targets exactly the AEvolution namespaces') cluster = sorted((w.get('group'), w.get('kind')) for w in spec.get('clusterResourceWhitelist', [])) check(cluster == [('', 'Namespace'), ('admissionregistration.k8s.io', 'ValidatingAdmissionPolicy'), ('admissionregistration.k8s.io', 'ValidatingAdmissionPolicyBinding'), ('storage.k8s.io', 'StorageClass')], f'{path}: the platform project may own only its four cluster kinds') kinds = {w.get('kind') for w in spec.get('namespaceResourceWhitelist', [])} check(not kinds & {'Secret', 'Role', 'RoleBinding', '*'}, f'{path}: platform whitelists a forbidden kind') elif doc['kind'] == 'AppProject' and name == 'aevolution-monitoring': spec = doc['spec'] check([d.get('namespace') for d in spec.get('destinations', [])] == ['aevolution-monitoring'] and spec.get('clusterResourceWhitelist') == [], f'{path}: monitoring targets only its namespace, with no cluster kinds') elif doc['kind'] == 'AppProject' and name.startswith('aevolution-'): env = name[len('aevolution-'):] spec = doc['spec'] namespaces = [d.get('namespace') for d in spec.get('destinations', [])] check(namespaces == [f'evolution-{env}'], f'{path}: {name} must target only evolution-{env}') check(spec.get('clusterResourceWhitelist') == [], f'{path}: {name} must allow no cluster kinds') kinds = {w.get('kind') for w in spec.get('namespaceResourceWhitelist', [])} check(not kinds & {'Secret', 'Namespace', 'ResourceQuota', 'LimitRange', 'Role', 'RoleBinding', '*'}, f'{path}: {name} whitelists a forbidden kind') if env == 'world0': windows = spec.get('syncWindows', []) check(any(w.get('kind') == 'deny' and w.get('manualSync') is True and w.get('duration') == '24h' and w.get('schedule') == '* * * * *' for w in windows), f'{path}: world0 must deny automated sync at all times') check('ServiceAccount' not in kinds, f'{path}: world0 service accounts are platform-managed') if doc['kind'] == 'Application': spec = doc['spec'] project = spec.get('project', '') destination = spec.get('destination', {}).get('namespace', '') automated = (spec.get('syncPolicy') or {}).get('automated') if name == 'aevolution-platform': # Self-heal only: never prune, so no Git change can delete durable objects. check(automated is not None and automated.get('prune') is False, f'{path}: the platform app self-heals and never prunes') check(spec.get('source', {}).get('path') == 'deploy/kubernetes/platform/cluster', f'{path}: the platform app syncs deploy/kubernetes/platform/cluster') else: check(automated is None, f'{path}: {name} must not auto-sync') check('resources-finalizer.argocd.argoproj.io' not in doc['metadata'].get('finalizers', []), f'{path}: {name} must not cascade-delete') if project.startswith('aevolution-') and project not in ('aevolution-platform', 'aevolution-monitoring'): check(destination == 'evolution-' + project[len('aevolution-'):], f'{path}: {name} leaves its project environment') check(destination != 'evolution-world0' or name in ('aevolution-world0', 'aevolution-platform'), f'{path}: only aevolution-world0 may target evolution-world0') # M8.3 storage (ADR-0070): durable volumes retain their data; database passwords are # only ever mounted as files, never placed in the environment. classes = {d['metadata']['name']: d for p, d in objects if d['kind'] == 'StorageClass'} retain = classes.get('aevolution-local-retain') check(retain is not None and retain.get('reclaimPolicy') == 'Retain' and retain.get('volumeBindingMode') == 'WaitForFirstConsumer' and 'storageclass.kubernetes.io/is-default-class' not in retain['metadata'].get('annotations', {}), 'aevolution-local-retain must be a non-default Retain class that binds on first consumer') for path, doc in objects: if doc['kind'] == 'PersistentVolumeClaim' and path.startswith(platform + '/'): check(doc['spec'].get('storageClassName') == 'aevolution-local-retain', f'{path}: platform volumes must use aevolution-local-retain') if doc['kind'] in ('Job', 'Deployment', 'StatefulSet', 'Pod'): pod = doc['spec'].get('template', {}).get('spec', doc['spec']) for container in pod.get('containers', []) + pod.get('initContainers', []): check(not any('secretRef' in source for source in container.get('envFrom', [])), f'{path}: never import a whole Secret into the environment') for variable in container.get('env', []): ref = variable.get('valueFrom', {}).get('secretKeyRef', {}) check(ref.get('key') not in ('PGPASSWORD', 'POSTGRES_PASSWORD', 'PGUSER', 'POSTGRES_USER'), f'{path}: {variable.get("name")} must not come from {ref.get("key")}') if pod.get('volumes') and 'store' in doc['metadata'].get('name', ''): check(doc['metadata'].get('namespace') != 'evolution-world0', f'{path}: store Jobs never target World0') # M8.4 region host (ADR-0071): one owner at a time, readiness means authority. for path, doc in objects: # Overlay patches (region-values.yaml) are partial objects; the base is complete. if (doc['kind'] == 'Deployment' and doc['metadata'].get('name') == 'evo-region' and not path.endswith('region-values.yaml')): spec = doc['spec'] pod = spec['template']['spec'] container = pod['containers'][0] check(spec.get('replicas') == 1 and spec.get('strategy', {}).get('type') == 'Recreate', f'{path}: the region host runs one replica with the Recreate strategy') check(container.get('readinessProbe', {}).get('httpGet', {}).get('path') == '/readyz' and container.get('livenessProbe', {}).get('httpGet', {}).get('path') == '/livez', f'{path}: readiness must be /readyz and liveness /livez') limits = container.get('resources', {}).get('limits', {}) check('cpu' not in limits and 'memory' in limits, f'{path}: memory limit required, no CPU limit (throttling delays renewals)') script = ' '.join(container.get('args', [])) drain = [int(x) for x in __import__('re').findall(r'--drain-ms (\d+)', script)] check(len(drain) == 1 and pod.get('terminationGracePeriodSeconds', 30) * 1000 > drain[0], f'{path}: the termination grace period must exceed --drain-ms') # M8.6 (ADR-0073): synthetic load is a drill-time override, never committed. for path, doc in objects: if path.startswith('deploy/kubernetes/region/') and doc['kind'] == 'Deployment': for container in doc['spec']['template']['spec'].get('containers', []): for variable in container.get('env', []): if variable.get('name') == 'REGION_INPUTS': check(variable.get('value') == '', f'{path}: synthetic inputs are never enabled in Git') for env in ('dev', 'stage'): overlay = f'deploy/kubernetes/region/{env}/' images = manifests.get(overlay + 'kustomization.yaml', [{}])[0].get('images', []) server = [i.get('digest', '') for i in images if i.get('name') == 'aevolution-server'] builds = [v.get('value') for d in manifests.get(overlay + 'region-values.yaml', []) for c in d['spec']['template']['spec']['containers'] for v in c.get('env', []) if v.get('name') == 'REGION_BUILD'] check(len(server) == 1 and len(builds) == 2 and all('sha256:' + b == server[0] for b in builds), f'{overlay}: the build identity must equal the server image digest') retained = [v.get('value', '') for d in manifests.get(overlay + 'region-values.yaml', []) for c in d['spec']['template']['spec']['containers'] for v in c.get('env', []) if v.get('name') == 'REGION_RETAINED'] for value in retained: entries = value.split() check(all(__import__('re').fullmatch(r'[0-9a-f]{64}', e) for e in entries) and not set(entries) & set(builds), f'{overlay}: REGION_RETAINED must list other builds as 64 hex digits') # M8.5 monitoring (ADR-0072): third-party images are pinned by digest, and Prometheus # needs no Kubernetes API access. for path, doc in objects: if path.startswith('deploy/kubernetes/monitoring/') and doc['kind'] == 'Deployment': pod = doc['spec']['template']['spec'] for container in pod['containers']: check('@sha256:' in container.get('image', ''), f'{path}: {container["name"]} must be pinned by digest') check(pod.get('automountServiceAccountToken') is False, f'{path}: monitoring pods mount no token') if path.startswith('deploy/') and doc['kind'] in ('ClusterRole', 'ClusterRoleBinding', 'Role', 'RoleBinding'): check(False, f'{path}: AEvolution grants no Kubernetes RBAC') # M8.8 World0 launch gate (ADR-0075). This host has no independent durable failure # domain, so an acknowledged death could be undone by host loss: World0 must not run # or acknowledge. Lifting this rule is itself the launch decision and needs its own # accepted ADR with host-loss RPO-0 evidence. workload_kinds = {'Deployment', 'StatefulSet', 'DaemonSet', 'Job', 'CronJob', 'Pod', 'ReplicaSet'} for path, doc in objects: in_world0 = doc['metadata'].get('namespace') == 'evolution-world0' or '/world0/' in path check(not (in_world0 and doc['kind'] in workload_kinds), f'{path}: World0 launch is blocked (ADR-0075); no {doc["kind"]} may target World0') if doc['kind'] == 'Application' and doc['metadata'].get('name') == 'aevolution-world0': check('REPLACE' in doc['spec'].get('source', {}).get('path', ''), f'{path}: the World0 Application must keep an unresolved path while launch is blocked') for component in ('region', 'stores'): check(not os.path.exists(os.path.join(ROOT, 'deploy', 'kubernetes', component, 'world0')), f'deploy/kubernetes/{component}/world0: World0 launch is blocked (ADR-0075)') # Promotion: World0 may only ever pin a digest that stage already runs. for directory, _, files in os.walk(os.path.join(ROOT, 'deploy', 'kubernetes')): rel = os.path.relpath(directory, ROOT).replace(os.sep, '/') if rel.endswith('/world0') and 'kustomization.yaml' in files: stage = rel[:-len('world0')] + 'stage/kustomization.yaml' staged = {(i.get('name'), i.get('digest')) for i in manifests.get(stage, [{}])[0].get('images', [])} for image in manifests[rel + '/kustomization.yaml'][0].get('images', []): check((image.get('name'), image.get('digest')) in staged, f'{rel}: {image.get("name")} must be promoted from {stage}, not pinned independently') # M8.9 scheduling (ADR-0076). Node-local volumes pin their pods: added CPU does not # make the objects volume portable. Compute is a preference so one node still works. for path, doc in objects: if doc['kind'] not in workload_kinds or path.endswith('region-values.yaml'): continue spec = doc.get('spec', {}) pod = spec.get('jobTemplate', {}).get('spec', {}).get('template', {}).get('spec') \ or spec.get('template', {}).get('spec') or spec claims = [v.get('persistentVolumeClaim', {}).get('claimName') for v in pod.get('volumes', [])] if 'aevolution-objects' in claims: check(pod.get('nodeSelector', {}).get('aevolution.io/stores') == 'true', f'{path}: a pod mounting aevolution-objects must select aevolution.io/stores') required = (pod.get('affinity', {}).get('nodeAffinity', {}) .get('requiredDuringSchedulingIgnoredDuringExecution', {})) check('aevolution.io/compute' not in yaml.safe_dump(required) and 'aevolution.io/compute' not in pod.get('nodeSelector', {}), f'{path}: aevolution.io/compute may only be preferred, never required') for container in pod.get('containers', []): text = ' '.join(container.get('args', []) + container.get('command', [])) check('--accelerate' not in text and '--synthetic-until' not in text, f'{path}: upgrade-rehearsal flags never run in a deployed workload') # The platform bundle: every rendered object is protected from prune and deletion. bundle = manifests.get(platform + '/cluster/kustomization.yaml', [{}])[0] check(bundle.get('commonAnnotations', {}).get('argocd.argoproj.io/sync-options') == 'Prune=false,Delete=false', 'the platform bundle must mark every object Prune=false,Delete=false') check(sorted(bundle.get('resources', [])) == ['../admission', '../dev', '../monitoring', '../stage', '../storage', '../world0'], 'the platform bundle must include admission, storage, the environments and monitoring') # Paths a real (non-example) Application syncs must be complete: no REPLACE markers # anywhere they render, following kustomization resources. def kustomize_files(directory, seen): directory = os.path.normpath(directory) if directory in seen: return [] seen.add(directory) found = [] k = os.path.join(directory, 'kustomization.yaml') if not os.path.isfile(k): return found found.append(k) spec = documents(k)[0] for entry in spec.get('resources', []) + spec.get('components', []): target = os.path.normpath(os.path.join(directory, entry)) if os.path.isdir(target): found += kustomize_files(target, seen) else: found.append(target) return found for path, doc in objects: # Examples count too once their path is real (the live dev app is only an example # here); an example whose path is itself still a marker is not yet deployable. source_path = doc.get('spec', {}).get('source', {}).get('path', '') if doc['kind'] == 'Application' and 'REPLACE' not in source_path: source = os.path.join(ROOT, source_path) for f in kustomize_files(source, set()): with open(f, encoding='utf-8') as handle: check('REPLACE' not in handle.read(), f'{os.path.relpath(f, ROOT)}: unresolved REPLACE marker in a synced path') # The live Argo path. for path, doc in objects: if path.startswith('deploy/kubernetes/dev/'): check(doc['kind'] not in {'Namespace', 'Secret', 'ResourceQuota', 'LimitRange', 'Role', 'RoleBinding', 'ClusterRole', 'ClusterRoleBinding'}, f'{path}: {doc["kind"]} must not live in the Argo workload path') # The cleanup, drill and backup tools refuse World0 before kubectl runs (no kubectl on PATH here). with tempfile.TemporaryDirectory() as empty: env = dict(os.environ, PATH=empty + os.pathsep + os.path.dirname(sys.executable)) for tool in ('tools/ops/cleanup-environment.sh', 'tools/ops/region-drill.sh', 'tools/ops/fault-drill.sh', 'tools/backup/cluster-backup.sh', 'tools/ops/scheduling-check.sh', 'tools/ops/upgrade-barrier.sh'): for bad in ('world0', 'evolution-world0', 'World0', 'prod', ''): result = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, tool), bad], capture_output=True, text=True, env=env) check(result.returncode == 2 and ('refused' in result.stderr or 'usage' in result.stderr) and 'kubectl' not in result.stderr, f'{tool} must refuse {bad!r} before kubectl (rc={result.returncode})') # Script-generated Pods (review M8 P2): rendered offline and checked like manifests. retained_hex = next((v.get('value', '').split()[0] for d in manifests.get( 'deploy/kubernetes/region/dev/region-values.yaml', []) for c in d['spec']['template']['spec']['containers'] for v in c.get('env', []) if v.get('name') == 'REGION_RETAINED' and v.get('value')), None) generators = [('tools/backup/cluster-backup.sh', ['dev', '--print-manifest'], 'backup')] if retained_hex: generators.append(('tools/ops/upgrade-barrier.sh', ['dev', '--retire', retained_hex, '--print-manifest'], 'migrate')) else: check(False, 'dev REGION_RETAINED is empty: the barrier Pod cannot be rendered') egress = next((d for p, d in objects if d['kind'] == 'NetworkPolicy' and d['metadata']['name'] == 'aevolution-database-egress'), {}) admitted = [v for e in egress.get('spec', {}).get('podSelector', {}).get('matchExpressions', []) for v in e.get('values', [])] for tool, args, component in generators: rendered = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, tool)] + args, capture_output=True, text=True) check(rendered.returncode == 0, f'{tool} --print-manifest failed: {rendered.stderr.strip()}') if rendered.returncode != 0: continue pod = yaml.safe_load(rendered.stdout) spec = pod['spec'] context = spec.get('securityContext', {}) check(context.get('runAsNonRoot') is True and context.get('runAsUser') == 10001 and context.get('runAsGroup') == 10001 and context.get('fsGroup') == 10001, f'{tool}: the generated Pod must run as 10001 with fsGroup 10001 (Secret access)') check(spec.get('automountServiceAccountToken') is False, f'{tool}: the generated Pod mounts no token') for container in spec['containers']: sc = container.get('securityContext', {}) check(sc.get('readOnlyRootFilesystem') is True and sc.get('allowPrivilegeEscalation') is False, f'{tool}: read-only root and no privilege escalation') for volume in spec.get('volumes', []): if 'secret' in volume: check(volume['secret'].get('defaultMode') == 0o440, f'{tool}: Secret files stay 0440 (never world-readable)') labels = pod['metadata'].get('labels', {}) check(labels.get('app.kubernetes.io/component') == component and component in admitted, f'{tool}: component {component} must be admitted by the database egress policy') check(pod['metadata'].get('namespace') == 'evolution-dev', f'{tool}: renders into its environment') for message in failures: print('FAIL', message) print(f'deploy policy: {len(objects)} objects in {len(manifests)} files, {len(failures)} failures') sys.exit(1 if failures else 0)