Deploy evo-region per environment (ADR-0071): one replica with the Recreate strategy (a rolling update would deadlock on the lease), /readyz readiness, /livez liveness, no CPU limit and a drain inside the grace period, with the image digest doubling as the build identity. A new idempotent evo-world-clone provision subcommand creates genesis from development_opening (byte-identical to the qualification fixture) as a Sync hook after the migration. Dev and stage paths include their region; a dev AppProject allows Deployment and Service; region-drill.sh times rollout and forced-deletion recovery; the policy check enforces the host's invariants. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
288 lines
17 KiB
Python
288 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""M8.2 offline deployment-policy check (ADR-0069). Needs Python 3 and PyYAML only.
|
|
|
|
Parses every manifest under deploy/ (including .yaml.example files) and fails if an
|
|
environment-isolation or World0-protection invariant is broken:
|
|
- no Secret objects in Git;
|
|
- every ServiceAccount disables token automounting;
|
|
- each platform overlay: labelled restricted namespace, quota, limit range,
|
|
default-deny and DNS-only policies, three service accounts; World0 alone is
|
|
labelled protected;
|
|
- the World0 admission policy denies what ADR-0069 lists, fails closed and is
|
|
bound with Deny;
|
|
- every AEvolution AppProject targets exactly its own namespace with no cluster
|
|
kinds; World0 blocks automated sync; no Application auto-syncs or carries the
|
|
cascade-delete finalizer, and each stays in its own environment;
|
|
- the live Argo path (deploy/kubernetes/dev) holds no Namespace, Secret, quota or
|
|
RBAC objects;
|
|
- the cleanup tool refuses World0 before invoking kubectl.
|
|
"""
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
import yaml
|
|
|
|
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
|
|
failures = []
|
|
|
|
|
|
def check(condition, message):
|
|
if not condition:
|
|
failures.append(message)
|
|
|
|
|
|
def documents(path):
|
|
with open(path, encoding='utf-8') as f:
|
|
return [d for d in yaml.safe_load_all(f) if isinstance(d, dict)]
|
|
|
|
|
|
manifests = {}
|
|
for base, _, files in os.walk(os.path.join(ROOT, 'deploy')):
|
|
for name in files:
|
|
if name.endswith(('.yaml', '.yml', '.yaml.example')):
|
|
path = os.path.join(base, name)
|
|
manifests[os.path.relpath(path, ROOT).replace(os.sep, '/')] = documents(path)
|
|
objects = [(p, d) for p, ds in manifests.items() for d in ds
|
|
if 'kind' in d and d['kind'] != 'Kustomization']
|
|
|
|
# Secrets and service-account tokens.
|
|
for path, doc in objects:
|
|
check(doc['kind'] != 'Secret', f'{path}: Secret objects never belong in Git')
|
|
if doc['kind'] == 'ServiceAccount':
|
|
check(doc.get('automountServiceAccountToken') is False,
|
|
f'{path}: ServiceAccount {doc["metadata"]["name"]} must not automount a token')
|
|
|
|
# Platform overlays.
|
|
platform = 'deploy/kubernetes/platform'
|
|
base = {d['kind'] + '/' + d['metadata']['name']: d
|
|
for p, d in objects if p.startswith(platform + '/base/')}
|
|
for key in ('ResourceQuota/aevolution-quota', 'LimitRange/aevolution-defaults',
|
|
'NetworkPolicy/aevolution-default-deny', 'NetworkPolicy/aevolution-allow-dns',
|
|
'ServiceAccount/aevolution-migrator', 'ServiceAccount/aevolution-region',
|
|
'ServiceAccount/aevolution-backup'):
|
|
check(key in base, f'{platform}/base: missing {key}')
|
|
deny = base.get('NetworkPolicy/aevolution-default-deny', {}).get('spec', {})
|
|
check(deny.get('podSelector') == {} and sorted(deny.get('policyTypes', [])) == ['Egress', 'Ingress']
|
|
and not deny.get('ingress') and not deny.get('egress'),
|
|
'default-deny must select every pod and allow nothing')
|
|
dns = base.get('NetworkPolicy/aevolution-allow-dns', {}).get('spec', {})
|
|
dns_ports = [p.get('port') for rule in dns.get('egress', []) for p in rule.get('ports', [])]
|
|
check(dns.get('policyTypes') == ['Egress'] and set(dns_ports) == {53} and len(dns.get('egress', [])) == 1,
|
|
'allow-dns must allow only egress to port 53')
|
|
quota = base.get('ResourceQuota/aevolution-quota', {}).get('spec', {}).get('hard', {})
|
|
check(quota.get('services.loadbalancers') == '0' and quota.get('services.nodeports') == '0',
|
|
'quota must forbid LoadBalancer and NodePort services')
|
|
for env in ('dev', 'stage', 'world0'):
|
|
k = f'{platform}/{env}/kustomization.yaml'
|
|
ns_path = f'{platform}/{env}/namespace.yaml'
|
|
check(k in manifests and ns_path in manifests, f'{platform}/{env}: overlay missing')
|
|
if k not in manifests or ns_path not in manifests:
|
|
continue
|
|
kustomization = manifests[k][0]
|
|
check(kustomization.get('namespace') == f'evolution-{env}' and '../base' in kustomization.get('resources', []),
|
|
f'{k}: must set namespace evolution-{env} and include ../base')
|
|
labels = manifests[ns_path][0]['metadata'].get('labels', {})
|
|
check(manifests[ns_path][0]['metadata']['name'] == f'evolution-{env}', f'{ns_path}: wrong name')
|
|
check(labels.get('aevolution.io/environment') == env, f'{ns_path}: environment label')
|
|
check(labels.get('pod-security.kubernetes.io/enforce') == 'restricted', f'{ns_path}: restricted pod security')
|
|
check((labels.get('aevolution.io/protected') == 'true') == (env == 'world0'),
|
|
f'{ns_path}: only world0 is protected')
|
|
|
|
# World0 admission policy.
|
|
policy = [d for p, d in objects if d['kind'] == 'ValidatingAdmissionPolicy'
|
|
and d['metadata']['name'] == 'aevolution-world0-protection']
|
|
binding = [d for p, d in objects if d['kind'] == 'ValidatingAdmissionPolicyBinding'
|
|
and d['metadata']['name'] == 'aevolution-world0-protection']
|
|
check(len(policy) == 1 and len(binding) == 1, 'world0 admission policy and binding must exist once')
|
|
if policy and binding:
|
|
spec = policy[0]['spec']
|
|
check(spec.get('failurePolicy') == 'Fail', 'world0 policy must fail closed')
|
|
check(binding[0]['spec'].get('validationActions') == ['Deny'] and 'paramRef' not in binding[0]['spec']
|
|
and 'matchResources' not in binding[0]['spec'], 'world0 binding must Deny everywhere')
|
|
covered = {(r, op) for rule in spec['matchConstraints']['resourceRules']
|
|
for r in rule['resources'] for op in rule['operations']}
|
|
for resource in ('namespaces', 'persistentvolumes', 'persistentvolumeclaims', 'secrets',
|
|
'serviceaccounts', 'applications', 'appprojects'):
|
|
check((resource, 'DELETE') in covered, f'world0 policy must cover DELETE {resource}')
|
|
for resource in ('namespaces', 'persistentvolumes'):
|
|
check((resource, 'UPDATE') in covered, f'world0 policy must cover UPDATE {resource}')
|
|
text = yaml.safe_dump(spec)
|
|
for needle in ("aevolution:world0-break-glass", "evolution-world0", "aevolution-world0",
|
|
"aevolution-platform",
|
|
"aevolution.io/protected", "Retain"):
|
|
check(needle in text, f'world0 policy must reference {needle}')
|
|
|
|
# Argo CD projects and applications.
|
|
for path, doc in objects:
|
|
name = doc['metadata'].get('name', '')
|
|
if doc['kind'] == 'AppProject' and name == 'aevolution-platform':
|
|
spec = doc['spec']
|
|
namespaces = sorted(d.get('namespace') for d in spec.get('destinations', []))
|
|
check(namespaces == ['evolution-dev', 'evolution-stage', 'evolution-world0'],
|
|
f'{path}: the platform project targets exactly the three AEvolution namespaces')
|
|
cluster = sorted((w.get('group'), w.get('kind')) for w in spec.get('clusterResourceWhitelist', []))
|
|
check(cluster == [('', 'Namespace'),
|
|
('admissionregistration.k8s.io', 'ValidatingAdmissionPolicy'),
|
|
('admissionregistration.k8s.io', 'ValidatingAdmissionPolicyBinding'),
|
|
('storage.k8s.io', 'StorageClass')],
|
|
f'{path}: the platform project may own only its four cluster kinds')
|
|
kinds = {w.get('kind') for w in spec.get('namespaceResourceWhitelist', [])}
|
|
check(not kinds & {'Secret', 'Role', 'RoleBinding', '*'}, f'{path}: platform whitelists a forbidden kind')
|
|
elif doc['kind'] == 'AppProject' and name.startswith('aevolution-'):
|
|
env = name[len('aevolution-'):]
|
|
spec = doc['spec']
|
|
namespaces = [d.get('namespace') for d in spec.get('destinations', [])]
|
|
check(namespaces == [f'evolution-{env}'], f'{path}: {name} must target only evolution-{env}')
|
|
check(spec.get('clusterResourceWhitelist') == [], f'{path}: {name} must allow no cluster kinds')
|
|
kinds = {w.get('kind') for w in spec.get('namespaceResourceWhitelist', [])}
|
|
check(not kinds & {'Secret', 'Namespace', 'ResourceQuota', 'LimitRange', 'Role', 'RoleBinding', '*'},
|
|
f'{path}: {name} whitelists a forbidden kind')
|
|
if env == 'world0':
|
|
windows = spec.get('syncWindows', [])
|
|
check(any(w.get('kind') == 'deny' and w.get('manualSync') is True and w.get('duration') == '24h'
|
|
and w.get('schedule') == '* * * * *' for w in windows),
|
|
f'{path}: world0 must deny automated sync at all times')
|
|
check('ServiceAccount' not in kinds, f'{path}: world0 service accounts are platform-managed')
|
|
if doc['kind'] == 'Application':
|
|
spec = doc['spec']
|
|
project = spec.get('project', '')
|
|
destination = spec.get('destination', {}).get('namespace', '')
|
|
automated = (spec.get('syncPolicy') or {}).get('automated')
|
|
if name == 'aevolution-platform':
|
|
# Self-heal only: never prune, so no Git change can delete durable objects.
|
|
check(automated is not None and automated.get('prune') is False,
|
|
f'{path}: the platform app self-heals and never prunes')
|
|
check(spec.get('source', {}).get('path') == 'deploy/kubernetes/platform/cluster',
|
|
f'{path}: the platform app syncs deploy/kubernetes/platform/cluster')
|
|
else:
|
|
check(automated is None, f'{path}: {name} must not auto-sync')
|
|
check('resources-finalizer.argocd.argoproj.io' not in doc['metadata'].get('finalizers', []),
|
|
f'{path}: {name} must not cascade-delete')
|
|
if project.startswith('aevolution-') and project != 'aevolution-platform':
|
|
check(destination == 'evolution-' + project[len('aevolution-'):],
|
|
f'{path}: {name} leaves its project environment')
|
|
check(destination != 'evolution-world0' or name in ('aevolution-world0', 'aevolution-platform'),
|
|
f'{path}: only aevolution-world0 may target evolution-world0')
|
|
|
|
# M8.3 storage (ADR-0070): durable volumes retain their data; database passwords are
|
|
# only ever mounted as files, never placed in the environment.
|
|
classes = {d['metadata']['name']: d for p, d in objects if d['kind'] == 'StorageClass'}
|
|
retain = classes.get('aevolution-local-retain')
|
|
check(retain is not None and retain.get('reclaimPolicy') == 'Retain'
|
|
and retain.get('volumeBindingMode') == 'WaitForFirstConsumer'
|
|
and 'storageclass.kubernetes.io/is-default-class' not in retain['metadata'].get('annotations', {}),
|
|
'aevolution-local-retain must be a non-default Retain class that binds on first consumer')
|
|
for path, doc in objects:
|
|
if doc['kind'] == 'PersistentVolumeClaim' and path.startswith(platform + '/'):
|
|
check(doc['spec'].get('storageClassName') == 'aevolution-local-retain',
|
|
f'{path}: platform volumes must use aevolution-local-retain')
|
|
if doc['kind'] in ('Job', 'Deployment', 'StatefulSet', 'Pod'):
|
|
pod = doc['spec'].get('template', {}).get('spec', doc['spec'])
|
|
for container in pod.get('containers', []) + pod.get('initContainers', []):
|
|
check(not any('secretRef' in source for source in container.get('envFrom', [])),
|
|
f'{path}: never import a whole Secret into the environment')
|
|
for variable in container.get('env', []):
|
|
ref = variable.get('valueFrom', {}).get('secretKeyRef', {})
|
|
check(ref.get('key') not in ('PGPASSWORD', 'POSTGRES_PASSWORD', 'PGUSER', 'POSTGRES_USER'),
|
|
f'{path}: {variable.get("name")} must not come from {ref.get("key")}')
|
|
if pod.get('volumes') and 'store' in doc['metadata'].get('name', ''):
|
|
check(doc['metadata'].get('namespace') != 'evolution-world0',
|
|
f'{path}: store Jobs never target World0')
|
|
|
|
# M8.4 region host (ADR-0071): one owner at a time, readiness means authority.
|
|
for path, doc in objects:
|
|
# Overlay patches (region-values.yaml) are partial objects; the base is complete.
|
|
if (doc['kind'] == 'Deployment' and doc['metadata'].get('name') == 'evo-region'
|
|
and not path.endswith('region-values.yaml')):
|
|
spec = doc['spec']
|
|
pod = spec['template']['spec']
|
|
container = pod['containers'][0]
|
|
check(spec.get('replicas') == 1 and spec.get('strategy', {}).get('type') == 'Recreate',
|
|
f'{path}: the region host runs one replica with the Recreate strategy')
|
|
check(container.get('readinessProbe', {}).get('httpGet', {}).get('path') == '/readyz'
|
|
and container.get('livenessProbe', {}).get('httpGet', {}).get('path') == '/livez',
|
|
f'{path}: readiness must be /readyz and liveness /livez')
|
|
limits = container.get('resources', {}).get('limits', {})
|
|
check('cpu' not in limits and 'memory' in limits,
|
|
f'{path}: memory limit required, no CPU limit (throttling delays renewals)')
|
|
script = ' '.join(container.get('args', []))
|
|
drain = [int(x) for x in __import__('re').findall(r'--drain-ms (\d+)', script)]
|
|
check(len(drain) == 1 and pod.get('terminationGracePeriodSeconds', 30) * 1000 > drain[0],
|
|
f'{path}: the termination grace period must exceed --drain-ms')
|
|
for env in ('dev', 'stage'):
|
|
overlay = f'deploy/kubernetes/region/{env}/'
|
|
images = manifests.get(overlay + 'kustomization.yaml', [{}])[0].get('images', [])
|
|
server = [i.get('digest', '') for i in images if i.get('name') == 'aevolution-server']
|
|
builds = [v.get('value') for d in manifests.get(overlay + 'region-values.yaml', [])
|
|
for c in d['spec']['template']['spec']['containers'] for v in c.get('env', [])
|
|
if v.get('name') == 'REGION_BUILD']
|
|
check(len(server) == 1 and len(builds) == 2 and all('sha256:' + b == server[0] for b in builds),
|
|
f'{overlay}: the build identity must equal the server image digest')
|
|
|
|
# The platform bundle: every rendered object is protected from prune and deletion.
|
|
bundle = manifests.get(platform + '/cluster/kustomization.yaml', [{}])[0]
|
|
check(bundle.get('commonAnnotations', {}).get('argocd.argoproj.io/sync-options') == 'Prune=false,Delete=false',
|
|
'the platform bundle must mark every object Prune=false,Delete=false')
|
|
check(sorted(bundle.get('resources', [])) == ['../admission', '../dev', '../stage', '../storage', '../world0'],
|
|
'the platform bundle must include admission, storage and the three environments')
|
|
|
|
|
|
# Paths a real (non-example) Application syncs must be complete: no REPLACE markers
|
|
# anywhere they render, following kustomization resources.
|
|
def kustomize_files(directory, seen):
|
|
directory = os.path.normpath(directory)
|
|
if directory in seen:
|
|
return []
|
|
seen.add(directory)
|
|
found = []
|
|
k = os.path.join(directory, 'kustomization.yaml')
|
|
if not os.path.isfile(k):
|
|
return found
|
|
found.append(k)
|
|
spec = documents(k)[0]
|
|
for entry in spec.get('resources', []) + spec.get('components', []):
|
|
target = os.path.normpath(os.path.join(directory, entry))
|
|
if os.path.isdir(target):
|
|
found += kustomize_files(target, seen)
|
|
else:
|
|
found.append(target)
|
|
return found
|
|
|
|
|
|
for path, doc in objects:
|
|
# Examples count too once their path is real (the live dev app is only an example
|
|
# here); an example whose path is itself still a marker is not yet deployable.
|
|
source_path = doc.get('spec', {}).get('source', {}).get('path', '')
|
|
if doc['kind'] == 'Application' and 'REPLACE' not in source_path:
|
|
source = os.path.join(ROOT, source_path)
|
|
for f in kustomize_files(source, set()):
|
|
with open(f, encoding='utf-8') as handle:
|
|
check('REPLACE' not in handle.read(),
|
|
f'{os.path.relpath(f, ROOT)}: unresolved REPLACE marker in a synced path')
|
|
|
|
# The live Argo path.
|
|
for path, doc in objects:
|
|
if path.startswith('deploy/kubernetes/dev/'):
|
|
check(doc['kind'] not in {'Namespace', 'Secret', 'ResourceQuota', 'LimitRange', 'Role',
|
|
'RoleBinding', 'ClusterRole', 'ClusterRoleBinding'},
|
|
f'{path}: {doc["kind"]} must not live in the Argo workload path')
|
|
|
|
# The cleanup tool refuses World0 before kubectl runs (no kubectl on PATH here).
|
|
with tempfile.TemporaryDirectory() as empty:
|
|
env = dict(os.environ, PATH=empty + os.pathsep + os.path.dirname(sys.executable))
|
|
for tool in ('tools/ops/cleanup-environment.sh', 'tools/ops/region-drill.sh'):
|
|
for bad in ('world0', 'evolution-world0', 'World0', 'prod', ''):
|
|
result = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, tool), bad],
|
|
capture_output=True, text=True, env=env)
|
|
check(result.returncode == 2 and ('refused' in result.stderr or 'usage' in result.stderr)
|
|
and 'kubectl' not in result.stderr,
|
|
f'{tool} must refuse {bad!r} before kubectl (rc={result.returncode})')
|
|
|
|
for message in failures:
|
|
print('FAIL', message)
|
|
print(f'deploy policy: {len(objects)} objects in {len(manifests)} files, {len(failures)} failures')
|
|
sys.exit(1 if failures else 0)
|