Files
aevolution/tools/ci/deploy-policy-selftest.py
T
dingoandClaude Opus 5.5 47b84bce74
Foundation / checks (client) (push) Successful in 4s
Foundation / checks (checks) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 5s
Foundation / checks (network) (push) Successful in 5s
Foundation / checks (postgres) (push) Successful in 5s
Foundation / checks (tsan) (push) Successful in 4s
Foundation / checks (deploy) (push) Successful in 3m31s
Foundation / postgres-tools (push) Successful in 5s
Foundation / server-image (push) Successful in 5s
Foundation / image (push) Successful in 5s
Refuse session windows beyond the server's 4-hour maximum in deploy policy
A window ending 5.2 h after start (b2f224d) put evo-region into a
crashloop (error m10-window-too-long); the server only sees it at
startup. The deploy policy now requires REGION_SESSION_WINDOW to name its
UTC activation and end within 4 h of it, with two selftest cases (one is
the b2f224d value). DEPLOYMENT.md documents the follow-on quarantine:
syncs recorded as Deployment-only run no hooks, so a promoted build is
never retained; the provision hook is re-run from the rendered manifest.
VALIDATION records the incident and the recovery.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 07:41:06 -04:00

309 lines
18 KiB
Python

#!/usr/bin/env python3
"""M9.8f fail-closed tests for deploy-policy-check.py (ADR-0087, ADR-0088). Needs
Python 3, PyYAML and the pinned kustomize (tools/ci/prepare-kustomize.sh). Copies the
manifests and the tools the check runs into a temporary tree, applies one unsafe change
at a time and requires the check to fail with the expected message; the unmodified copy
must pass. Includes inline strategic-merge and JSON patches and namespace changes, which
only the rendered-output checks can see.
"""
import os
import re
import shutil
import subprocess
import sys
import tempfile
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
ROUTE = 'deploy/kubernetes/region/dev-session-route/'
DEV = 'deploy/kubernetes/region/dev/kustomization.yaml'
REAL = [(ROUTE + 'service.yaml', '172.10.1.202', '10.20.30.40'),
(ROUTE + 'service.yaml', '172.10.1.0/24', '10.20.0.0/24'),
(ROUTE + 'networkpolicy.yaml', '172.10.1.0/24', '10.20.0.0/24')]
def edit(tree, path, old, new, count=1):
full = os.path.join(tree, path)
with open(full, encoding='utf-8') as f:
text = f.read()
if old not in text:
raise SystemExit(f'selftest setup: {old!r} not in {path}')
with open(full, 'w', encoding='utf-8') as f:
f.write(text.replace(old, new, count))
def append(tree, path, text):
with open(os.path.join(tree, path), 'a', encoding='utf-8') as f:
f.write(text)
def write(tree, path, text):
full = os.path.join(tree, path)
os.makedirs(os.path.dirname(full), exist_ok=True)
with open(full, 'w', encoding='utf-8') as f:
f.write(text)
def inline_patch(tree, path, patch, target=None):
"""Appends an inline patch (strategic merge, or JSON 6902 with a target)."""
import yaml
full = os.path.join(tree, path)
with open(full, encoding='utf-8') as f:
data = yaml.safe_load(f)
entry = {'patch': patch}
if target:
entry['target'] = target
data.setdefault('patches', []).append(entry)
with open(full, 'w', encoding='utf-8') as f:
yaml.safe_dump(data, f, sort_keys=False)
def env_patch(name, value):
return ('apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: evo-region\nspec:\n template:\n'
' spec:\n containers:\n - name: evo-server\n env:\n'
f' - {{name: {name}, value: "{value}"}}\n')
def add_resource(tree, kustomization, name, text):
"""Adds a manifest file to a kustomization's resources."""
import yaml
directory = os.path.dirname(kustomization)
write(tree, os.path.join(directory, name), text)
full = os.path.join(tree, kustomization)
with open(full, encoding='utf-8') as f:
data = yaml.safe_load(f)
data.setdefault('resources', []).append(name)
with open(full, 'w', encoding='utf-8') as f:
yaml.safe_dump(data, f, sort_keys=False)
def region_policy(selector, rule):
return ('apiVersion: networking.k8s.io/v1\nkind: NetworkPolicy\nmetadata:\n name: widened\nspec:\n'
f' podSelector: {selector}\n policyTypes: [Ingress]\n ingress:\n - {rule}\n')
def route_on(tree):
enable(tree)
resolve(tree)
quota_one(tree)
def enable(tree):
append(tree, DEV, 'components:\n - ../dev-session-route\n')
def resolve(tree):
for path, old, new in REAL:
edit(tree, path, old, new)
def quota_one(tree):
write(tree, 'deploy/kubernetes/platform/dev/quota-session.yaml',
'apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: aevolution-quota\n'
'spec:\n hard:\n services.loadbalancers: "1"\n services.nodeports: "0"\n')
CASES = [
('non-RFC 1918 VIP without approval', lambda t: edit(t, ROUTE + 'service.yaml', '172.10.1.202',
'172.11.1.210'), 'internal address'),
# Review M9.8f P1: inline/JSON patches and namespace changes, seen only when rendered.
('session window longer than the server maximum', lambda t: inline_patch(
t, 'deploy/kubernetes/dev/kustomization.yaml',
env_patch('REGION_SESSION_WINDOW', 'dev-20261008T111342-32432:1791477222')),
'end within 4 h of it'),
('session window without an activation time', lambda t: inline_patch(
t, 'deploy/kubernetes/dev/kustomization.yaml',
env_patch('REGION_SESSION_WINDOW', 'dev-manual:1791477222')),
'end within 4 h of it'),
('inline patch: synthetic inputs on', lambda t: inline_patch(
t, 'deploy/kubernetes/dev/kustomization.yaml', env_patch('REGION_INPUTS', 'development-v1')),
'sessions must be dev only with the route'),
('JSON patch: session port on the ops Service', lambda t: inline_patch(
t, DEV, '- op: add\n path: /spec/ports/-\n value: {name: session, port: 7400, targetPort: 7400, protocol: TCP}\n',
{'kind': 'Service', 'name': 'evo-region-ops'}), 'only the route may be a LoadBalancer'),
('JSON patch: LoadBalancer ops Service', lambda t: inline_patch(
t, 'deploy/kubernetes/stage/kustomization.yaml', '- op: replace\n path: /spec/type\n value: LoadBalancer\n',
{'kind': 'Service', 'name': 'evo-region-ops'}), 'only the route may be a LoadBalancer'),
('JSON patch: container port 7400', lambda t: inline_patch(
t, DEV, '- op: add\n path: /spec/template/spec/containers/0/ports/-\n'
' value: {name: session, containerPort: 7400, protocol: TCP}\n',
{'kind': 'Deployment', 'name': 'evo-region'}), 'container ports must be'),
('JSON patch: second listener argument', lambda t: inline_patch(
t, DEV, '- op: add\n path: /spec/template/spec/containers/0/args/-\n value: --m10-listen 0.0.0.0:7401\n',
{'kind': 'Deployment', 'name': 'evo-region'}), 'one M10 listener and absolute session window'),
('namespace transformation', lambda t: edit(t, DEV, 'namespace: evolution-dev', 'namespace: evolution-stage'),
'expected evolution-dev'),
('inline quota patch in stage', lambda t: inline_patch(
t, 'deploy/kubernetes/platform/stage/kustomization.yaml',
'apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: aevolution-quota\nspec:\n hard:\n'
' services.loadbalancers: "1"\n'), 'LoadBalancers only in evolution-dev'),
# Review 8a1e5ef P2: any effective ingress rule that can reach the region's session port.
('reviewer reproduction (default-deny admits all ingress)', lambda t: inline_patch(
t, 'deploy/kubernetes/platform/dev/kustomization.yaml',
'apiVersion: networking.k8s.io/v1\nkind: NetworkPolicy\nmetadata:\n name: aevolution-default-deny\n'
'spec:\n ingress:\n - {}\n'), 'default-deny must select every pod and allow nothing'),
('region policy without ports', lambda t: add_resource(
t, DEV, 'widened.yaml', region_policy('{matchLabels: {app.kubernetes.io/name: evo-region}}',
'{from: [{namespaceSelector: {}}]}')), 'admits the region session port'),
('region policy with empty ports', lambda t: add_resource(
t, DEV, 'widened.yaml', region_policy('{matchLabels: {app.kubernetes.io/name: evo-region}}',
'{from: [{ipBlock: {cidr: 10.0.0.0/8}}], ports: []}')),
'admits the region session port'),
('region policy by the session port name', lambda t: add_resource(
t, DEV, 'widened.yaml', region_policy('{matchLabels: {app.kubernetes.io/name: evo-region}}',
'{ports: [{protocol: TCP, port: session}]}')),
'admits the region session port'),
('region policy with a port range spanning 7400', lambda t: add_resource(
t, DEV, 'widened.yaml', region_policy('{matchLabels: {app.kubernetes.io/name: evo-region}}',
'{ports: [{protocol: TCP, port: 7000, endPort: 8000}]}')),
'admits the region session port'),
('policy for every pod on 7400', lambda t: add_resource(
t, DEV, 'widened.yaml', region_policy('{}', '{from: [{ipBlock: {cidr: 0.0.0.0/0}}], '
'ports: [{protocol: TCP, port: 7400}]}')),
'admits the region session port'),
('policy by expression selector', lambda t: add_resource(
t, DEV, 'widened.yaml', region_policy(
'{matchExpressions: [{key: app.kubernetes.io/name, operator: Exists}]}', '{ports: [{port: 7400}]}')),
'admits the region session port'),
('policy port without number', lambda t: add_resource(
t, DEV, 'widened.yaml', region_policy('{matchLabels: {app.kubernetes.io/part-of: aevolution}}',
'{ports: [{protocol: TCP}]}')), 'admits the region session port'),
# Activation follow-up: the rendered route keeps every reviewed field.
('rendered route selector patched', lambda t: (route_on(t), inline_patch(
t, DEV, '- op: replace\n path: /spec/selector\n value: {app.kubernetes.io/part-of: aevolution}\n',
{'kind': 'Service', 'name': 'evo-region-session'})),
'(rendered) Service/evo-region-session: the route selects only the region pod'),
('rendered route VIP patched', lambda t: (route_on(t), inline_patch(
t, DEV, '- op: replace\n path: /metadata/annotations/metallb.io~1loadBalancerIPs\n value: 8.8.8.8\n',
{'kind': 'Service', 'name': 'evo-region-session'})),
'(rendered) Service/evo-region-session: the VIP must be an internal address'),
('rendered route source range patched', lambda t: (route_on(t), inline_patch(
t, DEV, '- op: replace\n path: /spec/loadBalancerSourceRanges\n value: [0.0.0.0/0]\n',
{'kind': 'Service', 'name': 'evo-region-session'})),
'(rendered) Service/evo-region-session: 0.0.0.0/0 must be a private range'),
('secret generator', lambda t: append(t, DEV, 'secretGenerator:\n - name: leaked\n literals: [a=b]\n'),
'unsupported kustomize features'),
('helm chart inflation', lambda t: append(t, DEV, 'helmCharts:\n - name: x\n repo: https://example.invalid\n'),
'unsupported kustomize features'),
('route included by stage', lambda t: append(t, 'deploy/kubernetes/region/stage/kustomization.yaml',
'components:\n - ../dev-session-route\n'),
'only region/dev may include'),
('stage quota allowing a LoadBalancer', lambda t: write(
t, 'deploy/kubernetes/platform/stage/quota-lb.yaml',
'apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: aevolution-quota\n'
'spec:\n hard:\n services.loadbalancers: "1"\n'), 'LoadBalancers only in evolution-dev'),
('world-open source range', lambda t: (edit(t, ROUTE + 'service.yaml', '172.10.1.0/24', '0.0.0.0/0'),
edit(t, ROUTE + 'networkpolicy.yaml', '172.10.1.0/24', '0.0.0.0/0')),
'private range'),
('public VIP', lambda t: edit(t, ROUTE + 'service.yaml', '172.10.1.202', '8.8.8.8'), 'internal address'),
('policy CIDR differs from the Service', lambda t: edit(t, ROUTE + 'networkpolicy.yaml',
'172.10.1.0/24', '10.99.0.0/24'),
'exactly the Service source ranges'),
('policy admits another port', lambda t: edit(t, ROUTE + 'networkpolicy.yaml', 'port: 7400', 'port: 7401'),
'only UDP 7400'),
# ADR-0087 amendment 2026-10-05: the route is QUIC over UDP; a stale TCP route fails.
('route Service still TCP', lambda t: edit(t, ROUTE + 'service.yaml', 'protocol: UDP', 'protocol: TCP'),
'exactly UDP 7400 (QUIC)'),
('route policy still TCP', lambda t: edit(t, ROUTE + 'networkpolicy.yaml', 'protocol: UDP', 'protocol: TCP'),
'only UDP 7400'),
('route Service with TCP beside UDP', lambda t: append(
t, ROUTE + 'service.yaml', ' - {name: session-tcp, port: 7400, targetPort: 7400, protocol: TCP}\n'),
'exactly UDP 7400 (QUIC)'),
('route container port still TCP', lambda t: edit(t, ROUTE + 'listener.yaml',
'containerPort: 7400, protocol: UDP',
'containerPort: 7400, protocol: TCP'),
'container ports must be'),
('region policy admitting UDP 7400', lambda t: add_resource(
t, DEV, 'widened.yaml', region_policy('{matchLabels: {app.kubernetes.io/name: evo-region}}',
'{from: [{ipBlock: {cidr: 10.0.0.0/8}}], '
'ports: [{protocol: UDP, port: 7400}]}')),
'admits the region session port'),
('region policy admitting SCTP 7400', lambda t: add_resource(
t, DEV, 'widened.yaml', region_policy('{matchLabels: {app.kubernetes.io/name: evo-region}}',
'{ports: [{protocol: SCTP, port: 7400}]}')),
'admits the region session port'),
('cluster traffic policy', lambda t: edit(t, ROUTE + 'service.yaml', 'externalTrafficPolicy: Local',
'externalTrafficPolicy: Cluster'), 'Local traffic policy'),
('node ports allocated', lambda t: edit(t, ROUTE + 'service.yaml', 'allocateLoadBalancerNodePorts: false',
'allocateLoadBalancerNodePorts: true'), 'no node ports'),
('PROXY protocol annotation', lambda t: edit(t, ROUTE + 'service.yaml', ' metallb.io/loadBalancerIPs:',
' example.io/proxy-protocol: "true"\n metallb.io/loadBalancerIPs:'),
'no PROXY protocol'),
('extra route port', lambda t: append(t, ROUTE + 'service.yaml',
' - {name: ops, port: 9100, targetPort: ops, protocol: TCP}\n'),
'exactly UDP 7400'),
('another LoadBalancer', lambda t: edit(t, 'deploy/kubernetes/region/base/service.yaml', 'type: ClusterIP',
'type: LoadBalancer'), 'Services are forbidden'),
('session port on another Service', lambda t: append(t, 'deploy/kubernetes/region/base/service.yaml',
' - {name: session, port: 7400, targetPort: 7400, protocol: TCP}\n'),
'only the dev session route Service'),
('session Ingress', lambda t: write(
t, 'deploy/kubernetes/region/dev/ingress.yaml',
'apiVersion: networking.k8s.io/v1\nkind: Ingress\nmetadata:\n name: session\nspec:\n rules:\n'
' - host: void.aeternalisgames.com\n http:\n paths:\n - path: /\n'
' pathType: Prefix\n backend:\n service:\n'
' name: evo-region-session\n port: {number: 7400}\n'), 'never an Ingress'),
('base listener on the pod interface', lambda t: edit(
t, 'deploy/kubernetes/region/base/deployment.yaml', 'value: "127.0.0.1:7400"', 'value: "0.0.0.0:7400"'),
'REGION_SESSION_LISTEN must be 127.0.0.1:7400'),
('hard-coded listener address', lambda t: edit(
t, 'deploy/kubernetes/region/base/deployment.yaml', '--m10-listen "$REGION_SESSION_LISTEN"',
'--m10-listen 0.0.0.0:7400'), 'only from REGION_SESSION_LISTEN'),
('route listener patch elsewhere', lambda t: edit(
t, 'deploy/kubernetes/region/dev/region-values.yaml', '- {name: REGION_ID, value: "1"}',
'- {name: REGION_ID, value: "1"}\n - {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"}'),
'REGION_SESSION_LISTEN must be 127.0.0.1:7400'),
('route object in another namespace', lambda t: edit(
t, ROUTE + 'networkpolicy.yaml', ' name: evo-region-session-ingress',
' name: evo-region-session-ingress\n namespace: evolution-stage'), 'evolution-dev only'),
]
def copy_tree(target):
for directory in ('deploy', 'tools/ops', 'tools/backup'):
shutil.copytree(os.path.join(ROOT, directory), os.path.join(target, directory))
os.makedirs(os.path.join(target, 'tools', 'ci'))
for name in ('deploy-policy-check.py', 'prepare-kustomize.sh'):
shutil.copy(os.path.join(ROOT, 'tools', 'ci', name), os.path.join(target, 'tools', 'ci'))
def pinned_kustomize():
with open(os.path.join(ROOT, 'tools', 'ci', 'prepare-kustomize.sh'), encoding='utf-8') as f:
version = re.search(r'^version=(v[0-9.]+)$', f.read(), re.M).group(1)
for candidate in (os.environ.get('KUSTOMIZE', ''),
os.path.join(ROOT, '.cache', 'tools', f'kustomize-{version}', 'kustomize'),
shutil.which('kustomize') or ''):
if candidate and os.path.isfile(candidate):
return os.path.abspath(candidate)
raise SystemExit('selftest: the pinned kustomize is required (tools/ci/prepare-kustomize.sh)')
KUSTOMIZE = pinned_kustomize()
def run(tree):
result = subprocess.run([sys.executable, os.path.join(tree, 'tools', 'ci', 'deploy-policy-check.py')],
capture_output=True, text=True, env=dict(os.environ, KUSTOMIZE=KUSTOMIZE))
return result.returncode, result.stdout + result.stderr
failures = []
with tempfile.TemporaryDirectory() as clean:
copy_tree(clean)
code, output = run(clean)
if code != 0:
failures.append(f'baseline copy must pass:\n{output}')
for name, mutate, expected in CASES:
with tempfile.TemporaryDirectory() as tree:
copy_tree(tree)
mutate(tree)
code, output = run(tree)
refused = code != 0 and re.search(re.escape(expected), output)
print(f'{"refused" if refused else "NOT REFUSED"}: {name}')
if not refused:
failures.append(f'{name}: expected failure mentioning {expected!r}, got rc={code}\n{output}')
for message in failures:
print('FAIL', message)
print(f'deploy policy selftest: {len(CASES)} cases, {len(failures)} failures')
sys.exit(1 if failures else 0)