Add M9.8f dev session route design, disabled with fail-closed policy

ADR-0087 records the maintainer's route (void.aeternalisgames.com:7400 via a
dedicated internal MetalLB VIP to the evolution-dev region pod, TLS end to end,
CIDR-limited) and operator-opened windows enforced by evo-server. The route is a
kustomize component included nowhere; the policy check encodes its shape and
refuses enabling until VIP/CIDR, the dev quota and --session-window exist, with a
20-case self-test in the CI deploy stage. The base listener stays loopback.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-02 17:34:12 -04:00
co-authored by Claude Opus 5.5
parent 2a2d6fbbfe
commit 9f622cf6b3
18 changed files with 504 additions and 24 deletions
+3 -2
View File
@@ -90,11 +90,12 @@ RUN apt-get update \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /source
COPY deploy/ deploy/
COPY tools/ci/deploy-policy-check.py tools/ci/
COPY tools/ci/deploy-policy-check.py tools/ci/deploy-policy-selftest.py tools/ci/
COPY tools/ops/ tools/ops/
COPY tools/backup/ tools/backup/
COPY tools/ci/ops-lib-check.sh tools/ci/
RUN python3 tools/ci/deploy-policy-check.py && bash tools/ci/ops-lib-check.sh
RUN python3 tools/ci/deploy-policy-check.py && python3 tools/ci/deploy-policy-selftest.py \
&& bash tools/ci/ops-lib-check.sh
# M8.5: monitoring configuration and alert tests with the pinned images' own tools.
COPY --from=quay.io/prometheus/prometheus@sha256:6976aa8a60fec930796ce5772b8d12da7a318a5daa8d40d69c5c7819a05eeed7 /bin/promtool /usr/local/bin/promtool
COPY --from=quay.io/prometheus/alertmanager@sha256:e9733bafb1bdef9b00e25a21f8f99dc26a22224bf16641ad754d1649f4c3357a /bin/amtool /usr/local/bin/amtool
@@ -59,13 +59,14 @@ spec:
inputs=()
if [ -n "$REGION_INPUTS" ]; then inputs=(--synthetic-inputs "$REGION_INPUTS"); fi
# Dev sessions are a drill-time override too (tools/ops/session-drill.sh,
# ADR-0083): loopback only, reached by kubectl port-forward, never a Service.
# ADR-0083): loopback by default, reached by kubectl port-forward. Only the
# disabled dev-session-route component (ADR-0087) may bind the pod interface.
# evo-server refuses any environment but dev and needs the synthetic carrier.
# Consumed grants persist on the objects volume (outside the object
# namespaces, ignored by backups) so a restarted pod refuses them.
sessions=()
if [ -n "$REGION_SESSION" ]; then
sessions=(--session-listen 127.0.0.1:7400 --session-cert /run/session/server.pem
sessions=(--session-listen "$REGION_SESSION_LISTEN" --session-cert /run/session/server.pem
--session-key /run/session/server.key --session-grant-key /run/session/grant.key
--session-grant-ledger /objects/dev-session-grants.v1
--session-environment "$REGION_SESSION")
@@ -89,6 +90,9 @@ spec:
- {name: REGION_RESERVE_MIB, value: "1024"}
# M9.7 drill-only: "dev" enables the session listener; Git keeps it empty.
- {name: REGION_SESSION, value: ""}
# Session listener address; the policy check allows only this loopback value
# outside region/dev-session-route.
- {name: REGION_SESSION_LISTEN, value: "127.0.0.1:7400"}
ports:
- {name: ops, containerPort: 9100, protocol: TCP}
readinessProbe:
@@ -0,0 +1,16 @@
# M9.8f dev session route for evolution-dev ONLY (ADR-0087). DISABLED: no
# kustomization includes this component yet. Enabling it (components: in
# region/dev/kustomization.yaml) is refused by tools/ci/deploy-policy-check.py until:
# - the REPLACE markers carry the allocated internal MetalLB VIP and the permitted
# development/test client CIDR(s);
# - the platform dev quota allows exactly one LoadBalancer Service;
# - evo-server enforces operator-opened session windows itself (--session-window).
# Route: void.aeternalisgames.com:7400 -> VIP -> Service -> evo-region pod:7400 (TLS 1.3
# terminates in the pod). Raw TCP only: no Ingress, no TLS at nginx, no PROXY protocol.
apiVersion: kustomize.config.k8s.io/v1alpha1
kind: Component
resources:
- service.yaml
- networkpolicy.yaml
patches:
- path: listener.yaml
@@ -0,0 +1,15 @@
# With the route, the opt-in session listener binds the pod interface and names its
# port; it still starts only when sessions are enabled for a window (ADR-0087).
apiVersion: apps/v1
kind: Deployment
metadata:
name: evo-region
spec:
template:
spec:
containers:
- name: evo-server
env:
- {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"}
ports:
- {name: session, containerPort: 7400, protocol: TCP}
@@ -0,0 +1,18 @@
# Session ingress to the region pod: TCP 7400 from the permitted development/test
# client CIDR(s) only, which must equal the Service's loadBalancerSourceRanges.
# Everything else stays denied by the platform default-deny policy.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: evo-region-session-ingress
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: evo-region
policyTypes: [Ingress]
ingress:
- from:
- ipBlock:
cidr: REPLACE-dev-test-client-cidr
ports:
- {protocol: TCP, port: 7400}
@@ -0,0 +1,26 @@
# The one LoadBalancer Service AEvolution may own (ADR-0087), evolution-dev only.
# - metallb.io/loadBalancerIPs pins the dedicated internal VIP that the internal DNS
# name void.aeternalisgames.com resolves to (never the control-plane address).
# - externalTrafficPolicy Local keeps client source addresses, so the NetworkPolicy's
# ipBlock (same CIDRs) is enforceable; new connections reach only a ready pod.
# - allocateLoadBalancerNodePorts false: no node ports (the quota forbids them).
# - loadBalancerSourceRanges duplicates the CIDR at the Service; the edge firewall is
# the third layer. TLS 1.3, the dev CA and single-use grants remain mandatory.
apiVersion: v1
kind: Service
metadata:
name: evo-region-session
annotations:
metallb.io/loadBalancerIPs: REPLACE-dev-session-vip
labels:
app.kubernetes.io/name: evo-region
spec:
type: LoadBalancer
allocateLoadBalancerNodePorts: false
externalTrafficPolicy: Local
loadBalancerSourceRanges:
- REPLACE-dev-test-client-cidr
selector:
app.kubernetes.io/name: evo-region
ports:
- {name: session, port: 7400, targetPort: session, protocol: TCP}
+2 -1
View File
@@ -458,7 +458,8 @@ terrain meshing or procedural creature animation is required.
M9.8b adds bounded UI state, protected external input capture and bounded session
event delivery. M9.8c integrates the launcher, legend, labelled inspector, click
selection and session panel. M9.8d adds rebindable controls (ADR-0085). M9.8e
packages a local Windows x64 ZIP (ADR-0086). ADR-0084 records the toolkit scopes.
packages a local Windows x64 ZIP (ADR-0086). M9.8f designs the dev session route,
committed disabled (ADR-0087). ADR-0084 records the toolkit scopes.
**M9.8 planning (2026-10-02):** user-confirmed Windows ZIP, temporary Gitea Actions
artifacts, labelled engineering UI/launcher and control rebinding. See the
+5 -2
View File
@@ -10,10 +10,13 @@ bounded errors, Retry/Cancel), legend, labelled inspector, click selection, sess
panel with the intention outcome record, and visible notifications/dialog. M9.8d
adds rebindable controls (ADR-0085) with conflict checks and a validated file. M9.8e
builds a static-runtime Windows x64 ZIP locally with notices, identity, checksums and
credential guards (ADR-0086); Actions publication needs a Windows runner. Tests,
credential guards (ADR-0086); Actions publication needs a Windows runner. M9.8f
designs the dev session route (ADR-0087), committed disabled with fail-closed policy
tests; nothing was applied to the cluster. Tests,
visual review and limits are recorded in engineering/VALIDATION.md.
Next: M9.8f dev TCP route design, then M9.8g qualification on the low-spec VM.
Next: server-enforced session windows and the route inputs (VIP, CIDR, DNS) to enable
ADR-0087, and M9.8g qualification on the low-spec VM.
Windows packages will be built locally until a Windows runner exists. Testing is internal; 172.10.1.160 is the
RKE2 control-plane address. VM qualification and the cluster restart drill remain
outstanding, as agreed. See engineering/M9_8_DELIVERY_PLAN.md.
+2
View File
@@ -1,6 +1,8 @@
# ADR-0069: Environment isolation and World0 protection
## Status
ADR-0087 allows exactly one LoadBalancer Service, in evolution-dev only, for the dev
session route once enabled; node ports stay forbidden and stage/World0 are unchanged.
Accepted for M8.2, 2026-09-29. It builds on the M8.1 inventory
(infrastructure/INVENTORY.md), ADR-0020 (delivery), and OPERATIONS.md (World0
protection). Nothing here deploys a world; World0 launch stays gated by M8.8.
+2
View File
@@ -1,6 +1,8 @@
# ADR-0083: Dev region sessions, no-effect intentions and the session drill
## Status
Listener/Service rule narrowly superseded for evolution-dev by ADR-0087 (dev session
route), effective only when that route is enabled; until then this ADR applies as is.
Accepted for M9.7 (engineering client to one authoritative **dev** region). Decisions
confirmed with the maintainer on 2026-10-01: durable no-effect intentions, an
explicitly empty organism presentation, local real-process evidence plus a dev
+89
View File
@@ -0,0 +1,89 @@
# ADR-0087: Dev session route and operator-opened session windows
## Status
Accepted for M9.8f as design (2026-10-02), evolution-dev only. The route is committed
**disabled**; nothing was applied to the cluster. Enabling it is blocked by policy until
the inputs and server-side window enforcement below exist. Narrowly supersedes
ADR-0083 (loopback-only listener, never a Service) and ADR-0069 (no LoadBalancer
Services) for evolution-dev, and only once enabled.
## Context
Internal testers need the native client (M9.8e ZIP) to reach the dev region without
`kubectl port-forward`. The cluster inventory (INVENTORY.md, 2026-09-29) shows RKE2's
ingress-nginx 1.12.1 DaemonSet serving 80/443 on all four node addresses (public sites
included), MetalLB 0.15.2 in FRR mode with one LoadBalancer (Gitea SSH), and an existing
HTTP Ingress for void.aeternalisgames.com in namespace `void`. The session protocol is
TLS 1.3 over raw TCP (ADR-0082), not HTTP. Sessions need synthetic carrier inputs, which
ADR-0073 keeps drill-time only so dev history does not grow continuously.
## Decision (maintainer, 2026-10-02)
**Route.** `void.aeternalisgames.com:7400` resolves, in internal DNS, to a dedicated
internal MetalLB VIP (never automatically the control-plane address 172.10.1.160) that
fronts one LoadBalancer Service `evo-region-session` in evolution-dev, which forwards to
the region pod's port 7400. TLS 1.3 terminates in the region pod; its dev-CA server
certificate carries `void.aeternalisgames.com`, which the client verifies. No HTTP
Ingress, no TLS termination at nginx, no PROXY-protocol prefix. The Service uses
`externalTrafficPolicy: Local` (client addresses preserved), no node ports, a pinned
`metallb.io/loadBalancerIPs` annotation and `loadBalancerSourceRanges`. New connections
reach only a ready pod; after a replacement, clients reconnect with a fresh grant and
baseline (ADR-0083).
**Access.** Development workstation/test subnet only, in three layers: the edge
firewall, the Service source ranges and a NetworkPolicy admitting TCP 7400 to the region
pod from exactly those CIDRs. TLS with the dev CA and single-use, short-lived grants stay
mandatory; grants remain bearer tokens.
**Listener.** The base keeps the opt-in listener on loopback through
`REGION_SESSION_LISTEN=127.0.0.1:7400`. Only the route component
(`deploy/kubernetes/region/dev-session-route`) binds the pod interface and names the
session port. The listener still starts only when sessions are enabled.
**Windows.** Sessions and synthetic inputs run only during an operator-opened test window
with automatic expiry (default 30 minutes); always-on simulation waits for an explicit
disposable-world lifecycle and retention. The endpoint (DNS, MetalLB Service) may stay
configured permanently. A small dev-only GitOps configuration holds an activation id,
the exact world/region identity and an absolute UTC expiry. Operations, via a manually
triggered Gitea workflow: Open (default 30 min), Extend (new explicit expiry), Close
(stop accepting sessions, close existing connections, stop synthetic inputs, drain
pending commits) and Status (window, expiry, active sessions, committed tick).
Expiry is enforced **inside evo-server**, not by a delayed cleanup job:
- a restart keeps the original absolute deadline, never a fresh duration;
- an expired configuration cannot reopen the listener;
- expiry stops synthetic generation as well as connections;
- a failed workflow cannot leave simulation running; cleanup may later restore the
closed Git configuration. Grant expiry alone does not stop the generator.
## Enabling (all required; `tools/ci/deploy-policy-check.py` refuses otherwise)
1. The VIP (private, from the MetalLB pool) and permitted client CIDR(s) (private, no
wider than /16) replace the REPLACE markers; Service ranges equal the policy's.
2. `deploy/kubernetes/platform/dev` raises `services.loadbalancers` to exactly 1;
node ports stay 0; stage and World0 stay 0.
3. evo-server implements the session window (`--session-window`) as above, with tests
and an amendment to this ADR, and the base deployment passes it.
4. Only `region/dev` includes the component. Then: internal DNS for void to the VIP, the
edge firewall rule, and a server-side dry-run (quota accounting of the Local health
check port is to be confirmed there), a new dev server certificate for
void.aeternalisgames.com, and live connectivity recorded separately in VALIDATION.md.
## Open points for the maintainer
- **Argo delivery.** The aevolution-dev Application has no automated sync, and the
policy forbids auto-sync outside the platform app (ADR-0069). A workflow commit alone
will not deploy; either a manual sync per window or a scoped dev exception must be
decided before implementing the workflow. CI holds no cluster credentials (ADR-0020),
so Status can report Git state; live fields need a defined read path.
- **void namespace.** void.aeternalisgames.com already has an HTTP Ingress (namespace
`void`). Pointing internal DNS at the VIP changes what internal clients reach for that
name; confirm or retire the existing route.
- Window limits: maximum window length and clock-skew tolerance.
## Alternatives considered
- ingress-nginx tcp-services or SNI passthrough on 443: couples game sessions to the
shared web ingress (443 likely internet-facing) and needs controller changes.
- Port-forward only (ADR-0083): no tester-friendly endpoint.
- NodePort: opens a port on every node; forbidden by ADR-0069.
- Always-on sessions in dev: continuous synthetic history; rejected for now.
## Consequences
One reviewed exception to "no LoadBalancer" for evolution-dev. The policy check and its
fail-closed self-test (`tools/ci/deploy-policy-selftest.py`) encode the route's shape and
the enabling preconditions. World0 and stage are untouched.
+1
View File
@@ -97,5 +97,6 @@ design. No unadopted candidate is a link dependency of the proof.
- [ADR-0083: Dev region sessions](0083-dev-region-sessions.md) - session server in evo-server, durable no-effect intentions on synthetic carriers, empty organism presentation, reconnect with fresh baselines, loopback-only drill-time dev deployment.
- [ADR-0084: Engineering UI toolkit](0084-engineering-ui-toolkit.md) - Dear ImGui 1.92.9b pinned (M9.8a) and integrated as the engineering client overlay with toolkit-first input routing (M9.8c).
- [ADR-0087: Dev session route](0087-dev-session-route.md) - evolution-dev only: void.aeternalisgames.com:7400 via a dedicated internal MetalLB VIP to the region pod, TLS end to end, CIDR-limited, operator-opened windows enforced by evo-server; committed disabled with fail-closed policy tests.
- [ADR-0086: Windows client package](0086-windows-client-package.md) - static-runtime Windows x64 ZIP with explicit allowlist, licence texts, build identity, SHA256 manifest, credential guards and deterministic layout; local packaging until a Windows runner exists.
- [ADR-0085: Rebindable engineering controls](0085-rebindable-engineering-controls.md) - SDL-free action/HID-key bindings, conflict validation, versioned all-or-nothing controls file, binding capture before toolkit/world.
+7 -4
View File
@@ -4,7 +4,9 @@ Status: M9.8a probe, M9.8b owned state/capture/event bounds, M9.8c launcher,
legend, labelled inspector and click selection, and M9.8d rebindable controls
complete, 2026-10-02. M9.8e local Windows ZIP complete (ADR-0086); its Gitea
Actions publication waits for a Windows runner and its clean-VM run for M9.8g.
Remaining segments planned.
M9.8f route design, disabled manifests and fail-closed policy tests complete
(ADR-0087); enabling waits for the VIP, client CIDR, DNS, dev quota and server-enforced
session windows. M9.8g remains.
No application UI or deployment completion is claimed. See ADR-0084 for the toolkit boundary.
## Confirmed scope
@@ -64,9 +66,10 @@ All selected biological data remains a read-only presentation copy.
- No Windows runner exists yet (confirmed 2026-10-02). Local builds are the initial path. Do not silently route a Windows
package job to the existing Linux runner or assume an untested cross-compile.
The packaging script can run on the existing Windows development host meanwhile.
- Confirmed control plane: 172.10.1.160. Still needed for direct service routing:
nginx implementation/configuration, exposed TCP port,
source restrictions and certificate name. The protocol is TLS/TCP, not HTTP;
- Confirmed control plane: 172.10.1.160. Direct routing decided in ADR-0087: dedicated
internal MetalLB VIP, TCP 7400, void.aeternalisgames.com, CIDR-limited. Still needed:
the VIP from the MetalLB pool, the permitted client CIDR, internal DNS, and decisions
on Argo delivery of session windows and the existing void HTTP Ingress. The protocol is TLS/TCP, not HTTP;
a standard HTTP Ingress rule does not implement it. Port-forward remains supported.
- VM specification and whether graphics acceleration is available. Hardware-specific
performance acceptance cannot be declared before these are known.
+34
View File
@@ -3896,3 +3896,37 @@ Executed on the Windows development host (MSVC 19.44.35217, CMake 3.31.6):
Not done: Gitea Actions publication (no Windows runner), a run on a clean Windows
VM (M9.8g), code signing, hardware/DPI qualification, legal review of notices.
## 2026-10-02 — M9.8f dev session route design (ADR-0087)
Maintainer decisions: dedicated internal MetalLB VIP, TCP 7400,
void.aeternalisgames.com, TLS ending in the region pod, development/test CIDR only
(firewall + Service source ranges + NetworkPolicy), no Ingress/nginx TLS/PROXY
protocol; operator-opened windows with server-enforced absolute expiry; scope limited
to design, manifests and policy tests. Nothing was applied to the cluster.
Implemented: `REGION_SESSION_LISTEN` (base stays 127.0.0.1:7400); the disabled
component `deploy/kubernetes/region/dev-session-route` (LoadBalancer Service with
Local policy, no node ports, pinned VIP annotation and source ranges; CIDR-only
ingress policy on TCP 7400; listener patch to the pod interface) carrying REPLACE
markers; policy rules for the route's exact shape, LoadBalancer/NodePort and
session-port exclusivity, no session Ingress, evolution-dev-only placement, quota
limits, and enabling preconditions (no markers, dev quota of exactly one
LoadBalancer, server `--session-window`); patch files are now recognized generally.
`tools/ci/deploy-policy-selftest.py` copies the manifests and applies 20 unsafe changes.
Executed (WSL Ubuntu, Python 3 with PyYAML 6.0.1): deploy policy check on the
repository, 70 objects in 68 files, 0 failures; self-test 20/20 refused, baseline
copy passes; ops-lib and ci-path fixtures pass; session-drill usage unchanged.
kustomize v5.4.3 (official release, SHA-256 checked against its checksums.txt, in a
scratch directory, not a project dependency) rendered region/dev as committed (listener
127.0.0.1:7400, no session Service) and a temporary enabled copy with example private
values (LoadBalancer evo-region-session and the CIDR policy in evolution-dev, listener
0.0.0.0:7400, session container port merged).
Not run: the ci-deploy Docker stage (no Docker in this WSL distribution; its commands
were run directly), any cluster/server dry-run, MetalLB allocation, DNS, firewall or live
connectivity. Server-enforced session windows and the Gitea window workflow are not
implemented; Argo delivery for windows and the existing void HTTP Ingress are open
maintainer decisions recorded in ADR-0087.
@@ -82,3 +82,12 @@ exposed through a Service, NodePort or Ingress. A grant bundle lasts at most one
and each connection consumes one grant. WASD sends intentions; the small chip under
the intention dot turns white (admitted), green (durable, no effect) or red/grey
(rejected or outcome unknown). Console lines report every session event.
## 6. Planned direct route (M9.8f, disabled)
ADR-0087 designs `void.aeternalisgames.com:7400` through a dedicated internal MetalLB
VIP to the region pod (TLS end to end, development/test CIDR only), opened only in
operator windows that evo-server itself expires. The manifests are in
`deploy/kubernetes/region/dev-session-route` and are not included anywhere; the policy
check refuses to enable them until the VIP, client CIDR, dev quota and the server-side
`--session-window` exist. Until then use the port-forward procedures above.
+1 -1
View File
@@ -46,7 +46,7 @@ case "$suite" in
pattern="$common|$native|tests/|$postgres_tools|third_party/mbedtls/|tools/prepare-mbedtls\.cmake|tools/mbedtls-pin\.cmake|deploy/docker/Dockerfile"
;;
deploy)
pattern="$common|deploy/|tools/ops/|tools/backup/|tools/ci/deploy-policy-check\.py|tools/ci/monitoring-check\.sh|tools/ci/ops-lib-check\.sh"
pattern="$common|deploy/|tools/ops/|tools/backup/|tools/ci/deploy-policy-(check|selftest)\.py|tools/ci/monitoring-check\.sh|tools/ci/ops-lib-check\.sh"
;;
server-image)
pattern="$common|$native|migrations/|tools/prepare-postgres\.sh|tools/ci/server-image-smoke\.sh|deploy/docker/Dockerfile"
+120 -12
View File
@@ -24,8 +24,14 @@ environment-isolation or World0-protection invariant is broken:
required; upgrade-rehearsal flags never appear in a deployed workload;
- the Pods the backup and barrier scripts generate (rendered with --print-manifest)
keep the database-tool security contract: non-root 10001 with fsGroup 10001 so
the 0440 Secret files are readable, no token, read-only root, no escalation.
the 0440 Secret files are readable, no token, read-only root, no escalation;
- the dev session route (ADR-0087): loopback listener everywhere except the
evolution-dev route component; that component is the only LoadBalancer and the
only Service on the session port, narrowly shaped, and it cannot be enabled while
markers are unresolved or before server-enforced session windows exist.
"""
import ipaddress
import re
import os
import shutil
import subprocess
@@ -55,7 +61,15 @@ for base, _, files in os.walk(os.path.join(ROOT, 'deploy')):
path = os.path.join(base, name)
manifests[os.path.relpath(path, ROOT).replace(os.sep, '/')] = documents(path)
objects = [(p, d) for p, ds in manifests.items() for d in ds
if 'kind' in d and d['kind'] != 'Kustomization']
if 'kind' in d and d['kind'] not in ('Kustomization', 'Component')]
# Files used as kustomize patches hold partial objects (checked only for what they set).
patch_files = set()
for p, ds in manifests.items():
if p.endswith('kustomization.yaml') and ds:
for entry in ds[0].get('patches', []):
target = entry.get('path') if isinstance(entry, dict) else entry
if target:
patch_files.add(os.path.normpath(os.path.join(os.path.dirname(p), target)).replace(os.sep, '/'))
# Secrets and service-account tokens.
for path, doc in objects:
@@ -84,6 +98,21 @@ check(dns.get('policyTypes') == ['Egress'] and set(dns_ports) == {53} and len(dn
quota = base.get('ResourceQuota/aevolution-quota', {}).get('spec', {}).get('hard', {})
check(quota.get('services.loadbalancers') == '0' and quota.get('services.nodeports') == '0',
'quota must forbid LoadBalancer and NodePort services')
# ADR-0087: only the dev overlay may raise LoadBalancers, to exactly one, and only
# together with the session route; node ports stay forbidden everywhere.
ROUTE = 'deploy/kubernetes/region/dev-session-route/'
route_users = sorted(p for p, ds in manifests.items() if p.endswith('kustomization.yaml') and ds
and any('dev-session-route' in str(e) for e in ds[0].get('resources', []) + ds[0].get('components', [])))
route_enabled = route_users == ['deploy/kubernetes/region/dev/kustomization.yaml']
check(route_users in ([], ['deploy/kubernetes/region/dev/kustomization.yaml']),
f'only region/dev may include the dev session route: {route_users}')
for path, doc in objects:
if doc['kind'] == 'ResourceQuota' and path != f'{platform}/base/quota.yaml':
hard = doc.get('spec', {}).get('hard', {})
check(hard.get('services.nodeports', '0') == '0', f'{path}: node ports stay forbidden')
allowed = ('0', '1') if path.startswith(f'{platform}/dev/') and route_enabled else ('0',)
check(hard.get('services.loadbalancers', '0') in allowed,
f'{path}: LoadBalancers only in evolution-dev, at most one, with the session route')
for env in ('dev', 'stage', 'world0'):
k = f'{platform}/{env}/kustomization.yaml'
ns_path = f'{platform}/{env}/namespace.yaml'
@@ -212,7 +241,7 @@ for path, doc in objects:
for path, doc in objects:
# Overlay patches (region-values.yaml) are partial objects; the base is complete.
if (doc['kind'] == 'Deployment' and doc['metadata'].get('name') == 'evo-region'
and not path.endswith('region-values.yaml')):
and path not in patch_files):
spec = doc['spec']
pod = spec['template']['spec']
container = pod['containers'][0]
@@ -238,22 +267,101 @@ for path, doc in objects:
# M9.7 (ADR-0083): dev sessions are a drill-time override as well.
if variable.get('name') == 'REGION_SESSION':
check(variable.get('value') == '', f'{path}: dev sessions are never enabled in Git')
# M9.7 (ADR-0083): the session listener is loopback-only (kubectl port-forward), and its
# Secret is optional, 0440 and never exposed through a Service port.
# M9.7 (ADR-0083) and M9.8f (ADR-0087): the session listener is loopback (kubectl
# port-forward) everywhere except the evolution-dev route component; its Secret is
# optional and 0440; only that component's Service exposes the session port.
for path, doc in objects:
if path.startswith('deploy/kubernetes/region/') and doc['kind'] == 'Deployment' and not path.endswith('region-values.yaml'):
if path.startswith('deploy/kubernetes/region/') and doc['kind'] == 'Deployment':
pod = doc['spec']['template']['spec']
for container in pod.get('containers', []):
for variable in container.get('env', []):
if variable.get('name') == 'REGION_SESSION_LISTEN':
expected = '0.0.0.0:7400' if path.startswith(ROUTE) else '127.0.0.1:7400'
check(variable.get('value') == expected,
f'{path}: REGION_SESSION_LISTEN must be {expected} here')
if path in patch_files:
continue
script = ' '.join(pod['containers'][0].get('args', []))
check('--session-listen 127.0.0.1:' in script and '0.0.0.0:7400' not in script,
f'{path}: the session listener binds loopback only')
check('--session-listen "$REGION_SESSION_LISTEN"' in script and '0.0.0.0' not in script.replace('--ops-listen 0.0.0.0:9100', ''),
f'{path}: the session listener address comes only from REGION_SESSION_LISTEN')
for volume in pod.get('volumes', []):
if volume.get('secret', {}).get('secretName') == 'evolution-dev-session':
check(volume['secret'].get('optional') is True and volume['secret'].get('defaultMode') == 0o440,
f'{path}: the dev session Secret is optional and 0440')
route_ranges = []
for path, doc in objects:
if doc['kind'] == 'Service':
for port in doc['spec'].get('ports', []):
check(port.get('port') != 7400 and port.get('targetPort') not in (7400, 'session'),
f'{path}: the dev session port is never exposed by a Service')
spec = doc['spec']
session_port = any(port.get('port') == 7400 or port.get('targetPort') in (7400, 'session')
for port in spec.get('ports', []))
is_route = path == ROUTE + 'service.yaml' and doc['metadata'].get('name') == 'evo-region-session'
if not is_route:
check(not session_port, f'{path}: only the dev session route Service may expose the session port')
check(spec.get('type', 'ClusterIP') == 'ClusterIP',
f'{path}: {spec.get("type")} Services are forbidden (ADR-0069)')
continue
route_ranges = spec.get('loadBalancerSourceRanges', [])
check(spec.get('type') == 'LoadBalancer' and spec.get('externalTrafficPolicy') == 'Local'
and spec.get('allocateLoadBalancerNodePorts') is False,
f'{path}: the route is a LoadBalancer with Local traffic policy and no node ports')
check(spec.get('ports') == [{'name': 'session', 'port': 7400, 'targetPort': 'session', 'protocol': 'TCP'}],
f'{path}: the route exposes exactly TCP 7400')
check(spec.get('selector') == {'app.kubernetes.io/name': 'evo-region'},
f'{path}: the route selects only the region pod')
annotations = doc['metadata'].get('annotations', {})
check(set(annotations) == {'metallb.io/loadBalancerIPs'},
f'{path}: only a pinned MetalLB address annotation (no PROXY protocol or sharing)')
vip = annotations.get('metallb.io/loadBalancerIPs', '')
check(len(route_ranges) >= 1, f'{path}: loadBalancerSourceRanges must name the permitted CIDRs')
if not vip.startswith('REPLACE'):
try:
check(ipaddress.ip_address(vip).is_private, f'{path}: the VIP must be an internal address')
except ValueError:
check(False, f'{path}: the VIP must be one IP address')
for cidr in route_ranges:
if cidr.startswith('REPLACE'):
continue
try:
network = ipaddress.ip_network(cidr)
check(network.is_private and network.prefixlen >= (16 if network.version == 4 else 48),
f'{path}: {cidr} must be a private range no wider than /16 (IPv6 /48)')
except ValueError:
check(False, f'{path}: {cidr} is not a CIDR')
if doc['kind'] == 'Ingress':
text = yaml.safe_dump(doc)
check('7400' not in text and 'session' not in text,
f'{path}: the dev session is raw TCP, never an Ingress (ADR-0087)')
policies = [(p, d) for p, d in objects if p.startswith(ROUTE) and d['kind'] == 'NetworkPolicy']
check(len(policies) == 1, f'{ROUTE}: exactly one session ingress policy')
for path, doc in policies:
spec = doc['spec']
rules = spec.get('ingress', [])
blocks = [peer.get('ipBlock', {}) for rule in rules for peer in rule.get('from', [])]
check(spec.get('podSelector') == {'matchLabels': {'app.kubernetes.io/name': 'evo-region'}}
and spec.get('policyTypes') == ['Ingress'] and len(rules) == 1
and rules[0].get('ports') == [{'protocol': 'TCP', 'port': 7400}],
f'{path}: the session policy admits only TCP 7400 to the region pod')
check(all(set(b) == {'cidr'} for b in blocks) and len(blocks) == len(rules[0].get('from', []))
and sorted(b.get('cidr') for b in blocks) == sorted(route_ranges),
f'{path}: the session policy admits exactly the Service source ranges')
for path, doc in objects:
if path.startswith(ROUTE):
check(doc.get('metadata', {}).get('namespace') in (None, 'evolution-dev'),
f'{path}: the session route belongs to evolution-dev only')
# Enabling the route fails closed until its values are real and the server enforces
# operator-opened windows itself (expiry survives restarts; ADR-0087).
if route_enabled:
for name in os.listdir(os.path.join(ROOT, ROUTE)):
with open(os.path.join(ROOT, ROUTE, name), encoding='utf-8') as handle:
check('REPLACE' not in handle.read(), f'{ROUTE}{name}: unresolved REPLACE marker with the route enabled')
base_script = ' '.join(' '.join(c.get('args', [])) for p, d in objects
if p == 'deploy/kubernetes/region/base/deployment.yaml' and d['kind'] == 'Deployment'
for c in d['spec']['template']['spec']['containers'])
check('--session-window' in base_script,
'the dev session route needs server-enforced session windows (--session-window, ADR-0087)')
dev_quota = [d.get('spec', {}).get('hard', {}).get('services.loadbalancers') for p, d in objects
if d['kind'] == 'ResourceQuota' and p.startswith(f'{platform}/dev/')]
check(dev_quota == ['1'], 'the dev quota must allow exactly one LoadBalancer for the route')
for env in ('dev', 'stage'):
overlay = f'deploy/kubernetes/region/{env}/'
images = manifests.get(overlay + 'kustomization.yaml', [{}])[0].get('images', [])
@@ -311,7 +419,7 @@ for directory, _, files in os.walk(os.path.join(ROOT, 'deploy', 'kubernetes')):
# M8.9 scheduling (ADR-0076). Node-local volumes pin their pods: added CPU does not
# make the objects volume portable. Compute is a preference so one node still works.
for path, doc in objects:
if doc['kind'] not in workload_kinds or path.endswith('region-values.yaml'):
if doc['kind'] not in workload_kinds or path in patch_files:
continue
spec = doc.get('spec', {})
pod = spec.get('jobTemplate', {}).get('spec', {}).get('template', {}).get('spec') \
+148
View File
@@ -0,0 +1,148 @@
#!/usr/bin/env python3
"""M9.8f fail-closed tests for deploy-policy-check.py (ADR-0087). Needs Python 3 and
PyYAML only. Copies the manifests and the tools the check runs into a temporary tree,
applies one unsafe change at a time and requires the check to fail with the expected
message; the unmodified copy must pass.
"""
import os
import re
import shutil
import subprocess
import sys
import tempfile
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
ROUTE = 'deploy/kubernetes/region/dev-session-route/'
DEV = 'deploy/kubernetes/region/dev/kustomization.yaml'
REAL = [(ROUTE + 'service.yaml', 'REPLACE-dev-session-vip', '10.20.30.40'),
(ROUTE + 'service.yaml', 'REPLACE-dev-test-client-cidr', '10.20.0.0/24'),
(ROUTE + 'networkpolicy.yaml', 'REPLACE-dev-test-client-cidr', '10.20.0.0/24')]
def edit(tree, path, old, new, count=1):
full = os.path.join(tree, path)
with open(full, encoding='utf-8') as f:
text = f.read()
if old not in text:
raise SystemExit(f'selftest setup: {old!r} not in {path}')
with open(full, 'w', encoding='utf-8') as f:
f.write(text.replace(old, new, count))
def append(tree, path, text):
with open(os.path.join(tree, path), 'a', encoding='utf-8') as f:
f.write(text)
def write(tree, path, text):
full = os.path.join(tree, path)
os.makedirs(os.path.dirname(full), exist_ok=True)
with open(full, 'w', encoding='utf-8') as f:
f.write(text)
def enable(tree):
append(tree, DEV, 'components:\n - ../dev-session-route\n')
def resolve(tree):
for path, old, new in REAL:
edit(tree, path, old, new)
def quota_one(tree):
write(tree, 'deploy/kubernetes/platform/dev/quota-session.yaml',
'apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: aevolution-quota\n'
'spec:\n hard:\n services.loadbalancers: "1"\n services.nodeports: "0"\n')
CASES = [
('enabling with unresolved markers', lambda t: enable(t), 'unresolved REPLACE marker'),
('enabling before server-enforced windows', lambda t: (enable(t), resolve(t), quota_one(t)),
'server-enforced session windows'),
('enabling without the dev quota', lambda t: (enable(t), resolve(t)), 'exactly one LoadBalancer'),
('route included by stage', lambda t: append(t, 'deploy/kubernetes/region/stage/kustomization.yaml',
'components:\n - ../dev-session-route\n'),
'only region/dev may include'),
('stage quota allowing a LoadBalancer', lambda t: write(
t, 'deploy/kubernetes/platform/stage/quota-lb.yaml',
'apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: aevolution-quota\n'
'spec:\n hard:\n services.loadbalancers: "1"\n'), 'LoadBalancers only in evolution-dev'),
('world-open source range', lambda t: (edit(t, ROUTE + 'service.yaml', 'REPLACE-dev-test-client-cidr', '0.0.0.0/0'),
edit(t, ROUTE + 'networkpolicy.yaml', 'REPLACE-dev-test-client-cidr', '0.0.0.0/0')),
'private range'),
('public VIP', lambda t: edit(t, ROUTE + 'service.yaml', 'REPLACE-dev-session-vip', '8.8.8.8'), 'internal address'),
('policy CIDR differs from the Service', lambda t: edit(t, ROUTE + 'networkpolicy.yaml',
'REPLACE-dev-test-client-cidr', '10.99.0.0/24'),
'exactly the Service source ranges'),
('policy admits another port', lambda t: edit(t, ROUTE + 'networkpolicy.yaml', 'port: 7400', 'port: 7401'),
'only TCP 7400'),
('cluster traffic policy', lambda t: edit(t, ROUTE + 'service.yaml', 'externalTrafficPolicy: Local',
'externalTrafficPolicy: Cluster'), 'Local traffic policy'),
('node ports allocated', lambda t: edit(t, ROUTE + 'service.yaml', 'allocateLoadBalancerNodePorts: false',
'allocateLoadBalancerNodePorts: true'), 'no node ports'),
('PROXY protocol annotation', lambda t: edit(t, ROUTE + 'service.yaml', ' metallb.io/loadBalancerIPs:',
' example.io/proxy-protocol: "true"\n metallb.io/loadBalancerIPs:'),
'no PROXY protocol'),
('extra route port', lambda t: append(t, ROUTE + 'service.yaml',
' - {name: ops, port: 9100, targetPort: ops, protocol: TCP}\n'),
'exactly TCP 7400'),
('another LoadBalancer', lambda t: edit(t, 'deploy/kubernetes/region/base/service.yaml', 'type: ClusterIP',
'type: LoadBalancer'), 'Services are forbidden'),
('session port on another Service', lambda t: append(t, 'deploy/kubernetes/region/base/service.yaml',
' - {name: session, port: 7400, targetPort: 7400, protocol: TCP}\n'),
'only the dev session route Service'),
('session Ingress', lambda t: write(
t, 'deploy/kubernetes/region/dev/ingress.yaml',
'apiVersion: networking.k8s.io/v1\nkind: Ingress\nmetadata:\n name: session\nspec:\n rules:\n'
' - host: void.aeternalisgames.com\n http:\n paths:\n - path: /\n'
' pathType: Prefix\n backend:\n service:\n'
' name: evo-region-session\n port: {number: 7400}\n'), 'never an Ingress'),
('base listener on the pod interface', lambda t: edit(
t, 'deploy/kubernetes/region/base/deployment.yaml', 'value: "127.0.0.1:7400"', 'value: "0.0.0.0:7400"'),
'REGION_SESSION_LISTEN must be 127.0.0.1:7400'),
('hard-coded listener address', lambda t: edit(
t, 'deploy/kubernetes/region/base/deployment.yaml', '--session-listen "$REGION_SESSION_LISTEN"',
'--session-listen 0.0.0.0:7400'), 'only from REGION_SESSION_LISTEN'),
('route listener patch elsewhere', lambda t: edit(
t, 'deploy/kubernetes/region/dev/region-values.yaml', '- {name: REGION_ID, value: "1"}',
'- {name: REGION_ID, value: "1"}\n - {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"}'),
'REGION_SESSION_LISTEN must be 127.0.0.1:7400'),
('route object in another namespace', lambda t: edit(
t, ROUTE + 'networkpolicy.yaml', ' name: evo-region-session-ingress',
' name: evo-region-session-ingress\n namespace: evolution-stage'), 'evolution-dev only'),
]
def copy_tree(target):
for directory in ('deploy', 'tools/ops', 'tools/backup'):
shutil.copytree(os.path.join(ROOT, directory), os.path.join(target, directory))
os.makedirs(os.path.join(target, 'tools', 'ci'))
shutil.copy(os.path.join(ROOT, 'tools', 'ci', 'deploy-policy-check.py'), os.path.join(target, 'tools', 'ci'))
def run(tree):
result = subprocess.run([sys.executable, os.path.join(tree, 'tools', 'ci', 'deploy-policy-check.py')],
capture_output=True, text=True)
return result.returncode, result.stdout + result.stderr
failures = []
with tempfile.TemporaryDirectory() as clean:
copy_tree(clean)
code, output = run(clean)
if code != 0:
failures.append(f'baseline copy must pass:\n{output}')
for name, mutate, expected in CASES:
with tempfile.TemporaryDirectory() as tree:
copy_tree(tree)
mutate(tree)
code, output = run(tree)
refused = code != 0 and re.search(re.escape(expected), output)
print(f'{"refused" if refused else "NOT REFUSED"}: {name}')
if not refused:
failures.append(f'{name}: expected failure mentioning {expected!r}, got rc={code}\n{output}')
for message in failures:
print('FAIL', message)
print(f'deploy policy selftest: {len(CASES)} cases, {len(failures)} failures')
sys.exit(1 if failures else 0)