From 9f622cf6b348bccb2797d51879a50f4e1c212dd9 Mon Sep 17 00:00:00 2001 From: dingo Date: Fri, 2 Oct 2026 17:34:12 -0400 Subject: [PATCH] Add M9.8f dev session route design, disabled with fail-closed policy ADR-0087 records the maintainer's route (void.aeternalisgames.com:7400 via a dedicated internal MetalLB VIP to the evolution-dev region pod, TLS end to end, CIDR-limited) and operator-opened windows enforced by evo-server. The route is a kustomize component included nowhere; the policy check encodes its shape and refuses enabling until VIP/CIDR, the dev quota and --session-window exist, with a 20-case self-test in the CI deploy stage. The base listener stays loopback. Co-Authored-By: Claude Opus 5.5 --- deploy/docker/Dockerfile | 5 +- deploy/kubernetes/region/base/deployment.yaml | 8 +- .../dev-session-route/kustomization.yaml | 16 ++ .../region/dev-session-route/listener.yaml | 15 ++ .../dev-session-route/networkpolicy.yaml | 18 +++ .../region/dev-session-route/service.yaml | 26 +++ docs/ROADMAP.md | 3 +- docs/STATUS.md | 7 +- docs/adr/0069-environment-isolation.md | 2 + docs/adr/0083-dev-region-sessions.md | 2 + docs/adr/0087-dev-session-route.md | 89 +++++++++++ docs/adr/README.md | 1 + docs/engineering/M9_8_DELIVERY_PLAN.md | 11 +- docs/engineering/VALIDATION.md | 34 ++++ docs/networking/DEV_SESSION_OPERATIONS.md | 9 ++ tools/ci/ci-paths.sh | 2 +- tools/ci/deploy-policy-check.py | 132 ++++++++++++++-- tools/ci/deploy-policy-selftest.py | 148 ++++++++++++++++++ 18 files changed, 504 insertions(+), 24 deletions(-) create mode 100644 deploy/kubernetes/region/dev-session-route/kustomization.yaml create mode 100644 deploy/kubernetes/region/dev-session-route/listener.yaml create mode 100644 deploy/kubernetes/region/dev-session-route/networkpolicy.yaml create mode 100644 deploy/kubernetes/region/dev-session-route/service.yaml create mode 100644 docs/adr/0087-dev-session-route.md create mode 100644 tools/ci/deploy-policy-selftest.py diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index e5de8a5..01d9a37 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -90,11 +90,12 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* WORKDIR /source COPY deploy/ deploy/ -COPY tools/ci/deploy-policy-check.py tools/ci/ +COPY tools/ci/deploy-policy-check.py tools/ci/deploy-policy-selftest.py tools/ci/ COPY tools/ops/ tools/ops/ COPY tools/backup/ tools/backup/ COPY tools/ci/ops-lib-check.sh tools/ci/ -RUN python3 tools/ci/deploy-policy-check.py && bash tools/ci/ops-lib-check.sh +RUN python3 tools/ci/deploy-policy-check.py && python3 tools/ci/deploy-policy-selftest.py \ + && bash tools/ci/ops-lib-check.sh # M8.5: monitoring configuration and alert tests with the pinned images' own tools. COPY --from=quay.io/prometheus/prometheus@sha256:6976aa8a60fec930796ce5772b8d12da7a318a5daa8d40d69c5c7819a05eeed7 /bin/promtool /usr/local/bin/promtool COPY --from=quay.io/prometheus/alertmanager@sha256:e9733bafb1bdef9b00e25a21f8f99dc26a22224bf16641ad754d1649f4c3357a /bin/amtool /usr/local/bin/amtool diff --git a/deploy/kubernetes/region/base/deployment.yaml b/deploy/kubernetes/region/base/deployment.yaml index da190b7..0ade09d 100644 --- a/deploy/kubernetes/region/base/deployment.yaml +++ b/deploy/kubernetes/region/base/deployment.yaml @@ -59,13 +59,14 @@ spec: inputs=() if [ -n "$REGION_INPUTS" ]; then inputs=(--synthetic-inputs "$REGION_INPUTS"); fi # Dev sessions are a drill-time override too (tools/ops/session-drill.sh, - # ADR-0083): loopback only, reached by kubectl port-forward, never a Service. + # ADR-0083): loopback by default, reached by kubectl port-forward. Only the + # disabled dev-session-route component (ADR-0087) may bind the pod interface. # evo-server refuses any environment but dev and needs the synthetic carrier. # Consumed grants persist on the objects volume (outside the object # namespaces, ignored by backups) so a restarted pod refuses them. sessions=() if [ -n "$REGION_SESSION" ]; then - sessions=(--session-listen 127.0.0.1:7400 --session-cert /run/session/server.pem + sessions=(--session-listen "$REGION_SESSION_LISTEN" --session-cert /run/session/server.pem --session-key /run/session/server.key --session-grant-key /run/session/grant.key --session-grant-ledger /objects/dev-session-grants.v1 --session-environment "$REGION_SESSION") @@ -89,6 +90,9 @@ spec: - {name: REGION_RESERVE_MIB, value: "1024"} # M9.7 drill-only: "dev" enables the session listener; Git keeps it empty. - {name: REGION_SESSION, value: ""} + # Session listener address; the policy check allows only this loopback value + # outside region/dev-session-route. + - {name: REGION_SESSION_LISTEN, value: "127.0.0.1:7400"} ports: - {name: ops, containerPort: 9100, protocol: TCP} readinessProbe: diff --git a/deploy/kubernetes/region/dev-session-route/kustomization.yaml b/deploy/kubernetes/region/dev-session-route/kustomization.yaml new file mode 100644 index 0000000..debb834 --- /dev/null +++ b/deploy/kubernetes/region/dev-session-route/kustomization.yaml @@ -0,0 +1,16 @@ +# M9.8f dev session route for evolution-dev ONLY (ADR-0087). DISABLED: no +# kustomization includes this component yet. Enabling it (components: in +# region/dev/kustomization.yaml) is refused by tools/ci/deploy-policy-check.py until: +# - the REPLACE markers carry the allocated internal MetalLB VIP and the permitted +# development/test client CIDR(s); +# - the platform dev quota allows exactly one LoadBalancer Service; +# - evo-server enforces operator-opened session windows itself (--session-window). +# Route: void.aeternalisgames.com:7400 -> VIP -> Service -> evo-region pod:7400 (TLS 1.3 +# terminates in the pod). Raw TCP only: no Ingress, no TLS at nginx, no PROXY protocol. +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: + - service.yaml + - networkpolicy.yaml +patches: + - path: listener.yaml diff --git a/deploy/kubernetes/region/dev-session-route/listener.yaml b/deploy/kubernetes/region/dev-session-route/listener.yaml new file mode 100644 index 0000000..8865c52 --- /dev/null +++ b/deploy/kubernetes/region/dev-session-route/listener.yaml @@ -0,0 +1,15 @@ +# With the route, the opt-in session listener binds the pod interface and names its +# port; it still starts only when sessions are enabled for a window (ADR-0087). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: evo-region +spec: + template: + spec: + containers: + - name: evo-server + env: + - {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"} + ports: + - {name: session, containerPort: 7400, protocol: TCP} diff --git a/deploy/kubernetes/region/dev-session-route/networkpolicy.yaml b/deploy/kubernetes/region/dev-session-route/networkpolicy.yaml new file mode 100644 index 0000000..68da7a1 --- /dev/null +++ b/deploy/kubernetes/region/dev-session-route/networkpolicy.yaml @@ -0,0 +1,18 @@ +# Session ingress to the region pod: TCP 7400 from the permitted development/test +# client CIDR(s) only, which must equal the Service's loadBalancerSourceRanges. +# Everything else stays denied by the platform default-deny policy. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: evo-region-session-ingress +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: evo-region + policyTypes: [Ingress] + ingress: + - from: + - ipBlock: + cidr: REPLACE-dev-test-client-cidr + ports: + - {protocol: TCP, port: 7400} diff --git a/deploy/kubernetes/region/dev-session-route/service.yaml b/deploy/kubernetes/region/dev-session-route/service.yaml new file mode 100644 index 0000000..0c69680 --- /dev/null +++ b/deploy/kubernetes/region/dev-session-route/service.yaml @@ -0,0 +1,26 @@ +# The one LoadBalancer Service AEvolution may own (ADR-0087), evolution-dev only. +# - metallb.io/loadBalancerIPs pins the dedicated internal VIP that the internal DNS +# name void.aeternalisgames.com resolves to (never the control-plane address). +# - externalTrafficPolicy Local keeps client source addresses, so the NetworkPolicy's +# ipBlock (same CIDRs) is enforceable; new connections reach only a ready pod. +# - allocateLoadBalancerNodePorts false: no node ports (the quota forbids them). +# - loadBalancerSourceRanges duplicates the CIDR at the Service; the edge firewall is +# the third layer. TLS 1.3, the dev CA and single-use grants remain mandatory. +apiVersion: v1 +kind: Service +metadata: + name: evo-region-session + annotations: + metallb.io/loadBalancerIPs: REPLACE-dev-session-vip + labels: + app.kubernetes.io/name: evo-region +spec: + type: LoadBalancer + allocateLoadBalancerNodePorts: false + externalTrafficPolicy: Local + loadBalancerSourceRanges: + - REPLACE-dev-test-client-cidr + selector: + app.kubernetes.io/name: evo-region + ports: + - {name: session, port: 7400, targetPort: session, protocol: TCP} diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index a23044d..1aeb9af 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -458,7 +458,8 @@ terrain meshing or procedural creature animation is required. M9.8b adds bounded UI state, protected external input capture and bounded session event delivery. M9.8c integrates the launcher, legend, labelled inspector, click selection and session panel. M9.8d adds rebindable controls (ADR-0085). M9.8e -packages a local Windows x64 ZIP (ADR-0086). ADR-0084 records the toolkit scopes. +packages a local Windows x64 ZIP (ADR-0086). M9.8f designs the dev session route, +committed disabled (ADR-0087). ADR-0084 records the toolkit scopes. **M9.8 planning (2026-10-02):** user-confirmed Windows ZIP, temporary Gitea Actions artifacts, labelled engineering UI/launcher and control rebinding. See the diff --git a/docs/STATUS.md b/docs/STATUS.md index c30b1b7..db2e760 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -10,10 +10,13 @@ bounded errors, Retry/Cancel), legend, labelled inspector, click selection, sess panel with the intention outcome record, and visible notifications/dialog. M9.8d adds rebindable controls (ADR-0085) with conflict checks and a validated file. M9.8e builds a static-runtime Windows x64 ZIP locally with notices, identity, checksums and -credential guards (ADR-0086); Actions publication needs a Windows runner. Tests, +credential guards (ADR-0086); Actions publication needs a Windows runner. M9.8f +designs the dev session route (ADR-0087), committed disabled with fail-closed policy +tests; nothing was applied to the cluster. Tests, visual review and limits are recorded in engineering/VALIDATION.md. -Next: M9.8f dev TCP route design, then M9.8g qualification on the low-spec VM. +Next: server-enforced session windows and the route inputs (VIP, CIDR, DNS) to enable +ADR-0087, and M9.8g qualification on the low-spec VM. Windows packages will be built locally until a Windows runner exists. Testing is internal; 172.10.1.160 is the RKE2 control-plane address. VM qualification and the cluster restart drill remain outstanding, as agreed. See engineering/M9_8_DELIVERY_PLAN.md. diff --git a/docs/adr/0069-environment-isolation.md b/docs/adr/0069-environment-isolation.md index 52c2fe4..d5f3979 100644 --- a/docs/adr/0069-environment-isolation.md +++ b/docs/adr/0069-environment-isolation.md @@ -1,6 +1,8 @@ # ADR-0069: Environment isolation and World0 protection ## Status +ADR-0087 allows exactly one LoadBalancer Service, in evolution-dev only, for the dev +session route once enabled; node ports stay forbidden and stage/World0 are unchanged. Accepted for M8.2, 2026-09-29. It builds on the M8.1 inventory (infrastructure/INVENTORY.md), ADR-0020 (delivery), and OPERATIONS.md (World0 protection). Nothing here deploys a world; World0 launch stays gated by M8.8. diff --git a/docs/adr/0083-dev-region-sessions.md b/docs/adr/0083-dev-region-sessions.md index 81dcbb8..aa243fe 100644 --- a/docs/adr/0083-dev-region-sessions.md +++ b/docs/adr/0083-dev-region-sessions.md @@ -1,6 +1,8 @@ # ADR-0083: Dev region sessions, no-effect intentions and the session drill ## Status +Listener/Service rule narrowly superseded for evolution-dev by ADR-0087 (dev session +route), effective only when that route is enabled; until then this ADR applies as is. Accepted for M9.7 (engineering client to one authoritative **dev** region). Decisions confirmed with the maintainer on 2026-10-01: durable no-effect intentions, an explicitly empty organism presentation, local real-process evidence plus a dev diff --git a/docs/adr/0087-dev-session-route.md b/docs/adr/0087-dev-session-route.md new file mode 100644 index 0000000..ee09330 --- /dev/null +++ b/docs/adr/0087-dev-session-route.md @@ -0,0 +1,89 @@ +# ADR-0087: Dev session route and operator-opened session windows + +## Status +Accepted for M9.8f as design (2026-10-02), evolution-dev only. The route is committed +**disabled**; nothing was applied to the cluster. Enabling it is blocked by policy until +the inputs and server-side window enforcement below exist. Narrowly supersedes +ADR-0083 (loopback-only listener, never a Service) and ADR-0069 (no LoadBalancer +Services) for evolution-dev, and only once enabled. + +## Context +Internal testers need the native client (M9.8e ZIP) to reach the dev region without +`kubectl port-forward`. The cluster inventory (INVENTORY.md, 2026-09-29) shows RKE2's +ingress-nginx 1.12.1 DaemonSet serving 80/443 on all four node addresses (public sites +included), MetalLB 0.15.2 in FRR mode with one LoadBalancer (Gitea SSH), and an existing +HTTP Ingress for void.aeternalisgames.com in namespace `void`. The session protocol is +TLS 1.3 over raw TCP (ADR-0082), not HTTP. Sessions need synthetic carrier inputs, which +ADR-0073 keeps drill-time only so dev history does not grow continuously. + +## Decision (maintainer, 2026-10-02) +**Route.** `void.aeternalisgames.com:7400` resolves, in internal DNS, to a dedicated +internal MetalLB VIP (never automatically the control-plane address 172.10.1.160) that +fronts one LoadBalancer Service `evo-region-session` in evolution-dev, which forwards to +the region pod's port 7400. TLS 1.3 terminates in the region pod; its dev-CA server +certificate carries `void.aeternalisgames.com`, which the client verifies. No HTTP +Ingress, no TLS termination at nginx, no PROXY-protocol prefix. The Service uses +`externalTrafficPolicy: Local` (client addresses preserved), no node ports, a pinned +`metallb.io/loadBalancerIPs` annotation and `loadBalancerSourceRanges`. New connections +reach only a ready pod; after a replacement, clients reconnect with a fresh grant and +baseline (ADR-0083). + +**Access.** Development workstation/test subnet only, in three layers: the edge +firewall, the Service source ranges and a NetworkPolicy admitting TCP 7400 to the region +pod from exactly those CIDRs. TLS with the dev CA and single-use, short-lived grants stay +mandatory; grants remain bearer tokens. + +**Listener.** The base keeps the opt-in listener on loopback through +`REGION_SESSION_LISTEN=127.0.0.1:7400`. Only the route component +(`deploy/kubernetes/region/dev-session-route`) binds the pod interface and names the +session port. The listener still starts only when sessions are enabled. + +**Windows.** Sessions and synthetic inputs run only during an operator-opened test window +with automatic expiry (default 30 minutes); always-on simulation waits for an explicit +disposable-world lifecycle and retention. The endpoint (DNS, MetalLB Service) may stay +configured permanently. A small dev-only GitOps configuration holds an activation id, +the exact world/region identity and an absolute UTC expiry. Operations, via a manually +triggered Gitea workflow: Open (default 30 min), Extend (new explicit expiry), Close +(stop accepting sessions, close existing connections, stop synthetic inputs, drain +pending commits) and Status (window, expiry, active sessions, committed tick). +Expiry is enforced **inside evo-server**, not by a delayed cleanup job: +- a restart keeps the original absolute deadline, never a fresh duration; +- an expired configuration cannot reopen the listener; +- expiry stops synthetic generation as well as connections; +- a failed workflow cannot leave simulation running; cleanup may later restore the + closed Git configuration. Grant expiry alone does not stop the generator. + +## Enabling (all required; `tools/ci/deploy-policy-check.py` refuses otherwise) +1. The VIP (private, from the MetalLB pool) and permitted client CIDR(s) (private, no + wider than /16) replace the REPLACE markers; Service ranges equal the policy's. +2. `deploy/kubernetes/platform/dev` raises `services.loadbalancers` to exactly 1; + node ports stay 0; stage and World0 stay 0. +3. evo-server implements the session window (`--session-window`) as above, with tests + and an amendment to this ADR, and the base deployment passes it. +4. Only `region/dev` includes the component. Then: internal DNS for void to the VIP, the + edge firewall rule, and a server-side dry-run (quota accounting of the Local health + check port is to be confirmed there), a new dev server certificate for + void.aeternalisgames.com, and live connectivity recorded separately in VALIDATION.md. + +## Open points for the maintainer +- **Argo delivery.** The aevolution-dev Application has no automated sync, and the + policy forbids auto-sync outside the platform app (ADR-0069). A workflow commit alone + will not deploy; either a manual sync per window or a scoped dev exception must be + decided before implementing the workflow. CI holds no cluster credentials (ADR-0020), + so Status can report Git state; live fields need a defined read path. +- **void namespace.** void.aeternalisgames.com already has an HTTP Ingress (namespace + `void`). Pointing internal DNS at the VIP changes what internal clients reach for that + name; confirm or retire the existing route. +- Window limits: maximum window length and clock-skew tolerance. + +## Alternatives considered +- ingress-nginx tcp-services or SNI passthrough on 443: couples game sessions to the + shared web ingress (443 likely internet-facing) and needs controller changes. +- Port-forward only (ADR-0083): no tester-friendly endpoint. +- NodePort: opens a port on every node; forbidden by ADR-0069. +- Always-on sessions in dev: continuous synthetic history; rejected for now. + +## Consequences +One reviewed exception to "no LoadBalancer" for evolution-dev. The policy check and its +fail-closed self-test (`tools/ci/deploy-policy-selftest.py`) encode the route's shape and +the enabling preconditions. World0 and stage are untouched. diff --git a/docs/adr/README.md b/docs/adr/README.md index 192f057..cba56f5 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -97,5 +97,6 @@ design. No unadopted candidate is a link dependency of the proof. - [ADR-0083: Dev region sessions](0083-dev-region-sessions.md) - session server in evo-server, durable no-effect intentions on synthetic carriers, empty organism presentation, reconnect with fresh baselines, loopback-only drill-time dev deployment. - [ADR-0084: Engineering UI toolkit](0084-engineering-ui-toolkit.md) - Dear ImGui 1.92.9b pinned (M9.8a) and integrated as the engineering client overlay with toolkit-first input routing (M9.8c). +- [ADR-0087: Dev session route](0087-dev-session-route.md) - evolution-dev only: void.aeternalisgames.com:7400 via a dedicated internal MetalLB VIP to the region pod, TLS end to end, CIDR-limited, operator-opened windows enforced by evo-server; committed disabled with fail-closed policy tests. - [ADR-0086: Windows client package](0086-windows-client-package.md) - static-runtime Windows x64 ZIP with explicit allowlist, licence texts, build identity, SHA256 manifest, credential guards and deterministic layout; local packaging until a Windows runner exists. - [ADR-0085: Rebindable engineering controls](0085-rebindable-engineering-controls.md) - SDL-free action/HID-key bindings, conflict validation, versioned all-or-nothing controls file, binding capture before toolkit/world. diff --git a/docs/engineering/M9_8_DELIVERY_PLAN.md b/docs/engineering/M9_8_DELIVERY_PLAN.md index 216fda4..8c22a89 100644 --- a/docs/engineering/M9_8_DELIVERY_PLAN.md +++ b/docs/engineering/M9_8_DELIVERY_PLAN.md @@ -4,7 +4,9 @@ Status: M9.8a probe, M9.8b owned state/capture/event bounds, M9.8c launcher, legend, labelled inspector and click selection, and M9.8d rebindable controls complete, 2026-10-02. M9.8e local Windows ZIP complete (ADR-0086); its Gitea Actions publication waits for a Windows runner and its clean-VM run for M9.8g. -Remaining segments planned. +M9.8f route design, disabled manifests and fail-closed policy tests complete +(ADR-0087); enabling waits for the VIP, client CIDR, DNS, dev quota and server-enforced +session windows. M9.8g remains. No application UI or deployment completion is claimed. See ADR-0084 for the toolkit boundary. ## Confirmed scope @@ -64,9 +66,10 @@ All selected biological data remains a read-only presentation copy. - No Windows runner exists yet (confirmed 2026-10-02). Local builds are the initial path. Do not silently route a Windows package job to the existing Linux runner or assume an untested cross-compile. The packaging script can run on the existing Windows development host meanwhile. -- Confirmed control plane: 172.10.1.160. Still needed for direct service routing: - nginx implementation/configuration, exposed TCP port, - source restrictions and certificate name. The protocol is TLS/TCP, not HTTP; +- Confirmed control plane: 172.10.1.160. Direct routing decided in ADR-0087: dedicated + internal MetalLB VIP, TCP 7400, void.aeternalisgames.com, CIDR-limited. Still needed: + the VIP from the MetalLB pool, the permitted client CIDR, internal DNS, and decisions + on Argo delivery of session windows and the existing void HTTP Ingress. The protocol is TLS/TCP, not HTTP; a standard HTTP Ingress rule does not implement it. Port-forward remains supported. - VM specification and whether graphics acceleration is available. Hardware-specific performance acceptance cannot be declared before these are known. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index b92f829..82b8a23 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -3896,3 +3896,37 @@ Executed on the Windows development host (MSVC 19.44.35217, CMake 3.31.6): Not done: Gitea Actions publication (no Windows runner), a run on a clean Windows VM (M9.8g), code signing, hardware/DPI qualification, legal review of notices. + + +## 2026-10-02 — M9.8f dev session route design (ADR-0087) + +Maintainer decisions: dedicated internal MetalLB VIP, TCP 7400, +void.aeternalisgames.com, TLS ending in the region pod, development/test CIDR only +(firewall + Service source ranges + NetworkPolicy), no Ingress/nginx TLS/PROXY +protocol; operator-opened windows with server-enforced absolute expiry; scope limited +to design, manifests and policy tests. Nothing was applied to the cluster. + +Implemented: `REGION_SESSION_LISTEN` (base stays 127.0.0.1:7400); the disabled +component `deploy/kubernetes/region/dev-session-route` (LoadBalancer Service with +Local policy, no node ports, pinned VIP annotation and source ranges; CIDR-only +ingress policy on TCP 7400; listener patch to the pod interface) carrying REPLACE +markers; policy rules for the route's exact shape, LoadBalancer/NodePort and +session-port exclusivity, no session Ingress, evolution-dev-only placement, quota +limits, and enabling preconditions (no markers, dev quota of exactly one +LoadBalancer, server `--session-window`); patch files are now recognized generally. +`tools/ci/deploy-policy-selftest.py` copies the manifests and applies 20 unsafe changes. + +Executed (WSL Ubuntu, Python 3 with PyYAML 6.0.1): deploy policy check on the +repository, 70 objects in 68 files, 0 failures; self-test 20/20 refused, baseline +copy passes; ops-lib and ci-path fixtures pass; session-drill usage unchanged. +kustomize v5.4.3 (official release, SHA-256 checked against its checksums.txt, in a +scratch directory, not a project dependency) rendered region/dev as committed (listener +127.0.0.1:7400, no session Service) and a temporary enabled copy with example private +values (LoadBalancer evo-region-session and the CIDR policy in evolution-dev, listener +0.0.0.0:7400, session container port merged). + +Not run: the ci-deploy Docker stage (no Docker in this WSL distribution; its commands +were run directly), any cluster/server dry-run, MetalLB allocation, DNS, firewall or live +connectivity. Server-enforced session windows and the Gitea window workflow are not +implemented; Argo delivery for windows and the existing void HTTP Ingress are open +maintainer decisions recorded in ADR-0087. diff --git a/docs/networking/DEV_SESSION_OPERATIONS.md b/docs/networking/DEV_SESSION_OPERATIONS.md index 268e671..a75b9e9 100644 --- a/docs/networking/DEV_SESSION_OPERATIONS.md +++ b/docs/networking/DEV_SESSION_OPERATIONS.md @@ -82,3 +82,12 @@ exposed through a Service, NodePort or Ingress. A grant bundle lasts at most one and each connection consumes one grant. WASD sends intentions; the small chip under the intention dot turns white (admitted), green (durable, no effect) or red/grey (rejected or outcome unknown). Console lines report every session event. + +## 6. Planned direct route (M9.8f, disabled) + +ADR-0087 designs `void.aeternalisgames.com:7400` through a dedicated internal MetalLB +VIP to the region pod (TLS end to end, development/test CIDR only), opened only in +operator windows that evo-server itself expires. The manifests are in +`deploy/kubernetes/region/dev-session-route` and are not included anywhere; the policy +check refuses to enable them until the VIP, client CIDR, dev quota and the server-side +`--session-window` exist. Until then use the port-forward procedures above. diff --git a/tools/ci/ci-paths.sh b/tools/ci/ci-paths.sh index ad90495..85805c2 100755 --- a/tools/ci/ci-paths.sh +++ b/tools/ci/ci-paths.sh @@ -46,7 +46,7 @@ case "$suite" in pattern="$common|$native|tests/|$postgres_tools|third_party/mbedtls/|tools/prepare-mbedtls\.cmake|tools/mbedtls-pin\.cmake|deploy/docker/Dockerfile" ;; deploy) - pattern="$common|deploy/|tools/ops/|tools/backup/|tools/ci/deploy-policy-check\.py|tools/ci/monitoring-check\.sh|tools/ci/ops-lib-check\.sh" + pattern="$common|deploy/|tools/ops/|tools/backup/|tools/ci/deploy-policy-(check|selftest)\.py|tools/ci/monitoring-check\.sh|tools/ci/ops-lib-check\.sh" ;; server-image) pattern="$common|$native|migrations/|tools/prepare-postgres\.sh|tools/ci/server-image-smoke\.sh|deploy/docker/Dockerfile" diff --git a/tools/ci/deploy-policy-check.py b/tools/ci/deploy-policy-check.py index 1931a3c..cce664b 100644 --- a/tools/ci/deploy-policy-check.py +++ b/tools/ci/deploy-policy-check.py @@ -24,8 +24,14 @@ environment-isolation or World0-protection invariant is broken: required; upgrade-rehearsal flags never appear in a deployed workload; - the Pods the backup and barrier scripts generate (rendered with --print-manifest) keep the database-tool security contract: non-root 10001 with fsGroup 10001 so - the 0440 Secret files are readable, no token, read-only root, no escalation. + the 0440 Secret files are readable, no token, read-only root, no escalation; + - the dev session route (ADR-0087): loopback listener everywhere except the + evolution-dev route component; that component is the only LoadBalancer and the + only Service on the session port, narrowly shaped, and it cannot be enabled while + markers are unresolved or before server-enforced session windows exist. """ +import ipaddress +import re import os import shutil import subprocess @@ -55,7 +61,15 @@ for base, _, files in os.walk(os.path.join(ROOT, 'deploy')): path = os.path.join(base, name) manifests[os.path.relpath(path, ROOT).replace(os.sep, '/')] = documents(path) objects = [(p, d) for p, ds in manifests.items() for d in ds - if 'kind' in d and d['kind'] != 'Kustomization'] + if 'kind' in d and d['kind'] not in ('Kustomization', 'Component')] +# Files used as kustomize patches hold partial objects (checked only for what they set). +patch_files = set() +for p, ds in manifests.items(): + if p.endswith('kustomization.yaml') and ds: + for entry in ds[0].get('patches', []): + target = entry.get('path') if isinstance(entry, dict) else entry + if target: + patch_files.add(os.path.normpath(os.path.join(os.path.dirname(p), target)).replace(os.sep, '/')) # Secrets and service-account tokens. for path, doc in objects: @@ -84,6 +98,21 @@ check(dns.get('policyTypes') == ['Egress'] and set(dns_ports) == {53} and len(dn quota = base.get('ResourceQuota/aevolution-quota', {}).get('spec', {}).get('hard', {}) check(quota.get('services.loadbalancers') == '0' and quota.get('services.nodeports') == '0', 'quota must forbid LoadBalancer and NodePort services') +# ADR-0087: only the dev overlay may raise LoadBalancers, to exactly one, and only +# together with the session route; node ports stay forbidden everywhere. +ROUTE = 'deploy/kubernetes/region/dev-session-route/' +route_users = sorted(p for p, ds in manifests.items() if p.endswith('kustomization.yaml') and ds + and any('dev-session-route' in str(e) for e in ds[0].get('resources', []) + ds[0].get('components', []))) +route_enabled = route_users == ['deploy/kubernetes/region/dev/kustomization.yaml'] +check(route_users in ([], ['deploy/kubernetes/region/dev/kustomization.yaml']), + f'only region/dev may include the dev session route: {route_users}') +for path, doc in objects: + if doc['kind'] == 'ResourceQuota' and path != f'{platform}/base/quota.yaml': + hard = doc.get('spec', {}).get('hard', {}) + check(hard.get('services.nodeports', '0') == '0', f'{path}: node ports stay forbidden') + allowed = ('0', '1') if path.startswith(f'{platform}/dev/') and route_enabled else ('0',) + check(hard.get('services.loadbalancers', '0') in allowed, + f'{path}: LoadBalancers only in evolution-dev, at most one, with the session route') for env in ('dev', 'stage', 'world0'): k = f'{platform}/{env}/kustomization.yaml' ns_path = f'{platform}/{env}/namespace.yaml' @@ -212,7 +241,7 @@ for path, doc in objects: for path, doc in objects: # Overlay patches (region-values.yaml) are partial objects; the base is complete. if (doc['kind'] == 'Deployment' and doc['metadata'].get('name') == 'evo-region' - and not path.endswith('region-values.yaml')): + and path not in patch_files): spec = doc['spec'] pod = spec['template']['spec'] container = pod['containers'][0] @@ -238,22 +267,101 @@ for path, doc in objects: # M9.7 (ADR-0083): dev sessions are a drill-time override as well. if variable.get('name') == 'REGION_SESSION': check(variable.get('value') == '', f'{path}: dev sessions are never enabled in Git') -# M9.7 (ADR-0083): the session listener is loopback-only (kubectl port-forward), and its -# Secret is optional, 0440 and never exposed through a Service port. +# M9.7 (ADR-0083) and M9.8f (ADR-0087): the session listener is loopback (kubectl +# port-forward) everywhere except the evolution-dev route component; its Secret is +# optional and 0440; only that component's Service exposes the session port. for path, doc in objects: - if path.startswith('deploy/kubernetes/region/') and doc['kind'] == 'Deployment' and not path.endswith('region-values.yaml'): + if path.startswith('deploy/kubernetes/region/') and doc['kind'] == 'Deployment': pod = doc['spec']['template']['spec'] + for container in pod.get('containers', []): + for variable in container.get('env', []): + if variable.get('name') == 'REGION_SESSION_LISTEN': + expected = '0.0.0.0:7400' if path.startswith(ROUTE) else '127.0.0.1:7400' + check(variable.get('value') == expected, + f'{path}: REGION_SESSION_LISTEN must be {expected} here') + if path in patch_files: + continue script = ' '.join(pod['containers'][0].get('args', [])) - check('--session-listen 127.0.0.1:' in script and '0.0.0.0:7400' not in script, - f'{path}: the session listener binds loopback only') + check('--session-listen "$REGION_SESSION_LISTEN"' in script and '0.0.0.0' not in script.replace('--ops-listen 0.0.0.0:9100', ''), + f'{path}: the session listener address comes only from REGION_SESSION_LISTEN') for volume in pod.get('volumes', []): if volume.get('secret', {}).get('secretName') == 'evolution-dev-session': check(volume['secret'].get('optional') is True and volume['secret'].get('defaultMode') == 0o440, f'{path}: the dev session Secret is optional and 0440') +route_ranges = [] +for path, doc in objects: if doc['kind'] == 'Service': - for port in doc['spec'].get('ports', []): - check(port.get('port') != 7400 and port.get('targetPort') not in (7400, 'session'), - f'{path}: the dev session port is never exposed by a Service') + spec = doc['spec'] + session_port = any(port.get('port') == 7400 or port.get('targetPort') in (7400, 'session') + for port in spec.get('ports', [])) + is_route = path == ROUTE + 'service.yaml' and doc['metadata'].get('name') == 'evo-region-session' + if not is_route: + check(not session_port, f'{path}: only the dev session route Service may expose the session port') + check(spec.get('type', 'ClusterIP') == 'ClusterIP', + f'{path}: {spec.get("type")} Services are forbidden (ADR-0069)') + continue + route_ranges = spec.get('loadBalancerSourceRanges', []) + check(spec.get('type') == 'LoadBalancer' and spec.get('externalTrafficPolicy') == 'Local' + and spec.get('allocateLoadBalancerNodePorts') is False, + f'{path}: the route is a LoadBalancer with Local traffic policy and no node ports') + check(spec.get('ports') == [{'name': 'session', 'port': 7400, 'targetPort': 'session', 'protocol': 'TCP'}], + f'{path}: the route exposes exactly TCP 7400') + check(spec.get('selector') == {'app.kubernetes.io/name': 'evo-region'}, + f'{path}: the route selects only the region pod') + annotations = doc['metadata'].get('annotations', {}) + check(set(annotations) == {'metallb.io/loadBalancerIPs'}, + f'{path}: only a pinned MetalLB address annotation (no PROXY protocol or sharing)') + vip = annotations.get('metallb.io/loadBalancerIPs', '') + check(len(route_ranges) >= 1, f'{path}: loadBalancerSourceRanges must name the permitted CIDRs') + if not vip.startswith('REPLACE'): + try: + check(ipaddress.ip_address(vip).is_private, f'{path}: the VIP must be an internal address') + except ValueError: + check(False, f'{path}: the VIP must be one IP address') + for cidr in route_ranges: + if cidr.startswith('REPLACE'): + continue + try: + network = ipaddress.ip_network(cidr) + check(network.is_private and network.prefixlen >= (16 if network.version == 4 else 48), + f'{path}: {cidr} must be a private range no wider than /16 (IPv6 /48)') + except ValueError: + check(False, f'{path}: {cidr} is not a CIDR') + if doc['kind'] == 'Ingress': + text = yaml.safe_dump(doc) + check('7400' not in text and 'session' not in text, + f'{path}: the dev session is raw TCP, never an Ingress (ADR-0087)') +policies = [(p, d) for p, d in objects if p.startswith(ROUTE) and d['kind'] == 'NetworkPolicy'] +check(len(policies) == 1, f'{ROUTE}: exactly one session ingress policy') +for path, doc in policies: + spec = doc['spec'] + rules = spec.get('ingress', []) + blocks = [peer.get('ipBlock', {}) for rule in rules for peer in rule.get('from', [])] + check(spec.get('podSelector') == {'matchLabels': {'app.kubernetes.io/name': 'evo-region'}} + and spec.get('policyTypes') == ['Ingress'] and len(rules) == 1 + and rules[0].get('ports') == [{'protocol': 'TCP', 'port': 7400}], + f'{path}: the session policy admits only TCP 7400 to the region pod') + check(all(set(b) == {'cidr'} for b in blocks) and len(blocks) == len(rules[0].get('from', [])) + and sorted(b.get('cidr') for b in blocks) == sorted(route_ranges), + f'{path}: the session policy admits exactly the Service source ranges') +for path, doc in objects: + if path.startswith(ROUTE): + check(doc.get('metadata', {}).get('namespace') in (None, 'evolution-dev'), + f'{path}: the session route belongs to evolution-dev only') +# Enabling the route fails closed until its values are real and the server enforces +# operator-opened windows itself (expiry survives restarts; ADR-0087). +if route_enabled: + for name in os.listdir(os.path.join(ROOT, ROUTE)): + with open(os.path.join(ROOT, ROUTE, name), encoding='utf-8') as handle: + check('REPLACE' not in handle.read(), f'{ROUTE}{name}: unresolved REPLACE marker with the route enabled') + base_script = ' '.join(' '.join(c.get('args', [])) for p, d in objects + if p == 'deploy/kubernetes/region/base/deployment.yaml' and d['kind'] == 'Deployment' + for c in d['spec']['template']['spec']['containers']) + check('--session-window' in base_script, + 'the dev session route needs server-enforced session windows (--session-window, ADR-0087)') + dev_quota = [d.get('spec', {}).get('hard', {}).get('services.loadbalancers') for p, d in objects + if d['kind'] == 'ResourceQuota' and p.startswith(f'{platform}/dev/')] + check(dev_quota == ['1'], 'the dev quota must allow exactly one LoadBalancer for the route') for env in ('dev', 'stage'): overlay = f'deploy/kubernetes/region/{env}/' images = manifests.get(overlay + 'kustomization.yaml', [{}])[0].get('images', []) @@ -311,7 +419,7 @@ for directory, _, files in os.walk(os.path.join(ROOT, 'deploy', 'kubernetes')): # M8.9 scheduling (ADR-0076). Node-local volumes pin their pods: added CPU does not # make the objects volume portable. Compute is a preference so one node still works. for path, doc in objects: - if doc['kind'] not in workload_kinds or path.endswith('region-values.yaml'): + if doc['kind'] not in workload_kinds or path in patch_files: continue spec = doc.get('spec', {}) pod = spec.get('jobTemplate', {}).get('spec', {}).get('template', {}).get('spec') \ diff --git a/tools/ci/deploy-policy-selftest.py b/tools/ci/deploy-policy-selftest.py new file mode 100644 index 0000000..9fd1d7c --- /dev/null +++ b/tools/ci/deploy-policy-selftest.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""M9.8f fail-closed tests for deploy-policy-check.py (ADR-0087). Needs Python 3 and +PyYAML only. Copies the manifests and the tools the check runs into a temporary tree, +applies one unsafe change at a time and requires the check to fail with the expected +message; the unmodified copy must pass. +""" +import os +import re +import shutil +import subprocess +import sys +import tempfile + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) +ROUTE = 'deploy/kubernetes/region/dev-session-route/' +DEV = 'deploy/kubernetes/region/dev/kustomization.yaml' +REAL = [(ROUTE + 'service.yaml', 'REPLACE-dev-session-vip', '10.20.30.40'), + (ROUTE + 'service.yaml', 'REPLACE-dev-test-client-cidr', '10.20.0.0/24'), + (ROUTE + 'networkpolicy.yaml', 'REPLACE-dev-test-client-cidr', '10.20.0.0/24')] + + +def edit(tree, path, old, new, count=1): + full = os.path.join(tree, path) + with open(full, encoding='utf-8') as f: + text = f.read() + if old not in text: + raise SystemExit(f'selftest setup: {old!r} not in {path}') + with open(full, 'w', encoding='utf-8') as f: + f.write(text.replace(old, new, count)) + + +def append(tree, path, text): + with open(os.path.join(tree, path), 'a', encoding='utf-8') as f: + f.write(text) + + +def write(tree, path, text): + full = os.path.join(tree, path) + os.makedirs(os.path.dirname(full), exist_ok=True) + with open(full, 'w', encoding='utf-8') as f: + f.write(text) + + +def enable(tree): + append(tree, DEV, 'components:\n - ../dev-session-route\n') + + +def resolve(tree): + for path, old, new in REAL: + edit(tree, path, old, new) + + +def quota_one(tree): + write(tree, 'deploy/kubernetes/platform/dev/quota-session.yaml', + 'apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: aevolution-quota\n' + 'spec:\n hard:\n services.loadbalancers: "1"\n services.nodeports: "0"\n') + + +CASES = [ + ('enabling with unresolved markers', lambda t: enable(t), 'unresolved REPLACE marker'), + ('enabling before server-enforced windows', lambda t: (enable(t), resolve(t), quota_one(t)), + 'server-enforced session windows'), + ('enabling without the dev quota', lambda t: (enable(t), resolve(t)), 'exactly one LoadBalancer'), + ('route included by stage', lambda t: append(t, 'deploy/kubernetes/region/stage/kustomization.yaml', + 'components:\n - ../dev-session-route\n'), + 'only region/dev may include'), + ('stage quota allowing a LoadBalancer', lambda t: write( + t, 'deploy/kubernetes/platform/stage/quota-lb.yaml', + 'apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: aevolution-quota\n' + 'spec:\n hard:\n services.loadbalancers: "1"\n'), 'LoadBalancers only in evolution-dev'), + ('world-open source range', lambda t: (edit(t, ROUTE + 'service.yaml', 'REPLACE-dev-test-client-cidr', '0.0.0.0/0'), + edit(t, ROUTE + 'networkpolicy.yaml', 'REPLACE-dev-test-client-cidr', '0.0.0.0/0')), + 'private range'), + ('public VIP', lambda t: edit(t, ROUTE + 'service.yaml', 'REPLACE-dev-session-vip', '8.8.8.8'), 'internal address'), + ('policy CIDR differs from the Service', lambda t: edit(t, ROUTE + 'networkpolicy.yaml', + 'REPLACE-dev-test-client-cidr', '10.99.0.0/24'), + 'exactly the Service source ranges'), + ('policy admits another port', lambda t: edit(t, ROUTE + 'networkpolicy.yaml', 'port: 7400', 'port: 7401'), + 'only TCP 7400'), + ('cluster traffic policy', lambda t: edit(t, ROUTE + 'service.yaml', 'externalTrafficPolicy: Local', + 'externalTrafficPolicy: Cluster'), 'Local traffic policy'), + ('node ports allocated', lambda t: edit(t, ROUTE + 'service.yaml', 'allocateLoadBalancerNodePorts: false', + 'allocateLoadBalancerNodePorts: true'), 'no node ports'), + ('PROXY protocol annotation', lambda t: edit(t, ROUTE + 'service.yaml', ' metallb.io/loadBalancerIPs:', + ' example.io/proxy-protocol: "true"\n metallb.io/loadBalancerIPs:'), + 'no PROXY protocol'), + ('extra route port', lambda t: append(t, ROUTE + 'service.yaml', + ' - {name: ops, port: 9100, targetPort: ops, protocol: TCP}\n'), + 'exactly TCP 7400'), + ('another LoadBalancer', lambda t: edit(t, 'deploy/kubernetes/region/base/service.yaml', 'type: ClusterIP', + 'type: LoadBalancer'), 'Services are forbidden'), + ('session port on another Service', lambda t: append(t, 'deploy/kubernetes/region/base/service.yaml', + ' - {name: session, port: 7400, targetPort: 7400, protocol: TCP}\n'), + 'only the dev session route Service'), + ('session Ingress', lambda t: write( + t, 'deploy/kubernetes/region/dev/ingress.yaml', + 'apiVersion: networking.k8s.io/v1\nkind: Ingress\nmetadata:\n name: session\nspec:\n rules:\n' + ' - host: void.aeternalisgames.com\n http:\n paths:\n - path: /\n' + ' pathType: Prefix\n backend:\n service:\n' + ' name: evo-region-session\n port: {number: 7400}\n'), 'never an Ingress'), + ('base listener on the pod interface', lambda t: edit( + t, 'deploy/kubernetes/region/base/deployment.yaml', 'value: "127.0.0.1:7400"', 'value: "0.0.0.0:7400"'), + 'REGION_SESSION_LISTEN must be 127.0.0.1:7400'), + ('hard-coded listener address', lambda t: edit( + t, 'deploy/kubernetes/region/base/deployment.yaml', '--session-listen "$REGION_SESSION_LISTEN"', + '--session-listen 0.0.0.0:7400'), 'only from REGION_SESSION_LISTEN'), + ('route listener patch elsewhere', lambda t: edit( + t, 'deploy/kubernetes/region/dev/region-values.yaml', '- {name: REGION_ID, value: "1"}', + '- {name: REGION_ID, value: "1"}\n - {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"}'), + 'REGION_SESSION_LISTEN must be 127.0.0.1:7400'), + ('route object in another namespace', lambda t: edit( + t, ROUTE + 'networkpolicy.yaml', ' name: evo-region-session-ingress', + ' name: evo-region-session-ingress\n namespace: evolution-stage'), 'evolution-dev only'), +] + + +def copy_tree(target): + for directory in ('deploy', 'tools/ops', 'tools/backup'): + shutil.copytree(os.path.join(ROOT, directory), os.path.join(target, directory)) + os.makedirs(os.path.join(target, 'tools', 'ci')) + shutil.copy(os.path.join(ROOT, 'tools', 'ci', 'deploy-policy-check.py'), os.path.join(target, 'tools', 'ci')) + + +def run(tree): + result = subprocess.run([sys.executable, os.path.join(tree, 'tools', 'ci', 'deploy-policy-check.py')], + capture_output=True, text=True) + return result.returncode, result.stdout + result.stderr + + +failures = [] +with tempfile.TemporaryDirectory() as clean: + copy_tree(clean) + code, output = run(clean) + if code != 0: + failures.append(f'baseline copy must pass:\n{output}') +for name, mutate, expected in CASES: + with tempfile.TemporaryDirectory() as tree: + copy_tree(tree) + mutate(tree) + code, output = run(tree) + refused = code != 0 and re.search(re.escape(expected), output) + print(f'{"refused" if refused else "NOT REFUSED"}: {name}') + if not refused: + failures.append(f'{name}: expected failure mentioning {expected!r}, got rc={code}\n{output}') +for message in failures: + print('FAIL', message) +print(f'deploy policy selftest: {len(CASES)} cases, {len(failures)} failures') +sys.exit(1 if failures else 0)