From fad79973e27406d80f972eb29552ffe725824030 Mon Sep 17 00:00:00 2001 From: dingo Date: Mon, 19 Oct 2026 15:08:00 -0400 Subject: [PATCH] fix(deploy): constrain DEV workloads to worker nodes --- deploy/README.md | 4 ++ deploy/kubernetes/dev/kustomization.yaml | 2 + .../region/dev-session-route/listener.yaml | 2 +- .../kubernetes/region/dev/kustomization.yaml | 1 + .../kubernetes/stores/dev/kustomization.yaml | 2 + .../worker-only-dev/kustomization.yaml | 38 ++++++++++++++++ docs/STATUS.md | 13 ++++++ docs/adr/0170-dev-worker-placement.md | 43 +++++++++++++++++++ docs/adr/README.md | 2 + docs/engineering/M12_DEV_WORKER_PLACEMENT.md | 36 ++++++++++++++++ docs/engineering/M12_REMAINING_GATES.md | 5 +++ docs/engineering/VALIDATION.md | 31 +++++++++++++ tools/ci/deploy-policy-check.py | 18 ++++++++ tools/ci/deploy-policy-selftest.py | 28 +++++++++++- 14 files changed, 223 insertions(+), 2 deletions(-) create mode 100644 deploy/kubernetes/worker-only-dev/kustomization.yaml create mode 100644 docs/adr/0170-dev-worker-placement.md create mode 100644 docs/engineering/M12_DEV_WORKER_PLACEMENT.md diff --git a/deploy/README.md b/deploy/README.md index 39a5492..5edcb37 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -10,6 +10,10 @@ treated as a request to replace the current automation. DEV workloads use `evolution-dev`, run on worker nodes rather than the control plane, and retain the existing PostgreSQL server, network structure and URLs. +The full DEV and standalone region/store overlays enforce both standard +control-plane role exclusions through `worker-only-dev`; rendered policy checks +every required alternative and refuses a direct `nodeName` bypass. See +[placement evidence](../docs/engineering/M12_DEV_WORKER_PLACEMENT.md). Qualification data must remain isolated from existing worlds. Immediately before each push refresh `REGION_SESSION_WINDOW` in `kubernetes/region/dev-session-route/listener.yaml` with a unique activation and diff --git a/deploy/kubernetes/dev/kustomization.yaml b/deploy/kubernetes/dev/kustomization.yaml index 17a85a1..cc86696 100644 --- a/deploy/kubernetes/dev/kustomization.yaml +++ b/deploy/kubernetes/dev/kustomization.yaml @@ -1,6 +1,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: evolution-dev +components: + - ../worker-only-dev resources: - serviceaccount.yaml - networkpolicy.yaml diff --git a/deploy/kubernetes/region/dev-session-route/listener.yaml b/deploy/kubernetes/region/dev-session-route/listener.yaml index aec1ca9..b00c67f 100644 --- a/deploy/kubernetes/region/dev-session-route/listener.yaml +++ b/deploy/kubernetes/region/dev-session-route/listener.yaml @@ -12,7 +12,7 @@ spec: env: - {name: REGION_SESSION, value: "dev"} - {name: REGION_SESSION_LISTEN, value: "0.0.0.0:7400"} - - {name: REGION_SESSION_WINDOW, value: "dev-20261019T190609-a14a1852:1792451169"} + - {name: REGION_SESSION_WINDOW, value: "dev-20261019T190730-815046b1:1792451250"} - {name: REGION_M10_MAX_GRANT_LIFETIME, value: "14400"} ports: # QUIC (ADR-0090): UDP only; nothing listens on TCP 7400. diff --git a/deploy/kubernetes/region/dev/kustomization.yaml b/deploy/kubernetes/region/dev/kustomization.yaml index 4db3442..3ebe04d 100644 --- a/deploy/kubernetes/region/dev/kustomization.yaml +++ b/deploy/kubernetes/region/dev/kustomization.yaml @@ -9,6 +9,7 @@ resources: components: - ../dev-session-route - ../../grant-broker-dev + - ../../worker-only-dev images: - name: aevolution-server newName: registry.aeternalisgames.com/aeternalis/aevolution-server diff --git a/deploy/kubernetes/stores/dev/kustomization.yaml b/deploy/kubernetes/stores/dev/kustomization.yaml index 20937ce..c65bbc1 100644 --- a/deploy/kubernetes/stores/dev/kustomization.yaml +++ b/deploy/kubernetes/stores/dev/kustomization.yaml @@ -3,6 +3,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: evolution-dev +components: + - ../../worker-only-dev resources: - ../base images: diff --git a/deploy/kubernetes/worker-only-dev/kustomization.yaml b/deploy/kubernetes/worker-only-dev/kustomization.yaml new file mode 100644 index 0000000..4a1f03e --- /dev/null +++ b/deploy/kubernetes/worker-only-dev/kustomization.yaml @@ -0,0 +1,38 @@ +# Maintainer requirement: DEV workloads must never schedule on the control plane. +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +patches: + - target: + kind: Deployment + patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: worker-only + spec: + template: + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - {key: node-role.kubernetes.io/control-plane, operator: DoesNotExist} + - {key: node-role.kubernetes.io/master, operator: DoesNotExist} + - target: + kind: Job + patch: |- + apiVersion: batch/v1 + kind: Job + metadata: + name: worker-only + spec: + template: + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - {key: node-role.kubernetes.io/control-plane, operator: DoesNotExist} + - {key: node-role.kubernetes.io/master, operator: DoesNotExist} diff --git a/docs/STATUS.md b/docs/STATUS.md index d49f86f..b39699f 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1,5 +1,18 @@ # Project status and next steps +## Explicit DEV worker placement - 2026-10-19 + +[ADR-0170](adr/0170-dev-worker-placement.md) implements the maintainer's worker +placement requirement in the full DEV application and standalone region/store +overlays. Required affinity excludes both current and legacy control-plane role +labels. Existing store selectors, volumes, PostgreSQL endpoints and URLs remain. +Rendered checks reject an unrestricted affinity alternative or `nodeName` bypass. + +All 17 overlays render with zero policy failures; all 50 adversarial policy cases +pass. [Evidence](engineering/M12_DEV_WORKER_PLACEMENT.md) distinguishes offline +policy from actual running placement. Delivery uses Git Actions, Harbor, ArgoCD +and Kubernetes. Target-host drills and the remaining M12 gates remain open. + ## Retained-plan census and verified history - 2026-10-19 [ADR-0169](adr/0169-retained-plan-census-observations.md) adds raw owner-8 census diff --git a/docs/adr/0170-dev-worker-placement.md b/docs/adr/0170-dev-worker-placement.md new file mode 100644 index 0000000..121ad1e --- /dev/null +++ b/docs/adr/0170-dev-worker-placement.md @@ -0,0 +1,43 @@ +# ADR-0170: Explicit DEV worker placement + +## Status and scope + +Accepted operational clarification, 2026-10-19, from the maintainer's requirement +that DEV workloads run in `evolution-dev` and never on the control plane. Extends +ADR-0076/0088 without changing biological rules, node labels, storage placement, +database endpoints, credentials, URLs or World0 policy. + +## Decision + +Apply a reusable Kustomize component to the full DEV application and its +standalone region/store overlays. Every current DEV Deployment and Job requires +nodes without either `node-role.kubernetes.io/control-plane` or the older +`node-role.kubernetes.io/master` label. Both exclusions belong to the same +required selector term. Existing store selectors, volumes and resource settings +remain. No control-plane toleration or node selector override is introduced. + +The rendered deployment policy requires both exclusions in **every** required +node-affinity term, since terms are alternatives. It also refuses explicit +`nodeName`, which bypasses scheduler selection. Future DEV workload kinds must +provide the same restriction before passing the policy. Shared bases, stage, +monitoring and operator-only inventory examples keep their existing scope. + +## Alternatives and consequences + +Relying only on common control-plane taints is weaker than the explicit +requirement: taints are external cluster configuration. Replacing the store +selector with a particular hostname would couple the application to an unknown +node and could strand its node-local volume. Required negative role affinity +preserves current placement constraints while preventing scheduling onto a +correctly role-labelled control plane. + +If no eligible worker owns the existing volume, pods remain Pending. Do not +relax the rule or move retained storage merely to obtain readiness. Cluster role +labels remain an infrastructure responsibility; offline rendering cannot verify +their truth or prove a running pod's placement. + +Delivery uses the existing Git Actions -> Harbor -> ArgoCD -> Kubernetes flow. +Qualify effective rendered objects and adversarial patches (removed affinity, +missing legacy-role exclusion, unrestricted OR term and direct `nodeName`). +Record actual rollout, placement and target-host measurements separately. No +direct deployment, live reachability or M12 completion is inferred from a push. diff --git a/docs/adr/README.md b/docs/adr/README.md index 75268bf..4e633f4 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -1,5 +1,7 @@ # Architecture decision records +- [ADR-0170: Explicit DEV worker placement](0170-dev-worker-placement.md) - required control-plane/master exclusion for DEV workloads in full and standalone overlays, with rendered-policy regression checks; no live-placement claim. + - [ADR-0169: Retained-plan census observations](0169-retained-plan-census-observations.md) - raw owner-8 projection 2 separates expressed resident stock from living offspring-plan references and dead-plan history; verified history and registered capture retain exact boundaries. - [ADR-0168: Durable raw retained-plan admission](0168-durable-retained-plan-admission.md) - schema 55, explicit raw bundle-11 -> 12 fenced upgrades and trace-4 mutation/birth publication with exact recovery; no automatic opening or population admission. diff --git a/docs/engineering/M12_DEV_WORKER_PLACEMENT.md b/docs/engineering/M12_DEV_WORKER_PLACEMENT.md new file mode 100644 index 0000000..e5c56d0 --- /dev/null +++ b/docs/engineering/M12_DEV_WORKER_PLACEMENT.md @@ -0,0 +1,36 @@ +# DEV worker placement qualification + +2026-10-19, [ADR-0170](../adr/0170-dev-worker-placement.md). The maintainer +confirmed Git Actions -> Harbor -> ArgoCD -> Kubernetes delivery from `deploy/`, +with DEV workloads in `evolution-dev`, on workers, and the existing PostgreSQL +server/network/URLs. This segment makes the placement requirement explicit in +the delivered manifests; it does not replace the configured pipeline. + +The full DEV overlay and standalone region/store overlays apply the same +component. All current DEV Deployments and Jobs exclude nodes carrying either +standard control-plane role label. Existing store selectors and volume claims +remain, so a missing eligible worker leaves the workload unscheduled rather +than silently using the control plane. The component does not allocate/move +storage, change database endpoints or grant credentials. + +Executed locally with the repository-pinned Kustomize v5.8.1: + +- Rendered deployment policy: **75 objects in 75 files, 138 rendered objects in + 17 overlays, zero failures**. +- Deployment policy self-test: **50 cases, zero failures**. The four added + cases remove required affinity, remove the older master-role exclusion, + append an unrestricted OR term and set direct `nodeName`. All are refused. + Existing namespace, route, window, port and NetworkPolicy regressions remain. +- The route self-test helper now preserves existing components when enabling + the route, rather than appending a duplicate YAML key that drops other rules. + +The policy checks effective rendered objects, not only patch text. Every required +selector alternative must exclude both roles. It also refuses a `nodeName` +bypass for every DEV workload kind. Future workload kinds need equivalent +affinity before entering the delivery path; no new placeholder workload is added. + +This is offline policy evidence. Actual node role labels, running pod placement, +image rollout, readiness, multi-host failures and M12 target-host performance +need results from the delivered environment. A Git push or local policy pass +alone cannot supply those results. No direct cluster operation or World0 +deployment/reset/pruning occurred. diff --git a/docs/engineering/M12_REMAINING_GATES.md b/docs/engineering/M12_REMAINING_GATES.md index 33d0b7c..a3f7409 100644 --- a/docs/engineering/M12_REMAINING_GATES.md +++ b/docs/engineering/M12_REMAINING_GATES.md @@ -36,6 +36,11 @@ on worker nodes, using the existing PostgreSQL server/network endpoints and isolated test data. A successful local check or Git push alone is not evidence that a target-host fault drill or deployed workload passed. Direct workstation Kubernetes access is not a prerequisite for this delivery flow. +[ADR-0170](../adr/0170-dev-worker-placement.md) now enforces required worker +placement in full/standalone DEV overlays and rendered policy, including all +affinity alternatives and direct-node bypasses. Its +[50-case policy proof](M12_DEV_WORKER_PLACEMENT.md) does not establish actual +node labels, deployed placement, rollout success or target-host performance. ## Persistence boundary resolved by 12.3b diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 137c0aa..9bc5173 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -1,5 +1,36 @@ # Validation report +## 2026-10-19 - Explicit worker-only DEV scheduling + +[ADR-0170](../adr/0170-dev-worker-placement.md) applies required negative +control-plane/master role affinity to current DEV Deployments and Jobs in the +full application and standalone region/store overlays. It preserves store +selectors, claims, requests and network/database endpoints. The rendered policy +checks every required selector alternative and refuses direct `nodeName`. + +Executed locally with pinned Kustomize v5.8.1: + +- Deployment policy: **75 objects in 75 files, 138 rendered objects in 17 + overlays, zero failures**. +- Full policy self-test: **50 cases, zero failures**. Four new cases refuse + removed affinity, missing legacy master exclusion, an unrestricted OR term + and a direct node bypass. Existing route/window/network/namespace checks pass. +- Staged/working diff checks pass. No C++/biological rule or golden changes; + no further compiler/sanitizer rerun is warranted for these manifest/Python/docs + changes. The preceding census section records the full C++ validation. + +The self-test route helper now preserves all existing components when enabling +the route, without duplicate YAML keys. Detailed scope is in +[the placement report](M12_DEV_WORKER_PLACEMENT.md). + +Delivery remains Git Actions -> Harbor -> ArgoCD -> Kubernetes. Offline policy +does not verify actual cluster role labels, deployed placement, successful +image rollout, reachability, target-host costs or multi-host failures. No direct +cluster operation, new workload service or World0 operation occurred. These +checks do not complete M12's ecology, control/UI, actual-days or product gates. +The worker-placement push refreshes the DEV session window to +**2026-10-19 23:07:30 UTC**; live reachability still requires deployment evidence. + ## 2026-10-19 - Retained-plan census projection 2 and committed observations [ADR-0169](../adr/0169-retained-plan-census-observations.md) and the diff --git a/tools/ci/deploy-policy-check.py b/tools/ci/deploy-policy-check.py index e56a127..88c9b56 100644 --- a/tools/ci/deploy-policy-check.py +++ b/tools/ci/deploy-policy-check.py @@ -22,6 +22,8 @@ environment-isolation or World0-protection invariant is broken: - scheduling (ADR-0076): every pod that mounts the node-local objects volume selects the store node; aevolution.io/compute is only ever preferred, never required; upgrade-rehearsal flags never appear in a deployed workload; + - DEV placement (ADR-0170): every required node-affinity alternative excludes + both control-plane roles; direct nodeName bypasses are forbidden; - the Pods the backup and barrier scripts generate (rendered with --print-manifest) keep the database-tool security contract: non-root 10001 with fsGroup 10001 so the 0440 Secret files are readable, no token, read-only root, no escalation; @@ -672,6 +674,22 @@ for overlay, docs in renders.items(): if expected and kind not in CLUSTER_KINDS: check(namespace == expected, f'{where}: renders into {namespace}, expected {expected}') check(kind != 'Secret', f'{where}: rendered Secrets never come from Git') + if namespace == 'evolution-dev' and kind in workload_kinds: + spec = doc.get('spec', {}) + pod = (spec.get('jobTemplate', {}).get('spec', {}).get('template', {}).get('spec') + or spec.get('template', {}).get('spec') or spec) + terms = (pod.get('affinity', {}).get('nodeAffinity', {}) + .get('requiredDuringSchedulingIgnoredDuringExecution', {}) + .get('nodeSelectorTerms', [])) + control_roles = {'node-role.kubernetes.io/control-plane', 'node-role.kubernetes.io/master'} + # Selector terms are OR-ed: every possible term must exclude both roles. + excludes_control = bool(terms) and all( + control_roles <= {e.get('key') for e in term.get('matchExpressions', []) + if e.get('operator') == 'DoesNotExist' and not e.get('values')} + for term in terms) + check(excludes_control and not pod.get('nodeName'), + f'{where}: DEV workloads must exclude control-plane and master nodes in every required term ' + 'and must not bypass scheduling with nodeName') if kind == 'Deployment' and name == 'evo-region': pod = doc['spec']['template']['spec'] for container in pod.get('containers', []) + pod.get('initContainers', []): diff --git a/tools/ci/deploy-policy-selftest.py b/tools/ci/deploy-policy-selftest.py index 2880758..e1c6da0 100644 --- a/tools/ci/deploy-policy-selftest.py +++ b/tools/ci/deploy-policy-selftest.py @@ -88,7 +88,15 @@ def route_on(tree): def enable(tree): - append(tree, DEV, 'components:\n - ../dev-session-route\n') + import yaml + full = os.path.join(tree, DEV) + with open(full, encoding='utf-8') as f: + data = yaml.safe_load(f) + components = data.setdefault('components', []) + if '../dev-session-route' not in components: + components.append('../dev-session-route') + with open(full, 'w', encoding='utf-8') as f: + yaml.safe_dump(data, f, sort_keys=False) def resolve(tree): @@ -103,6 +111,24 @@ def quota_one(tree): CASES = [ + ('DEV required worker affinity removed', lambda t: inline_patch( + t, 'deploy/kubernetes/dev/kustomization.yaml', + '- op: remove\n path: /spec/template/spec/affinity\n', + {'kind': 'Deployment', 'name': 'evo-region'}), 'must exclude control-plane and master'), + ('DEV affinity has an unrestricted OR term', lambda t: inline_patch( + t, 'deploy/kubernetes/dev/kustomization.yaml', + '- op: add\n path: /spec/template/spec/affinity/nodeAffinity/' + 'requiredDuringSchedulingIgnoredDuringExecution/nodeSelectorTerms/-\n' + ' value: {matchExpressions: [{key: aevolution.io/stores, operator: Exists}]}\n', + {'kind': 'Deployment', 'name': 'evo-region'}), 'must exclude control-plane and master'), + ('DEV direct nodeName bypass', lambda t: inline_patch( + t, 'deploy/kubernetes/dev/kustomization.yaml', + '- op: add\n path: /spec/template/spec/nodeName\n value: arbitrary-node\n', + {'kind': 'Job', 'name': 'aevolution-db-migrate'}), 'must not bypass scheduling with nodeName'), + ('DEV legacy master exclusion removed', lambda t: edit( + t, 'deploy/kubernetes/worker-only-dev/kustomization.yaml', + ' - {key: node-role.kubernetes.io/master, operator: DoesNotExist}\n', + '', 2), 'must exclude control-plane and master'), ('non-RFC 1918 VIP without approval', lambda t: edit(t, ROUTE + 'service.yaml', '172.10.1.202', '172.11.1.210'), 'internal address'), # Review M9.8f P1: inline/JSON patches and namespace changes, seen only when rendered.