From df0c4f4efb01b9e0e978a0e51ac69c12256c008c Mon Sep 17 00:00:00 2001 From: dingo Date: Tue, 29 Sep 2026 12:43:58 -0400 Subject: [PATCH] Add M8.2 environment isolation and World0 protection Per-environment platform layers (dev, stage, world0) with restricted pod security, quotas, limit ranges, default-deny networking and token-less service accounts, applied by an administrator rather than Argo. A native ValidatingAdmissionPolicy denies deleting World0 data, identity and Argo objects to every requester without the break-glass group. Stage and World0 Argo projects are scoped to one namespace, World0 blocks automated sync, and a cleanup tool refuses World0. An offline policy check runs as a new CI suite; a live verification script uses only can-i, reads and server dry runs. Co-Authored-By: Claude Opus 5.5 --- .gitea/workflows/build.yaml | 2 +- deploy/README.md | 11 ++ deploy/argocd/stage/application.yaml.example | 21 +++ deploy/argocd/stage/project.yaml.example | 26 +++ deploy/argocd/world0/application.yaml.example | 25 +++ deploy/argocd/world0/project.yaml.example | 33 ++++ deploy/docker/Dockerfile | 13 ++ .../platform/admission/world0-protection.yaml | 88 +++++++++ .../platform/base/kustomization.yaml | 15 ++ .../kubernetes/platform/base/limitrange.yaml | 17 ++ .../platform/base/networkpolicies.yaml | 30 ++++ deploy/kubernetes/platform/base/quota.yaml | 17 ++ .../platform/base/serviceaccounts.yaml | 26 +++ .../platform/dev/kustomization.yaml | 9 + deploy/kubernetes/platform/dev/namespace.yaml | 10 ++ .../platform/stage/kustomization.yaml | 9 + .../kubernetes/platform/stage/namespace.yaml | 10 ++ .../platform/world0/kustomization.yaml | 29 +++ .../kubernetes/platform/world0/namespace.yaml | 11 ++ docs/ROADMAP.md | 10 ++ docs/STATUS.md | 18 +- docs/adr/0069-environment-isolation.md | 106 +++++++++++ docs/adr/README.md | 1 + docs/engineering/VALIDATION.md | 24 +++ docs/infrastructure/ENVIRONMENTS.md | 74 ++++++++ tools/ci/deploy-policy-check.py | 168 ++++++++++++++++++ tools/ops/cleanup-environment.sh | 42 +++++ tools/ops/verify-isolation.sh | 110 ++++++++++++ 28 files changed, 953 insertions(+), 2 deletions(-) create mode 100644 deploy/argocd/stage/application.yaml.example create mode 100644 deploy/argocd/stage/project.yaml.example create mode 100644 deploy/argocd/world0/application.yaml.example create mode 100644 deploy/argocd/world0/project.yaml.example create mode 100644 deploy/kubernetes/platform/admission/world0-protection.yaml create mode 100644 deploy/kubernetes/platform/base/kustomization.yaml create mode 100644 deploy/kubernetes/platform/base/limitrange.yaml create mode 100644 deploy/kubernetes/platform/base/networkpolicies.yaml create mode 100644 deploy/kubernetes/platform/base/quota.yaml create mode 100644 deploy/kubernetes/platform/base/serviceaccounts.yaml create mode 100644 deploy/kubernetes/platform/dev/kustomization.yaml create mode 100644 deploy/kubernetes/platform/dev/namespace.yaml create mode 100644 deploy/kubernetes/platform/stage/kustomization.yaml create mode 100644 deploy/kubernetes/platform/stage/namespace.yaml create mode 100644 deploy/kubernetes/platform/world0/kustomization.yaml create mode 100644 deploy/kubernetes/platform/world0/namespace.yaml create mode 100644 docs/adr/0069-environment-isolation.md create mode 100644 docs/infrastructure/ENVIRONMENTS.md create mode 100644 tools/ci/deploy-policy-check.py create mode 100644 tools/ops/cleanup-environment.sh create mode 100644 tools/ops/verify-isolation.sh diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml index 5bb87f5..ede48c7 100644 --- a/.gitea/workflows/build.yaml +++ b/.gitea/workflows/build.yaml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - suite: [checks, tsan, ecs, postgres] + suite: [checks, tsan, ecs, postgres, deploy] steps: - name: Checkout uses: https://github.com/actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 diff --git a/deploy/README.md b/deploy/README.md index 6aed57a..e27d4b5 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -152,3 +152,14 @@ plus the pinned libpq, UID 10001, no credentials). It smoke-tests the image with - the runtime Secret, object PVC and a NetworkPolicy. Pin the digest and provision the region first. Without a gateway, the region receives no inputs. + +## Environments and World0 protection (M8.2) + +`kubernetes/platform/` holds each environment's namespace, quota, limit range, +default-deny policies and service accounts. `kubernetes/platform/admission` holds +the World0 admission policy. A cluster administrator applies these by hand; they +are never part of an Argo Application. The stage and World0 Argo projects and +applications are in `argocd/stage` and `argocd/world0` as examples. + +See docs/infrastructure/ENVIRONMENTS.md for the runbook, and ADR-0069. The CI +`deploy` suite runs `tools/ci/deploy-policy-check.py` over every manifest. diff --git a/deploy/argocd/stage/application.yaml.example b/deploy/argocd/stage/application.yaml.example new file mode 100644 index 0000000..2fcc1f2 --- /dev/null +++ b/deploy/argocd/stage/application.yaml.example @@ -0,0 +1,21 @@ +# M8.2 (stage). REPLACE_WORKLOAD_PATH is deliberately unresolved: the stage +# workload is M8.4 work. Manual sync only; no automated policy, no prune on sync of +# durable objects (PVCs carry argocd.argoproj.io/sync-options: Prune=false,Delete=false). +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: aevolution-stage + namespace: argocd +spec: + project: aevolution-stage + source: + repoURL: https://git.aeternalisgames.com/dingo/AEvolution.git + targetRevision: main + path: REPLACE_WORKLOAD_PATH + destination: + server: https://kubernetes.default.svc + namespace: evolution-stage + syncPolicy: + syncOptions: + - FailOnSharedResource=true + - PruneLast=true diff --git a/deploy/argocd/stage/project.yaml.example b/deploy/argocd/stage/project.yaml.example new file mode 100644 index 0000000..eed4587 --- /dev/null +++ b/deploy/argocd/stage/project.yaml.example @@ -0,0 +1,26 @@ +# M8.2 (stage). Apply by hand to the argocd namespace; never from an Application. +# One destination namespace only, no cluster-scoped kinds, no Secrets or quotas. +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: aevolution-stage + namespace: argocd +spec: + description: AEvolution stage (evolution-stage only) + sourceRepos: + - https://git.aeternalisgames.com/dingo/AEvolution.git + destinations: + - server: https://kubernetes.default.svc + namespace: evolution-stage + clusterResourceWhitelist: [] + namespaceResourceWhitelist: + - {group: "", kind: Service} + - {group: "", kind: ConfigMap} + - {group: "", kind: PersistentVolumeClaim} + - {group: apps, kind: Deployment} + - {group: batch, kind: Job} + - {group: networking.k8s.io, kind: NetworkPolicy} + # Not listed, so never Argo-managed: Namespaces, ServiceAccounts (platform layer), + # Secrets, quotas, limit ranges and RBAC. + orphanedResources: + warn: true diff --git a/deploy/argocd/world0/application.yaml.example b/deploy/argocd/world0/application.yaml.example new file mode 100644 index 0000000..03537f4 --- /dev/null +++ b/deploy/argocd/world0/application.yaml.example @@ -0,0 +1,25 @@ +# M8.2 (world0). REPLACE_WORKLOAD_PATH is deliberately unresolved: the world0 +# workload is M8.4 work. Manual sync only; no automated policy, no prune on sync of +# durable objects (PVCs carry argocd.argoproj.io/sync-options: Prune=false,Delete=false). +# World0: never add the resources-finalizer.argocd.argoproj.io finalizer (deleting +# the Application must orphan, not cascade), never add an automated policy, and never +# list this Application in a parent app-of-apps with prune. The admission policy +# also denies deleting this Application without break-glass. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: aevolution-world0 + namespace: argocd +spec: + project: aevolution-world0 + source: + repoURL: https://git.aeternalisgames.com/dingo/AEvolution.git + targetRevision: main + path: REPLACE_WORKLOAD_PATH + destination: + server: https://kubernetes.default.svc + namespace: evolution-world0 + syncPolicy: + syncOptions: + - FailOnSharedResource=true + - PruneLast=true diff --git a/deploy/argocd/world0/project.yaml.example b/deploy/argocd/world0/project.yaml.example new file mode 100644 index 0000000..e39ce9e --- /dev/null +++ b/deploy/argocd/world0/project.yaml.example @@ -0,0 +1,33 @@ +# M8.2 (world0). Apply by hand to the argocd namespace; never from an Application. +# One destination namespace only, no cluster-scoped kinds, no Secrets or quotas. +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: aevolution-world0 + namespace: argocd +spec: + description: AEvolution world0 (evolution-world0 only) + sourceRepos: + - https://git.aeternalisgames.com/dingo/AEvolution.git + destinations: + - server: https://kubernetes.default.svc + namespace: evolution-world0 + clusterResourceWhitelist: [] + namespaceResourceWhitelist: + - {group: "", kind: Service} + - {group: "", kind: ConfigMap} + - {group: "", kind: PersistentVolumeClaim} + - {group: apps, kind: Deployment} + - {group: batch, kind: Job} + - {group: networking.k8s.io, kind: NetworkPolicy} + # Not listed, so never Argo-managed: Namespaces, ServiceAccounts (platform layer), + # Secrets, quotas, limit ranges and RBAC. + orphanedResources: + warn: true + # No automated sync can ever run; a human may still sync by hand. + syncWindows: + - kind: deny + schedule: "* * * * *" + duration: 24h + applications: ["*"] + manualSync: true diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index f5ca1dc..86ee298 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -45,6 +45,19 @@ RUN cmake --preset analysis -B build/ecs-analysis -DCMAKE_CXX_COMPILER=clang++-1 && cmake --build build/ecs-analysis --target evo-ecs-experiment --parallel 2 \ && ctest --test-dir build/ecs-analysis -R ecs-experiment --output-on-failure --no-tests=error +# M8.2 offline deployment-policy check (ADR-0069): Ubuntu's python3-yaml only, no +# cluster access. Fails the build when an isolation or World0 invariant breaks. +FROM ${UBUNTU_IMAGE} AS ci-deploy +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update \ + && apt-get install -y --no-install-recommends python3 python3-yaml \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /source +COPY deploy/ deploy/ +COPY tools/ci/deploy-policy-check.py tools/ci/ +COPY tools/ops/ tools/ops/ +RUN python3 tools/ci/deploy-policy-check.py + FROM ci-toolchain AS postgres-build RUN apt-get update && apt-get install -y --no-install-recommends \ curl bzip2 make flex bison perl libssl-dev zlib1g-dev pkg-config \ diff --git a/deploy/kubernetes/platform/admission/world0-protection.yaml b/deploy/kubernetes/platform/admission/world0-protection.yaml new file mode 100644 index 0000000..b9f4c18 --- /dev/null +++ b/deploy/kubernetes/platform/admission/world0-protection.yaml @@ -0,0 +1,88 @@ +# M8.2 World0 protection (ADR-0069). A native ValidatingAdmissionPolicy (GA since +# Kubernetes 1.30; the cluster runs 1.32): no webhook, no new dependency. It applies +# to every requester, including cluster-admin (system:masters), the Argo CD +# application controller and any Application in any AppProject, so no permissive +# app, prune, parent-app cascade or credential can remove World0 data or identity. +# Only a member of the group `aevolution:world0-break-glass` passes. Membership is +# granted out of band (client certificate O= or OIDC group) for a recorded operation. +# +# Denied without break-glass: +# - deleting Namespace evolution-world0, or removing its aevolution.io/protected label; +# - deleting PersistentVolumeClaims, Secrets or ServiceAccounts in evolution-world0; +# - deleting a PersistentVolume bound to evolution-world0, or changing its reclaim +# policy away from Retain once set (set Retain when the volume is bound: the +# only StorageClass, local-path, provisions with Delete); +# - deleting the aevolution-world0 Argo CD Application or AppProject. +# Kubernetes does not run admission policies on policy objects themselves, so +# removing this policy is itself the audited break-glass step (cluster-admin). +# Apply by hand: kubectl apply -f deploy/kubernetes/platform/admission/ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: aevolution-world0-protection + labels: + app.kubernetes.io/part-of: aevolution +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["DELETE", "UPDATE"] + resources: ["namespaces", "persistentvolumes"] + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["DELETE"] + resources: ["persistentvolumeclaims", "secrets", "serviceaccounts"] + - apiGroups: ["argoproj.io"] + apiVersions: ["*"] + operations: ["DELETE"] + resources: ["applications", "appprojects"] + variables: + - name: breakGlass + expression: "request.userInfo.groups.exists(g, g == 'aevolution:world0-break-glass')" + - name: resource + expression: "request.resource.resource" + - name: world0Namespace + expression: "variables.resource == 'namespaces' && request.name == 'evolution-world0'" + - name: world0Volume + expression: >- + variables.resource == 'persistentvolumes' && + has(oldObject.spec.claimRef) && has(oldObject.spec.claimRef.namespace) && + oldObject.spec.claimRef.namespace == 'evolution-world0' + - name: world0Object + expression: >- + request.namespace == 'evolution-world0' && + variables.resource in ['persistentvolumeclaims', 'secrets', 'serviceaccounts'] + - name: world0Delivery + expression: >- + request.resource.group == 'argoproj.io' && request.name == 'aevolution-world0' + validations: + - expression: >- + variables.breakGlass || request.operation != 'DELETE' || + !(variables.world0Namespace || variables.world0Volume || variables.world0Object || + variables.world0Delivery) + message: "World0 data, identity and delivery objects cannot be deleted without the aevolution:world0-break-glass group" + reason: Forbidden + - expression: >- + variables.breakGlass || request.operation != 'UPDATE' || !variables.world0Namespace || + (has(object.metadata.labels) && 'aevolution.io/protected' in object.metadata.labels && + object.metadata.labels['aevolution.io/protected'] == 'true') + message: "The World0 namespace must keep aevolution.io/protected=true" + reason: Forbidden + - expression: >- + variables.breakGlass || request.operation != 'UPDATE' || !variables.world0Volume || + oldObject.spec.persistentVolumeReclaimPolicy != 'Retain' || + object.spec.persistentVolumeReclaimPolicy == 'Retain' + message: "A World0 PersistentVolume set to Retain cannot be changed back" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: aevolution-world0-protection + labels: + app.kubernetes.io/part-of: aevolution +spec: + policyName: aevolution-world0-protection + validationActions: [Deny] diff --git a/deploy/kubernetes/platform/base/kustomization.yaml b/deploy/kubernetes/platform/base/kustomization.yaml new file mode 100644 index 0000000..e2f1092 --- /dev/null +++ b/deploy/kubernetes/platform/base/kustomization.yaml @@ -0,0 +1,15 @@ +# M8.2 per-environment guardrails (docs/infrastructure/ENVIRONMENTS.md). Applied by a +# cluster administrator with `kubectl apply -k deploy/kubernetes/platform/`, never +# through an Argo CD Application: namespaces, quotas and identities must not be +# prunable by a workload sync. Each overlay sets the namespace and its quota values. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - quota.yaml + - limitrange.yaml + - networkpolicies.yaml + - serviceaccounts.yaml +labels: + - pairs: + app.kubernetes.io/part-of: aevolution + app.kubernetes.io/managed-by: aevolution-platform diff --git a/deploy/kubernetes/platform/base/limitrange.yaml b/deploy/kubernetes/platform/base/limitrange.yaml new file mode 100644 index 0000000..6a16f44 --- /dev/null +++ b/deploy/kubernetes/platform/base/limitrange.yaml @@ -0,0 +1,17 @@ +# Defaults for containers that omit resources (the quota requires them), sized from +# the measured region-host profile (peak RSS about 9.7 MiB): 64Mi request, 256Mi +# limit, 50m CPU request and no default CPU limit. No container may exceed 2Gi. +apiVersion: v1 +kind: LimitRange +metadata: + name: aevolution-defaults +spec: + limits: + - type: Container + defaultRequest: + cpu: 50m + memory: 64Mi + default: + memory: 256Mi + max: + memory: 2Gi diff --git a/deploy/kubernetes/platform/base/networkpolicies.yaml b/deploy/kubernetes/platform/base/networkpolicies.yaml new file mode 100644 index 0000000..6f2d74a --- /dev/null +++ b/deploy/kubernetes/platform/base/networkpolicies.yaml @@ -0,0 +1,30 @@ +# Deny by default: every pod in the namespace has no ingress or egress except DNS. +# Workloads add narrow allowances with their own policies (for example the region +# host's egress to exactly the database VM address and port, and ingress to its +# operations port from the metrics scraper) in M8.4. Enforced by Canal/Calico. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: aevolution-default-deny +spec: + podSelector: {} + policyTypes: [Ingress, Egress] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: aevolution-allow-dns +spec: + podSelector: {} + policyTypes: [Egress] + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - {protocol: UDP, port: 53} + - {protocol: TCP, port: 53} diff --git a/deploy/kubernetes/platform/base/quota.yaml b/deploy/kubernetes/platform/base/quota.yaml new file mode 100644 index 0000000..72714eb --- /dev/null +++ b/deploy/kubernetes/platform/base/quota.yaml @@ -0,0 +1,17 @@ +# Values are replaced per environment. Memory is quota'd on requests and limits; +# CPU only on requests (region hosts run without CPU limits: throttling would delay +# lease renewals, docs/infrastructure/INVENTORY.md). No LoadBalancer or NodePort. +apiVersion: v1 +kind: ResourceQuota +metadata: + name: aevolution-quota +spec: + hard: + requests.cpu: "2" + requests.memory: 2Gi + limits.memory: 4Gi + pods: "20" + persistentvolumeclaims: "4" + requests.storage: 40Gi + services.loadbalancers: "0" + services.nodeports: "0" diff --git a/deploy/kubernetes/platform/base/serviceaccounts.yaml b/deploy/kubernetes/platform/base/serviceaccounts.yaml new file mode 100644 index 0000000..52e7e59 --- /dev/null +++ b/deploy/kubernetes/platform/base/serviceaccounts.yaml @@ -0,0 +1,26 @@ +# One identity per role; none needs the Kubernetes API, so none mounts a token and +# no Role or RoleBinding is granted. Secrets are referenced by name only and are +# created per environment by an operator (ENVIRONMENTS.md); they never enter Git. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: aevolution-migrator # Runs migrations with the migrator database login. +automountServiceAccountToken: false +imagePullSecrets: + - name: harbor-pull +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: aevolution-region # evo-server with the runtime database login. +automountServiceAccountToken: false +imagePullSecrets: + - name: harbor-pull +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: aevolution-backup # Read-only backup/clone jobs (M8.7). +automountServiceAccountToken: false +imagePullSecrets: + - name: harbor-pull diff --git a/deploy/kubernetes/platform/dev/kustomization.yaml b/deploy/kubernetes/platform/dev/kustomization.yaml new file mode 100644 index 0000000..0bf27c0 --- /dev/null +++ b/deploy/kubernetes/platform/dev/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: evolution-dev +resources: + - namespace.yaml + - ../base +labels: + - pairs: + aevolution.io/environment: dev diff --git a/deploy/kubernetes/platform/dev/namespace.yaml b/deploy/kubernetes/platform/dev/namespace.yaml new file mode 100644 index 0000000..3486b5e --- /dev/null +++ b/deploy/kubernetes/platform/dev/namespace.yaml @@ -0,0 +1,10 @@ +# Disposable development. Existing namespace; applying adds labels and guardrails. +apiVersion: v1 +kind: Namespace +metadata: + name: evolution-dev + labels: + aevolution.io/environment: dev + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/enforce-version: v1.32 + pod-security.kubernetes.io/warn: restricted diff --git a/deploy/kubernetes/platform/stage/kustomization.yaml b/deploy/kubernetes/platform/stage/kustomization.yaml new file mode 100644 index 0000000..7bae733 --- /dev/null +++ b/deploy/kubernetes/platform/stage/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: evolution-stage +resources: + - namespace.yaml + - ../base +labels: + - pairs: + aevolution.io/environment: stage diff --git a/deploy/kubernetes/platform/stage/namespace.yaml b/deploy/kubernetes/platform/stage/namespace.yaml new file mode 100644 index 0000000..fd35dea --- /dev/null +++ b/deploy/kubernetes/platform/stage/namespace.yaml @@ -0,0 +1,10 @@ +# Clone, migration and soak rehearsals (M8.6/8.9). Disposable, never World0 data. +apiVersion: v1 +kind: Namespace +metadata: + name: evolution-stage + labels: + aevolution.io/environment: stage + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/enforce-version: v1.32 + pod-security.kubernetes.io/warn: restricted diff --git a/deploy/kubernetes/platform/world0/kustomization.yaml b/deploy/kubernetes/platform/world0/kustomization.yaml new file mode 100644 index 0000000..210db13 --- /dev/null +++ b/deploy/kubernetes/platform/world0/kustomization.yaml @@ -0,0 +1,29 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: evolution-world0 +resources: + - namespace.yaml + - ../base +patches: + - target: + kind: ResourceQuota + name: aevolution-quota + patch: |- + - op: replace + path: /spec/hard/requests.cpu + value: "4" + - op: replace + path: /spec/hard/requests.memory + value: 4Gi + - op: replace + path: /spec/hard/limits.memory + value: 8Gi + - op: replace + path: /spec/hard/persistentvolumeclaims + value: "8" + - op: replace + path: /spec/hard/requests.storage + value: 200Gi +labels: + - pairs: + aevolution.io/environment: world0 diff --git a/deploy/kubernetes/platform/world0/namespace.yaml b/deploy/kubernetes/platform/world0/namespace.yaml new file mode 100644 index 0000000..4ff6f49 --- /dev/null +++ b/deploy/kubernetes/platform/world0/namespace.yaml @@ -0,0 +1,11 @@ +# Persistent World0. Guarded by deploy/kubernetes/platform/admission. Nothing here deploys a world; launch stays gated by M8.8. +apiVersion: v1 +kind: Namespace +metadata: + name: evolution-world0 + labels: + aevolution.io/environment: world0 + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/enforce-version: v1.32 + pod-security.kubernetes.io/warn: restricted + aevolution.io/protected: "true" diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 9fecadf..6e2bffe 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -606,3 +606,13 @@ M8.1 complete (2026-09-29): docs/infrastructure/INVENTORY.md records the cluster the separate PostgreSQL VM, the measured workload profile and a per-worker `local-path` storage probe. Storage figures are provisional until the planned RAID and storage change. Next: M8.2, environment isolation and secrets. + +M8.2 designed and checked offline (2026-09-29), under ADR-0069: +- dev, stage and World0 namespaces, each with its own guardrails, identities and + secret references; +- a native admission policy that protects World0 from every requester except a + break-glass group; +- scoped Argo projects, and a cleanup tool that refuses World0; +- an offline policy check in CI, and a live isolation check. +The live checks run once the maintainer applies the platform layer. Next: M8.3, +durable stores in dev and stage. diff --git a/docs/STATUS.md b/docs/STATUS.md index ad7cf98..5e25fac 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -993,4 +993,20 @@ M8.1 complete (2026-09-29) for the current disk layout. The per-worker These are well inside the lease deadlines. Re-probe after the planned RAID and storage change. Host memory was raised by about 500 GB. -Next: M8.2, environment isolation and secrets. \ No newline at end of file +Next: M8.2, environment isolation and secrets. +## M8.2 environment isolation (2026-09-29) + +ADR-0069 adds three environments, each with its own guardrails, service accounts, +secret references and database: +- `evolution-dev`, `evolution-stage` and `evolution-world0`; +- guardrails: quota, limit range, default-deny networking, restricted pod + security. +A native ValidatingAdmissionPolicy denies deleting World0 data, identity or Argo +objects to everyone, including cluster-admin and Argo, unless the requester is in +`aevolution:world0-break-glass`. The stage and World0 Argo projects are scoped to +one namespace each, and World0 permanently denies automated sync. The cleanup tool +refuses World0. + +The offline policy check (now a CI suite) passes, and nine injected violations +were each caught. Applying the layer and running `tools/ops/verify-isolation.sh` +is waiting on the maintainer. diff --git a/docs/adr/0069-environment-isolation.md b/docs/adr/0069-environment-isolation.md new file mode 100644 index 0000000..5d0c0f0 --- /dev/null +++ b/docs/adr/0069-environment-isolation.md @@ -0,0 +1,106 @@ +# ADR-0069: Environment isolation and World0 protection + +## Status +Accepted for M8.2, 2026-09-29. It builds on the M8.1 inventory +(infrastructure/INVENTORY.md), ADR-0020 (delivery), and OPERATIONS.md (World0 +protection). Nothing here deploys a world; World0 launch stays gated by M8.8. + +## Context +The inventory found five things that matter here: +- Argo CD's `default` AppProject allows every cluster resource in every namespace. + Three applications use it, two with automated prune and self-heal. +- The only StorageClass reclaims with Delete. +- Collection ran as `system:admin`. +- There are no quotas, limit ranges or priority classes. +- Only the AEvolution application is manual-sync. + +The roadmap's named risk is that one permissive application or credential could +destroy every world. Argo project boundaries cannot address that alone, because a +`default`-project application, a cascading parent application or a cluster-admin +is not constrained by an AEvolution AppProject. + +## Decision +**Three environments,** each in its own namespace: `evolution-dev`, +`evolution-stage` and `evolution-world0`. Each has its own AppProject, service +accounts, Secrets, database, database logins and object PVC. No identity, Secret or +database is shared between them. + +**Platform layer** (`deploy/kubernetes/platform/`), applied by a cluster +administrator and never by an Argo CD Application, so no sync or prune can remove +it. It contains: +- The Namespace, labelled with its environment and with Pod Security `restricted` + enforced. Only World0 carries `aevolution.io/protected=true`. +- A ResourceQuota: + - dev and stage: 2 CPU and 2 GiB requests, 4 GiB of memory limits, 4 PVCs and + 40 GiB; + - World0: 4 CPU and 4 GiB requests, 8 GiB of limits, 8 PVCs and 200 GiB; + - no LoadBalancer or NodePort services anywhere. +- A LimitRange (defaults 50m/64Mi with a 256Mi limit, at most 2Gi per container) + sized from the measured host profile. +- A default-deny NetworkPolicy for every pod, plus DNS egress only. Workloads add + narrow rules in M8.4. +- Service accounts `aevolution-migrator`, `aevolution-region` and + `aevolution-backup`, with no token and no RBAC; nothing needs the Kubernetes API. + They are platform-owned, so workload projects cannot create them. + +**Secrets never enter Git.** Operators create them per environment, by reference +name (ENVIRONMENTS.md). CI holds no cluster or data credentials (ADR-0020). + +**World0 admission protection.** A native ValidatingAdmissionPolicy +(`deploy/kubernetes/platform/admission`) is GA on the cluster's Kubernetes 1.32. It +needs no webhook or dependency, and it fails closed. It applies to everyone, +including system:masters and Argo's controller. Unless the requester is in the +group `aevolution:world0-break-glass`, it denies: +- deleting the World0 namespace, or removing its protected label; +- deleting its PVCs, Secrets or service accounts; +- deleting a PV bound to it, or moving such a PV away from `Retain`; +- deleting the `aevolution-world0` Application or AppProject. + +A cascading parent app, a prune, a `default`-project application or a stray +`kubectl delete` therefore fails at admission. Kubernetes does not apply policies +to policy objects, so removing the policy is itself the break-glass action, +reserved to cluster-admins. + +**Delivery.** +- The stage and World0 AppProjects (examples) each target one namespace. They + allow no cluster kinds, and whitelist only namespaced workload kinds. +- World0 carries a permanent deny sync window with `manualSync: true`, so no + automated sync can ever run. +- Applications never auto-sync and never carry the cascade finalizer. World0's must + not appear in a pruning parent. +- Durable PVCs will carry `Prune=false,Delete=false`. + +**Cleanup.** `tools/ops/cleanup-environment.sh` accepts only `dev` or `stage`. It +refuses anything else before invoking kubectl, re-checks the namespace labels, +never touches platform objects or Secrets, and dry-runs unless `--execute` is given. + +**Verification.** +- `tools/ci/deploy-policy-check.py` runs offline in CI (Ubuntu `python3-yaml`) and + asserts every invariant above over all manifests. +- `tools/ops/verify-isolation.sh` runs live. It makes only `can-i` queries, reads + and server dry-runs, and covers negative RBAC, cross-environment denial, World0 + admission denials, project scopes and the cleanup refusal. + +## Consequences +- **Residual risk: creation.** A `default`-project application can still *create* + objects in AEvolution namespaces; admission protects only World0's data and + identity from deletion. Recommended (owner's decision): restrict `default` or + move those applications to scoped projects. +- **Residual risk: advisory locks.** The runtime login's advisory-lock capability + (ADR-0068) remains. +- **Local-path storage.** Its PVs are Delete-reclaim. A World0 PV must be patched + to Retain when bound, and the policy then keeps it there. Volumes remain + node-local until M8.3. +- **Quotas and defaults.** Quotas are starting values from measured profiles. They + are raised by editing the platform overlay, never by the workload. +- **Existing dev namespace.** Applying the dev overlay to the existing namespace + adds labels, the quota and the default-deny policy. Its proof Job already + complies. + +## Alternatives rejected +- **An admission webhook or policy engine (Kyverno, Gatekeeper).** Either is a new + dependency with its own availability risk; the native policy suffices. +- **Relying on AppProjects alone.** They do not constrain other projects, parents + or cluster-admins. +- **Namespaces created by Argo `CreateNamespace`.** A prune or cascade could then + remove them. diff --git a/docs/adr/README.md b/docs/adr/README.md index 5039cde..50dcfb2 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -79,3 +79,4 @@ design. No unadopted candidate is a link dependency of the proof. - [ADR-0066: Operations endpoints](0066-operations-endpoints.md) - accepted for M7.8 liveness vs authority readiness, bounded metrics, dependency-free responder and the server image. - [ADR-0067: Region clones and retained-history audit](0067-region-clones.md) - accepted for M7.9a re-identified clone genesis naming its source cut, isolated clone databases and full-chain retention audit. - [ADR-0068: Upgrade barrier and durability qualification](0068-upgrade-barrier.md) - accepted for M7.9b build-bound authority, retirement barrier fencing old writers, explicit refusal of rule/format changes and the measured recovery fixture. +- [ADR-0069: Environment isolation and World0 protection](0069-environment-isolation.md) - accepted for M8.2 per-environment guardrails, native World0 admission protection, scoped Argo projects and verification. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 2f2f850..e186bb4 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -2639,3 +2639,27 @@ Evidence: - Collection is manual, with no automation planned. - No credential values were read. The 18.6 version, role and TLS requirements are enforced by the adapter and migrations when M8.3 connects. + +## M8.2 environment isolation - 2026-09-29 + +- `tools/ci/deploy-policy-check.py` (WSL, Python 3 with PyYAML): 35 objects in 27 + files, 0 failures. +- Nine mutations were each caught, then restored: + - World0 auto-sync; + - the World0 cascade finalizer; + - the policy set to failurePolicy Ignore; + - the policy no longer covering Secrets; + - an egress hole in default-deny; + - the World0 protected label removed; + - the cleanup tool accepting world0 (rc 127, it reached kubectl); + - service-account token automount; + - a wildcard stage destination. +- `kubectl kustomize` (client v1.31.4) renders the dev, stage and World0 platform + overlays (9 objects each), and still renders the live `deploy/kubernetes/dev` + path unchanged. +- `bash -n` passes for both ops scripts. +- The CI `deploy` suite (a Docker `ci-deploy` stage) has not run yet; Docker is + not running locally. +- Not executed: applying anything to the cluster, the admission policy's CEL + compilation (it needs an API server; the runbook validates it with a server + dry run first), and `tools/ops/verify-isolation.sh`. diff --git a/docs/infrastructure/ENVIRONMENTS.md b/docs/infrastructure/ENVIRONMENTS.md new file mode 100644 index 0000000..1e369a1 --- /dev/null +++ b/docs/infrastructure/ENVIRONMENTS.md @@ -0,0 +1,74 @@ +# AEvolution environments (M8.2) + +The decision is recorded in ADR-0069. This runbook applies the platform layer, +creates the per-environment references, and verifies isolation. Every step is +manual, and none is performed by Argo CD. + +## Environments + +| | dev | stage | world0 | +|---|---|---|---| +| Namespace | `evolution-dev` (exists) | `evolution-stage` | `evolution-world0` | +| Purpose | Development | Clone, migration and soak rehearsal | Persistent world (launch gated by M8.8) | +| Protected by admission | No | No | Yes | +| Cleanup tool | Allowed | Allowed | Refused | +| AppProject / Application | `aevolution-dev` (live, manual) | `deploy/argocd/stage/*.example` | `deploy/argocd/world0/*.example` (deny window) | +| Database | Its own database and logins on the AEvolution PostgreSQL VM | Its own | Its own | +| World IDs | Development range | Clone targets (ADR-0067) | World0's own; never reused elsewhere | + +## Secret references (created by an operator, never committed) + +Each namespace gets its own Secrets, with environment-specific values: +- `harbor-pull`: a pull-only Harbor robot (docker-registry type). +- `evolution-postgres-migrator`: the migrator/owner login for that environment's + database. +- `evolution-postgres-runtime`: the runtime login, a member of that environment's + NOLOGIN runtime group (ADR-0060). +- `aevolution-db`: connection settings (host, address, port, database, TLS mode + and root certificate). Mount it as files. The adapter reads a password file, + never `PGPASSWORD`. + +A World0 credential must never exist in the dev or stage namespaces, and no dev or +stage credential may reach the World0 database. Rotate by replacing the Secret and +restarting the consumer. + +## Apply (cluster administrator) + +```sh +# 1. Platform guardrails, per environment (dev adds labels, quota and policies to +# the existing namespace; its completed proof Job already complies). +kubectl apply -k deploy/kubernetes/platform/dev +kubectl apply -k deploy/kubernetes/platform/stage +kubectl apply -k deploy/kubernetes/platform/world0 # an empty, protected namespace +# 2. World0 admission protection (validate first, then apply). +kubectl apply --dry-run=server -f deploy/kubernetes/platform/admission/ +kubectl apply -f deploy/kubernetes/platform/admission/ +# 3. Projects (resolve the Application paths only when M8.4 adds workloads). +kubectl apply -f deploy/argocd/stage/project.yaml.example +kubectl apply -f deploy/argocd/world0/project.yaml.example +# 4. Verify: reads, can-i and server dry runs only. +bash tools/ops/verify-isolation.sh +``` + +Create the Secrets listed above in each namespace before any workload syncs. + +## Break-glass + +To delete World0 data or identity on purpose: +1. Record the operation and its approval. +2. Use a credential whose groups include `aevolution:world0-break-glass`, for + example a short-lived client certificate with that organization. +3. Take and verify a backup first (M8.7). + +Removing the admission policy itself is the break-glass of last resort. It needs +cluster-admin and must be recorded the same way. Never test by deleting World0: the +verify script uses server-side dry runs. + +## Residual risks +- **`default` AppProject.** It still allows every destination, so its applications + (other projects, two with automated prune) can create objects in AEvolution + namespaces. Admission denies only World0 deletions. Recommended: restrict + `default`, or move those applications to scoped projects. +- **World0 volumes.** `local-path` provisions volumes with Delete. Patch each World0 + PV to `Retain` when it is bound, and the policy then keeps it there. Volumes stay + node-local until M8.3. diff --git a/tools/ci/deploy-policy-check.py b/tools/ci/deploy-policy-check.py new file mode 100644 index 0000000..de3ba60 --- /dev/null +++ b/tools/ci/deploy-policy-check.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""M8.2 offline deployment-policy check (ADR-0069). Needs Python 3 and PyYAML only. + +Parses every manifest under deploy/ (including .yaml.example files) and fails if an +environment-isolation or World0-protection invariant is broken: + - no Secret objects in Git; + - every ServiceAccount disables token automounting; + - each platform overlay: labelled restricted namespace, quota, limit range, + default-deny and DNS-only policies, three service accounts; World0 alone is + labelled protected; + - the World0 admission policy denies what ADR-0069 lists, fails closed and is + bound with Deny; + - every AEvolution AppProject targets exactly its own namespace with no cluster + kinds; World0 blocks automated sync; no Application auto-syncs or carries the + cascade-delete finalizer, and each stays in its own environment; + - the live Argo path (deploy/kubernetes/dev) holds no Namespace, Secret, quota or + RBAC objects; + - the cleanup tool refuses World0 before invoking kubectl. +""" +import os +import shutil +import subprocess +import sys +import tempfile + +import yaml + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) +failures = [] + + +def check(condition, message): + if not condition: + failures.append(message) + + +def documents(path): + with open(path, encoding='utf-8') as f: + return [d for d in yaml.safe_load_all(f) if isinstance(d, dict)] + + +manifests = {} +for base, _, files in os.walk(os.path.join(ROOT, 'deploy')): + for name in files: + if name.endswith(('.yaml', '.yml', '.yaml.example')): + path = os.path.join(base, name) + manifests[os.path.relpath(path, ROOT).replace(os.sep, '/')] = documents(path) +objects = [(p, d) for p, ds in manifests.items() for d in ds + if 'kind' in d and d['kind'] != 'Kustomization'] + +# Secrets and service-account tokens. +for path, doc in objects: + check(doc['kind'] != 'Secret', f'{path}: Secret objects never belong in Git') + if doc['kind'] == 'ServiceAccount': + check(doc.get('automountServiceAccountToken') is False, + f'{path}: ServiceAccount {doc["metadata"]["name"]} must not automount a token') + +# Platform overlays. +platform = 'deploy/kubernetes/platform' +base = {d['kind'] + '/' + d['metadata']['name']: d + for p, d in objects if p.startswith(platform + '/base/')} +for key in ('ResourceQuota/aevolution-quota', 'LimitRange/aevolution-defaults', + 'NetworkPolicy/aevolution-default-deny', 'NetworkPolicy/aevolution-allow-dns', + 'ServiceAccount/aevolution-migrator', 'ServiceAccount/aevolution-region', + 'ServiceAccount/aevolution-backup'): + check(key in base, f'{platform}/base: missing {key}') +deny = base.get('NetworkPolicy/aevolution-default-deny', {}).get('spec', {}) +check(deny.get('podSelector') == {} and sorted(deny.get('policyTypes', [])) == ['Egress', 'Ingress'] + and not deny.get('ingress') and not deny.get('egress'), + 'default-deny must select every pod and allow nothing') +dns = base.get('NetworkPolicy/aevolution-allow-dns', {}).get('spec', {}) +dns_ports = [p.get('port') for rule in dns.get('egress', []) for p in rule.get('ports', [])] +check(dns.get('policyTypes') == ['Egress'] and set(dns_ports) == {53} and len(dns.get('egress', [])) == 1, + 'allow-dns must allow only egress to port 53') +quota = base.get('ResourceQuota/aevolution-quota', {}).get('spec', {}).get('hard', {}) +check(quota.get('services.loadbalancers') == '0' and quota.get('services.nodeports') == '0', + 'quota must forbid LoadBalancer and NodePort services') +for env in ('dev', 'stage', 'world0'): + k = f'{platform}/{env}/kustomization.yaml' + ns_path = f'{platform}/{env}/namespace.yaml' + check(k in manifests and ns_path in manifests, f'{platform}/{env}: overlay missing') + if k not in manifests or ns_path not in manifests: + continue + kustomization = manifests[k][0] + check(kustomization.get('namespace') == f'evolution-{env}' and '../base' in kustomization.get('resources', []), + f'{k}: must set namespace evolution-{env} and include ../base') + labels = manifests[ns_path][0]['metadata'].get('labels', {}) + check(manifests[ns_path][0]['metadata']['name'] == f'evolution-{env}', f'{ns_path}: wrong name') + check(labels.get('aevolution.io/environment') == env, f'{ns_path}: environment label') + check(labels.get('pod-security.kubernetes.io/enforce') == 'restricted', f'{ns_path}: restricted pod security') + check((labels.get('aevolution.io/protected') == 'true') == (env == 'world0'), + f'{ns_path}: only world0 is protected') + +# World0 admission policy. +policy = [d for p, d in objects if d['kind'] == 'ValidatingAdmissionPolicy' + and d['metadata']['name'] == 'aevolution-world0-protection'] +binding = [d for p, d in objects if d['kind'] == 'ValidatingAdmissionPolicyBinding' + and d['metadata']['name'] == 'aevolution-world0-protection'] +check(len(policy) == 1 and len(binding) == 1, 'world0 admission policy and binding must exist once') +if policy and binding: + spec = policy[0]['spec'] + check(spec.get('failurePolicy') == 'Fail', 'world0 policy must fail closed') + check(binding[0]['spec'].get('validationActions') == ['Deny'] and 'paramRef' not in binding[0]['spec'] + and 'matchResources' not in binding[0]['spec'], 'world0 binding must Deny everywhere') + covered = {(r, op) for rule in spec['matchConstraints']['resourceRules'] + for r in rule['resources'] for op in rule['operations']} + for resource in ('namespaces', 'persistentvolumes', 'persistentvolumeclaims', 'secrets', + 'serviceaccounts', 'applications', 'appprojects'): + check((resource, 'DELETE') in covered, f'world0 policy must cover DELETE {resource}') + for resource in ('namespaces', 'persistentvolumes'): + check((resource, 'UPDATE') in covered, f'world0 policy must cover UPDATE {resource}') + text = yaml.safe_dump(spec) + for needle in ("aevolution:world0-break-glass", "evolution-world0", "aevolution-world0", + "aevolution.io/protected", "Retain"): + check(needle in text, f'world0 policy must reference {needle}') + +# Argo CD projects and applications. +for path, doc in objects: + name = doc['metadata'].get('name', '') + if doc['kind'] == 'AppProject' and name.startswith('aevolution-'): + env = name[len('aevolution-'):] + spec = doc['spec'] + namespaces = [d.get('namespace') for d in spec.get('destinations', [])] + check(namespaces == [f'evolution-{env}'], f'{path}: {name} must target only evolution-{env}') + check(spec.get('clusterResourceWhitelist') == [], f'{path}: {name} must allow no cluster kinds') + kinds = {w.get('kind') for w in spec.get('namespaceResourceWhitelist', [])} + check(not kinds & {'Secret', 'Namespace', 'ResourceQuota', 'LimitRange', 'Role', 'RoleBinding', '*'}, + f'{path}: {name} whitelists a forbidden kind') + if env == 'world0': + windows = spec.get('syncWindows', []) + check(any(w.get('kind') == 'deny' and w.get('manualSync') is True and w.get('duration') == '24h' + and w.get('schedule') == '* * * * *' for w in windows), + f'{path}: world0 must deny automated sync at all times') + check('ServiceAccount' not in kinds, f'{path}: world0 service accounts are platform-managed') + if doc['kind'] == 'Application': + spec = doc['spec'] + project = spec.get('project', '') + destination = spec.get('destination', {}).get('namespace', '') + check('automated' not in (spec.get('syncPolicy') or {}), f'{path}: {name} must not auto-sync') + check('resources-finalizer.argocd.argoproj.io' not in doc['metadata'].get('finalizers', []), + f'{path}: {name} must not cascade-delete') + if project.startswith('aevolution-'): + check(destination == 'evolution-' + project[len('aevolution-'):], + f'{path}: {name} leaves its project environment') + check(destination != 'evolution-world0' or name == 'aevolution-world0', + f'{path}: only aevolution-world0 may target evolution-world0') + +# The live Argo path. +for path, doc in objects: + if path.startswith('deploy/kubernetes/dev/'): + check(doc['kind'] not in {'Namespace', 'Secret', 'ResourceQuota', 'LimitRange', 'Role', + 'RoleBinding', 'ClusterRole', 'ClusterRoleBinding'}, + f'{path}: {doc["kind"]} must not live in the Argo workload path') + +# The cleanup tool refuses World0 before kubectl runs (no kubectl on PATH here). +with tempfile.TemporaryDirectory() as empty: + env = dict(os.environ, PATH=empty + os.pathsep + os.path.dirname(sys.executable)) + for bad in ('world0', 'evolution-world0', 'World0', 'prod', ''): + result = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, 'tools/ops/cleanup-environment.sh'), bad], + capture_output=True, text=True, env=env) + check(result.returncode == 2 and ('refused' in result.stderr or 'usage' in result.stderr) + and 'kubectl' not in result.stderr, + f'cleanup must refuse {bad!r} before kubectl (rc={result.returncode})') + +for message in failures: + print('FAIL', message) +print(f'deploy policy: {len(objects)} objects in {len(manifests)} files, {len(failures)} failures') +sys.exit(1 if failures else 0) diff --git a/tools/ops/cleanup-environment.sh b/tools/ops/cleanup-environment.sh new file mode 100644 index 0000000..52ad457 --- /dev/null +++ b/tools/ops/cleanup-environment.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# M8.2 environment cleanup for disposable AEvolution environments only. +# +# tools/ops/cleanup-environment.sh [--execute] [--context ] +# +# Removes AEvolution workload objects (Deployments, StatefulSets, Jobs, Services, +# ConfigMaps and PersistentVolumeClaims labelled app.kubernetes.io/part-of=aevolution) +# from evolution-. Platform guardrails (namespace, quota, limit range, network +# policies, service accounts) and Secrets are never touched. Without --execute every +# delete is a server-side dry run. World0, or any namespace that is not labelled as +# the named disposable environment or is labelled protected, is refused before any +# change; the refusal of World0 happens before kubectl is even invoked. +set -euo pipefail +usage() { echo "usage: $0 [--execute] [--context ]" >&2; exit 2; } +[ $# -ge 1 ] || usage +env="$1"; shift +case "$env" in + dev|stage) ;; + *) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2 + exit 2 ;; +esac +dry_run=(--dry-run=server) +k=(kubectl --request-timeout=30s) +while [ $# -gt 0 ]; do + case "$1" in + --execute) dry_run=() ;; + --context) [ $# -ge 2 ] || usage; k+=(--context "$2"); shift ;; + *) usage ;; + esac + shift +done +ns="evolution-$env" +labels="$("${k[@]}" get namespace "$ns" -o jsonpath='{.metadata.labels.aevolution\.io/environment}/{.metadata.labels.aevolution\.io/protected}')" +if [ "$labels" != "$env/" ]; then + echo "refused: namespace $ns labels are '$labels' (need aevolution.io/environment=$env and no aevolution.io/protected)" >&2 + exit 2 +fi +selector='app.kubernetes.io/part-of=aevolution,app.kubernetes.io/managed-by!=aevolution-platform' +"${k[@]}" delete -n "$ns" "${dry_run[@]}" --wait=false -l "$selector" \ + deployments,statefulsets,jobs,services,configmaps,persistentvolumeclaims +[ ${#dry_run[@]} -eq 0 ] && echo "deleted AEvolution workloads in $ns" || + echo "dry run only (server-side); rerun with --execute to delete in $ns" diff --git a/tools/ops/verify-isolation.sh b/tools/ops/verify-isolation.sh new file mode 100644 index 0000000..4bb654d --- /dev/null +++ b/tools/ops/verify-isolation.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# M8.2 live isolation checks. Changes nothing: it only asks `kubectl auth can-i` +# (impersonating the environments' service accounts), reads objects, and sends +# DELETE requests with --dry-run=server, which run admission but never persist. +# +# tools/ops/verify-isolation.sh [--context ] +# +# Needs an identity that may impersonate service accounts and read namespaces, +# policies and Argo CD projects (a cluster administrator). Prints PASS/FAIL/SKIP per +# check and exits nonzero if any check fails. +set -uo pipefail +k=(kubectl --request-timeout=20s) +[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2") +fails=0 +pass() { echo "PASS $*"; } +fail() { echo "FAIL $*"; fails=$((fails + 1)); } +skip() { echo "SKIP $*"; } + +envs=(dev stage world0) +present=() +for env in "${envs[@]}"; do + if "${k[@]}" get namespace "evolution-$env" > /dev/null 2>&1; then + present+=("$env") + else + skip "namespace evolution-$env does not exist" + fi +done + +# 1. Guardrails and labels in every present environment. +for env in "${present[@]}"; do + ns="evolution-$env" + got="$("${k[@]}" get namespace "$ns" -o jsonpath='{.metadata.labels.aevolution\.io/environment} {.metadata.labels.pod-security\.kubernetes\.io/enforce} {.metadata.labels.aevolution\.io/protected}')" + want="$env restricted"; [ "$env" = world0 ] && want="$env restricted true" + [ "$(echo "$got" | xargs)" = "$want" ] && pass "$ns labels ($got)" || fail "$ns labels '$got' (want '$want')" + for object in resourcequota/aevolution-quota limitrange/aevolution-defaults \ + networkpolicy/aevolution-default-deny networkpolicy/aevolution-allow-dns \ + serviceaccount/aevolution-migrator serviceaccount/aevolution-region serviceaccount/aevolution-backup; do + "${k[@]}" get -n "$ns" "$object" > /dev/null 2>&1 && pass "$ns has $object" || fail "$ns lacks $object" + done +done + +# 2. No environment identity can read secrets or change anything anywhere. +for env in "${present[@]}"; do + for sa in aevolution-migrator aevolution-region aevolution-backup; do + who="system:serviceaccount:evolution-$env:$sa" + for target in "${envs[@]}"; do + for action in "get secrets" "list secrets" "create pods" "delete persistentvolumeclaims" "create rolebindings"; do + answer="$("${k[@]}" auth can-i $action -n "evolution-$target" --as "$who" 2>/dev/null)" + [ "$answer" = "no" ] && pass "$who cannot $action in evolution-$target" || + fail "$who can $action in evolution-$target" + done + done + answer="$("${k[@]}" auth can-i delete namespaces --as "$who" 2>/dev/null)" + [ "$answer" = "no" ] && pass "$who cannot delete namespaces" || fail "$who can delete namespaces" + done +done + +# 3. World0 admission protection, exercised with server-side dry runs. +if "${k[@]}" get validatingadmissionpolicybinding aevolution-world0-protection > /dev/null 2>&1; then + pass "world0 protection policy is bound" +else + fail "world0 protection policy binding missing" +fi +denied() { # denied (always --dry-run=server) + local what="$1"; shift + local out + out="$("${k[@]}" delete --dry-run=server --wait=false "$@" 2>&1)" + if echo "$out" | grep -q "aevolution:world0-break-glass"; then + pass "denied: $what" + elif echo "$out" | grep -qi "not found"; then + skip "$what (object absent)" + else + fail "not denied: $what -> $out" + fi +} +if printf '%s\n' "${present[@]}" | grep -qx world0; then + denied "delete namespace evolution-world0" namespace evolution-world0 + denied "delete service account in evolution-world0" -n evolution-world0 serviceaccount aevolution-region + secret="$("${k[@]}" get secrets -n evolution-world0 -o name 2>/dev/null | head -1)" + [ -n "$secret" ] && denied "delete $secret in evolution-world0" -n evolution-world0 "$secret" || + skip "no Secret in evolution-world0 to test" + pvc="$("${k[@]}" get pvc -n evolution-world0 -o name 2>/dev/null | head -1)" + [ -n "$pvc" ] && denied "delete $pvc in evolution-world0" -n evolution-world0 "$pvc" || + skip "no PVC in evolution-world0 to test" +else + skip "world0 admission dry runs (namespace absent)" +fi + +# 4. Argo CD projects: each AEvolution project targets only its own namespace. +for env in "${envs[@]}"; do + dest="$("${k[@]}" get appproject "aevolution-$env" -n argocd -o jsonpath='{range .spec.destinations[*]}{.namespace}{" "}{end}{.spec.clusterResourceWhitelist}' 2>/dev/null)" || { + skip "appproject aevolution-$env not present"; continue; } + [ "$(echo "$dest" | xargs)" = "evolution-$env" ] && pass "appproject aevolution-$env destinations '$dest'" || + fail "appproject aevolution-$env destinations/cluster kinds '$dest'" +done +wide="$("${k[@]}" get appproject default -n argocd -o jsonpath='{.spec.destinations[*].namespace}' 2>/dev/null)" +[ "$wide" = "*" ] && echo "WARN appproject default allows every namespace; its applications could create objects in evolution-* (deletion is still denied in world0 by admission)" + +# 5. The cleanup tool refuses World0 before touching the cluster. +root="$(cd "$(dirname "$0")/../.." && pwd)" +for bad in world0 evolution-world0 World0 ""; do + if bash "$root/tools/ops/cleanup-environment.sh" "$bad" > /dev/null 2>&1; then + fail "cleanup accepted '$bad'" + else + pass "cleanup refused '$bad'" + fi +done + +echo "failures=$fails" +[ "$fails" -eq 0 ]