Add M8.2 environment isolation and World0 protection
Per-environment platform layers (dev, stage, world0) with restricted pod security, quotas, limit ranges, default-deny networking and token-less service accounts, applied by an administrator rather than Argo. A native ValidatingAdmissionPolicy denies deleting World0 data, identity and Argo objects to every requester without the break-glass group. Stage and World0 Argo projects are scoped to one namespace, World0 blocks automated sync, and a cleanup tool refuses World0. An offline policy check runs as a new CI suite; a live verification script uses only can-i, reads and server dry runs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -19,7 +19,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
suite: [checks, tsan, ecs, postgres]
|
||||
suite: [checks, tsan, ecs, postgres, deploy]
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: https://github.com/actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
@@ -152,3 +152,14 @@ plus the pinned libpq, UID 10001, no credentials). It smoke-tests the image with
|
||||
- the runtime Secret, object PVC and a NetworkPolicy.
|
||||
Pin the digest and provision the region first. Without a gateway, the region
|
||||
receives no inputs.
|
||||
|
||||
## Environments and World0 protection (M8.2)
|
||||
|
||||
`kubernetes/platform/<env>` holds each environment's namespace, quota, limit range,
|
||||
default-deny policies and service accounts. `kubernetes/platform/admission` holds
|
||||
the World0 admission policy. A cluster administrator applies these by hand; they
|
||||
are never part of an Argo Application. The stage and World0 Argo projects and
|
||||
applications are in `argocd/stage` and `argocd/world0` as examples.
|
||||
|
||||
See docs/infrastructure/ENVIRONMENTS.md for the runbook, and ADR-0069. The CI
|
||||
`deploy` suite runs `tools/ci/deploy-policy-check.py` over every manifest.
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# M8.2 (stage). REPLACE_WORKLOAD_PATH is deliberately unresolved: the stage
|
||||
# workload is M8.4 work. Manual sync only; no automated policy, no prune on sync of
|
||||
# durable objects (PVCs carry argocd.argoproj.io/sync-options: Prune=false,Delete=false).
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: aevolution-stage
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: aevolution-stage
|
||||
source:
|
||||
repoURL: https://git.aeternalisgames.com/dingo/AEvolution.git
|
||||
targetRevision: main
|
||||
path: REPLACE_WORKLOAD_PATH
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: evolution-stage
|
||||
syncPolicy:
|
||||
syncOptions:
|
||||
- FailOnSharedResource=true
|
||||
- PruneLast=true
|
||||
@@ -0,0 +1,26 @@
|
||||
# M8.2 (stage). Apply by hand to the argocd namespace; never from an Application.
|
||||
# One destination namespace only, no cluster-scoped kinds, no Secrets or quotas.
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: AppProject
|
||||
metadata:
|
||||
name: aevolution-stage
|
||||
namespace: argocd
|
||||
spec:
|
||||
description: AEvolution stage (evolution-stage only)
|
||||
sourceRepos:
|
||||
- https://git.aeternalisgames.com/dingo/AEvolution.git
|
||||
destinations:
|
||||
- server: https://kubernetes.default.svc
|
||||
namespace: evolution-stage
|
||||
clusterResourceWhitelist: []
|
||||
namespaceResourceWhitelist:
|
||||
- {group: "", kind: Service}
|
||||
- {group: "", kind: ConfigMap}
|
||||
- {group: "", kind: PersistentVolumeClaim}
|
||||
- {group: apps, kind: Deployment}
|
||||
- {group: batch, kind: Job}
|
||||
- {group: networking.k8s.io, kind: NetworkPolicy}
|
||||
# Not listed, so never Argo-managed: Namespaces, ServiceAccounts (platform layer),
|
||||
# Secrets, quotas, limit ranges and RBAC.
|
||||
orphanedResources:
|
||||
warn: true
|
||||
@@ -0,0 +1,25 @@
|
||||
# M8.2 (world0). REPLACE_WORKLOAD_PATH is deliberately unresolved: the world0
|
||||
# workload is M8.4 work. Manual sync only; no automated policy, no prune on sync of
|
||||
# durable objects (PVCs carry argocd.argoproj.io/sync-options: Prune=false,Delete=false).
|
||||
# World0: never add the resources-finalizer.argocd.argoproj.io finalizer (deleting
|
||||
# the Application must orphan, not cascade), never add an automated policy, and never
|
||||
# list this Application in a parent app-of-apps with prune. The admission policy
|
||||
# also denies deleting this Application without break-glass.
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: aevolution-world0
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: aevolution-world0
|
||||
source:
|
||||
repoURL: https://git.aeternalisgames.com/dingo/AEvolution.git
|
||||
targetRevision: main
|
||||
path: REPLACE_WORKLOAD_PATH
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: evolution-world0
|
||||
syncPolicy:
|
||||
syncOptions:
|
||||
- FailOnSharedResource=true
|
||||
- PruneLast=true
|
||||
@@ -0,0 +1,33 @@
|
||||
# M8.2 (world0). Apply by hand to the argocd namespace; never from an Application.
|
||||
# One destination namespace only, no cluster-scoped kinds, no Secrets or quotas.
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: AppProject
|
||||
metadata:
|
||||
name: aevolution-world0
|
||||
namespace: argocd
|
||||
spec:
|
||||
description: AEvolution world0 (evolution-world0 only)
|
||||
sourceRepos:
|
||||
- https://git.aeternalisgames.com/dingo/AEvolution.git
|
||||
destinations:
|
||||
- server: https://kubernetes.default.svc
|
||||
namespace: evolution-world0
|
||||
clusterResourceWhitelist: []
|
||||
namespaceResourceWhitelist:
|
||||
- {group: "", kind: Service}
|
||||
- {group: "", kind: ConfigMap}
|
||||
- {group: "", kind: PersistentVolumeClaim}
|
||||
- {group: apps, kind: Deployment}
|
||||
- {group: batch, kind: Job}
|
||||
- {group: networking.k8s.io, kind: NetworkPolicy}
|
||||
# Not listed, so never Argo-managed: Namespaces, ServiceAccounts (platform layer),
|
||||
# Secrets, quotas, limit ranges and RBAC.
|
||||
orphanedResources:
|
||||
warn: true
|
||||
# No automated sync can ever run; a human may still sync by hand.
|
||||
syncWindows:
|
||||
- kind: deny
|
||||
schedule: "* * * * *"
|
||||
duration: 24h
|
||||
applications: ["*"]
|
||||
manualSync: true
|
||||
@@ -45,6 +45,19 @@ RUN cmake --preset analysis -B build/ecs-analysis -DCMAKE_CXX_COMPILER=clang++-1
|
||||
&& cmake --build build/ecs-analysis --target evo-ecs-experiment --parallel 2 \
|
||||
&& ctest --test-dir build/ecs-analysis -R ecs-experiment --output-on-failure --no-tests=error
|
||||
|
||||
# M8.2 offline deployment-policy check (ADR-0069): Ubuntu's python3-yaml only, no
|
||||
# cluster access. Fails the build when an isolation or World0 invariant breaks.
|
||||
FROM ${UBUNTU_IMAGE} AS ci-deploy
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends python3 python3-yaml \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /source
|
||||
COPY deploy/ deploy/
|
||||
COPY tools/ci/deploy-policy-check.py tools/ci/
|
||||
COPY tools/ops/ tools/ops/
|
||||
RUN python3 tools/ci/deploy-policy-check.py
|
||||
|
||||
FROM ci-toolchain AS postgres-build
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl bzip2 make flex bison perl libssl-dev zlib1g-dev pkg-config \
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
# M8.2 World0 protection (ADR-0069). A native ValidatingAdmissionPolicy (GA since
|
||||
# Kubernetes 1.30; the cluster runs 1.32): no webhook, no new dependency. It applies
|
||||
# to every requester, including cluster-admin (system:masters), the Argo CD
|
||||
# application controller and any Application in any AppProject, so no permissive
|
||||
# app, prune, parent-app cascade or credential can remove World0 data or identity.
|
||||
# Only a member of the group `aevolution:world0-break-glass` passes. Membership is
|
||||
# granted out of band (client certificate O= or OIDC group) for a recorded operation.
|
||||
#
|
||||
# Denied without break-glass:
|
||||
# - deleting Namespace evolution-world0, or removing its aevolution.io/protected label;
|
||||
# - deleting PersistentVolumeClaims, Secrets or ServiceAccounts in evolution-world0;
|
||||
# - deleting a PersistentVolume bound to evolution-world0, or changing its reclaim
|
||||
# policy away from Retain once set (set Retain when the volume is bound: the
|
||||
# only StorageClass, local-path, provisions with Delete);
|
||||
# - deleting the aevolution-world0 Argo CD Application or AppProject.
|
||||
# Kubernetes does not run admission policies on policy objects themselves, so
|
||||
# removing this policy is itself the audited break-glass step (cluster-admin).
|
||||
# Apply by hand: kubectl apply -f deploy/kubernetes/platform/admission/
|
||||
apiVersion: admissionregistration.k8s.io/v1
|
||||
kind: ValidatingAdmissionPolicy
|
||||
metadata:
|
||||
name: aevolution-world0-protection
|
||||
labels:
|
||||
app.kubernetes.io/part-of: aevolution
|
||||
spec:
|
||||
failurePolicy: Fail
|
||||
matchConstraints:
|
||||
resourceRules:
|
||||
- apiGroups: [""]
|
||||
apiVersions: ["v1"]
|
||||
operations: ["DELETE", "UPDATE"]
|
||||
resources: ["namespaces", "persistentvolumes"]
|
||||
- apiGroups: [""]
|
||||
apiVersions: ["v1"]
|
||||
operations: ["DELETE"]
|
||||
resources: ["persistentvolumeclaims", "secrets", "serviceaccounts"]
|
||||
- apiGroups: ["argoproj.io"]
|
||||
apiVersions: ["*"]
|
||||
operations: ["DELETE"]
|
||||
resources: ["applications", "appprojects"]
|
||||
variables:
|
||||
- name: breakGlass
|
||||
expression: "request.userInfo.groups.exists(g, g == 'aevolution:world0-break-glass')"
|
||||
- name: resource
|
||||
expression: "request.resource.resource"
|
||||
- name: world0Namespace
|
||||
expression: "variables.resource == 'namespaces' && request.name == 'evolution-world0'"
|
||||
- name: world0Volume
|
||||
expression: >-
|
||||
variables.resource == 'persistentvolumes' &&
|
||||
has(oldObject.spec.claimRef) && has(oldObject.spec.claimRef.namespace) &&
|
||||
oldObject.spec.claimRef.namespace == 'evolution-world0'
|
||||
- name: world0Object
|
||||
expression: >-
|
||||
request.namespace == 'evolution-world0' &&
|
||||
variables.resource in ['persistentvolumeclaims', 'secrets', 'serviceaccounts']
|
||||
- name: world0Delivery
|
||||
expression: >-
|
||||
request.resource.group == 'argoproj.io' && request.name == 'aevolution-world0'
|
||||
validations:
|
||||
- expression: >-
|
||||
variables.breakGlass || request.operation != 'DELETE' ||
|
||||
!(variables.world0Namespace || variables.world0Volume || variables.world0Object ||
|
||||
variables.world0Delivery)
|
||||
message: "World0 data, identity and delivery objects cannot be deleted without the aevolution:world0-break-glass group"
|
||||
reason: Forbidden
|
||||
- expression: >-
|
||||
variables.breakGlass || request.operation != 'UPDATE' || !variables.world0Namespace ||
|
||||
(has(object.metadata.labels) && 'aevolution.io/protected' in object.metadata.labels &&
|
||||
object.metadata.labels['aevolution.io/protected'] == 'true')
|
||||
message: "The World0 namespace must keep aevolution.io/protected=true"
|
||||
reason: Forbidden
|
||||
- expression: >-
|
||||
variables.breakGlass || request.operation != 'UPDATE' || !variables.world0Volume ||
|
||||
oldObject.spec.persistentVolumeReclaimPolicy != 'Retain' ||
|
||||
object.spec.persistentVolumeReclaimPolicy == 'Retain'
|
||||
message: "A World0 PersistentVolume set to Retain cannot be changed back"
|
||||
reason: Forbidden
|
||||
---
|
||||
apiVersion: admissionregistration.k8s.io/v1
|
||||
kind: ValidatingAdmissionPolicyBinding
|
||||
metadata:
|
||||
name: aevolution-world0-protection
|
||||
labels:
|
||||
app.kubernetes.io/part-of: aevolution
|
||||
spec:
|
||||
policyName: aevolution-world0-protection
|
||||
validationActions: [Deny]
|
||||
@@ -0,0 +1,15 @@
|
||||
# M8.2 per-environment guardrails (docs/infrastructure/ENVIRONMENTS.md). Applied by a
|
||||
# cluster administrator with `kubectl apply -k deploy/kubernetes/platform/<env>`, never
|
||||
# through an Argo CD Application: namespaces, quotas and identities must not be
|
||||
# prunable by a workload sync. Each overlay sets the namespace and its quota values.
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- quota.yaml
|
||||
- limitrange.yaml
|
||||
- networkpolicies.yaml
|
||||
- serviceaccounts.yaml
|
||||
labels:
|
||||
- pairs:
|
||||
app.kubernetes.io/part-of: aevolution
|
||||
app.kubernetes.io/managed-by: aevolution-platform
|
||||
@@ -0,0 +1,17 @@
|
||||
# Defaults for containers that omit resources (the quota requires them), sized from
|
||||
# the measured region-host profile (peak RSS about 9.7 MiB): 64Mi request, 256Mi
|
||||
# limit, 50m CPU request and no default CPU limit. No container may exceed 2Gi.
|
||||
apiVersion: v1
|
||||
kind: LimitRange
|
||||
metadata:
|
||||
name: aevolution-defaults
|
||||
spec:
|
||||
limits:
|
||||
- type: Container
|
||||
defaultRequest:
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
default:
|
||||
memory: 256Mi
|
||||
max:
|
||||
memory: 2Gi
|
||||
@@ -0,0 +1,30 @@
|
||||
# Deny by default: every pod in the namespace has no ingress or egress except DNS.
|
||||
# Workloads add narrow allowances with their own policies (for example the region
|
||||
# host's egress to exactly the database VM address and port, and ingress to its
|
||||
# operations port from the metrics scraper) in M8.4. Enforced by Canal/Calico.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: aevolution-default-deny
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes: [Ingress, Egress]
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: aevolution-allow-dns
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes: [Egress]
|
||||
egress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: kube-system
|
||||
podSelector:
|
||||
matchLabels:
|
||||
k8s-app: kube-dns
|
||||
ports:
|
||||
- {protocol: UDP, port: 53}
|
||||
- {protocol: TCP, port: 53}
|
||||
@@ -0,0 +1,17 @@
|
||||
# Values are replaced per environment. Memory is quota'd on requests and limits;
|
||||
# CPU only on requests (region hosts run without CPU limits: throttling would delay
|
||||
# lease renewals, docs/infrastructure/INVENTORY.md). No LoadBalancer or NodePort.
|
||||
apiVersion: v1
|
||||
kind: ResourceQuota
|
||||
metadata:
|
||||
name: aevolution-quota
|
||||
spec:
|
||||
hard:
|
||||
requests.cpu: "2"
|
||||
requests.memory: 2Gi
|
||||
limits.memory: 4Gi
|
||||
pods: "20"
|
||||
persistentvolumeclaims: "4"
|
||||
requests.storage: 40Gi
|
||||
services.loadbalancers: "0"
|
||||
services.nodeports: "0"
|
||||
@@ -0,0 +1,26 @@
|
||||
# One identity per role; none needs the Kubernetes API, so none mounts a token and
|
||||
# no Role or RoleBinding is granted. Secrets are referenced by name only and are
|
||||
# created per environment by an operator (ENVIRONMENTS.md); they never enter Git.
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: aevolution-migrator # Runs migrations with the migrator database login.
|
||||
automountServiceAccountToken: false
|
||||
imagePullSecrets:
|
||||
- name: harbor-pull
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: aevolution-region # evo-server with the runtime database login.
|
||||
automountServiceAccountToken: false
|
||||
imagePullSecrets:
|
||||
- name: harbor-pull
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: aevolution-backup # Read-only backup/clone jobs (M8.7).
|
||||
automountServiceAccountToken: false
|
||||
imagePullSecrets:
|
||||
- name: harbor-pull
|
||||
@@ -0,0 +1,9 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: evolution-dev
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- ../base
|
||||
labels:
|
||||
- pairs:
|
||||
aevolution.io/environment: dev
|
||||
@@ -0,0 +1,10 @@
|
||||
# Disposable development. Existing namespace; applying adds labels and guardrails.
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: evolution-dev
|
||||
labels:
|
||||
aevolution.io/environment: dev
|
||||
pod-security.kubernetes.io/enforce: restricted
|
||||
pod-security.kubernetes.io/enforce-version: v1.32
|
||||
pod-security.kubernetes.io/warn: restricted
|
||||
@@ -0,0 +1,9 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: evolution-stage
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- ../base
|
||||
labels:
|
||||
- pairs:
|
||||
aevolution.io/environment: stage
|
||||
@@ -0,0 +1,10 @@
|
||||
# Clone, migration and soak rehearsals (M8.6/8.9). Disposable, never World0 data.
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: evolution-stage
|
||||
labels:
|
||||
aevolution.io/environment: stage
|
||||
pod-security.kubernetes.io/enforce: restricted
|
||||
pod-security.kubernetes.io/enforce-version: v1.32
|
||||
pod-security.kubernetes.io/warn: restricted
|
||||
@@ -0,0 +1,29 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: evolution-world0
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- ../base
|
||||
patches:
|
||||
- target:
|
||||
kind: ResourceQuota
|
||||
name: aevolution-quota
|
||||
patch: |-
|
||||
- op: replace
|
||||
path: /spec/hard/requests.cpu
|
||||
value: "4"
|
||||
- op: replace
|
||||
path: /spec/hard/requests.memory
|
||||
value: 4Gi
|
||||
- op: replace
|
||||
path: /spec/hard/limits.memory
|
||||
value: 8Gi
|
||||
- op: replace
|
||||
path: /spec/hard/persistentvolumeclaims
|
||||
value: "8"
|
||||
- op: replace
|
||||
path: /spec/hard/requests.storage
|
||||
value: 200Gi
|
||||
labels:
|
||||
- pairs:
|
||||
aevolution.io/environment: world0
|
||||
@@ -0,0 +1,11 @@
|
||||
# Persistent World0. Guarded by deploy/kubernetes/platform/admission. Nothing here deploys a world; launch stays gated by M8.8.
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: evolution-world0
|
||||
labels:
|
||||
aevolution.io/environment: world0
|
||||
pod-security.kubernetes.io/enforce: restricted
|
||||
pod-security.kubernetes.io/enforce-version: v1.32
|
||||
pod-security.kubernetes.io/warn: restricted
|
||||
aevolution.io/protected: "true"
|
||||
@@ -606,3 +606,13 @@ M8.1 complete (2026-09-29): docs/infrastructure/INVENTORY.md records the cluster
|
||||
the separate PostgreSQL VM, the measured workload profile and a per-worker
|
||||
`local-path` storage probe. Storage figures are provisional until the planned RAID
|
||||
and storage change. Next: M8.2, environment isolation and secrets.
|
||||
|
||||
M8.2 designed and checked offline (2026-09-29), under ADR-0069:
|
||||
- dev, stage and World0 namespaces, each with its own guardrails, identities and
|
||||
secret references;
|
||||
- a native admission policy that protects World0 from every requester except a
|
||||
break-glass group;
|
||||
- scoped Argo projects, and a cleanup tool that refuses World0;
|
||||
- an offline policy check in CI, and a live isolation check.
|
||||
The live checks run once the maintainer applies the platform layer. Next: M8.3,
|
||||
durable stores in dev and stage.
|
||||
|
||||
+17
-1
@@ -993,4 +993,20 @@ M8.1 complete (2026-09-29) for the current disk layout. The per-worker
|
||||
These are well inside the lease deadlines. Re-probe after the planned RAID and
|
||||
storage change. Host memory was raised by about 500 GB.
|
||||
|
||||
Next: M8.2, environment isolation and secrets.
|
||||
Next: M8.2, environment isolation and secrets.
|
||||
## M8.2 environment isolation (2026-09-29)
|
||||
|
||||
ADR-0069 adds three environments, each with its own guardrails, service accounts,
|
||||
secret references and database:
|
||||
- `evolution-dev`, `evolution-stage` and `evolution-world0`;
|
||||
- guardrails: quota, limit range, default-deny networking, restricted pod
|
||||
security.
|
||||
A native ValidatingAdmissionPolicy denies deleting World0 data, identity or Argo
|
||||
objects to everyone, including cluster-admin and Argo, unless the requester is in
|
||||
`aevolution:world0-break-glass`. The stage and World0 Argo projects are scoped to
|
||||
one namespace each, and World0 permanently denies automated sync. The cleanup tool
|
||||
refuses World0.
|
||||
|
||||
The offline policy check (now a CI suite) passes, and nine injected violations
|
||||
were each caught. Applying the layer and running `tools/ops/verify-isolation.sh`
|
||||
is waiting on the maintainer.
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
# ADR-0069: Environment isolation and World0 protection
|
||||
|
||||
## Status
|
||||
Accepted for M8.2, 2026-09-29. It builds on the M8.1 inventory
|
||||
(infrastructure/INVENTORY.md), ADR-0020 (delivery), and OPERATIONS.md (World0
|
||||
protection). Nothing here deploys a world; World0 launch stays gated by M8.8.
|
||||
|
||||
## Context
|
||||
The inventory found five things that matter here:
|
||||
- Argo CD's `default` AppProject allows every cluster resource in every namespace.
|
||||
Three applications use it, two with automated prune and self-heal.
|
||||
- The only StorageClass reclaims with Delete.
|
||||
- Collection ran as `system:admin`.
|
||||
- There are no quotas, limit ranges or priority classes.
|
||||
- Only the AEvolution application is manual-sync.
|
||||
|
||||
The roadmap's named risk is that one permissive application or credential could
|
||||
destroy every world. Argo project boundaries cannot address that alone, because a
|
||||
`default`-project application, a cascading parent application or a cluster-admin
|
||||
is not constrained by an AEvolution AppProject.
|
||||
|
||||
## Decision
|
||||
**Three environments,** each in its own namespace: `evolution-dev`,
|
||||
`evolution-stage` and `evolution-world0`. Each has its own AppProject, service
|
||||
accounts, Secrets, database, database logins and object PVC. No identity, Secret or
|
||||
database is shared between them.
|
||||
|
||||
**Platform layer** (`deploy/kubernetes/platform/<env>`), applied by a cluster
|
||||
administrator and never by an Argo CD Application, so no sync or prune can remove
|
||||
it. It contains:
|
||||
- The Namespace, labelled with its environment and with Pod Security `restricted`
|
||||
enforced. Only World0 carries `aevolution.io/protected=true`.
|
||||
- A ResourceQuota:
|
||||
- dev and stage: 2 CPU and 2 GiB requests, 4 GiB of memory limits, 4 PVCs and
|
||||
40 GiB;
|
||||
- World0: 4 CPU and 4 GiB requests, 8 GiB of limits, 8 PVCs and 200 GiB;
|
||||
- no LoadBalancer or NodePort services anywhere.
|
||||
- A LimitRange (defaults 50m/64Mi with a 256Mi limit, at most 2Gi per container)
|
||||
sized from the measured host profile.
|
||||
- A default-deny NetworkPolicy for every pod, plus DNS egress only. Workloads add
|
||||
narrow rules in M8.4.
|
||||
- Service accounts `aevolution-migrator`, `aevolution-region` and
|
||||
`aevolution-backup`, with no token and no RBAC; nothing needs the Kubernetes API.
|
||||
They are platform-owned, so workload projects cannot create them.
|
||||
|
||||
**Secrets never enter Git.** Operators create them per environment, by reference
|
||||
name (ENVIRONMENTS.md). CI holds no cluster or data credentials (ADR-0020).
|
||||
|
||||
**World0 admission protection.** A native ValidatingAdmissionPolicy
|
||||
(`deploy/kubernetes/platform/admission`) is GA on the cluster's Kubernetes 1.32. It
|
||||
needs no webhook or dependency, and it fails closed. It applies to everyone,
|
||||
including system:masters and Argo's controller. Unless the requester is in the
|
||||
group `aevolution:world0-break-glass`, it denies:
|
||||
- deleting the World0 namespace, or removing its protected label;
|
||||
- deleting its PVCs, Secrets or service accounts;
|
||||
- deleting a PV bound to it, or moving such a PV away from `Retain`;
|
||||
- deleting the `aevolution-world0` Application or AppProject.
|
||||
|
||||
A cascading parent app, a prune, a `default`-project application or a stray
|
||||
`kubectl delete` therefore fails at admission. Kubernetes does not apply policies
|
||||
to policy objects, so removing the policy is itself the break-glass action,
|
||||
reserved to cluster-admins.
|
||||
|
||||
**Delivery.**
|
||||
- The stage and World0 AppProjects (examples) each target one namespace. They
|
||||
allow no cluster kinds, and whitelist only namespaced workload kinds.
|
||||
- World0 carries a permanent deny sync window with `manualSync: true`, so no
|
||||
automated sync can ever run.
|
||||
- Applications never auto-sync and never carry the cascade finalizer. World0's must
|
||||
not appear in a pruning parent.
|
||||
- Durable PVCs will carry `Prune=false,Delete=false`.
|
||||
|
||||
**Cleanup.** `tools/ops/cleanup-environment.sh` accepts only `dev` or `stage`. It
|
||||
refuses anything else before invoking kubectl, re-checks the namespace labels,
|
||||
never touches platform objects or Secrets, and dry-runs unless `--execute` is given.
|
||||
|
||||
**Verification.**
|
||||
- `tools/ci/deploy-policy-check.py` runs offline in CI (Ubuntu `python3-yaml`) and
|
||||
asserts every invariant above over all manifests.
|
||||
- `tools/ops/verify-isolation.sh` runs live. It makes only `can-i` queries, reads
|
||||
and server dry-runs, and covers negative RBAC, cross-environment denial, World0
|
||||
admission denials, project scopes and the cleanup refusal.
|
||||
|
||||
## Consequences
|
||||
- **Residual risk: creation.** A `default`-project application can still *create*
|
||||
objects in AEvolution namespaces; admission protects only World0's data and
|
||||
identity from deletion. Recommended (owner's decision): restrict `default` or
|
||||
move those applications to scoped projects.
|
||||
- **Residual risk: advisory locks.** The runtime login's advisory-lock capability
|
||||
(ADR-0068) remains.
|
||||
- **Local-path storage.** Its PVs are Delete-reclaim. A World0 PV must be patched
|
||||
to Retain when bound, and the policy then keeps it there. Volumes remain
|
||||
node-local until M8.3.
|
||||
- **Quotas and defaults.** Quotas are starting values from measured profiles. They
|
||||
are raised by editing the platform overlay, never by the workload.
|
||||
- **Existing dev namespace.** Applying the dev overlay to the existing namespace
|
||||
adds labels, the quota and the default-deny policy. Its proof Job already
|
||||
complies.
|
||||
|
||||
## Alternatives rejected
|
||||
- **An admission webhook or policy engine (Kyverno, Gatekeeper).** Either is a new
|
||||
dependency with its own availability risk; the native policy suffices.
|
||||
- **Relying on AppProjects alone.** They do not constrain other projects, parents
|
||||
or cluster-admins.
|
||||
- **Namespaces created by Argo `CreateNamespace`.** A prune or cascade could then
|
||||
remove them.
|
||||
@@ -79,3 +79,4 @@ design. No unadopted candidate is a link dependency of the proof.
|
||||
- [ADR-0066: Operations endpoints](0066-operations-endpoints.md) - accepted for M7.8 liveness vs authority readiness, bounded metrics, dependency-free responder and the server image.
|
||||
- [ADR-0067: Region clones and retained-history audit](0067-region-clones.md) - accepted for M7.9a re-identified clone genesis naming its source cut, isolated clone databases and full-chain retention audit.
|
||||
- [ADR-0068: Upgrade barrier and durability qualification](0068-upgrade-barrier.md) - accepted for M7.9b build-bound authority, retirement barrier fencing old writers, explicit refusal of rule/format changes and the measured recovery fixture.
|
||||
- [ADR-0069: Environment isolation and World0 protection](0069-environment-isolation.md) - accepted for M8.2 per-environment guardrails, native World0 admission protection, scoped Argo projects and verification.
|
||||
|
||||
@@ -2639,3 +2639,27 @@ Evidence:
|
||||
- Collection is manual, with no automation planned.
|
||||
- No credential values were read. The 18.6 version, role and TLS requirements are
|
||||
enforced by the adapter and migrations when M8.3 connects.
|
||||
|
||||
## M8.2 environment isolation - 2026-09-29
|
||||
|
||||
- `tools/ci/deploy-policy-check.py` (WSL, Python 3 with PyYAML): 35 objects in 27
|
||||
files, 0 failures.
|
||||
- Nine mutations were each caught, then restored:
|
||||
- World0 auto-sync;
|
||||
- the World0 cascade finalizer;
|
||||
- the policy set to failurePolicy Ignore;
|
||||
- the policy no longer covering Secrets;
|
||||
- an egress hole in default-deny;
|
||||
- the World0 protected label removed;
|
||||
- the cleanup tool accepting world0 (rc 127, it reached kubectl);
|
||||
- service-account token automount;
|
||||
- a wildcard stage destination.
|
||||
- `kubectl kustomize` (client v1.31.4) renders the dev, stage and World0 platform
|
||||
overlays (9 objects each), and still renders the live `deploy/kubernetes/dev`
|
||||
path unchanged.
|
||||
- `bash -n` passes for both ops scripts.
|
||||
- The CI `deploy` suite (a Docker `ci-deploy` stage) has not run yet; Docker is
|
||||
not running locally.
|
||||
- Not executed: applying anything to the cluster, the admission policy's CEL
|
||||
compilation (it needs an API server; the runbook validates it with a server
|
||||
dry run first), and `tools/ops/verify-isolation.sh`.
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
# AEvolution environments (M8.2)
|
||||
|
||||
The decision is recorded in ADR-0069. This runbook applies the platform layer,
|
||||
creates the per-environment references, and verifies isolation. Every step is
|
||||
manual, and none is performed by Argo CD.
|
||||
|
||||
## Environments
|
||||
|
||||
| | dev | stage | world0 |
|
||||
|---|---|---|---|
|
||||
| Namespace | `evolution-dev` (exists) | `evolution-stage` | `evolution-world0` |
|
||||
| Purpose | Development | Clone, migration and soak rehearsal | Persistent world (launch gated by M8.8) |
|
||||
| Protected by admission | No | No | Yes |
|
||||
| Cleanup tool | Allowed | Allowed | Refused |
|
||||
| AppProject / Application | `aevolution-dev` (live, manual) | `deploy/argocd/stage/*.example` | `deploy/argocd/world0/*.example` (deny window) |
|
||||
| Database | Its own database and logins on the AEvolution PostgreSQL VM | Its own | Its own |
|
||||
| World IDs | Development range | Clone targets (ADR-0067) | World0's own; never reused elsewhere |
|
||||
|
||||
## Secret references (created by an operator, never committed)
|
||||
|
||||
Each namespace gets its own Secrets, with environment-specific values:
|
||||
- `harbor-pull`: a pull-only Harbor robot (docker-registry type).
|
||||
- `evolution-postgres-migrator`: the migrator/owner login for that environment's
|
||||
database.
|
||||
- `evolution-postgres-runtime`: the runtime login, a member of that environment's
|
||||
NOLOGIN runtime group (ADR-0060).
|
||||
- `aevolution-db`: connection settings (host, address, port, database, TLS mode
|
||||
and root certificate). Mount it as files. The adapter reads a password file,
|
||||
never `PGPASSWORD`.
|
||||
|
||||
A World0 credential must never exist in the dev or stage namespaces, and no dev or
|
||||
stage credential may reach the World0 database. Rotate by replacing the Secret and
|
||||
restarting the consumer.
|
||||
|
||||
## Apply (cluster administrator)
|
||||
|
||||
```sh
|
||||
# 1. Platform guardrails, per environment (dev adds labels, quota and policies to
|
||||
# the existing namespace; its completed proof Job already complies).
|
||||
kubectl apply -k deploy/kubernetes/platform/dev
|
||||
kubectl apply -k deploy/kubernetes/platform/stage
|
||||
kubectl apply -k deploy/kubernetes/platform/world0 # an empty, protected namespace
|
||||
# 2. World0 admission protection (validate first, then apply).
|
||||
kubectl apply --dry-run=server -f deploy/kubernetes/platform/admission/
|
||||
kubectl apply -f deploy/kubernetes/platform/admission/
|
||||
# 3. Projects (resolve the Application paths only when M8.4 adds workloads).
|
||||
kubectl apply -f deploy/argocd/stage/project.yaml.example
|
||||
kubectl apply -f deploy/argocd/world0/project.yaml.example
|
||||
# 4. Verify: reads, can-i and server dry runs only.
|
||||
bash tools/ops/verify-isolation.sh
|
||||
```
|
||||
|
||||
Create the Secrets listed above in each namespace before any workload syncs.
|
||||
|
||||
## Break-glass
|
||||
|
||||
To delete World0 data or identity on purpose:
|
||||
1. Record the operation and its approval.
|
||||
2. Use a credential whose groups include `aevolution:world0-break-glass`, for
|
||||
example a short-lived client certificate with that organization.
|
||||
3. Take and verify a backup first (M8.7).
|
||||
|
||||
Removing the admission policy itself is the break-glass of last resort. It needs
|
||||
cluster-admin and must be recorded the same way. Never test by deleting World0: the
|
||||
verify script uses server-side dry runs.
|
||||
|
||||
## Residual risks
|
||||
- **`default` AppProject.** It still allows every destination, so its applications
|
||||
(other projects, two with automated prune) can create objects in AEvolution
|
||||
namespaces. Admission denies only World0 deletions. Recommended: restrict
|
||||
`default`, or move those applications to scoped projects.
|
||||
- **World0 volumes.** `local-path` provisions volumes with Delete. Patch each World0
|
||||
PV to `Retain` when it is bound, and the policy then keeps it there. Volumes stay
|
||||
node-local until M8.3.
|
||||
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env python3
|
||||
"""M8.2 offline deployment-policy check (ADR-0069). Needs Python 3 and PyYAML only.
|
||||
|
||||
Parses every manifest under deploy/ (including .yaml.example files) and fails if an
|
||||
environment-isolation or World0-protection invariant is broken:
|
||||
- no Secret objects in Git;
|
||||
- every ServiceAccount disables token automounting;
|
||||
- each platform overlay: labelled restricted namespace, quota, limit range,
|
||||
default-deny and DNS-only policies, three service accounts; World0 alone is
|
||||
labelled protected;
|
||||
- the World0 admission policy denies what ADR-0069 lists, fails closed and is
|
||||
bound with Deny;
|
||||
- every AEvolution AppProject targets exactly its own namespace with no cluster
|
||||
kinds; World0 blocks automated sync; no Application auto-syncs or carries the
|
||||
cascade-delete finalizer, and each stays in its own environment;
|
||||
- the live Argo path (deploy/kubernetes/dev) holds no Namespace, Secret, quota or
|
||||
RBAC objects;
|
||||
- the cleanup tool refuses World0 before invoking kubectl.
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
import yaml
|
||||
|
||||
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
|
||||
failures = []
|
||||
|
||||
|
||||
def check(condition, message):
|
||||
if not condition:
|
||||
failures.append(message)
|
||||
|
||||
|
||||
def documents(path):
|
||||
with open(path, encoding='utf-8') as f:
|
||||
return [d for d in yaml.safe_load_all(f) if isinstance(d, dict)]
|
||||
|
||||
|
||||
manifests = {}
|
||||
for base, _, files in os.walk(os.path.join(ROOT, 'deploy')):
|
||||
for name in files:
|
||||
if name.endswith(('.yaml', '.yml', '.yaml.example')):
|
||||
path = os.path.join(base, name)
|
||||
manifests[os.path.relpath(path, ROOT).replace(os.sep, '/')] = documents(path)
|
||||
objects = [(p, d) for p, ds in manifests.items() for d in ds
|
||||
if 'kind' in d and d['kind'] != 'Kustomization']
|
||||
|
||||
# Secrets and service-account tokens.
|
||||
for path, doc in objects:
|
||||
check(doc['kind'] != 'Secret', f'{path}: Secret objects never belong in Git')
|
||||
if doc['kind'] == 'ServiceAccount':
|
||||
check(doc.get('automountServiceAccountToken') is False,
|
||||
f'{path}: ServiceAccount {doc["metadata"]["name"]} must not automount a token')
|
||||
|
||||
# Platform overlays.
|
||||
platform = 'deploy/kubernetes/platform'
|
||||
base = {d['kind'] + '/' + d['metadata']['name']: d
|
||||
for p, d in objects if p.startswith(platform + '/base/')}
|
||||
for key in ('ResourceQuota/aevolution-quota', 'LimitRange/aevolution-defaults',
|
||||
'NetworkPolicy/aevolution-default-deny', 'NetworkPolicy/aevolution-allow-dns',
|
||||
'ServiceAccount/aevolution-migrator', 'ServiceAccount/aevolution-region',
|
||||
'ServiceAccount/aevolution-backup'):
|
||||
check(key in base, f'{platform}/base: missing {key}')
|
||||
deny = base.get('NetworkPolicy/aevolution-default-deny', {}).get('spec', {})
|
||||
check(deny.get('podSelector') == {} and sorted(deny.get('policyTypes', [])) == ['Egress', 'Ingress']
|
||||
and not deny.get('ingress') and not deny.get('egress'),
|
||||
'default-deny must select every pod and allow nothing')
|
||||
dns = base.get('NetworkPolicy/aevolution-allow-dns', {}).get('spec', {})
|
||||
dns_ports = [p.get('port') for rule in dns.get('egress', []) for p in rule.get('ports', [])]
|
||||
check(dns.get('policyTypes') == ['Egress'] and set(dns_ports) == {53} and len(dns.get('egress', [])) == 1,
|
||||
'allow-dns must allow only egress to port 53')
|
||||
quota = base.get('ResourceQuota/aevolution-quota', {}).get('spec', {}).get('hard', {})
|
||||
check(quota.get('services.loadbalancers') == '0' and quota.get('services.nodeports') == '0',
|
||||
'quota must forbid LoadBalancer and NodePort services')
|
||||
for env in ('dev', 'stage', 'world0'):
|
||||
k = f'{platform}/{env}/kustomization.yaml'
|
||||
ns_path = f'{platform}/{env}/namespace.yaml'
|
||||
check(k in manifests and ns_path in manifests, f'{platform}/{env}: overlay missing')
|
||||
if k not in manifests or ns_path not in manifests:
|
||||
continue
|
||||
kustomization = manifests[k][0]
|
||||
check(kustomization.get('namespace') == f'evolution-{env}' and '../base' in kustomization.get('resources', []),
|
||||
f'{k}: must set namespace evolution-{env} and include ../base')
|
||||
labels = manifests[ns_path][0]['metadata'].get('labels', {})
|
||||
check(manifests[ns_path][0]['metadata']['name'] == f'evolution-{env}', f'{ns_path}: wrong name')
|
||||
check(labels.get('aevolution.io/environment') == env, f'{ns_path}: environment label')
|
||||
check(labels.get('pod-security.kubernetes.io/enforce') == 'restricted', f'{ns_path}: restricted pod security')
|
||||
check((labels.get('aevolution.io/protected') == 'true') == (env == 'world0'),
|
||||
f'{ns_path}: only world0 is protected')
|
||||
|
||||
# World0 admission policy.
|
||||
policy = [d for p, d in objects if d['kind'] == 'ValidatingAdmissionPolicy'
|
||||
and d['metadata']['name'] == 'aevolution-world0-protection']
|
||||
binding = [d for p, d in objects if d['kind'] == 'ValidatingAdmissionPolicyBinding'
|
||||
and d['metadata']['name'] == 'aevolution-world0-protection']
|
||||
check(len(policy) == 1 and len(binding) == 1, 'world0 admission policy and binding must exist once')
|
||||
if policy and binding:
|
||||
spec = policy[0]['spec']
|
||||
check(spec.get('failurePolicy') == 'Fail', 'world0 policy must fail closed')
|
||||
check(binding[0]['spec'].get('validationActions') == ['Deny'] and 'paramRef' not in binding[0]['spec']
|
||||
and 'matchResources' not in binding[0]['spec'], 'world0 binding must Deny everywhere')
|
||||
covered = {(r, op) for rule in spec['matchConstraints']['resourceRules']
|
||||
for r in rule['resources'] for op in rule['operations']}
|
||||
for resource in ('namespaces', 'persistentvolumes', 'persistentvolumeclaims', 'secrets',
|
||||
'serviceaccounts', 'applications', 'appprojects'):
|
||||
check((resource, 'DELETE') in covered, f'world0 policy must cover DELETE {resource}')
|
||||
for resource in ('namespaces', 'persistentvolumes'):
|
||||
check((resource, 'UPDATE') in covered, f'world0 policy must cover UPDATE {resource}')
|
||||
text = yaml.safe_dump(spec)
|
||||
for needle in ("aevolution:world0-break-glass", "evolution-world0", "aevolution-world0",
|
||||
"aevolution.io/protected", "Retain"):
|
||||
check(needle in text, f'world0 policy must reference {needle}')
|
||||
|
||||
# Argo CD projects and applications.
|
||||
for path, doc in objects:
|
||||
name = doc['metadata'].get('name', '')
|
||||
if doc['kind'] == 'AppProject' and name.startswith('aevolution-'):
|
||||
env = name[len('aevolution-'):]
|
||||
spec = doc['spec']
|
||||
namespaces = [d.get('namespace') for d in spec.get('destinations', [])]
|
||||
check(namespaces == [f'evolution-{env}'], f'{path}: {name} must target only evolution-{env}')
|
||||
check(spec.get('clusterResourceWhitelist') == [], f'{path}: {name} must allow no cluster kinds')
|
||||
kinds = {w.get('kind') for w in spec.get('namespaceResourceWhitelist', [])}
|
||||
check(not kinds & {'Secret', 'Namespace', 'ResourceQuota', 'LimitRange', 'Role', 'RoleBinding', '*'},
|
||||
f'{path}: {name} whitelists a forbidden kind')
|
||||
if env == 'world0':
|
||||
windows = spec.get('syncWindows', [])
|
||||
check(any(w.get('kind') == 'deny' and w.get('manualSync') is True and w.get('duration') == '24h'
|
||||
and w.get('schedule') == '* * * * *' for w in windows),
|
||||
f'{path}: world0 must deny automated sync at all times')
|
||||
check('ServiceAccount' not in kinds, f'{path}: world0 service accounts are platform-managed')
|
||||
if doc['kind'] == 'Application':
|
||||
spec = doc['spec']
|
||||
project = spec.get('project', '')
|
||||
destination = spec.get('destination', {}).get('namespace', '')
|
||||
check('automated' not in (spec.get('syncPolicy') or {}), f'{path}: {name} must not auto-sync')
|
||||
check('resources-finalizer.argocd.argoproj.io' not in doc['metadata'].get('finalizers', []),
|
||||
f'{path}: {name} must not cascade-delete')
|
||||
if project.startswith('aevolution-'):
|
||||
check(destination == 'evolution-' + project[len('aevolution-'):],
|
||||
f'{path}: {name} leaves its project environment')
|
||||
check(destination != 'evolution-world0' or name == 'aevolution-world0',
|
||||
f'{path}: only aevolution-world0 may target evolution-world0')
|
||||
|
||||
# The live Argo path.
|
||||
for path, doc in objects:
|
||||
if path.startswith('deploy/kubernetes/dev/'):
|
||||
check(doc['kind'] not in {'Namespace', 'Secret', 'ResourceQuota', 'LimitRange', 'Role',
|
||||
'RoleBinding', 'ClusterRole', 'ClusterRoleBinding'},
|
||||
f'{path}: {doc["kind"]} must not live in the Argo workload path')
|
||||
|
||||
# The cleanup tool refuses World0 before kubectl runs (no kubectl on PATH here).
|
||||
with tempfile.TemporaryDirectory() as empty:
|
||||
env = dict(os.environ, PATH=empty + os.pathsep + os.path.dirname(sys.executable))
|
||||
for bad in ('world0', 'evolution-world0', 'World0', 'prod', ''):
|
||||
result = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, 'tools/ops/cleanup-environment.sh'), bad],
|
||||
capture_output=True, text=True, env=env)
|
||||
check(result.returncode == 2 and ('refused' in result.stderr or 'usage' in result.stderr)
|
||||
and 'kubectl' not in result.stderr,
|
||||
f'cleanup must refuse {bad!r} before kubectl (rc={result.returncode})')
|
||||
|
||||
for message in failures:
|
||||
print('FAIL', message)
|
||||
print(f'deploy policy: {len(objects)} objects in {len(manifests)} files, {len(failures)} failures')
|
||||
sys.exit(1 if failures else 0)
|
||||
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env bash
|
||||
# M8.2 environment cleanup for disposable AEvolution environments only.
|
||||
#
|
||||
# tools/ops/cleanup-environment.sh <dev|stage> [--execute] [--context <name>]
|
||||
#
|
||||
# Removes AEvolution workload objects (Deployments, StatefulSets, Jobs, Services,
|
||||
# ConfigMaps and PersistentVolumeClaims labelled app.kubernetes.io/part-of=aevolution)
|
||||
# from evolution-<env>. Platform guardrails (namespace, quota, limit range, network
|
||||
# policies, service accounts) and Secrets are never touched. Without --execute every
|
||||
# delete is a server-side dry run. World0, or any namespace that is not labelled as
|
||||
# the named disposable environment or is labelled protected, is refused before any
|
||||
# change; the refusal of World0 happens before kubectl is even invoked.
|
||||
set -euo pipefail
|
||||
usage() { echo "usage: $0 <dev|stage> [--execute] [--context <name>]" >&2; exit 2; }
|
||||
[ $# -ge 1 ] || usage
|
||||
env="$1"; shift
|
||||
case "$env" in
|
||||
dev|stage) ;;
|
||||
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
|
||||
exit 2 ;;
|
||||
esac
|
||||
dry_run=(--dry-run=server)
|
||||
k=(kubectl --request-timeout=30s)
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--execute) dry_run=() ;;
|
||||
--context) [ $# -ge 2 ] || usage; k+=(--context "$2"); shift ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
ns="evolution-$env"
|
||||
labels="$("${k[@]}" get namespace "$ns" -o jsonpath='{.metadata.labels.aevolution\.io/environment}/{.metadata.labels.aevolution\.io/protected}')"
|
||||
if [ "$labels" != "$env/" ]; then
|
||||
echo "refused: namespace $ns labels are '$labels' (need aevolution.io/environment=$env and no aevolution.io/protected)" >&2
|
||||
exit 2
|
||||
fi
|
||||
selector='app.kubernetes.io/part-of=aevolution,app.kubernetes.io/managed-by!=aevolution-platform'
|
||||
"${k[@]}" delete -n "$ns" "${dry_run[@]}" --wait=false -l "$selector" \
|
||||
deployments,statefulsets,jobs,services,configmaps,persistentvolumeclaims
|
||||
[ ${#dry_run[@]} -eq 0 ] && echo "deleted AEvolution workloads in $ns" ||
|
||||
echo "dry run only (server-side); rerun with --execute to delete in $ns"
|
||||
@@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env bash
|
||||
# M8.2 live isolation checks. Changes nothing: it only asks `kubectl auth can-i`
|
||||
# (impersonating the environments' service accounts), reads objects, and sends
|
||||
# DELETE requests with --dry-run=server, which run admission but never persist.
|
||||
#
|
||||
# tools/ops/verify-isolation.sh [--context <name>]
|
||||
#
|
||||
# Needs an identity that may impersonate service accounts and read namespaces,
|
||||
# policies and Argo CD projects (a cluster administrator). Prints PASS/FAIL/SKIP per
|
||||
# check and exits nonzero if any check fails.
|
||||
set -uo pipefail
|
||||
k=(kubectl --request-timeout=20s)
|
||||
[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2")
|
||||
fails=0
|
||||
pass() { echo "PASS $*"; }
|
||||
fail() { echo "FAIL $*"; fails=$((fails + 1)); }
|
||||
skip() { echo "SKIP $*"; }
|
||||
|
||||
envs=(dev stage world0)
|
||||
present=()
|
||||
for env in "${envs[@]}"; do
|
||||
if "${k[@]}" get namespace "evolution-$env" > /dev/null 2>&1; then
|
||||
present+=("$env")
|
||||
else
|
||||
skip "namespace evolution-$env does not exist"
|
||||
fi
|
||||
done
|
||||
|
||||
# 1. Guardrails and labels in every present environment.
|
||||
for env in "${present[@]}"; do
|
||||
ns="evolution-$env"
|
||||
got="$("${k[@]}" get namespace "$ns" -o jsonpath='{.metadata.labels.aevolution\.io/environment} {.metadata.labels.pod-security\.kubernetes\.io/enforce} {.metadata.labels.aevolution\.io/protected}')"
|
||||
want="$env restricted"; [ "$env" = world0 ] && want="$env restricted true"
|
||||
[ "$(echo "$got" | xargs)" = "$want" ] && pass "$ns labels ($got)" || fail "$ns labels '$got' (want '$want')"
|
||||
for object in resourcequota/aevolution-quota limitrange/aevolution-defaults \
|
||||
networkpolicy/aevolution-default-deny networkpolicy/aevolution-allow-dns \
|
||||
serviceaccount/aevolution-migrator serviceaccount/aevolution-region serviceaccount/aevolution-backup; do
|
||||
"${k[@]}" get -n "$ns" "$object" > /dev/null 2>&1 && pass "$ns has $object" || fail "$ns lacks $object"
|
||||
done
|
||||
done
|
||||
|
||||
# 2. No environment identity can read secrets or change anything anywhere.
|
||||
for env in "${present[@]}"; do
|
||||
for sa in aevolution-migrator aevolution-region aevolution-backup; do
|
||||
who="system:serviceaccount:evolution-$env:$sa"
|
||||
for target in "${envs[@]}"; do
|
||||
for action in "get secrets" "list secrets" "create pods" "delete persistentvolumeclaims" "create rolebindings"; do
|
||||
answer="$("${k[@]}" auth can-i $action -n "evolution-$target" --as "$who" 2>/dev/null)"
|
||||
[ "$answer" = "no" ] && pass "$who cannot $action in evolution-$target" ||
|
||||
fail "$who can $action in evolution-$target"
|
||||
done
|
||||
done
|
||||
answer="$("${k[@]}" auth can-i delete namespaces --as "$who" 2>/dev/null)"
|
||||
[ "$answer" = "no" ] && pass "$who cannot delete namespaces" || fail "$who can delete namespaces"
|
||||
done
|
||||
done
|
||||
|
||||
# 3. World0 admission protection, exercised with server-side dry runs.
|
||||
if "${k[@]}" get validatingadmissionpolicybinding aevolution-world0-protection > /dev/null 2>&1; then
|
||||
pass "world0 protection policy is bound"
|
||||
else
|
||||
fail "world0 protection policy binding missing"
|
||||
fi
|
||||
denied() { # denied <description> <kubectl delete args...> (always --dry-run=server)
|
||||
local what="$1"; shift
|
||||
local out
|
||||
out="$("${k[@]}" delete --dry-run=server --wait=false "$@" 2>&1)"
|
||||
if echo "$out" | grep -q "aevolution:world0-break-glass"; then
|
||||
pass "denied: $what"
|
||||
elif echo "$out" | grep -qi "not found"; then
|
||||
skip "$what (object absent)"
|
||||
else
|
||||
fail "not denied: $what -> $out"
|
||||
fi
|
||||
}
|
||||
if printf '%s\n' "${present[@]}" | grep -qx world0; then
|
||||
denied "delete namespace evolution-world0" namespace evolution-world0
|
||||
denied "delete service account in evolution-world0" -n evolution-world0 serviceaccount aevolution-region
|
||||
secret="$("${k[@]}" get secrets -n evolution-world0 -o name 2>/dev/null | head -1)"
|
||||
[ -n "$secret" ] && denied "delete $secret in evolution-world0" -n evolution-world0 "$secret" ||
|
||||
skip "no Secret in evolution-world0 to test"
|
||||
pvc="$("${k[@]}" get pvc -n evolution-world0 -o name 2>/dev/null | head -1)"
|
||||
[ -n "$pvc" ] && denied "delete $pvc in evolution-world0" -n evolution-world0 "$pvc" ||
|
||||
skip "no PVC in evolution-world0 to test"
|
||||
else
|
||||
skip "world0 admission dry runs (namespace absent)"
|
||||
fi
|
||||
|
||||
# 4. Argo CD projects: each AEvolution project targets only its own namespace.
|
||||
for env in "${envs[@]}"; do
|
||||
dest="$("${k[@]}" get appproject "aevolution-$env" -n argocd -o jsonpath='{range .spec.destinations[*]}{.namespace}{" "}{end}{.spec.clusterResourceWhitelist}' 2>/dev/null)" || {
|
||||
skip "appproject aevolution-$env not present"; continue; }
|
||||
[ "$(echo "$dest" | xargs)" = "evolution-$env" ] && pass "appproject aevolution-$env destinations '$dest'" ||
|
||||
fail "appproject aevolution-$env destinations/cluster kinds '$dest'"
|
||||
done
|
||||
wide="$("${k[@]}" get appproject default -n argocd -o jsonpath='{.spec.destinations[*].namespace}' 2>/dev/null)"
|
||||
[ "$wide" = "*" ] && echo "WARN appproject default allows every namespace; its applications could create objects in evolution-* (deletion is still denied in world0 by admission)"
|
||||
|
||||
# 5. The cleanup tool refuses World0 before touching the cluster.
|
||||
root="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
for bad in world0 evolution-world0 World0 ""; do
|
||||
if bash "$root/tools/ops/cleanup-environment.sh" "$bad" > /dev/null 2>&1; then
|
||||
fail "cleanup accepted '$bad'"
|
||||
else
|
||||
pass "cleanup refused '$bad'"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "failures=$fails"
|
||||
[ "$fails" -eq 0 ]
|
||||
Reference in New Issue
Block a user