Add M8.2 environment isolation and World0 protection

Per-environment platform layers (dev, stage, world0) with restricted pod
security, quotas, limit ranges, default-deny networking and token-less
service accounts, applied by an administrator rather than Argo. A native
ValidatingAdmissionPolicy denies deleting World0 data, identity and Argo
objects to every requester without the break-glass group. Stage and
World0 Argo projects are scoped to one namespace, World0 blocks
automated sync, and a cleanup tool refuses World0. An offline policy
check runs as a new CI suite; a live verification script uses only
can-i, reads and server dry runs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-29 12:43:58 -04:00
co-authored by Claude Opus 5.5
parent 18708340c3
commit df0c4f4efb
28 changed files with 953 additions and 2 deletions
+1 -1
View File
@@ -19,7 +19,7 @@ jobs:
strategy:
fail-fast: false
matrix:
suite: [checks, tsan, ecs, postgres]
suite: [checks, tsan, ecs, postgres, deploy]
steps:
- name: Checkout
uses: https://github.com/actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+11
View File
@@ -152,3 +152,14 @@ plus the pinned libpq, UID 10001, no credentials). It smoke-tests the image with
- the runtime Secret, object PVC and a NetworkPolicy.
Pin the digest and provision the region first. Without a gateway, the region
receives no inputs.
## Environments and World0 protection (M8.2)
`kubernetes/platform/<env>` holds each environment's namespace, quota, limit range,
default-deny policies and service accounts. `kubernetes/platform/admission` holds
the World0 admission policy. A cluster administrator applies these by hand; they
are never part of an Argo Application. The stage and World0 Argo projects and
applications are in `argocd/stage` and `argocd/world0` as examples.
See docs/infrastructure/ENVIRONMENTS.md for the runbook, and ADR-0069. The CI
`deploy` suite runs `tools/ci/deploy-policy-check.py` over every manifest.
@@ -0,0 +1,21 @@
# M8.2 (stage). REPLACE_WORKLOAD_PATH is deliberately unresolved: the stage
# workload is M8.4 work. Manual sync only; no automated policy, no prune on sync of
# durable objects (PVCs carry argocd.argoproj.io/sync-options: Prune=false,Delete=false).
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: aevolution-stage
namespace: argocd
spec:
project: aevolution-stage
source:
repoURL: https://git.aeternalisgames.com/dingo/AEvolution.git
targetRevision: main
path: REPLACE_WORKLOAD_PATH
destination:
server: https://kubernetes.default.svc
namespace: evolution-stage
syncPolicy:
syncOptions:
- FailOnSharedResource=true
- PruneLast=true
+26
View File
@@ -0,0 +1,26 @@
# M8.2 (stage). Apply by hand to the argocd namespace; never from an Application.
# One destination namespace only, no cluster-scoped kinds, no Secrets or quotas.
apiVersion: argoproj.io/v1alpha1
kind: AppProject
metadata:
name: aevolution-stage
namespace: argocd
spec:
description: AEvolution stage (evolution-stage only)
sourceRepos:
- https://git.aeternalisgames.com/dingo/AEvolution.git
destinations:
- server: https://kubernetes.default.svc
namespace: evolution-stage
clusterResourceWhitelist: []
namespaceResourceWhitelist:
- {group: "", kind: Service}
- {group: "", kind: ConfigMap}
- {group: "", kind: PersistentVolumeClaim}
- {group: apps, kind: Deployment}
- {group: batch, kind: Job}
- {group: networking.k8s.io, kind: NetworkPolicy}
# Not listed, so never Argo-managed: Namespaces, ServiceAccounts (platform layer),
# Secrets, quotas, limit ranges and RBAC.
orphanedResources:
warn: true
@@ -0,0 +1,25 @@
# M8.2 (world0). REPLACE_WORKLOAD_PATH is deliberately unresolved: the world0
# workload is M8.4 work. Manual sync only; no automated policy, no prune on sync of
# durable objects (PVCs carry argocd.argoproj.io/sync-options: Prune=false,Delete=false).
# World0: never add the resources-finalizer.argocd.argoproj.io finalizer (deleting
# the Application must orphan, not cascade), never add an automated policy, and never
# list this Application in a parent app-of-apps with prune. The admission policy
# also denies deleting this Application without break-glass.
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: aevolution-world0
namespace: argocd
spec:
project: aevolution-world0
source:
repoURL: https://git.aeternalisgames.com/dingo/AEvolution.git
targetRevision: main
path: REPLACE_WORKLOAD_PATH
destination:
server: https://kubernetes.default.svc
namespace: evolution-world0
syncPolicy:
syncOptions:
- FailOnSharedResource=true
- PruneLast=true
+33
View File
@@ -0,0 +1,33 @@
# M8.2 (world0). Apply by hand to the argocd namespace; never from an Application.
# One destination namespace only, no cluster-scoped kinds, no Secrets or quotas.
apiVersion: argoproj.io/v1alpha1
kind: AppProject
metadata:
name: aevolution-world0
namespace: argocd
spec:
description: AEvolution world0 (evolution-world0 only)
sourceRepos:
- https://git.aeternalisgames.com/dingo/AEvolution.git
destinations:
- server: https://kubernetes.default.svc
namespace: evolution-world0
clusterResourceWhitelist: []
namespaceResourceWhitelist:
- {group: "", kind: Service}
- {group: "", kind: ConfigMap}
- {group: "", kind: PersistentVolumeClaim}
- {group: apps, kind: Deployment}
- {group: batch, kind: Job}
- {group: networking.k8s.io, kind: NetworkPolicy}
# Not listed, so never Argo-managed: Namespaces, ServiceAccounts (platform layer),
# Secrets, quotas, limit ranges and RBAC.
orphanedResources:
warn: true
# No automated sync can ever run; a human may still sync by hand.
syncWindows:
- kind: deny
schedule: "* * * * *"
duration: 24h
applications: ["*"]
manualSync: true
+13
View File
@@ -45,6 +45,19 @@ RUN cmake --preset analysis -B build/ecs-analysis -DCMAKE_CXX_COMPILER=clang++-1
&& cmake --build build/ecs-analysis --target evo-ecs-experiment --parallel 2 \
&& ctest --test-dir build/ecs-analysis -R ecs-experiment --output-on-failure --no-tests=error
# M8.2 offline deployment-policy check (ADR-0069): Ubuntu's python3-yaml only, no
# cluster access. Fails the build when an isolation or World0 invariant breaks.
FROM ${UBUNTU_IMAGE} AS ci-deploy
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update \
&& apt-get install -y --no-install-recommends python3 python3-yaml \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /source
COPY deploy/ deploy/
COPY tools/ci/deploy-policy-check.py tools/ci/
COPY tools/ops/ tools/ops/
RUN python3 tools/ci/deploy-policy-check.py
FROM ci-toolchain AS postgres-build
RUN apt-get update && apt-get install -y --no-install-recommends \
curl bzip2 make flex bison perl libssl-dev zlib1g-dev pkg-config \
@@ -0,0 +1,88 @@
# M8.2 World0 protection (ADR-0069). A native ValidatingAdmissionPolicy (GA since
# Kubernetes 1.30; the cluster runs 1.32): no webhook, no new dependency. It applies
# to every requester, including cluster-admin (system:masters), the Argo CD
# application controller and any Application in any AppProject, so no permissive
# app, prune, parent-app cascade or credential can remove World0 data or identity.
# Only a member of the group `aevolution:world0-break-glass` passes. Membership is
# granted out of band (client certificate O= or OIDC group) for a recorded operation.
#
# Denied without break-glass:
# - deleting Namespace evolution-world0, or removing its aevolution.io/protected label;
# - deleting PersistentVolumeClaims, Secrets or ServiceAccounts in evolution-world0;
# - deleting a PersistentVolume bound to evolution-world0, or changing its reclaim
# policy away from Retain once set (set Retain when the volume is bound: the
# only StorageClass, local-path, provisions with Delete);
# - deleting the aevolution-world0 Argo CD Application or AppProject.
# Kubernetes does not run admission policies on policy objects themselves, so
# removing this policy is itself the audited break-glass step (cluster-admin).
# Apply by hand: kubectl apply -f deploy/kubernetes/platform/admission/
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingAdmissionPolicy
metadata:
name: aevolution-world0-protection
labels:
app.kubernetes.io/part-of: aevolution
spec:
failurePolicy: Fail
matchConstraints:
resourceRules:
- apiGroups: [""]
apiVersions: ["v1"]
operations: ["DELETE", "UPDATE"]
resources: ["namespaces", "persistentvolumes"]
- apiGroups: [""]
apiVersions: ["v1"]
operations: ["DELETE"]
resources: ["persistentvolumeclaims", "secrets", "serviceaccounts"]
- apiGroups: ["argoproj.io"]
apiVersions: ["*"]
operations: ["DELETE"]
resources: ["applications", "appprojects"]
variables:
- name: breakGlass
expression: "request.userInfo.groups.exists(g, g == 'aevolution:world0-break-glass')"
- name: resource
expression: "request.resource.resource"
- name: world0Namespace
expression: "variables.resource == 'namespaces' && request.name == 'evolution-world0'"
- name: world0Volume
expression: >-
variables.resource == 'persistentvolumes' &&
has(oldObject.spec.claimRef) && has(oldObject.spec.claimRef.namespace) &&
oldObject.spec.claimRef.namespace == 'evolution-world0'
- name: world0Object
expression: >-
request.namespace == 'evolution-world0' &&
variables.resource in ['persistentvolumeclaims', 'secrets', 'serviceaccounts']
- name: world0Delivery
expression: >-
request.resource.group == 'argoproj.io' && request.name == 'aevolution-world0'
validations:
- expression: >-
variables.breakGlass || request.operation != 'DELETE' ||
!(variables.world0Namespace || variables.world0Volume || variables.world0Object ||
variables.world0Delivery)
message: "World0 data, identity and delivery objects cannot be deleted without the aevolution:world0-break-glass group"
reason: Forbidden
- expression: >-
variables.breakGlass || request.operation != 'UPDATE' || !variables.world0Namespace ||
(has(object.metadata.labels) && 'aevolution.io/protected' in object.metadata.labels &&
object.metadata.labels['aevolution.io/protected'] == 'true')
message: "The World0 namespace must keep aevolution.io/protected=true"
reason: Forbidden
- expression: >-
variables.breakGlass || request.operation != 'UPDATE' || !variables.world0Volume ||
oldObject.spec.persistentVolumeReclaimPolicy != 'Retain' ||
object.spec.persistentVolumeReclaimPolicy == 'Retain'
message: "A World0 PersistentVolume set to Retain cannot be changed back"
reason: Forbidden
---
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingAdmissionPolicyBinding
metadata:
name: aevolution-world0-protection
labels:
app.kubernetes.io/part-of: aevolution
spec:
policyName: aevolution-world0-protection
validationActions: [Deny]
@@ -0,0 +1,15 @@
# M8.2 per-environment guardrails (docs/infrastructure/ENVIRONMENTS.md). Applied by a
# cluster administrator with `kubectl apply -k deploy/kubernetes/platform/<env>`, never
# through an Argo CD Application: namespaces, quotas and identities must not be
# prunable by a workload sync. Each overlay sets the namespace and its quota values.
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- quota.yaml
- limitrange.yaml
- networkpolicies.yaml
- serviceaccounts.yaml
labels:
- pairs:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/managed-by: aevolution-platform
@@ -0,0 +1,17 @@
# Defaults for containers that omit resources (the quota requires them), sized from
# the measured region-host profile (peak RSS about 9.7 MiB): 64Mi request, 256Mi
# limit, 50m CPU request and no default CPU limit. No container may exceed 2Gi.
apiVersion: v1
kind: LimitRange
metadata:
name: aevolution-defaults
spec:
limits:
- type: Container
defaultRequest:
cpu: 50m
memory: 64Mi
default:
memory: 256Mi
max:
memory: 2Gi
@@ -0,0 +1,30 @@
# Deny by default: every pod in the namespace has no ingress or egress except DNS.
# Workloads add narrow allowances with their own policies (for example the region
# host's egress to exactly the database VM address and port, and ingress to its
# operations port from the metrics scraper) in M8.4. Enforced by Canal/Calico.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: aevolution-default-deny
spec:
podSelector: {}
policyTypes: [Ingress, Egress]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: aevolution-allow-dns
spec:
podSelector: {}
policyTypes: [Egress]
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
podSelector:
matchLabels:
k8s-app: kube-dns
ports:
- {protocol: UDP, port: 53}
- {protocol: TCP, port: 53}
@@ -0,0 +1,17 @@
# Values are replaced per environment. Memory is quota'd on requests and limits;
# CPU only on requests (region hosts run without CPU limits: throttling would delay
# lease renewals, docs/infrastructure/INVENTORY.md). No LoadBalancer or NodePort.
apiVersion: v1
kind: ResourceQuota
metadata:
name: aevolution-quota
spec:
hard:
requests.cpu: "2"
requests.memory: 2Gi
limits.memory: 4Gi
pods: "20"
persistentvolumeclaims: "4"
requests.storage: 40Gi
services.loadbalancers: "0"
services.nodeports: "0"
@@ -0,0 +1,26 @@
# One identity per role; none needs the Kubernetes API, so none mounts a token and
# no Role or RoleBinding is granted. Secrets are referenced by name only and are
# created per environment by an operator (ENVIRONMENTS.md); they never enter Git.
apiVersion: v1
kind: ServiceAccount
metadata:
name: aevolution-migrator # Runs migrations with the migrator database login.
automountServiceAccountToken: false
imagePullSecrets:
- name: harbor-pull
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: aevolution-region # evo-server with the runtime database login.
automountServiceAccountToken: false
imagePullSecrets:
- name: harbor-pull
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: aevolution-backup # Read-only backup/clone jobs (M8.7).
automountServiceAccountToken: false
imagePullSecrets:
- name: harbor-pull
@@ -0,0 +1,9 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: evolution-dev
resources:
- namespace.yaml
- ../base
labels:
- pairs:
aevolution.io/environment: dev
@@ -0,0 +1,10 @@
# Disposable development. Existing namespace; applying adds labels and guardrails.
apiVersion: v1
kind: Namespace
metadata:
name: evolution-dev
labels:
aevolution.io/environment: dev
pod-security.kubernetes.io/enforce: restricted
pod-security.kubernetes.io/enforce-version: v1.32
pod-security.kubernetes.io/warn: restricted
@@ -0,0 +1,9 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: evolution-stage
resources:
- namespace.yaml
- ../base
labels:
- pairs:
aevolution.io/environment: stage
@@ -0,0 +1,10 @@
# Clone, migration and soak rehearsals (M8.6/8.9). Disposable, never World0 data.
apiVersion: v1
kind: Namespace
metadata:
name: evolution-stage
labels:
aevolution.io/environment: stage
pod-security.kubernetes.io/enforce: restricted
pod-security.kubernetes.io/enforce-version: v1.32
pod-security.kubernetes.io/warn: restricted
@@ -0,0 +1,29 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: evolution-world0
resources:
- namespace.yaml
- ../base
patches:
- target:
kind: ResourceQuota
name: aevolution-quota
patch: |-
- op: replace
path: /spec/hard/requests.cpu
value: "4"
- op: replace
path: /spec/hard/requests.memory
value: 4Gi
- op: replace
path: /spec/hard/limits.memory
value: 8Gi
- op: replace
path: /spec/hard/persistentvolumeclaims
value: "8"
- op: replace
path: /spec/hard/requests.storage
value: 200Gi
labels:
- pairs:
aevolution.io/environment: world0
@@ -0,0 +1,11 @@
# Persistent World0. Guarded by deploy/kubernetes/platform/admission. Nothing here deploys a world; launch stays gated by M8.8.
apiVersion: v1
kind: Namespace
metadata:
name: evolution-world0
labels:
aevolution.io/environment: world0
pod-security.kubernetes.io/enforce: restricted
pod-security.kubernetes.io/enforce-version: v1.32
pod-security.kubernetes.io/warn: restricted
aevolution.io/protected: "true"
+10
View File
@@ -606,3 +606,13 @@ M8.1 complete (2026-09-29): docs/infrastructure/INVENTORY.md records the cluster
the separate PostgreSQL VM, the measured workload profile and a per-worker
`local-path` storage probe. Storage figures are provisional until the planned RAID
and storage change. Next: M8.2, environment isolation and secrets.
M8.2 designed and checked offline (2026-09-29), under ADR-0069:
- dev, stage and World0 namespaces, each with its own guardrails, identities and
secret references;
- a native admission policy that protects World0 from every requester except a
break-glass group;
- scoped Argo projects, and a cleanup tool that refuses World0;
- an offline policy check in CI, and a live isolation check.
The live checks run once the maintainer applies the platform layer. Next: M8.3,
durable stores in dev and stage.
+17 -1
View File
@@ -993,4 +993,20 @@ M8.1 complete (2026-09-29) for the current disk layout. The per-worker
These are well inside the lease deadlines. Re-probe after the planned RAID and
storage change. Host memory was raised by about 500 GB.
Next: M8.2, environment isolation and secrets.
Next: M8.2, environment isolation and secrets.
## M8.2 environment isolation (2026-09-29)
ADR-0069 adds three environments, each with its own guardrails, service accounts,
secret references and database:
- `evolution-dev`, `evolution-stage` and `evolution-world0`;
- guardrails: quota, limit range, default-deny networking, restricted pod
security.
A native ValidatingAdmissionPolicy denies deleting World0 data, identity or Argo
objects to everyone, including cluster-admin and Argo, unless the requester is in
`aevolution:world0-break-glass`. The stage and World0 Argo projects are scoped to
one namespace each, and World0 permanently denies automated sync. The cleanup tool
refuses World0.
The offline policy check (now a CI suite) passes, and nine injected violations
were each caught. Applying the layer and running `tools/ops/verify-isolation.sh`
is waiting on the maintainer.
+106
View File
@@ -0,0 +1,106 @@
# ADR-0069: Environment isolation and World0 protection
## Status
Accepted for M8.2, 2026-09-29. It builds on the M8.1 inventory
(infrastructure/INVENTORY.md), ADR-0020 (delivery), and OPERATIONS.md (World0
protection). Nothing here deploys a world; World0 launch stays gated by M8.8.
## Context
The inventory found five things that matter here:
- Argo CD's `default` AppProject allows every cluster resource in every namespace.
Three applications use it, two with automated prune and self-heal.
- The only StorageClass reclaims with Delete.
- Collection ran as `system:admin`.
- There are no quotas, limit ranges or priority classes.
- Only the AEvolution application is manual-sync.
The roadmap's named risk is that one permissive application or credential could
destroy every world. Argo project boundaries cannot address that alone, because a
`default`-project application, a cascading parent application or a cluster-admin
is not constrained by an AEvolution AppProject.
## Decision
**Three environments,** each in its own namespace: `evolution-dev`,
`evolution-stage` and `evolution-world0`. Each has its own AppProject, service
accounts, Secrets, database, database logins and object PVC. No identity, Secret or
database is shared between them.
**Platform layer** (`deploy/kubernetes/platform/<env>`), applied by a cluster
administrator and never by an Argo CD Application, so no sync or prune can remove
it. It contains:
- The Namespace, labelled with its environment and with Pod Security `restricted`
enforced. Only World0 carries `aevolution.io/protected=true`.
- A ResourceQuota:
- dev and stage: 2 CPU and 2 GiB requests, 4 GiB of memory limits, 4 PVCs and
40 GiB;
- World0: 4 CPU and 4 GiB requests, 8 GiB of limits, 8 PVCs and 200 GiB;
- no LoadBalancer or NodePort services anywhere.
- A LimitRange (defaults 50m/64Mi with a 256Mi limit, at most 2Gi per container)
sized from the measured host profile.
- A default-deny NetworkPolicy for every pod, plus DNS egress only. Workloads add
narrow rules in M8.4.
- Service accounts `aevolution-migrator`, `aevolution-region` and
`aevolution-backup`, with no token and no RBAC; nothing needs the Kubernetes API.
They are platform-owned, so workload projects cannot create them.
**Secrets never enter Git.** Operators create them per environment, by reference
name (ENVIRONMENTS.md). CI holds no cluster or data credentials (ADR-0020).
**World0 admission protection.** A native ValidatingAdmissionPolicy
(`deploy/kubernetes/platform/admission`) is GA on the cluster's Kubernetes 1.32. It
needs no webhook or dependency, and it fails closed. It applies to everyone,
including system:masters and Argo's controller. Unless the requester is in the
group `aevolution:world0-break-glass`, it denies:
- deleting the World0 namespace, or removing its protected label;
- deleting its PVCs, Secrets or service accounts;
- deleting a PV bound to it, or moving such a PV away from `Retain`;
- deleting the `aevolution-world0` Application or AppProject.
A cascading parent app, a prune, a `default`-project application or a stray
`kubectl delete` therefore fails at admission. Kubernetes does not apply policies
to policy objects, so removing the policy is itself the break-glass action,
reserved to cluster-admins.
**Delivery.**
- The stage and World0 AppProjects (examples) each target one namespace. They
allow no cluster kinds, and whitelist only namespaced workload kinds.
- World0 carries a permanent deny sync window with `manualSync: true`, so no
automated sync can ever run.
- Applications never auto-sync and never carry the cascade finalizer. World0's must
not appear in a pruning parent.
- Durable PVCs will carry `Prune=false,Delete=false`.
**Cleanup.** `tools/ops/cleanup-environment.sh` accepts only `dev` or `stage`. It
refuses anything else before invoking kubectl, re-checks the namespace labels,
never touches platform objects or Secrets, and dry-runs unless `--execute` is given.
**Verification.**
- `tools/ci/deploy-policy-check.py` runs offline in CI (Ubuntu `python3-yaml`) and
asserts every invariant above over all manifests.
- `tools/ops/verify-isolation.sh` runs live. It makes only `can-i` queries, reads
and server dry-runs, and covers negative RBAC, cross-environment denial, World0
admission denials, project scopes and the cleanup refusal.
## Consequences
- **Residual risk: creation.** A `default`-project application can still *create*
objects in AEvolution namespaces; admission protects only World0's data and
identity from deletion. Recommended (owner's decision): restrict `default` or
move those applications to scoped projects.
- **Residual risk: advisory locks.** The runtime login's advisory-lock capability
(ADR-0068) remains.
- **Local-path storage.** Its PVs are Delete-reclaim. A World0 PV must be patched
to Retain when bound, and the policy then keeps it there. Volumes remain
node-local until M8.3.
- **Quotas and defaults.** Quotas are starting values from measured profiles. They
are raised by editing the platform overlay, never by the workload.
- **Existing dev namespace.** Applying the dev overlay to the existing namespace
adds labels, the quota and the default-deny policy. Its proof Job already
complies.
## Alternatives rejected
- **An admission webhook or policy engine (Kyverno, Gatekeeper).** Either is a new
dependency with its own availability risk; the native policy suffices.
- **Relying on AppProjects alone.** They do not constrain other projects, parents
or cluster-admins.
- **Namespaces created by Argo `CreateNamespace`.** A prune or cascade could then
remove them.
+1
View File
@@ -79,3 +79,4 @@ design. No unadopted candidate is a link dependency of the proof.
- [ADR-0066: Operations endpoints](0066-operations-endpoints.md) - accepted for M7.8 liveness vs authority readiness, bounded metrics, dependency-free responder and the server image.
- [ADR-0067: Region clones and retained-history audit](0067-region-clones.md) - accepted for M7.9a re-identified clone genesis naming its source cut, isolated clone databases and full-chain retention audit.
- [ADR-0068: Upgrade barrier and durability qualification](0068-upgrade-barrier.md) - accepted for M7.9b build-bound authority, retirement barrier fencing old writers, explicit refusal of rule/format changes and the measured recovery fixture.
- [ADR-0069: Environment isolation and World0 protection](0069-environment-isolation.md) - accepted for M8.2 per-environment guardrails, native World0 admission protection, scoped Argo projects and verification.
+24
View File
@@ -2639,3 +2639,27 @@ Evidence:
- Collection is manual, with no automation planned.
- No credential values were read. The 18.6 version, role and TLS requirements are
enforced by the adapter and migrations when M8.3 connects.
## M8.2 environment isolation - 2026-09-29
- `tools/ci/deploy-policy-check.py` (WSL, Python 3 with PyYAML): 35 objects in 27
files, 0 failures.
- Nine mutations were each caught, then restored:
- World0 auto-sync;
- the World0 cascade finalizer;
- the policy set to failurePolicy Ignore;
- the policy no longer covering Secrets;
- an egress hole in default-deny;
- the World0 protected label removed;
- the cleanup tool accepting world0 (rc 127, it reached kubectl);
- service-account token automount;
- a wildcard stage destination.
- `kubectl kustomize` (client v1.31.4) renders the dev, stage and World0 platform
overlays (9 objects each), and still renders the live `deploy/kubernetes/dev`
path unchanged.
- `bash -n` passes for both ops scripts.
- The CI `deploy` suite (a Docker `ci-deploy` stage) has not run yet; Docker is
not running locally.
- Not executed: applying anything to the cluster, the admission policy's CEL
compilation (it needs an API server; the runbook validates it with a server
dry run first), and `tools/ops/verify-isolation.sh`.
+74
View File
@@ -0,0 +1,74 @@
# AEvolution environments (M8.2)
The decision is recorded in ADR-0069. This runbook applies the platform layer,
creates the per-environment references, and verifies isolation. Every step is
manual, and none is performed by Argo CD.
## Environments
| | dev | stage | world0 |
|---|---|---|---|
| Namespace | `evolution-dev` (exists) | `evolution-stage` | `evolution-world0` |
| Purpose | Development | Clone, migration and soak rehearsal | Persistent world (launch gated by M8.8) |
| Protected by admission | No | No | Yes |
| Cleanup tool | Allowed | Allowed | Refused |
| AppProject / Application | `aevolution-dev` (live, manual) | `deploy/argocd/stage/*.example` | `deploy/argocd/world0/*.example` (deny window) |
| Database | Its own database and logins on the AEvolution PostgreSQL VM | Its own | Its own |
| World IDs | Development range | Clone targets (ADR-0067) | World0's own; never reused elsewhere |
## Secret references (created by an operator, never committed)
Each namespace gets its own Secrets, with environment-specific values:
- `harbor-pull`: a pull-only Harbor robot (docker-registry type).
- `evolution-postgres-migrator`: the migrator/owner login for that environment's
database.
- `evolution-postgres-runtime`: the runtime login, a member of that environment's
NOLOGIN runtime group (ADR-0060).
- `aevolution-db`: connection settings (host, address, port, database, TLS mode
and root certificate). Mount it as files. The adapter reads a password file,
never `PGPASSWORD`.
A World0 credential must never exist in the dev or stage namespaces, and no dev or
stage credential may reach the World0 database. Rotate by replacing the Secret and
restarting the consumer.
## Apply (cluster administrator)
```sh
# 1. Platform guardrails, per environment (dev adds labels, quota and policies to
# the existing namespace; its completed proof Job already complies).
kubectl apply -k deploy/kubernetes/platform/dev
kubectl apply -k deploy/kubernetes/platform/stage
kubectl apply -k deploy/kubernetes/platform/world0 # an empty, protected namespace
# 2. World0 admission protection (validate first, then apply).
kubectl apply --dry-run=server -f deploy/kubernetes/platform/admission/
kubectl apply -f deploy/kubernetes/platform/admission/
# 3. Projects (resolve the Application paths only when M8.4 adds workloads).
kubectl apply -f deploy/argocd/stage/project.yaml.example
kubectl apply -f deploy/argocd/world0/project.yaml.example
# 4. Verify: reads, can-i and server dry runs only.
bash tools/ops/verify-isolation.sh
```
Create the Secrets listed above in each namespace before any workload syncs.
## Break-glass
To delete World0 data or identity on purpose:
1. Record the operation and its approval.
2. Use a credential whose groups include `aevolution:world0-break-glass`, for
example a short-lived client certificate with that organization.
3. Take and verify a backup first (M8.7).
Removing the admission policy itself is the break-glass of last resort. It needs
cluster-admin and must be recorded the same way. Never test by deleting World0: the
verify script uses server-side dry runs.
## Residual risks
- **`default` AppProject.** It still allows every destination, so its applications
(other projects, two with automated prune) can create objects in AEvolution
namespaces. Admission denies only World0 deletions. Recommended: restrict
`default`, or move those applications to scoped projects.
- **World0 volumes.** `local-path` provisions volumes with Delete. Patch each World0
PV to `Retain` when it is bound, and the policy then keeps it there. Volumes stay
node-local until M8.3.
+168
View File
@@ -0,0 +1,168 @@
#!/usr/bin/env python3
"""M8.2 offline deployment-policy check (ADR-0069). Needs Python 3 and PyYAML only.
Parses every manifest under deploy/ (including .yaml.example files) and fails if an
environment-isolation or World0-protection invariant is broken:
- no Secret objects in Git;
- every ServiceAccount disables token automounting;
- each platform overlay: labelled restricted namespace, quota, limit range,
default-deny and DNS-only policies, three service accounts; World0 alone is
labelled protected;
- the World0 admission policy denies what ADR-0069 lists, fails closed and is
bound with Deny;
- every AEvolution AppProject targets exactly its own namespace with no cluster
kinds; World0 blocks automated sync; no Application auto-syncs or carries the
cascade-delete finalizer, and each stays in its own environment;
- the live Argo path (deploy/kubernetes/dev) holds no Namespace, Secret, quota or
RBAC objects;
- the cleanup tool refuses World0 before invoking kubectl.
"""
import os
import shutil
import subprocess
import sys
import tempfile
import yaml
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
failures = []
def check(condition, message):
if not condition:
failures.append(message)
def documents(path):
with open(path, encoding='utf-8') as f:
return [d for d in yaml.safe_load_all(f) if isinstance(d, dict)]
manifests = {}
for base, _, files in os.walk(os.path.join(ROOT, 'deploy')):
for name in files:
if name.endswith(('.yaml', '.yml', '.yaml.example')):
path = os.path.join(base, name)
manifests[os.path.relpath(path, ROOT).replace(os.sep, '/')] = documents(path)
objects = [(p, d) for p, ds in manifests.items() for d in ds
if 'kind' in d and d['kind'] != 'Kustomization']
# Secrets and service-account tokens.
for path, doc in objects:
check(doc['kind'] != 'Secret', f'{path}: Secret objects never belong in Git')
if doc['kind'] == 'ServiceAccount':
check(doc.get('automountServiceAccountToken') is False,
f'{path}: ServiceAccount {doc["metadata"]["name"]} must not automount a token')
# Platform overlays.
platform = 'deploy/kubernetes/platform'
base = {d['kind'] + '/' + d['metadata']['name']: d
for p, d in objects if p.startswith(platform + '/base/')}
for key in ('ResourceQuota/aevolution-quota', 'LimitRange/aevolution-defaults',
'NetworkPolicy/aevolution-default-deny', 'NetworkPolicy/aevolution-allow-dns',
'ServiceAccount/aevolution-migrator', 'ServiceAccount/aevolution-region',
'ServiceAccount/aevolution-backup'):
check(key in base, f'{platform}/base: missing {key}')
deny = base.get('NetworkPolicy/aevolution-default-deny', {}).get('spec', {})
check(deny.get('podSelector') == {} and sorted(deny.get('policyTypes', [])) == ['Egress', 'Ingress']
and not deny.get('ingress') and not deny.get('egress'),
'default-deny must select every pod and allow nothing')
dns = base.get('NetworkPolicy/aevolution-allow-dns', {}).get('spec', {})
dns_ports = [p.get('port') for rule in dns.get('egress', []) for p in rule.get('ports', [])]
check(dns.get('policyTypes') == ['Egress'] and set(dns_ports) == {53} and len(dns.get('egress', [])) == 1,
'allow-dns must allow only egress to port 53')
quota = base.get('ResourceQuota/aevolution-quota', {}).get('spec', {}).get('hard', {})
check(quota.get('services.loadbalancers') == '0' and quota.get('services.nodeports') == '0',
'quota must forbid LoadBalancer and NodePort services')
for env in ('dev', 'stage', 'world0'):
k = f'{platform}/{env}/kustomization.yaml'
ns_path = f'{platform}/{env}/namespace.yaml'
check(k in manifests and ns_path in manifests, f'{platform}/{env}: overlay missing')
if k not in manifests or ns_path not in manifests:
continue
kustomization = manifests[k][0]
check(kustomization.get('namespace') == f'evolution-{env}' and '../base' in kustomization.get('resources', []),
f'{k}: must set namespace evolution-{env} and include ../base')
labels = manifests[ns_path][0]['metadata'].get('labels', {})
check(manifests[ns_path][0]['metadata']['name'] == f'evolution-{env}', f'{ns_path}: wrong name')
check(labels.get('aevolution.io/environment') == env, f'{ns_path}: environment label')
check(labels.get('pod-security.kubernetes.io/enforce') == 'restricted', f'{ns_path}: restricted pod security')
check((labels.get('aevolution.io/protected') == 'true') == (env == 'world0'),
f'{ns_path}: only world0 is protected')
# World0 admission policy.
policy = [d for p, d in objects if d['kind'] == 'ValidatingAdmissionPolicy'
and d['metadata']['name'] == 'aevolution-world0-protection']
binding = [d for p, d in objects if d['kind'] == 'ValidatingAdmissionPolicyBinding'
and d['metadata']['name'] == 'aevolution-world0-protection']
check(len(policy) == 1 and len(binding) == 1, 'world0 admission policy and binding must exist once')
if policy and binding:
spec = policy[0]['spec']
check(spec.get('failurePolicy') == 'Fail', 'world0 policy must fail closed')
check(binding[0]['spec'].get('validationActions') == ['Deny'] and 'paramRef' not in binding[0]['spec']
and 'matchResources' not in binding[0]['spec'], 'world0 binding must Deny everywhere')
covered = {(r, op) for rule in spec['matchConstraints']['resourceRules']
for r in rule['resources'] for op in rule['operations']}
for resource in ('namespaces', 'persistentvolumes', 'persistentvolumeclaims', 'secrets',
'serviceaccounts', 'applications', 'appprojects'):
check((resource, 'DELETE') in covered, f'world0 policy must cover DELETE {resource}')
for resource in ('namespaces', 'persistentvolumes'):
check((resource, 'UPDATE') in covered, f'world0 policy must cover UPDATE {resource}')
text = yaml.safe_dump(spec)
for needle in ("aevolution:world0-break-glass", "evolution-world0", "aevolution-world0",
"aevolution.io/protected", "Retain"):
check(needle in text, f'world0 policy must reference {needle}')
# Argo CD projects and applications.
for path, doc in objects:
name = doc['metadata'].get('name', '')
if doc['kind'] == 'AppProject' and name.startswith('aevolution-'):
env = name[len('aevolution-'):]
spec = doc['spec']
namespaces = [d.get('namespace') for d in spec.get('destinations', [])]
check(namespaces == [f'evolution-{env}'], f'{path}: {name} must target only evolution-{env}')
check(spec.get('clusterResourceWhitelist') == [], f'{path}: {name} must allow no cluster kinds')
kinds = {w.get('kind') for w in spec.get('namespaceResourceWhitelist', [])}
check(not kinds & {'Secret', 'Namespace', 'ResourceQuota', 'LimitRange', 'Role', 'RoleBinding', '*'},
f'{path}: {name} whitelists a forbidden kind')
if env == 'world0':
windows = spec.get('syncWindows', [])
check(any(w.get('kind') == 'deny' and w.get('manualSync') is True and w.get('duration') == '24h'
and w.get('schedule') == '* * * * *' for w in windows),
f'{path}: world0 must deny automated sync at all times')
check('ServiceAccount' not in kinds, f'{path}: world0 service accounts are platform-managed')
if doc['kind'] == 'Application':
spec = doc['spec']
project = spec.get('project', '')
destination = spec.get('destination', {}).get('namespace', '')
check('automated' not in (spec.get('syncPolicy') or {}), f'{path}: {name} must not auto-sync')
check('resources-finalizer.argocd.argoproj.io' not in doc['metadata'].get('finalizers', []),
f'{path}: {name} must not cascade-delete')
if project.startswith('aevolution-'):
check(destination == 'evolution-' + project[len('aevolution-'):],
f'{path}: {name} leaves its project environment')
check(destination != 'evolution-world0' or name == 'aevolution-world0',
f'{path}: only aevolution-world0 may target evolution-world0')
# The live Argo path.
for path, doc in objects:
if path.startswith('deploy/kubernetes/dev/'):
check(doc['kind'] not in {'Namespace', 'Secret', 'ResourceQuota', 'LimitRange', 'Role',
'RoleBinding', 'ClusterRole', 'ClusterRoleBinding'},
f'{path}: {doc["kind"]} must not live in the Argo workload path')
# The cleanup tool refuses World0 before kubectl runs (no kubectl on PATH here).
with tempfile.TemporaryDirectory() as empty:
env = dict(os.environ, PATH=empty + os.pathsep + os.path.dirname(sys.executable))
for bad in ('world0', 'evolution-world0', 'World0', 'prod', ''):
result = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, 'tools/ops/cleanup-environment.sh'), bad],
capture_output=True, text=True, env=env)
check(result.returncode == 2 and ('refused' in result.stderr or 'usage' in result.stderr)
and 'kubectl' not in result.stderr,
f'cleanup must refuse {bad!r} before kubectl (rc={result.returncode})')
for message in failures:
print('FAIL', message)
print(f'deploy policy: {len(objects)} objects in {len(manifests)} files, {len(failures)} failures')
sys.exit(1 if failures else 0)
+42
View File
@@ -0,0 +1,42 @@
#!/usr/bin/env bash
# M8.2 environment cleanup for disposable AEvolution environments only.
#
# tools/ops/cleanup-environment.sh <dev|stage> [--execute] [--context <name>]
#
# Removes AEvolution workload objects (Deployments, StatefulSets, Jobs, Services,
# ConfigMaps and PersistentVolumeClaims labelled app.kubernetes.io/part-of=aevolution)
# from evolution-<env>. Platform guardrails (namespace, quota, limit range, network
# policies, service accounts) and Secrets are never touched. Without --execute every
# delete is a server-side dry run. World0, or any namespace that is not labelled as
# the named disposable environment or is labelled protected, is refused before any
# change; the refusal of World0 happens before kubectl is even invoked.
set -euo pipefail
usage() { echo "usage: $0 <dev|stage> [--execute] [--context <name>]" >&2; exit 2; }
[ $# -ge 1 ] || usage
env="$1"; shift
case "$env" in
dev|stage) ;;
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
exit 2 ;;
esac
dry_run=(--dry-run=server)
k=(kubectl --request-timeout=30s)
while [ $# -gt 0 ]; do
case "$1" in
--execute) dry_run=() ;;
--context) [ $# -ge 2 ] || usage; k+=(--context "$2"); shift ;;
*) usage ;;
esac
shift
done
ns="evolution-$env"
labels="$("${k[@]}" get namespace "$ns" -o jsonpath='{.metadata.labels.aevolution\.io/environment}/{.metadata.labels.aevolution\.io/protected}')"
if [ "$labels" != "$env/" ]; then
echo "refused: namespace $ns labels are '$labels' (need aevolution.io/environment=$env and no aevolution.io/protected)" >&2
exit 2
fi
selector='app.kubernetes.io/part-of=aevolution,app.kubernetes.io/managed-by!=aevolution-platform'
"${k[@]}" delete -n "$ns" "${dry_run[@]}" --wait=false -l "$selector" \
deployments,statefulsets,jobs,services,configmaps,persistentvolumeclaims
[ ${#dry_run[@]} -eq 0 ] && echo "deleted AEvolution workloads in $ns" ||
echo "dry run only (server-side); rerun with --execute to delete in $ns"
+110
View File
@@ -0,0 +1,110 @@
#!/usr/bin/env bash
# M8.2 live isolation checks. Changes nothing: it only asks `kubectl auth can-i`
# (impersonating the environments' service accounts), reads objects, and sends
# DELETE requests with --dry-run=server, which run admission but never persist.
#
# tools/ops/verify-isolation.sh [--context <name>]
#
# Needs an identity that may impersonate service accounts and read namespaces,
# policies and Argo CD projects (a cluster administrator). Prints PASS/FAIL/SKIP per
# check and exits nonzero if any check fails.
set -uo pipefail
k=(kubectl --request-timeout=20s)
[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2")
fails=0
pass() { echo "PASS $*"; }
fail() { echo "FAIL $*"; fails=$((fails + 1)); }
skip() { echo "SKIP $*"; }
envs=(dev stage world0)
present=()
for env in "${envs[@]}"; do
if "${k[@]}" get namespace "evolution-$env" > /dev/null 2>&1; then
present+=("$env")
else
skip "namespace evolution-$env does not exist"
fi
done
# 1. Guardrails and labels in every present environment.
for env in "${present[@]}"; do
ns="evolution-$env"
got="$("${k[@]}" get namespace "$ns" -o jsonpath='{.metadata.labels.aevolution\.io/environment} {.metadata.labels.pod-security\.kubernetes\.io/enforce} {.metadata.labels.aevolution\.io/protected}')"
want="$env restricted"; [ "$env" = world0 ] && want="$env restricted true"
[ "$(echo "$got" | xargs)" = "$want" ] && pass "$ns labels ($got)" || fail "$ns labels '$got' (want '$want')"
for object in resourcequota/aevolution-quota limitrange/aevolution-defaults \
networkpolicy/aevolution-default-deny networkpolicy/aevolution-allow-dns \
serviceaccount/aevolution-migrator serviceaccount/aevolution-region serviceaccount/aevolution-backup; do
"${k[@]}" get -n "$ns" "$object" > /dev/null 2>&1 && pass "$ns has $object" || fail "$ns lacks $object"
done
done
# 2. No environment identity can read secrets or change anything anywhere.
for env in "${present[@]}"; do
for sa in aevolution-migrator aevolution-region aevolution-backup; do
who="system:serviceaccount:evolution-$env:$sa"
for target in "${envs[@]}"; do
for action in "get secrets" "list secrets" "create pods" "delete persistentvolumeclaims" "create rolebindings"; do
answer="$("${k[@]}" auth can-i $action -n "evolution-$target" --as "$who" 2>/dev/null)"
[ "$answer" = "no" ] && pass "$who cannot $action in evolution-$target" ||
fail "$who can $action in evolution-$target"
done
done
answer="$("${k[@]}" auth can-i delete namespaces --as "$who" 2>/dev/null)"
[ "$answer" = "no" ] && pass "$who cannot delete namespaces" || fail "$who can delete namespaces"
done
done
# 3. World0 admission protection, exercised with server-side dry runs.
if "${k[@]}" get validatingadmissionpolicybinding aevolution-world0-protection > /dev/null 2>&1; then
pass "world0 protection policy is bound"
else
fail "world0 protection policy binding missing"
fi
denied() { # denied <description> <kubectl delete args...> (always --dry-run=server)
local what="$1"; shift
local out
out="$("${k[@]}" delete --dry-run=server --wait=false "$@" 2>&1)"
if echo "$out" | grep -q "aevolution:world0-break-glass"; then
pass "denied: $what"
elif echo "$out" | grep -qi "not found"; then
skip "$what (object absent)"
else
fail "not denied: $what -> $out"
fi
}
if printf '%s\n' "${present[@]}" | grep -qx world0; then
denied "delete namespace evolution-world0" namespace evolution-world0
denied "delete service account in evolution-world0" -n evolution-world0 serviceaccount aevolution-region
secret="$("${k[@]}" get secrets -n evolution-world0 -o name 2>/dev/null | head -1)"
[ -n "$secret" ] && denied "delete $secret in evolution-world0" -n evolution-world0 "$secret" ||
skip "no Secret in evolution-world0 to test"
pvc="$("${k[@]}" get pvc -n evolution-world0 -o name 2>/dev/null | head -1)"
[ -n "$pvc" ] && denied "delete $pvc in evolution-world0" -n evolution-world0 "$pvc" ||
skip "no PVC in evolution-world0 to test"
else
skip "world0 admission dry runs (namespace absent)"
fi
# 4. Argo CD projects: each AEvolution project targets only its own namespace.
for env in "${envs[@]}"; do
dest="$("${k[@]}" get appproject "aevolution-$env" -n argocd -o jsonpath='{range .spec.destinations[*]}{.namespace}{" "}{end}{.spec.clusterResourceWhitelist}' 2>/dev/null)" || {
skip "appproject aevolution-$env not present"; continue; }
[ "$(echo "$dest" | xargs)" = "evolution-$env" ] && pass "appproject aevolution-$env destinations '$dest'" ||
fail "appproject aevolution-$env destinations/cluster kinds '$dest'"
done
wide="$("${k[@]}" get appproject default -n argocd -o jsonpath='{.spec.destinations[*].namespace}' 2>/dev/null)"
[ "$wide" = "*" ] && echo "WARN appproject default allows every namespace; its applications could create objects in evolution-* (deletion is still denied in world0 by admission)"
# 5. The cleanup tool refuses World0 before touching the cluster.
root="$(cd "$(dirname "$0")/../.." && pwd)"
for bad in world0 evolution-world0 World0 ""; do
if bash "$root/tools/ops/cleanup-environment.sh" "$bad" > /dev/null 2>&1; then
fail "cleanup accepted '$bad'"
else
pass "cleanup refused '$bad'"
fi
done
echo "failures=$fails"
[ "$fails" -eq 0 ]