Deliver the platform layer and store Jobs through Argo CD

At the maintainer's request everything but Secrets and the store node
label now goes through Argo CD. aevolution-platform syncs one bundle
(World0 admission policy, Retain storage class, each environment's
namespace, guardrails, service accounts and object claim) with
self-heal, no prune and Prune=false,Delete=false on every object; the
admission policy also protects the platform app and project. Store Jobs
become kustomize components used as PreSync (migrate) and PostSync
(check-store) hooks per environment, and the policy check refuses any
unresolved marker in a path an Application syncs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-29 17:05:34 -04:00
co-authored by Claude Opus 5.5
parent 8d594f28af
commit 7318d82a14
20 changed files with 505 additions and 223 deletions
+26
View File
@@ -0,0 +1,26 @@
# M8.2/M8.3 platform Application (ADR-0069 amendment). Self-heal restores drift
# (a deleted quota, policy or label comes back); prune is off and every object
# carries Delete=false, so removing a file from Git or deleting this Application
# never deletes a namespace, claim, class or policy. No cascade finalizer. The
# World0 admission policy also denies deleting this Application and its project.
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: aevolution-platform
namespace: argocd
spec:
project: aevolution-platform
source:
repoURL: https://git.aeternalisgames.com/dingo/AEvolution.git
targetRevision: main
path: deploy/kubernetes/platform/cluster
destination:
server: https://kubernetes.default.svc
namespace: evolution-dev
syncPolicy:
automated:
prune: false
selfHeal: true
syncOptions:
- FailOnSharedResource=true
- ServerSideApply=true
+33
View File
@@ -0,0 +1,33 @@
# M8.2/M8.3 platform project (ADR-0069 amendment). Bootstrap once by hand; Argo CD
# cannot create its own first project: kubectl apply -f deploy/argocd/platform/
# Only the three AEvolution namespaces, and only the cluster kinds the platform
# layer owns. Workload projects (aevolution-dev/-stage/-world0) stay separate.
apiVersion: argoproj.io/v1alpha1
kind: AppProject
metadata:
name: aevolution-platform
namespace: argocd
spec:
description: AEvolution platform layer (namespaces, guardrails, storage, World0 protection)
sourceRepos:
- https://git.aeternalisgames.com/dingo/AEvolution.git
destinations:
- server: https://kubernetes.default.svc
namespace: evolution-dev
- server: https://kubernetes.default.svc
namespace: evolution-stage
- server: https://kubernetes.default.svc
namespace: evolution-world0
clusterResourceWhitelist:
- {group: "", kind: Namespace}
- {group: storage.k8s.io, kind: StorageClass}
- {group: admissionregistration.k8s.io, kind: ValidatingAdmissionPolicy}
- {group: admissionregistration.k8s.io, kind: ValidatingAdmissionPolicyBinding}
namespaceResourceWhitelist:
- {group: "", kind: ResourceQuota}
- {group: "", kind: LimitRange}
- {group: "", kind: ServiceAccount}
- {group: "", kind: PersistentVolumeClaim}
- {group: networking.k8s.io, kind: NetworkPolicy}
orphanedResources:
warn: true
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- world0-protection.yaml
@@ -12,7 +12,8 @@
# - deleting a PersistentVolume bound to evolution-world0, or changing its reclaim
# policy away from Retain once set (set Retain when the volume is bound: the
# only StorageClass, local-path, provisions with Delete);
# - deleting the aevolution-world0 Argo CD Application or AppProject.
# - deleting the aevolution-world0 or aevolution-platform Argo CD Application or
# AppProject (the platform app owns this policy and the World0 namespace).
# Kubernetes does not run admission policies on policy objects themselves, so
# removing this policy is itself the audited break-glass step (cluster-admin).
# Apply by hand: kubectl apply -f deploy/kubernetes/platform/admission/
@@ -56,7 +57,8 @@ spec:
variables.resource in ['persistentvolumeclaims', 'secrets', 'serviceaccounts']
- name: world0Delivery
expression: >-
request.resource.group == 'argoproj.io' && request.name == 'aevolution-world0'
request.resource.group == 'argoproj.io' &&
request.name in ['aevolution-world0', 'aevolution-platform']
validations:
- expression: >-
variables.breakGlass || request.operation != 'DELETE' ||
@@ -0,0 +1,16 @@
# M8.2/M8.3 platform layer as one Argo CD Application, aevolution-platform
# (deploy/argocd/platform, ADR-0069 amendment): the World0 admission policy, the
# Retain storage class, and each environment's namespace, guardrails, service
# accounts and object claim. Self-heal restores drift; prune is never enabled and
# every durable object carries Delete=false, so neither a Git removal nor deleting
# the Application removes a namespace, claim, class or policy.
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../admission
- ../storage
- ../dev
- ../stage
- ../world0
commonAnnotations:
argocd.argoproj.io/sync-options: Prune=false,Delete=false
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- storageclass.yaml
@@ -0,0 +1,23 @@
# Egress from migrate, store-check and region pods to exactly the AEvolution
# PostgreSQL VM (pg-forge-01) and nothing else (ADR-0070).
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: aevolution-database-egress
labels:
app.kubernetes.io/part-of: aevolution
annotations:
argocd.argoproj.io/sync-wave: "-1"
spec:
podSelector:
matchExpressions:
- key: app.kubernetes.io/component
operator: In
values: [migrate, store-check, region]
policyTypes: [Egress]
egress:
- to:
- ipBlock:
cidr: 172.10.1.127/32
ports:
- {protocol: TCP, port: 5432}
@@ -0,0 +1,12 @@
# M8.3 store Jobs as Argo CD hooks (ADR-0070 amendment). Included by an
# environment's workload path through its overlay (stores/dev, stores/stage), which
# pins the image digests and names the environment's NOLOGIN runtime role.
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- database-egress.yaml
- migrate-job.yaml
- store-check-job.yaml
labels:
- pairs:
app.kubernetes.io/part-of: aevolution
@@ -0,0 +1,60 @@
# PreSync hook: every sync of the environment first migrates its database with the
# migrator login (version pin and runtime privilege audit included); a failure
# stops the sync before anything else changes.
apiVersion: batch/v1
kind: Job
metadata:
name: aevolution-db-migrate
labels:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: migrate
annotations:
argocd.argoproj.io/hook: PreSync
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
spec:
backoffLimit: 0
activeDeadlineSeconds: 180
template:
metadata:
labels:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: migrate
spec:
serviceAccountName: aevolution-migrator
automountServiceAccountToken: false
restartPolicy: Never
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile: {type: RuntimeDefault}
containers:
- name: migrate
image: aevolution-postgres-tools
env:
- {name: PGSERVICE, value: evolution}
- {name: PGSERVICEFILE, value: /run/postgres/pg_service.conf}
- {name: EVO_PGPASS_SOURCE, value: /run/postgres/pgpass}
- {name: EVO_PG_RUNTIME_ROLE, value: set-by-overlay}
resources:
requests: {cpu: 100m, memory: 64Mi}
limits: {memory: 128Mi}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
volumeMounts:
- {name: connection, mountPath: /run/postgres, readOnly: true}
- {name: scratch, mountPath: /tmp}
volumes:
- name: connection
secret:
secretName: evolution-postgres-migrator
defaultMode: 0440
items:
- {key: pg_service.conf, path: pg_service.conf}
- {key: pgpass, path: pgpass}
- {key: ca.crt, path: ca.crt}
- name: scratch
emptyDir: {medium: Memory, sizeLimit: 8Mi}
@@ -0,0 +1,72 @@
# PostSync hook: qualifies the object volume and the runtime login after each sync
# (evo-world-clone check-store, ADR-0070). Its log is the store evidence.
apiVersion: batch/v1
kind: Job
metadata:
name: aevolution-store-check
labels:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: store-check
annotations:
argocd.argoproj.io/hook: PostSync
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
spec:
backoffLimit: 0
activeDeadlineSeconds: 120
template:
metadata:
labels:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: store-check
spec:
serviceAccountName: aevolution-region
automountServiceAccountToken: false
restartPolicy: Never
nodeSelector:
aevolution.io/stores: "true"
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile: {type: RuntimeDefault}
containers:
- name: check-store
image: aevolution-server
# libpq ignores a group-readable password file, and Secret volumes are
# group-readable for a non-root fsGroup, so copy it to private memory first.
# w1/r1 is a reserved probe namespace, never a real world.
command: ["/bin/bash", "-c"]
args:
- |
set -eu
umask 077
cp /run/postgres/pgpass /tmp/pgpass
exec /usr/local/bin/evo-world-clone check-store --world 1 --region 1 \
--objects /objects --pg-service /run/postgres/pg_service.conf \
--pg-passfile /tmp/pgpass
resources:
requests: {cpu: 50m, memory: 64Mi}
limits: {memory: 256Mi}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
volumeMounts:
- {name: objects, mountPath: /objects}
- {name: connection, mountPath: /run/postgres, readOnly: true}
- {name: scratch, mountPath: /tmp}
volumes:
- name: objects
persistentVolumeClaim:
claimName: aevolution-objects
- name: scratch
emptyDir: {medium: Memory, sizeLimit: 1Mi}
- name: connection
secret:
secretName: evolution-postgres-runtime
defaultMode: 0440
items:
- {key: pg_service.conf, path: pg_service.conf}
- {key: pgpass, path: pgpass}
- {key: ca.crt, path: ca.crt}
@@ -0,0 +1,23 @@
# Store Jobs for evolution-dev. Values still to supply before this is included in
# the dev workload path: the published registry digests (from the CI Publish
# step's "Digest:" line) and the dev database's NOLOGIN runtime group role.
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: evolution-dev
resources:
- ../base
images:
- name: aevolution-postgres-tools
newName: registry.aeternalisgames.com/aeternalis/aevolution-postgres-tools
digest: sha256:REPLACE_PUBLISHED_TOOLS_DIGEST
- name: aevolution-server
newName: registry.aeternalisgames.com/aeternalis/aevolution-server
digest: sha256:REPLACE_PUBLISHED_SERVER_DIGEST
patches:
- target:
kind: Job
name: aevolution-db-migrate
patch: |-
- op: replace
path: /spec/template/spec/containers/0/env/3/value
value: REPLACE_DEV_RUNTIME_GROUP_ROLE
@@ -0,0 +1,23 @@
# Store Jobs for evolution-stage. Values still to supply before this is included in
# the stage workload path: the published registry digests (from the CI Publish
# step's "Digest:" line) and the stage database's NOLOGIN runtime group role.
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: evolution-stage
resources:
- ../base
images:
- name: aevolution-postgres-tools
newName: registry.aeternalisgames.com/aeternalis/aevolution-postgres-tools
digest: sha256:REPLACE_PUBLISHED_TOOLS_DIGEST
- name: aevolution-server
newName: registry.aeternalisgames.com/aeternalis/aevolution-server
digest: sha256:REPLACE_PUBLISHED_SERVER_DIGEST
patches:
- target:
kind: Job
name: aevolution-db-migrate
patch: |-
- op: replace
path: /spec/template/spec/containers/0/env/3/value
value: REPLACE_STAGE_RUNTIME_GROUP_ROLE
@@ -1,163 +0,0 @@
# M8.3 store Jobs for ONE disposable environment (dev or stage). Not an Argo path.
# Copy, then replace every REPLACE_* marker (the markers make this file unappliable
# as committed), and apply by hand:
# REPLACE_NAMESPACE evolution-dev | evolution-stage (never evolution-world0)
# REPLACE_DB_CIDR the database VM's address as a /32 (PGHOSTADDR)
# REPLACE_DB_PORT its port (PGPORT)
# REPLACE_RUNTIME_ROLE that environment's NOLOGIN runtime group role
# REPLACE_TOOLS_DIGEST aevolution-postgres-tools image digest (sha256:...)
# REPLACE_SERVER_DIGEST aevolution server-runtime image digest (sha256:...)
# Secret contract (docs/infrastructure/ENVIRONMENTS.md, never in Git). Each login
# Secret is self-contained, with keys pg_service.conf (section [evolution], with
# sslrootcert=/run/postgres/ca.crt), pgpass and ca.crt:
# evolution-postgres-migrator the environment's migrator login (migrate-postgres.sh)
# evolution-postgres-runtime the environment's runtime login (check-store, region)
# Order: apply the NetworkPolicy, run migrate, then check-store (twice: the second
# run must report "probe verified", proving the volume survived a new pod).
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: aevolution-database-egress
namespace: REPLACE_NAMESPACE
labels:
app.kubernetes.io/part-of: aevolution
spec:
podSelector:
matchExpressions:
- key: app.kubernetes.io/component
operator: In
values: [migrate, store-check, region]
policyTypes: [Egress]
egress:
- to:
- ipBlock:
cidr: REPLACE_DB_CIDR
ports:
- {protocol: TCP, port: REPLACE_DB_PORT}
---
apiVersion: batch/v1
kind: Job
metadata:
name: aevolution-db-migrate
namespace: REPLACE_NAMESPACE
labels:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: migrate
spec:
backoffLimit: 0
activeDeadlineSeconds: 180
ttlSecondsAfterFinished: 86400
template:
metadata:
labels:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: migrate
spec:
serviceAccountName: aevolution-migrator
automountServiceAccountToken: false
restartPolicy: Never
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile: {type: RuntimeDefault}
containers:
- name: migrate
image: registry.aeternalisgames.com/aeternalis/aevolution-postgres-tools@REPLACE_TOOLS_DIGEST
env:
- {name: PGSERVICE, value: evolution}
- {name: PGSERVICEFILE, value: /run/postgres/pg_service.conf}
- {name: EVO_PGPASS_SOURCE, value: /run/postgres/pgpass}
- {name: EVO_PG_RUNTIME_ROLE, value: REPLACE_RUNTIME_ROLE}
resources:
requests: {cpu: 100m, memory: 64Mi}
limits: {memory: 128Mi}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
volumeMounts:
- {name: connection, mountPath: /run/postgres, readOnly: true}
- {name: scratch, mountPath: /tmp}
volumes:
- name: connection
secret:
secretName: evolution-postgres-migrator
defaultMode: 0440
items:
- {key: pg_service.conf, path: pg_service.conf}
- {key: pgpass, path: pgpass}
- {key: ca.crt, path: ca.crt}
- name: scratch
emptyDir: {medium: Memory, sizeLimit: 8Mi}
---
apiVersion: batch/v1
kind: Job
metadata:
name: aevolution-store-check
namespace: REPLACE_NAMESPACE
labels:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: store-check
spec:
backoffLimit: 0
activeDeadlineSeconds: 120
ttlSecondsAfterFinished: 86400
template:
metadata:
labels:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: store-check
spec:
serviceAccountName: aevolution-region
automountServiceAccountToken: false
restartPolicy: Never
nodeSelector:
aevolution.io/stores: "true"
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile: {type: RuntimeDefault}
containers:
- name: check-store
image: registry.aeternalisgames.com/aeternalis/aevolution-server@REPLACE_SERVER_DIGEST
# libpq ignores a group-readable password file, and Secret volumes are
# group-readable for a non-root fsGroup, so copy it to private memory first.
# w1/r1 is a reserved probe namespace, never a real world.
command: ["/bin/bash", "-c"]
args:
- |
set -eu
umask 077
cp /run/postgres/pgpass /tmp/pgpass
exec /usr/local/bin/evo-world-clone check-store --world 1 --region 1 \
--objects /objects --pg-service /run/postgres/pg_service.conf \
--pg-passfile /tmp/pgpass
resources:
requests: {cpu: 50m, memory: 64Mi}
limits: {memory: 256Mi}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
volumeMounts:
- {name: objects, mountPath: /objects}
- {name: connection, mountPath: /run/postgres, readOnly: true}
- {name: scratch, mountPath: /tmp}
volumes:
- name: objects
persistentVolumeClaim:
claimName: aevolution-objects
- name: scratch
emptyDir: {medium: Memory, sizeLimit: 1Mi}
- name: connection
secret:
secretName: evolution-postgres-runtime
defaultMode: 0440
items:
- {key: pg_service.conf, path: pg_service.conf}
- {key: pgpass, path: pgpass}
- {key: ca.crt, path: ca.crt}
+2
View File
@@ -8,6 +8,8 @@ names it. Remove an item only when it is done, and record where it was done.
|---|---|---|---|
| **World0 break-glass membership.** Decide who may act as `aevolution:world0-break-glass`, and how that group is granted: for example a short-lived client certificate with that organization, or an OIDC group. Decide also how its use is approved and recorded. The admission policy already enforces the group (ADR-0069). | M8.2, 2026-09-29 | Maintainer: figure out later | Any intentional deletion of World0 data or identity; M8.8 World0 launch |
| **Backup destination for dev/stage stores.** There are no backups yet (maintainer, 2026-09-29). The plan is to build backup tooling for the AEvolution PostgreSQL VM and each `aevolution-objects` volume up to a pluggable destination, then add the destination. It must be off the host for anything beyond disposable environments. | M8.3, 2026-09-29 | The maintainer will supply a destination later | M8.7 backup and restore; any World0 data |
| **Secret manager for Argo-delivered configuration.** Secrets are still created by hand (ADR-0069 amendment). Choose how they reach the cluster through Git or Argo: for example Vault with the Agent Injector (as in the maintainer's other environment), or sealed or external secrets. Until then, rotate by hand. | M8.3, 2026-09-29 | The maintainer wants everything through Argo; a secret manager is not chosen | Stage and World0 credentials; M8.4 |
| **Store Job values.** Supply the published registry digests for `aevolution-postgres-tools` and `aevolution-server` (from each CI Publish step's `Digest:` line), and the NOLOGIN runtime group role for dev (`aevolution`) and for stage (`aevolution_stage`). Then include `stores/<env>` in each environment's path. | M8.3, 2026-09-29 | Values pending | M8.3 in-cluster evidence |
| **Storage after the RAID/disk rework.** Re-run `deploy/kubernetes/inventory/disk-probe-local-path.yaml` (or a new class), and update INVENTORY.md. | M8.1, 2026-09-29 | Maintainer is reorganizing disks | M8.3's final storage decision |
## Accepted risks (maintainer decisions)
+8
View File
@@ -1036,3 +1036,11 @@ Not yet done:
- the in-cluster run (it needs Secrets with the documented key names and
published image digests);
- choosing a backup destination (docs/BACKLOG.md).
M8.3 delivery moved to Argo CD, at the maintainer's request:
- **Platform app:** `aevolution-platform` (self-heal, never prune, `Delete=false`,
protected by admission) owns the namespaces, guardrails, storage class, object
claims and World0 policy.
- **Store Jobs:** they become sync hooks in each environment's path once their
digests and runtime roles are supplied.
- **Outside Argo:** Secrets and the store node label.
+26
View File
@@ -104,3 +104,29 @@ never touches platform objects or Secrets, and dry-runs unless `--execute` is gi
or cluster-admins.
- **Namespaces created by Argo `CreateNamespace`.** A prune or cascade could then
remove them.
## Amendment: platform delivered by Argo CD (2026-09-29)
The maintainer prefers that every deployment and configuration goes through Argo
CD. The platform layer is therefore an Argo Application, `aevolution-platform`
(`deploy/argocd/platform`), no longer a hand-applied layer. The original concern,
that a sync, prune or cascade could delete namespaces and durable objects, is met
differently:
- **Never prune.** The Application self-heals but never prunes.
- **Protected objects.** Every object in the bundle carries
`argocd.argoproj.io/sync-options: Prune=false,Delete=false`, so neither a Git
removal nor deleting the Application deletes it.
- **No cascade.** There is no cascade finalizer.
- **Protected by admission.** The World0 admission policy also denies deleting the
`aevolution-platform` Application and AppProject.
- **Scoped project.** Its project may target only the three AEvolution namespaces,
and only four cluster kinds: Namespace, StorageClass, ValidatingAdmissionPolicy
and its binding.
Self-heal adds a protection the hand-applied layer lacked: drift, such as a
deleted quota or label, is restored. The admission policy itself is still not
subject to admission, so a policy change is a reviewed Git change.
What stays outside Argo:
- Secrets, until a secret manager is chosen (docs/BACKLOG.md);
- the store node label;
- the one-time bootstrap of the platform project and Application.
+18
View File
@@ -100,3 +100,21 @@ policy check forbids `PGPASSWORD`/`PGUSER` references and whole-Secret `envFrom`
- **Leaving `local-path`/Delete for durable data.** Deleting the claim or an Argo
prune would erase the store.
- **Trusting PVC sizes.** `local-path` does not enforce them.
## Amendment: Argo delivery and the real Secret layout (2026-09-29)
- **Secret layout.** The maintainer's login Secrets have the keys
`pg_service.conf` (section `[evolution]`), `pgpass` and `ca.crt`. `evo-server`
and `evo-world-clone` read the service file with `--pg-service`, a strict parser
that refuses passwords, other options and weaker TLS. The Jobs mount each login
Secret whole, and `aevolution-db` is not used.
- **Argo delivery.** The storage class and object claims are delivered by the
`aevolution-platform` Application (ADR-0069 amendment). The store Jobs are
kustomize components (`deploy/kubernetes/stores`) that an environment's workload
path includes as Argo hooks:
- the migration runs PreSync, so a failure stops the sync;
- `check-store` runs PostSync.
- **Values.** Each environment overlay pins the published image digests and its
runtime group role. A synced path may not contain placeholders; the policy check
enforces this.
- **Stage database.** Stage uses its own database, `aevolution_stage`, on the same
VM, with its own migrator login and runtime group.
+24
View File
@@ -2757,3 +2757,27 @@ admission policy and the stage and World0 AppProjects, then ran
file.
- PostgreSQL Release and ASan/UBSan passed. The analysis build passed after one
loop copy was fixed. Deploy policy: 0 failures.
## M8.3 Argo delivery - 2026-09-29
- The platform layer is now `deploy/kubernetes/platform/cluster`, delivered by the
`aevolution-platform` Application. It renders 29 objects, all marked
`Prune=false,Delete=false`: 3 namespaces, 3 quotas, 3 limit ranges, 6 network
policies, 9 service accounts, 2 claims, the storage class, the policy and its
binding.
- The store Jobs are kustomize components used as Argo hooks. The dev overlay
renders into `evolution-dev` with PreSync and PostSync hooks; its digests and
role are still markers.
- `deploy-policy-check.py`: 43 objects in 41 files, 0 failures. Five new mutations
were each caught:
- platform prune enabled;
- a missing Delete guard;
- an unfinished stores overlay included in the live dev path;
- an extra cluster kind in the platform project;
- the admission policy no longer covering the platform app.
The unfinished-path rule first missed the live dev Application, which exists
here only as an `.example`. It now covers every Application whose path is
resolved.
- The digests the maintainer pasted are local image IDs from the build step, not
registry digests. The registry digest is printed by the Publish step.
- Not executed: bootstrapping the platform Application, or any Argo sync.
+60 -54
View File
@@ -47,68 +47,74 @@ A World0 credential must never exist in the dev or stage namespaces, and no dev
stage credential may reach the World0 database. Rotate by replacing the Secret and
restarting the consumer.
## Apply (cluster administrator)
## Delivery through Argo CD (maintainer preference, 2026-09-29)
Everything except Secrets and one node label is delivered from Git by Argo CD.
**Bootstrap once.** Argo CD cannot create its own first project and application:
```sh
# 1. Platform guardrails, per environment (dev adds labels, quota and policies to
# the existing namespace; its completed proof Job already complies).
kubectl apply -k deploy/kubernetes/platform/dev
kubectl apply -k deploy/kubernetes/platform/stage
kubectl apply -k deploy/kubernetes/platform/world0 # an empty, protected namespace
# 2. World0 admission protection (validate first, then apply).
kubectl apply --dry-run=server -f deploy/kubernetes/platform/admission/
kubectl apply -f deploy/kubernetes/platform/admission/
# 3. Projects (resolve the Application paths only when M8.4 adds workloads).
kubectl apply -f deploy/argocd/stage/project.yaml.example
kubectl apply -f deploy/argocd/world0/project.yaml.example
# 4. Verify: reads, can-i and server dry runs only.
bash tools/ops/verify-isolation.sh
kubectl apply -f deploy/argocd/platform/project.yaml -f deploy/argocd/platform/application.yaml
```
`aevolution-platform` then syncs `deploy/kubernetes/platform/cluster`:
- the World0 admission policy;
- the `aevolution-local-retain` storage class;
- each environment's namespace, quota, limit range, default-deny policies,
service accounts and object claim (dev and stage).
Create the Secrets listed above in each namespace before any workload syncs.
It self-heals, so a deleted quota, policy or label comes back. It never prunes,
and every object carries `Prune=false,Delete=false`, so removing a file or
deleting the Application never deletes a namespace, claim, class or policy. The
admission policy also denies deleting the platform Application or project. Objects
applied by hand earlier are adopted (server-side apply).
**Workloads.** Each environment's Application syncs its path, currently manually:
- `aevolution-dev` syncs `deploy/kubernetes/dev`, which is live.
- Stage uses `deploy/argocd/stage`, whose path is resolved when its values are
supplied.
The store Jobs (`deploy/kubernetes/stores/<env>`) are included in the environment
path as sync hooks:
- the database migration runs as a PreSync hook, so a failed migration stops the
sync;
- `check-store` runs as a PostSync hook.
An environment's stores overlay is included only once it has its published image
digests and its runtime group role. The policy check refuses any `REPLACE_` marker
in a synced path.
**Outside Argo (by design, for now).**
- Secrets are created by an operator; a secret manager is in docs/BACKLOG.md.
- One node label: `kubectl label node rke2-worker-1 aevolution.io/stores=true`.
The maintainer allows any worker; worker-1 is chosen because it had the lowest
measured latency, and worker-3's disk is 87 % full.
**Verify:** `bash tools/ops/verify-isolation.sh` makes only reads, can-i queries and
server dry runs.
## Stores (M8.3, ADR-0070)
Run this once per disposable environment (`dev`, then `stage`).
**1. Label the store node, and apply the storage class and claim.**
```sh
# Label exactly one worker. The maintainer allows any worker (never the control plane);
# worker-1 is chosen (lowest measured latency; worker-3's disk is 87 % full).
kubectl label node rke2-worker-1 aevolution.io/stores=true
kubectl apply -f deploy/kubernetes/platform/storage/storageclass.yaml
kubectl apply -k deploy/kubernetes/platform/dev # adds the aevolution-objects claim
```
**2. Prepare the store Jobs.** Copy `deploy/kubernetes/stores/stores.yaml.example`,
replace its `REPLACE_*` markers (namespace, database address and port, runtime
role, image digests), and create the Secrets above.
**3. Run and qualify.**
```sh
kubectl apply -f stores-dev.yaml # policy + both Jobs
kubectl logs -n evolution-dev job/aevolution-db-migrate # schema and audit
kubectl logs -n evolution-dev job/aevolution-store-check # expect exit 0 and "probe created"
```
Evidence to record in docs/engineering/VALIDATION.md:
- **Pod restart:** delete the check Job and apply it again. It must print
`probe verified`, with the same digest.
- **Missing volume:** run the check with the `objects` volume removed. It must exit
4 with `objects refused missing-or-not-a-directory`.
- **Full volume:** add `--reserve-mib 1048576` and use a new probe region, such as
`--region 2`. It must exit 4 with `probe refused full`.
- **Retention:** `kubectl delete pvc aevolution-objects -n evolution-dev`. The PV
must become `Released`, not deleted. To rebind:
- clear the PV's `spec.claimRef`;
- recreate the claim with `volumeName` set to that PV;
- rerun the check, which must print `probe verified`.
- **Credential rotation:**
- Change the runtime login's password on the database VM, and run the check
before updating the Secret. It must exit 3 with `database refused connect`.
- Update the Secret's `pgpass` and rerun. It must exit 0.
- **Privilege boundary:** every run must print
Once `deploy/kubernetes/stores/dev` has its values and is included in
`deploy/kubernetes/dev`, a manual sync of `aevolution-dev` runs the migration and
then the store check. The hook Jobs keep their logs until the next sync. Record
this evidence in docs/engineering/VALIDATION.md:
- **Pod restart:** sync again. The check must print `probe verified` with the same
digest.
- **Privilege boundary:** every run prints
`database tables=denied functions=reachable`.
- **Missing volume:** temporarily remove the `objects` volume in a branch, or run the
image by hand without it. The check must exit 4 with
`objects refused missing-or-not-a-directory`.
- **Full volume:** add `--reserve-mib 1048576` and a new probe region, such as
`--region 2`. The check must exit 4 with `probe refused full`.
- **Retention:** delete the claim by hand, since Argo never prunes it. The PV must
become `Released`, not deleted. To rebind:
- clear the PV's `spec.claimRef`;
- let the platform app recreate the claim, with `volumeName` pinned in Git;
- sync, and check that it prints `probe verified`.
- **Credential rotation:**
- Change the runtime login's password on the database VM and sync before updating
the Secret. The check must exit 3 with `database refused connect`.
- Update the Secret's `pgpass` and sync again. It must exit 0.
## Break-glass
+67 -4
View File
@@ -111,13 +111,27 @@ if policy and binding:
check((resource, 'UPDATE') in covered, f'world0 policy must cover UPDATE {resource}')
text = yaml.safe_dump(spec)
for needle in ("aevolution:world0-break-glass", "evolution-world0", "aevolution-world0",
"aevolution-platform",
"aevolution.io/protected", "Retain"):
check(needle in text, f'world0 policy must reference {needle}')
# Argo CD projects and applications.
for path, doc in objects:
name = doc['metadata'].get('name', '')
if doc['kind'] == 'AppProject' and name.startswith('aevolution-'):
if doc['kind'] == 'AppProject' and name == 'aevolution-platform':
spec = doc['spec']
namespaces = sorted(d.get('namespace') for d in spec.get('destinations', []))
check(namespaces == ['evolution-dev', 'evolution-stage', 'evolution-world0'],
f'{path}: the platform project targets exactly the three AEvolution namespaces')
cluster = sorted((w.get('group'), w.get('kind')) for w in spec.get('clusterResourceWhitelist', []))
check(cluster == [('', 'Namespace'),
('admissionregistration.k8s.io', 'ValidatingAdmissionPolicy'),
('admissionregistration.k8s.io', 'ValidatingAdmissionPolicyBinding'),
('storage.k8s.io', 'StorageClass')],
f'{path}: the platform project may own only its four cluster kinds')
kinds = {w.get('kind') for w in spec.get('namespaceResourceWhitelist', [])}
check(not kinds & {'Secret', 'Role', 'RoleBinding', '*'}, f'{path}: platform whitelists a forbidden kind')
elif doc['kind'] == 'AppProject' and name.startswith('aevolution-'):
env = name[len('aevolution-'):]
spec = doc['spec']
namespaces = [d.get('namespace') for d in spec.get('destinations', [])]
@@ -136,13 +150,21 @@ for path, doc in objects:
spec = doc['spec']
project = spec.get('project', '')
destination = spec.get('destination', {}).get('namespace', '')
check('automated' not in (spec.get('syncPolicy') or {}), f'{path}: {name} must not auto-sync')
automated = (spec.get('syncPolicy') or {}).get('automated')
if name == 'aevolution-platform':
# Self-heal only: never prune, so no Git change can delete durable objects.
check(automated is not None and automated.get('prune') is False,
f'{path}: the platform app self-heals and never prunes')
check(spec.get('source', {}).get('path') == 'deploy/kubernetes/platform/cluster',
f'{path}: the platform app syncs deploy/kubernetes/platform/cluster')
else:
check(automated is None, f'{path}: {name} must not auto-sync')
check('resources-finalizer.argocd.argoproj.io' not in doc['metadata'].get('finalizers', []),
f'{path}: {name} must not cascade-delete')
if project.startswith('aevolution-'):
if project.startswith('aevolution-') and project != 'aevolution-platform':
check(destination == 'evolution-' + project[len('aevolution-'):],
f'{path}: {name} leaves its project environment')
check(destination != 'evolution-world0' or name == 'aevolution-world0',
check(destination != 'evolution-world0' or name in ('aevolution-world0', 'aevolution-platform'),
f'{path}: only aevolution-world0 may target evolution-world0')
# M8.3 storage (ADR-0070): durable volumes retain their data; database passwords are
@@ -170,6 +192,47 @@ for path, doc in objects:
check(doc['metadata'].get('namespace') != 'evolution-world0',
f'{path}: store Jobs never target World0')
# The platform bundle: every rendered object is protected from prune and deletion.
bundle = manifests.get(platform + '/cluster/kustomization.yaml', [{}])[0]
check(bundle.get('commonAnnotations', {}).get('argocd.argoproj.io/sync-options') == 'Prune=false,Delete=false',
'the platform bundle must mark every object Prune=false,Delete=false')
check(sorted(bundle.get('resources', [])) == ['../admission', '../dev', '../stage', '../storage', '../world0'],
'the platform bundle must include admission, storage and the three environments')
# Paths a real (non-example) Application syncs must be complete: no REPLACE markers
# anywhere they render, following kustomization resources.
def kustomize_files(directory, seen):
directory = os.path.normpath(directory)
if directory in seen:
return []
seen.add(directory)
found = []
k = os.path.join(directory, 'kustomization.yaml')
if not os.path.isfile(k):
return found
found.append(k)
spec = documents(k)[0]
for entry in spec.get('resources', []) + spec.get('components', []):
target = os.path.normpath(os.path.join(directory, entry))
if os.path.isdir(target):
found += kustomize_files(target, seen)
else:
found.append(target)
return found
for path, doc in objects:
# Examples count too once their path is real (the live dev app is only an example
# here); an example whose path is itself still a marker is not yet deployable.
source_path = doc.get('spec', {}).get('source', {}).get('path', '')
if doc['kind'] == 'Application' and 'REPLACE' not in source_path:
source = os.path.join(ROOT, source_path)
for f in kustomize_files(source, set()):
with open(f, encoding='utf-8') as handle:
check('REPLACE' not in handle.read(),
f'{os.path.relpath(f, ROOT)}: unresolved REPLACE marker in a synced path')
# The live Argo path.
for path, doc in objects:
if path.startswith('deploy/kubernetes/dev/'):