diff --git a/deploy/argocd/platform/application.yaml b/deploy/argocd/platform/application.yaml new file mode 100644 index 0000000..f0377f8 --- /dev/null +++ b/deploy/argocd/platform/application.yaml @@ -0,0 +1,26 @@ +# M8.2/M8.3 platform Application (ADR-0069 amendment). Self-heal restores drift +# (a deleted quota, policy or label comes back); prune is off and every object +# carries Delete=false, so removing a file from Git or deleting this Application +# never deletes a namespace, claim, class or policy. No cascade finalizer. The +# World0 admission policy also denies deleting this Application and its project. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: aevolution-platform + namespace: argocd +spec: + project: aevolution-platform + source: + repoURL: https://git.aeternalisgames.com/dingo/AEvolution.git + targetRevision: main + path: deploy/kubernetes/platform/cluster + destination: + server: https://kubernetes.default.svc + namespace: evolution-dev + syncPolicy: + automated: + prune: false + selfHeal: true + syncOptions: + - FailOnSharedResource=true + - ServerSideApply=true diff --git a/deploy/argocd/platform/project.yaml b/deploy/argocd/platform/project.yaml new file mode 100644 index 0000000..ea23ca6 --- /dev/null +++ b/deploy/argocd/platform/project.yaml @@ -0,0 +1,33 @@ +# M8.2/M8.3 platform project (ADR-0069 amendment). Bootstrap once by hand; Argo CD +# cannot create its own first project: kubectl apply -f deploy/argocd/platform/ +# Only the three AEvolution namespaces, and only the cluster kinds the platform +# layer owns. Workload projects (aevolution-dev/-stage/-world0) stay separate. +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: aevolution-platform + namespace: argocd +spec: + description: AEvolution platform layer (namespaces, guardrails, storage, World0 protection) + sourceRepos: + - https://git.aeternalisgames.com/dingo/AEvolution.git + destinations: + - server: https://kubernetes.default.svc + namespace: evolution-dev + - server: https://kubernetes.default.svc + namespace: evolution-stage + - server: https://kubernetes.default.svc + namespace: evolution-world0 + clusterResourceWhitelist: + - {group: "", kind: Namespace} + - {group: storage.k8s.io, kind: StorageClass} + - {group: admissionregistration.k8s.io, kind: ValidatingAdmissionPolicy} + - {group: admissionregistration.k8s.io, kind: ValidatingAdmissionPolicyBinding} + namespaceResourceWhitelist: + - {group: "", kind: ResourceQuota} + - {group: "", kind: LimitRange} + - {group: "", kind: ServiceAccount} + - {group: "", kind: PersistentVolumeClaim} + - {group: networking.k8s.io, kind: NetworkPolicy} + orphanedResources: + warn: true diff --git a/deploy/kubernetes/platform/admission/kustomization.yaml b/deploy/kubernetes/platform/admission/kustomization.yaml new file mode 100644 index 0000000..cef8c67 --- /dev/null +++ b/deploy/kubernetes/platform/admission/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - world0-protection.yaml diff --git a/deploy/kubernetes/platform/admission/world0-protection.yaml b/deploy/kubernetes/platform/admission/world0-protection.yaml index b9f4c18..74faa52 100644 --- a/deploy/kubernetes/platform/admission/world0-protection.yaml +++ b/deploy/kubernetes/platform/admission/world0-protection.yaml @@ -12,7 +12,8 @@ # - deleting a PersistentVolume bound to evolution-world0, or changing its reclaim # policy away from Retain once set (set Retain when the volume is bound: the # only StorageClass, local-path, provisions with Delete); -# - deleting the aevolution-world0 Argo CD Application or AppProject. +# - deleting the aevolution-world0 or aevolution-platform Argo CD Application or +# AppProject (the platform app owns this policy and the World0 namespace). # Kubernetes does not run admission policies on policy objects themselves, so # removing this policy is itself the audited break-glass step (cluster-admin). # Apply by hand: kubectl apply -f deploy/kubernetes/platform/admission/ @@ -56,7 +57,8 @@ spec: variables.resource in ['persistentvolumeclaims', 'secrets', 'serviceaccounts'] - name: world0Delivery expression: >- - request.resource.group == 'argoproj.io' && request.name == 'aevolution-world0' + request.resource.group == 'argoproj.io' && + request.name in ['aevolution-world0', 'aevolution-platform'] validations: - expression: >- variables.breakGlass || request.operation != 'DELETE' || diff --git a/deploy/kubernetes/platform/cluster/kustomization.yaml b/deploy/kubernetes/platform/cluster/kustomization.yaml new file mode 100644 index 0000000..faed6a2 --- /dev/null +++ b/deploy/kubernetes/platform/cluster/kustomization.yaml @@ -0,0 +1,16 @@ +# M8.2/M8.3 platform layer as one Argo CD Application, aevolution-platform +# (deploy/argocd/platform, ADR-0069 amendment): the World0 admission policy, the +# Retain storage class, and each environment's namespace, guardrails, service +# accounts and object claim. Self-heal restores drift; prune is never enabled and +# every durable object carries Delete=false, so neither a Git removal nor deleting +# the Application removes a namespace, claim, class or policy. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../admission + - ../storage + - ../dev + - ../stage + - ../world0 +commonAnnotations: + argocd.argoproj.io/sync-options: Prune=false,Delete=false diff --git a/deploy/kubernetes/platform/storage/kustomization.yaml b/deploy/kubernetes/platform/storage/kustomization.yaml new file mode 100644 index 0000000..23c2e0c --- /dev/null +++ b/deploy/kubernetes/platform/storage/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - storageclass.yaml diff --git a/deploy/kubernetes/stores/base/database-egress.yaml b/deploy/kubernetes/stores/base/database-egress.yaml new file mode 100644 index 0000000..a9ba859 --- /dev/null +++ b/deploy/kubernetes/stores/base/database-egress.yaml @@ -0,0 +1,23 @@ +# Egress from migrate, store-check and region pods to exactly the AEvolution +# PostgreSQL VM (pg-forge-01) and nothing else (ADR-0070). +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: aevolution-database-egress + labels: + app.kubernetes.io/part-of: aevolution + annotations: + argocd.argoproj.io/sync-wave: "-1" +spec: + podSelector: + matchExpressions: + - key: app.kubernetes.io/component + operator: In + values: [migrate, store-check, region] + policyTypes: [Egress] + egress: + - to: + - ipBlock: + cidr: 172.10.1.127/32 + ports: + - {protocol: TCP, port: 5432} diff --git a/deploy/kubernetes/stores/base/kustomization.yaml b/deploy/kubernetes/stores/base/kustomization.yaml new file mode 100644 index 0000000..2c49b6b --- /dev/null +++ b/deploy/kubernetes/stores/base/kustomization.yaml @@ -0,0 +1,12 @@ +# M8.3 store Jobs as Argo CD hooks (ADR-0070 amendment). Included by an +# environment's workload path through its overlay (stores/dev, stores/stage), which +# pins the image digests and names the environment's NOLOGIN runtime role. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - database-egress.yaml + - migrate-job.yaml + - store-check-job.yaml +labels: + - pairs: + app.kubernetes.io/part-of: aevolution diff --git a/deploy/kubernetes/stores/base/migrate-job.yaml b/deploy/kubernetes/stores/base/migrate-job.yaml new file mode 100644 index 0000000..855fea8 --- /dev/null +++ b/deploy/kubernetes/stores/base/migrate-job.yaml @@ -0,0 +1,60 @@ +# PreSync hook: every sync of the environment first migrates its database with the +# migrator login (version pin and runtime privilege audit included); a failure +# stops the sync before anything else changes. +apiVersion: batch/v1 +kind: Job +metadata: + name: aevolution-db-migrate + labels: + app.kubernetes.io/part-of: aevolution + app.kubernetes.io/component: migrate + annotations: + argocd.argoproj.io/hook: PreSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +spec: + backoffLimit: 0 + activeDeadlineSeconds: 180 + template: + metadata: + labels: + app.kubernetes.io/part-of: aevolution + app.kubernetes.io/component: migrate + spec: + serviceAccountName: aevolution-migrator + automountServiceAccountToken: false + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: {type: RuntimeDefault} + containers: + - name: migrate + image: aevolution-postgres-tools + env: + - {name: PGSERVICE, value: evolution} + - {name: PGSERVICEFILE, value: /run/postgres/pg_service.conf} + - {name: EVO_PGPASS_SOURCE, value: /run/postgres/pgpass} + - {name: EVO_PG_RUNTIME_ROLE, value: set-by-overlay} + resources: + requests: {cpu: 100m, memory: 64Mi} + limits: {memory: 128Mi} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + volumeMounts: + - {name: connection, mountPath: /run/postgres, readOnly: true} + - {name: scratch, mountPath: /tmp} + volumes: + - name: connection + secret: + secretName: evolution-postgres-migrator + defaultMode: 0440 + items: + - {key: pg_service.conf, path: pg_service.conf} + - {key: pgpass, path: pgpass} + - {key: ca.crt, path: ca.crt} + - name: scratch + emptyDir: {medium: Memory, sizeLimit: 8Mi} diff --git a/deploy/kubernetes/stores/base/store-check-job.yaml b/deploy/kubernetes/stores/base/store-check-job.yaml new file mode 100644 index 0000000..48c6395 --- /dev/null +++ b/deploy/kubernetes/stores/base/store-check-job.yaml @@ -0,0 +1,72 @@ +# PostSync hook: qualifies the object volume and the runtime login after each sync +# (evo-world-clone check-store, ADR-0070). Its log is the store evidence. +apiVersion: batch/v1 +kind: Job +metadata: + name: aevolution-store-check + labels: + app.kubernetes.io/part-of: aevolution + app.kubernetes.io/component: store-check + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +spec: + backoffLimit: 0 + activeDeadlineSeconds: 120 + template: + metadata: + labels: + app.kubernetes.io/part-of: aevolution + app.kubernetes.io/component: store-check + spec: + serviceAccountName: aevolution-region + automountServiceAccountToken: false + restartPolicy: Never + nodeSelector: + aevolution.io/stores: "true" + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: {type: RuntimeDefault} + containers: + - name: check-store + image: aevolution-server + # libpq ignores a group-readable password file, and Secret volumes are + # group-readable for a non-root fsGroup, so copy it to private memory first. + # w1/r1 is a reserved probe namespace, never a real world. + command: ["/bin/bash", "-c"] + args: + - | + set -eu + umask 077 + cp /run/postgres/pgpass /tmp/pgpass + exec /usr/local/bin/evo-world-clone check-store --world 1 --region 1 \ + --objects /objects --pg-service /run/postgres/pg_service.conf \ + --pg-passfile /tmp/pgpass + resources: + requests: {cpu: 50m, memory: 64Mi} + limits: {memory: 256Mi} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + volumeMounts: + - {name: objects, mountPath: /objects} + - {name: connection, mountPath: /run/postgres, readOnly: true} + - {name: scratch, mountPath: /tmp} + volumes: + - name: objects + persistentVolumeClaim: + claimName: aevolution-objects + - name: scratch + emptyDir: {medium: Memory, sizeLimit: 1Mi} + - name: connection + secret: + secretName: evolution-postgres-runtime + defaultMode: 0440 + items: + - {key: pg_service.conf, path: pg_service.conf} + - {key: pgpass, path: pgpass} + - {key: ca.crt, path: ca.crt} diff --git a/deploy/kubernetes/stores/dev/kustomization.yaml b/deploy/kubernetes/stores/dev/kustomization.yaml new file mode 100644 index 0000000..e609c6a --- /dev/null +++ b/deploy/kubernetes/stores/dev/kustomization.yaml @@ -0,0 +1,23 @@ +# Store Jobs for evolution-dev. Values still to supply before this is included in +# the dev workload path: the published registry digests (from the CI Publish +# step's "Digest:" line) and the dev database's NOLOGIN runtime group role. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: evolution-dev +resources: + - ../base +images: + - name: aevolution-postgres-tools + newName: registry.aeternalisgames.com/aeternalis/aevolution-postgres-tools + digest: sha256:REPLACE_PUBLISHED_TOOLS_DIGEST + - name: aevolution-server + newName: registry.aeternalisgames.com/aeternalis/aevolution-server + digest: sha256:REPLACE_PUBLISHED_SERVER_DIGEST +patches: + - target: + kind: Job + name: aevolution-db-migrate + patch: |- + - op: replace + path: /spec/template/spec/containers/0/env/3/value + value: REPLACE_DEV_RUNTIME_GROUP_ROLE diff --git a/deploy/kubernetes/stores/stage/kustomization.yaml b/deploy/kubernetes/stores/stage/kustomization.yaml new file mode 100644 index 0000000..3fbcfdb --- /dev/null +++ b/deploy/kubernetes/stores/stage/kustomization.yaml @@ -0,0 +1,23 @@ +# Store Jobs for evolution-stage. Values still to supply before this is included in +# the stage workload path: the published registry digests (from the CI Publish +# step's "Digest:" line) and the stage database's NOLOGIN runtime group role. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: evolution-stage +resources: + - ../base +images: + - name: aevolution-postgres-tools + newName: registry.aeternalisgames.com/aeternalis/aevolution-postgres-tools + digest: sha256:REPLACE_PUBLISHED_TOOLS_DIGEST + - name: aevolution-server + newName: registry.aeternalisgames.com/aeternalis/aevolution-server + digest: sha256:REPLACE_PUBLISHED_SERVER_DIGEST +patches: + - target: + kind: Job + name: aevolution-db-migrate + patch: |- + - op: replace + path: /spec/template/spec/containers/0/env/3/value + value: REPLACE_STAGE_RUNTIME_GROUP_ROLE diff --git a/deploy/kubernetes/stores/stores.yaml.example b/deploy/kubernetes/stores/stores.yaml.example deleted file mode 100644 index d971968..0000000 --- a/deploy/kubernetes/stores/stores.yaml.example +++ /dev/null @@ -1,163 +0,0 @@ -# M8.3 store Jobs for ONE disposable environment (dev or stage). Not an Argo path. -# Copy, then replace every REPLACE_* marker (the markers make this file unappliable -# as committed), and apply by hand: -# REPLACE_NAMESPACE evolution-dev | evolution-stage (never evolution-world0) -# REPLACE_DB_CIDR the database VM's address as a /32 (PGHOSTADDR) -# REPLACE_DB_PORT its port (PGPORT) -# REPLACE_RUNTIME_ROLE that environment's NOLOGIN runtime group role -# REPLACE_TOOLS_DIGEST aevolution-postgres-tools image digest (sha256:...) -# REPLACE_SERVER_DIGEST aevolution server-runtime image digest (sha256:...) -# Secret contract (docs/infrastructure/ENVIRONMENTS.md, never in Git). Each login -# Secret is self-contained, with keys pg_service.conf (section [evolution], with -# sslrootcert=/run/postgres/ca.crt), pgpass and ca.crt: -# evolution-postgres-migrator the environment's migrator login (migrate-postgres.sh) -# evolution-postgres-runtime the environment's runtime login (check-store, region) -# Order: apply the NetworkPolicy, run migrate, then check-store (twice: the second -# run must report "probe verified", proving the volume survived a new pod). -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: aevolution-database-egress - namespace: REPLACE_NAMESPACE - labels: - app.kubernetes.io/part-of: aevolution -spec: - podSelector: - matchExpressions: - - key: app.kubernetes.io/component - operator: In - values: [migrate, store-check, region] - policyTypes: [Egress] - egress: - - to: - - ipBlock: - cidr: REPLACE_DB_CIDR - ports: - - {protocol: TCP, port: REPLACE_DB_PORT} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: aevolution-db-migrate - namespace: REPLACE_NAMESPACE - labels: - app.kubernetes.io/part-of: aevolution - app.kubernetes.io/component: migrate -spec: - backoffLimit: 0 - activeDeadlineSeconds: 180 - ttlSecondsAfterFinished: 86400 - template: - metadata: - labels: - app.kubernetes.io/part-of: aevolution - app.kubernetes.io/component: migrate - spec: - serviceAccountName: aevolution-migrator - automountServiceAccountToken: false - restartPolicy: Never - securityContext: - runAsNonRoot: true - runAsUser: 10001 - runAsGroup: 10001 - fsGroup: 10001 - seccompProfile: {type: RuntimeDefault} - containers: - - name: migrate - image: registry.aeternalisgames.com/aeternalis/aevolution-postgres-tools@REPLACE_TOOLS_DIGEST - env: - - {name: PGSERVICE, value: evolution} - - {name: PGSERVICEFILE, value: /run/postgres/pg_service.conf} - - {name: EVO_PGPASS_SOURCE, value: /run/postgres/pgpass} - - {name: EVO_PG_RUNTIME_ROLE, value: REPLACE_RUNTIME_ROLE} - resources: - requests: {cpu: 100m, memory: 64Mi} - limits: {memory: 128Mi} - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: {drop: [ALL]} - volumeMounts: - - {name: connection, mountPath: /run/postgres, readOnly: true} - - {name: scratch, mountPath: /tmp} - volumes: - - name: connection - secret: - secretName: evolution-postgres-migrator - defaultMode: 0440 - items: - - {key: pg_service.conf, path: pg_service.conf} - - {key: pgpass, path: pgpass} - - {key: ca.crt, path: ca.crt} - - name: scratch - emptyDir: {medium: Memory, sizeLimit: 8Mi} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: aevolution-store-check - namespace: REPLACE_NAMESPACE - labels: - app.kubernetes.io/part-of: aevolution - app.kubernetes.io/component: store-check -spec: - backoffLimit: 0 - activeDeadlineSeconds: 120 - ttlSecondsAfterFinished: 86400 - template: - metadata: - labels: - app.kubernetes.io/part-of: aevolution - app.kubernetes.io/component: store-check - spec: - serviceAccountName: aevolution-region - automountServiceAccountToken: false - restartPolicy: Never - nodeSelector: - aevolution.io/stores: "true" - securityContext: - runAsNonRoot: true - runAsUser: 10001 - runAsGroup: 10001 - fsGroup: 10001 - seccompProfile: {type: RuntimeDefault} - containers: - - name: check-store - image: registry.aeternalisgames.com/aeternalis/aevolution-server@REPLACE_SERVER_DIGEST - # libpq ignores a group-readable password file, and Secret volumes are - # group-readable for a non-root fsGroup, so copy it to private memory first. - # w1/r1 is a reserved probe namespace, never a real world. - command: ["/bin/bash", "-c"] - args: - - | - set -eu - umask 077 - cp /run/postgres/pgpass /tmp/pgpass - exec /usr/local/bin/evo-world-clone check-store --world 1 --region 1 \ - --objects /objects --pg-service /run/postgres/pg_service.conf \ - --pg-passfile /tmp/pgpass - resources: - requests: {cpu: 50m, memory: 64Mi} - limits: {memory: 256Mi} - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: {drop: [ALL]} - volumeMounts: - - {name: objects, mountPath: /objects} - - {name: connection, mountPath: /run/postgres, readOnly: true} - - {name: scratch, mountPath: /tmp} - volumes: - - name: objects - persistentVolumeClaim: - claimName: aevolution-objects - - name: scratch - emptyDir: {medium: Memory, sizeLimit: 1Mi} - - name: connection - secret: - secretName: evolution-postgres-runtime - defaultMode: 0440 - items: - - {key: pg_service.conf, path: pg_service.conf} - - {key: pgpass, path: pgpass} - - {key: ca.crt, path: ca.crt} diff --git a/docs/BACKLOG.md b/docs/BACKLOG.md index 2a4e73c..7f528ed 100644 --- a/docs/BACKLOG.md +++ b/docs/BACKLOG.md @@ -8,6 +8,8 @@ names it. Remove an item only when it is done, and record where it was done. |---|---|---|---| | **World0 break-glass membership.** Decide who may act as `aevolution:world0-break-glass`, and how that group is granted: for example a short-lived client certificate with that organization, or an OIDC group. Decide also how its use is approved and recorded. The admission policy already enforces the group (ADR-0069). | M8.2, 2026-09-29 | Maintainer: figure out later | Any intentional deletion of World0 data or identity; M8.8 World0 launch | | **Backup destination for dev/stage stores.** There are no backups yet (maintainer, 2026-09-29). The plan is to build backup tooling for the AEvolution PostgreSQL VM and each `aevolution-objects` volume up to a pluggable destination, then add the destination. It must be off the host for anything beyond disposable environments. | M8.3, 2026-09-29 | The maintainer will supply a destination later | M8.7 backup and restore; any World0 data | +| **Secret manager for Argo-delivered configuration.** Secrets are still created by hand (ADR-0069 amendment). Choose how they reach the cluster through Git or Argo: for example Vault with the Agent Injector (as in the maintainer's other environment), or sealed or external secrets. Until then, rotate by hand. | M8.3, 2026-09-29 | The maintainer wants everything through Argo; a secret manager is not chosen | Stage and World0 credentials; M8.4 | +| **Store Job values.** Supply the published registry digests for `aevolution-postgres-tools` and `aevolution-server` (from each CI Publish step's `Digest:` line), and the NOLOGIN runtime group role for dev (`aevolution`) and for stage (`aevolution_stage`). Then include `stores/` in each environment's path. | M8.3, 2026-09-29 | Values pending | M8.3 in-cluster evidence | | **Storage after the RAID/disk rework.** Re-run `deploy/kubernetes/inventory/disk-probe-local-path.yaml` (or a new class), and update INVENTORY.md. | M8.1, 2026-09-29 | Maintainer is reorganizing disks | M8.3's final storage decision | ## Accepted risks (maintainer decisions) diff --git a/docs/STATUS.md b/docs/STATUS.md index 1c4e050..a72f674 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1036,3 +1036,11 @@ Not yet done: - the in-cluster run (it needs Secrets with the documented key names and published image digests); - choosing a backup destination (docs/BACKLOG.md). + +M8.3 delivery moved to Argo CD, at the maintainer's request: +- **Platform app:** `aevolution-platform` (self-heal, never prune, `Delete=false`, + protected by admission) owns the namespaces, guardrails, storage class, object + claims and World0 policy. +- **Store Jobs:** they become sync hooks in each environment's path once their + digests and runtime roles are supplied. +- **Outside Argo:** Secrets and the store node label. diff --git a/docs/adr/0069-environment-isolation.md b/docs/adr/0069-environment-isolation.md index da916eb..e4dbf6b 100644 --- a/docs/adr/0069-environment-isolation.md +++ b/docs/adr/0069-environment-isolation.md @@ -104,3 +104,29 @@ never touches platform objects or Secrets, and dry-runs unless `--execute` is gi or cluster-admins. - **Namespaces created by Argo `CreateNamespace`.** A prune or cascade could then remove them. + +## Amendment: platform delivered by Argo CD (2026-09-29) +The maintainer prefers that every deployment and configuration goes through Argo +CD. The platform layer is therefore an Argo Application, `aevolution-platform` +(`deploy/argocd/platform`), no longer a hand-applied layer. The original concern, +that a sync, prune or cascade could delete namespaces and durable objects, is met +differently: +- **Never prune.** The Application self-heals but never prunes. +- **Protected objects.** Every object in the bundle carries + `argocd.argoproj.io/sync-options: Prune=false,Delete=false`, so neither a Git + removal nor deleting the Application deletes it. +- **No cascade.** There is no cascade finalizer. +- **Protected by admission.** The World0 admission policy also denies deleting the + `aevolution-platform` Application and AppProject. +- **Scoped project.** Its project may target only the three AEvolution namespaces, + and only four cluster kinds: Namespace, StorageClass, ValidatingAdmissionPolicy + and its binding. + +Self-heal adds a protection the hand-applied layer lacked: drift, such as a +deleted quota or label, is restored. The admission policy itself is still not +subject to admission, so a policy change is a reviewed Git change. + +What stays outside Argo: +- Secrets, until a secret manager is chosen (docs/BACKLOG.md); +- the store node label; +- the one-time bootstrap of the platform project and Application. diff --git a/docs/adr/0070-dev-stage-stores.md b/docs/adr/0070-dev-stage-stores.md index 7a05465..cb6b466 100644 --- a/docs/adr/0070-dev-stage-stores.md +++ b/docs/adr/0070-dev-stage-stores.md @@ -100,3 +100,21 @@ policy check forbids `PGPASSWORD`/`PGUSER` references and whole-Secret `envFrom` - **Leaving `local-path`/Delete for durable data.** Deleting the claim or an Argo prune would erase the store. - **Trusting PVC sizes.** `local-path` does not enforce them. + +## Amendment: Argo delivery and the real Secret layout (2026-09-29) +- **Secret layout.** The maintainer's login Secrets have the keys + `pg_service.conf` (section `[evolution]`), `pgpass` and `ca.crt`. `evo-server` + and `evo-world-clone` read the service file with `--pg-service`, a strict parser + that refuses passwords, other options and weaker TLS. The Jobs mount each login + Secret whole, and `aevolution-db` is not used. +- **Argo delivery.** The storage class and object claims are delivered by the + `aevolution-platform` Application (ADR-0069 amendment). The store Jobs are + kustomize components (`deploy/kubernetes/stores`) that an environment's workload + path includes as Argo hooks: + - the migration runs PreSync, so a failure stops the sync; + - `check-store` runs PostSync. +- **Values.** Each environment overlay pins the published image digests and its + runtime group role. A synced path may not contain placeholders; the policy check + enforces this. +- **Stage database.** Stage uses its own database, `aevolution_stage`, on the same + VM, with its own migrator login and runtime group. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 63b248d..c7c38d4 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -2757,3 +2757,27 @@ admission policy and the stage and World0 AppProjects, then ran file. - PostgreSQL Release and ASan/UBSan passed. The analysis build passed after one loop copy was fixed. Deploy policy: 0 failures. + +## M8.3 Argo delivery - 2026-09-29 + +- The platform layer is now `deploy/kubernetes/platform/cluster`, delivered by the + `aevolution-platform` Application. It renders 29 objects, all marked + `Prune=false,Delete=false`: 3 namespaces, 3 quotas, 3 limit ranges, 6 network + policies, 9 service accounts, 2 claims, the storage class, the policy and its + binding. +- The store Jobs are kustomize components used as Argo hooks. The dev overlay + renders into `evolution-dev` with PreSync and PostSync hooks; its digests and + role are still markers. +- `deploy-policy-check.py`: 43 objects in 41 files, 0 failures. Five new mutations + were each caught: + - platform prune enabled; + - a missing Delete guard; + - an unfinished stores overlay included in the live dev path; + - an extra cluster kind in the platform project; + - the admission policy no longer covering the platform app. + The unfinished-path rule first missed the live dev Application, which exists + here only as an `.example`. It now covers every Application whose path is + resolved. +- The digests the maintainer pasted are local image IDs from the build step, not + registry digests. The registry digest is printed by the Publish step. +- Not executed: bootstrapping the platform Application, or any Argo sync. diff --git a/docs/infrastructure/ENVIRONMENTS.md b/docs/infrastructure/ENVIRONMENTS.md index f57532a..10d7f75 100644 --- a/docs/infrastructure/ENVIRONMENTS.md +++ b/docs/infrastructure/ENVIRONMENTS.md @@ -47,68 +47,74 @@ A World0 credential must never exist in the dev or stage namespaces, and no dev stage credential may reach the World0 database. Rotate by replacing the Secret and restarting the consumer. -## Apply (cluster administrator) +## Delivery through Argo CD (maintainer preference, 2026-09-29) +Everything except Secrets and one node label is delivered from Git by Argo CD. + +**Bootstrap once.** Argo CD cannot create its own first project and application: ```sh -# 1. Platform guardrails, per environment (dev adds labels, quota and policies to -# the existing namespace; its completed proof Job already complies). -kubectl apply -k deploy/kubernetes/platform/dev -kubectl apply -k deploy/kubernetes/platform/stage -kubectl apply -k deploy/kubernetes/platform/world0 # an empty, protected namespace -# 2. World0 admission protection (validate first, then apply). -kubectl apply --dry-run=server -f deploy/kubernetes/platform/admission/ -kubectl apply -f deploy/kubernetes/platform/admission/ -# 3. Projects (resolve the Application paths only when M8.4 adds workloads). -kubectl apply -f deploy/argocd/stage/project.yaml.example -kubectl apply -f deploy/argocd/world0/project.yaml.example -# 4. Verify: reads, can-i and server dry runs only. -bash tools/ops/verify-isolation.sh +kubectl apply -f deploy/argocd/platform/project.yaml -f deploy/argocd/platform/application.yaml ``` +`aevolution-platform` then syncs `deploy/kubernetes/platform/cluster`: +- the World0 admission policy; +- the `aevolution-local-retain` storage class; +- each environment's namespace, quota, limit range, default-deny policies, + service accounts and object claim (dev and stage). -Create the Secrets listed above in each namespace before any workload syncs. +It self-heals, so a deleted quota, policy or label comes back. It never prunes, +and every object carries `Prune=false,Delete=false`, so removing a file or +deleting the Application never deletes a namespace, claim, class or policy. The +admission policy also denies deleting the platform Application or project. Objects +applied by hand earlier are adopted (server-side apply). + +**Workloads.** Each environment's Application syncs its path, currently manually: +- `aevolution-dev` syncs `deploy/kubernetes/dev`, which is live. +- Stage uses `deploy/argocd/stage`, whose path is resolved when its values are + supplied. + +The store Jobs (`deploy/kubernetes/stores/`) are included in the environment +path as sync hooks: +- the database migration runs as a PreSync hook, so a failed migration stops the + sync; +- `check-store` runs as a PostSync hook. + +An environment's stores overlay is included only once it has its published image +digests and its runtime group role. The policy check refuses any `REPLACE_` marker +in a synced path. + +**Outside Argo (by design, for now).** +- Secrets are created by an operator; a secret manager is in docs/BACKLOG.md. +- One node label: `kubectl label node rke2-worker-1 aevolution.io/stores=true`. + The maintainer allows any worker; worker-1 is chosen because it had the lowest + measured latency, and worker-3's disk is 87 % full. + +**Verify:** `bash tools/ops/verify-isolation.sh` makes only reads, can-i queries and +server dry runs. ## Stores (M8.3, ADR-0070) -Run this once per disposable environment (`dev`, then `stage`). - -**1. Label the store node, and apply the storage class and claim.** -```sh -# Label exactly one worker. The maintainer allows any worker (never the control plane); -# worker-1 is chosen (lowest measured latency; worker-3's disk is 87 % full). -kubectl label node rke2-worker-1 aevolution.io/stores=true -kubectl apply -f deploy/kubernetes/platform/storage/storageclass.yaml -kubectl apply -k deploy/kubernetes/platform/dev # adds the aevolution-objects claim -``` - -**2. Prepare the store Jobs.** Copy `deploy/kubernetes/stores/stores.yaml.example`, -replace its `REPLACE_*` markers (namespace, database address and port, runtime -role, image digests), and create the Secrets above. - -**3. Run and qualify.** -```sh -kubectl apply -f stores-dev.yaml # policy + both Jobs -kubectl logs -n evolution-dev job/aevolution-db-migrate # schema and audit -kubectl logs -n evolution-dev job/aevolution-store-check # expect exit 0 and "probe created" -``` - -Evidence to record in docs/engineering/VALIDATION.md: -- **Pod restart:** delete the check Job and apply it again. It must print - `probe verified`, with the same digest. -- **Missing volume:** run the check with the `objects` volume removed. It must exit - 4 with `objects refused missing-or-not-a-directory`. -- **Full volume:** add `--reserve-mib 1048576` and use a new probe region, such as - `--region 2`. It must exit 4 with `probe refused full`. -- **Retention:** `kubectl delete pvc aevolution-objects -n evolution-dev`. The PV - must become `Released`, not deleted. To rebind: - - clear the PV's `spec.claimRef`; - - recreate the claim with `volumeName` set to that PV; - - rerun the check, which must print `probe verified`. -- **Credential rotation:** - - Change the runtime login's password on the database VM, and run the check - before updating the Secret. It must exit 3 with `database refused connect`. - - Update the Secret's `pgpass` and rerun. It must exit 0. -- **Privilege boundary:** every run must print +Once `deploy/kubernetes/stores/dev` has its values and is included in +`deploy/kubernetes/dev`, a manual sync of `aevolution-dev` runs the migration and +then the store check. The hook Jobs keep their logs until the next sync. Record +this evidence in docs/engineering/VALIDATION.md: +- **Pod restart:** sync again. The check must print `probe verified` with the same + digest. +- **Privilege boundary:** every run prints `database tables=denied functions=reachable`. +- **Missing volume:** temporarily remove the `objects` volume in a branch, or run the + image by hand without it. The check must exit 4 with + `objects refused missing-or-not-a-directory`. +- **Full volume:** add `--reserve-mib 1048576` and a new probe region, such as + `--region 2`. The check must exit 4 with `probe refused full`. +- **Retention:** delete the claim by hand, since Argo never prunes it. The PV must + become `Released`, not deleted. To rebind: + - clear the PV's `spec.claimRef`; + - let the platform app recreate the claim, with `volumeName` pinned in Git; + - sync, and check that it prints `probe verified`. +- **Credential rotation:** + - Change the runtime login's password on the database VM and sync before updating + the Secret. The check must exit 3 with `database refused connect`. + - Update the Secret's `pgpass` and sync again. It must exit 0. ## Break-glass diff --git a/tools/ci/deploy-policy-check.py b/tools/ci/deploy-policy-check.py index 995cc1e..f3c873a 100644 --- a/tools/ci/deploy-policy-check.py +++ b/tools/ci/deploy-policy-check.py @@ -111,13 +111,27 @@ if policy and binding: check((resource, 'UPDATE') in covered, f'world0 policy must cover UPDATE {resource}') text = yaml.safe_dump(spec) for needle in ("aevolution:world0-break-glass", "evolution-world0", "aevolution-world0", + "aevolution-platform", "aevolution.io/protected", "Retain"): check(needle in text, f'world0 policy must reference {needle}') # Argo CD projects and applications. for path, doc in objects: name = doc['metadata'].get('name', '') - if doc['kind'] == 'AppProject' and name.startswith('aevolution-'): + if doc['kind'] == 'AppProject' and name == 'aevolution-platform': + spec = doc['spec'] + namespaces = sorted(d.get('namespace') for d in spec.get('destinations', [])) + check(namespaces == ['evolution-dev', 'evolution-stage', 'evolution-world0'], + f'{path}: the platform project targets exactly the three AEvolution namespaces') + cluster = sorted((w.get('group'), w.get('kind')) for w in spec.get('clusterResourceWhitelist', [])) + check(cluster == [('', 'Namespace'), + ('admissionregistration.k8s.io', 'ValidatingAdmissionPolicy'), + ('admissionregistration.k8s.io', 'ValidatingAdmissionPolicyBinding'), + ('storage.k8s.io', 'StorageClass')], + f'{path}: the platform project may own only its four cluster kinds') + kinds = {w.get('kind') for w in spec.get('namespaceResourceWhitelist', [])} + check(not kinds & {'Secret', 'Role', 'RoleBinding', '*'}, f'{path}: platform whitelists a forbidden kind') + elif doc['kind'] == 'AppProject' and name.startswith('aevolution-'): env = name[len('aevolution-'):] spec = doc['spec'] namespaces = [d.get('namespace') for d in spec.get('destinations', [])] @@ -136,13 +150,21 @@ for path, doc in objects: spec = doc['spec'] project = spec.get('project', '') destination = spec.get('destination', {}).get('namespace', '') - check('automated' not in (spec.get('syncPolicy') or {}), f'{path}: {name} must not auto-sync') + automated = (spec.get('syncPolicy') or {}).get('automated') + if name == 'aevolution-platform': + # Self-heal only: never prune, so no Git change can delete durable objects. + check(automated is not None and automated.get('prune') is False, + f'{path}: the platform app self-heals and never prunes') + check(spec.get('source', {}).get('path') == 'deploy/kubernetes/platform/cluster', + f'{path}: the platform app syncs deploy/kubernetes/platform/cluster') + else: + check(automated is None, f'{path}: {name} must not auto-sync') check('resources-finalizer.argocd.argoproj.io' not in doc['metadata'].get('finalizers', []), f'{path}: {name} must not cascade-delete') - if project.startswith('aevolution-'): + if project.startswith('aevolution-') and project != 'aevolution-platform': check(destination == 'evolution-' + project[len('aevolution-'):], f'{path}: {name} leaves its project environment') - check(destination != 'evolution-world0' or name == 'aevolution-world0', + check(destination != 'evolution-world0' or name in ('aevolution-world0', 'aevolution-platform'), f'{path}: only aevolution-world0 may target evolution-world0') # M8.3 storage (ADR-0070): durable volumes retain their data; database passwords are @@ -170,6 +192,47 @@ for path, doc in objects: check(doc['metadata'].get('namespace') != 'evolution-world0', f'{path}: store Jobs never target World0') +# The platform bundle: every rendered object is protected from prune and deletion. +bundle = manifests.get(platform + '/cluster/kustomization.yaml', [{}])[0] +check(bundle.get('commonAnnotations', {}).get('argocd.argoproj.io/sync-options') == 'Prune=false,Delete=false', + 'the platform bundle must mark every object Prune=false,Delete=false') +check(sorted(bundle.get('resources', [])) == ['../admission', '../dev', '../stage', '../storage', '../world0'], + 'the platform bundle must include admission, storage and the three environments') + + +# Paths a real (non-example) Application syncs must be complete: no REPLACE markers +# anywhere they render, following kustomization resources. +def kustomize_files(directory, seen): + directory = os.path.normpath(directory) + if directory in seen: + return [] + seen.add(directory) + found = [] + k = os.path.join(directory, 'kustomization.yaml') + if not os.path.isfile(k): + return found + found.append(k) + spec = documents(k)[0] + for entry in spec.get('resources', []) + spec.get('components', []): + target = os.path.normpath(os.path.join(directory, entry)) + if os.path.isdir(target): + found += kustomize_files(target, seen) + else: + found.append(target) + return found + + +for path, doc in objects: + # Examples count too once their path is real (the live dev app is only an example + # here); an example whose path is itself still a marker is not yet deployable. + source_path = doc.get('spec', {}).get('source', {}).get('path', '') + if doc['kind'] == 'Application' and 'REPLACE' not in source_path: + source = os.path.join(ROOT, source_path) + for f in kustomize_files(source, set()): + with open(f, encoding='utf-8') as handle: + check('REPLACE' not in handle.read(), + f'{os.path.relpath(f, ROOT)}: unresolved REPLACE marker in a synced path') + # The live Argo path. for path, doc in objects: if path.startswith('deploy/kubernetes/dev/'):