Add M8.3 dev/stage stores: Retain volumes, free-space reserve, store check

Keep the filesystem object store on a node-local Retain-class PVC per
environment (ADR-0070), with the database on the external PostgreSQL VM.
Because local-path does not enforce sizes, the object store now refuses
new objects below a free-space reserve with a distinct full error, so a
full volume publishes and acknowledges nothing. evo-world-clone gains
check-store to qualify volume retention, reserve, database version/TLS
and the runtime privilege boundary from inside the cluster; it now ships
in the server image. Add the storage class, object claims, hand-applied
store Jobs, policy rules forbidding password env vars, and the runbook.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-29 13:51:15 -04:00
co-authored by Claude Opus 5.5
parent 9d9e00c307
commit 2d950060db
23 changed files with 833 additions and 24 deletions
+2
View File
@@ -100,6 +100,8 @@ ENTRYPOINT ["bash", "/opt/evo/tools/migrate-postgres.sh"]
FROM ${UBUNTU_IMAGE} AS server-runtime
RUN apt-get update && apt-get install -y --no-install-recommends libstdc++6 libgcc-s1 libssl3t64 zlib1g ca-certificates && rm -rf /var/lib/apt/lists/* && groupadd --gid 10001 evo && useradd --uid 10001 --gid 10001 --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin evo
COPY --from=postgres-build /source/build/postgres/evo-server /usr/local/bin/evo-server
# M8.3 operations tool: check-store qualification, clone and history audit.
COPY --from=postgres-build /source/build/postgres/evo-world-clone /usr/local/bin/evo-world-clone
COPY --from=postgres-build /source/.cache/postgres/install/lib/libpq.so* /opt/postgres/lib/
COPY --from=postgres-build /source/.cache/postgres/postgresql-18.6/COPYRIGHT /usr/share/doc/evo-postgres/COPYRIGHT
ENV LD_LIBRARY_PATH=/opt/postgres/lib
@@ -4,6 +4,7 @@ namespace: evolution-dev
resources:
- namespace.yaml
- ../base
- objects-pvc.yaml
labels:
- pairs:
aevolution.io/environment: dev
@@ -0,0 +1,15 @@
# M8.3 region object store volume (ADR-0070): durable data, so it belongs to the
# platform layer (applied by hand), never to an Argo Application. The volume is
# created on the node of its first consumer; every consumer selects
# aevolution.io/stores=true, so label exactly one worker first (ENVIRONMENTS.md).
# Retain: deleting this claim leaves the volume and its data Released, not deleted.
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: aevolution-objects
spec:
accessModes: [ReadWriteOnce]
storageClassName: aevolution-local-retain
resources:
requests:
storage: 10Gi
@@ -4,6 +4,7 @@ namespace: evolution-stage
resources:
- namespace.yaml
- ../base
- objects-pvc.yaml
labels:
- pairs:
aevolution.io/environment: stage
@@ -0,0 +1,15 @@
# M8.3 region object store volume (ADR-0070): durable data, so it belongs to the
# platform layer (applied by hand), never to an Argo Application. The volume is
# created on the node of its first consumer; every consumer selects
# aevolution.io/stores=true, so label exactly one worker first (ENVIRONMENTS.md).
# Retain: deleting this claim leaves the volume and its data Released, not deleted.
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: aevolution-objects
spec:
accessModes: [ReadWriteOnce]
storageClassName: aevolution-local-retain
resources:
requests:
storage: 10Gi
@@ -0,0 +1,17 @@
# M8.3 (ADR-0070). Cluster-scoped; a cluster administrator applies it by hand:
# kubectl apply -f deploy/kubernetes/platform/storage/storageclass.yaml
# Same provisioner as the cluster's only class (rancher.io/local-path: a directory on
# the node that first runs the consumer), but deleting a claim keeps the volume and
# its data (Released) instead of deleting it. Volumes are node-local: no replication,
# no failover, and the requested size is not enforced (the object store's free-space
# reserve and M8.5 alerts cover that). Never the default class.
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: aevolution-local-retain
labels:
app.kubernetes.io/part-of: aevolution
provisioner: rancher.io/local-path
reclaimPolicy: Retain
volumeBindingMode: WaitForFirstConsumer
allowVolumeExpansion: false
@@ -0,0 +1,174 @@
# M8.3 store Jobs for ONE disposable environment (dev or stage). Not an Argo path.
# Copy, then replace every REPLACE_* marker (the markers make this file unappliable
# as committed), and apply by hand:
# REPLACE_NAMESPACE evolution-dev | evolution-stage (never evolution-world0)
# REPLACE_DB_CIDR the database VM's address as a /32 (PGHOSTADDR)
# REPLACE_DB_PORT its port (PGPORT)
# REPLACE_RUNTIME_ROLE that environment's NOLOGIN runtime group role
# REPLACE_TOOLS_DIGEST aevolution-postgres-tools image digest (sha256:...)
# REPLACE_SERVER_DIGEST aevolution server-runtime image digest (sha256:...)
# Secret contract (docs/infrastructure/ENVIRONMENTS.md, never in Git):
# aevolution-db: PGHOST, PGHOSTADDR, PGPORT, PGDATABASE (only these
# keys are read; PGPASSWORD/PGUSER are never mounted)
# evolution-postgres-migrator: pg_service.conf, pgpass, ca.crt (migrate-postgres.sh)
# evolution-postgres-runtime: user, pgpass, ca.crt
# Order: apply the NetworkPolicy, run migrate, then check-store (twice: the second
# run must report "probe verified", proving the volume survived a new pod).
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: aevolution-database-egress
namespace: REPLACE_NAMESPACE
labels:
app.kubernetes.io/part-of: aevolution
spec:
podSelector:
matchExpressions:
- key: app.kubernetes.io/component
operator: In
values: [migrate, store-check, region]
policyTypes: [Egress]
egress:
- to:
- ipBlock:
cidr: REPLACE_DB_CIDR
ports:
- {protocol: TCP, port: REPLACE_DB_PORT}
---
apiVersion: batch/v1
kind: Job
metadata:
name: aevolution-db-migrate
namespace: REPLACE_NAMESPACE
labels:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: migrate
spec:
backoffLimit: 0
activeDeadlineSeconds: 180
ttlSecondsAfterFinished: 86400
template:
metadata:
labels:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: migrate
spec:
serviceAccountName: aevolution-migrator
automountServiceAccountToken: false
restartPolicy: Never
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile: {type: RuntimeDefault}
containers:
- name: migrate
image: registry.aeternalisgames.com/aeternalis/aevolution-postgres-tools@REPLACE_TOOLS_DIGEST
env:
- {name: PGSERVICE, value: evolution-migrator}
- {name: PGSERVICEFILE, value: /run/postgres/pg_service.conf}
- {name: EVO_PGPASS_SOURCE, value: /run/postgres/pgpass}
- {name: EVO_PG_RUNTIME_ROLE, value: REPLACE_RUNTIME_ROLE}
resources:
requests: {cpu: 100m, memory: 64Mi}
limits: {memory: 128Mi}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
volumeMounts:
- {name: connection, mountPath: /run/postgres, readOnly: true}
- {name: scratch, mountPath: /tmp}
volumes:
- name: connection
secret:
secretName: evolution-postgres-migrator
defaultMode: 0440
items:
- {key: pg_service.conf, path: pg_service.conf}
- {key: pgpass, path: pgpass}
- {key: ca.crt, path: ca.crt}
- name: scratch
emptyDir: {medium: Memory, sizeLimit: 8Mi}
---
apiVersion: batch/v1
kind: Job
metadata:
name: aevolution-store-check
namespace: REPLACE_NAMESPACE
labels:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: store-check
spec:
backoffLimit: 0
activeDeadlineSeconds: 120
ttlSecondsAfterFinished: 86400
template:
metadata:
labels:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: store-check
spec:
serviceAccountName: aevolution-region
automountServiceAccountToken: false
restartPolicy: Never
nodeSelector:
aevolution.io/stores: "true"
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile: {type: RuntimeDefault}
containers:
- name: check-store
image: registry.aeternalisgames.com/aeternalis/aevolution-server@REPLACE_SERVER_DIGEST
# libpq ignores a group-readable password file, and Secret volumes are
# group-readable for a non-root fsGroup, so copy it to private memory first.
# w1/r1 is a reserved probe namespace, never a real world.
command: ["/bin/bash", "-c"]
args:
- |
set -eu
umask 077
cp /run/postgres/pgpass /tmp/pgpass
exec /usr/local/bin/evo-world-clone check-store --world 1 --region 1 \
--objects /objects --pg-host "$DB_HOST" --pg-hostaddr "$DB_HOSTADDR" \
--pg-port "$DB_PORT" --pg-database "$DB_NAME" --pg-user "$DB_USER" \
--pg-passfile /tmp/pgpass --pg-ca /run/postgres/ca.crt
env:
- name: DB_HOST
valueFrom: {secretKeyRef: {name: aevolution-db, key: PGHOST}}
- name: DB_HOSTADDR
valueFrom: {secretKeyRef: {name: aevolution-db, key: PGHOSTADDR}}
- name: DB_PORT
valueFrom: {secretKeyRef: {name: aevolution-db, key: PGPORT}}
- name: DB_NAME
valueFrom: {secretKeyRef: {name: aevolution-db, key: PGDATABASE}}
- name: DB_USER
valueFrom: {secretKeyRef: {name: evolution-postgres-runtime, key: user}}
resources:
requests: {cpu: 50m, memory: 64Mi}
limits: {memory: 256Mi}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
volumeMounts:
- {name: objects, mountPath: /objects}
- {name: connection, mountPath: /run/postgres, readOnly: true}
- {name: scratch, mountPath: /tmp}
volumes:
- name: objects
persistentVolumeClaim:
claimName: aevolution-objects
- name: scratch
emptyDir: {medium: Memory, sizeLimit: 1Mi}
- name: connection
secret:
secretName: evolution-postgres-runtime
defaultMode: 0440
items:
- {key: pgpass, path: pgpass}
- {key: ca.crt, path: ca.crt}