Summarize the maintainer-collected read-only inventory: RKE2 v1.32.5, an untainted control plane and three 32 GB workers on one hypervisor, no kubelet reservations or quotas, local-path as the only (node-pinned, reclaim Delete) storage, Canal/ingress-nginx/MetalLB/cert-manager, a manually synced aevolution-dev application and no PostgreSQL 18.6. Propose a measured budget and dev placement, and add a ready-to-apply local-path probe with one PVC and Job pinned to each worker. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
292 lines
9.7 KiB
YAML
292 lines
9.7 KiB
YAML
# M8.1 storage latency probe for the cluster's only StorageClass, local-path
|
|
# (rancher.io/local-path: node-local directories, reclaim Delete,
|
|
# WaitForFirstConsumer). A local-path volume is created on the node that first
|
|
# runs its pod, so each worker gets its own PVC and Job pinned by hostname.
|
|
# Generated from disk-probe.yaml.example; outside every Argo path, applied by hand
|
|
# with the maintainer's approval (2026-09-29):
|
|
# kubectl apply -f deploy/kubernetes/inventory/disk-probe-local-path.yaml
|
|
# kubectl wait -n evolution-dev --for=condition=complete --timeout=600s \
|
|
# job -l app.kubernetes.io/name=evo-disk-probe
|
|
# kubectl logs -n evolution-dev -l app.kubernetes.io/name=evo-disk-probe --prefix
|
|
# kubectl delete -f deploy/kubernetes/inventory/disk-probe-local-path.yaml
|
|
# Deleting the PVCs deletes the probe data (reclaim Delete); nothing else is touched.
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: evo-disk-probe
|
|
namespace: evolution-dev
|
|
labels:
|
|
app.kubernetes.io/name: evo-disk-probe
|
|
spec:
|
|
podSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: evo-disk-probe
|
|
policyTypes: [Ingress, Egress] # No traffic in or out.
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: evo-disk-probe-w1
|
|
namespace: evolution-dev
|
|
labels:
|
|
app.kubernetes.io/name: evo-disk-probe
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
storageClassName: local-path
|
|
resources:
|
|
requests:
|
|
storage: 2Gi
|
|
---
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: evo-disk-probe-w1
|
|
namespace: evolution-dev
|
|
labels:
|
|
app.kubernetes.io/name: evo-disk-probe
|
|
spec:
|
|
backoffLimit: 0
|
|
activeDeadlineSeconds: 600
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: evo-disk-probe
|
|
spec:
|
|
restartPolicy: Never
|
|
automountServiceAccountToken: false
|
|
imagePullSecrets:
|
|
- name: harbor-pull
|
|
nodeSelector:
|
|
kubernetes.io/hostname: rke2-worker-1
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 10001
|
|
runAsGroup: 10001
|
|
fsGroup: 10001
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containers:
|
|
- name: probe
|
|
# Same repository and digest as deploy/kubernetes/dev (verified run 1399).
|
|
image: registry.aeternalisgames.com/aeternalis/aevolution@sha256:0c6c6e1ac28de2ce0648fd0992ca694963e303ef5c9414fe5ac668937136800d
|
|
command: ["/bin/bash", "-c"]
|
|
args:
|
|
- |
|
|
set -eu
|
|
cd /data
|
|
ns() { date +%s%N; }
|
|
echo "node=rke2-worker-1 filesystem: $(df -hT /data | tail -1)"
|
|
t0=$(ns)
|
|
dd if=/dev/zero of=dsync.bin bs=4k count=1000 oflag=dsync status=none
|
|
t1=$(ns)
|
|
echo "dsync_4k_writes=1000 mean_us=$(( (t1 - t0) / 1000 / 1000 ))"
|
|
head -c 40960 /dev/urandom > payload.src
|
|
mkdir -p objects
|
|
worst=0; t0=$(ns)
|
|
for i in $(seq 1 200); do
|
|
s=$(ns)
|
|
dd if=payload.src of=objects/.tmp status=none conv=fsync
|
|
mv objects/.tmp objects/o$i
|
|
sync objects
|
|
e=$(( ($(ns) - s) / 1000 )); [ "$e" -gt "$worst" ] && worst=$e
|
|
done
|
|
t1=$(ns)
|
|
echo "object_publish_40k=200 mean_us=$(( (t1 - t0) / 200 / 1000 )) max_us=$worst"
|
|
t0=$(ns)
|
|
dd if=/dev/zero of=seq.bin bs=1M count=256 conv=fdatasync status=none
|
|
t1=$(ns)
|
|
echo "sequential_256m_ms=$(( (t1 - t0) / 1000000 ))"
|
|
rm -rf dsync.bin payload.src objects seq.bin
|
|
resources:
|
|
requests: {cpu: 100m, memory: 64Mi}
|
|
limits: {cpu: 500m, memory: 128Mi}
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
capabilities:
|
|
drop: [ALL]
|
|
volumeMounts:
|
|
- {name: data, mountPath: /data}
|
|
volumes:
|
|
- name: data
|
|
persistentVolumeClaim:
|
|
claimName: evo-disk-probe-w1
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: evo-disk-probe-w2
|
|
namespace: evolution-dev
|
|
labels:
|
|
app.kubernetes.io/name: evo-disk-probe
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
storageClassName: local-path
|
|
resources:
|
|
requests:
|
|
storage: 2Gi
|
|
---
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: evo-disk-probe-w2
|
|
namespace: evolution-dev
|
|
labels:
|
|
app.kubernetes.io/name: evo-disk-probe
|
|
spec:
|
|
backoffLimit: 0
|
|
activeDeadlineSeconds: 600
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: evo-disk-probe
|
|
spec:
|
|
restartPolicy: Never
|
|
automountServiceAccountToken: false
|
|
imagePullSecrets:
|
|
- name: harbor-pull
|
|
nodeSelector:
|
|
kubernetes.io/hostname: rke2-worker-2
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 10001
|
|
runAsGroup: 10001
|
|
fsGroup: 10001
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containers:
|
|
- name: probe
|
|
# Same repository and digest as deploy/kubernetes/dev (verified run 1399).
|
|
image: registry.aeternalisgames.com/aeternalis/aevolution@sha256:0c6c6e1ac28de2ce0648fd0992ca694963e303ef5c9414fe5ac668937136800d
|
|
command: ["/bin/bash", "-c"]
|
|
args:
|
|
- |
|
|
set -eu
|
|
cd /data
|
|
ns() { date +%s%N; }
|
|
echo "node=rke2-worker-2 filesystem: $(df -hT /data | tail -1)"
|
|
t0=$(ns)
|
|
dd if=/dev/zero of=dsync.bin bs=4k count=1000 oflag=dsync status=none
|
|
t1=$(ns)
|
|
echo "dsync_4k_writes=1000 mean_us=$(( (t1 - t0) / 1000 / 1000 ))"
|
|
head -c 40960 /dev/urandom > payload.src
|
|
mkdir -p objects
|
|
worst=0; t0=$(ns)
|
|
for i in $(seq 1 200); do
|
|
s=$(ns)
|
|
dd if=payload.src of=objects/.tmp status=none conv=fsync
|
|
mv objects/.tmp objects/o$i
|
|
sync objects
|
|
e=$(( ($(ns) - s) / 1000 )); [ "$e" -gt "$worst" ] && worst=$e
|
|
done
|
|
t1=$(ns)
|
|
echo "object_publish_40k=200 mean_us=$(( (t1 - t0) / 200 / 1000 )) max_us=$worst"
|
|
t0=$(ns)
|
|
dd if=/dev/zero of=seq.bin bs=1M count=256 conv=fdatasync status=none
|
|
t1=$(ns)
|
|
echo "sequential_256m_ms=$(( (t1 - t0) / 1000000 ))"
|
|
rm -rf dsync.bin payload.src objects seq.bin
|
|
resources:
|
|
requests: {cpu: 100m, memory: 64Mi}
|
|
limits: {cpu: 500m, memory: 128Mi}
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
capabilities:
|
|
drop: [ALL]
|
|
volumeMounts:
|
|
- {name: data, mountPath: /data}
|
|
volumes:
|
|
- name: data
|
|
persistentVolumeClaim:
|
|
claimName: evo-disk-probe-w2
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: evo-disk-probe-w3
|
|
namespace: evolution-dev
|
|
labels:
|
|
app.kubernetes.io/name: evo-disk-probe
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
storageClassName: local-path
|
|
resources:
|
|
requests:
|
|
storage: 2Gi
|
|
---
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: evo-disk-probe-w3
|
|
namespace: evolution-dev
|
|
labels:
|
|
app.kubernetes.io/name: evo-disk-probe
|
|
spec:
|
|
backoffLimit: 0
|
|
activeDeadlineSeconds: 600
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: evo-disk-probe
|
|
spec:
|
|
restartPolicy: Never
|
|
automountServiceAccountToken: false
|
|
imagePullSecrets:
|
|
- name: harbor-pull
|
|
nodeSelector:
|
|
kubernetes.io/hostname: rke2-worker-3
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 10001
|
|
runAsGroup: 10001
|
|
fsGroup: 10001
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containers:
|
|
- name: probe
|
|
# Same repository and digest as deploy/kubernetes/dev (verified run 1399).
|
|
image: registry.aeternalisgames.com/aeternalis/aevolution@sha256:0c6c6e1ac28de2ce0648fd0992ca694963e303ef5c9414fe5ac668937136800d
|
|
command: ["/bin/bash", "-c"]
|
|
args:
|
|
- |
|
|
set -eu
|
|
cd /data
|
|
ns() { date +%s%N; }
|
|
echo "node=rke2-worker-3 filesystem: $(df -hT /data | tail -1)"
|
|
t0=$(ns)
|
|
dd if=/dev/zero of=dsync.bin bs=4k count=1000 oflag=dsync status=none
|
|
t1=$(ns)
|
|
echo "dsync_4k_writes=1000 mean_us=$(( (t1 - t0) / 1000 / 1000 ))"
|
|
head -c 40960 /dev/urandom > payload.src
|
|
mkdir -p objects
|
|
worst=0; t0=$(ns)
|
|
for i in $(seq 1 200); do
|
|
s=$(ns)
|
|
dd if=payload.src of=objects/.tmp status=none conv=fsync
|
|
mv objects/.tmp objects/o$i
|
|
sync objects
|
|
e=$(( ($(ns) - s) / 1000 )); [ "$e" -gt "$worst" ] && worst=$e
|
|
done
|
|
t1=$(ns)
|
|
echo "object_publish_40k=200 mean_us=$(( (t1 - t0) / 200 / 1000 )) max_us=$worst"
|
|
t0=$(ns)
|
|
dd if=/dev/zero of=seq.bin bs=1M count=256 conv=fdatasync status=none
|
|
t1=$(ns)
|
|
echo "sequential_256m_ms=$(( (t1 - t0) / 1000000 ))"
|
|
rm -rf dsync.bin payload.src objects seq.bin
|
|
resources:
|
|
requests: {cpu: 100m, memory: 64Mi}
|
|
limits: {cpu: 500m, memory: 128Mi}
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
capabilities:
|
|
drop: [ALL]
|
|
volumeMounts:
|
|
- {name: data, mountPath: /data}
|
|
volumes:
|
|
- name: data
|
|
persistentVolumeClaim:
|
|
claimName: evo-disk-probe-w3
|