Files
aevolution/tools/backup/cluster-backup.sh
T
dingoandClaude Opus 5.5 5512fdec54 feat(backup): M8.7 world backup bundles and verified restore (ADR-0074)
Online, snapshot-consistent bundles (pg_dump at an exported snapshot, immutable
objects verified by digest, MANIFEST of versions/builds/heads, SHA256SUMS); a
fail-closed restore that audits every region's full history on independent
storage; an isolated rebuild through clones that serves under load; an
operator-run in-cluster backup streamed off the cluster. backup-check.sh runs in
every PostgreSQL build, with corruption, missing-object, tampered-manifest and
non-empty-target refusals. Destination, encryption, schedule and WAL archiving
remain on the backlog.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 15:51:04 -04:00

136 lines
5.5 KiB
Bash

#!/usr/bin/env bash
# M8.7 operator-run backup of a disposable environment to storage off the cluster
# (ADR-0074).
#
# tools/backup/cluster-backup.sh <dev|stage> --to DIR [--verify WORLD] [--context NAME]
#
# Runs one short-lived Pod in evolution-<env> (image: the postgres-tools digest
# pinned in deploy/kubernetes/stores/<env>; migrator login; the aevolution-objects
# volume mounted read-only on the store node) that runs backup-world.sh into an
# emptyDir. The bundle is then streamed out with `kubectl exec tar` to
# DIR/<env>-<UTC time>/, its checksums are verified here, and the Pod is deleted
# (also on failure). DIR is the destination: until the maintainer chooses the
# permanent one (docs/BACKLOG.md), it is any directory on a machine outside the
# hypervisor host. --verify WORLD then runs verify-backup.sh on this machine
# (restore on a private cluster, rebuilt and served as isolated world WORLD); it
# needs the PostgreSQL 18.6 binaries and evo-server/evo-world-clone.
# World0 is refused before anything touches the cluster. The Pod is an operator
# action, like fault-drill.sh; it is not part of any Argo application. A scheduled
# backup (CronJob in Git, synced by Argo) follows once the destination exists.
set -euo pipefail
usage() { echo "usage: $0 <dev|stage> --to DIR [--verify WORLD] [--context NAME]" >&2; exit 2; }
[ $# -ge 1 ] || usage
env="$1"; shift
case "$env" in
dev|stage) ;;
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
exit 2 ;;
esac
to="" verify="" k=(kubectl --request-timeout=30s)
while [ $# -gt 0 ]; do
case "$1" in
--to) to="${2:-}"; shift 2 ;;
--verify) verify="${2:-}"; shift 2 ;;
--context) k+=(--context "${2:?}"); shift 2 ;;
*) usage ;;
esac
done
[ -n "$to" ] && [ -d "$to" ] || { echo "--to must be an existing directory" >&2; exit 2; }
root="$(cd "$(dirname "$0")/../.." && pwd)"
ns="evolution-$env"
image="$(awk '/name: aevolution-postgres-tools/ {f = 1} f && /newName:/ {n = $2} f && /digest:/ {print n "@" $2; exit}' \
"$root/deploy/kubernetes/stores/$env/kustomization.yaml")"
[[ "$image" =~ ^[a-z0-9./-]+@sha256:[0-9a-f]{64}$ ]] || { echo "no pinned postgres-tools image for $env" >&2; exit 2; }
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
pod="aevolution-backup-$(date -u +%Y%m%d%H%M%S)"
dest="$to/$env-$stamp"
say() { echo "$(date -u +%H:%M:%S) $*"; }
cleanup() { "${k[@]}" delete pod -n "$ns" "$pod" --wait=false > /dev/null 2>&1 || true; }
trap cleanup EXIT
say "starting $pod in $ns ($image)"
"${k[@]}" apply -n "$ns" -f - > /dev/null <<YAML
apiVersion: v1
kind: Pod
metadata:
name: $pod
labels:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: backup
spec:
serviceAccountName: aevolution-migrator
automountServiceAccountToken: false
restartPolicy: Never
activeDeadlineSeconds: 3600
nodeSelector:
aevolution.io/stores: "true"
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile: {type: RuntimeDefault}
containers:
- name: backup
image: $image
command: ["/bin/bash", "-c"]
args:
- |
set -eu
bash /opt/evo/tools/backup-world.sh --objects /objects --out /work/bundle --label "$env-$stamp"
touch /work/complete
exec sleep 1800
env:
- {name: PGSERVICE, value: evolution}
- {name: PGSERVICEFILE, value: /run/postgres/pg_service.conf}
- {name: EVO_PGPASS_SOURCE, value: /run/postgres/pgpass}
resources:
requests: {cpu: 100m, memory: 128Mi}
limits: {memory: 512Mi}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
volumeMounts:
- {name: objects, mountPath: /objects, readOnly: true}
- {name: connection, mountPath: /run/postgres, readOnly: true}
- {name: work, mountPath: /work}
- {name: scratch, mountPath: /tmp}
volumes:
- name: objects
persistentVolumeClaim: {claimName: aevolution-objects, readOnly: true}
- name: connection
secret:
secretName: evolution-postgres-migrator
defaultMode: 0440
items:
- {key: pg_service.conf, path: pg_service.conf}
- {key: pgpass, path: pgpass}
- {key: ca.crt, path: ca.crt}
- name: work
emptyDir: {sizeLimit: 12Gi}
- name: scratch
emptyDir: {medium: Memory, sizeLimit: 8Mi}
YAML
for _ in $(seq 1 1800); do
phase="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}' 2> /dev/null || true)"
[ "$phase" = Failed ] && break
[ "$phase" = Running ] && "${k[@]}" exec -n "$ns" "$pod" -- test -f /work/complete 2> /dev/null && break
sleep 1
done
"${k[@]}" logs -n "$ns" "$pod" 2>&1 | sed 's/^/ /'
"${k[@]}" exec -n "$ns" "$pod" -- test -f /work/complete 2> /dev/null ||
{ say "FAIL backup did not complete (phase ${phase:-unknown})"; exit 3; }
mkdir -p "$dest"
"${k[@]}" exec -n "$ns" "$pod" -- tar -C /work/bundle -cf - . | tar -C "$dest" -xf -
(cd "$dest" && sha256sum --check --strict --quiet SHA256SUMS) || { say "FAIL fetched bundle checksum mismatch"; exit 3; }
say "bundle verified at $dest ($(du -sh "$dest" | cut -f1)); $(grep -c '^head ' "$dest/MANIFEST") region(s):"
grep '^head ' "$dest/MANIFEST" | sed 's/^/ /'
if [ -n "$verify" ]; then
bash "$root/tools/backup/verify-backup.sh" "$dest" --isolate "$verify"
else
say "next: tools/backup/verify-backup.sh $dest --isolate <world> (restore test on independent storage)"
fi