P1: restore-world.sh claims its objects path (mkdir) and the database (a held session with a restore try-lock and the exclusive upgrade lock), and removes only what it created, so a losing restore never drops a winner's schema. P2: barrier Ready-pod selection via shared jsonpath helper with fixtures; barrier pod fsGroup; script-generated pods inspected by the policy check; World0 PV claim references protected; ci-paths.sh uses the event's real base and runs on uncertainty; fault-drill asserts RTO and needs positive quarantine evidence. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
147 lines
5.8 KiB
Bash
147 lines
5.8 KiB
Bash
#!/usr/bin/env bash
|
|
# M8.7 operator-run backup of a disposable environment to storage off the cluster
|
|
# (ADR-0074).
|
|
#
|
|
# tools/backup/cluster-backup.sh <dev|stage> --to DIR [--verify WORLD] [--context NAME]
|
|
# tools/backup/cluster-backup.sh <dev|stage> --print-manifest
|
|
#
|
|
# Runs one short-lived Pod in evolution-<env> (image: the postgres-tools digest
|
|
# pinned in deploy/kubernetes/stores/<env>; migrator login; the aevolution-objects
|
|
# volume mounted read-only on the store node) that runs backup-world.sh into an
|
|
# emptyDir. The bundle is then streamed out with `kubectl exec tar` to
|
|
# DIR/<env>-<UTC time>/, its checksums are verified here, and the Pod is deleted
|
|
# (also on failure). DIR is the destination: until the maintainer chooses the
|
|
# permanent one (docs/BACKLOG.md), it is any directory on a machine outside the
|
|
# hypervisor host. --verify WORLD then runs verify-backup.sh on this machine
|
|
# (restore on a private cluster, rebuilt and served as isolated world WORLD); it
|
|
# needs the PostgreSQL 18.6 binaries and evo-server/evo-world-clone.
|
|
# --print-manifest prints the Pod and exits without touching the cluster (the deploy
|
|
# policy check inspects it). World0 is refused before anything touches the cluster. The Pod is an operator
|
|
# action, like fault-drill.sh; it is not part of any Argo application. A scheduled
|
|
# backup (CronJob in Git, synced by Argo) follows once the destination exists.
|
|
set -euo pipefail
|
|
usage() { echo "usage: $0 <dev|stage> --to DIR [--verify WORLD] [--context NAME] | --print-manifest" >&2; exit 2; }
|
|
[ $# -ge 1 ] || usage
|
|
env="$1"; shift
|
|
case "$env" in
|
|
dev|stage) ;;
|
|
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
|
|
exit 2 ;;
|
|
esac
|
|
to="" verify="" print=0 k=(kubectl --request-timeout=30s)
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--to) to="${2:-}"; shift 2 ;;
|
|
--verify) verify="${2:-}"; shift 2 ;;
|
|
--print-manifest) print=1; shift ;;
|
|
--context) k+=(--context "${2:?}"); shift 2 ;;
|
|
*) usage ;;
|
|
esac
|
|
done
|
|
if [ "$print" != 1 ]; then
|
|
[ -n "$to" ] && [ -d "$to" ] || { echo "--to must be an existing directory" >&2; exit 2; }
|
|
fi
|
|
root="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
ns="evolution-$env"
|
|
image="$(awk '/name: aevolution-postgres-tools/ {f = 1} f && /newName:/ {n = $2} f && /digest:/ {print n "@" $2; exit}' \
|
|
"$root/deploy/kubernetes/stores/$env/kustomization.yaml")"
|
|
[[ "$image" =~ ^[a-z0-9./-]+@sha256:[0-9a-f]{64}$ ]] || { echo "no pinned postgres-tools image for $env" >&2; exit 2; }
|
|
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
pod="aevolution-backup-$(date -u +%Y%m%d%H%M%S)"
|
|
dest="$to/$env-$stamp"
|
|
say() { echo "$(date -u +%H:%M:%S) $*"; }
|
|
|
|
manifest() {
|
|
cat <<YAML
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
name: $pod
|
|
namespace: $ns
|
|
labels:
|
|
app.kubernetes.io/part-of: aevolution
|
|
app.kubernetes.io/component: backup
|
|
spec:
|
|
serviceAccountName: aevolution-migrator
|
|
automountServiceAccountToken: false
|
|
restartPolicy: Never
|
|
activeDeadlineSeconds: 3600
|
|
nodeSelector:
|
|
aevolution.io/stores: "true"
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 10001
|
|
runAsGroup: 10001
|
|
fsGroup: 10001
|
|
seccompProfile: {type: RuntimeDefault}
|
|
containers:
|
|
- name: backup
|
|
image: $image
|
|
command: ["/bin/bash", "-c"]
|
|
args:
|
|
- |
|
|
set -eu
|
|
bash /opt/evo/tools/backup-world.sh --objects /objects --out /work/bundle --label "$env-$stamp"
|
|
touch /work/complete
|
|
exec sleep 1800
|
|
env:
|
|
- {name: PGSERVICE, value: evolution}
|
|
- {name: PGSERVICEFILE, value: /run/postgres/pg_service.conf}
|
|
- {name: EVO_PGPASS_SOURCE, value: /run/postgres/pgpass}
|
|
resources:
|
|
requests: {cpu: 100m, memory: 128Mi}
|
|
limits: {memory: 512Mi}
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
capabilities: {drop: [ALL]}
|
|
volumeMounts:
|
|
- {name: objects, mountPath: /objects, readOnly: true}
|
|
- {name: connection, mountPath: /run/postgres, readOnly: true}
|
|
- {name: work, mountPath: /work}
|
|
- {name: scratch, mountPath: /tmp}
|
|
volumes:
|
|
- name: objects
|
|
persistentVolumeClaim: {claimName: aevolution-objects, readOnly: true}
|
|
- name: connection
|
|
secret:
|
|
secretName: evolution-postgres-migrator
|
|
defaultMode: 0440
|
|
items:
|
|
- {key: pg_service.conf, path: pg_service.conf}
|
|
- {key: pgpass, path: pgpass}
|
|
- {key: ca.crt, path: ca.crt}
|
|
- name: work
|
|
emptyDir: {sizeLimit: 12Gi}
|
|
- name: scratch
|
|
emptyDir: {medium: Memory, sizeLimit: 8Mi}
|
|
YAML
|
|
}
|
|
if [ "$print" = 1 ]; then manifest; exit 0; fi
|
|
|
|
cleanup() { "${k[@]}" delete pod -n "$ns" "$pod" --wait=false > /dev/null 2>&1 || true; }
|
|
trap cleanup EXIT
|
|
say "starting $pod in $ns ($image)"
|
|
manifest | "${k[@]}" apply -f - > /dev/null
|
|
|
|
for _ in $(seq 1 1800); do
|
|
phase="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}' 2> /dev/null || true)"
|
|
[ "$phase" = Failed ] && break
|
|
[ "$phase" = Running ] && "${k[@]}" exec -n "$ns" "$pod" -- test -f /work/complete 2> /dev/null && break
|
|
sleep 1
|
|
done
|
|
"${k[@]}" logs -n "$ns" "$pod" 2>&1 | sed 's/^/ /'
|
|
"${k[@]}" exec -n "$ns" "$pod" -- test -f /work/complete 2> /dev/null ||
|
|
{ say "FAIL backup did not complete (phase ${phase:-unknown})"; exit 3; }
|
|
|
|
mkdir -p "$dest"
|
|
"${k[@]}" exec -n "$ns" "$pod" -- tar -C /work/bundle -cf - . | tar -C "$dest" -xf -
|
|
(cd "$dest" && sha256sum --check --strict --quiet SHA256SUMS) || { say "FAIL fetched bundle checksum mismatch"; exit 3; }
|
|
say "bundle verified at $dest ($(du -sh "$dest" | cut -f1)); $(grep -c '^head ' "$dest/MANIFEST") region(s):"
|
|
grep '^head ' "$dest/MANIFEST" | sed 's/^/ /'
|
|
if [ -n "$verify" ]; then
|
|
bash "$root/tools/backup/verify-backup.sh" "$dest" --isolate "$verify"
|
|
else
|
|
say "next: tools/backup/verify-backup.sh $dest --isolate <world> (restore test on independent storage)"
|
|
fi
|