P1: restore-world.sh claims its objects path (mkdir) and the database (a held session with a restore try-lock and the exclusive upgrade lock), and removes only what it created, so a losing restore never drops a winner's schema. P2: barrier Ready-pod selection via shared jsonpath helper with fixtures; barrier pod fsGroup; script-generated pods inspected by the policy check; World0 PV claim references protected; ci-paths.sh uses the event's real base and runs on uncertainty; fault-drill asserts RTO and needs positive quarantine evidence. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
157 lines
6.6 KiB
Bash
157 lines
6.6 KiB
Bash
#!/usr/bin/env bash
|
|
# M8.9 version barrier in a disposable environment (ADR-0068, ADR-0076).
|
|
#
|
|
# tools/ops/upgrade-barrier.sh <dev|stage> --retire HEX64 [--context NAME] [--print-manifest]
|
|
#
|
|
# The last step of the upgrade order: migrate -> retain the new build (the provision
|
|
# hook does) -> deploy it with the old build in REGION_RETAINED -> retire the old
|
|
# build here. Refused unless:
|
|
# - HEX64 is not the build Git currently deploys (deploy/kubernetes/region/<env>);
|
|
# - HEX64 is listed in REGION_RETAINED (its history must stay recoverable);
|
|
# - exactly one evo-region pod is Ready (and not being deleted), and it runs exactly
|
|
# the Git-pinned image, so the new build already serves.
|
|
# Then one short Pod (the postgres-tools image pinned for <env>, migrator login, the
|
|
# same fsGroup Secret-access contract as the other database-tool pods) runs
|
|
# evolution.upgrade_barrier(HEX64): the old build can never acquire, renew or commit
|
|
# again, and every lease is fenced. The new host re-acquires within one lease. The
|
|
# script waits for Ready again and prints the time without a ready host.
|
|
# --print-manifest prints that Pod after the offline checks and exits without
|
|
# touching the cluster (the deploy policy check inspects it).
|
|
# Retirement is irreversible here. World0 is refused before kubectl runs.
|
|
set -euo pipefail
|
|
usage() { echo "usage: $0 <dev|stage> --retire HEX64 [--context NAME] [--print-manifest]" >&2; exit 2; }
|
|
[ $# -ge 1 ] || usage
|
|
env="$1"; shift
|
|
case "$env" in
|
|
dev|stage) ;;
|
|
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
|
|
exit 2 ;;
|
|
esac
|
|
retire="" print=0 k=(kubectl --request-timeout=30s)
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--retire) retire="${2:-}"; shift 2 ;;
|
|
--context) k+=(--context "${2:?}"); shift 2 ;;
|
|
--print-manifest) print=1; shift ;;
|
|
*) usage ;;
|
|
esac
|
|
done
|
|
[[ "$retire" =~ ^[0-9a-f]{64}$ ]] || usage
|
|
root="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
# shellcheck source=lib/pods.sh
|
|
. "$root/tools/ops/lib/pods.sh"
|
|
ns="evolution-$env"
|
|
values="$root/deploy/kubernetes/region/$env/region-values.yaml"
|
|
current="$(sed -n 's/.*name: REGION_BUILD, value: \([0-9a-f]\{64\}\)}.*/\1/p' "$values" | sort -u)"
|
|
retained="$(sed -n 's/.*name: REGION_RETAINED, value: "\([0-9a-f ]*\)"}.*/\1/p' "$values" | head -1)"
|
|
[ "$(wc -l <<< "$current")" = 1 ] || { echo "cannot read one REGION_BUILD from $values" >&2; exit 2; }
|
|
[ "$retire" != "$current" ] || { echo "refused: $retire is the build Git deploys" >&2; exit 2; }
|
|
grep -qw "$retire" <<< "$retained" || { echo "refused: $retire is not in REGION_RETAINED" >&2; exit 2; }
|
|
image="$(awk '/name: aevolution-postgres-tools/ {f = 1} f && /newName:/ {n = $2} f && /digest:/ {print n "@" $2; exit}' \
|
|
"$root/deploy/kubernetes/stores/$env/kustomization.yaml")"
|
|
[[ "$image" =~ @sha256:[0-9a-f]{64}$ ]] || { echo "no pinned postgres-tools image for $env" >&2; exit 2; }
|
|
pod="aevolution-barrier-$(date -u +%Y%m%d%H%M%S)"
|
|
|
|
manifest() {
|
|
cat <<YAML
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
name: $pod
|
|
namespace: $ns
|
|
labels:
|
|
app.kubernetes.io/part-of: aevolution
|
|
app.kubernetes.io/component: migrate
|
|
spec:
|
|
serviceAccountName: aevolution-migrator
|
|
automountServiceAccountToken: false
|
|
restartPolicy: Never
|
|
activeDeadlineSeconds: 120
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 10001
|
|
runAsGroup: 10001
|
|
fsGroup: 10001
|
|
seccompProfile: {type: RuntimeDefault}
|
|
containers:
|
|
- name: barrier
|
|
image: $image
|
|
command: ["/bin/bash", "-c"]
|
|
args:
|
|
- |
|
|
set -eu
|
|
umask 077
|
|
cp /run/postgres/pgpass /tmp/pgpass
|
|
export PGPASSFILE=/tmp/pgpass
|
|
psql -X --no-password -At -v ON_ERROR_STOP=1 \\
|
|
-c "SELECT 'fenced ' || evolution.upgrade_barrier(decode('$retire', 'hex'))" \\
|
|
-c "SELECT 'retired ' || encode(build_identity, 'hex') FROM evolution.retained_builds WHERE retired_at IS NOT NULL ORDER BY 1"
|
|
env:
|
|
- {name: PGSERVICE, value: evolution}
|
|
- {name: PGSERVICEFILE, value: /run/postgres/pg_service.conf}
|
|
resources:
|
|
requests: {cpu: 50m, memory: 64Mi}
|
|
limits: {memory: 128Mi}
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
capabilities: {drop: [ALL]}
|
|
volumeMounts:
|
|
- {name: connection, mountPath: /run/postgres, readOnly: true}
|
|
- {name: scratch, mountPath: /tmp}
|
|
volumes:
|
|
- name: connection
|
|
secret:
|
|
secretName: evolution-postgres-migrator
|
|
defaultMode: 0440
|
|
items:
|
|
- {key: pg_service.conf, path: pg_service.conf}
|
|
- {key: pgpass, path: pgpass}
|
|
- {key: ca.crt, path: ca.crt}
|
|
- name: scratch
|
|
emptyDir: {medium: Memory, sizeLimit: 1Mi}
|
|
YAML
|
|
}
|
|
if [ "$print" = 1 ]; then manifest; exit 0; fi
|
|
|
|
say() { echo "$(date -u +%H:%M:%S) $*"; }
|
|
ready_pod() { # The single Ready evo-region pod, or failure (none, several, or unreadable).
|
|
"${k[@]}" get pods -n "$ns" -l app.kubernetes.io/name=evo-region -o jsonpath="$POD_STATUS_JSONPATH" |
|
|
single_ready_pod
|
|
}
|
|
serving() { # The single Ready pod's running image digest.
|
|
local name
|
|
name="$(ready_pod 2> /dev/null)" || return 1
|
|
"${k[@]}" get pod -n "$ns" "$name" -o jsonpath='{.status.containerStatuses[0].imageID}'
|
|
}
|
|
ready_pod > /dev/null || { echo "refused: no single Ready evo-region pod" >&2; exit 3; }
|
|
[[ "$(serving)" == *"@sha256:$current" ]] ||
|
|
{ echo "refused: the Ready evo-region pod does not run sha256:$current (deploy the new build first)" >&2; exit 3; }
|
|
|
|
cleanup() { "${k[@]}" delete pod -n "$ns" "$pod" --wait=false > /dev/null 2>&1 || true; }
|
|
trap cleanup EXIT
|
|
say "retiring $retire in $ns (serving sha256:$current)"
|
|
manifest | "${k[@]}" apply -f - > /dev/null
|
|
start="$(date +%s%3N)"
|
|
phase=""
|
|
for _ in $(seq 1 120); do
|
|
phase="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}' 2> /dev/null || true)"
|
|
{ [ "$phase" = Succeeded ] || [ "$phase" = Failed ]; } && break
|
|
sleep 1
|
|
done
|
|
"${k[@]}" logs -n "$ns" "$pod" 2>&1 | sed 's/^/ /'
|
|
[ "$phase" = Succeeded ] || { say "FAIL barrier pod ${phase:-unknown}"; exit 3; }
|
|
# The fenced host loses readiness at its next lease check, then re-acquires.
|
|
not_ready_since=""
|
|
for _ in $(seq 1 600); do
|
|
if ready_pod > /dev/null 2>&1; then
|
|
[ -z "$not_ready_since" ] && [ $(( $(date +%s%3N) - start )) -lt 3000 ] && { sleep 0.5; continue; }
|
|
break
|
|
fi
|
|
[ -n "$not_ready_since" ] || not_ready_since="$(date +%s%3N)"
|
|
sleep 0.5
|
|
done
|
|
[[ "$(serving)" == *"@sha256:$current" ]] || { say "FAIL no Ready pod serves sha256:$current after the barrier"; exit 3; }
|
|
gap=0; [ -n "$not_ready_since" ] && gap=$(( $(date +%s%3N) - not_ready_since ))
|
|
say "PASS $retire retired in $ns; not-ready gap ${gap} ms; sha256:$current serving"
|