Files
aevolution/tools/ops/upgrade-barrier.sh
dingoandClaude Opus 5.5 3a34ae39bc fix: M8 review findings (restore ownership, barrier tool, World0 claimRef, CI selection, drill verdicts)
P1: restore-world.sh claims its objects path (mkdir) and the database (a held
session with a restore try-lock and the exclusive upgrade lock), and removes only
what it created, so a losing restore never drops a winner's schema.
P2: barrier Ready-pod selection via shared jsonpath helper with fixtures; barrier
pod fsGroup; script-generated pods inspected by the policy check; World0 PV claim
references protected; ci-paths.sh uses the event's real base and runs on
uncertainty; fault-drill asserts RTO and needs positive quarantine evidence.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 22:53:11 -04:00

157 lines
6.6 KiB
Bash

#!/usr/bin/env bash
# M8.9 version barrier in a disposable environment (ADR-0068, ADR-0076).
#
# tools/ops/upgrade-barrier.sh <dev|stage> --retire HEX64 [--context NAME] [--print-manifest]
#
# The last step of the upgrade order: migrate -> retain the new build (the provision
# hook does) -> deploy it with the old build in REGION_RETAINED -> retire the old
# build here. Refused unless:
# - HEX64 is not the build Git currently deploys (deploy/kubernetes/region/<env>);
# - HEX64 is listed in REGION_RETAINED (its history must stay recoverable);
# - exactly one evo-region pod is Ready (and not being deleted), and it runs exactly
# the Git-pinned image, so the new build already serves.
# Then one short Pod (the postgres-tools image pinned for <env>, migrator login, the
# same fsGroup Secret-access contract as the other database-tool pods) runs
# evolution.upgrade_barrier(HEX64): the old build can never acquire, renew or commit
# again, and every lease is fenced. The new host re-acquires within one lease. The
# script waits for Ready again and prints the time without a ready host.
# --print-manifest prints that Pod after the offline checks and exits without
# touching the cluster (the deploy policy check inspects it).
# Retirement is irreversible here. World0 is refused before kubectl runs.
set -euo pipefail
usage() { echo "usage: $0 <dev|stage> --retire HEX64 [--context NAME] [--print-manifest]" >&2; exit 2; }
[ $# -ge 1 ] || usage
env="$1"; shift
case "$env" in
dev|stage) ;;
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
exit 2 ;;
esac
retire="" print=0 k=(kubectl --request-timeout=30s)
while [ $# -gt 0 ]; do
case "$1" in
--retire) retire="${2:-}"; shift 2 ;;
--context) k+=(--context "${2:?}"); shift 2 ;;
--print-manifest) print=1; shift ;;
*) usage ;;
esac
done
[[ "$retire" =~ ^[0-9a-f]{64}$ ]] || usage
root="$(cd "$(dirname "$0")/../.." && pwd)"
# shellcheck source=lib/pods.sh
. "$root/tools/ops/lib/pods.sh"
ns="evolution-$env"
values="$root/deploy/kubernetes/region/$env/region-values.yaml"
current="$(sed -n 's/.*name: REGION_BUILD, value: \([0-9a-f]\{64\}\)}.*/\1/p' "$values" | sort -u)"
retained="$(sed -n 's/.*name: REGION_RETAINED, value: "\([0-9a-f ]*\)"}.*/\1/p' "$values" | head -1)"
[ "$(wc -l <<< "$current")" = 1 ] || { echo "cannot read one REGION_BUILD from $values" >&2; exit 2; }
[ "$retire" != "$current" ] || { echo "refused: $retire is the build Git deploys" >&2; exit 2; }
grep -qw "$retire" <<< "$retained" || { echo "refused: $retire is not in REGION_RETAINED" >&2; exit 2; }
image="$(awk '/name: aevolution-postgres-tools/ {f = 1} f && /newName:/ {n = $2} f && /digest:/ {print n "@" $2; exit}' \
"$root/deploy/kubernetes/stores/$env/kustomization.yaml")"
[[ "$image" =~ @sha256:[0-9a-f]{64}$ ]] || { echo "no pinned postgres-tools image for $env" >&2; exit 2; }
pod="aevolution-barrier-$(date -u +%Y%m%d%H%M%S)"
manifest() {
cat <<YAML
apiVersion: v1
kind: Pod
metadata:
name: $pod
namespace: $ns
labels:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: migrate
spec:
serviceAccountName: aevolution-migrator
automountServiceAccountToken: false
restartPolicy: Never
activeDeadlineSeconds: 120
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile: {type: RuntimeDefault}
containers:
- name: barrier
image: $image
command: ["/bin/bash", "-c"]
args:
- |
set -eu
umask 077
cp /run/postgres/pgpass /tmp/pgpass
export PGPASSFILE=/tmp/pgpass
psql -X --no-password -At -v ON_ERROR_STOP=1 \\
-c "SELECT 'fenced ' || evolution.upgrade_barrier(decode('$retire', 'hex'))" \\
-c "SELECT 'retired ' || encode(build_identity, 'hex') FROM evolution.retained_builds WHERE retired_at IS NOT NULL ORDER BY 1"
env:
- {name: PGSERVICE, value: evolution}
- {name: PGSERVICEFILE, value: /run/postgres/pg_service.conf}
resources:
requests: {cpu: 50m, memory: 64Mi}
limits: {memory: 128Mi}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
volumeMounts:
- {name: connection, mountPath: /run/postgres, readOnly: true}
- {name: scratch, mountPath: /tmp}
volumes:
- name: connection
secret:
secretName: evolution-postgres-migrator
defaultMode: 0440
items:
- {key: pg_service.conf, path: pg_service.conf}
- {key: pgpass, path: pgpass}
- {key: ca.crt, path: ca.crt}
- name: scratch
emptyDir: {medium: Memory, sizeLimit: 1Mi}
YAML
}
if [ "$print" = 1 ]; then manifest; exit 0; fi
say() { echo "$(date -u +%H:%M:%S) $*"; }
ready_pod() { # The single Ready evo-region pod, or failure (none, several, or unreadable).
"${k[@]}" get pods -n "$ns" -l app.kubernetes.io/name=evo-region -o jsonpath="$POD_STATUS_JSONPATH" |
single_ready_pod
}
serving() { # The single Ready pod's running image digest.
local name
name="$(ready_pod 2> /dev/null)" || return 1
"${k[@]}" get pod -n "$ns" "$name" -o jsonpath='{.status.containerStatuses[0].imageID}'
}
ready_pod > /dev/null || { echo "refused: no single Ready evo-region pod" >&2; exit 3; }
[[ "$(serving)" == *"@sha256:$current" ]] ||
{ echo "refused: the Ready evo-region pod does not run sha256:$current (deploy the new build first)" >&2; exit 3; }
cleanup() { "${k[@]}" delete pod -n "$ns" "$pod" --wait=false > /dev/null 2>&1 || true; }
trap cleanup EXIT
say "retiring $retire in $ns (serving sha256:$current)"
manifest | "${k[@]}" apply -f - > /dev/null
start="$(date +%s%3N)"
phase=""
for _ in $(seq 1 120); do
phase="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}' 2> /dev/null || true)"
{ [ "$phase" = Succeeded ] || [ "$phase" = Failed ]; } && break
sleep 1
done
"${k[@]}" logs -n "$ns" "$pod" 2>&1 | sed 's/^/ /'
[ "$phase" = Succeeded ] || { say "FAIL barrier pod ${phase:-unknown}"; exit 3; }
# The fenced host loses readiness at its next lease check, then re-acquires.
not_ready_since=""
for _ in $(seq 1 600); do
if ready_pod > /dev/null 2>&1; then
[ -z "$not_ready_since" ] && [ $(( $(date +%s%3N) - start )) -lt 3000 ] && { sleep 0.5; continue; }
break
fi
[ -n "$not_ready_since" ] || not_ready_since="$(date +%s%3N)"
sleep 0.5
done
[[ "$(serving)" == *"@sha256:$current" ]] || { say "FAIL no Ready pod serves sha256:$current after the barrier"; exit 3; }
gap=0; [ -n "$not_ready_since" ] && gap=$(( $(date +%s%3N) - not_ready_since ))
say "PASS $retire retired in $ns; not-ready gap ${gap} ms; sha256:$current serving"