#!/usr/bin/env bash # M8.9 version barrier in a disposable environment (ADR-0068, ADR-0076). # # tools/ops/upgrade-barrier.sh --retire HEX64 [--context NAME] [--print-manifest] # # The last step of the upgrade order: migrate -> retain the new build (the provision # hook does) -> deploy it with the old build in REGION_RETAINED -> retire the old # build here. Refused unless: # - HEX64 is not the build Git currently deploys (deploy/kubernetes/region/); # - HEX64 is listed in REGION_RETAINED (its history must stay recoverable); # - exactly one evo-region pod is Ready (and not being deleted), and it runs exactly # the Git-pinned image, so the new build already serves. # Then one short Pod (the postgres-tools image pinned for , migrator login, the # same fsGroup Secret-access contract as the other database-tool pods) runs # evolution.upgrade_barrier(HEX64): the old build can never acquire, renew or commit # again, and every lease is fenced. The new host re-acquires within one lease. The # script waits for Ready again and prints the time without a ready host. # --print-manifest prints that Pod after the offline checks and exits without # touching the cluster (the deploy policy check inspects it). # Retirement is irreversible here. World0 is refused before kubectl runs. set -euo pipefail usage() { echo "usage: $0 --retire HEX64 [--context NAME] [--print-manifest]" >&2; exit 2; } [ $# -ge 1 ] || usage env="$1"; shift case "$env" in dev|stage) ;; *) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2 exit 2 ;; esac retire="" print=0 k=(kubectl --request-timeout=30s) while [ $# -gt 0 ]; do case "$1" in --retire) retire="${2:-}"; shift 2 ;; --context) k+=(--context "${2:?}"); shift 2 ;; --print-manifest) print=1; shift ;; *) usage ;; esac done [[ "$retire" =~ ^[0-9a-f]{64}$ ]] || usage root="$(cd "$(dirname "$0")/../.." && pwd)" # shellcheck source=lib/pods.sh . "$root/tools/ops/lib/pods.sh" ns="evolution-$env" values="$root/deploy/kubernetes/region/$env/region-values.yaml" current="$(sed -n 's/.*name: REGION_BUILD, value: \([0-9a-f]\{64\}\)}.*/\1/p' "$values" | sort -u)" retained="$(sed -n 's/.*name: REGION_RETAINED, value: "\([0-9a-f ]*\)"}.*/\1/p' "$values" | head -1)" [ "$(wc -l <<< "$current")" = 1 ] || { echo "cannot read one REGION_BUILD from $values" >&2; exit 2; } [ "$retire" != "$current" ] || { echo "refused: $retire is the build Git deploys" >&2; exit 2; } grep -qw "$retire" <<< "$retained" || { echo "refused: $retire is not in REGION_RETAINED" >&2; exit 2; } image="$(awk '/name: aevolution-postgres-tools/ {f = 1} f && /newName:/ {n = $2} f && /digest:/ {print n "@" $2; exit}' \ "$root/deploy/kubernetes/stores/$env/kustomization.yaml")" [[ "$image" =~ @sha256:[0-9a-f]{64}$ ]] || { echo "no pinned postgres-tools image for $env" >&2; exit 2; } pod="aevolution-barrier-$(date -u +%Y%m%d%H%M%S)" manifest() { cat < /dev/null || { echo "refused: no single Ready evo-region pod" >&2; exit 3; } [[ "$(serving)" == *"@sha256:$current" ]] || { echo "refused: the Ready evo-region pod does not run sha256:$current (deploy the new build first)" >&2; exit 3; } cleanup() { "${k[@]}" delete pod -n "$ns" "$pod" --wait=false > /dev/null 2>&1 || true; } trap cleanup EXIT say "retiring $retire in $ns (serving sha256:$current)" manifest | "${k[@]}" apply -f - > /dev/null start="$(date +%s%3N)" phase="" for _ in $(seq 1 120); do phase="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}' 2> /dev/null || true)" { [ "$phase" = Succeeded ] || [ "$phase" = Failed ]; } && break sleep 1 done "${k[@]}" logs -n "$ns" "$pod" 2>&1 | sed 's/^/ /' [ "$phase" = Succeeded ] || { say "FAIL barrier pod ${phase:-unknown}"; exit 3; } # The fenced host loses readiness at its next lease check, then re-acquires. not_ready_since="" for _ in $(seq 1 600); do if ready_pod > /dev/null 2>&1; then [ -z "$not_ready_since" ] && [ $(( $(date +%s%3N) - start )) -lt 3000 ] && { sleep 0.5; continue; } break fi [ -n "$not_ready_since" ] || not_ready_since="$(date +%s%3N)" sleep 0.5 done [[ "$(serving)" == *"@sha256:$current" ]] || { say "FAIL no Ready pod serves sha256:$current after the barrier"; exit 3; } gap=0; [ -n "$not_ready_since" ] && gap=$(( $(date +%s%3N) - not_ready_since )) say "PASS $retire retired in $ns; not-ready gap ${gap} ms; sha256:$current serving"