feat(ops): M8.9 upgrade rehearsal on clones, barrier and node plan (ADR-0076)
Drill-only --synthetic-until/--accelerate and evo-world-clone compare (compare_region_histories) prove a candidate build runs the same world on clones of one cut before any barrier. upgrade-check.sh rehearses a failed migration under load and the version barrier with real hosts; upgrade-barrier.sh retires an old build in dev/stage only once the new digest is Ready. Nodes: aevolution.io/stores pins every objects-volume pod, aevolution.io/compute is preference-only (Alertmanager first), checked by policy and scheduling-check.sh. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env bash
|
||||
# M8.9 read-only placement check (ADR-0076) for a disposable environment.
|
||||
#
|
||||
# tools/ops/scheduling-check.sh <dev|stage> [--context NAME]
|
||||
#
|
||||
# Verifies the node plan against the live cluster, changing nothing:
|
||||
# - exactly one Ready worker carries aevolution.io/stores=true, and it is not a
|
||||
# control-plane node;
|
||||
# - the environment's aevolution-objects volume is pinned to that node (local
|
||||
# volumes cannot follow a region to another node);
|
||||
# - the evo-region pod runs there;
|
||||
# - nodes labelled aevolution.io/compute=true are listed, with whether they can
|
||||
# take work (Ready, schedulable). None is valid: compute is only a preference.
|
||||
# Adding a node: label it aevolution.io/compute=true and re-run. Only stateless pods
|
||||
# move to it; a region moves between nodes only through a backup restore or a clone
|
||||
# onto that node's storage (ADR-0074/0075), never by rescheduling.
|
||||
set -euo pipefail
|
||||
usage() { echo "usage: $0 <dev|stage> [--context NAME]" >&2; exit 2; }
|
||||
[ $# -ge 1 ] || usage
|
||||
env="$1"; shift
|
||||
case "$env" in
|
||||
dev|stage) ;;
|
||||
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
|
||||
exit 2 ;;
|
||||
esac
|
||||
k=(kubectl --request-timeout=30s)
|
||||
[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2")
|
||||
ns="evolution-$env"
|
||||
failures=0
|
||||
ok() { echo " ok $*"; }
|
||||
bad() { echo " FAIL $*"; failures=$((failures + 1)); }
|
||||
ready='{range .status.conditions[?(@.type=="Ready")]}{.status}{end}'
|
||||
|
||||
echo "stores node"
|
||||
stores="$("${k[@]}" get nodes -l aevolution.io/stores=true -o name | sed 's#^node/##')"
|
||||
if [ "$(grep -c . <<< "$stores")" = 1 ]; then ok "one stores node: $stores"; else bad "stores nodes: '${stores//$'\n'/ }' (need exactly one)"; fi
|
||||
store_node="$(head -1 <<< "$stores")"
|
||||
if [ -n "$store_node" ]; then
|
||||
[ "$("${k[@]}" get node "$store_node" -o jsonpath="$ready")" = True ] && ok "$store_node Ready" || bad "$store_node not Ready"
|
||||
"${k[@]}" get node "$store_node" -o jsonpath='{.metadata.labels}' | grep -q 'node-role.kubernetes.io/control-plane' &&
|
||||
bad "$store_node is a control-plane node" || ok "$store_node is a worker"
|
||||
fi
|
||||
|
||||
echo "objects volume ($ns)"
|
||||
pv="$("${k[@]}" get pvc -n "$ns" aevolution-objects -o jsonpath='{.spec.volumeName}' 2> /dev/null || true)"
|
||||
if [ -z "$pv" ]; then
|
||||
bad "aevolution-objects is not bound"
|
||||
else
|
||||
pinned="$("${k[@]}" get pv "$pv" -o jsonpath='{.spec.nodeAffinity.required.nodeSelectorTerms[*].matchExpressions[*].values[*]}')"
|
||||
[ "$pinned" = "$store_node" ] && ok "$pv pinned to $pinned" || bad "$pv pinned to '$pinned', not the stores node '$store_node'"
|
||||
fi
|
||||
|
||||
echo "region host"
|
||||
placed="$("${k[@]}" get pods -n "$ns" -l app.kubernetes.io/name=evo-region -o jsonpath='{range .items[*]}{.metadata.name}={.spec.nodeName}{"\n"}{end}')"
|
||||
if [ -z "$placed" ]; then bad "no evo-region pod"; fi
|
||||
while IFS='=' read -r pod node; do
|
||||
[ -n "$pod" ] || continue
|
||||
[ "$node" = "$store_node" ] && ok "$pod on $node" || bad "$pod on '$node', not the stores node"
|
||||
done <<< "$placed"
|
||||
|
||||
echo "compute nodes (preferred for stateless AEvolution pods)"
|
||||
compute="$("${k[@]}" get nodes -l aevolution.io/compute=true -o name | sed 's#^node/##')"
|
||||
if [ -z "$compute" ]; then
|
||||
echo " none labelled: stateless pods schedule anywhere (single-node plan)"
|
||||
else
|
||||
while read -r node; do
|
||||
state="$("${k[@]}" get node "$node" -o jsonpath="$ready")"
|
||||
cordoned="$("${k[@]}" get node "$node" -o jsonpath='{.spec.unschedulable}')"
|
||||
echo " $node Ready=$state schedulable=$([ "$cordoned" = true ] && echo no || echo yes)"
|
||||
done <<< "$compute"
|
||||
fi
|
||||
placed_am="$("${k[@]}" get pods -n aevolution-monitoring -l app.kubernetes.io/name=alertmanager -o jsonpath='{.items[0].spec.nodeName}' 2> /dev/null || true)"
|
||||
[ -n "$placed_am" ] && echo " alertmanager runs on $placed_am"
|
||||
echo
|
||||
[ "$failures" = 0 ] && echo "PASS scheduling plan in $ns" || { echo "FAIL $failures check(s) in $ns"; exit 1; }
|
||||
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env bash
|
||||
# M8.9 version barrier in a disposable environment (ADR-0068, ADR-0076).
|
||||
#
|
||||
# tools/ops/upgrade-barrier.sh <dev|stage> --retire HEX64 [--context NAME]
|
||||
#
|
||||
# The last step of the upgrade order: migrate -> retain the new build (the provision
|
||||
# hook does) -> deploy it with the old build in REGION_RETAINED -> retire the old
|
||||
# build here. Refused unless:
|
||||
# - HEX64 is not the build Git currently deploys (deploy/kubernetes/region/<env>);
|
||||
# - HEX64 is listed in REGION_RETAINED (its history must stay recoverable);
|
||||
# - the evo-region pod is Ready and runs exactly the Git-pinned image, so the new
|
||||
# build already serves.
|
||||
# Then one short Pod (the postgres-tools image pinned for <env>, migrator login) runs
|
||||
# evolution.upgrade_barrier(HEX64): the old build can never acquire, renew or commit
|
||||
# again, and every lease is fenced. The new host re-acquires within one lease. The
|
||||
# script waits for Ready again and prints the time without a ready host.
|
||||
# Retirement is irreversible here. World0 is refused before kubectl runs.
|
||||
set -euo pipefail
|
||||
usage() { echo "usage: $0 <dev|stage> --retire HEX64 [--context NAME]" >&2; exit 2; }
|
||||
[ $# -ge 1 ] || usage
|
||||
env="$1"; shift
|
||||
case "$env" in
|
||||
dev|stage) ;;
|
||||
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
|
||||
exit 2 ;;
|
||||
esac
|
||||
retire="" k=(kubectl --request-timeout=30s)
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--retire) retire="${2:-}"; shift 2 ;;
|
||||
--context) k+=(--context "${2:?}"); shift 2 ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
done
|
||||
[[ "$retire" =~ ^[0-9a-f]{64}$ ]] || usage
|
||||
root="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
ns="evolution-$env"
|
||||
values="$root/deploy/kubernetes/region/$env/region-values.yaml"
|
||||
current="$(sed -n 's/.*name: REGION_BUILD, value: \([0-9a-f]\{64\}\)}.*/\1/p' "$values" | sort -u)"
|
||||
retained="$(sed -n 's/.*name: REGION_RETAINED, value: "\([0-9a-f ]*\)"}.*/\1/p' "$values" | head -1)"
|
||||
[ "$(wc -l <<< "$current")" = 1 ] || { echo "cannot read one REGION_BUILD from $values" >&2; exit 2; }
|
||||
[ "$retire" != "$current" ] || { echo "refused: $retire is the build Git deploys" >&2; exit 2; }
|
||||
grep -qw "$retire" <<< "$retained" || { echo "refused: $retire is not in REGION_RETAINED" >&2; exit 2; }
|
||||
image="$(awk '/name: aevolution-postgres-tools/ {f = 1} f && /newName:/ {n = $2} f && /digest:/ {print n "@" $2; exit}' \
|
||||
"$root/deploy/kubernetes/stores/$env/kustomization.yaml")"
|
||||
[[ "$image" =~ @sha256:[0-9a-f]{64}$ ]] || { echo "no pinned postgres-tools image for $env" >&2; exit 2; }
|
||||
say() { echo "$(date -u +%H:%M:%S) $*"; }
|
||||
ready_pod() {
|
||||
"${k[@]}" get pods -n "$ns" -l app.kubernetes.io/name=evo-region -o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{range .status.conditions}}{{if and (eq .type "Ready") (eq .status "True")}}{{$.metadata.name}}{{end}}{{end}}{{end}}{{end}}' 2> /dev/null || true
|
||||
}
|
||||
serving() { # The Ready pod's running image digest.
|
||||
local pod; pod="$(ready_pod)"
|
||||
[ -n "$pod" ] && "${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.containerStatuses[0].imageID}'
|
||||
}
|
||||
[[ "$(serving)" == *"@sha256:$current" ]] ||
|
||||
{ echo "refused: no Ready evo-region pod runs sha256:$current yet (deploy the new build first)" >&2; exit 3; }
|
||||
|
||||
pod="aevolution-barrier-$(date -u +%Y%m%d%H%M%S)"
|
||||
cleanup() { "${k[@]}" delete pod -n "$ns" "$pod" --wait=false > /dev/null 2>&1 || true; }
|
||||
trap cleanup EXIT
|
||||
say "retiring $retire in $ns (serving sha256:$current)"
|
||||
"${k[@]}" apply -n "$ns" -f - > /dev/null <<YAML
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: $pod
|
||||
labels:
|
||||
app.kubernetes.io/part-of: aevolution
|
||||
app.kubernetes.io/component: migrate
|
||||
spec:
|
||||
serviceAccountName: aevolution-migrator
|
||||
automountServiceAccountToken: false
|
||||
restartPolicy: Never
|
||||
activeDeadlineSeconds: 120
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
containers:
|
||||
- name: barrier
|
||||
image: $image
|
||||
command: ["/bin/bash", "-c"]
|
||||
args:
|
||||
- |
|
||||
set -eu
|
||||
umask 077
|
||||
cp /run/postgres/pgpass /tmp/pgpass
|
||||
export PGPASSFILE=/tmp/pgpass
|
||||
psql -X --no-password -At -v ON_ERROR_STOP=1 \
|
||||
-c "SELECT 'fenced ' || evolution.upgrade_barrier(decode('$retire', 'hex'))" \
|
||||
-c "SELECT 'retired ' || encode(build_identity, 'hex') FROM evolution.retained_builds WHERE retired_at IS NOT NULL ORDER BY 1"
|
||||
env:
|
||||
- {name: PGSERVICE, value: evolution}
|
||||
- {name: PGSERVICEFILE, value: /run/postgres/pg_service.conf}
|
||||
resources:
|
||||
requests: {cpu: 50m, memory: 64Mi}
|
||||
limits: {memory: 128Mi}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities: {drop: [ALL]}
|
||||
volumeMounts:
|
||||
- {name: connection, mountPath: /run/postgres, readOnly: true}
|
||||
- {name: scratch, mountPath: /tmp}
|
||||
volumes:
|
||||
- name: connection
|
||||
secret:
|
||||
secretName: evolution-postgres-migrator
|
||||
defaultMode: 0440
|
||||
items:
|
||||
- {key: pg_service.conf, path: pg_service.conf}
|
||||
- {key: pgpass, path: pgpass}
|
||||
- {key: ca.crt, path: ca.crt}
|
||||
- name: scratch
|
||||
emptyDir: {medium: Memory, sizeLimit: 1Mi}
|
||||
YAML
|
||||
start="$(date +%s%3N)"
|
||||
for _ in $(seq 1 120); do
|
||||
phase="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}' 2> /dev/null || true)"
|
||||
[ "$phase" = Succeeded ] || [ "$phase" = Failed ] && break
|
||||
sleep 1
|
||||
done
|
||||
"${k[@]}" logs -n "$ns" "$pod" 2>&1 | sed 's/^/ /'
|
||||
[ "$phase" = Succeeded ] || { say "FAIL barrier pod $phase"; exit 3; }
|
||||
not_ready_since=""
|
||||
for _ in $(seq 1 600); do
|
||||
if [ -n "$(ready_pod)" ]; then
|
||||
[ -z "$not_ready_since" ] && [ $(( $(date +%s%3N) - start )) -lt 3000 ] && { sleep 0.5; continue; }
|
||||
break
|
||||
fi
|
||||
[ -n "$not_ready_since" ] || not_ready_since="$(date +%s%3N)"
|
||||
sleep 0.5
|
||||
done
|
||||
[[ "$(serving)" == *"@sha256:$current" ]] || { say "FAIL no Ready pod serves sha256:$current after the barrier"; exit 3; }
|
||||
gap=0; [ -n "$not_ready_since" ] && gap=$(( $(date +%s%3N) - not_ready_since ))
|
||||
say "PASS $retire retired in $ns; not-ready gap ${gap} ms; sha256:$current serving"
|
||||
Reference in New Issue
Block a user