feat(ops): M8.9 upgrade rehearsal on clones, barrier and node plan (ADR-0076)

Drill-only --synthetic-until/--accelerate and evo-world-clone compare
(compare_region_histories) prove a candidate build runs the same world on
clones of one cut before any barrier. upgrade-check.sh rehearses a failed
migration under load and the version barrier with real hosts; upgrade-barrier.sh
retires an old build in dev/stage only once the new digest is Ready. Nodes:
aevolution.io/stores pins every objects-volume pod, aevolution.io/compute is
preference-only (Alertmanager first), checked by policy and scheduling-check.sh.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-30 17:51:17 -04:00
co-authored by Claude Opus 5.5
parent 645d16b27a
commit 5b482ad6a3
19 changed files with 1028 additions and 12 deletions
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env bash
# M8.9 read-only placement check (ADR-0076) for a disposable environment.
#
# tools/ops/scheduling-check.sh <dev|stage> [--context NAME]
#
# Verifies the node plan against the live cluster, changing nothing:
# - exactly one Ready worker carries aevolution.io/stores=true, and it is not a
# control-plane node;
# - the environment's aevolution-objects volume is pinned to that node (local
# volumes cannot follow a region to another node);
# - the evo-region pod runs there;
# - nodes labelled aevolution.io/compute=true are listed, with whether they can
# take work (Ready, schedulable). None is valid: compute is only a preference.
# Adding a node: label it aevolution.io/compute=true and re-run. Only stateless pods
# move to it; a region moves between nodes only through a backup restore or a clone
# onto that node's storage (ADR-0074/0075), never by rescheduling.
set -euo pipefail
usage() { echo "usage: $0 <dev|stage> [--context NAME]" >&2; exit 2; }
[ $# -ge 1 ] || usage
env="$1"; shift
case "$env" in
dev|stage) ;;
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
exit 2 ;;
esac
k=(kubectl --request-timeout=30s)
[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2")
ns="evolution-$env"
failures=0
ok() { echo " ok $*"; }
bad() { echo " FAIL $*"; failures=$((failures + 1)); }
ready='{range .status.conditions[?(@.type=="Ready")]}{.status}{end}'
echo "stores node"
stores="$("${k[@]}" get nodes -l aevolution.io/stores=true -o name | sed 's#^node/##')"
if [ "$(grep -c . <<< "$stores")" = 1 ]; then ok "one stores node: $stores"; else bad "stores nodes: '${stores//$'\n'/ }' (need exactly one)"; fi
store_node="$(head -1 <<< "$stores")"
if [ -n "$store_node" ]; then
[ "$("${k[@]}" get node "$store_node" -o jsonpath="$ready")" = True ] && ok "$store_node Ready" || bad "$store_node not Ready"
"${k[@]}" get node "$store_node" -o jsonpath='{.metadata.labels}' | grep -q 'node-role.kubernetes.io/control-plane' &&
bad "$store_node is a control-plane node" || ok "$store_node is a worker"
fi
echo "objects volume ($ns)"
pv="$("${k[@]}" get pvc -n "$ns" aevolution-objects -o jsonpath='{.spec.volumeName}' 2> /dev/null || true)"
if [ -z "$pv" ]; then
bad "aevolution-objects is not bound"
else
pinned="$("${k[@]}" get pv "$pv" -o jsonpath='{.spec.nodeAffinity.required.nodeSelectorTerms[*].matchExpressions[*].values[*]}')"
[ "$pinned" = "$store_node" ] && ok "$pv pinned to $pinned" || bad "$pv pinned to '$pinned', not the stores node '$store_node'"
fi
echo "region host"
placed="$("${k[@]}" get pods -n "$ns" -l app.kubernetes.io/name=evo-region -o jsonpath='{range .items[*]}{.metadata.name}={.spec.nodeName}{"\n"}{end}')"
if [ -z "$placed" ]; then bad "no evo-region pod"; fi
while IFS='=' read -r pod node; do
[ -n "$pod" ] || continue
[ "$node" = "$store_node" ] && ok "$pod on $node" || bad "$pod on '$node', not the stores node"
done <<< "$placed"
echo "compute nodes (preferred for stateless AEvolution pods)"
compute="$("${k[@]}" get nodes -l aevolution.io/compute=true -o name | sed 's#^node/##')"
if [ -z "$compute" ]; then
echo " none labelled: stateless pods schedule anywhere (single-node plan)"
else
while read -r node; do
state="$("${k[@]}" get node "$node" -o jsonpath="$ready")"
cordoned="$("${k[@]}" get node "$node" -o jsonpath='{.spec.unschedulable}')"
echo " $node Ready=$state schedulable=$([ "$cordoned" = true ] && echo no || echo yes)"
done <<< "$compute"
fi
placed_am="$("${k[@]}" get pods -n aevolution-monitoring -l app.kubernetes.io/name=alertmanager -o jsonpath='{.items[0].spec.nodeName}' 2> /dev/null || true)"
[ -n "$placed_am" ] && echo " alertmanager runs on $placed_am"
echo
[ "$failures" = 0 ] && echo "PASS scheduling plan in $ns" || { echo "FAIL $failures check(s) in $ns"; exit 1; }
+137
View File
@@ -0,0 +1,137 @@
#!/usr/bin/env bash
# M8.9 version barrier in a disposable environment (ADR-0068, ADR-0076).
#
# tools/ops/upgrade-barrier.sh <dev|stage> --retire HEX64 [--context NAME]
#
# The last step of the upgrade order: migrate -> retain the new build (the provision
# hook does) -> deploy it with the old build in REGION_RETAINED -> retire the old
# build here. Refused unless:
# - HEX64 is not the build Git currently deploys (deploy/kubernetes/region/<env>);
# - HEX64 is listed in REGION_RETAINED (its history must stay recoverable);
# - the evo-region pod is Ready and runs exactly the Git-pinned image, so the new
# build already serves.
# Then one short Pod (the postgres-tools image pinned for <env>, migrator login) runs
# evolution.upgrade_barrier(HEX64): the old build can never acquire, renew or commit
# again, and every lease is fenced. The new host re-acquires within one lease. The
# script waits for Ready again and prints the time without a ready host.
# Retirement is irreversible here. World0 is refused before kubectl runs.
set -euo pipefail
usage() { echo "usage: $0 <dev|stage> --retire HEX64 [--context NAME]" >&2; exit 2; }
[ $# -ge 1 ] || usage
env="$1"; shift
case "$env" in
dev|stage) ;;
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
exit 2 ;;
esac
retire="" k=(kubectl --request-timeout=30s)
while [ $# -gt 0 ]; do
case "$1" in
--retire) retire="${2:-}"; shift 2 ;;
--context) k+=(--context "${2:?}"); shift 2 ;;
*) usage ;;
esac
done
[[ "$retire" =~ ^[0-9a-f]{64}$ ]] || usage
root="$(cd "$(dirname "$0")/../.." && pwd)"
ns="evolution-$env"
values="$root/deploy/kubernetes/region/$env/region-values.yaml"
current="$(sed -n 's/.*name: REGION_BUILD, value: \([0-9a-f]\{64\}\)}.*/\1/p' "$values" | sort -u)"
retained="$(sed -n 's/.*name: REGION_RETAINED, value: "\([0-9a-f ]*\)"}.*/\1/p' "$values" | head -1)"
[ "$(wc -l <<< "$current")" = 1 ] || { echo "cannot read one REGION_BUILD from $values" >&2; exit 2; }
[ "$retire" != "$current" ] || { echo "refused: $retire is the build Git deploys" >&2; exit 2; }
grep -qw "$retire" <<< "$retained" || { echo "refused: $retire is not in REGION_RETAINED" >&2; exit 2; }
image="$(awk '/name: aevolution-postgres-tools/ {f = 1} f && /newName:/ {n = $2} f && /digest:/ {print n "@" $2; exit}' \
"$root/deploy/kubernetes/stores/$env/kustomization.yaml")"
[[ "$image" =~ @sha256:[0-9a-f]{64}$ ]] || { echo "no pinned postgres-tools image for $env" >&2; exit 2; }
say() { echo "$(date -u +%H:%M:%S) $*"; }
ready_pod() {
"${k[@]}" get pods -n "$ns" -l app.kubernetes.io/name=evo-region -o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{range .status.conditions}}{{if and (eq .type "Ready") (eq .status "True")}}{{$.metadata.name}}{{end}}{{end}}{{end}}{{end}}' 2> /dev/null || true
}
serving() { # The Ready pod's running image digest.
local pod; pod="$(ready_pod)"
[ -n "$pod" ] && "${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.containerStatuses[0].imageID}'
}
[[ "$(serving)" == *"@sha256:$current" ]] ||
{ echo "refused: no Ready evo-region pod runs sha256:$current yet (deploy the new build first)" >&2; exit 3; }
pod="aevolution-barrier-$(date -u +%Y%m%d%H%M%S)"
cleanup() { "${k[@]}" delete pod -n "$ns" "$pod" --wait=false > /dev/null 2>&1 || true; }
trap cleanup EXIT
say "retiring $retire in $ns (serving sha256:$current)"
"${k[@]}" apply -n "$ns" -f - > /dev/null <<YAML
apiVersion: v1
kind: Pod
metadata:
name: $pod
labels:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: migrate
spec:
serviceAccountName: aevolution-migrator
automountServiceAccountToken: false
restartPolicy: Never
activeDeadlineSeconds: 120
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
seccompProfile: {type: RuntimeDefault}
containers:
- name: barrier
image: $image
command: ["/bin/bash", "-c"]
args:
- |
set -eu
umask 077
cp /run/postgres/pgpass /tmp/pgpass
export PGPASSFILE=/tmp/pgpass
psql -X --no-password -At -v ON_ERROR_STOP=1 \
-c "SELECT 'fenced ' || evolution.upgrade_barrier(decode('$retire', 'hex'))" \
-c "SELECT 'retired ' || encode(build_identity, 'hex') FROM evolution.retained_builds WHERE retired_at IS NOT NULL ORDER BY 1"
env:
- {name: PGSERVICE, value: evolution}
- {name: PGSERVICEFILE, value: /run/postgres/pg_service.conf}
resources:
requests: {cpu: 50m, memory: 64Mi}
limits: {memory: 128Mi}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
volumeMounts:
- {name: connection, mountPath: /run/postgres, readOnly: true}
- {name: scratch, mountPath: /tmp}
volumes:
- name: connection
secret:
secretName: evolution-postgres-migrator
defaultMode: 0440
items:
- {key: pg_service.conf, path: pg_service.conf}
- {key: pgpass, path: pgpass}
- {key: ca.crt, path: ca.crt}
- name: scratch
emptyDir: {medium: Memory, sizeLimit: 1Mi}
YAML
start="$(date +%s%3N)"
for _ in $(seq 1 120); do
phase="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}' 2> /dev/null || true)"
[ "$phase" = Succeeded ] || [ "$phase" = Failed ] && break
sleep 1
done
"${k[@]}" logs -n "$ns" "$pod" 2>&1 | sed 's/^/ /'
[ "$phase" = Succeeded ] || { say "FAIL barrier pod $phase"; exit 3; }
not_ready_since=""
for _ in $(seq 1 600); do
if [ -n "$(ready_pod)" ]; then
[ -z "$not_ready_since" ] && [ $(( $(date +%s%3N) - start )) -lt 3000 ] && { sleep 0.5; continue; }
break
fi
[ -n "$not_ready_since" ] || not_ready_since="$(date +%s%3N)"
sleep 0.5
done
[[ "$(serving)" == *"@sha256:$current" ]] || { say "FAIL no Ready pod serves sha256:$current after the barrier"; exit 3; }
gap=0; [ -n "$not_ready_since" ] && gap=$(( $(date +%s%3N) - not_ready_since ))
say "PASS $retire retired in $ns; not-ready gap ${gap} ms; sha256:$current serving"