feat(ops): M8.9 upgrade rehearsal on clones, barrier and node plan (ADR-0076)

Drill-only --synthetic-until/--accelerate and evo-world-clone compare
(compare_region_histories) prove a candidate build runs the same world on
clones of one cut before any barrier. upgrade-check.sh rehearses a failed
migration under load and the version barrier with real hosts; upgrade-barrier.sh
retires an old build in dev/stage only once the new digest is Ready. Nodes:
aevolution.io/stores pins every objects-volume pod, aevolution.io/compute is
preference-only (Alertmanager first), checked by policy and scheduling-check.sh.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-30 17:51:17 -04:00
co-authored by Claude Opus 5.5
parent 645d16b27a
commit 5b482ad6a3
19 changed files with 1028 additions and 12 deletions
+28 -2
View File
@@ -18,7 +18,10 @@ environment-isolation or World0-protection invariant is broken:
- the cleanup tool refuses World0 before invoking kubectl;
- World0 launch stays blocked (ADR-0075): no workload targets World0 and its
Application keeps an unresolved path; any image a World0 overlay pins must be
the digest already pinned in stage (promotion, never a fresh build).
the digest already pinned in stage (promotion, never a fresh build);
- scheduling (ADR-0076): every pod that mounts the node-local objects volume
selects the store node; aevolution.io/compute is only ever preferred, never
required; upgrade-rehearsal flags never appear in a deployed workload.
"""
import os
import shutil
@@ -281,6 +284,28 @@ for directory, _, files in os.walk(os.path.join(ROOT, 'deploy', 'kubernetes')):
check((image.get('name'), image.get('digest')) in staged,
f'{rel}: {image.get("name")} must be promoted from {stage}, not pinned independently')
# M8.9 scheduling (ADR-0076). Node-local volumes pin their pods: added CPU does not
# make the objects volume portable. Compute is a preference so one node still works.
for path, doc in objects:
if doc['kind'] not in workload_kinds or path.endswith('region-values.yaml'):
continue
spec = doc.get('spec', {})
pod = spec.get('jobTemplate', {}).get('spec', {}).get('template', {}).get('spec') \
or spec.get('template', {}).get('spec') or spec
claims = [v.get('persistentVolumeClaim', {}).get('claimName') for v in pod.get('volumes', [])]
if 'aevolution-objects' in claims:
check(pod.get('nodeSelector', {}).get('aevolution.io/stores') == 'true',
f'{path}: a pod mounting aevolution-objects must select aevolution.io/stores')
required = (pod.get('affinity', {}).get('nodeAffinity', {})
.get('requiredDuringSchedulingIgnoredDuringExecution', {}))
check('aevolution.io/compute' not in yaml.safe_dump(required)
and 'aevolution.io/compute' not in pod.get('nodeSelector', {}),
f'{path}: aevolution.io/compute may only be preferred, never required')
for container in pod.get('containers', []):
text = ' '.join(container.get('args', []) + container.get('command', []))
check('--accelerate' not in text and '--synthetic-until' not in text,
f'{path}: upgrade-rehearsal flags never run in a deployed workload')
# The platform bundle: every rendered object is protected from prune and deletion.
bundle = manifests.get(platform + '/cluster/kustomization.yaml', [{}])[0]
check(bundle.get('commonAnnotations', {}).get('argocd.argoproj.io/sync-options') == 'Prune=false,Delete=false',
@@ -334,7 +359,8 @@ for path, doc in objects:
with tempfile.TemporaryDirectory() as empty:
env = dict(os.environ, PATH=empty + os.pathsep + os.path.dirname(sys.executable))
for tool in ('tools/ops/cleanup-environment.sh', 'tools/ops/region-drill.sh', 'tools/ops/fault-drill.sh',
'tools/backup/cluster-backup.sh'):
'tools/backup/cluster-backup.sh', 'tools/ops/scheduling-check.sh',
'tools/ops/upgrade-barrier.sh'):
for bad in ('world0', 'evolution-world0', 'World0', 'prod', ''):
result = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, tool), bad],
capture_output=True, text=True, env=env)
+2
View File
@@ -34,6 +34,8 @@ EVO_WORLD_CLONE_BIN="$(dirname "$1")/evo-world-clone"
bash "$root/tools/ci/backup-check.sh"
# M8.8: host-loss rehearsal: in-place restore, fenced old environment, measured rewind.
bash "$root/tools/ci/dr-check.sh"
# M8.9: failed migration under load and the version barrier with real hosts.
bash "$root/tools/ci/upgrade-check.sh"
v1_sha="$(sha256sum "$root/migrations/0001_region_bootstrap.sql")"
v1_sha="${v1_sha%% *}"
# Migration failure must not publish a partial schema.
+95
View File
@@ -0,0 +1,95 @@
#!/usr/bin/env bash
# M8.9 version-barrier rehearsal (ADR-0076). Run by postgres-check.sh inside its
# throwaway cluster (PG* set; EVO_SERVER_BIN, EVO_WORLD_CLONE_BIN, runtime roles
# exported). Real evo-server processes on world 500:
# 1. Failed migration: a migration that takes the migration lock, alters a history
# table, holds its locks for 3 s and then fails runs while build b1 serves.
# Nothing it did remains (schema still v9, no new column); the host keeps its
# authority, and every acknowledgement stays durable.
# 2. Barrier (ADR-0068 operator order): build b2 is retained, and a b2 host is started
# while b1 still holds the lease (it waits, never ready). upgrade_barrier(b1) fences
# b1, which quarantines (exit 4); b2 resumes at the committed head under a newer
# epoch. A b1 host started afterwards quarantines at once, and the history across
# both builds passes the full audit.
set -euo pipefail
root="$(cd "$(dirname "$0")/../.." && pwd)"
work="$(mktemp -d /tmp/evo-upgrade-check.XXXXXXXX)"
chmod 700 "$work"
pids=()
trap 'for p in "${pids[@]}"; do kill -9 "$p" 2> /dev/null || true; done; rm -rf -- "$work"' EXIT
fail() { echo "upgrade rehearsal FAILED: $*" >&2; exit 1; }
b1=b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1
b2=b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2
createdb evo_upgrade
export PGDATABASE=evo_upgrade
bash "$root/tools/migrate-postgres.sh" > /dev/null
q() { psql -XAt -v ON_ERROR_STOP=1 "$@"; }
mkdir -m 700 "$work/objects"
pg=(--pg-host "$PGHOST" --pg-port "$PGPORT" --pg-database evo_upgrade)
"$EVO_WORLD_CLONE_BIN" provision --world 500 --region 1 --opening development-v1 "${pg[@]}" \
--pg-user "$PGUSER" --objects "$work/objects" --build "$b1" --retain-build true > /dev/null
serve() { # serve <build> <log> [retained...]
local build="$1" log="$2"; shift 2
local retained=()
for r in "$@"; do retained+=(--retained "$r"); done
"$EVO_SERVER_BIN" --profile lineage --world 500 --region 1 --frame 9 "${pg[@]}" \
--pg-user "$EVO_PG_RUNTIME_USER" --objects "$work/objects" --build "$build" "${retained[@]}" \
--flush-ms 200 --lease-profile fast --synthetic-inputs development-v1 < /dev/null > "$log" 2>&1 &
pids+=($!)
server_pid=$!
}
acked_through() { awk -v t="$2" '$1 == "ack" && $3 >= t {f = 1} END {exit !f}' "$1"; }
max_ack() { awk '$1 == "ack" && $3 > m {m = $3} END {print m + 0}' "$1"; }
wait_ack() { for _ in $(seq 1 300); do acked_through "$1" "$2" && return 0; sleep 0.1; done; return 1; }
head_tick() { q -c 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 500'; }
serve "$b1" "$work/b1.log"; b1_pid=$server_pid
wait_ack "$work/b1.log" 30 || fail "b1 acknowledged nothing"
# 1. A failing migration while b1 serves.
before="$(max_ack "$work/b1.log")"
code=0
q -v ON_ERROR_STOP=1 > "$work/migration.log" 2>&1 <<'SQL' || code=$?
BEGIN;
SET LOCAL lock_timeout = '5s';
SELECT pg_catalog.pg_advisory_xact_lock(71420001);
ALTER TABLE evolution.region_commits ADD COLUMN rehearsal_column integer;
SELECT pg_catalog.pg_sleep(3);
SELECT 1 / 0; -- The migration fails after holding its locks.
COMMIT;
SQL
[ "$code" != 0 ] && grep -q 'division by zero' "$work/migration.log" || fail "the rehearsal migration did not fail"
[ "$(q -c "SELECT count(*) FROM information_schema.columns WHERE table_name = 'region_commits' AND column_name = 'rehearsal_column'")" = 0 ] ||
fail "the failed migration left a column"
[ "$(q -c 'SELECT max(version) FROM evolution.schema_migrations')" = 9 ] || fail "schema version moved"
wait_ack "$work/b1.log" $((before + 40)) || fail "b1 did not resume after the failed migration"
kill -0 "$b1_pid" 2> /dev/null || fail "b1 exited during the failed migration"
bash "$root/tools/migrate-postgres.sh" > /dev/null || fail "migrate after the failure"
# 2. The barrier: retain b2, start its host, then retire b1.
q -c "INSERT INTO evolution.retained_builds(build_identity) VALUES (decode('$b2', 'hex'))" > /dev/null
serve "$b2" "$work/b2.log" "$b1"; b2_pid=$server_pid
sleep 2
! grep -q '^ready 1' "$work/b2.log" || fail "b2 became ready while b1 held the lease"
[ "$(q -c "SELECT evolution.upgrade_barrier(decode('$b1', 'hex'))")" = 1 ] || fail "barrier fenced no lease"
code=0; wait "$b1_pid" || code=$?
[ "$code" = 4 ] || { tail "$work/b1.log" >&2; fail "b1 exited $code after the barrier, not 4"; }
b1_final="$(max_ack "$work/b1.log")"
wait_ack "$work/b2.log" $((b1_final + 30)) || { tail "$work/b2.log" >&2; fail "b2 did not take over"; }
resumed="$(sed -n 's/^phase running epoch=\([0-9]*\) committed=\([0-9]*\) .*/\1 \2/p' "$work/b2.log" | head -1)"
read -r epoch committed <<< "$resumed"
[ "$epoch" -ge 2 ] && [ "$committed" -ge "$b1_final" ] ||
fail "b2 resumed at epoch=$epoch committed=$committed; b1 acknowledged $b1_final"
[ "$(awk '$1 == "ack" {print $3; exit}' "$work/b2.log")" = $((committed + 1)) ] || fail "b2 skipped or repeated ticks"
serve "$b1" "$work/late.log"; late_pid=$server_pid
code=0; wait "$late_pid" || code=$?
[ "$code" = 4 ] && ! grep -q '^ready 1' "$work/late.log" || fail "a late b1 host exited $code or served"
kill -TERM "$b2_pid"; code=0; wait "$b2_pid" || code=$?
[ "$code" = 0 ] || fail "b2 exited $code"
audit="$("$EVO_WORLD_CLONE_BIN" audit --world 500 --region 1 "${pg[@]}" --pg-user "$PGUSER" \
--objects "$work/objects" --retained "$b1" --retained "$b2")" || fail "audit across builds"
[[ "$audit" == *" tick=$(head_tick) "* ]] || fail "audit: $audit"
builds="$(q -c 'SELECT count(DISTINCT build_identity) FROM evolution.region_commits WHERE world_id = 500')"
[ "$builds" = 2 ] || fail "history holds $builds builds, expected 2"
echo "upgrade rehearsal passed: failed migration held locks 3 s, b1 acknowledged $b1_final," \
"b2 resumed epoch=$epoch committed=$committed, head $(head_tick) ($audit)"
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env bash
# M8.9 read-only placement check (ADR-0076) for a disposable environment.
#
# tools/ops/scheduling-check.sh <dev|stage> [--context NAME]
#
# Verifies the node plan against the live cluster, changing nothing:
# - exactly one Ready worker carries aevolution.io/stores=true, and it is not a
# control-plane node;
# - the environment's aevolution-objects volume is pinned to that node (local
# volumes cannot follow a region to another node);
# - the evo-region pod runs there;
# - nodes labelled aevolution.io/compute=true are listed, with whether they can
# take work (Ready, schedulable). None is valid: compute is only a preference.
# Adding a node: label it aevolution.io/compute=true and re-run. Only stateless pods
# move to it; a region moves between nodes only through a backup restore or a clone
# onto that node's storage (ADR-0074/0075), never by rescheduling.
set -euo pipefail
usage() { echo "usage: $0 <dev|stage> [--context NAME]" >&2; exit 2; }
[ $# -ge 1 ] || usage
env="$1"; shift
case "$env" in
dev|stage) ;;
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
exit 2 ;;
esac
k=(kubectl --request-timeout=30s)
[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2")
ns="evolution-$env"
failures=0
ok() { echo " ok $*"; }
bad() { echo " FAIL $*"; failures=$((failures + 1)); }
ready='{range .status.conditions[?(@.type=="Ready")]}{.status}{end}'
echo "stores node"
stores="$("${k[@]}" get nodes -l aevolution.io/stores=true -o name | sed 's#^node/##')"
if [ "$(grep -c . <<< "$stores")" = 1 ]; then ok "one stores node: $stores"; else bad "stores nodes: '${stores//$'\n'/ }' (need exactly one)"; fi
store_node="$(head -1 <<< "$stores")"
if [ -n "$store_node" ]; then
[ "$("${k[@]}" get node "$store_node" -o jsonpath="$ready")" = True ] && ok "$store_node Ready" || bad "$store_node not Ready"
"${k[@]}" get node "$store_node" -o jsonpath='{.metadata.labels}' | grep -q 'node-role.kubernetes.io/control-plane' &&
bad "$store_node is a control-plane node" || ok "$store_node is a worker"
fi
echo "objects volume ($ns)"
pv="$("${k[@]}" get pvc -n "$ns" aevolution-objects -o jsonpath='{.spec.volumeName}' 2> /dev/null || true)"
if [ -z "$pv" ]; then
bad "aevolution-objects is not bound"
else
pinned="$("${k[@]}" get pv "$pv" -o jsonpath='{.spec.nodeAffinity.required.nodeSelectorTerms[*].matchExpressions[*].values[*]}')"
[ "$pinned" = "$store_node" ] && ok "$pv pinned to $pinned" || bad "$pv pinned to '$pinned', not the stores node '$store_node'"
fi
echo "region host"
placed="$("${k[@]}" get pods -n "$ns" -l app.kubernetes.io/name=evo-region -o jsonpath='{range .items[*]}{.metadata.name}={.spec.nodeName}{"\n"}{end}')"
if [ -z "$placed" ]; then bad "no evo-region pod"; fi
while IFS='=' read -r pod node; do
[ -n "$pod" ] || continue
[ "$node" = "$store_node" ] && ok "$pod on $node" || bad "$pod on '$node', not the stores node"
done <<< "$placed"
echo "compute nodes (preferred for stateless AEvolution pods)"
compute="$("${k[@]}" get nodes -l aevolution.io/compute=true -o name | sed 's#^node/##')"
if [ -z "$compute" ]; then
echo " none labelled: stateless pods schedule anywhere (single-node plan)"
else
while read -r node; do
state="$("${k[@]}" get node "$node" -o jsonpath="$ready")"
cordoned="$("${k[@]}" get node "$node" -o jsonpath='{.spec.unschedulable}')"
echo " $node Ready=$state schedulable=$([ "$cordoned" = true ] && echo no || echo yes)"
done <<< "$compute"
fi
placed_am="$("${k[@]}" get pods -n aevolution-monitoring -l app.kubernetes.io/name=alertmanager -o jsonpath='{.items[0].spec.nodeName}' 2> /dev/null || true)"
[ -n "$placed_am" ] && echo " alertmanager runs on $placed_am"
echo
[ "$failures" = 0 ] && echo "PASS scheduling plan in $ns" || { echo "FAIL $failures check(s) in $ns"; exit 1; }
+137
View File
@@ -0,0 +1,137 @@
#!/usr/bin/env bash
# M8.9 version barrier in a disposable environment (ADR-0068, ADR-0076).
#
# tools/ops/upgrade-barrier.sh <dev|stage> --retire HEX64 [--context NAME]
#
# The last step of the upgrade order: migrate -> retain the new build (the provision
# hook does) -> deploy it with the old build in REGION_RETAINED -> retire the old
# build here. Refused unless:
# - HEX64 is not the build Git currently deploys (deploy/kubernetes/region/<env>);
# - HEX64 is listed in REGION_RETAINED (its history must stay recoverable);
# - the evo-region pod is Ready and runs exactly the Git-pinned image, so the new
# build already serves.
# Then one short Pod (the postgres-tools image pinned for <env>, migrator login) runs
# evolution.upgrade_barrier(HEX64): the old build can never acquire, renew or commit
# again, and every lease is fenced. The new host re-acquires within one lease. The
# script waits for Ready again and prints the time without a ready host.
# Retirement is irreversible here. World0 is refused before kubectl runs.
set -euo pipefail
usage() { echo "usage: $0 <dev|stage> --retire HEX64 [--context NAME]" >&2; exit 2; }
[ $# -ge 1 ] || usage
env="$1"; shift
case "$env" in
dev|stage) ;;
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
exit 2 ;;
esac
retire="" k=(kubectl --request-timeout=30s)
while [ $# -gt 0 ]; do
case "$1" in
--retire) retire="${2:-}"; shift 2 ;;
--context) k+=(--context "${2:?}"); shift 2 ;;
*) usage ;;
esac
done
[[ "$retire" =~ ^[0-9a-f]{64}$ ]] || usage
root="$(cd "$(dirname "$0")/../.." && pwd)"
ns="evolution-$env"
values="$root/deploy/kubernetes/region/$env/region-values.yaml"
current="$(sed -n 's/.*name: REGION_BUILD, value: \([0-9a-f]\{64\}\)}.*/\1/p' "$values" | sort -u)"
retained="$(sed -n 's/.*name: REGION_RETAINED, value: "\([0-9a-f ]*\)"}.*/\1/p' "$values" | head -1)"
[ "$(wc -l <<< "$current")" = 1 ] || { echo "cannot read one REGION_BUILD from $values" >&2; exit 2; }
[ "$retire" != "$current" ] || { echo "refused: $retire is the build Git deploys" >&2; exit 2; }
grep -qw "$retire" <<< "$retained" || { echo "refused: $retire is not in REGION_RETAINED" >&2; exit 2; }
image="$(awk '/name: aevolution-postgres-tools/ {f = 1} f && /newName:/ {n = $2} f && /digest:/ {print n "@" $2; exit}' \
"$root/deploy/kubernetes/stores/$env/kustomization.yaml")"
[[ "$image" =~ @sha256:[0-9a-f]{64}$ ]] || { echo "no pinned postgres-tools image for $env" >&2; exit 2; }
say() { echo "$(date -u +%H:%M:%S) $*"; }
ready_pod() {
"${k[@]}" get pods -n "$ns" -l app.kubernetes.io/name=evo-region -o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{range .status.conditions}}{{if and (eq .type "Ready") (eq .status "True")}}{{$.metadata.name}}{{end}}{{end}}{{end}}{{end}}' 2> /dev/null || true
}
serving() { # The Ready pod's running image digest.
local pod; pod="$(ready_pod)"
[ -n "$pod" ] && "${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.containerStatuses[0].imageID}'
}
[[ "$(serving)" == *"@sha256:$current" ]] ||
{ echo "refused: no Ready evo-region pod runs sha256:$current yet (deploy the new build first)" >&2; exit 3; }
pod="aevolution-barrier-$(date -u +%Y%m%d%H%M%S)"
cleanup() { "${k[@]}" delete pod -n "$ns" "$pod" --wait=false > /dev/null 2>&1 || true; }
trap cleanup EXIT
say "retiring $retire in $ns (serving sha256:$current)"
"${k[@]}" apply -n "$ns" -f - > /dev/null <<YAML
apiVersion: v1
kind: Pod
metadata:
name: $pod
labels:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: migrate
spec:
serviceAccountName: aevolution-migrator
automountServiceAccountToken: false
restartPolicy: Never
activeDeadlineSeconds: 120
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
seccompProfile: {type: RuntimeDefault}
containers:
- name: barrier
image: $image
command: ["/bin/bash", "-c"]
args:
- |
set -eu
umask 077
cp /run/postgres/pgpass /tmp/pgpass
export PGPASSFILE=/tmp/pgpass
psql -X --no-password -At -v ON_ERROR_STOP=1 \
-c "SELECT 'fenced ' || evolution.upgrade_barrier(decode('$retire', 'hex'))" \
-c "SELECT 'retired ' || encode(build_identity, 'hex') FROM evolution.retained_builds WHERE retired_at IS NOT NULL ORDER BY 1"
env:
- {name: PGSERVICE, value: evolution}
- {name: PGSERVICEFILE, value: /run/postgres/pg_service.conf}
resources:
requests: {cpu: 50m, memory: 64Mi}
limits: {memory: 128Mi}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
volumeMounts:
- {name: connection, mountPath: /run/postgres, readOnly: true}
- {name: scratch, mountPath: /tmp}
volumes:
- name: connection
secret:
secretName: evolution-postgres-migrator
defaultMode: 0440
items:
- {key: pg_service.conf, path: pg_service.conf}
- {key: pgpass, path: pgpass}
- {key: ca.crt, path: ca.crt}
- name: scratch
emptyDir: {medium: Memory, sizeLimit: 1Mi}
YAML
start="$(date +%s%3N)"
for _ in $(seq 1 120); do
phase="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}' 2> /dev/null || true)"
[ "$phase" = Succeeded ] || [ "$phase" = Failed ] && break
sleep 1
done
"${k[@]}" logs -n "$ns" "$pod" 2>&1 | sed 's/^/ /'
[ "$phase" = Succeeded ] || { say "FAIL barrier pod $phase"; exit 3; }
not_ready_since=""
for _ in $(seq 1 600); do
if [ -n "$(ready_pod)" ]; then
[ -z "$not_ready_since" ] && [ $(( $(date +%s%3N) - start )) -lt 3000 ] && { sleep 0.5; continue; }
break
fi
[ -n "$not_ready_since" ] || not_ready_since="$(date +%s%3N)"
sleep 0.5
done
[[ "$(serving)" == *"@sha256:$current" ]] || { say "FAIL no Ready pod serves sha256:$current after the barrier"; exit 3; }
gap=0; [ -n "$not_ready_since" ] && gap=$(( $(date +%s%3N) - not_ready_since ))
say "PASS $retire retired in $ns; not-ready gap ${gap} ms; sha256:$current serving"