feat(ops): M8.9 upgrade rehearsal on clones, barrier and node plan (ADR-0076)
Drill-only --synthetic-until/--accelerate and evo-world-clone compare (compare_region_histories) prove a candidate build runs the same world on clones of one cut before any barrier. upgrade-check.sh rehearses a failed migration under load and the version barrier with real hosts; upgrade-barrier.sh retires an old build in dev/stage only once the new digest is Ready. Nodes: aevolution.io/stores pins every objects-volume pod, aevolution.io/compute is preference-only (Alertmanager first), checked by policy and scheduling-check.sh. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -18,7 +18,10 @@ environment-isolation or World0-protection invariant is broken:
|
||||
- the cleanup tool refuses World0 before invoking kubectl;
|
||||
- World0 launch stays blocked (ADR-0075): no workload targets World0 and its
|
||||
Application keeps an unresolved path; any image a World0 overlay pins must be
|
||||
the digest already pinned in stage (promotion, never a fresh build).
|
||||
the digest already pinned in stage (promotion, never a fresh build);
|
||||
- scheduling (ADR-0076): every pod that mounts the node-local objects volume
|
||||
selects the store node; aevolution.io/compute is only ever preferred, never
|
||||
required; upgrade-rehearsal flags never appear in a deployed workload.
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
@@ -281,6 +284,28 @@ for directory, _, files in os.walk(os.path.join(ROOT, 'deploy', 'kubernetes')):
|
||||
check((image.get('name'), image.get('digest')) in staged,
|
||||
f'{rel}: {image.get("name")} must be promoted from {stage}, not pinned independently')
|
||||
|
||||
# M8.9 scheduling (ADR-0076). Node-local volumes pin their pods: added CPU does not
|
||||
# make the objects volume portable. Compute is a preference so one node still works.
|
||||
for path, doc in objects:
|
||||
if doc['kind'] not in workload_kinds or path.endswith('region-values.yaml'):
|
||||
continue
|
||||
spec = doc.get('spec', {})
|
||||
pod = spec.get('jobTemplate', {}).get('spec', {}).get('template', {}).get('spec') \
|
||||
or spec.get('template', {}).get('spec') or spec
|
||||
claims = [v.get('persistentVolumeClaim', {}).get('claimName') for v in pod.get('volumes', [])]
|
||||
if 'aevolution-objects' in claims:
|
||||
check(pod.get('nodeSelector', {}).get('aevolution.io/stores') == 'true',
|
||||
f'{path}: a pod mounting aevolution-objects must select aevolution.io/stores')
|
||||
required = (pod.get('affinity', {}).get('nodeAffinity', {})
|
||||
.get('requiredDuringSchedulingIgnoredDuringExecution', {}))
|
||||
check('aevolution.io/compute' not in yaml.safe_dump(required)
|
||||
and 'aevolution.io/compute' not in pod.get('nodeSelector', {}),
|
||||
f'{path}: aevolution.io/compute may only be preferred, never required')
|
||||
for container in pod.get('containers', []):
|
||||
text = ' '.join(container.get('args', []) + container.get('command', []))
|
||||
check('--accelerate' not in text and '--synthetic-until' not in text,
|
||||
f'{path}: upgrade-rehearsal flags never run in a deployed workload')
|
||||
|
||||
# The platform bundle: every rendered object is protected from prune and deletion.
|
||||
bundle = manifests.get(platform + '/cluster/kustomization.yaml', [{}])[0]
|
||||
check(bundle.get('commonAnnotations', {}).get('argocd.argoproj.io/sync-options') == 'Prune=false,Delete=false',
|
||||
@@ -334,7 +359,8 @@ for path, doc in objects:
|
||||
with tempfile.TemporaryDirectory() as empty:
|
||||
env = dict(os.environ, PATH=empty + os.pathsep + os.path.dirname(sys.executable))
|
||||
for tool in ('tools/ops/cleanup-environment.sh', 'tools/ops/region-drill.sh', 'tools/ops/fault-drill.sh',
|
||||
'tools/backup/cluster-backup.sh'):
|
||||
'tools/backup/cluster-backup.sh', 'tools/ops/scheduling-check.sh',
|
||||
'tools/ops/upgrade-barrier.sh'):
|
||||
for bad in ('world0', 'evolution-world0', 'World0', 'prod', ''):
|
||||
result = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, tool), bad],
|
||||
capture_output=True, text=True, env=env)
|
||||
|
||||
@@ -34,6 +34,8 @@ EVO_WORLD_CLONE_BIN="$(dirname "$1")/evo-world-clone"
|
||||
bash "$root/tools/ci/backup-check.sh"
|
||||
# M8.8: host-loss rehearsal: in-place restore, fenced old environment, measured rewind.
|
||||
bash "$root/tools/ci/dr-check.sh"
|
||||
# M8.9: failed migration under load and the version barrier with real hosts.
|
||||
bash "$root/tools/ci/upgrade-check.sh"
|
||||
v1_sha="$(sha256sum "$root/migrations/0001_region_bootstrap.sql")"
|
||||
v1_sha="${v1_sha%% *}"
|
||||
# Migration failure must not publish a partial schema.
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env bash
|
||||
# M8.9 version-barrier rehearsal (ADR-0076). Run by postgres-check.sh inside its
|
||||
# throwaway cluster (PG* set; EVO_SERVER_BIN, EVO_WORLD_CLONE_BIN, runtime roles
|
||||
# exported). Real evo-server processes on world 500:
|
||||
# 1. Failed migration: a migration that takes the migration lock, alters a history
|
||||
# table, holds its locks for 3 s and then fails runs while build b1 serves.
|
||||
# Nothing it did remains (schema still v9, no new column); the host keeps its
|
||||
# authority, and every acknowledgement stays durable.
|
||||
# 2. Barrier (ADR-0068 operator order): build b2 is retained, and a b2 host is started
|
||||
# while b1 still holds the lease (it waits, never ready). upgrade_barrier(b1) fences
|
||||
# b1, which quarantines (exit 4); b2 resumes at the committed head under a newer
|
||||
# epoch. A b1 host started afterwards quarantines at once, and the history across
|
||||
# both builds passes the full audit.
|
||||
set -euo pipefail
|
||||
root="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
work="$(mktemp -d /tmp/evo-upgrade-check.XXXXXXXX)"
|
||||
chmod 700 "$work"
|
||||
pids=()
|
||||
trap 'for p in "${pids[@]}"; do kill -9 "$p" 2> /dev/null || true; done; rm -rf -- "$work"' EXIT
|
||||
fail() { echo "upgrade rehearsal FAILED: $*" >&2; exit 1; }
|
||||
b1=b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1
|
||||
b2=b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2
|
||||
createdb evo_upgrade
|
||||
export PGDATABASE=evo_upgrade
|
||||
bash "$root/tools/migrate-postgres.sh" > /dev/null
|
||||
q() { psql -XAt -v ON_ERROR_STOP=1 "$@"; }
|
||||
mkdir -m 700 "$work/objects"
|
||||
pg=(--pg-host "$PGHOST" --pg-port "$PGPORT" --pg-database evo_upgrade)
|
||||
"$EVO_WORLD_CLONE_BIN" provision --world 500 --region 1 --opening development-v1 "${pg[@]}" \
|
||||
--pg-user "$PGUSER" --objects "$work/objects" --build "$b1" --retain-build true > /dev/null
|
||||
serve() { # serve <build> <log> [retained...]
|
||||
local build="$1" log="$2"; shift 2
|
||||
local retained=()
|
||||
for r in "$@"; do retained+=(--retained "$r"); done
|
||||
"$EVO_SERVER_BIN" --profile lineage --world 500 --region 1 --frame 9 "${pg[@]}" \
|
||||
--pg-user "$EVO_PG_RUNTIME_USER" --objects "$work/objects" --build "$build" "${retained[@]}" \
|
||||
--flush-ms 200 --lease-profile fast --synthetic-inputs development-v1 < /dev/null > "$log" 2>&1 &
|
||||
pids+=($!)
|
||||
server_pid=$!
|
||||
}
|
||||
acked_through() { awk -v t="$2" '$1 == "ack" && $3 >= t {f = 1} END {exit !f}' "$1"; }
|
||||
max_ack() { awk '$1 == "ack" && $3 > m {m = $3} END {print m + 0}' "$1"; }
|
||||
wait_ack() { for _ in $(seq 1 300); do acked_through "$1" "$2" && return 0; sleep 0.1; done; return 1; }
|
||||
head_tick() { q -c 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 500'; }
|
||||
|
||||
serve "$b1" "$work/b1.log"; b1_pid=$server_pid
|
||||
wait_ack "$work/b1.log" 30 || fail "b1 acknowledged nothing"
|
||||
|
||||
# 1. A failing migration while b1 serves.
|
||||
before="$(max_ack "$work/b1.log")"
|
||||
code=0
|
||||
q -v ON_ERROR_STOP=1 > "$work/migration.log" 2>&1 <<'SQL' || code=$?
|
||||
BEGIN;
|
||||
SET LOCAL lock_timeout = '5s';
|
||||
SELECT pg_catalog.pg_advisory_xact_lock(71420001);
|
||||
ALTER TABLE evolution.region_commits ADD COLUMN rehearsal_column integer;
|
||||
SELECT pg_catalog.pg_sleep(3);
|
||||
SELECT 1 / 0; -- The migration fails after holding its locks.
|
||||
COMMIT;
|
||||
SQL
|
||||
[ "$code" != 0 ] && grep -q 'division by zero' "$work/migration.log" || fail "the rehearsal migration did not fail"
|
||||
[ "$(q -c "SELECT count(*) FROM information_schema.columns WHERE table_name = 'region_commits' AND column_name = 'rehearsal_column'")" = 0 ] ||
|
||||
fail "the failed migration left a column"
|
||||
[ "$(q -c 'SELECT max(version) FROM evolution.schema_migrations')" = 9 ] || fail "schema version moved"
|
||||
wait_ack "$work/b1.log" $((before + 40)) || fail "b1 did not resume after the failed migration"
|
||||
kill -0 "$b1_pid" 2> /dev/null || fail "b1 exited during the failed migration"
|
||||
bash "$root/tools/migrate-postgres.sh" > /dev/null || fail "migrate after the failure"
|
||||
|
||||
# 2. The barrier: retain b2, start its host, then retire b1.
|
||||
q -c "INSERT INTO evolution.retained_builds(build_identity) VALUES (decode('$b2', 'hex'))" > /dev/null
|
||||
serve "$b2" "$work/b2.log" "$b1"; b2_pid=$server_pid
|
||||
sleep 2
|
||||
! grep -q '^ready 1' "$work/b2.log" || fail "b2 became ready while b1 held the lease"
|
||||
[ "$(q -c "SELECT evolution.upgrade_barrier(decode('$b1', 'hex'))")" = 1 ] || fail "barrier fenced no lease"
|
||||
code=0; wait "$b1_pid" || code=$?
|
||||
[ "$code" = 4 ] || { tail "$work/b1.log" >&2; fail "b1 exited $code after the barrier, not 4"; }
|
||||
b1_final="$(max_ack "$work/b1.log")"
|
||||
wait_ack "$work/b2.log" $((b1_final + 30)) || { tail "$work/b2.log" >&2; fail "b2 did not take over"; }
|
||||
resumed="$(sed -n 's/^phase running epoch=\([0-9]*\) committed=\([0-9]*\) .*/\1 \2/p' "$work/b2.log" | head -1)"
|
||||
read -r epoch committed <<< "$resumed"
|
||||
[ "$epoch" -ge 2 ] && [ "$committed" -ge "$b1_final" ] ||
|
||||
fail "b2 resumed at epoch=$epoch committed=$committed; b1 acknowledged $b1_final"
|
||||
[ "$(awk '$1 == "ack" {print $3; exit}' "$work/b2.log")" = $((committed + 1)) ] || fail "b2 skipped or repeated ticks"
|
||||
serve "$b1" "$work/late.log"; late_pid=$server_pid
|
||||
code=0; wait "$late_pid" || code=$?
|
||||
[ "$code" = 4 ] && ! grep -q '^ready 1' "$work/late.log" || fail "a late b1 host exited $code or served"
|
||||
kill -TERM "$b2_pid"; code=0; wait "$b2_pid" || code=$?
|
||||
[ "$code" = 0 ] || fail "b2 exited $code"
|
||||
audit="$("$EVO_WORLD_CLONE_BIN" audit --world 500 --region 1 "${pg[@]}" --pg-user "$PGUSER" \
|
||||
--objects "$work/objects" --retained "$b1" --retained "$b2")" || fail "audit across builds"
|
||||
[[ "$audit" == *" tick=$(head_tick) "* ]] || fail "audit: $audit"
|
||||
builds="$(q -c 'SELECT count(DISTINCT build_identity) FROM evolution.region_commits WHERE world_id = 500')"
|
||||
[ "$builds" = 2 ] || fail "history holds $builds builds, expected 2"
|
||||
echo "upgrade rehearsal passed: failed migration held locks 3 s, b1 acknowledged $b1_final," \
|
||||
"b2 resumed epoch=$epoch committed=$committed, head $(head_tick) ($audit)"
|
||||
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env bash
|
||||
# M8.9 read-only placement check (ADR-0076) for a disposable environment.
|
||||
#
|
||||
# tools/ops/scheduling-check.sh <dev|stage> [--context NAME]
|
||||
#
|
||||
# Verifies the node plan against the live cluster, changing nothing:
|
||||
# - exactly one Ready worker carries aevolution.io/stores=true, and it is not a
|
||||
# control-plane node;
|
||||
# - the environment's aevolution-objects volume is pinned to that node (local
|
||||
# volumes cannot follow a region to another node);
|
||||
# - the evo-region pod runs there;
|
||||
# - nodes labelled aevolution.io/compute=true are listed, with whether they can
|
||||
# take work (Ready, schedulable). None is valid: compute is only a preference.
|
||||
# Adding a node: label it aevolution.io/compute=true and re-run. Only stateless pods
|
||||
# move to it; a region moves between nodes only through a backup restore or a clone
|
||||
# onto that node's storage (ADR-0074/0075), never by rescheduling.
|
||||
set -euo pipefail
|
||||
usage() { echo "usage: $0 <dev|stage> [--context NAME]" >&2; exit 2; }
|
||||
[ $# -ge 1 ] || usage
|
||||
env="$1"; shift
|
||||
case "$env" in
|
||||
dev|stage) ;;
|
||||
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
|
||||
exit 2 ;;
|
||||
esac
|
||||
k=(kubectl --request-timeout=30s)
|
||||
[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2")
|
||||
ns="evolution-$env"
|
||||
failures=0
|
||||
ok() { echo " ok $*"; }
|
||||
bad() { echo " FAIL $*"; failures=$((failures + 1)); }
|
||||
ready='{range .status.conditions[?(@.type=="Ready")]}{.status}{end}'
|
||||
|
||||
echo "stores node"
|
||||
stores="$("${k[@]}" get nodes -l aevolution.io/stores=true -o name | sed 's#^node/##')"
|
||||
if [ "$(grep -c . <<< "$stores")" = 1 ]; then ok "one stores node: $stores"; else bad "stores nodes: '${stores//$'\n'/ }' (need exactly one)"; fi
|
||||
store_node="$(head -1 <<< "$stores")"
|
||||
if [ -n "$store_node" ]; then
|
||||
[ "$("${k[@]}" get node "$store_node" -o jsonpath="$ready")" = True ] && ok "$store_node Ready" || bad "$store_node not Ready"
|
||||
"${k[@]}" get node "$store_node" -o jsonpath='{.metadata.labels}' | grep -q 'node-role.kubernetes.io/control-plane' &&
|
||||
bad "$store_node is a control-plane node" || ok "$store_node is a worker"
|
||||
fi
|
||||
|
||||
echo "objects volume ($ns)"
|
||||
pv="$("${k[@]}" get pvc -n "$ns" aevolution-objects -o jsonpath='{.spec.volumeName}' 2> /dev/null || true)"
|
||||
if [ -z "$pv" ]; then
|
||||
bad "aevolution-objects is not bound"
|
||||
else
|
||||
pinned="$("${k[@]}" get pv "$pv" -o jsonpath='{.spec.nodeAffinity.required.nodeSelectorTerms[*].matchExpressions[*].values[*]}')"
|
||||
[ "$pinned" = "$store_node" ] && ok "$pv pinned to $pinned" || bad "$pv pinned to '$pinned', not the stores node '$store_node'"
|
||||
fi
|
||||
|
||||
echo "region host"
|
||||
placed="$("${k[@]}" get pods -n "$ns" -l app.kubernetes.io/name=evo-region -o jsonpath='{range .items[*]}{.metadata.name}={.spec.nodeName}{"\n"}{end}')"
|
||||
if [ -z "$placed" ]; then bad "no evo-region pod"; fi
|
||||
while IFS='=' read -r pod node; do
|
||||
[ -n "$pod" ] || continue
|
||||
[ "$node" = "$store_node" ] && ok "$pod on $node" || bad "$pod on '$node', not the stores node"
|
||||
done <<< "$placed"
|
||||
|
||||
echo "compute nodes (preferred for stateless AEvolution pods)"
|
||||
compute="$("${k[@]}" get nodes -l aevolution.io/compute=true -o name | sed 's#^node/##')"
|
||||
if [ -z "$compute" ]; then
|
||||
echo " none labelled: stateless pods schedule anywhere (single-node plan)"
|
||||
else
|
||||
while read -r node; do
|
||||
state="$("${k[@]}" get node "$node" -o jsonpath="$ready")"
|
||||
cordoned="$("${k[@]}" get node "$node" -o jsonpath='{.spec.unschedulable}')"
|
||||
echo " $node Ready=$state schedulable=$([ "$cordoned" = true ] && echo no || echo yes)"
|
||||
done <<< "$compute"
|
||||
fi
|
||||
placed_am="$("${k[@]}" get pods -n aevolution-monitoring -l app.kubernetes.io/name=alertmanager -o jsonpath='{.items[0].spec.nodeName}' 2> /dev/null || true)"
|
||||
[ -n "$placed_am" ] && echo " alertmanager runs on $placed_am"
|
||||
echo
|
||||
[ "$failures" = 0 ] && echo "PASS scheduling plan in $ns" || { echo "FAIL $failures check(s) in $ns"; exit 1; }
|
||||
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env bash
|
||||
# M8.9 version barrier in a disposable environment (ADR-0068, ADR-0076).
|
||||
#
|
||||
# tools/ops/upgrade-barrier.sh <dev|stage> --retire HEX64 [--context NAME]
|
||||
#
|
||||
# The last step of the upgrade order: migrate -> retain the new build (the provision
|
||||
# hook does) -> deploy it with the old build in REGION_RETAINED -> retire the old
|
||||
# build here. Refused unless:
|
||||
# - HEX64 is not the build Git currently deploys (deploy/kubernetes/region/<env>);
|
||||
# - HEX64 is listed in REGION_RETAINED (its history must stay recoverable);
|
||||
# - the evo-region pod is Ready and runs exactly the Git-pinned image, so the new
|
||||
# build already serves.
|
||||
# Then one short Pod (the postgres-tools image pinned for <env>, migrator login) runs
|
||||
# evolution.upgrade_barrier(HEX64): the old build can never acquire, renew or commit
|
||||
# again, and every lease is fenced. The new host re-acquires within one lease. The
|
||||
# script waits for Ready again and prints the time without a ready host.
|
||||
# Retirement is irreversible here. World0 is refused before kubectl runs.
|
||||
set -euo pipefail
|
||||
usage() { echo "usage: $0 <dev|stage> --retire HEX64 [--context NAME]" >&2; exit 2; }
|
||||
[ $# -ge 1 ] || usage
|
||||
env="$1"; shift
|
||||
case "$env" in
|
||||
dev|stage) ;;
|
||||
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
|
||||
exit 2 ;;
|
||||
esac
|
||||
retire="" k=(kubectl --request-timeout=30s)
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--retire) retire="${2:-}"; shift 2 ;;
|
||||
--context) k+=(--context "${2:?}"); shift 2 ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
done
|
||||
[[ "$retire" =~ ^[0-9a-f]{64}$ ]] || usage
|
||||
root="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
ns="evolution-$env"
|
||||
values="$root/deploy/kubernetes/region/$env/region-values.yaml"
|
||||
current="$(sed -n 's/.*name: REGION_BUILD, value: \([0-9a-f]\{64\}\)}.*/\1/p' "$values" | sort -u)"
|
||||
retained="$(sed -n 's/.*name: REGION_RETAINED, value: "\([0-9a-f ]*\)"}.*/\1/p' "$values" | head -1)"
|
||||
[ "$(wc -l <<< "$current")" = 1 ] || { echo "cannot read one REGION_BUILD from $values" >&2; exit 2; }
|
||||
[ "$retire" != "$current" ] || { echo "refused: $retire is the build Git deploys" >&2; exit 2; }
|
||||
grep -qw "$retire" <<< "$retained" || { echo "refused: $retire is not in REGION_RETAINED" >&2; exit 2; }
|
||||
image="$(awk '/name: aevolution-postgres-tools/ {f = 1} f && /newName:/ {n = $2} f && /digest:/ {print n "@" $2; exit}' \
|
||||
"$root/deploy/kubernetes/stores/$env/kustomization.yaml")"
|
||||
[[ "$image" =~ @sha256:[0-9a-f]{64}$ ]] || { echo "no pinned postgres-tools image for $env" >&2; exit 2; }
|
||||
say() { echo "$(date -u +%H:%M:%S) $*"; }
|
||||
ready_pod() {
|
||||
"${k[@]}" get pods -n "$ns" -l app.kubernetes.io/name=evo-region -o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{range .status.conditions}}{{if and (eq .type "Ready") (eq .status "True")}}{{$.metadata.name}}{{end}}{{end}}{{end}}{{end}}' 2> /dev/null || true
|
||||
}
|
||||
serving() { # The Ready pod's running image digest.
|
||||
local pod; pod="$(ready_pod)"
|
||||
[ -n "$pod" ] && "${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.containerStatuses[0].imageID}'
|
||||
}
|
||||
[[ "$(serving)" == *"@sha256:$current" ]] ||
|
||||
{ echo "refused: no Ready evo-region pod runs sha256:$current yet (deploy the new build first)" >&2; exit 3; }
|
||||
|
||||
pod="aevolution-barrier-$(date -u +%Y%m%d%H%M%S)"
|
||||
cleanup() { "${k[@]}" delete pod -n "$ns" "$pod" --wait=false > /dev/null 2>&1 || true; }
|
||||
trap cleanup EXIT
|
||||
say "retiring $retire in $ns (serving sha256:$current)"
|
||||
"${k[@]}" apply -n "$ns" -f - > /dev/null <<YAML
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: $pod
|
||||
labels:
|
||||
app.kubernetes.io/part-of: aevolution
|
||||
app.kubernetes.io/component: migrate
|
||||
spec:
|
||||
serviceAccountName: aevolution-migrator
|
||||
automountServiceAccountToken: false
|
||||
restartPolicy: Never
|
||||
activeDeadlineSeconds: 120
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
containers:
|
||||
- name: barrier
|
||||
image: $image
|
||||
command: ["/bin/bash", "-c"]
|
||||
args:
|
||||
- |
|
||||
set -eu
|
||||
umask 077
|
||||
cp /run/postgres/pgpass /tmp/pgpass
|
||||
export PGPASSFILE=/tmp/pgpass
|
||||
psql -X --no-password -At -v ON_ERROR_STOP=1 \
|
||||
-c "SELECT 'fenced ' || evolution.upgrade_barrier(decode('$retire', 'hex'))" \
|
||||
-c "SELECT 'retired ' || encode(build_identity, 'hex') FROM evolution.retained_builds WHERE retired_at IS NOT NULL ORDER BY 1"
|
||||
env:
|
||||
- {name: PGSERVICE, value: evolution}
|
||||
- {name: PGSERVICEFILE, value: /run/postgres/pg_service.conf}
|
||||
resources:
|
||||
requests: {cpu: 50m, memory: 64Mi}
|
||||
limits: {memory: 128Mi}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities: {drop: [ALL]}
|
||||
volumeMounts:
|
||||
- {name: connection, mountPath: /run/postgres, readOnly: true}
|
||||
- {name: scratch, mountPath: /tmp}
|
||||
volumes:
|
||||
- name: connection
|
||||
secret:
|
||||
secretName: evolution-postgres-migrator
|
||||
defaultMode: 0440
|
||||
items:
|
||||
- {key: pg_service.conf, path: pg_service.conf}
|
||||
- {key: pgpass, path: pgpass}
|
||||
- {key: ca.crt, path: ca.crt}
|
||||
- name: scratch
|
||||
emptyDir: {medium: Memory, sizeLimit: 1Mi}
|
||||
YAML
|
||||
start="$(date +%s%3N)"
|
||||
for _ in $(seq 1 120); do
|
||||
phase="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}' 2> /dev/null || true)"
|
||||
[ "$phase" = Succeeded ] || [ "$phase" = Failed ] && break
|
||||
sleep 1
|
||||
done
|
||||
"${k[@]}" logs -n "$ns" "$pod" 2>&1 | sed 's/^/ /'
|
||||
[ "$phase" = Succeeded ] || { say "FAIL barrier pod $phase"; exit 3; }
|
||||
not_ready_since=""
|
||||
for _ in $(seq 1 600); do
|
||||
if [ -n "$(ready_pod)" ]; then
|
||||
[ -z "$not_ready_since" ] && [ $(( $(date +%s%3N) - start )) -lt 3000 ] && { sleep 0.5; continue; }
|
||||
break
|
||||
fi
|
||||
[ -n "$not_ready_since" ] || not_ready_since="$(date +%s%3N)"
|
||||
sleep 0.5
|
||||
done
|
||||
[[ "$(serving)" == *"@sha256:$current" ]] || { say "FAIL no Ready pod serves sha256:$current after the barrier"; exit 3; }
|
||||
gap=0; [ -n "$not_ready_since" ] && gap=$(( $(date +%s%3N) - not_ready_since ))
|
||||
say "PASS $retire retired in $ns; not-ready gap ${gap} ms; sha256:$current serving"
|
||||
Reference in New Issue
Block a user