Files
aevolution/tools/ops/verify-isolation.sh
dingoandClaude Opus 5.5 3a34ae39bc fix: M8 review findings (restore ownership, barrier tool, World0 claimRef, CI selection, drill verdicts)
P1: restore-world.sh claims its objects path (mkdir) and the database (a held
session with a restore try-lock and the exclusive upgrade lock), and removes only
what it created, so a losing restore never drops a winner's schema.
P2: barrier Ready-pod selection via shared jsonpath helper with fixtures; barrier
pod fsGroup; script-generated pods inspected by the policy check; World0 PV claim
references protected; ci-paths.sh uses the event's real base and runs on
uncertainty; fault-drill asserts RTO and needs positive quarantine evidence.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 22:53:11 -04:00

129 lines
6.3 KiB
Bash

#!/usr/bin/env bash
# M8.2 live isolation checks. Changes nothing: it only asks `kubectl auth can-i`
# (impersonating the environments' service accounts), reads objects, and sends
# DELETE requests with --dry-run=server, which run admission but never persist.
#
# tools/ops/verify-isolation.sh [--context <name>]
#
# Needs an identity that may impersonate service accounts and read namespaces,
# policies and Argo CD projects (a cluster administrator). Prints PASS/FAIL/SKIP per
# check and exits nonzero if any check fails.
set -uo pipefail
k=(kubectl --request-timeout=20s)
[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2")
fails=0
pass() { echo "PASS $*"; }
fail() { echo "FAIL $*"; fails=$((fails + 1)); }
skip() { echo "SKIP $*"; }
envs=(dev stage world0)
present=()
for env in "${envs[@]}"; do
if "${k[@]}" get namespace "evolution-$env" > /dev/null 2>&1; then
present+=("$env")
else
skip "namespace evolution-$env does not exist"
fi
done
# 1. Guardrails and labels in every present environment.
for env in "${present[@]}"; do
ns="evolution-$env"
got="$("${k[@]}" get namespace "$ns" -o jsonpath='{.metadata.labels.aevolution\.io/environment} {.metadata.labels.pod-security\.kubernetes\.io/enforce} {.metadata.labels.aevolution\.io/protected}')"
want="$env restricted"; [ "$env" = world0 ] && want="$env restricted true"
[ "$(echo "$got" | xargs)" = "$want" ] && pass "$ns labels ($got)" || fail "$ns labels '$got' (want '$want')"
for object in resourcequota/aevolution-quota limitrange/aevolution-defaults \
networkpolicy/aevolution-default-deny networkpolicy/aevolution-allow-dns \
serviceaccount/aevolution-migrator serviceaccount/aevolution-region serviceaccount/aevolution-backup; do
"${k[@]}" get -n "$ns" "$object" > /dev/null 2>&1 && pass "$ns has $object" || fail "$ns lacks $object"
done
done
# 2. No environment identity can read secrets or change anything anywhere.
for env in "${present[@]}"; do
for sa in aevolution-migrator aevolution-region aevolution-backup; do
who="system:serviceaccount:evolution-$env:$sa"
for target in "${envs[@]}"; do
for action in "get secrets" "list secrets" "create pods" "delete persistentvolumeclaims" "create rolebindings"; do
answer="$("${k[@]}" auth can-i $action -n "evolution-$target" --as "$who" 2>/dev/null)"
[ "$answer" = "no" ] && pass "$who cannot $action in evolution-$target" ||
fail "$who can $action in evolution-$target"
done
done
answer="$("${k[@]}" auth can-i delete namespaces --as "$who" 2>/dev/null)"
[ "$answer" = "no" ] && pass "$who cannot delete namespaces" || fail "$who can delete namespaces"
done
done
# 3. World0 admission protection, exercised with server-side dry runs.
if "${k[@]}" get validatingadmissionpolicybinding aevolution-world0-protection > /dev/null 2>&1; then
pass "world0 protection policy is bound"
else
fail "world0 protection policy binding missing"
fi
denied() { # denied <description> <kubectl delete args...> (always --dry-run=server)
local what="$1"; shift
local out
out="$("${k[@]}" delete --dry-run=server --wait=false "$@" 2>&1)"
if echo "$out" | grep -q "aevolution:world0-break-glass"; then
pass "denied: $what"
elif echo "$out" | grep -qi "not found"; then
skip "$what (object absent)"
else
fail "not denied: $what -> $out"
fi
}
if printf '%s\n' "${present[@]}" | grep -qx world0; then
denied "delete namespace evolution-world0" namespace evolution-world0
denied "delete service account in evolution-world0" -n evolution-world0 serviceaccount aevolution-region
secret="$("${k[@]}" get secrets -n evolution-world0 -o name 2>/dev/null | head -1)"
[ -n "$secret" ] && denied "delete $secret in evolution-world0" -n evolution-world0 "$secret" ||
skip "no Secret in evolution-world0 to test"
pvc="$("${k[@]}" get pvc -n evolution-world0 -o name 2>/dev/null | head -1)"
[ -n "$pvc" ] && denied "delete $pvc in evolution-world0" -n evolution-world0 "$pvc" ||
skip "no PVC in evolution-world0 to test"
# Clearing a World0 volume's claim reference must be denied too (it would strip
# the protection before a delete). Server-side dry run: never persisted.
pv="$("${k[@]}" get pv -o jsonpath='{range .items[?(@.spec.claimRef.namespace=="evolution-world0")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | head -1)"
if [ -n "$pv" ]; then
out="$("${k[@]}" patch pv "$pv" --dry-run=server --type json \
-p '[{"op":"remove","path":"/spec/claimRef"}]' 2>&1 || true)"
if echo "$out" | grep -q "aevolution:world0-break-glass"; then
pass "clearing the claim reference of World0 volume $pv is denied"
else
fail "clearing the claim reference of World0 volume $pv was not denied: $out"
fi
else
skip "no PersistentVolume bound to evolution-world0 to test"
fi
else
skip "world0 admission dry runs (namespace absent)"
fi
# 4. Argo CD projects: each AEvolution project targets only its own namespace.
for env in "${envs[@]}"; do
dest="$("${k[@]}" get appproject "aevolution-$env" -n argocd -o jsonpath='{range .spec.destinations[*]}{.namespace}{" "}{end}' 2>/dev/null)" || {
skip "appproject aevolution-$env not present"; continue; }
cluster="$("${k[@]}" get appproject "aevolution-$env" -n argocd -o jsonpath='{.spec.clusterResourceWhitelist}' 2>/dev/null)"
[ "$(echo "$dest" | xargs)" = "evolution-$env" ] && pass "appproject aevolution-$env destinations '$dest'" ||
fail "appproject aevolution-$env destinations '$dest'"
# An empty whitelist prints as nothing or as [] depending on how it was stored.
case "$cluster" in ""|"[]") pass "appproject aevolution-$env allows no cluster kinds" ;;
*) fail "appproject aevolution-$env cluster kinds '$cluster'" ;; esac
done
wide="$("${k[@]}" get appproject default -n argocd -o jsonpath='{.spec.destinations[*].namespace}' 2>/dev/null)"
[ "$wide" = "*" ] && echo "WARN appproject default allows every namespace; its applications could create objects in evolution-* (deletion is still denied in world0 by admission)"
# 5. The cleanup tool refuses World0 before touching the cluster.
root="$(cd "$(dirname "$0")/../.." && pwd)"
for bad in world0 evolution-world0 World0 ""; do
if bash "$root/tools/ops/cleanup-environment.sh" "$bad" > /dev/null 2>&1; then
fail "cleanup accepted '$bad'"
else
pass "cleanup refused '$bad'"
fi
done
echo "failures=$fails"
[ "$fails" -eq 0 ]