P1: restore-world.sh claims its objects path (mkdir) and the database (a held session with a restore try-lock and the exclusive upgrade lock), and removes only what it created, so a losing restore never drops a winner's schema. P2: barrier Ready-pod selection via shared jsonpath helper with fixtures; barrier pod fsGroup; script-generated pods inspected by the policy check; World0 PV claim references protected; ci-paths.sh uses the event's real base and runs on uncertainty; fault-drill asserts RTO and needs positive quarantine evidence. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
129 lines
6.3 KiB
Bash
129 lines
6.3 KiB
Bash
#!/usr/bin/env bash
|
|
# M8.2 live isolation checks. Changes nothing: it only asks `kubectl auth can-i`
|
|
# (impersonating the environments' service accounts), reads objects, and sends
|
|
# DELETE requests with --dry-run=server, which run admission but never persist.
|
|
#
|
|
# tools/ops/verify-isolation.sh [--context <name>]
|
|
#
|
|
# Needs an identity that may impersonate service accounts and read namespaces,
|
|
# policies and Argo CD projects (a cluster administrator). Prints PASS/FAIL/SKIP per
|
|
# check and exits nonzero if any check fails.
|
|
set -uo pipefail
|
|
k=(kubectl --request-timeout=20s)
|
|
[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2")
|
|
fails=0
|
|
pass() { echo "PASS $*"; }
|
|
fail() { echo "FAIL $*"; fails=$((fails + 1)); }
|
|
skip() { echo "SKIP $*"; }
|
|
|
|
envs=(dev stage world0)
|
|
present=()
|
|
for env in "${envs[@]}"; do
|
|
if "${k[@]}" get namespace "evolution-$env" > /dev/null 2>&1; then
|
|
present+=("$env")
|
|
else
|
|
skip "namespace evolution-$env does not exist"
|
|
fi
|
|
done
|
|
|
|
# 1. Guardrails and labels in every present environment.
|
|
for env in "${present[@]}"; do
|
|
ns="evolution-$env"
|
|
got="$("${k[@]}" get namespace "$ns" -o jsonpath='{.metadata.labels.aevolution\.io/environment} {.metadata.labels.pod-security\.kubernetes\.io/enforce} {.metadata.labels.aevolution\.io/protected}')"
|
|
want="$env restricted"; [ "$env" = world0 ] && want="$env restricted true"
|
|
[ "$(echo "$got" | xargs)" = "$want" ] && pass "$ns labels ($got)" || fail "$ns labels '$got' (want '$want')"
|
|
for object in resourcequota/aevolution-quota limitrange/aevolution-defaults \
|
|
networkpolicy/aevolution-default-deny networkpolicy/aevolution-allow-dns \
|
|
serviceaccount/aevolution-migrator serviceaccount/aevolution-region serviceaccount/aevolution-backup; do
|
|
"${k[@]}" get -n "$ns" "$object" > /dev/null 2>&1 && pass "$ns has $object" || fail "$ns lacks $object"
|
|
done
|
|
done
|
|
|
|
# 2. No environment identity can read secrets or change anything anywhere.
|
|
for env in "${present[@]}"; do
|
|
for sa in aevolution-migrator aevolution-region aevolution-backup; do
|
|
who="system:serviceaccount:evolution-$env:$sa"
|
|
for target in "${envs[@]}"; do
|
|
for action in "get secrets" "list secrets" "create pods" "delete persistentvolumeclaims" "create rolebindings"; do
|
|
answer="$("${k[@]}" auth can-i $action -n "evolution-$target" --as "$who" 2>/dev/null)"
|
|
[ "$answer" = "no" ] && pass "$who cannot $action in evolution-$target" ||
|
|
fail "$who can $action in evolution-$target"
|
|
done
|
|
done
|
|
answer="$("${k[@]}" auth can-i delete namespaces --as "$who" 2>/dev/null)"
|
|
[ "$answer" = "no" ] && pass "$who cannot delete namespaces" || fail "$who can delete namespaces"
|
|
done
|
|
done
|
|
|
|
# 3. World0 admission protection, exercised with server-side dry runs.
|
|
if "${k[@]}" get validatingadmissionpolicybinding aevolution-world0-protection > /dev/null 2>&1; then
|
|
pass "world0 protection policy is bound"
|
|
else
|
|
fail "world0 protection policy binding missing"
|
|
fi
|
|
denied() { # denied <description> <kubectl delete args...> (always --dry-run=server)
|
|
local what="$1"; shift
|
|
local out
|
|
out="$("${k[@]}" delete --dry-run=server --wait=false "$@" 2>&1)"
|
|
if echo "$out" | grep -q "aevolution:world0-break-glass"; then
|
|
pass "denied: $what"
|
|
elif echo "$out" | grep -qi "not found"; then
|
|
skip "$what (object absent)"
|
|
else
|
|
fail "not denied: $what -> $out"
|
|
fi
|
|
}
|
|
if printf '%s\n' "${present[@]}" | grep -qx world0; then
|
|
denied "delete namespace evolution-world0" namespace evolution-world0
|
|
denied "delete service account in evolution-world0" -n evolution-world0 serviceaccount aevolution-region
|
|
secret="$("${k[@]}" get secrets -n evolution-world0 -o name 2>/dev/null | head -1)"
|
|
[ -n "$secret" ] && denied "delete $secret in evolution-world0" -n evolution-world0 "$secret" ||
|
|
skip "no Secret in evolution-world0 to test"
|
|
pvc="$("${k[@]}" get pvc -n evolution-world0 -o name 2>/dev/null | head -1)"
|
|
[ -n "$pvc" ] && denied "delete $pvc in evolution-world0" -n evolution-world0 "$pvc" ||
|
|
skip "no PVC in evolution-world0 to test"
|
|
# Clearing a World0 volume's claim reference must be denied too (it would strip
|
|
# the protection before a delete). Server-side dry run: never persisted.
|
|
pv="$("${k[@]}" get pv -o jsonpath='{range .items[?(@.spec.claimRef.namespace=="evolution-world0")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | head -1)"
|
|
if [ -n "$pv" ]; then
|
|
out="$("${k[@]}" patch pv "$pv" --dry-run=server --type json \
|
|
-p '[{"op":"remove","path":"/spec/claimRef"}]' 2>&1 || true)"
|
|
if echo "$out" | grep -q "aevolution:world0-break-glass"; then
|
|
pass "clearing the claim reference of World0 volume $pv is denied"
|
|
else
|
|
fail "clearing the claim reference of World0 volume $pv was not denied: $out"
|
|
fi
|
|
else
|
|
skip "no PersistentVolume bound to evolution-world0 to test"
|
|
fi
|
|
else
|
|
skip "world0 admission dry runs (namespace absent)"
|
|
fi
|
|
|
|
# 4. Argo CD projects: each AEvolution project targets only its own namespace.
|
|
for env in "${envs[@]}"; do
|
|
dest="$("${k[@]}" get appproject "aevolution-$env" -n argocd -o jsonpath='{range .spec.destinations[*]}{.namespace}{" "}{end}' 2>/dev/null)" || {
|
|
skip "appproject aevolution-$env not present"; continue; }
|
|
cluster="$("${k[@]}" get appproject "aevolution-$env" -n argocd -o jsonpath='{.spec.clusterResourceWhitelist}' 2>/dev/null)"
|
|
[ "$(echo "$dest" | xargs)" = "evolution-$env" ] && pass "appproject aevolution-$env destinations '$dest'" ||
|
|
fail "appproject aevolution-$env destinations '$dest'"
|
|
# An empty whitelist prints as nothing or as [] depending on how it was stored.
|
|
case "$cluster" in ""|"[]") pass "appproject aevolution-$env allows no cluster kinds" ;;
|
|
*) fail "appproject aevolution-$env cluster kinds '$cluster'" ;; esac
|
|
done
|
|
wide="$("${k[@]}" get appproject default -n argocd -o jsonpath='{.spec.destinations[*].namespace}' 2>/dev/null)"
|
|
[ "$wide" = "*" ] && echo "WARN appproject default allows every namespace; its applications could create objects in evolution-* (deletion is still denied in world0 by admission)"
|
|
|
|
# 5. The cleanup tool refuses World0 before touching the cluster.
|
|
root="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
for bad in world0 evolution-world0 World0 ""; do
|
|
if bash "$root/tools/ops/cleanup-environment.sh" "$bad" > /dev/null 2>&1; then
|
|
fail "cleanup accepted '$bad'"
|
|
else
|
|
pass "cleanup refused '$bad'"
|
|
fi
|
|
done
|
|
|
|
echo "failures=$fails"
|
|
[ "$fails" -eq 0 ]
|