#!/usr/bin/env bash # M9.7 dev session drill (ADR-0083) for evolution-dev only. # # tools/ops/session-drill.sh [--context ] [--subject ] # # Requires: kubectl access to evolution-dev, an `evo-dev-session` binary (EVO_DEV_SESSION, # default: on PATH) and an operator credentials directory made by # evo-dev-session credentials --environment dev --server-name --out # whose server.pem/server.key/grant.key are already in the `evolution-dev-session` # Secret (docs/networking/DEV_SESSION_OPERATIONS.md). The grant key never leaves the # operator machine; the drill issues a short-lived single-use grant bundle locally. # # Steps: enable synthetic inputs and dev sessions on evo-region (`kubectl set env`), # wait for Ready, port-forward 127.0.0.1: to the pod's loopback listener, run # `evo-dev-session probe --expect-restart`: one durable no-effect acknowledgement, # then a forced pod deletion (the "server pod restart"), reconnection at a newer epoch # with a fresh baseline, and a second durable acknowledgement. Everything is reverted # on exit (including on failure) to the Git values, so Argo returns to Synced. # Stage and World0 are refused before anything touches the cluster. set -euo pipefail usage() { echo "usage: $0 [--context ] [--subject ]" >&2; exit 2; } [ $# -ge 1 ] || usage creds="$1"; shift k=(kubectl --request-timeout=30s) subject=7 while [ $# -gt 0 ]; do case "$1" in --context) [ -n "${2:-}" ] || usage; k+=(--context "$2"); shift 2 ;; --subject) [ -n "${2:-}" ] || usage; subject="$2"; shift 2 ;; *) usage ;; esac done ns=evolution-dev # Fixed: the dev identity exists nowhere else. tool="${EVO_DEV_SESSION:-evo-dev-session}" for f in dev-ca.pem server.pem grant.key; do [ -f "$creds/$f" ] || { echo "missing $creds/$f" >&2; exit 2; } done command -v "$tool" > /dev/null || { echo "evo-dev-session not found (set EVO_DEV_SESSION)" >&2; exit 2; } selector=app.kubernetes.io/name=evo-region work="$(mktemp -d)" chmod 700 "$work" forward_pid="" say() { echo "$(date -u +%H:%M:%S) $*"; } fail() { say "FAIL $*"; exit 1; } "${k[@]}" get secret -n "$ns" evolution-dev-session > /dev/null || fail "Secret evolution-dev-session is missing in $ns (see DEV_SESSION_OPERATIONS.md)" server_name="$(openssl x509 -noout -subject -nameopt RFC2253 -in "$creds/server.pem" 2> /dev/null | sed -n 's/^subject=CN=//p')" [ -n "$server_name" ] || fail "cannot read the server name from $creds/server.pem" ready_pod() { # Waits up to $2 seconds for a Ready pod other than $1. local old="$1" limit="${2:-180}" for _ in $(seq 1 $((limit * 5))); do for p in $("${k[@]}" get pods -n "$ns" -l "$selector" \ -o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}'); do [ "$p" = "$old" ] && continue [ "$("${k[@]}" get pod -n "$ns" "$p" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2> /dev/null)" = True ] && { echo "$p"; return 0; } done sleep 0.2 done return 1 } forward() { # forward : local 17400 -> pod loopback 7400 (no Service involved). [ -n "$forward_pid" ] && kill "$forward_pid" 2> /dev/null || true "${k[@]}" port-forward -n "$ns" "pod/$1" 17400:7400 > "$work/forward-$1.log" 2>&1 & forward_pid=$! } restore() { set +e [ -n "$forward_pid" ] && kill "$forward_pid" 2> /dev/null say "restoring the Git configuration of evo-region" "${k[@]}" set env -n "$ns" deployment/evo-region REGION_INPUTS= REGION_SESSION= > /dev/null ready_pod "" 180 > /dev/null && say "restored: evo-region Ready" || say "WARN evo-region not Ready after restore" rm -f "$work/grants.bin" say "logs kept in $work" } trap restore EXIT world="$("${k[@]}" get deployment -n "$ns" evo-region -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="REGION_WORLD")].value}')" region="$("${k[@]}" get deployment -n "$ns" evo-region -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="REGION_ID")].value}')" frame="$("${k[@]}" get deployment -n "$ns" evo-region -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="REGION_FRAME")].value}')" "$tool" grants --environment dev --key "$creds/grant.key" --world "$world" --frame "$frame" \ --region "$region" --subject "$subject" --count 8 --lifetime 1800 --out "$work/grants.bin" > /dev/null || fail "could not issue the grant bundle" say "enabling synthetic inputs and dev sessions in $ns (world $world region $region)" before="$("${k[@]}" get pods -n "$ns" -l "$selector" -o jsonpath='{.items[0].metadata.name}')" "${k[@]}" set env -n "$ns" deployment/evo-region REGION_INPUTS=development-v1 REGION_SESSION=dev > /dev/null pod="$(ready_pod "$before" 180)" || fail "no Ready evo-region pod with sessions" "${k[@]}" logs -n "$ns" -f "$pod" > "$work/$pod.log" 2> /dev/null & forward "$pod" sleep 2 "$tool" probe --connect 127.0.0.1:17400 --server-name "$server_name" --ca "$creds/dev-ca.pem" \ --grants "$work/grants.bin" --expect-restart --timeout 300 > "$work/probe.log" 2>&1 & probe_pid=$! for _ in $(seq 1 600); do grep -q "probe awaiting-restart" "$work/probe.log" && break kill -0 "$probe_pid" 2> /dev/null || break; sleep 0.2; done grep -q "probe awaiting-restart" "$work/probe.log" || { cat "$work/probe.log"; fail "first acknowledgement not observed"; } grep "^probe durable" "$work/probe.log" | head -1 say "deleting $pod (server pod restart)" started=$(date +%s%3N) "${k[@]}" delete pod -n "$ns" "$pod" --wait=false > /dev/null successor="$(ready_pod "$pod" 180)" || fail "no successor became Ready" say "successor $successor Ready after $(( $(date +%s%3N) - started )) ms" "${k[@]}" logs -n "$ns" -f "$successor" > "$work/$successor.log" 2> /dev/null & forward "$successor" wait "$probe_pid" && verdict=0 || verdict=$? cat "$work/probe.log" [ "$verdict" = 0 ] || fail "probe failed (exit $verdict)" grep -q "^session opened" "$work/$successor.log" || say "WARN successor log shows no session (log stream lag?)" say "PASS session drill: durable acknowledgement before and after a pod restart, new epoch, fresh baseline"