fix: M8 review findings (restore ownership, barrier tool, World0 claimRef, CI selection, drill verdicts)

P1: restore-world.sh claims its objects path (mkdir) and the database (a held
session with a restore try-lock and the exclusive upgrade lock), and removes only
what it created, so a losing restore never drops a winner's schema.
P2: barrier Ready-pod selection via shared jsonpath helper with fixtures; barrier
pod fsGroup; script-generated pods inspected by the policy check; World0 PV claim
references protected; ci-paths.sh uses the event's real base and runs on
uncertainty; fault-drill asserts RTO and needs positive quarantine evidence.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-30 22:53:11 -04:00
co-authored by Claude Opus 5.5
parent 5b482ad6a3
commit 3a34ae39bc
21 changed files with 584 additions and 80 deletions
+8
View File
@@ -20,6 +20,11 @@ env:
HARBOR_USERNAME: ${{ secrets.HARBOR_USERNAME }} HARBOR_USERNAME: ${{ secrets.HARBOR_USERNAME }}
HARBOR_PASSWORD: ${{ secrets.HARBOR_PASSWORD }} HARBOR_PASSWORD: ${{ secrets.HARBOR_PASSWORD }}
CACHE_WRITE: '0' CACHE_WRITE: '0'
# Change selection (tools/ci/ci-paths.sh) compares against the event's real base:
# the push's before SHA or the pull request's base; manual runs always run.
CI_EVENT: ${{ github.event_name }}
CI_BEFORE: ${{ github.event.before }}
CI_PR_BASE: ${{ github.event.pull_request.base.sha }}
jobs: jobs:
checks: checks:
@@ -40,6 +45,9 @@ jobs:
CHECK_SUITE: ${{ matrix.suite }} CHECK_SUITE: ${{ matrix.suite }}
run: | run: |
set -eu set -eu
# The selector's own regression runs first, so a broken selector cannot
# silently skip the suites.
if [ "$CHECK_SUITE" = deploy ]; then bash tools/ci/ci-paths-check.sh; fi
if ! bash tools/ci/ci-paths.sh "$CHECK_SUITE"; then if ! bash tools/ci/ci-paths.sh "$CHECK_SUITE"; then
echo "No files relevant to $CHECK_SUITE changed; gate skipped." echo "No files relevant to $CHECK_SUITE changed; gate skipped."
exit 0 exit 0
+2 -1
View File
@@ -60,7 +60,8 @@ COPY deploy/ deploy/
COPY tools/ci/deploy-policy-check.py tools/ci/ COPY tools/ci/deploy-policy-check.py tools/ci/
COPY tools/ops/ tools/ops/ COPY tools/ops/ tools/ops/
COPY tools/backup/ tools/backup/ COPY tools/backup/ tools/backup/
RUN python3 tools/ci/deploy-policy-check.py COPY tools/ci/ops-lib-check.sh tools/ci/
RUN python3 tools/ci/deploy-policy-check.py && bash tools/ci/ops-lib-check.sh
# M8.5: monitoring configuration and alert tests with the pinned images' own tools. # M8.5: monitoring configuration and alert tests with the pinned images' own tools.
COPY --from=quay.io/prometheus/prometheus@sha256:6976aa8a60fec930796ce5772b8d12da7a318a5daa8d40d69c5c7819a05eeed7 /bin/promtool /usr/local/bin/promtool COPY --from=quay.io/prometheus/prometheus@sha256:6976aa8a60fec930796ce5772b8d12da7a318a5daa8d40d69c5c7819a05eeed7 /bin/promtool /usr/local/bin/promtool
COPY --from=quay.io/prometheus/alertmanager@sha256:e9733bafb1bdef9b00e25a21f8f99dc26a22224bf16641ad754d1649f4c3357a /bin/amtool /usr/local/bin/amtool COPY --from=quay.io/prometheus/alertmanager@sha256:e9733bafb1bdef9b00e25a21f8f99dc26a22224bf16641ad754d1649f4c3357a /bin/amtool /usr/local/bin/amtool
@@ -12,6 +12,10 @@
# - deleting a PersistentVolume bound to evolution-world0, or changing its reclaim # - deleting a PersistentVolume bound to evolution-world0, or changing its reclaim
# policy away from Retain once set (set Retain when the volume is bound: the # policy away from Retain once set (set Retain when the volume is bound: the
# only StorageClass, local-path, provisions with Delete); # only StorageClass, local-path, provisions with Delete);
# - removing or reassigning such a volume's claim reference (namespace or claim
# name): the reference is how this policy recognizes a World0 volume, so clearing
# it first would otherwise strip the protection (review M8 P2). The binder may
# still fill in uid/resourceVersion; reassignment is a break-glass operation;
# - deleting the aevolution-world0 or aevolution-platform Argo CD Application or # - deleting the aevolution-world0 or aevolution-platform Argo CD Application or
# AppProject (the platform app owns this policy and the World0 namespace). # AppProject (the platform app owns this policy and the World0 namespace).
# Kubernetes does not run admission policies on policy objects themselves, so # Kubernetes does not run admission policies on policy objects themselves, so
@@ -78,6 +82,14 @@ spec:
object.spec.persistentVolumeReclaimPolicy == 'Retain' object.spec.persistentVolumeReclaimPolicy == 'Retain'
message: "A World0 PersistentVolume set to Retain cannot be changed back" message: "A World0 PersistentVolume set to Retain cannot be changed back"
reason: Forbidden reason: Forbidden
- expression: >-
variables.breakGlass || request.operation != 'UPDATE' || !variables.world0Volume ||
(has(object.spec.claimRef) && has(object.spec.claimRef.namespace) &&
object.spec.claimRef.namespace == 'evolution-world0' &&
(!has(oldObject.spec.claimRef.name) ||
(has(object.spec.claimRef.name) && object.spec.claimRef.name == oldObject.spec.claimRef.name)))
message: "A World0 PersistentVolume keeps its World0 claim reference without the aevolution:world0-break-glass group"
reason: Forbidden
--- ---
apiVersion: admissionregistration.k8s.io/v1 apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingAdmissionPolicyBinding kind: ValidatingAdmissionPolicyBinding
+16 -2
View File
@@ -827,8 +827,9 @@ measurements (worker loss RPO 0 / RTO <= 60 s; host loss = explicit rewind to th
last verified bundle); **World0 launch stays blocked** because no independent last verified bundle); **World0 launch stays blocked** because no independent
durable failure domain exists, enforced by the deploy policy check; host-loss durable failure domain exists, enforced by the deploy policy check; host-loss
restore in place with barrier fencing of the superseded database rehearsed restore in place with barrier fencing of the superseded database rehearsed
locally; promotion only of stage-pinned digests. **M8-dev/stage passed; locally; promotion only of stage-pinned digests. **M8-World0 launch not passed**
M8-World0 launch not passed.** (blocked by design). M8-dev/stage is implemented and locally qualified, not yet
passed: see "M8 gate status" below.
M8.9 implemented and locally qualified (2026-09-30), under ADR-0076: accelerated M8.9 implemented and locally qualified (2026-09-30), under ADR-0076: accelerated
clone runs of the current and candidate builds compared state-for-state and clone runs of the current and candidate builds compared state-for-state and
@@ -837,3 +838,16 @@ version barrier rehearsed with real hosts; `upgrade-barrier.sh` for dev/stage;
stores/compute node labels with policy-checked placement and `scheduling-check.sh`. stores/compute node labels with policy-checked placement and `scheduling-check.sh`.
Rules changes stay refused until a converter ADR. In-cluster barrier and Rules changes stay refused until a converter ADR. In-cluster barrier and
additional-node evidence wait on the maintainer. additional-node evidence wait on the maintainer.
**M8 gate status (2026-09-30, after the M8.1–M8.9 review).** Implementation and
local qualification are complete; these live gates are still open, so M8-dev/stage
is **not passed**:
- M8.6: a valid dev database-outage run, and a re-run of full/empty with the new
image (positive quarantine evidence, RTO asserted against 60 s);
- M8.7: an in-cluster `cluster-backup.sh` bundle restore-tested off the host; the
destination, schedule, retention, encryption and WAL archiving (backlog);
- M8.5: an alert receiver and kubelet log-rotation check (backlog);
- M8.9: `upgrade-barrier.sh` on dev and `scheduling-check.sh` with a second labelled
node;
- M8.1/M8.3: storage re-measured after the RAID rework.
M8-World0 launch stays blocked by ADR-0075.
+8
View File
@@ -130,3 +130,11 @@ What stays outside Argo:
- Secrets, until a secret manager is chosen (docs/BACKLOG.md); - Secrets, until a secret manager is chosen (docs/BACKLOG.md);
- the store node label; - the store node label;
- the one-time bootstrap of the platform project and Application. - the one-time bootstrap of the platform project and Application.
## Amendment: World0 volume claim references (M8 review, 2026-09-30)
The policy recognizes a World0 PersistentVolume by its claim reference. Clearing
or reassigning that reference first would have removed the protection. An UPDATE
that removes the reference, or changes its namespace or claim name, is now denied
without the break-glass group. The binder may still fill in uid and
resourceVersion. `verify-isolation.sh` tests this with a server-side dry run when
a World0 volume exists.
+14
View File
@@ -90,3 +90,17 @@ PostgreSQL:
without a shared pipe that the image does not have. without a shared pipe that the image does not have.
- **Partitioning with NetworkPolicy.** It does not cut established connections - **Partitioning with NetworkPolicy.** It does not cut established connections
under Calico. under Calico.
## Amendment: drill verdicts (M8 review, 2026-09-30)
A missing observation no longer passes. The verdicts are in `tools/ops/lib/drill.sh`
and are fixture-tested by `tools/ci/ops-lib-check.sh`:
- Each kill must recover within `RTO_BUDGET_MS`, default 60 000 (ADR-0075).
- The empty scenario passes only on positive evidence: a `phase quarantined` line,
and exit 4 if the container ended. Any other exit fails. No evidence is
INCONCLUSIVE, which fails the drill.
- A dead or drained owner's acknowledgements are read from its log after the
stream has ended.
Limit: the drill reads acknowledgements from logs. A line that was written but not
streamed before a forced kill is not observed, so the zero-lost-acknowledgement
check covers observed acknowledgements only.
+19
View File
@@ -172,3 +172,22 @@ every PostgreSQL build:
authority. Isolation goes through the clone path, which re-identifies the world. authority. Isolation goes through the clone path, which re-identifies the world.
- **A new object-storage dependency (MinIO, S3 SDK).** The destination is not - **A new object-storage dependency (MinIO, S3 SDK).** The destination is not
chosen; a directory and `tar` need no new dependency. chosen; a directory and `tar` need no new dependency.
## Amendment: restore ownership (M8 review, 2026-09-30)
The review reproduced a race. A restore checked that the target was empty, a
competing actor then created the schema, the restore's `pg_restore` failed, and
its cleanup dropped the other actor's schema. Restores now own what they touch:
- `--objects` is claimed by creating it (`mkdir`, never `-p`). An existing path is
refused and never touched.
- One psql session, held open for the whole restore, takes the restore claim
(advisory 71420004, try-lock, so a concurrent restore refuses at once) and the
upgrade lock (71420002, exclusive). Provisioning, acquisition and lease checks in
that database wait until the restore is verified or removed.
- Cleanup removes only what the invocation created: the schema only if its own
one-transaction `pg_restore` committed, and the directory only if its own `mkdir`
created it.
`backup-check.sh` covers three cases:
- a competing schema created just before `pg_restore` survives;
- a concurrent restore refuses, and the first completes;
- a lease acquisition times out while the restore holds the database.
+12
View File
@@ -150,3 +150,15 @@ In the cluster, still to be run by the maintainer:
exactly the named risk. exactly the named risk.
- **Retiring the old build automatically on every sync.** Retirement is - **Retiring the old build automatically on every sync.** Retirement is
irreversible. It stays a deliberate step, guarded by the checks above. irreversible. It stays a deliberate step, guarded by the checks above.
## Amendment: barrier tool fixes (M8 review, 2026-09-30)
Two defects in `upgrade-barrier.sh` would have blocked the in-cluster barrier:
- Its Ready-pod template printed the list root instead of the pod name. Selection
now uses a plain jsonpath (`tools/ops/lib/pods.sh`) and requires exactly one
Ready pod that is not being deleted. Fixtures cover one pod, not Ready, being
deleted, no condition and two Ready pods.
- Its pod lacked `fsGroup`, so it could not read the 0440 Secret files. The pod
now follows the same contract as the other database-tool pods.
Both script-generated pods (backup and barrier) render with `--print-manifest`,
and the deploy policy check inspects them.
+53
View File
@@ -3034,3 +3034,56 @@ Fixes (this segment):
- scheduling onto a second labelled node; - scheduling onto a second labelled node;
- the Windows MSVC build (no Windows target changed; the server and PostgreSQL - the Windows MSVC build (no Windows target changed; the server and PostgreSQL
code are Linux-only). code are Linux-only).
## M8.1–M8.9 review fixes - 2026-09-30
Review: `.cache/reviews/M8_REVIEW_680e7e5-5b482ad.md` (one P1, five P2).
- **P1 restore ownership.** `backup-check.sh` passed in the Release, analysis and
ASan/UBSan PostgreSQL builds, including three new cases:
- A schema created after the checks but before `pg_restore`: exit 4 ("pg_restore
failed; the database was not changed"). The competing table survived, and only
this restore's own objects directory was removed.
- A second concurrent restore of the same database: exit 4 ("another restore
holds this database"). It left no directory, and the first restore was
verified.
- While the first restore held the database, a lease acquisition there ended in
`lock timeout`. An occupied objects path was refused and left untouched.
- The reviewer's `restore-race.sh` now ends with `remaining_evolution_schemas=1`
(`schema=no, objects=yes`).
- **P2 barrier Ready-pod selection.** `tools/ops/lib/pods.sh` uses a plain
jsonpath and needs exactly one Ready pod that is not being deleted.
`ops-lib-check.sh` passed its fixtures: one pod, not Ready, being deleted, no
condition, Ready beside a deleting pod, two Ready pods, and no pods. Not run:
the jsonpath against a live API server (no kubectl or Go in this environment).
- **P2 barrier Secret access.** The generated pod has `fsGroup: 10001`. The deploy
policy check now renders the backup and barrier pods with `--print-manifest` and
checks: non-root 10001 with fsGroup 10001, no token, read-only root, no
escalation, Secret files 0440, and egress admission of the component. Not run: a
disposable pod reading the Secret in the cluster.
- **P2 World0 claimRef.** A new admission validation denies removing a World0
volume's claim reference, or changing its namespace or name, without
break-glass. The policy check requires the rule, and `verify-isolation.sh`
dry-runs the removal when a World0 PV exists. Not run: no admission test cluster
here, and no World0 PV exists.
- **P2 CI change selection.** `ci-paths.sh` compares against the event's base (the
push's `before`, or the PR base, fetched if missing). It runs on manual
dispatch, local runs and any uncertainty, and the path lists now include the
check scripts. `ci-paths-check.sh` passed 11 fixtures, and the deploy suite runs
it first. The reviewer's `ci-range.sh` now runs in both cases. Not run: a real
Gitea push, so the event-context mapping is unverified.
- **P2 drill verdicts.** Every kill RTO is asserted against `RTO_BUDGET_MS`
(60 000). The empty scenario needs positive quarantine evidence; INCONCLUSIVE and
other exits fail. Acknowledgements are read from the dead or drained owner's
finalized log. The fixtures passed. Not run: the drill itself.
- Mutations, each caught:
- dropping a schema that was not installed: `backup-check.sh` failed;
- removing `fsGroup`: the policy check failed;
- accepting several Ready pods, or treating never-ran as PASS: the
`ops-lib-check.sh` fixtures failed;
- falling back to `HEAD^` when there is no base: the `ci-paths-check.sh` fixtures
failed.
- The host-loss and upgrade rehearsals, and the PostgreSQL migration and
permission suites, still pass in all three builds. The deploy policy check: 67
objects, 0 failures.
- ROADMAP corrected: M8-dev/stage is implemented and locally qualified, but **not
passed** until the live gates listed there are run.
+19 -8
View File
@@ -3,6 +3,7 @@
# (ADR-0074). # (ADR-0074).
# #
# tools/backup/cluster-backup.sh <dev|stage> --to DIR [--verify WORLD] [--context NAME] # tools/backup/cluster-backup.sh <dev|stage> --to DIR [--verify WORLD] [--context NAME]
# tools/backup/cluster-backup.sh <dev|stage> --print-manifest
# #
# Runs one short-lived Pod in evolution-<env> (image: the postgres-tools digest # Runs one short-lived Pod in evolution-<env> (image: the postgres-tools digest
# pinned in deploy/kubernetes/stores/<env>; migrator login; the aevolution-objects # pinned in deploy/kubernetes/stores/<env>; migrator login; the aevolution-objects
@@ -14,11 +15,12 @@
# hypervisor host. --verify WORLD then runs verify-backup.sh on this machine # hypervisor host. --verify WORLD then runs verify-backup.sh on this machine
# (restore on a private cluster, rebuilt and served as isolated world WORLD); it # (restore on a private cluster, rebuilt and served as isolated world WORLD); it
# needs the PostgreSQL 18.6 binaries and evo-server/evo-world-clone. # needs the PostgreSQL 18.6 binaries and evo-server/evo-world-clone.
# World0 is refused before anything touches the cluster. The Pod is an operator # --print-manifest prints the Pod and exits without touching the cluster (the deploy
# policy check inspects it). World0 is refused before anything touches the cluster. The Pod is an operator
# action, like fault-drill.sh; it is not part of any Argo application. A scheduled # action, like fault-drill.sh; it is not part of any Argo application. A scheduled
# backup (CronJob in Git, synced by Argo) follows once the destination exists. # backup (CronJob in Git, synced by Argo) follows once the destination exists.
set -euo pipefail set -euo pipefail
usage() { echo "usage: $0 <dev|stage> --to DIR [--verify WORLD] [--context NAME]" >&2; exit 2; } usage() { echo "usage: $0 <dev|stage> --to DIR [--verify WORLD] [--context NAME] | --print-manifest" >&2; exit 2; }
[ $# -ge 1 ] || usage [ $# -ge 1 ] || usage
env="$1"; shift env="$1"; shift
case "$env" in case "$env" in
@@ -26,16 +28,19 @@ case "$env" in
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2 *) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
exit 2 ;; exit 2 ;;
esac esac
to="" verify="" k=(kubectl --request-timeout=30s) to="" verify="" print=0 k=(kubectl --request-timeout=30s)
while [ $# -gt 0 ]; do while [ $# -gt 0 ]; do
case "$1" in case "$1" in
--to) to="${2:-}"; shift 2 ;; --to) to="${2:-}"; shift 2 ;;
--verify) verify="${2:-}"; shift 2 ;; --verify) verify="${2:-}"; shift 2 ;;
--print-manifest) print=1; shift ;;
--context) k+=(--context "${2:?}"); shift 2 ;; --context) k+=(--context "${2:?}"); shift 2 ;;
*) usage ;; *) usage ;;
esac esac
done done
[ -n "$to" ] && [ -d "$to" ] || { echo "--to must be an existing directory" >&2; exit 2; } if [ "$print" != 1 ]; then
[ -n "$to" ] && [ -d "$to" ] || { echo "--to must be an existing directory" >&2; exit 2; }
fi
root="$(cd "$(dirname "$0")/../.." && pwd)" root="$(cd "$(dirname "$0")/../.." && pwd)"
ns="evolution-$env" ns="evolution-$env"
image="$(awk '/name: aevolution-postgres-tools/ {f = 1} f && /newName:/ {n = $2} f && /digest:/ {print n "@" $2; exit}' \ image="$(awk '/name: aevolution-postgres-tools/ {f = 1} f && /newName:/ {n = $2} f && /digest:/ {print n "@" $2; exit}' \
@@ -45,15 +50,14 @@ stamp="$(date -u +%Y%m%dT%H%M%SZ)"
pod="aevolution-backup-$(date -u +%Y%m%d%H%M%S)" pod="aevolution-backup-$(date -u +%Y%m%d%H%M%S)"
dest="$to/$env-$stamp" dest="$to/$env-$stamp"
say() { echo "$(date -u +%H:%M:%S) $*"; } say() { echo "$(date -u +%H:%M:%S) $*"; }
cleanup() { "${k[@]}" delete pod -n "$ns" "$pod" --wait=false > /dev/null 2>&1 || true; }
trap cleanup EXIT
say "starting $pod in $ns ($image)" manifest() {
"${k[@]}" apply -n "$ns" -f - > /dev/null <<YAML cat <<YAML
apiVersion: v1 apiVersion: v1
kind: Pod kind: Pod
metadata: metadata:
name: $pod name: $pod
namespace: $ns
labels: labels:
app.kubernetes.io/part-of: aevolution app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: backup app.kubernetes.io/component: backup
@@ -112,6 +116,13 @@ spec:
- name: scratch - name: scratch
emptyDir: {medium: Memory, sizeLimit: 8Mi} emptyDir: {medium: Memory, sizeLimit: 8Mi}
YAML YAML
}
if [ "$print" = 1 ]; then manifest; exit 0; fi
cleanup() { "${k[@]}" delete pod -n "$ns" "$pod" --wait=false > /dev/null 2>&1 || true; }
trap cleanup EXIT
say "starting $pod in $ns ($image)"
manifest | "${k[@]}" apply -f - > /dev/null
for _ in $(seq 1 1800); do for _ in $(seq 1 1800); do
phase="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}' 2> /dev/null || true)" phase="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}' 2> /dev/null || true)"
+67 -20
View File
@@ -14,13 +14,23 @@
# Order, failing closed at each step: # Order, failing closed at each step:
# 1. every bundle file matches SHA256SUMS, no file is unlisted, and every object's # 1. every bundle file matches SHA256SUMS, no file is unlisted, and every object's
# name is its digest; nothing is touched before this passes; # name is its digest; nothing is touched before this passes;
# 2. the target has no `evolution` schema and runs the manifest's server version; # 2. the targets are claimed for this invocation alone (below), then the target
# has no `evolution` schema and runs the manifest's server version;
# 3. pg_restore in one transaction, then objects are copied; # 3. pg_restore in one transaction, then objects are copied;
# 4. the restored heads equal the MANIFEST heads exactly; # 4. the restored heads equal the MANIFEST heads exactly;
# 5. every region's full history is audited (evo-world-clone audit: each link's # 5. every region's full history is audited (evo-world-clone audit: each link's
# manifest, payload, trace replay and outcomes, from genesis to the head). # manifest, payload, trace replay and outcomes, from genesis to the head).
# On a failure after step 2 the restored schema and the copied objects are removed, # Ownership (review M8 P1): --objects is claimed by creating it (mkdir, never -p: an
# so an unverified restore can never be mistaken for a world. The result is a # existing path is refused and never touched), and the database by one psql session
# held open for the whole restore. That session takes the restore claim (advisory
# 71420004, try-lock: a concurrent restore of the same database refuses at once) and
# the upgrade lock exclusively (71420002: provisioning, acquisition and lease checks
# in that database wait until this restore is verified or removed). On failure only
# what this invocation created is removed: the schema only if its own pg_restore
# committed, the objects directory only if its own mkdir created it. So an
# unverified restore can never be mistaken for a world, and a losing restore never
# deletes a winner's state. EVO_RESTORE_TEST_HOOK (tests only) is run by bash with
# $1 = before-restore (claimed, checked) and after-restore (installed, objects copied). The result is a
# verbatim, owner-only copy with the source's world identities: it has no runtime # verbatim, owner-only copy with the source's world identities: it has no runtime
# grants and is never served. To run it, clone it into an isolated world # grants and is never served. To run it, clone it into an isolated world
# (tools/backup/verify-backup.sh --isolate). # (tools/backup/verify-backup.sh --isolate).
@@ -89,32 +99,69 @@ if [ "$in_place" = 1 ]; then
privileges=() privileges=()
fi fi
# 2. Target: empty, same server version. target="${PGDATABASE:-}"
[ ! -e "$objects" ] || refuse "objects target already exists: $objects" [ -n "${PGSERVICE:-}" ] && target="service=$PGSERVICE"
[ -n "$target" ] || { echo "set PGDATABASE or PGSERVICE for the target" >&2; exit 2; }
# 2. Claim the targets, then check them under the claim.
created_objects=0 installed=0 restored=0 lock_pid="" claim=""
cleanup() {
local code=$?
if [ "$restored" != 1 ]; then
if [ "$installed" = 1 ]; then
psql -X --no-password -q -v ON_ERROR_STOP=1 -c 'DROP SCHEMA evolution CASCADE' > /dev/null 2>&1 || true
fi
[ "$created_objects" = 1 ] && rm -rf -- "$objects"
if [ "$installed" = 1 ] || [ "$created_objects" = 1 ]; then
echo "RESTORE FAILED: removed only what this restore created" \
"(schema=$([ "$installed" = 1 ] && echo yes || echo no), objects=$([ "$created_objects" = 1 ] && echo yes || echo no))" >&2
fi
fi
[ -n "$claim" ] && exec 7>&- # Ends the claim session: its locks are released.
[ -n "$lock_pid" ] && { wait "$lock_pid" 2> /dev/null || true; }
[ -n "$claim" ] && rm -rf -- "$claim"
return "$code"
}
trap cleanup EXIT
mkdir -m 700 -- "$objects" 2> /dev/null || refuse "objects target already exists or cannot be created: $objects"
created_objects=1
claim="$(mktemp -d /tmp/evo-restore-claim.XXXXXXXX)"
mkfifo "$claim/in"
psql -X --no-password -q -A -t -v ON_ERROR_STOP=1 < "$claim/in" > "$claim/out" 2>&1 &
lock_pid=$!
exec 7> "$claim/in"
cat >&7 <<SQL
SELECT pg_catalog.pg_try_advisory_lock(71420004) AS claimed \gset
\if :claimed
SELECT pg_catalog.pg_advisory_lock(71420002);
\! touch '$claim/held'
\else
\! touch '$claim/busy'
\endif
SQL
for _ in $(seq 1 600); do
[ -e "$claim/held" ] || [ -e "$claim/busy" ] && break
kill -0 "$lock_pid" 2> /dev/null || break
sleep 0.1
done
[ -e "$claim/busy" ] && refuse "another restore holds this database"
[ -e "$claim/held" ] || refuse "could not claim the target database ($(head -c 200 "$claim/out"))"
version="$(psql -X --no-password -A -t -v ON_ERROR_STOP=1 -c "SELECT current_setting('server_version')")" version="$(psql -X --no-password -A -t -v ON_ERROR_STOP=1 -c "SELECT current_setting('server_version')")"
grep -qx "database $version" "$bundle/MANIFEST" || refuse "target server $version differs from the manifest" grep -qx "database $version" "$bundle/MANIFEST" || refuse "target server $version differs from the manifest"
existing="$(psql -X --no-password -A -t -v ON_ERROR_STOP=1 -c "SELECT count(*) FROM pg_namespace WHERE nspname = 'evolution'")" existing="$(psql -X --no-password -A -t -v ON_ERROR_STOP=1 -c "SELECT count(*) FROM pg_namespace WHERE nspname = 'evolution'")"
[ "$existing" = 0 ] || refuse "target database already has an evolution schema" [ "$existing" = 0 ] || refuse "target database already has an evolution schema"
restored=0 # 3. Database in one transaction, then objects. A failed pg_restore installed
cleanup() { # nothing (one transaction), so nothing of the database is removed after it.
[ "$restored" = 1 ] && return hook() { [ -z "${EVO_RESTORE_TEST_HOOK:-}" ] || bash -c "$EVO_RESTORE_TEST_HOOK" hook "$1"; }
psql -X --no-password -q -v ON_ERROR_STOP=1 -c 'DROP SCHEMA IF EXISTS evolution CASCADE' > /dev/null 2>&1 || true hook before-restore
rm -rf -- "$objects"
echo "RESTORE FAILED: the restored schema and objects were removed" >&2
}
trap cleanup EXIT
# 3. Database in one transaction, then objects.
target="${PGDATABASE:-}"
[ -n "${PGSERVICE:-}" ] && target="service=$PGSERVICE"
[ -n "$target" ] || { echo "set PGDATABASE or PGSERVICE for the target" >&2; exit 2; }
pg_restore --no-password --exit-on-error --single-transaction --no-owner "${privileges[@]}" \ pg_restore --no-password --exit-on-error --single-transaction --no-owner "${privileges[@]}" \
--dbname="$target" "$bundle/database.dump" --dbname="$target" "$bundle/database.dump" || refuse "pg_restore failed; the database was not changed"
mkdir -m 700 -- "$objects" installed=1
(cd "$bundle/objects" && find . -mindepth 2 -maxdepth 2 -type d -printf '%P\0' | (cd "$bundle/objects" && find . -mindepth 2 -maxdepth 2 -type d -printf '%P\0' |
xargs -0 -r -I{} mkdir -p -m 700 -- "$objects/{}") xargs -0 -r -I{} mkdir -p -m 700 -- "$objects/{}")
(cd "$bundle/objects" && find . -type f -printf '%P\0' | xargs -0 -r -I{} cp -- {} "$objects/{}") (cd "$bundle/objects" && find . -type f -printf '%P\0' | xargs -0 -r -I{} cp -- {} "$objects/{}")
hook after-restore
# 4. Heads exactly as recorded. # 4. Heads exactly as recorded.
actual_heads="$(psql -X --no-password -A -t -F ' ' -v ON_ERROR_STOP=1 -c " actual_heads="$(psql -X --no-password -A -t -F ' ' -v ON_ERROR_STOP=1 -c "
+42
View File
@@ -90,4 +90,46 @@ PGDATABASE=evo_restore_twice bash "$root/tools/backup/restore-world.sh" --bundle
[ "$code" = 4 ] && grep -q 'already has an evolution schema' "$work/twice.log" || fail "a second restore into one database" [ "$code" = 4 ] && grep -q 'already has an evolution schema' "$work/twice.log" || fail "a second restore into one database"
[ "$(PGDATABASE=evo_restore_twice psql -XAtc 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 300')" = "$backup_tick" ] || [ "$(PGDATABASE=evo_restore_twice psql -XAtc 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 300')" = "$backup_tick" ] ||
fail "the refused second restore changed the first" fail "the refused second restore changed the first"
# Ownership races (review M8 P1). A competing actor creates the schema between this
# restore's checks and its pg_restore: the restore fails and leaves that schema alone.
createdb evo_restore_race
code=0
PGDATABASE=evo_restore_race EVO_RESTORE_TEST_HOOK='[ "$1" = before-restore ] && psql -XAtq -c "CREATE SCHEMA evolution" -c "CREATE TABLE evolution.winner(x int)" || true' \
bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" --objects "$work/race-a" \
> "$work/race-a.log" 2>&1 || code=$?
[ "$code" = 4 ] && grep -q 'pg_restore failed; the database was not changed' "$work/race-a.log" ||
{ cat "$work/race-a.log" >&2; fail "a pre-empted restore: exit $code"; }
[ "$(PGDATABASE=evo_restore_race psql -XAtc "SELECT count(*) FROM pg_tables WHERE schemaname = 'evolution' AND tablename = 'winner'")" = 1 ] ||
fail "a pre-empted restore deleted the competing schema"
[ ! -e "$work/race-a" ] || fail "a pre-empted restore left its own objects directory"
# Two restores of one database: the second refuses at once and touches nothing of
# the first, which completes. While the first holds the database (after its install),
# a lease acquisition there waits instead of running against unverified state.
createdb evo_restore_pair
PGDATABASE=evo_restore_pair EVO_RESTORE_TEST_HOOK='
if [ "$1" = before-restore ]; then touch "'"$work"'/pair-claimed"; sleep 3; fi
if [ "$1" = after-restore ]; then
psql -XAt -c "SET lock_timeout = '"'"'1s'"'"'" -c "SELECT * FROM evolution.acquire_region_lease(300, 1, 20000, decode(repeat('"'"'b1'"'"', 32), '"'"'hex'"'"'))" > "'"$work"'/pair-acquire.log" 2>&1 || true
fi' bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" --objects "$work/pair-a" \
> "$work/pair-a.log" 2>&1 &
pair_pid=$!
for _ in $(seq 1 100); do [ -e "$work/pair-claimed" ] && break; sleep 0.1; done
code=0
PGDATABASE=evo_restore_pair bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" \
--objects "$work/pair-b" > "$work/pair-b.log" 2>&1 || code=$?
[ "$code" = 4 ] && grep -q 'another restore holds this database' "$work/pair-b.log" ||
{ cat "$work/pair-b.log" >&2; fail "a concurrent restore: exit $code"; }
[ ! -e "$work/pair-b" ] || fail "the refused concurrent restore left its objects directory"
code=0; wait "$pair_pid" || code=$?
[ "$code" = 0 ] && grep -q '^RESTORE VERIFIED regions=1 ' "$work/pair-a.log" ||
{ cat "$work/pair-a.log" >&2; fail "the first of two restores: exit $code"; }
grep -q 'lock timeout' "$work/pair-acquire.log" ||
{ cat "$work/pair-acquire.log" >&2; fail "a lease acquisition ran during the restore"; }
# An existing objects path is never claimed or removed.
mkdir "$work/occupied"; echo keep > "$work/occupied/mine"
createdb evo_restore_occupied
code=0
PGDATABASE=evo_restore_occupied bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" \
--objects "$work/occupied" > "$work/occupied.log" 2>&1 || code=$?
[ "$code" = 4 ] && [ "$(cat "$work/occupied/mine")" = keep ] || fail "an occupied objects path: exit $code"
echo "backup and restore qualification passed (backup tick $backup_tick, acknowledged $acked_before, source $source_tick)" echo "backup and restore qualification passed (backup tick $backup_tick, acknowledged $acked_before, source $source_tick)"
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env bash
# Offline regression for tools/ci/ci-paths.sh (review M8 P2) in a disposable Git
# repository: multi-commit pushes ending in documentation, missing and unknown bases,
# pull requests, manual dispatch, and changes to only a check script. Uncertainty
# must run validation. Run by the deploy suite before it decides anything.
set -euo pipefail
script="$(cd "$(dirname "$0")" && pwd)/ci-paths.sh"
repo="$(mktemp -d /tmp/evo-ci-paths.XXXXXXXX)"
trap 'rm -rf -- "$repo"' EXIT
cd "$repo"
git init -q
git config user.email ci@example.invalid
git config user.name ci
mkdir -p src docs tools/ci
echo base > src/a.cpp; echo base > docs/a.md; echo base > tools/ci/dr-check.sh
git add .; git commit -qm base
base="$(git rev-parse HEAD)"
echo change >> src/a.cpp; git commit -qam implementation
echo change >> docs/a.md; git commit -qam documentation
docs_only_base="$(git rev-parse HEAD~1)"
failures=0
expect() { # expect <label> <0 run|1 skip> <env...> -- <suite>
local label="$1" want="$2"; shift 2
local envs=()
while [ "$1" != -- ]; do envs+=("$1"); shift; done
shift
local got=0
env -u GITHUB_EVENT_BEFORE -u GITHUB_EVENT_NAME -u CI_EVENT -u CI_BEFORE -u CI_PR_BASE \
"${envs[@]}" bash "$script" "$1" || got=$?
if [ "$got" = "$want" ]; then echo "ok $label"; else echo "FAIL $label: exit $got, wanted $want"; failures=$((failures + 1)); fi
}
expect "push: code then docs runs checks" 0 CI_EVENT=push "CI_BEFORE=$base" -- checks
expect "push: docs-only commit skips checks" 1 CI_EVENT=push "CI_BEFORE=$docs_only_base" -- checks
expect "push: no before SHA runs" 0 CI_EVENT=push -- checks
expect "push: all-zero before (new branch) runs" 0 CI_EVENT=push CI_BEFORE=0000000000000000000000000000000000000000 -- checks
expect "push: unknown, unfetchable base runs" 0 CI_EVENT=push CI_BEFORE=1111111111111111111111111111111111111111 -- checks
expect "pull request: base diff runs checks" 0 CI_EVENT=pull_request "CI_PR_BASE=$base" -- checks
expect "pull request: no base runs" 0 CI_EVENT=pull_request -- deploy
expect "manual dispatch always runs" 0 CI_EVENT=workflow_dispatch "CI_BEFORE=$docs_only_base" -- checks
expect "local run (no event) runs" 0 -- postgres
echo change >> tools/ci/dr-check.sh; git commit -qam dr-script
script_base="$(git rev-parse HEAD~1)"
expect "push: only dr-check.sh changed runs postgres" 0 CI_EVENT=push "CI_BEFORE=$script_base" -- postgres
expect "push: only dr-check.sh changed skips image" 1 CI_EVENT=push "CI_BEFORE=$script_base" -- image
[ "$failures" = 0 ] && echo "ci path selection fixtures passed" || { echo "$failures fixture(s) failed"; exit 1; }
+32 -12
View File
@@ -1,35 +1,55 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# Decides whether a CI suite must run: exit 0 = run, exit 1 = nothing relevant changed.
#
# The comparison base comes from the event (review M8 P2), never from HEAD^ alone:
# - push: CI_BEFORE (the push's `before` SHA), so every commit of a multi-commit push
# counts, not just the last;
# - pull_request: CI_PR_BASE (the base SHA), diffed against the checked-out merge;
# - workflow_dispatch, schedule, a local run or any other event: always run.
# A missing, all-zero or unfetchable base, or any git failure, also runs the suite:
# uncertainty never skips validation. tools/ci/ci-paths-check.sh tests this offline.
set -euo pipefail set -euo pipefail
suite=${1:?suite required} suite=${1:?suite required}
base=${GITHUB_EVENT_BEFORE:-} event="${CI_EVENT:-${GITHUB_EVENT_NAME:-}}"
if [[ -z "$base" || "$base" == 0000000000000000000000000000000000000000 ]]; then case "$event" in
base=HEAD^ push) base="${CI_BEFORE:-${GITHUB_EVENT_BEFORE:-}}" ;;
pull_request) base="${CI_PR_BASE:-}" ;;
*) exit 0 ;;
esac
if [[ ! "$base" =~ ^[0-9a-f]{40}$ || "$base" == 0000000000000000000000000000000000000000 ]]; then
exit 0
fi fi
if ! git cat-file -e "$base^{commit}" 2> /dev/null; then
# A single-parent diff is sufficient for push and the merge commit generated for git fetch --no-tags --quiet --depth=1 origin "$base" > /dev/null 2>&1 || exit 0
# pull requests. If history is unavailable, fail open and run the gate. git cat-file -e "$base^{commit}" 2> /dev/null || exit 0
if ! files=$(git diff --name-only "$base" HEAD 2>/dev/null); then fi
if ! files=$(git diff --name-only "$base" HEAD 2> /dev/null); then
exit 0 exit 0
fi fi
# Every suite runs when the workflow or this selector changes.
common='\.gitea/workflows/|tools/ci/ci-paths\.sh'
native='CMakeLists\.txt|CMakePresets\.json|cmake/|include/|src/|apps/'
postgres_tools='migrations/|tools/prepare-postgres\.sh|tools/migrate-postgres\.sh|tools/backup/|tools/ci/postgres-check\.sh|tools/ci/backup-check\.sh|tools/ci/dr-check\.sh|tools/ci/upgrade-check\.sh'
case "$suite" in case "$suite" in
checks|tsan|ecs) checks|tsan|ecs)
grep -Eq '^(CMakeLists.txt|CMakePresets.txt|CMakePresets.json|cmake/|include/|src/|apps/|tests/|experiments/|tools/ci/linux-check.sh|tools/ci/tsan-profile.mjs|deploy/docker/Dockerfile|\.gitea/workflows/)' <<<"$files" pattern="$common|$native|tests/|experiments/|tools/ci/linux-check\.sh|tools/ci/tsan-profile\.mjs|deploy/docker/Dockerfile"
;; ;;
postgres|postgres-tools) postgres|postgres-tools)
grep -Eq '^(CMakeLists.txt|CMakePresets.json|cmake/|include/|src/|apps/|tests/|migrations/|tools/prepare-postgres.sh|tools/ci/postgres-check.sh|tools/ci/backup-check.sh|tools/backup/|tools/migrate-postgres.sh|deploy/docker/Dockerfile|\.gitea/workflows/)' <<<"$files" pattern="$common|$native|tests/|$postgres_tools|deploy/docker/Dockerfile"
;; ;;
deploy) deploy)
grep -Eq '^(deploy/|tools/ci/deploy-policy-check.py|tools/ops/|tools/backup/|\.gitea/workflows/)' <<<"$files" pattern="$common|deploy/|tools/ops/|tools/backup/|tools/ci/deploy-policy-check\.py|tools/ci/monitoring-check\.sh|tools/ci/ops-lib-check\.sh"
;; ;;
server-image) server-image)
grep -Eq '^(CMakeLists.txt|CMakePresets.json|cmake/|include/|src/|apps/|migrations/|tools/prepare-postgres.sh|tools/ci/server-image-smoke.sh|deploy/docker/Dockerfile|\.gitea/workflows/)' <<<"$files" pattern="$common|$native|migrations/|tools/prepare-postgres\.sh|tools/ci/server-image-smoke\.sh|deploy/docker/Dockerfile"
;; ;;
image) image)
grep -Eq '^(CMakeLists.txt|CMakePresets.json|cmake/|include/|src/|apps/|tests/|deploy/docker/Dockerfile|\.gitea/workflows/)' <<<"$files" pattern="$common|$native|tests/|deploy/docker/Dockerfile"
;; ;;
*) *)
exit 0 exit 0
;; ;;
esac esac
grep -Eq "^($pattern)" <<< "$files"
+48 -2
View File
@@ -21,7 +21,10 @@ environment-isolation or World0-protection invariant is broken:
the digest already pinned in stage (promotion, never a fresh build); the digest already pinned in stage (promotion, never a fresh build);
- scheduling (ADR-0076): every pod that mounts the node-local objects volume - scheduling (ADR-0076): every pod that mounts the node-local objects volume
selects the store node; aevolution.io/compute is only ever preferred, never selects the store node; aevolution.io/compute is only ever preferred, never
required; upgrade-rehearsal flags never appear in a deployed workload. required; upgrade-rehearsal flags never appear in a deployed workload;
- the Pods the backup and barrier scripts generate (rendered with --print-manifest)
keep the database-tool security contract: non-root 10001 with fsGroup 10001 so
the 0440 Secret files are readable, no token, read-only root, no escalation.
""" """
import os import os
import shutil import shutil
@@ -118,7 +121,9 @@ if policy and binding:
text = yaml.safe_dump(spec) text = yaml.safe_dump(spec)
for needle in ("aevolution:world0-break-glass", "evolution-world0", "aevolution-world0", for needle in ("aevolution:world0-break-glass", "evolution-world0", "aevolution-world0",
"aevolution-platform", "aevolution-platform",
"aevolution.io/protected", "Retain"): "aevolution.io/protected", "Retain",
"has(object.spec.claimRef.namespace)",
"object.spec.claimRef.name == oldObject.spec.claimRef.name"):
check(needle in text, f'world0 policy must reference {needle}') check(needle in text, f'world0 policy must reference {needle}')
# Argo CD projects and applications. # Argo CD projects and applications.
@@ -368,6 +373,47 @@ with tempfile.TemporaryDirectory() as empty:
and 'kubectl' not in result.stderr, and 'kubectl' not in result.stderr,
f'{tool} must refuse {bad!r} before kubectl (rc={result.returncode})') f'{tool} must refuse {bad!r} before kubectl (rc={result.returncode})')
# Script-generated Pods (review M8 P2): rendered offline and checked like manifests.
retained_hex = next((v.get('value', '').split()[0] for d in manifests.get(
'deploy/kubernetes/region/dev/region-values.yaml', [])
for c in d['spec']['template']['spec']['containers'] for v in c.get('env', [])
if v.get('name') == 'REGION_RETAINED' and v.get('value')), None)
generators = [('tools/backup/cluster-backup.sh', ['dev', '--print-manifest'], 'backup')]
if retained_hex:
generators.append(('tools/ops/upgrade-barrier.sh',
['dev', '--retire', retained_hex, '--print-manifest'], 'migrate'))
else:
check(False, 'dev REGION_RETAINED is empty: the barrier Pod cannot be rendered')
egress = next((d for p, d in objects if d['kind'] == 'NetworkPolicy'
and d['metadata']['name'] == 'aevolution-database-egress'), {})
admitted = [v for e in egress.get('spec', {}).get('podSelector', {}).get('matchExpressions', [])
for v in e.get('values', [])]
for tool, args, component in generators:
rendered = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, tool)] + args,
capture_output=True, text=True)
check(rendered.returncode == 0, f'{tool} --print-manifest failed: {rendered.stderr.strip()}')
if rendered.returncode != 0:
continue
pod = yaml.safe_load(rendered.stdout)
spec = pod['spec']
context = spec.get('securityContext', {})
check(context.get('runAsNonRoot') is True and context.get('runAsUser') == 10001
and context.get('runAsGroup') == 10001 and context.get('fsGroup') == 10001,
f'{tool}: the generated Pod must run as 10001 with fsGroup 10001 (Secret access)')
check(spec.get('automountServiceAccountToken') is False, f'{tool}: the generated Pod mounts no token')
for container in spec['containers']:
sc = container.get('securityContext', {})
check(sc.get('readOnlyRootFilesystem') is True and sc.get('allowPrivilegeEscalation') is False,
f'{tool}: read-only root and no privilege escalation')
for volume in spec.get('volumes', []):
if 'secret' in volume:
check(volume['secret'].get('defaultMode') == 0o440,
f'{tool}: Secret files stay 0440 (never world-readable)')
labels = pod['metadata'].get('labels', {})
check(labels.get('app.kubernetes.io/component') == component and component in admitted,
f'{tool}: component {component} must be admitted by the database egress policy')
check(pod['metadata'].get('namespace') == 'evolution-dev', f'{tool}: renders into its environment')
for message in failures: for message in failures:
print('FAIL', message) print('FAIL', message)
print(f'deploy policy: {len(objects)} objects in {len(manifests)} files, {len(failures)} failures') print(f'deploy policy: {len(objects)} objects in {len(manifests)} files, {len(failures)} failures')
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env bash
# Offline fixtures for the operations helpers (review M8 P2): Ready-pod selection
# (tools/ops/lib/pods.sh) and the fault-drill verdicts (tools/ops/lib/drill.sh). No
# cluster: the lines below are what POD_STATUS_JSONPATH renders as
# "<name>|<Ready status>|<deletionTimestamp>". Run by the ci-deploy image build.
set -euo pipefail
root="$(cd "$(dirname "$0")/../.." && pwd)"
. "$root/tools/ops/lib/pods.sh"
. "$root/tools/ops/lib/drill.sh"
failures=0
expect() { # expect <label> <wanted> <actual>
if [ "$2" = "$3" ]; then echo "ok $1"; else echo "FAIL $1: wanted '$2', got '$3'"; failures=$((failures + 1)); fi
}
pick() { single_ready_pod 2> /dev/null <<< "$1" || echo "<none>"; }
expect "one Ready pod" evo-region-a "$(pick 'evo-region-a|True|')"
expect "not Ready" "<none>" "$(pick 'evo-region-a|False|')"
expect "being deleted" "<none>" "$(pick 'evo-region-a|True|2026-09-30T10:00:00Z')"
expect "no Ready condition yet" "<none>" "$(pick 'evo-region-a||')"
expect "Ready beside a deleting pod" evo-region-b "$(pick $'evo-region-a|True|2026-09-30T10:00:00Z\nevo-region-b|True|')"
expect "two Ready pods" "<none>" "$(pick $'evo-region-a|True|\nevo-region-b|True|')"
expect "no pods" "<none>" "$(pick '')"
expect "jsonpath selects by the Ready condition" 1 \
"$(grep -c 'conditions\[?(@.type=="Ready")\].status' <<< "$POD_STATUS_JSONPATH")"
expect "RTO within budget" PASS "$(rto_verdict 20147 60000)"
expect "RTO just above budget" "FAIL rto 60001 ms > 60000 ms" "$(rto_verdict 60001 60000)"
expect "RTO not measured" "FAIL rto unknown ms > 60000 ms" "$(rto_verdict '' 60000)"
expect "empty: quarantined, exit 4" PASS "$(empty_verdict 2 4 2 6)"
expect "empty: quarantined, still running" PASS "$(empty_verdict 0 '' 1 2)"
expect "empty: crashed otherwise" "FAIL the host exited 1, not 4 (quarantined)" "$(empty_verdict 3 1 0 4)"
expect "empty: image pull, never ran" \
"INCONCLUSIVE the replacement host never ran (Pending, image pull or start failure?)" "$(empty_verdict 0 '' 0 0)"
expect "empty: ran without quarantine" "INCONCLUSIVE the host ran but logged no quarantine" "$(empty_verdict 0 '' 0 3)"
expect "acks covered" PASS "$(acks_verdict 209 209)"
expect "acks lost" "FAIL acknowledged 210 > recovered 209" "$(acks_verdict 210 209)"
expect "acks unobserved" "FAIL acknowledged ? > recovered 209" "$(acks_verdict '' 209)"
[ "$failures" = 0 ] && echo "operations helper fixtures passed" || { echo "$failures fixture(s) failed"; exit 1; }
+34 -9
View File
@@ -21,6 +21,13 @@
# made with `kubectl set env` / `kubectl patch` on the evo-region Deployment and # made with `kubectl set env` / `kubectl patch` on the evo-region Deployment and
# reverted on exit, including on failure, back to the Git values (so Argo returns to # reverted on exit, including on failure, back to the Git values (so Argo returns to
# Synced); World0 is refused before anything touches the cluster. # Synced); World0 is refused before anything touches the cluster.
#
# Verdicts (tools/ops/lib/drill.sh): each kill must recover within RTO_BUDGET_MS
# (default 60000, ADR-0075); the empty scenario needs positive quarantine evidence.
# A missing observation fails the drill. Limits of log-based evidence: an owner's
# acknowledgements are read from its streamed log, finalized after it is gone; a line
# written but never streamed before a forced kill is not observed, so the
# zero-lost-acknowledgement check covers every observed acknowledgement only.
set -euo pipefail set -euo pipefail
usage() { echo "usage: $0 <dev|stage> [kills|full|empty|db-outage|all] [--context <name>]" >&2; exit 2; } usage() { echo "usage: $0 <dev|stage> [kills|full|empty|db-outage|all] [--context <name>]" >&2; exit 2; }
[ $# -ge 1 ] || usage [ $# -ge 1 ] || usage
@@ -35,9 +42,13 @@ case "$scenario" in kills|full|empty|db-outage|all) shift || true ;; --context)
k=(kubectl --request-timeout=30s) k=(kubectl --request-timeout=30s)
[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2") [ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2")
ns="evolution-$env" ns="evolution-$env"
rto_budget="${RTO_BUDGET_MS:-60000}"
# shellcheck source=lib/drill.sh
. "$(cd "$(dirname "$0")" && pwd)/lib/drill.sh"
selector=app.kubernetes.io/name=evo-region selector=app.kubernetes.io/name=evo-region
work="$(mktemp -d)" work="$(mktemp -d)"
log_pids=() log_pids=()
declare -A capture_pid=()
now_ms() { date +%s%3N; } now_ms() { date +%s%3N; }
say() { echo "$(date -u +%H:%M:%S) $*"; } say() { echo "$(date -u +%H:%M:%S) $*"; }
@@ -65,6 +76,13 @@ metric() { # metric <pod> <series>: from the pod's own /metrics (bash /dev/tcp).
capture() { # Streams a pod's log to $work/<pod>.log for acknowledgement accounting. capture() { # Streams a pod's log to $work/<pod>.log for acknowledgement accounting.
"${k[@]}" logs -n "$ns" -f "$1" > "$work/$1.log" 2>/dev/null & "${k[@]}" logs -n "$ns" -f "$1" > "$work/$1.log" 2>/dev/null &
log_pids+=($!) log_pids+=($!)
capture_pid[$1]=$!
}
finalize() { # Waits (up to 15 s) for a gone pod's log stream to end, so its log is complete.
local pid="${capture_pid[$1]:-}"
[ -n "$pid" ] || return 0
for _ in $(seq 1 75); do kill -0 "$pid" 2> /dev/null || return 0; sleep 0.2; done
kill "$pid" 2> /dev/null || true
} }
max_ack() { awk '$1 == "ack" {if ($3 > m) m = $3} END {print m + 0}' "$work/$1.log"; } max_ack() { awk '$1 == "ack" {if ($3 > m) m = $3} END {print m + 0}' "$work/$1.log"; }
recovered_at() { # committed tick from the pod's "phase running epoch=E committed=C" line. recovered_at() { # committed tick from the pod's "phase running epoch=E committed=C" line.
@@ -104,24 +122,31 @@ say "loaded pod=$pod ($(recovered_at "$pod"))"
if [ "$scenario" = kills ] || [ "$scenario" = all ]; then if [ "$scenario" = kills ] || [ "$scenario" = all ]; then
for round in 1 2 3; do for round in 1 2 3; do
sleep 10 sleep 10
acked="$(max_ack "$pod")"
start="$(now_ms)" start="$(now_ms)"
"${k[@]}" delete pod -n "$ns" "$pod" --grace-period=0 --force > /dev/null 2>&1 "${k[@]}" delete pod -n "$ns" "$pod" --grace-period=0 --force > /dev/null 2>&1
loaded_pod "$pod"; next="$POD" POD="$(ready_pod "$pod" 120)" || fail "round $round: no Ready successor within 120 s"
rto=$(( $(now_ms) - start )) rto=$(( $(now_ms) - start ))
next="$POD"
capture "$next"
wait_for_line "$next" '^ack ' 60 || fail "$next acknowledges nothing under load"
finalize "$pod"
acked="$(max_ack "$pod")" # Everything the dead owner was observed to acknowledge.
read -r epoch committed <<< "$(recovered_at "$next")" read -r epoch committed <<< "$(recovered_at "$next")"
say "kill round=$round rto_ms=$rto dead_acked_max=$acked recovered_committed=$committed epoch=$epoch pod=$next" say "kill round=$round rto_ms=$rto budget_ms=$rto_budget dead_acked_max=$acked recovered_committed=$committed epoch=$epoch pod=$next"
[ "$acked" -le "$committed" ] || fail "round $round lost acknowledgements ($acked > $committed)" verdict="$(rto_verdict "$rto" "$rto_budget")"
[ "$verdict" = PASS ] || fail "round $round: $verdict"
verdict="$(acks_verdict "$acked" "$committed")"
[ "$verdict" = PASS ] || fail "round $round lost acknowledgements: $verdict"
pod="$next" pod="$next"
done done
fi fi
if [ "$scenario" = full ] || [ "$scenario" = all ]; then if [ "$scenario" = full ] || [ "$scenario" = all ]; then
sleep 5 sleep 5
acked="$(max_ack "$pod")"
set_env REGION_RESERVE_MIB=1048576 set_env REGION_RESERVE_MIB=1048576
full_pod="$(ready_pod "$pod" 120)" || fail "full: host did not recover (recovery needs no space)" full_pod="$(ready_pod "$pod" 120)" || fail "full: host did not recover (recovery needs no space)"
capture "$full_pod" capture "$full_pod"
finalize "$pod"; acked="$(max_ack "$pod")" # The drained owner's complete log.
sleep 30 sleep 30
new_acks="$(awk '$1 == "ack"' "$work/$full_pod.log" | wc -l)" new_acks="$(awk '$1 == "ack"' "$work/$full_pod.log" | wc -l)"
available="$(metric "$full_pod" evo_region_objects_available_bytes)" available="$(metric "$full_pod" evo_region_objects_available_bytes)"
@@ -138,10 +163,10 @@ fi
if [ "$scenario" = empty ] || [ "$scenario" = all ]; then if [ "$scenario" = empty ] || [ "$scenario" = all ]; then
sleep 5 sleep 5
acked="$(max_ack "$pod")"
"${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \ "${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \
'[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","emptyDir":{"sizeLimit":"64Mi"}}}]' > /dev/null '[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","emptyDir":{"sizeLimit":"64Mi"}}}]' > /dev/null
sleep 45 sleep 45
finalize "$pod"; acked="$(max_ack "$pod")" # The drained owner's complete log.
if ready_pod "$pod" 1 > /dev/null; then fail "empty: a host became Ready on an empty volume"; fi if ready_pod "$pod" 1 > /dev/null; then fail "empty: a host became Ready on an empty volume"; fi
empty_pod="$(current_pods | grep -v "^$pod$" | head -1)" empty_pod="$(current_pods | grep -v "^$pod$" | head -1)"
[ -n "$empty_pod" ] || fail "empty: no replacement pod was created" [ -n "$empty_pod" ] || fail "empty: no replacement pod was created"
@@ -153,9 +178,9 @@ if [ "$scenario" = empty ] || [ "$scenario" = all ]; then
quarantined="$(cat "$work/$empty_pod.current.log" "$work/$empty_pod.previous.log" | grep -c '^phase quarantined' || true)" quarantined="$(cat "$work/$empty_pod.current.log" "$work/$empty_pod.previous.log" | grep -c '^phase quarantined' || true)"
say "empty pod=$empty_pod ready=never restarts=${restarts:-0} last_exit=${exit_code:-none} quarantined_lines=$quarantined" say "empty pod=$empty_pod ready=never restarts=${restarts:-0} last_exit=${exit_code:-none} quarantined_lines=$quarantined"
say "empty last log lines:"; tail -n 5 "$work/$empty_pod.previous.log" "$work/$empty_pod.current.log" | sed 's/^/ /' say "empty last log lines:"; tail -n 5 "$work/$empty_pod.previous.log" "$work/$empty_pod.current.log" | sed 's/^/ /'
if [ "${exit_code:-}" != 4 ] && [ "$quarantined" -eq 0 ]; then lines="$(cat "$work/$empty_pod.current.log" "$work/$empty_pod.previous.log" | grep -c '^phase ' || true)"
say "WARN empty: no quarantine evidence (exit 4 or a 'phase quarantined' line); see the logs above" verdict="$(empty_verdict "${restarts:-0}" "${exit_code:-}" "$quarantined" "$lines")"
fi [ "$verdict" = PASS ] || fail "empty: $verdict"
"${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \ "${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \
'[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","persistentVolumeClaim":{"claimName":"aevolution-objects"}}}]' > /dev/null '[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","persistentVolumeClaim":{"claimName":"aevolution-objects"}}}]' > /dev/null
loaded_pod "$empty_pod"; pod="$POD" loaded_pod "$empty_pod"; pod="$POD"
+34
View File
@@ -0,0 +1,34 @@
# Fault-drill verdicts (sourced by tools/ops/fault-drill.sh; tested offline with
# fixtures by tools/ci/ops-lib-check.sh). Each prints PASS, or FAIL/INCONCLUSIVE with
# a reason; the drill passes a scenario only on PASS (review M8 P2: a missing
# observation is never a pass).
# rto_verdict <rto_ms> <budget_ms>: the ratified objective (ADR-0075: 60 s for dev/stage).
rto_verdict() {
if [[ "${1:-}" =~ ^[0-9]+$ ]] && [ "$1" -le "$2" ]; then echo PASS; else echo "FAIL rto ${1:-unknown} ms > ${2} ms"; fi
}
# empty_verdict <restarts> <last_exit_code> <quarantined_lines> <log_lines>
# Positive evidence only: the replacement host ran and logged "phase quarantined"
# (on an empty volume the database and builds are unchanged, so unavailable committed
# state is the only cause), and if its container ended, it ended with exit 4.
# Another exit (image, permission, crash) is a FAIL; no evidence is INCONCLUSIVE.
empty_verdict() {
local restarts="${1:-0}" code="${2:-}" quarantined="${3:-0}" lines="${4:-0}"
if [ -n "$code" ] && [ "$code" != 4 ]; then
echo "FAIL the host exited $code, not 4 (quarantined)"
elif [ "$quarantined" -gt 0 ]; then
echo PASS
elif [ "$restarts" = 0 ] && [ "$lines" = 0 ]; then
echo "INCONCLUSIVE the replacement host never ran (Pending, image pull or start failure?)"
else
echo "INCONCLUSIVE the host ran but logged no quarantine"
fi
}
# acks_verdict <dead_acked_max> <recovered_committed>: every acknowledgement the dead
# owner was observed to make is at or below the head its successor recovered.
acks_verdict() {
if [[ "${1:-}" =~ ^[0-9]+$ && "${2:-}" =~ ^[0-9]+$ ]] && [ "$1" -le "$2" ]; then echo PASS
else echo "FAIL acknowledged ${1:-?} > recovered ${2:-?}"; fi
}
+22
View File
@@ -0,0 +1,22 @@
# Shared pod-status helpers for the operations scripts (sourced, not executed).
#
# POD_STATUS_JSONPATH renders one line per pod, "<name>|<Ready status>|<deletion>",
# with plain kubectl jsonpath (no template variables): the Ready status is "True",
# "False" or empty when the pod has no Ready condition yet, and the deletion field is
# empty unless the pod is being deleted. single_ready_pod reads such lines on stdin
# and prints the one Ready pod not being deleted; zero or several candidates fail
# (review M8 P2: never concatenate names or guess). tools/ci/ops-lib-check.sh tests it
# offline with fixtures.
# shellcheck disable=SC2034 # Used by the scripts that source this file.
POD_STATUS_JSONPATH='{range .items[*]}{.metadata.name}{"|"}{.status.conditions[?(@.type=="Ready")].status}{"|"}{.metadata.deletionTimestamp}{"\n"}{end}'
single_ready_pod() {
local ready count
ready="$(awk -F'|' 'NF >= 2 && $1 != "" && $2 == "True" && $3 == "" {print $1}')"
count="$(grep -c . <<< "$ready" || true)"
if [ "$count" != 1 ]; then
echo "expected exactly one Ready pod, found $count${ready:+: ${ready//$'\n'/ }}" >&2
return 1
fi
printf '%s\n' "$ready"
}
+45 -26
View File
@@ -1,22 +1,25 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# M8.9 version barrier in a disposable environment (ADR-0068, ADR-0076). # M8.9 version barrier in a disposable environment (ADR-0068, ADR-0076).
# #
# tools/ops/upgrade-barrier.sh <dev|stage> --retire HEX64 [--context NAME] # tools/ops/upgrade-barrier.sh <dev|stage> --retire HEX64 [--context NAME] [--print-manifest]
# #
# The last step of the upgrade order: migrate -> retain the new build (the provision # The last step of the upgrade order: migrate -> retain the new build (the provision
# hook does) -> deploy it with the old build in REGION_RETAINED -> retire the old # hook does) -> deploy it with the old build in REGION_RETAINED -> retire the old
# build here. Refused unless: # build here. Refused unless:
# - HEX64 is not the build Git currently deploys (deploy/kubernetes/region/<env>); # - HEX64 is not the build Git currently deploys (deploy/kubernetes/region/<env>);
# - HEX64 is listed in REGION_RETAINED (its history must stay recoverable); # - HEX64 is listed in REGION_RETAINED (its history must stay recoverable);
# - the evo-region pod is Ready and runs exactly the Git-pinned image, so the new # - exactly one evo-region pod is Ready (and not being deleted), and it runs exactly
# build already serves. # the Git-pinned image, so the new build already serves.
# Then one short Pod (the postgres-tools image pinned for <env>, migrator login) runs # Then one short Pod (the postgres-tools image pinned for <env>, migrator login, the
# same fsGroup Secret-access contract as the other database-tool pods) runs
# evolution.upgrade_barrier(HEX64): the old build can never acquire, renew or commit # evolution.upgrade_barrier(HEX64): the old build can never acquire, renew or commit
# again, and every lease is fenced. The new host re-acquires within one lease. The # again, and every lease is fenced. The new host re-acquires within one lease. The
# script waits for Ready again and prints the time without a ready host. # script waits for Ready again and prints the time without a ready host.
# --print-manifest prints that Pod after the offline checks and exits without
# touching the cluster (the deploy policy check inspects it).
# Retirement is irreversible here. World0 is refused before kubectl runs. # Retirement is irreversible here. World0 is refused before kubectl runs.
set -euo pipefail set -euo pipefail
usage() { echo "usage: $0 <dev|stage> --retire HEX64 [--context NAME]" >&2; exit 2; } usage() { echo "usage: $0 <dev|stage> --retire HEX64 [--context NAME] [--print-manifest]" >&2; exit 2; }
[ $# -ge 1 ] || usage [ $# -ge 1 ] || usage
env="$1"; shift env="$1"; shift
case "$env" in case "$env" in
@@ -24,16 +27,19 @@ case "$env" in
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2 *) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
exit 2 ;; exit 2 ;;
esac esac
retire="" k=(kubectl --request-timeout=30s) retire="" print=0 k=(kubectl --request-timeout=30s)
while [ $# -gt 0 ]; do while [ $# -gt 0 ]; do
case "$1" in case "$1" in
--retire) retire="${2:-}"; shift 2 ;; --retire) retire="${2:-}"; shift 2 ;;
--context) k+=(--context "${2:?}"); shift 2 ;; --context) k+=(--context "${2:?}"); shift 2 ;;
--print-manifest) print=1; shift ;;
*) usage ;; *) usage ;;
esac esac
done done
[[ "$retire" =~ ^[0-9a-f]{64}$ ]] || usage [[ "$retire" =~ ^[0-9a-f]{64}$ ]] || usage
root="$(cd "$(dirname "$0")/../.." && pwd)" root="$(cd "$(dirname "$0")/../.." && pwd)"
# shellcheck source=lib/pods.sh
. "$root/tools/ops/lib/pods.sh"
ns="evolution-$env" ns="evolution-$env"
values="$root/deploy/kubernetes/region/$env/region-values.yaml" values="$root/deploy/kubernetes/region/$env/region-values.yaml"
current="$(sed -n 's/.*name: REGION_BUILD, value: \([0-9a-f]\{64\}\)}.*/\1/p' "$values" | sort -u)" current="$(sed -n 's/.*name: REGION_BUILD, value: \([0-9a-f]\{64\}\)}.*/\1/p' "$values" | sort -u)"
@@ -44,26 +50,15 @@ grep -qw "$retire" <<< "$retained" || { echo "refused: $retire is not in REGION_
image="$(awk '/name: aevolution-postgres-tools/ {f = 1} f && /newName:/ {n = $2} f && /digest:/ {print n "@" $2; exit}' \ image="$(awk '/name: aevolution-postgres-tools/ {f = 1} f && /newName:/ {n = $2} f && /digest:/ {print n "@" $2; exit}' \
"$root/deploy/kubernetes/stores/$env/kustomization.yaml")" "$root/deploy/kubernetes/stores/$env/kustomization.yaml")"
[[ "$image" =~ @sha256:[0-9a-f]{64}$ ]] || { echo "no pinned postgres-tools image for $env" >&2; exit 2; } [[ "$image" =~ @sha256:[0-9a-f]{64}$ ]] || { echo "no pinned postgres-tools image for $env" >&2; exit 2; }
say() { echo "$(date -u +%H:%M:%S) $*"; }
ready_pod() {
"${k[@]}" get pods -n "$ns" -l app.kubernetes.io/name=evo-region -o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{range .status.conditions}}{{if and (eq .type "Ready") (eq .status "True")}}{{$.metadata.name}}{{end}}{{end}}{{end}}{{end}}' 2> /dev/null || true
}
serving() { # The Ready pod's running image digest.
local pod; pod="$(ready_pod)"
[ -n "$pod" ] && "${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.containerStatuses[0].imageID}'
}
[[ "$(serving)" == *"@sha256:$current" ]] ||
{ echo "refused: no Ready evo-region pod runs sha256:$current yet (deploy the new build first)" >&2; exit 3; }
pod="aevolution-barrier-$(date -u +%Y%m%d%H%M%S)" pod="aevolution-barrier-$(date -u +%Y%m%d%H%M%S)"
cleanup() { "${k[@]}" delete pod -n "$ns" "$pod" --wait=false > /dev/null 2>&1 || true; }
trap cleanup EXIT manifest() {
say "retiring $retire in $ns (serving sha256:$current)" cat <<YAML
"${k[@]}" apply -n "$ns" -f - > /dev/null <<YAML
apiVersion: v1 apiVersion: v1
kind: Pod kind: Pod
metadata: metadata:
name: $pod name: $pod
namespace: $ns
labels: labels:
app.kubernetes.io/part-of: aevolution app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: migrate app.kubernetes.io/component: migrate
@@ -76,6 +71,7 @@ spec:
runAsNonRoot: true runAsNonRoot: true
runAsUser: 10001 runAsUser: 10001
runAsGroup: 10001 runAsGroup: 10001
fsGroup: 10001
seccompProfile: {type: RuntimeDefault} seccompProfile: {type: RuntimeDefault}
containers: containers:
- name: barrier - name: barrier
@@ -87,8 +83,8 @@ spec:
umask 077 umask 077
cp /run/postgres/pgpass /tmp/pgpass cp /run/postgres/pgpass /tmp/pgpass
export PGPASSFILE=/tmp/pgpass export PGPASSFILE=/tmp/pgpass
psql -X --no-password -At -v ON_ERROR_STOP=1 \ psql -X --no-password -At -v ON_ERROR_STOP=1 \\
-c "SELECT 'fenced ' || evolution.upgrade_barrier(decode('$retire', 'hex'))" \ -c "SELECT 'fenced ' || evolution.upgrade_barrier(decode('$retire', 'hex'))" \\
-c "SELECT 'retired ' || encode(build_identity, 'hex') FROM evolution.retained_builds WHERE retired_at IS NOT NULL ORDER BY 1" -c "SELECT 'retired ' || encode(build_identity, 'hex') FROM evolution.retained_builds WHERE retired_at IS NOT NULL ORDER BY 1"
env: env:
- {name: PGSERVICE, value: evolution} - {name: PGSERVICE, value: evolution}
@@ -115,17 +111,40 @@ spec:
- name: scratch - name: scratch
emptyDir: {medium: Memory, sizeLimit: 1Mi} emptyDir: {medium: Memory, sizeLimit: 1Mi}
YAML YAML
}
if [ "$print" = 1 ]; then manifest; exit 0; fi
say() { echo "$(date -u +%H:%M:%S) $*"; }
ready_pod() { # The single Ready evo-region pod, or failure (none, several, or unreadable).
"${k[@]}" get pods -n "$ns" -l app.kubernetes.io/name=evo-region -o jsonpath="$POD_STATUS_JSONPATH" |
single_ready_pod
}
serving() { # The single Ready pod's running image digest.
local name
name="$(ready_pod 2> /dev/null)" || return 1
"${k[@]}" get pod -n "$ns" "$name" -o jsonpath='{.status.containerStatuses[0].imageID}'
}
ready_pod > /dev/null || { echo "refused: no single Ready evo-region pod" >&2; exit 3; }
[[ "$(serving)" == *"@sha256:$current" ]] ||
{ echo "refused: the Ready evo-region pod does not run sha256:$current (deploy the new build first)" >&2; exit 3; }
cleanup() { "${k[@]}" delete pod -n "$ns" "$pod" --wait=false > /dev/null 2>&1 || true; }
trap cleanup EXIT
say "retiring $retire in $ns (serving sha256:$current)"
manifest | "${k[@]}" apply -f - > /dev/null
start="$(date +%s%3N)" start="$(date +%s%3N)"
phase=""
for _ in $(seq 1 120); do for _ in $(seq 1 120); do
phase="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}' 2> /dev/null || true)" phase="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}' 2> /dev/null || true)"
[ "$phase" = Succeeded ] || [ "$phase" = Failed ] && break { [ "$phase" = Succeeded ] || [ "$phase" = Failed ]; } && break
sleep 1 sleep 1
done done
"${k[@]}" logs -n "$ns" "$pod" 2>&1 | sed 's/^/ /' "${k[@]}" logs -n "$ns" "$pod" 2>&1 | sed 's/^/ /'
[ "$phase" = Succeeded ] || { say "FAIL barrier pod $phase"; exit 3; } [ "$phase" = Succeeded ] || { say "FAIL barrier pod ${phase:-unknown}"; exit 3; }
# The fenced host loses readiness at its next lease check, then re-acquires.
not_ready_since="" not_ready_since=""
for _ in $(seq 1 600); do for _ in $(seq 1 600); do
if [ -n "$(ready_pod)" ]; then if ready_pod > /dev/null 2>&1; then
[ -z "$not_ready_since" ] && [ $(( $(date +%s%3N) - start )) -lt 3000 ] && { sleep 0.5; continue; } [ -z "$not_ready_since" ] && [ $(( $(date +%s%3N) - start )) -lt 3000 ] && { sleep 0.5; continue; }
break break
fi fi
+14
View File
@@ -82,6 +82,20 @@ if printf '%s\n' "${present[@]}" | grep -qx world0; then
pvc="$("${k[@]}" get pvc -n evolution-world0 -o name 2>/dev/null | head -1)" pvc="$("${k[@]}" get pvc -n evolution-world0 -o name 2>/dev/null | head -1)"
[ -n "$pvc" ] && denied "delete $pvc in evolution-world0" -n evolution-world0 "$pvc" || [ -n "$pvc" ] && denied "delete $pvc in evolution-world0" -n evolution-world0 "$pvc" ||
skip "no PVC in evolution-world0 to test" skip "no PVC in evolution-world0 to test"
# Clearing a World0 volume's claim reference must be denied too (it would strip
# the protection before a delete). Server-side dry run: never persisted.
pv="$("${k[@]}" get pv -o jsonpath='{range .items[?(@.spec.claimRef.namespace=="evolution-world0")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | head -1)"
if [ -n "$pv" ]; then
out="$("${k[@]}" patch pv "$pv" --dry-run=server --type json \
-p '[{"op":"remove","path":"/spec/claimRef"}]' 2>&1 || true)"
if echo "$out" | grep -q "aevolution:world0-break-glass"; then
pass "clearing the claim reference of World0 volume $pv is denied"
else
fail "clearing the claim reference of World0 volume $pv was not denied: $out"
fi
else
skip "no PersistentVolume bound to evolution-world0 to test"
fi
else else
skip "world0 admission dry runs (namespace absent)" skip "world0 admission dry runs (namespace absent)"
fi fi