fix: M8 review findings (restore ownership, barrier tool, World0 claimRef, CI selection, drill verdicts)
P1: restore-world.sh claims its objects path (mkdir) and the database (a held session with a restore try-lock and the exclusive upgrade lock), and removes only what it created, so a losing restore never drops a winner's schema. P2: barrier Ready-pod selection via shared jsonpath helper with fixtures; barrier pod fsGroup; script-generated pods inspected by the policy check; World0 PV claim references protected; ci-paths.sh uses the event's real base and runs on uncertainty; fault-drill asserts RTO and needs positive quarantine evidence. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -20,6 +20,11 @@ env:
|
|||||||
HARBOR_USERNAME: ${{ secrets.HARBOR_USERNAME }}
|
HARBOR_USERNAME: ${{ secrets.HARBOR_USERNAME }}
|
||||||
HARBOR_PASSWORD: ${{ secrets.HARBOR_PASSWORD }}
|
HARBOR_PASSWORD: ${{ secrets.HARBOR_PASSWORD }}
|
||||||
CACHE_WRITE: '0'
|
CACHE_WRITE: '0'
|
||||||
|
# Change selection (tools/ci/ci-paths.sh) compares against the event's real base:
|
||||||
|
# the push's before SHA or the pull request's base; manual runs always run.
|
||||||
|
CI_EVENT: ${{ github.event_name }}
|
||||||
|
CI_BEFORE: ${{ github.event.before }}
|
||||||
|
CI_PR_BASE: ${{ github.event.pull_request.base.sha }}
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
checks:
|
checks:
|
||||||
@@ -40,6 +45,9 @@ jobs:
|
|||||||
CHECK_SUITE: ${{ matrix.suite }}
|
CHECK_SUITE: ${{ matrix.suite }}
|
||||||
run: |
|
run: |
|
||||||
set -eu
|
set -eu
|
||||||
|
# The selector's own regression runs first, so a broken selector cannot
|
||||||
|
# silently skip the suites.
|
||||||
|
if [ "$CHECK_SUITE" = deploy ]; then bash tools/ci/ci-paths-check.sh; fi
|
||||||
if ! bash tools/ci/ci-paths.sh "$CHECK_SUITE"; then
|
if ! bash tools/ci/ci-paths.sh "$CHECK_SUITE"; then
|
||||||
echo "No files relevant to $CHECK_SUITE changed; gate skipped."
|
echo "No files relevant to $CHECK_SUITE changed; gate skipped."
|
||||||
exit 0
|
exit 0
|
||||||
|
|||||||
@@ -60,7 +60,8 @@ COPY deploy/ deploy/
|
|||||||
COPY tools/ci/deploy-policy-check.py tools/ci/
|
COPY tools/ci/deploy-policy-check.py tools/ci/
|
||||||
COPY tools/ops/ tools/ops/
|
COPY tools/ops/ tools/ops/
|
||||||
COPY tools/backup/ tools/backup/
|
COPY tools/backup/ tools/backup/
|
||||||
RUN python3 tools/ci/deploy-policy-check.py
|
COPY tools/ci/ops-lib-check.sh tools/ci/
|
||||||
|
RUN python3 tools/ci/deploy-policy-check.py && bash tools/ci/ops-lib-check.sh
|
||||||
# M8.5: monitoring configuration and alert tests with the pinned images' own tools.
|
# M8.5: monitoring configuration and alert tests with the pinned images' own tools.
|
||||||
COPY --from=quay.io/prometheus/prometheus@sha256:6976aa8a60fec930796ce5772b8d12da7a318a5daa8d40d69c5c7819a05eeed7 /bin/promtool /usr/local/bin/promtool
|
COPY --from=quay.io/prometheus/prometheus@sha256:6976aa8a60fec930796ce5772b8d12da7a318a5daa8d40d69c5c7819a05eeed7 /bin/promtool /usr/local/bin/promtool
|
||||||
COPY --from=quay.io/prometheus/alertmanager@sha256:e9733bafb1bdef9b00e25a21f8f99dc26a22224bf16641ad754d1649f4c3357a /bin/amtool /usr/local/bin/amtool
|
COPY --from=quay.io/prometheus/alertmanager@sha256:e9733bafb1bdef9b00e25a21f8f99dc26a22224bf16641ad754d1649f4c3357a /bin/amtool /usr/local/bin/amtool
|
||||||
|
|||||||
@@ -12,6 +12,10 @@
|
|||||||
# - deleting a PersistentVolume bound to evolution-world0, or changing its reclaim
|
# - deleting a PersistentVolume bound to evolution-world0, or changing its reclaim
|
||||||
# policy away from Retain once set (set Retain when the volume is bound: the
|
# policy away from Retain once set (set Retain when the volume is bound: the
|
||||||
# only StorageClass, local-path, provisions with Delete);
|
# only StorageClass, local-path, provisions with Delete);
|
||||||
|
# - removing or reassigning such a volume's claim reference (namespace or claim
|
||||||
|
# name): the reference is how this policy recognizes a World0 volume, so clearing
|
||||||
|
# it first would otherwise strip the protection (review M8 P2). The binder may
|
||||||
|
# still fill in uid/resourceVersion; reassignment is a break-glass operation;
|
||||||
# - deleting the aevolution-world0 or aevolution-platform Argo CD Application or
|
# - deleting the aevolution-world0 or aevolution-platform Argo CD Application or
|
||||||
# AppProject (the platform app owns this policy and the World0 namespace).
|
# AppProject (the platform app owns this policy and the World0 namespace).
|
||||||
# Kubernetes does not run admission policies on policy objects themselves, so
|
# Kubernetes does not run admission policies on policy objects themselves, so
|
||||||
@@ -78,6 +82,14 @@ spec:
|
|||||||
object.spec.persistentVolumeReclaimPolicy == 'Retain'
|
object.spec.persistentVolumeReclaimPolicy == 'Retain'
|
||||||
message: "A World0 PersistentVolume set to Retain cannot be changed back"
|
message: "A World0 PersistentVolume set to Retain cannot be changed back"
|
||||||
reason: Forbidden
|
reason: Forbidden
|
||||||
|
- expression: >-
|
||||||
|
variables.breakGlass || request.operation != 'UPDATE' || !variables.world0Volume ||
|
||||||
|
(has(object.spec.claimRef) && has(object.spec.claimRef.namespace) &&
|
||||||
|
object.spec.claimRef.namespace == 'evolution-world0' &&
|
||||||
|
(!has(oldObject.spec.claimRef.name) ||
|
||||||
|
(has(object.spec.claimRef.name) && object.spec.claimRef.name == oldObject.spec.claimRef.name)))
|
||||||
|
message: "A World0 PersistentVolume keeps its World0 claim reference without the aevolution:world0-break-glass group"
|
||||||
|
reason: Forbidden
|
||||||
---
|
---
|
||||||
apiVersion: admissionregistration.k8s.io/v1
|
apiVersion: admissionregistration.k8s.io/v1
|
||||||
kind: ValidatingAdmissionPolicyBinding
|
kind: ValidatingAdmissionPolicyBinding
|
||||||
|
|||||||
+16
-2
@@ -827,8 +827,9 @@ measurements (worker loss RPO 0 / RTO <= 60 s; host loss = explicit rewind to th
|
|||||||
last verified bundle); **World0 launch stays blocked** because no independent
|
last verified bundle); **World0 launch stays blocked** because no independent
|
||||||
durable failure domain exists, enforced by the deploy policy check; host-loss
|
durable failure domain exists, enforced by the deploy policy check; host-loss
|
||||||
restore in place with barrier fencing of the superseded database rehearsed
|
restore in place with barrier fencing of the superseded database rehearsed
|
||||||
locally; promotion only of stage-pinned digests. **M8-dev/stage passed;
|
locally; promotion only of stage-pinned digests. **M8-World0 launch not passed**
|
||||||
M8-World0 launch not passed.**
|
(blocked by design). M8-dev/stage is implemented and locally qualified, not yet
|
||||||
|
passed: see "M8 gate status" below.
|
||||||
|
|
||||||
M8.9 implemented and locally qualified (2026-09-30), under ADR-0076: accelerated
|
M8.9 implemented and locally qualified (2026-09-30), under ADR-0076: accelerated
|
||||||
clone runs of the current and candidate builds compared state-for-state and
|
clone runs of the current and candidate builds compared state-for-state and
|
||||||
@@ -837,3 +838,16 @@ version barrier rehearsed with real hosts; `upgrade-barrier.sh` for dev/stage;
|
|||||||
stores/compute node labels with policy-checked placement and `scheduling-check.sh`.
|
stores/compute node labels with policy-checked placement and `scheduling-check.sh`.
|
||||||
Rules changes stay refused until a converter ADR. In-cluster barrier and
|
Rules changes stay refused until a converter ADR. In-cluster barrier and
|
||||||
additional-node evidence wait on the maintainer.
|
additional-node evidence wait on the maintainer.
|
||||||
|
|
||||||
|
**M8 gate status (2026-09-30, after the M8.1–M8.9 review).** Implementation and
|
||||||
|
local qualification are complete; these live gates are still open, so M8-dev/stage
|
||||||
|
is **not passed**:
|
||||||
|
- M8.6: a valid dev database-outage run, and a re-run of full/empty with the new
|
||||||
|
image (positive quarantine evidence, RTO asserted against 60 s);
|
||||||
|
- M8.7: an in-cluster `cluster-backup.sh` bundle restore-tested off the host; the
|
||||||
|
destination, schedule, retention, encryption and WAL archiving (backlog);
|
||||||
|
- M8.5: an alert receiver and kubelet log-rotation check (backlog);
|
||||||
|
- M8.9: `upgrade-barrier.sh` on dev and `scheduling-check.sh` with a second labelled
|
||||||
|
node;
|
||||||
|
- M8.1/M8.3: storage re-measured after the RAID rework.
|
||||||
|
M8-World0 launch stays blocked by ADR-0075.
|
||||||
|
|||||||
@@ -130,3 +130,11 @@ What stays outside Argo:
|
|||||||
- Secrets, until a secret manager is chosen (docs/BACKLOG.md);
|
- Secrets, until a secret manager is chosen (docs/BACKLOG.md);
|
||||||
- the store node label;
|
- the store node label;
|
||||||
- the one-time bootstrap of the platform project and Application.
|
- the one-time bootstrap of the platform project and Application.
|
||||||
|
|
||||||
|
## Amendment: World0 volume claim references (M8 review, 2026-09-30)
|
||||||
|
The policy recognizes a World0 PersistentVolume by its claim reference. Clearing
|
||||||
|
or reassigning that reference first would have removed the protection. An UPDATE
|
||||||
|
that removes the reference, or changes its namespace or claim name, is now denied
|
||||||
|
without the break-glass group. The binder may still fill in uid and
|
||||||
|
resourceVersion. `verify-isolation.sh` tests this with a server-side dry run when
|
||||||
|
a World0 volume exists.
|
||||||
|
|||||||
@@ -90,3 +90,17 @@ PostgreSQL:
|
|||||||
without a shared pipe that the image does not have.
|
without a shared pipe that the image does not have.
|
||||||
- **Partitioning with NetworkPolicy.** It does not cut established connections
|
- **Partitioning with NetworkPolicy.** It does not cut established connections
|
||||||
under Calico.
|
under Calico.
|
||||||
|
|
||||||
|
## Amendment: drill verdicts (M8 review, 2026-09-30)
|
||||||
|
A missing observation no longer passes. The verdicts are in `tools/ops/lib/drill.sh`
|
||||||
|
and are fixture-tested by `tools/ci/ops-lib-check.sh`:
|
||||||
|
- Each kill must recover within `RTO_BUDGET_MS`, default 60 000 (ADR-0075).
|
||||||
|
- The empty scenario passes only on positive evidence: a `phase quarantined` line,
|
||||||
|
and exit 4 if the container ended. Any other exit fails. No evidence is
|
||||||
|
INCONCLUSIVE, which fails the drill.
|
||||||
|
- A dead or drained owner's acknowledgements are read from its log after the
|
||||||
|
stream has ended.
|
||||||
|
|
||||||
|
Limit: the drill reads acknowledgements from logs. A line that was written but not
|
||||||
|
streamed before a forced kill is not observed, so the zero-lost-acknowledgement
|
||||||
|
check covers observed acknowledgements only.
|
||||||
|
|||||||
@@ -172,3 +172,22 @@ every PostgreSQL build:
|
|||||||
authority. Isolation goes through the clone path, which re-identifies the world.
|
authority. Isolation goes through the clone path, which re-identifies the world.
|
||||||
- **A new object-storage dependency (MinIO, S3 SDK).** The destination is not
|
- **A new object-storage dependency (MinIO, S3 SDK).** The destination is not
|
||||||
chosen; a directory and `tar` need no new dependency.
|
chosen; a directory and `tar` need no new dependency.
|
||||||
|
|
||||||
|
## Amendment: restore ownership (M8 review, 2026-09-30)
|
||||||
|
The review reproduced a race. A restore checked that the target was empty, a
|
||||||
|
competing actor then created the schema, the restore's `pg_restore` failed, and
|
||||||
|
its cleanup dropped the other actor's schema. Restores now own what they touch:
|
||||||
|
- `--objects` is claimed by creating it (`mkdir`, never `-p`). An existing path is
|
||||||
|
refused and never touched.
|
||||||
|
- One psql session, held open for the whole restore, takes the restore claim
|
||||||
|
(advisory 71420004, try-lock, so a concurrent restore refuses at once) and the
|
||||||
|
upgrade lock (71420002, exclusive). Provisioning, acquisition and lease checks in
|
||||||
|
that database wait until the restore is verified or removed.
|
||||||
|
- Cleanup removes only what the invocation created: the schema only if its own
|
||||||
|
one-transaction `pg_restore` committed, and the directory only if its own `mkdir`
|
||||||
|
created it.
|
||||||
|
|
||||||
|
`backup-check.sh` covers three cases:
|
||||||
|
- a competing schema created just before `pg_restore` survives;
|
||||||
|
- a concurrent restore refuses, and the first completes;
|
||||||
|
- a lease acquisition times out while the restore holds the database.
|
||||||
|
|||||||
@@ -150,3 +150,15 @@ In the cluster, still to be run by the maintainer:
|
|||||||
exactly the named risk.
|
exactly the named risk.
|
||||||
- **Retiring the old build automatically on every sync.** Retirement is
|
- **Retiring the old build automatically on every sync.** Retirement is
|
||||||
irreversible. It stays a deliberate step, guarded by the checks above.
|
irreversible. It stays a deliberate step, guarded by the checks above.
|
||||||
|
|
||||||
|
## Amendment: barrier tool fixes (M8 review, 2026-09-30)
|
||||||
|
Two defects in `upgrade-barrier.sh` would have blocked the in-cluster barrier:
|
||||||
|
- Its Ready-pod template printed the list root instead of the pod name. Selection
|
||||||
|
now uses a plain jsonpath (`tools/ops/lib/pods.sh`) and requires exactly one
|
||||||
|
Ready pod that is not being deleted. Fixtures cover one pod, not Ready, being
|
||||||
|
deleted, no condition and two Ready pods.
|
||||||
|
- Its pod lacked `fsGroup`, so it could not read the 0440 Secret files. The pod
|
||||||
|
now follows the same contract as the other database-tool pods.
|
||||||
|
|
||||||
|
Both script-generated pods (backup and barrier) render with `--print-manifest`,
|
||||||
|
and the deploy policy check inspects them.
|
||||||
|
|||||||
@@ -3034,3 +3034,56 @@ Fixes (this segment):
|
|||||||
- scheduling onto a second labelled node;
|
- scheduling onto a second labelled node;
|
||||||
- the Windows MSVC build (no Windows target changed; the server and PostgreSQL
|
- the Windows MSVC build (no Windows target changed; the server and PostgreSQL
|
||||||
code are Linux-only).
|
code are Linux-only).
|
||||||
|
|
||||||
|
## M8.1–M8.9 review fixes - 2026-09-30
|
||||||
|
|
||||||
|
Review: `.cache/reviews/M8_REVIEW_680e7e5-5b482ad.md` (one P1, five P2).
|
||||||
|
- **P1 restore ownership.** `backup-check.sh` passed in the Release, analysis and
|
||||||
|
ASan/UBSan PostgreSQL builds, including three new cases:
|
||||||
|
- A schema created after the checks but before `pg_restore`: exit 4 ("pg_restore
|
||||||
|
failed; the database was not changed"). The competing table survived, and only
|
||||||
|
this restore's own objects directory was removed.
|
||||||
|
- A second concurrent restore of the same database: exit 4 ("another restore
|
||||||
|
holds this database"). It left no directory, and the first restore was
|
||||||
|
verified.
|
||||||
|
- While the first restore held the database, a lease acquisition there ended in
|
||||||
|
`lock timeout`. An occupied objects path was refused and left untouched.
|
||||||
|
- The reviewer's `restore-race.sh` now ends with `remaining_evolution_schemas=1`
|
||||||
|
(`schema=no, objects=yes`).
|
||||||
|
- **P2 barrier Ready-pod selection.** `tools/ops/lib/pods.sh` uses a plain
|
||||||
|
jsonpath and needs exactly one Ready pod that is not being deleted.
|
||||||
|
`ops-lib-check.sh` passed its fixtures: one pod, not Ready, being deleted, no
|
||||||
|
condition, Ready beside a deleting pod, two Ready pods, and no pods. Not run:
|
||||||
|
the jsonpath against a live API server (no kubectl or Go in this environment).
|
||||||
|
- **P2 barrier Secret access.** The generated pod has `fsGroup: 10001`. The deploy
|
||||||
|
policy check now renders the backup and barrier pods with `--print-manifest` and
|
||||||
|
checks: non-root 10001 with fsGroup 10001, no token, read-only root, no
|
||||||
|
escalation, Secret files 0440, and egress admission of the component. Not run: a
|
||||||
|
disposable pod reading the Secret in the cluster.
|
||||||
|
- **P2 World0 claimRef.** A new admission validation denies removing a World0
|
||||||
|
volume's claim reference, or changing its namespace or name, without
|
||||||
|
break-glass. The policy check requires the rule, and `verify-isolation.sh`
|
||||||
|
dry-runs the removal when a World0 PV exists. Not run: no admission test cluster
|
||||||
|
here, and no World0 PV exists.
|
||||||
|
- **P2 CI change selection.** `ci-paths.sh` compares against the event's base (the
|
||||||
|
push's `before`, or the PR base, fetched if missing). It runs on manual
|
||||||
|
dispatch, local runs and any uncertainty, and the path lists now include the
|
||||||
|
check scripts. `ci-paths-check.sh` passed 11 fixtures, and the deploy suite runs
|
||||||
|
it first. The reviewer's `ci-range.sh` now runs in both cases. Not run: a real
|
||||||
|
Gitea push, so the event-context mapping is unverified.
|
||||||
|
- **P2 drill verdicts.** Every kill RTO is asserted against `RTO_BUDGET_MS`
|
||||||
|
(60 000). The empty scenario needs positive quarantine evidence; INCONCLUSIVE and
|
||||||
|
other exits fail. Acknowledgements are read from the dead or drained owner's
|
||||||
|
finalized log. The fixtures passed. Not run: the drill itself.
|
||||||
|
- Mutations, each caught:
|
||||||
|
- dropping a schema that was not installed: `backup-check.sh` failed;
|
||||||
|
- removing `fsGroup`: the policy check failed;
|
||||||
|
- accepting several Ready pods, or treating never-ran as PASS: the
|
||||||
|
`ops-lib-check.sh` fixtures failed;
|
||||||
|
- falling back to `HEAD^` when there is no base: the `ci-paths-check.sh` fixtures
|
||||||
|
failed.
|
||||||
|
- The host-loss and upgrade rehearsals, and the PostgreSQL migration and
|
||||||
|
permission suites, still pass in all three builds. The deploy policy check: 67
|
||||||
|
objects, 0 failures.
|
||||||
|
- ROADMAP corrected: M8-dev/stage is implemented and locally qualified, but **not
|
||||||
|
passed** until the live gates listed there are run.
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
# (ADR-0074).
|
# (ADR-0074).
|
||||||
#
|
#
|
||||||
# tools/backup/cluster-backup.sh <dev|stage> --to DIR [--verify WORLD] [--context NAME]
|
# tools/backup/cluster-backup.sh <dev|stage> --to DIR [--verify WORLD] [--context NAME]
|
||||||
|
# tools/backup/cluster-backup.sh <dev|stage> --print-manifest
|
||||||
#
|
#
|
||||||
# Runs one short-lived Pod in evolution-<env> (image: the postgres-tools digest
|
# Runs one short-lived Pod in evolution-<env> (image: the postgres-tools digest
|
||||||
# pinned in deploy/kubernetes/stores/<env>; migrator login; the aevolution-objects
|
# pinned in deploy/kubernetes/stores/<env>; migrator login; the aevolution-objects
|
||||||
@@ -14,11 +15,12 @@
|
|||||||
# hypervisor host. --verify WORLD then runs verify-backup.sh on this machine
|
# hypervisor host. --verify WORLD then runs verify-backup.sh on this machine
|
||||||
# (restore on a private cluster, rebuilt and served as isolated world WORLD); it
|
# (restore on a private cluster, rebuilt and served as isolated world WORLD); it
|
||||||
# needs the PostgreSQL 18.6 binaries and evo-server/evo-world-clone.
|
# needs the PostgreSQL 18.6 binaries and evo-server/evo-world-clone.
|
||||||
# World0 is refused before anything touches the cluster. The Pod is an operator
|
# --print-manifest prints the Pod and exits without touching the cluster (the deploy
|
||||||
|
# policy check inspects it). World0 is refused before anything touches the cluster. The Pod is an operator
|
||||||
# action, like fault-drill.sh; it is not part of any Argo application. A scheduled
|
# action, like fault-drill.sh; it is not part of any Argo application. A scheduled
|
||||||
# backup (CronJob in Git, synced by Argo) follows once the destination exists.
|
# backup (CronJob in Git, synced by Argo) follows once the destination exists.
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
usage() { echo "usage: $0 <dev|stage> --to DIR [--verify WORLD] [--context NAME]" >&2; exit 2; }
|
usage() { echo "usage: $0 <dev|stage> --to DIR [--verify WORLD] [--context NAME] | --print-manifest" >&2; exit 2; }
|
||||||
[ $# -ge 1 ] || usage
|
[ $# -ge 1 ] || usage
|
||||||
env="$1"; shift
|
env="$1"; shift
|
||||||
case "$env" in
|
case "$env" in
|
||||||
@@ -26,16 +28,19 @@ case "$env" in
|
|||||||
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
|
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
|
||||||
exit 2 ;;
|
exit 2 ;;
|
||||||
esac
|
esac
|
||||||
to="" verify="" k=(kubectl --request-timeout=30s)
|
to="" verify="" print=0 k=(kubectl --request-timeout=30s)
|
||||||
while [ $# -gt 0 ]; do
|
while [ $# -gt 0 ]; do
|
||||||
case "$1" in
|
case "$1" in
|
||||||
--to) to="${2:-}"; shift 2 ;;
|
--to) to="${2:-}"; shift 2 ;;
|
||||||
--verify) verify="${2:-}"; shift 2 ;;
|
--verify) verify="${2:-}"; shift 2 ;;
|
||||||
|
--print-manifest) print=1; shift ;;
|
||||||
--context) k+=(--context "${2:?}"); shift 2 ;;
|
--context) k+=(--context "${2:?}"); shift 2 ;;
|
||||||
*) usage ;;
|
*) usage ;;
|
||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
|
if [ "$print" != 1 ]; then
|
||||||
[ -n "$to" ] && [ -d "$to" ] || { echo "--to must be an existing directory" >&2; exit 2; }
|
[ -n "$to" ] && [ -d "$to" ] || { echo "--to must be an existing directory" >&2; exit 2; }
|
||||||
|
fi
|
||||||
root="$(cd "$(dirname "$0")/../.." && pwd)"
|
root="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||||
ns="evolution-$env"
|
ns="evolution-$env"
|
||||||
image="$(awk '/name: aevolution-postgres-tools/ {f = 1} f && /newName:/ {n = $2} f && /digest:/ {print n "@" $2; exit}' \
|
image="$(awk '/name: aevolution-postgres-tools/ {f = 1} f && /newName:/ {n = $2} f && /digest:/ {print n "@" $2; exit}' \
|
||||||
@@ -45,15 +50,14 @@ stamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
|||||||
pod="aevolution-backup-$(date -u +%Y%m%d%H%M%S)"
|
pod="aevolution-backup-$(date -u +%Y%m%d%H%M%S)"
|
||||||
dest="$to/$env-$stamp"
|
dest="$to/$env-$stamp"
|
||||||
say() { echo "$(date -u +%H:%M:%S) $*"; }
|
say() { echo "$(date -u +%H:%M:%S) $*"; }
|
||||||
cleanup() { "${k[@]}" delete pod -n "$ns" "$pod" --wait=false > /dev/null 2>&1 || true; }
|
|
||||||
trap cleanup EXIT
|
|
||||||
|
|
||||||
say "starting $pod in $ns ($image)"
|
manifest() {
|
||||||
"${k[@]}" apply -n "$ns" -f - > /dev/null <<YAML
|
cat <<YAML
|
||||||
apiVersion: v1
|
apiVersion: v1
|
||||||
kind: Pod
|
kind: Pod
|
||||||
metadata:
|
metadata:
|
||||||
name: $pod
|
name: $pod
|
||||||
|
namespace: $ns
|
||||||
labels:
|
labels:
|
||||||
app.kubernetes.io/part-of: aevolution
|
app.kubernetes.io/part-of: aevolution
|
||||||
app.kubernetes.io/component: backup
|
app.kubernetes.io/component: backup
|
||||||
@@ -112,6 +116,13 @@ spec:
|
|||||||
- name: scratch
|
- name: scratch
|
||||||
emptyDir: {medium: Memory, sizeLimit: 8Mi}
|
emptyDir: {medium: Memory, sizeLimit: 8Mi}
|
||||||
YAML
|
YAML
|
||||||
|
}
|
||||||
|
if [ "$print" = 1 ]; then manifest; exit 0; fi
|
||||||
|
|
||||||
|
cleanup() { "${k[@]}" delete pod -n "$ns" "$pod" --wait=false > /dev/null 2>&1 || true; }
|
||||||
|
trap cleanup EXIT
|
||||||
|
say "starting $pod in $ns ($image)"
|
||||||
|
manifest | "${k[@]}" apply -f - > /dev/null
|
||||||
|
|
||||||
for _ in $(seq 1 1800); do
|
for _ in $(seq 1 1800); do
|
||||||
phase="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}' 2> /dev/null || true)"
|
phase="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}' 2> /dev/null || true)"
|
||||||
|
|||||||
@@ -14,13 +14,23 @@
|
|||||||
# Order, failing closed at each step:
|
# Order, failing closed at each step:
|
||||||
# 1. every bundle file matches SHA256SUMS, no file is unlisted, and every object's
|
# 1. every bundle file matches SHA256SUMS, no file is unlisted, and every object's
|
||||||
# name is its digest; nothing is touched before this passes;
|
# name is its digest; nothing is touched before this passes;
|
||||||
# 2. the target has no `evolution` schema and runs the manifest's server version;
|
# 2. the targets are claimed for this invocation alone (below), then the target
|
||||||
|
# has no `evolution` schema and runs the manifest's server version;
|
||||||
# 3. pg_restore in one transaction, then objects are copied;
|
# 3. pg_restore in one transaction, then objects are copied;
|
||||||
# 4. the restored heads equal the MANIFEST heads exactly;
|
# 4. the restored heads equal the MANIFEST heads exactly;
|
||||||
# 5. every region's full history is audited (evo-world-clone audit: each link's
|
# 5. every region's full history is audited (evo-world-clone audit: each link's
|
||||||
# manifest, payload, trace replay and outcomes, from genesis to the head).
|
# manifest, payload, trace replay and outcomes, from genesis to the head).
|
||||||
# On a failure after step 2 the restored schema and the copied objects are removed,
|
# Ownership (review M8 P1): --objects is claimed by creating it (mkdir, never -p: an
|
||||||
# so an unverified restore can never be mistaken for a world. The result is a
|
# existing path is refused and never touched), and the database by one psql session
|
||||||
|
# held open for the whole restore. That session takes the restore claim (advisory
|
||||||
|
# 71420004, try-lock: a concurrent restore of the same database refuses at once) and
|
||||||
|
# the upgrade lock exclusively (71420002: provisioning, acquisition and lease checks
|
||||||
|
# in that database wait until this restore is verified or removed). On failure only
|
||||||
|
# what this invocation created is removed: the schema only if its own pg_restore
|
||||||
|
# committed, the objects directory only if its own mkdir created it. So an
|
||||||
|
# unverified restore can never be mistaken for a world, and a losing restore never
|
||||||
|
# deletes a winner's state. EVO_RESTORE_TEST_HOOK (tests only) is run by bash with
|
||||||
|
# $1 = before-restore (claimed, checked) and after-restore (installed, objects copied). The result is a
|
||||||
# verbatim, owner-only copy with the source's world identities: it has no runtime
|
# verbatim, owner-only copy with the source's world identities: it has no runtime
|
||||||
# grants and is never served. To run it, clone it into an isolated world
|
# grants and is never served. To run it, clone it into an isolated world
|
||||||
# (tools/backup/verify-backup.sh --isolate).
|
# (tools/backup/verify-backup.sh --isolate).
|
||||||
@@ -89,32 +99,69 @@ if [ "$in_place" = 1 ]; then
|
|||||||
privileges=()
|
privileges=()
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# 2. Target: empty, same server version.
|
target="${PGDATABASE:-}"
|
||||||
[ ! -e "$objects" ] || refuse "objects target already exists: $objects"
|
[ -n "${PGSERVICE:-}" ] && target="service=$PGSERVICE"
|
||||||
|
[ -n "$target" ] || { echo "set PGDATABASE or PGSERVICE for the target" >&2; exit 2; }
|
||||||
|
|
||||||
|
# 2. Claim the targets, then check them under the claim.
|
||||||
|
created_objects=0 installed=0 restored=0 lock_pid="" claim=""
|
||||||
|
cleanup() {
|
||||||
|
local code=$?
|
||||||
|
if [ "$restored" != 1 ]; then
|
||||||
|
if [ "$installed" = 1 ]; then
|
||||||
|
psql -X --no-password -q -v ON_ERROR_STOP=1 -c 'DROP SCHEMA evolution CASCADE' > /dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
[ "$created_objects" = 1 ] && rm -rf -- "$objects"
|
||||||
|
if [ "$installed" = 1 ] || [ "$created_objects" = 1 ]; then
|
||||||
|
echo "RESTORE FAILED: removed only what this restore created" \
|
||||||
|
"(schema=$([ "$installed" = 1 ] && echo yes || echo no), objects=$([ "$created_objects" = 1 ] && echo yes || echo no))" >&2
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
[ -n "$claim" ] && exec 7>&- # Ends the claim session: its locks are released.
|
||||||
|
[ -n "$lock_pid" ] && { wait "$lock_pid" 2> /dev/null || true; }
|
||||||
|
[ -n "$claim" ] && rm -rf -- "$claim"
|
||||||
|
return "$code"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
mkdir -m 700 -- "$objects" 2> /dev/null || refuse "objects target already exists or cannot be created: $objects"
|
||||||
|
created_objects=1
|
||||||
|
claim="$(mktemp -d /tmp/evo-restore-claim.XXXXXXXX)"
|
||||||
|
mkfifo "$claim/in"
|
||||||
|
psql -X --no-password -q -A -t -v ON_ERROR_STOP=1 < "$claim/in" > "$claim/out" 2>&1 &
|
||||||
|
lock_pid=$!
|
||||||
|
exec 7> "$claim/in"
|
||||||
|
cat >&7 <<SQL
|
||||||
|
SELECT pg_catalog.pg_try_advisory_lock(71420004) AS claimed \gset
|
||||||
|
\if :claimed
|
||||||
|
SELECT pg_catalog.pg_advisory_lock(71420002);
|
||||||
|
\! touch '$claim/held'
|
||||||
|
\else
|
||||||
|
\! touch '$claim/busy'
|
||||||
|
\endif
|
||||||
|
SQL
|
||||||
|
for _ in $(seq 1 600); do
|
||||||
|
[ -e "$claim/held" ] || [ -e "$claim/busy" ] && break
|
||||||
|
kill -0 "$lock_pid" 2> /dev/null || break
|
||||||
|
sleep 0.1
|
||||||
|
done
|
||||||
|
[ -e "$claim/busy" ] && refuse "another restore holds this database"
|
||||||
|
[ -e "$claim/held" ] || refuse "could not claim the target database ($(head -c 200 "$claim/out"))"
|
||||||
version="$(psql -X --no-password -A -t -v ON_ERROR_STOP=1 -c "SELECT current_setting('server_version')")"
|
version="$(psql -X --no-password -A -t -v ON_ERROR_STOP=1 -c "SELECT current_setting('server_version')")"
|
||||||
grep -qx "database $version" "$bundle/MANIFEST" || refuse "target server $version differs from the manifest"
|
grep -qx "database $version" "$bundle/MANIFEST" || refuse "target server $version differs from the manifest"
|
||||||
existing="$(psql -X --no-password -A -t -v ON_ERROR_STOP=1 -c "SELECT count(*) FROM pg_namespace WHERE nspname = 'evolution'")"
|
existing="$(psql -X --no-password -A -t -v ON_ERROR_STOP=1 -c "SELECT count(*) FROM pg_namespace WHERE nspname = 'evolution'")"
|
||||||
[ "$existing" = 0 ] || refuse "target database already has an evolution schema"
|
[ "$existing" = 0 ] || refuse "target database already has an evolution schema"
|
||||||
|
|
||||||
restored=0
|
# 3. Database in one transaction, then objects. A failed pg_restore installed
|
||||||
cleanup() {
|
# nothing (one transaction), so nothing of the database is removed after it.
|
||||||
[ "$restored" = 1 ] && return
|
hook() { [ -z "${EVO_RESTORE_TEST_HOOK:-}" ] || bash -c "$EVO_RESTORE_TEST_HOOK" hook "$1"; }
|
||||||
psql -X --no-password -q -v ON_ERROR_STOP=1 -c 'DROP SCHEMA IF EXISTS evolution CASCADE' > /dev/null 2>&1 || true
|
hook before-restore
|
||||||
rm -rf -- "$objects"
|
|
||||||
echo "RESTORE FAILED: the restored schema and objects were removed" >&2
|
|
||||||
}
|
|
||||||
trap cleanup EXIT
|
|
||||||
|
|
||||||
# 3. Database in one transaction, then objects.
|
|
||||||
target="${PGDATABASE:-}"
|
|
||||||
[ -n "${PGSERVICE:-}" ] && target="service=$PGSERVICE"
|
|
||||||
[ -n "$target" ] || { echo "set PGDATABASE or PGSERVICE for the target" >&2; exit 2; }
|
|
||||||
pg_restore --no-password --exit-on-error --single-transaction --no-owner "${privileges[@]}" \
|
pg_restore --no-password --exit-on-error --single-transaction --no-owner "${privileges[@]}" \
|
||||||
--dbname="$target" "$bundle/database.dump"
|
--dbname="$target" "$bundle/database.dump" || refuse "pg_restore failed; the database was not changed"
|
||||||
mkdir -m 700 -- "$objects"
|
installed=1
|
||||||
(cd "$bundle/objects" && find . -mindepth 2 -maxdepth 2 -type d -printf '%P\0' |
|
(cd "$bundle/objects" && find . -mindepth 2 -maxdepth 2 -type d -printf '%P\0' |
|
||||||
xargs -0 -r -I{} mkdir -p -m 700 -- "$objects/{}")
|
xargs -0 -r -I{} mkdir -p -m 700 -- "$objects/{}")
|
||||||
(cd "$bundle/objects" && find . -type f -printf '%P\0' | xargs -0 -r -I{} cp -- {} "$objects/{}")
|
(cd "$bundle/objects" && find . -type f -printf '%P\0' | xargs -0 -r -I{} cp -- {} "$objects/{}")
|
||||||
|
hook after-restore
|
||||||
|
|
||||||
# 4. Heads exactly as recorded.
|
# 4. Heads exactly as recorded.
|
||||||
actual_heads="$(psql -X --no-password -A -t -F ' ' -v ON_ERROR_STOP=1 -c "
|
actual_heads="$(psql -X --no-password -A -t -F ' ' -v ON_ERROR_STOP=1 -c "
|
||||||
|
|||||||
@@ -90,4 +90,46 @@ PGDATABASE=evo_restore_twice bash "$root/tools/backup/restore-world.sh" --bundle
|
|||||||
[ "$code" = 4 ] && grep -q 'already has an evolution schema' "$work/twice.log" || fail "a second restore into one database"
|
[ "$code" = 4 ] && grep -q 'already has an evolution schema' "$work/twice.log" || fail "a second restore into one database"
|
||||||
[ "$(PGDATABASE=evo_restore_twice psql -XAtc 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 300')" = "$backup_tick" ] ||
|
[ "$(PGDATABASE=evo_restore_twice psql -XAtc 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 300')" = "$backup_tick" ] ||
|
||||||
fail "the refused second restore changed the first"
|
fail "the refused second restore changed the first"
|
||||||
|
# Ownership races (review M8 P1). A competing actor creates the schema between this
|
||||||
|
# restore's checks and its pg_restore: the restore fails and leaves that schema alone.
|
||||||
|
createdb evo_restore_race
|
||||||
|
code=0
|
||||||
|
PGDATABASE=evo_restore_race EVO_RESTORE_TEST_HOOK='[ "$1" = before-restore ] && psql -XAtq -c "CREATE SCHEMA evolution" -c "CREATE TABLE evolution.winner(x int)" || true' \
|
||||||
|
bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" --objects "$work/race-a" \
|
||||||
|
> "$work/race-a.log" 2>&1 || code=$?
|
||||||
|
[ "$code" = 4 ] && grep -q 'pg_restore failed; the database was not changed' "$work/race-a.log" ||
|
||||||
|
{ cat "$work/race-a.log" >&2; fail "a pre-empted restore: exit $code"; }
|
||||||
|
[ "$(PGDATABASE=evo_restore_race psql -XAtc "SELECT count(*) FROM pg_tables WHERE schemaname = 'evolution' AND tablename = 'winner'")" = 1 ] ||
|
||||||
|
fail "a pre-empted restore deleted the competing schema"
|
||||||
|
[ ! -e "$work/race-a" ] || fail "a pre-empted restore left its own objects directory"
|
||||||
|
# Two restores of one database: the second refuses at once and touches nothing of
|
||||||
|
# the first, which completes. While the first holds the database (after its install),
|
||||||
|
# a lease acquisition there waits instead of running against unverified state.
|
||||||
|
createdb evo_restore_pair
|
||||||
|
PGDATABASE=evo_restore_pair EVO_RESTORE_TEST_HOOK='
|
||||||
|
if [ "$1" = before-restore ]; then touch "'"$work"'/pair-claimed"; sleep 3; fi
|
||||||
|
if [ "$1" = after-restore ]; then
|
||||||
|
psql -XAt -c "SET lock_timeout = '"'"'1s'"'"'" -c "SELECT * FROM evolution.acquire_region_lease(300, 1, 20000, decode(repeat('"'"'b1'"'"', 32), '"'"'hex'"'"'))" > "'"$work"'/pair-acquire.log" 2>&1 || true
|
||||||
|
fi' bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" --objects "$work/pair-a" \
|
||||||
|
> "$work/pair-a.log" 2>&1 &
|
||||||
|
pair_pid=$!
|
||||||
|
for _ in $(seq 1 100); do [ -e "$work/pair-claimed" ] && break; sleep 0.1; done
|
||||||
|
code=0
|
||||||
|
PGDATABASE=evo_restore_pair bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" \
|
||||||
|
--objects "$work/pair-b" > "$work/pair-b.log" 2>&1 || code=$?
|
||||||
|
[ "$code" = 4 ] && grep -q 'another restore holds this database' "$work/pair-b.log" ||
|
||||||
|
{ cat "$work/pair-b.log" >&2; fail "a concurrent restore: exit $code"; }
|
||||||
|
[ ! -e "$work/pair-b" ] || fail "the refused concurrent restore left its objects directory"
|
||||||
|
code=0; wait "$pair_pid" || code=$?
|
||||||
|
[ "$code" = 0 ] && grep -q '^RESTORE VERIFIED regions=1 ' "$work/pair-a.log" ||
|
||||||
|
{ cat "$work/pair-a.log" >&2; fail "the first of two restores: exit $code"; }
|
||||||
|
grep -q 'lock timeout' "$work/pair-acquire.log" ||
|
||||||
|
{ cat "$work/pair-acquire.log" >&2; fail "a lease acquisition ran during the restore"; }
|
||||||
|
# An existing objects path is never claimed or removed.
|
||||||
|
mkdir "$work/occupied"; echo keep > "$work/occupied/mine"
|
||||||
|
createdb evo_restore_occupied
|
||||||
|
code=0
|
||||||
|
PGDATABASE=evo_restore_occupied bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" \
|
||||||
|
--objects "$work/occupied" > "$work/occupied.log" 2>&1 || code=$?
|
||||||
|
[ "$code" = 4 ] && [ "$(cat "$work/occupied/mine")" = keep ] || fail "an occupied objects path: exit $code"
|
||||||
echo "backup and restore qualification passed (backup tick $backup_tick, acknowledged $acked_before, source $source_tick)"
|
echo "backup and restore qualification passed (backup tick $backup_tick, acknowledged $acked_before, source $source_tick)"
|
||||||
|
|||||||
@@ -0,0 +1,45 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Offline regression for tools/ci/ci-paths.sh (review M8 P2) in a disposable Git
|
||||||
|
# repository: multi-commit pushes ending in documentation, missing and unknown bases,
|
||||||
|
# pull requests, manual dispatch, and changes to only a check script. Uncertainty
|
||||||
|
# must run validation. Run by the deploy suite before it decides anything.
|
||||||
|
set -euo pipefail
|
||||||
|
script="$(cd "$(dirname "$0")" && pwd)/ci-paths.sh"
|
||||||
|
repo="$(mktemp -d /tmp/evo-ci-paths.XXXXXXXX)"
|
||||||
|
trap 'rm -rf -- "$repo"' EXIT
|
||||||
|
cd "$repo"
|
||||||
|
git init -q
|
||||||
|
git config user.email ci@example.invalid
|
||||||
|
git config user.name ci
|
||||||
|
mkdir -p src docs tools/ci
|
||||||
|
echo base > src/a.cpp; echo base > docs/a.md; echo base > tools/ci/dr-check.sh
|
||||||
|
git add .; git commit -qm base
|
||||||
|
base="$(git rev-parse HEAD)"
|
||||||
|
echo change >> src/a.cpp; git commit -qam implementation
|
||||||
|
echo change >> docs/a.md; git commit -qam documentation
|
||||||
|
docs_only_base="$(git rev-parse HEAD~1)"
|
||||||
|
failures=0
|
||||||
|
expect() { # expect <label> <0 run|1 skip> <env...> -- <suite>
|
||||||
|
local label="$1" want="$2"; shift 2
|
||||||
|
local envs=()
|
||||||
|
while [ "$1" != -- ]; do envs+=("$1"); shift; done
|
||||||
|
shift
|
||||||
|
local got=0
|
||||||
|
env -u GITHUB_EVENT_BEFORE -u GITHUB_EVENT_NAME -u CI_EVENT -u CI_BEFORE -u CI_PR_BASE \
|
||||||
|
"${envs[@]}" bash "$script" "$1" || got=$?
|
||||||
|
if [ "$got" = "$want" ]; then echo "ok $label"; else echo "FAIL $label: exit $got, wanted $want"; failures=$((failures + 1)); fi
|
||||||
|
}
|
||||||
|
expect "push: code then docs runs checks" 0 CI_EVENT=push "CI_BEFORE=$base" -- checks
|
||||||
|
expect "push: docs-only commit skips checks" 1 CI_EVENT=push "CI_BEFORE=$docs_only_base" -- checks
|
||||||
|
expect "push: no before SHA runs" 0 CI_EVENT=push -- checks
|
||||||
|
expect "push: all-zero before (new branch) runs" 0 CI_EVENT=push CI_BEFORE=0000000000000000000000000000000000000000 -- checks
|
||||||
|
expect "push: unknown, unfetchable base runs" 0 CI_EVENT=push CI_BEFORE=1111111111111111111111111111111111111111 -- checks
|
||||||
|
expect "pull request: base diff runs checks" 0 CI_EVENT=pull_request "CI_PR_BASE=$base" -- checks
|
||||||
|
expect "pull request: no base runs" 0 CI_EVENT=pull_request -- deploy
|
||||||
|
expect "manual dispatch always runs" 0 CI_EVENT=workflow_dispatch "CI_BEFORE=$docs_only_base" -- checks
|
||||||
|
expect "local run (no event) runs" 0 -- postgres
|
||||||
|
echo change >> tools/ci/dr-check.sh; git commit -qam dr-script
|
||||||
|
script_base="$(git rev-parse HEAD~1)"
|
||||||
|
expect "push: only dr-check.sh changed runs postgres" 0 CI_EVENT=push "CI_BEFORE=$script_base" -- postgres
|
||||||
|
expect "push: only dr-check.sh changed skips image" 1 CI_EVENT=push "CI_BEFORE=$script_base" -- image
|
||||||
|
[ "$failures" = 0 ] && echo "ci path selection fixtures passed" || { echo "$failures fixture(s) failed"; exit 1; }
|
||||||
+31
-11
@@ -1,35 +1,55 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
|
# Decides whether a CI suite must run: exit 0 = run, exit 1 = nothing relevant changed.
|
||||||
|
#
|
||||||
|
# The comparison base comes from the event (review M8 P2), never from HEAD^ alone:
|
||||||
|
# - push: CI_BEFORE (the push's `before` SHA), so every commit of a multi-commit push
|
||||||
|
# counts, not just the last;
|
||||||
|
# - pull_request: CI_PR_BASE (the base SHA), diffed against the checked-out merge;
|
||||||
|
# - workflow_dispatch, schedule, a local run or any other event: always run.
|
||||||
|
# A missing, all-zero or unfetchable base, or any git failure, also runs the suite:
|
||||||
|
# uncertainty never skips validation. tools/ci/ci-paths-check.sh tests this offline.
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
suite=${1:?suite required}
|
suite=${1:?suite required}
|
||||||
base=${GITHUB_EVENT_BEFORE:-}
|
event="${CI_EVENT:-${GITHUB_EVENT_NAME:-}}"
|
||||||
if [[ -z "$base" || "$base" == 0000000000000000000000000000000000000000 ]]; then
|
case "$event" in
|
||||||
base=HEAD^
|
push) base="${CI_BEFORE:-${GITHUB_EVENT_BEFORE:-}}" ;;
|
||||||
|
pull_request) base="${CI_PR_BASE:-}" ;;
|
||||||
|
*) exit 0 ;;
|
||||||
|
esac
|
||||||
|
if [[ ! "$base" =~ ^[0-9a-f]{40}$ || "$base" == 0000000000000000000000000000000000000000 ]]; then
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
if ! git cat-file -e "$base^{commit}" 2> /dev/null; then
|
||||||
|
git fetch --no-tags --quiet --depth=1 origin "$base" > /dev/null 2>&1 || exit 0
|
||||||
|
git cat-file -e "$base^{commit}" 2> /dev/null || exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# A single-parent diff is sufficient for push and the merge commit generated for
|
|
||||||
# pull requests. If history is unavailable, fail open and run the gate.
|
|
||||||
if ! files=$(git diff --name-only "$base" HEAD 2> /dev/null); then
|
if ! files=$(git diff --name-only "$base" HEAD 2> /dev/null); then
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Every suite runs when the workflow or this selector changes.
|
||||||
|
common='\.gitea/workflows/|tools/ci/ci-paths\.sh'
|
||||||
|
native='CMakeLists\.txt|CMakePresets\.json|cmake/|include/|src/|apps/'
|
||||||
|
postgres_tools='migrations/|tools/prepare-postgres\.sh|tools/migrate-postgres\.sh|tools/backup/|tools/ci/postgres-check\.sh|tools/ci/backup-check\.sh|tools/ci/dr-check\.sh|tools/ci/upgrade-check\.sh'
|
||||||
case "$suite" in
|
case "$suite" in
|
||||||
checks|tsan|ecs)
|
checks|tsan|ecs)
|
||||||
grep -Eq '^(CMakeLists.txt|CMakePresets.txt|CMakePresets.json|cmake/|include/|src/|apps/|tests/|experiments/|tools/ci/linux-check.sh|tools/ci/tsan-profile.mjs|deploy/docker/Dockerfile|\.gitea/workflows/)' <<<"$files"
|
pattern="$common|$native|tests/|experiments/|tools/ci/linux-check\.sh|tools/ci/tsan-profile\.mjs|deploy/docker/Dockerfile"
|
||||||
;;
|
;;
|
||||||
postgres|postgres-tools)
|
postgres|postgres-tools)
|
||||||
grep -Eq '^(CMakeLists.txt|CMakePresets.json|cmake/|include/|src/|apps/|tests/|migrations/|tools/prepare-postgres.sh|tools/ci/postgres-check.sh|tools/ci/backup-check.sh|tools/backup/|tools/migrate-postgres.sh|deploy/docker/Dockerfile|\.gitea/workflows/)' <<<"$files"
|
pattern="$common|$native|tests/|$postgres_tools|deploy/docker/Dockerfile"
|
||||||
;;
|
;;
|
||||||
deploy)
|
deploy)
|
||||||
grep -Eq '^(deploy/|tools/ci/deploy-policy-check.py|tools/ops/|tools/backup/|\.gitea/workflows/)' <<<"$files"
|
pattern="$common|deploy/|tools/ops/|tools/backup/|tools/ci/deploy-policy-check\.py|tools/ci/monitoring-check\.sh|tools/ci/ops-lib-check\.sh"
|
||||||
;;
|
;;
|
||||||
server-image)
|
server-image)
|
||||||
grep -Eq '^(CMakeLists.txt|CMakePresets.json|cmake/|include/|src/|apps/|migrations/|tools/prepare-postgres.sh|tools/ci/server-image-smoke.sh|deploy/docker/Dockerfile|\.gitea/workflows/)' <<<"$files"
|
pattern="$common|$native|migrations/|tools/prepare-postgres\.sh|tools/ci/server-image-smoke\.sh|deploy/docker/Dockerfile"
|
||||||
;;
|
;;
|
||||||
image)
|
image)
|
||||||
grep -Eq '^(CMakeLists.txt|CMakePresets.json|cmake/|include/|src/|apps/|tests/|deploy/docker/Dockerfile|\.gitea/workflows/)' <<<"$files"
|
pattern="$common|$native|tests/|deploy/docker/Dockerfile"
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
exit 0
|
exit 0
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
grep -Eq "^($pattern)" <<< "$files"
|
||||||
|
|||||||
@@ -21,7 +21,10 @@ environment-isolation or World0-protection invariant is broken:
|
|||||||
the digest already pinned in stage (promotion, never a fresh build);
|
the digest already pinned in stage (promotion, never a fresh build);
|
||||||
- scheduling (ADR-0076): every pod that mounts the node-local objects volume
|
- scheduling (ADR-0076): every pod that mounts the node-local objects volume
|
||||||
selects the store node; aevolution.io/compute is only ever preferred, never
|
selects the store node; aevolution.io/compute is only ever preferred, never
|
||||||
required; upgrade-rehearsal flags never appear in a deployed workload.
|
required; upgrade-rehearsal flags never appear in a deployed workload;
|
||||||
|
- the Pods the backup and barrier scripts generate (rendered with --print-manifest)
|
||||||
|
keep the database-tool security contract: non-root 10001 with fsGroup 10001 so
|
||||||
|
the 0440 Secret files are readable, no token, read-only root, no escalation.
|
||||||
"""
|
"""
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
@@ -118,7 +121,9 @@ if policy and binding:
|
|||||||
text = yaml.safe_dump(spec)
|
text = yaml.safe_dump(spec)
|
||||||
for needle in ("aevolution:world0-break-glass", "evolution-world0", "aevolution-world0",
|
for needle in ("aevolution:world0-break-glass", "evolution-world0", "aevolution-world0",
|
||||||
"aevolution-platform",
|
"aevolution-platform",
|
||||||
"aevolution.io/protected", "Retain"):
|
"aevolution.io/protected", "Retain",
|
||||||
|
"has(object.spec.claimRef.namespace)",
|
||||||
|
"object.spec.claimRef.name == oldObject.spec.claimRef.name"):
|
||||||
check(needle in text, f'world0 policy must reference {needle}')
|
check(needle in text, f'world0 policy must reference {needle}')
|
||||||
|
|
||||||
# Argo CD projects and applications.
|
# Argo CD projects and applications.
|
||||||
@@ -368,6 +373,47 @@ with tempfile.TemporaryDirectory() as empty:
|
|||||||
and 'kubectl' not in result.stderr,
|
and 'kubectl' not in result.stderr,
|
||||||
f'{tool} must refuse {bad!r} before kubectl (rc={result.returncode})')
|
f'{tool} must refuse {bad!r} before kubectl (rc={result.returncode})')
|
||||||
|
|
||||||
|
# Script-generated Pods (review M8 P2): rendered offline and checked like manifests.
|
||||||
|
retained_hex = next((v.get('value', '').split()[0] for d in manifests.get(
|
||||||
|
'deploy/kubernetes/region/dev/region-values.yaml', [])
|
||||||
|
for c in d['spec']['template']['spec']['containers'] for v in c.get('env', [])
|
||||||
|
if v.get('name') == 'REGION_RETAINED' and v.get('value')), None)
|
||||||
|
generators = [('tools/backup/cluster-backup.sh', ['dev', '--print-manifest'], 'backup')]
|
||||||
|
if retained_hex:
|
||||||
|
generators.append(('tools/ops/upgrade-barrier.sh',
|
||||||
|
['dev', '--retire', retained_hex, '--print-manifest'], 'migrate'))
|
||||||
|
else:
|
||||||
|
check(False, 'dev REGION_RETAINED is empty: the barrier Pod cannot be rendered')
|
||||||
|
egress = next((d for p, d in objects if d['kind'] == 'NetworkPolicy'
|
||||||
|
and d['metadata']['name'] == 'aevolution-database-egress'), {})
|
||||||
|
admitted = [v for e in egress.get('spec', {}).get('podSelector', {}).get('matchExpressions', [])
|
||||||
|
for v in e.get('values', [])]
|
||||||
|
for tool, args, component in generators:
|
||||||
|
rendered = subprocess.run([shutil.which('bash') or '/bin/bash', os.path.join(ROOT, tool)] + args,
|
||||||
|
capture_output=True, text=True)
|
||||||
|
check(rendered.returncode == 0, f'{tool} --print-manifest failed: {rendered.stderr.strip()}')
|
||||||
|
if rendered.returncode != 0:
|
||||||
|
continue
|
||||||
|
pod = yaml.safe_load(rendered.stdout)
|
||||||
|
spec = pod['spec']
|
||||||
|
context = spec.get('securityContext', {})
|
||||||
|
check(context.get('runAsNonRoot') is True and context.get('runAsUser') == 10001
|
||||||
|
and context.get('runAsGroup') == 10001 and context.get('fsGroup') == 10001,
|
||||||
|
f'{tool}: the generated Pod must run as 10001 with fsGroup 10001 (Secret access)')
|
||||||
|
check(spec.get('automountServiceAccountToken') is False, f'{tool}: the generated Pod mounts no token')
|
||||||
|
for container in spec['containers']:
|
||||||
|
sc = container.get('securityContext', {})
|
||||||
|
check(sc.get('readOnlyRootFilesystem') is True and sc.get('allowPrivilegeEscalation') is False,
|
||||||
|
f'{tool}: read-only root and no privilege escalation')
|
||||||
|
for volume in spec.get('volumes', []):
|
||||||
|
if 'secret' in volume:
|
||||||
|
check(volume['secret'].get('defaultMode') == 0o440,
|
||||||
|
f'{tool}: Secret files stay 0440 (never world-readable)')
|
||||||
|
labels = pod['metadata'].get('labels', {})
|
||||||
|
check(labels.get('app.kubernetes.io/component') == component and component in admitted,
|
||||||
|
f'{tool}: component {component} must be admitted by the database egress policy')
|
||||||
|
check(pod['metadata'].get('namespace') == 'evolution-dev', f'{tool}: renders into its environment')
|
||||||
|
|
||||||
for message in failures:
|
for message in failures:
|
||||||
print('FAIL', message)
|
print('FAIL', message)
|
||||||
print(f'deploy policy: {len(objects)} objects in {len(manifests)} files, {len(failures)} failures')
|
print(f'deploy policy: {len(objects)} objects in {len(manifests)} files, {len(failures)} failures')
|
||||||
|
|||||||
@@ -0,0 +1,38 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Offline fixtures for the operations helpers (review M8 P2): Ready-pod selection
|
||||||
|
# (tools/ops/lib/pods.sh) and the fault-drill verdicts (tools/ops/lib/drill.sh). No
|
||||||
|
# cluster: the lines below are what POD_STATUS_JSONPATH renders as
|
||||||
|
# "<name>|<Ready status>|<deletionTimestamp>". Run by the ci-deploy image build.
|
||||||
|
set -euo pipefail
|
||||||
|
root="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||||
|
. "$root/tools/ops/lib/pods.sh"
|
||||||
|
. "$root/tools/ops/lib/drill.sh"
|
||||||
|
failures=0
|
||||||
|
expect() { # expect <label> <wanted> <actual>
|
||||||
|
if [ "$2" = "$3" ]; then echo "ok $1"; else echo "FAIL $1: wanted '$2', got '$3'"; failures=$((failures + 1)); fi
|
||||||
|
}
|
||||||
|
pick() { single_ready_pod 2> /dev/null <<< "$1" || echo "<none>"; }
|
||||||
|
|
||||||
|
expect "one Ready pod" evo-region-a "$(pick 'evo-region-a|True|')"
|
||||||
|
expect "not Ready" "<none>" "$(pick 'evo-region-a|False|')"
|
||||||
|
expect "being deleted" "<none>" "$(pick 'evo-region-a|True|2026-09-30T10:00:00Z')"
|
||||||
|
expect "no Ready condition yet" "<none>" "$(pick 'evo-region-a||')"
|
||||||
|
expect "Ready beside a deleting pod" evo-region-b "$(pick $'evo-region-a|True|2026-09-30T10:00:00Z\nevo-region-b|True|')"
|
||||||
|
expect "two Ready pods" "<none>" "$(pick $'evo-region-a|True|\nevo-region-b|True|')"
|
||||||
|
expect "no pods" "<none>" "$(pick '')"
|
||||||
|
expect "jsonpath selects by the Ready condition" 1 \
|
||||||
|
"$(grep -c 'conditions\[?(@.type=="Ready")\].status' <<< "$POD_STATUS_JSONPATH")"
|
||||||
|
|
||||||
|
expect "RTO within budget" PASS "$(rto_verdict 20147 60000)"
|
||||||
|
expect "RTO just above budget" "FAIL rto 60001 ms > 60000 ms" "$(rto_verdict 60001 60000)"
|
||||||
|
expect "RTO not measured" "FAIL rto unknown ms > 60000 ms" "$(rto_verdict '' 60000)"
|
||||||
|
expect "empty: quarantined, exit 4" PASS "$(empty_verdict 2 4 2 6)"
|
||||||
|
expect "empty: quarantined, still running" PASS "$(empty_verdict 0 '' 1 2)"
|
||||||
|
expect "empty: crashed otherwise" "FAIL the host exited 1, not 4 (quarantined)" "$(empty_verdict 3 1 0 4)"
|
||||||
|
expect "empty: image pull, never ran" \
|
||||||
|
"INCONCLUSIVE the replacement host never ran (Pending, image pull or start failure?)" "$(empty_verdict 0 '' 0 0)"
|
||||||
|
expect "empty: ran without quarantine" "INCONCLUSIVE the host ran but logged no quarantine" "$(empty_verdict 0 '' 0 3)"
|
||||||
|
expect "acks covered" PASS "$(acks_verdict 209 209)"
|
||||||
|
expect "acks lost" "FAIL acknowledged 210 > recovered 209" "$(acks_verdict 210 209)"
|
||||||
|
expect "acks unobserved" "FAIL acknowledged ? > recovered 209" "$(acks_verdict '' 209)"
|
||||||
|
[ "$failures" = 0 ] && echo "operations helper fixtures passed" || { echo "$failures fixture(s) failed"; exit 1; }
|
||||||
@@ -21,6 +21,13 @@
|
|||||||
# made with `kubectl set env` / `kubectl patch` on the evo-region Deployment and
|
# made with `kubectl set env` / `kubectl patch` on the evo-region Deployment and
|
||||||
# reverted on exit, including on failure, back to the Git values (so Argo returns to
|
# reverted on exit, including on failure, back to the Git values (so Argo returns to
|
||||||
# Synced); World0 is refused before anything touches the cluster.
|
# Synced); World0 is refused before anything touches the cluster.
|
||||||
|
#
|
||||||
|
# Verdicts (tools/ops/lib/drill.sh): each kill must recover within RTO_BUDGET_MS
|
||||||
|
# (default 60000, ADR-0075); the empty scenario needs positive quarantine evidence.
|
||||||
|
# A missing observation fails the drill. Limits of log-based evidence: an owner's
|
||||||
|
# acknowledgements are read from its streamed log, finalized after it is gone; a line
|
||||||
|
# written but never streamed before a forced kill is not observed, so the
|
||||||
|
# zero-lost-acknowledgement check covers every observed acknowledgement only.
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
usage() { echo "usage: $0 <dev|stage> [kills|full|empty|db-outage|all] [--context <name>]" >&2; exit 2; }
|
usage() { echo "usage: $0 <dev|stage> [kills|full|empty|db-outage|all] [--context <name>]" >&2; exit 2; }
|
||||||
[ $# -ge 1 ] || usage
|
[ $# -ge 1 ] || usage
|
||||||
@@ -35,9 +42,13 @@ case "$scenario" in kills|full|empty|db-outage|all) shift || true ;; --context)
|
|||||||
k=(kubectl --request-timeout=30s)
|
k=(kubectl --request-timeout=30s)
|
||||||
[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2")
|
[ "${1:-}" = "--context" ] && [ -n "${2:-}" ] && k+=(--context "$2")
|
||||||
ns="evolution-$env"
|
ns="evolution-$env"
|
||||||
|
rto_budget="${RTO_BUDGET_MS:-60000}"
|
||||||
|
# shellcheck source=lib/drill.sh
|
||||||
|
. "$(cd "$(dirname "$0")" && pwd)/lib/drill.sh"
|
||||||
selector=app.kubernetes.io/name=evo-region
|
selector=app.kubernetes.io/name=evo-region
|
||||||
work="$(mktemp -d)"
|
work="$(mktemp -d)"
|
||||||
log_pids=()
|
log_pids=()
|
||||||
|
declare -A capture_pid=()
|
||||||
|
|
||||||
now_ms() { date +%s%3N; }
|
now_ms() { date +%s%3N; }
|
||||||
say() { echo "$(date -u +%H:%M:%S) $*"; }
|
say() { echo "$(date -u +%H:%M:%S) $*"; }
|
||||||
@@ -65,6 +76,13 @@ metric() { # metric <pod> <series>: from the pod's own /metrics (bash /dev/tcp).
|
|||||||
capture() { # Streams a pod's log to $work/<pod>.log for acknowledgement accounting.
|
capture() { # Streams a pod's log to $work/<pod>.log for acknowledgement accounting.
|
||||||
"${k[@]}" logs -n "$ns" -f "$1" > "$work/$1.log" 2>/dev/null &
|
"${k[@]}" logs -n "$ns" -f "$1" > "$work/$1.log" 2>/dev/null &
|
||||||
log_pids+=($!)
|
log_pids+=($!)
|
||||||
|
capture_pid[$1]=$!
|
||||||
|
}
|
||||||
|
finalize() { # Waits (up to 15 s) for a gone pod's log stream to end, so its log is complete.
|
||||||
|
local pid="${capture_pid[$1]:-}"
|
||||||
|
[ -n "$pid" ] || return 0
|
||||||
|
for _ in $(seq 1 75); do kill -0 "$pid" 2> /dev/null || return 0; sleep 0.2; done
|
||||||
|
kill "$pid" 2> /dev/null || true
|
||||||
}
|
}
|
||||||
max_ack() { awk '$1 == "ack" {if ($3 > m) m = $3} END {print m + 0}' "$work/$1.log"; }
|
max_ack() { awk '$1 == "ack" {if ($3 > m) m = $3} END {print m + 0}' "$work/$1.log"; }
|
||||||
recovered_at() { # committed tick from the pod's "phase running epoch=E committed=C" line.
|
recovered_at() { # committed tick from the pod's "phase running epoch=E committed=C" line.
|
||||||
@@ -104,24 +122,31 @@ say "loaded pod=$pod ($(recovered_at "$pod"))"
|
|||||||
if [ "$scenario" = kills ] || [ "$scenario" = all ]; then
|
if [ "$scenario" = kills ] || [ "$scenario" = all ]; then
|
||||||
for round in 1 2 3; do
|
for round in 1 2 3; do
|
||||||
sleep 10
|
sleep 10
|
||||||
acked="$(max_ack "$pod")"
|
|
||||||
start="$(now_ms)"
|
start="$(now_ms)"
|
||||||
"${k[@]}" delete pod -n "$ns" "$pod" --grace-period=0 --force > /dev/null 2>&1
|
"${k[@]}" delete pod -n "$ns" "$pod" --grace-period=0 --force > /dev/null 2>&1
|
||||||
loaded_pod "$pod"; next="$POD"
|
POD="$(ready_pod "$pod" 120)" || fail "round $round: no Ready successor within 120 s"
|
||||||
rto=$(( $(now_ms) - start ))
|
rto=$(( $(now_ms) - start ))
|
||||||
|
next="$POD"
|
||||||
|
capture "$next"
|
||||||
|
wait_for_line "$next" '^ack ' 60 || fail "$next acknowledges nothing under load"
|
||||||
|
finalize "$pod"
|
||||||
|
acked="$(max_ack "$pod")" # Everything the dead owner was observed to acknowledge.
|
||||||
read -r epoch committed <<< "$(recovered_at "$next")"
|
read -r epoch committed <<< "$(recovered_at "$next")"
|
||||||
say "kill round=$round rto_ms=$rto dead_acked_max=$acked recovered_committed=$committed epoch=$epoch pod=$next"
|
say "kill round=$round rto_ms=$rto budget_ms=$rto_budget dead_acked_max=$acked recovered_committed=$committed epoch=$epoch pod=$next"
|
||||||
[ "$acked" -le "$committed" ] || fail "round $round lost acknowledgements ($acked > $committed)"
|
verdict="$(rto_verdict "$rto" "$rto_budget")"
|
||||||
|
[ "$verdict" = PASS ] || fail "round $round: $verdict"
|
||||||
|
verdict="$(acks_verdict "$acked" "$committed")"
|
||||||
|
[ "$verdict" = PASS ] || fail "round $round lost acknowledgements: $verdict"
|
||||||
pod="$next"
|
pod="$next"
|
||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ "$scenario" = full ] || [ "$scenario" = all ]; then
|
if [ "$scenario" = full ] || [ "$scenario" = all ]; then
|
||||||
sleep 5
|
sleep 5
|
||||||
acked="$(max_ack "$pod")"
|
|
||||||
set_env REGION_RESERVE_MIB=1048576
|
set_env REGION_RESERVE_MIB=1048576
|
||||||
full_pod="$(ready_pod "$pod" 120)" || fail "full: host did not recover (recovery needs no space)"
|
full_pod="$(ready_pod "$pod" 120)" || fail "full: host did not recover (recovery needs no space)"
|
||||||
capture "$full_pod"
|
capture "$full_pod"
|
||||||
|
finalize "$pod"; acked="$(max_ack "$pod")" # The drained owner's complete log.
|
||||||
sleep 30
|
sleep 30
|
||||||
new_acks="$(awk '$1 == "ack"' "$work/$full_pod.log" | wc -l)"
|
new_acks="$(awk '$1 == "ack"' "$work/$full_pod.log" | wc -l)"
|
||||||
available="$(metric "$full_pod" evo_region_objects_available_bytes)"
|
available="$(metric "$full_pod" evo_region_objects_available_bytes)"
|
||||||
@@ -138,10 +163,10 @@ fi
|
|||||||
|
|
||||||
if [ "$scenario" = empty ] || [ "$scenario" = all ]; then
|
if [ "$scenario" = empty ] || [ "$scenario" = all ]; then
|
||||||
sleep 5
|
sleep 5
|
||||||
acked="$(max_ack "$pod")"
|
|
||||||
"${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \
|
"${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \
|
||||||
'[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","emptyDir":{"sizeLimit":"64Mi"}}}]' > /dev/null
|
'[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","emptyDir":{"sizeLimit":"64Mi"}}}]' > /dev/null
|
||||||
sleep 45
|
sleep 45
|
||||||
|
finalize "$pod"; acked="$(max_ack "$pod")" # The drained owner's complete log.
|
||||||
if ready_pod "$pod" 1 > /dev/null; then fail "empty: a host became Ready on an empty volume"; fi
|
if ready_pod "$pod" 1 > /dev/null; then fail "empty: a host became Ready on an empty volume"; fi
|
||||||
empty_pod="$(current_pods | grep -v "^$pod$" | head -1)"
|
empty_pod="$(current_pods | grep -v "^$pod$" | head -1)"
|
||||||
[ -n "$empty_pod" ] || fail "empty: no replacement pod was created"
|
[ -n "$empty_pod" ] || fail "empty: no replacement pod was created"
|
||||||
@@ -153,9 +178,9 @@ if [ "$scenario" = empty ] || [ "$scenario" = all ]; then
|
|||||||
quarantined="$(cat "$work/$empty_pod.current.log" "$work/$empty_pod.previous.log" | grep -c '^phase quarantined' || true)"
|
quarantined="$(cat "$work/$empty_pod.current.log" "$work/$empty_pod.previous.log" | grep -c '^phase quarantined' || true)"
|
||||||
say "empty pod=$empty_pod ready=never restarts=${restarts:-0} last_exit=${exit_code:-none} quarantined_lines=$quarantined"
|
say "empty pod=$empty_pod ready=never restarts=${restarts:-0} last_exit=${exit_code:-none} quarantined_lines=$quarantined"
|
||||||
say "empty last log lines:"; tail -n 5 "$work/$empty_pod.previous.log" "$work/$empty_pod.current.log" | sed 's/^/ /'
|
say "empty last log lines:"; tail -n 5 "$work/$empty_pod.previous.log" "$work/$empty_pod.current.log" | sed 's/^/ /'
|
||||||
if [ "${exit_code:-}" != 4 ] && [ "$quarantined" -eq 0 ]; then
|
lines="$(cat "$work/$empty_pod.current.log" "$work/$empty_pod.previous.log" | grep -c '^phase ' || true)"
|
||||||
say "WARN empty: no quarantine evidence (exit 4 or a 'phase quarantined' line); see the logs above"
|
verdict="$(empty_verdict "${restarts:-0}" "${exit_code:-}" "$quarantined" "$lines")"
|
||||||
fi
|
[ "$verdict" = PASS ] || fail "empty: $verdict"
|
||||||
"${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \
|
"${k[@]}" patch -n "$ns" deployment/evo-region --type json -p \
|
||||||
'[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","persistentVolumeClaim":{"claimName":"aevolution-objects"}}}]' > /dev/null
|
'[{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"objects","persistentVolumeClaim":{"claimName":"aevolution-objects"}}}]' > /dev/null
|
||||||
loaded_pod "$empty_pod"; pod="$POD"
|
loaded_pod "$empty_pod"; pod="$POD"
|
||||||
|
|||||||
@@ -0,0 +1,34 @@
|
|||||||
|
# Fault-drill verdicts (sourced by tools/ops/fault-drill.sh; tested offline with
|
||||||
|
# fixtures by tools/ci/ops-lib-check.sh). Each prints PASS, or FAIL/INCONCLUSIVE with
|
||||||
|
# a reason; the drill passes a scenario only on PASS (review M8 P2: a missing
|
||||||
|
# observation is never a pass).
|
||||||
|
|
||||||
|
# rto_verdict <rto_ms> <budget_ms>: the ratified objective (ADR-0075: 60 s for dev/stage).
|
||||||
|
rto_verdict() {
|
||||||
|
if [[ "${1:-}" =~ ^[0-9]+$ ]] && [ "$1" -le "$2" ]; then echo PASS; else echo "FAIL rto ${1:-unknown} ms > ${2} ms"; fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# empty_verdict <restarts> <last_exit_code> <quarantined_lines> <log_lines>
|
||||||
|
# Positive evidence only: the replacement host ran and logged "phase quarantined"
|
||||||
|
# (on an empty volume the database and builds are unchanged, so unavailable committed
|
||||||
|
# state is the only cause), and if its container ended, it ended with exit 4.
|
||||||
|
# Another exit (image, permission, crash) is a FAIL; no evidence is INCONCLUSIVE.
|
||||||
|
empty_verdict() {
|
||||||
|
local restarts="${1:-0}" code="${2:-}" quarantined="${3:-0}" lines="${4:-0}"
|
||||||
|
if [ -n "$code" ] && [ "$code" != 4 ]; then
|
||||||
|
echo "FAIL the host exited $code, not 4 (quarantined)"
|
||||||
|
elif [ "$quarantined" -gt 0 ]; then
|
||||||
|
echo PASS
|
||||||
|
elif [ "$restarts" = 0 ] && [ "$lines" = 0 ]; then
|
||||||
|
echo "INCONCLUSIVE the replacement host never ran (Pending, image pull or start failure?)"
|
||||||
|
else
|
||||||
|
echo "INCONCLUSIVE the host ran but logged no quarantine"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# acks_verdict <dead_acked_max> <recovered_committed>: every acknowledgement the dead
|
||||||
|
# owner was observed to make is at or below the head its successor recovered.
|
||||||
|
acks_verdict() {
|
||||||
|
if [[ "${1:-}" =~ ^[0-9]+$ && "${2:-}" =~ ^[0-9]+$ ]] && [ "$1" -le "$2" ]; then echo PASS
|
||||||
|
else echo "FAIL acknowledged ${1:-?} > recovered ${2:-?}"; fi
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
# Shared pod-status helpers for the operations scripts (sourced, not executed).
|
||||||
|
#
|
||||||
|
# POD_STATUS_JSONPATH renders one line per pod, "<name>|<Ready status>|<deletion>",
|
||||||
|
# with plain kubectl jsonpath (no template variables): the Ready status is "True",
|
||||||
|
# "False" or empty when the pod has no Ready condition yet, and the deletion field is
|
||||||
|
# empty unless the pod is being deleted. single_ready_pod reads such lines on stdin
|
||||||
|
# and prints the one Ready pod not being deleted; zero or several candidates fail
|
||||||
|
# (review M8 P2: never concatenate names or guess). tools/ci/ops-lib-check.sh tests it
|
||||||
|
# offline with fixtures.
|
||||||
|
# shellcheck disable=SC2034 # Used by the scripts that source this file.
|
||||||
|
POD_STATUS_JSONPATH='{range .items[*]}{.metadata.name}{"|"}{.status.conditions[?(@.type=="Ready")].status}{"|"}{.metadata.deletionTimestamp}{"\n"}{end}'
|
||||||
|
|
||||||
|
single_ready_pod() {
|
||||||
|
local ready count
|
||||||
|
ready="$(awk -F'|' 'NF >= 2 && $1 != "" && $2 == "True" && $3 == "" {print $1}')"
|
||||||
|
count="$(grep -c . <<< "$ready" || true)"
|
||||||
|
if [ "$count" != 1 ]; then
|
||||||
|
echo "expected exactly one Ready pod, found $count${ready:+: ${ready//$'\n'/ }}" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
printf '%s\n' "$ready"
|
||||||
|
}
|
||||||
@@ -1,22 +1,25 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# M8.9 version barrier in a disposable environment (ADR-0068, ADR-0076).
|
# M8.9 version barrier in a disposable environment (ADR-0068, ADR-0076).
|
||||||
#
|
#
|
||||||
# tools/ops/upgrade-barrier.sh <dev|stage> --retire HEX64 [--context NAME]
|
# tools/ops/upgrade-barrier.sh <dev|stage> --retire HEX64 [--context NAME] [--print-manifest]
|
||||||
#
|
#
|
||||||
# The last step of the upgrade order: migrate -> retain the new build (the provision
|
# The last step of the upgrade order: migrate -> retain the new build (the provision
|
||||||
# hook does) -> deploy it with the old build in REGION_RETAINED -> retire the old
|
# hook does) -> deploy it with the old build in REGION_RETAINED -> retire the old
|
||||||
# build here. Refused unless:
|
# build here. Refused unless:
|
||||||
# - HEX64 is not the build Git currently deploys (deploy/kubernetes/region/<env>);
|
# - HEX64 is not the build Git currently deploys (deploy/kubernetes/region/<env>);
|
||||||
# - HEX64 is listed in REGION_RETAINED (its history must stay recoverable);
|
# - HEX64 is listed in REGION_RETAINED (its history must stay recoverable);
|
||||||
# - the evo-region pod is Ready and runs exactly the Git-pinned image, so the new
|
# - exactly one evo-region pod is Ready (and not being deleted), and it runs exactly
|
||||||
# build already serves.
|
# the Git-pinned image, so the new build already serves.
|
||||||
# Then one short Pod (the postgres-tools image pinned for <env>, migrator login) runs
|
# Then one short Pod (the postgres-tools image pinned for <env>, migrator login, the
|
||||||
|
# same fsGroup Secret-access contract as the other database-tool pods) runs
|
||||||
# evolution.upgrade_barrier(HEX64): the old build can never acquire, renew or commit
|
# evolution.upgrade_barrier(HEX64): the old build can never acquire, renew or commit
|
||||||
# again, and every lease is fenced. The new host re-acquires within one lease. The
|
# again, and every lease is fenced. The new host re-acquires within one lease. The
|
||||||
# script waits for Ready again and prints the time without a ready host.
|
# script waits for Ready again and prints the time without a ready host.
|
||||||
|
# --print-manifest prints that Pod after the offline checks and exits without
|
||||||
|
# touching the cluster (the deploy policy check inspects it).
|
||||||
# Retirement is irreversible here. World0 is refused before kubectl runs.
|
# Retirement is irreversible here. World0 is refused before kubectl runs.
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
usage() { echo "usage: $0 <dev|stage> --retire HEX64 [--context NAME]" >&2; exit 2; }
|
usage() { echo "usage: $0 <dev|stage> --retire HEX64 [--context NAME] [--print-manifest]" >&2; exit 2; }
|
||||||
[ $# -ge 1 ] || usage
|
[ $# -ge 1 ] || usage
|
||||||
env="$1"; shift
|
env="$1"; shift
|
||||||
case "$env" in
|
case "$env" in
|
||||||
@@ -24,16 +27,19 @@ case "$env" in
|
|||||||
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
|
*) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2
|
||||||
exit 2 ;;
|
exit 2 ;;
|
||||||
esac
|
esac
|
||||||
retire="" k=(kubectl --request-timeout=30s)
|
retire="" print=0 k=(kubectl --request-timeout=30s)
|
||||||
while [ $# -gt 0 ]; do
|
while [ $# -gt 0 ]; do
|
||||||
case "$1" in
|
case "$1" in
|
||||||
--retire) retire="${2:-}"; shift 2 ;;
|
--retire) retire="${2:-}"; shift 2 ;;
|
||||||
--context) k+=(--context "${2:?}"); shift 2 ;;
|
--context) k+=(--context "${2:?}"); shift 2 ;;
|
||||||
|
--print-manifest) print=1; shift ;;
|
||||||
*) usage ;;
|
*) usage ;;
|
||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
[[ "$retire" =~ ^[0-9a-f]{64}$ ]] || usage
|
[[ "$retire" =~ ^[0-9a-f]{64}$ ]] || usage
|
||||||
root="$(cd "$(dirname "$0")/../.." && pwd)"
|
root="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||||
|
# shellcheck source=lib/pods.sh
|
||||||
|
. "$root/tools/ops/lib/pods.sh"
|
||||||
ns="evolution-$env"
|
ns="evolution-$env"
|
||||||
values="$root/deploy/kubernetes/region/$env/region-values.yaml"
|
values="$root/deploy/kubernetes/region/$env/region-values.yaml"
|
||||||
current="$(sed -n 's/.*name: REGION_BUILD, value: \([0-9a-f]\{64\}\)}.*/\1/p' "$values" | sort -u)"
|
current="$(sed -n 's/.*name: REGION_BUILD, value: \([0-9a-f]\{64\}\)}.*/\1/p' "$values" | sort -u)"
|
||||||
@@ -44,26 +50,15 @@ grep -qw "$retire" <<< "$retained" || { echo "refused: $retire is not in REGION_
|
|||||||
image="$(awk '/name: aevolution-postgres-tools/ {f = 1} f && /newName:/ {n = $2} f && /digest:/ {print n "@" $2; exit}' \
|
image="$(awk '/name: aevolution-postgres-tools/ {f = 1} f && /newName:/ {n = $2} f && /digest:/ {print n "@" $2; exit}' \
|
||||||
"$root/deploy/kubernetes/stores/$env/kustomization.yaml")"
|
"$root/deploy/kubernetes/stores/$env/kustomization.yaml")"
|
||||||
[[ "$image" =~ @sha256:[0-9a-f]{64}$ ]] || { echo "no pinned postgres-tools image for $env" >&2; exit 2; }
|
[[ "$image" =~ @sha256:[0-9a-f]{64}$ ]] || { echo "no pinned postgres-tools image for $env" >&2; exit 2; }
|
||||||
say() { echo "$(date -u +%H:%M:%S) $*"; }
|
|
||||||
ready_pod() {
|
|
||||||
"${k[@]}" get pods -n "$ns" -l app.kubernetes.io/name=evo-region -o go-template='{{range .items}}{{if not .metadata.deletionTimestamp}}{{range .status.conditions}}{{if and (eq .type "Ready") (eq .status "True")}}{{$.metadata.name}}{{end}}{{end}}{{end}}{{end}}' 2> /dev/null || true
|
|
||||||
}
|
|
||||||
serving() { # The Ready pod's running image digest.
|
|
||||||
local pod; pod="$(ready_pod)"
|
|
||||||
[ -n "$pod" ] && "${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.containerStatuses[0].imageID}'
|
|
||||||
}
|
|
||||||
[[ "$(serving)" == *"@sha256:$current" ]] ||
|
|
||||||
{ echo "refused: no Ready evo-region pod runs sha256:$current yet (deploy the new build first)" >&2; exit 3; }
|
|
||||||
|
|
||||||
pod="aevolution-barrier-$(date -u +%Y%m%d%H%M%S)"
|
pod="aevolution-barrier-$(date -u +%Y%m%d%H%M%S)"
|
||||||
cleanup() { "${k[@]}" delete pod -n "$ns" "$pod" --wait=false > /dev/null 2>&1 || true; }
|
|
||||||
trap cleanup EXIT
|
manifest() {
|
||||||
say "retiring $retire in $ns (serving sha256:$current)"
|
cat <<YAML
|
||||||
"${k[@]}" apply -n "$ns" -f - > /dev/null <<YAML
|
|
||||||
apiVersion: v1
|
apiVersion: v1
|
||||||
kind: Pod
|
kind: Pod
|
||||||
metadata:
|
metadata:
|
||||||
name: $pod
|
name: $pod
|
||||||
|
namespace: $ns
|
||||||
labels:
|
labels:
|
||||||
app.kubernetes.io/part-of: aevolution
|
app.kubernetes.io/part-of: aevolution
|
||||||
app.kubernetes.io/component: migrate
|
app.kubernetes.io/component: migrate
|
||||||
@@ -76,6 +71,7 @@ spec:
|
|||||||
runAsNonRoot: true
|
runAsNonRoot: true
|
||||||
runAsUser: 10001
|
runAsUser: 10001
|
||||||
runAsGroup: 10001
|
runAsGroup: 10001
|
||||||
|
fsGroup: 10001
|
||||||
seccompProfile: {type: RuntimeDefault}
|
seccompProfile: {type: RuntimeDefault}
|
||||||
containers:
|
containers:
|
||||||
- name: barrier
|
- name: barrier
|
||||||
@@ -87,8 +83,8 @@ spec:
|
|||||||
umask 077
|
umask 077
|
||||||
cp /run/postgres/pgpass /tmp/pgpass
|
cp /run/postgres/pgpass /tmp/pgpass
|
||||||
export PGPASSFILE=/tmp/pgpass
|
export PGPASSFILE=/tmp/pgpass
|
||||||
psql -X --no-password -At -v ON_ERROR_STOP=1 \
|
psql -X --no-password -At -v ON_ERROR_STOP=1 \\
|
||||||
-c "SELECT 'fenced ' || evolution.upgrade_barrier(decode('$retire', 'hex'))" \
|
-c "SELECT 'fenced ' || evolution.upgrade_barrier(decode('$retire', 'hex'))" \\
|
||||||
-c "SELECT 'retired ' || encode(build_identity, 'hex') FROM evolution.retained_builds WHERE retired_at IS NOT NULL ORDER BY 1"
|
-c "SELECT 'retired ' || encode(build_identity, 'hex') FROM evolution.retained_builds WHERE retired_at IS NOT NULL ORDER BY 1"
|
||||||
env:
|
env:
|
||||||
- {name: PGSERVICE, value: evolution}
|
- {name: PGSERVICE, value: evolution}
|
||||||
@@ -115,17 +111,40 @@ spec:
|
|||||||
- name: scratch
|
- name: scratch
|
||||||
emptyDir: {medium: Memory, sizeLimit: 1Mi}
|
emptyDir: {medium: Memory, sizeLimit: 1Mi}
|
||||||
YAML
|
YAML
|
||||||
|
}
|
||||||
|
if [ "$print" = 1 ]; then manifest; exit 0; fi
|
||||||
|
|
||||||
|
say() { echo "$(date -u +%H:%M:%S) $*"; }
|
||||||
|
ready_pod() { # The single Ready evo-region pod, or failure (none, several, or unreadable).
|
||||||
|
"${k[@]}" get pods -n "$ns" -l app.kubernetes.io/name=evo-region -o jsonpath="$POD_STATUS_JSONPATH" |
|
||||||
|
single_ready_pod
|
||||||
|
}
|
||||||
|
serving() { # The single Ready pod's running image digest.
|
||||||
|
local name
|
||||||
|
name="$(ready_pod 2> /dev/null)" || return 1
|
||||||
|
"${k[@]}" get pod -n "$ns" "$name" -o jsonpath='{.status.containerStatuses[0].imageID}'
|
||||||
|
}
|
||||||
|
ready_pod > /dev/null || { echo "refused: no single Ready evo-region pod" >&2; exit 3; }
|
||||||
|
[[ "$(serving)" == *"@sha256:$current" ]] ||
|
||||||
|
{ echo "refused: the Ready evo-region pod does not run sha256:$current (deploy the new build first)" >&2; exit 3; }
|
||||||
|
|
||||||
|
cleanup() { "${k[@]}" delete pod -n "$ns" "$pod" --wait=false > /dev/null 2>&1 || true; }
|
||||||
|
trap cleanup EXIT
|
||||||
|
say "retiring $retire in $ns (serving sha256:$current)"
|
||||||
|
manifest | "${k[@]}" apply -f - > /dev/null
|
||||||
start="$(date +%s%3N)"
|
start="$(date +%s%3N)"
|
||||||
|
phase=""
|
||||||
for _ in $(seq 1 120); do
|
for _ in $(seq 1 120); do
|
||||||
phase="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}' 2> /dev/null || true)"
|
phase="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}' 2> /dev/null || true)"
|
||||||
[ "$phase" = Succeeded ] || [ "$phase" = Failed ] && break
|
{ [ "$phase" = Succeeded ] || [ "$phase" = Failed ]; } && break
|
||||||
sleep 1
|
sleep 1
|
||||||
done
|
done
|
||||||
"${k[@]}" logs -n "$ns" "$pod" 2>&1 | sed 's/^/ /'
|
"${k[@]}" logs -n "$ns" "$pod" 2>&1 | sed 's/^/ /'
|
||||||
[ "$phase" = Succeeded ] || { say "FAIL barrier pod $phase"; exit 3; }
|
[ "$phase" = Succeeded ] || { say "FAIL barrier pod ${phase:-unknown}"; exit 3; }
|
||||||
|
# The fenced host loses readiness at its next lease check, then re-acquires.
|
||||||
not_ready_since=""
|
not_ready_since=""
|
||||||
for _ in $(seq 1 600); do
|
for _ in $(seq 1 600); do
|
||||||
if [ -n "$(ready_pod)" ]; then
|
if ready_pod > /dev/null 2>&1; then
|
||||||
[ -z "$not_ready_since" ] && [ $(( $(date +%s%3N) - start )) -lt 3000 ] && { sleep 0.5; continue; }
|
[ -z "$not_ready_since" ] && [ $(( $(date +%s%3N) - start )) -lt 3000 ] && { sleep 0.5; continue; }
|
||||||
break
|
break
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -82,6 +82,20 @@ if printf '%s\n' "${present[@]}" | grep -qx world0; then
|
|||||||
pvc="$("${k[@]}" get pvc -n evolution-world0 -o name 2>/dev/null | head -1)"
|
pvc="$("${k[@]}" get pvc -n evolution-world0 -o name 2>/dev/null | head -1)"
|
||||||
[ -n "$pvc" ] && denied "delete $pvc in evolution-world0" -n evolution-world0 "$pvc" ||
|
[ -n "$pvc" ] && denied "delete $pvc in evolution-world0" -n evolution-world0 "$pvc" ||
|
||||||
skip "no PVC in evolution-world0 to test"
|
skip "no PVC in evolution-world0 to test"
|
||||||
|
# Clearing a World0 volume's claim reference must be denied too (it would strip
|
||||||
|
# the protection before a delete). Server-side dry run: never persisted.
|
||||||
|
pv="$("${k[@]}" get pv -o jsonpath='{range .items[?(@.spec.claimRef.namespace=="evolution-world0")]}{.metadata.name}{"\n"}{end}' 2>/dev/null | head -1)"
|
||||||
|
if [ -n "$pv" ]; then
|
||||||
|
out="$("${k[@]}" patch pv "$pv" --dry-run=server --type json \
|
||||||
|
-p '[{"op":"remove","path":"/spec/claimRef"}]' 2>&1 || true)"
|
||||||
|
if echo "$out" | grep -q "aevolution:world0-break-glass"; then
|
||||||
|
pass "clearing the claim reference of World0 volume $pv is denied"
|
||||||
|
else
|
||||||
|
fail "clearing the claim reference of World0 volume $pv was not denied: $out"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
skip "no PersistentVolume bound to evolution-world0 to test"
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
skip "world0 admission dry runs (namespace absent)"
|
skip "world0 admission dry runs (namespace absent)"
|
||||||
fi
|
fi
|
||||||
|
|||||||
Reference in New Issue
Block a user