From 3a34ae39bc836c16786363a99b16c5d89456607c Mon Sep 17 00:00:00 2001 From: dingo Date: Wed, 30 Sep 2026 22:53:11 -0400 Subject: [PATCH] fix: M8 review findings (restore ownership, barrier tool, World0 claimRef, CI selection, drill verdicts) P1: restore-world.sh claims its objects path (mkdir) and the database (a held session with a restore try-lock and the exclusive upgrade lock), and removes only what it created, so a losing restore never drops a winner's schema. P2: barrier Ready-pod selection via shared jsonpath helper with fixtures; barrier pod fsGroup; script-generated pods inspected by the policy check; World0 PV claim references protected; ci-paths.sh uses the event's real base and runs on uncertainty; fault-drill asserts RTO and needs positive quarantine evidence. Co-Authored-By: Claude Opus 5.5 --- .gitea/workflows/build.yaml | 8 ++ deploy/docker/Dockerfile | 3 +- .../platform/admission/world0-protection.yaml | 12 +++ docs/ROADMAP.md | 18 +++- docs/adr/0069-environment-isolation.md | 8 ++ docs/adr/0073-fault-drills.md | 14 +++ docs/adr/0074-world-backup-and-restore.md | 19 ++++ docs/adr/0076-upgrade-and-growth.md | 12 +++ docs/engineering/VALIDATION.md | 53 +++++++++++ tools/backup/cluster-backup.sh | 27 ++++-- tools/backup/restore-world.sh | 87 ++++++++++++++----- tools/ci/backup-check.sh | 42 +++++++++ tools/ci/ci-paths-check.sh | 45 ++++++++++ tools/ci/ci-paths.sh | 44 +++++++--- tools/ci/deploy-policy-check.py | 50 ++++++++++- tools/ci/ops-lib-check.sh | 38 ++++++++ tools/ops/fault-drill.sh | 43 +++++++-- tools/ops/lib/drill.sh | 34 ++++++++ tools/ops/lib/pods.sh | 22 +++++ tools/ops/upgrade-barrier.sh | 71 +++++++++------ tools/ops/verify-isolation.sh | 14 +++ 21 files changed, 584 insertions(+), 80 deletions(-) create mode 100644 tools/ci/ci-paths-check.sh create mode 100644 tools/ci/ops-lib-check.sh create mode 100644 tools/ops/lib/drill.sh create mode 100644 tools/ops/lib/pods.sh diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml index 4df592d..03dfd2f 100644 --- a/.gitea/workflows/build.yaml +++ b/.gitea/workflows/build.yaml @@ -20,6 +20,11 @@ env: HARBOR_USERNAME: ${{ secrets.HARBOR_USERNAME }} HARBOR_PASSWORD: ${{ secrets.HARBOR_PASSWORD }} CACHE_WRITE: '0' + # Change selection (tools/ci/ci-paths.sh) compares against the event's real base: + # the push's before SHA or the pull request's base; manual runs always run. + CI_EVENT: ${{ github.event_name }} + CI_BEFORE: ${{ github.event.before }} + CI_PR_BASE: ${{ github.event.pull_request.base.sha }} jobs: checks: @@ -40,6 +45,9 @@ jobs: CHECK_SUITE: ${{ matrix.suite }} run: | set -eu + # The selector's own regression runs first, so a broken selector cannot + # silently skip the suites. + if [ "$CHECK_SUITE" = deploy ]; then bash tools/ci/ci-paths-check.sh; fi if ! bash tools/ci/ci-paths.sh "$CHECK_SUITE"; then echo "No files relevant to $CHECK_SUITE changed; gate skipped." exit 0 diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 3e197e0..33dc422 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -60,7 +60,8 @@ COPY deploy/ deploy/ COPY tools/ci/deploy-policy-check.py tools/ci/ COPY tools/ops/ tools/ops/ COPY tools/backup/ tools/backup/ -RUN python3 tools/ci/deploy-policy-check.py +COPY tools/ci/ops-lib-check.sh tools/ci/ +RUN python3 tools/ci/deploy-policy-check.py && bash tools/ci/ops-lib-check.sh # M8.5: monitoring configuration and alert tests with the pinned images' own tools. COPY --from=quay.io/prometheus/prometheus@sha256:6976aa8a60fec930796ce5772b8d12da7a318a5daa8d40d69c5c7819a05eeed7 /bin/promtool /usr/local/bin/promtool COPY --from=quay.io/prometheus/alertmanager@sha256:e9733bafb1bdef9b00e25a21f8f99dc26a22224bf16641ad754d1649f4c3357a /bin/amtool /usr/local/bin/amtool diff --git a/deploy/kubernetes/platform/admission/world0-protection.yaml b/deploy/kubernetes/platform/admission/world0-protection.yaml index 74faa52..522b59b 100644 --- a/deploy/kubernetes/platform/admission/world0-protection.yaml +++ b/deploy/kubernetes/platform/admission/world0-protection.yaml @@ -12,6 +12,10 @@ # - deleting a PersistentVolume bound to evolution-world0, or changing its reclaim # policy away from Retain once set (set Retain when the volume is bound: the # only StorageClass, local-path, provisions with Delete); +# - removing or reassigning such a volume's claim reference (namespace or claim +# name): the reference is how this policy recognizes a World0 volume, so clearing +# it first would otherwise strip the protection (review M8 P2). The binder may +# still fill in uid/resourceVersion; reassignment is a break-glass operation; # - deleting the aevolution-world0 or aevolution-platform Argo CD Application or # AppProject (the platform app owns this policy and the World0 namespace). # Kubernetes does not run admission policies on policy objects themselves, so @@ -78,6 +82,14 @@ spec: object.spec.persistentVolumeReclaimPolicy == 'Retain' message: "A World0 PersistentVolume set to Retain cannot be changed back" reason: Forbidden + - expression: >- + variables.breakGlass || request.operation != 'UPDATE' || !variables.world0Volume || + (has(object.spec.claimRef) && has(object.spec.claimRef.namespace) && + object.spec.claimRef.namespace == 'evolution-world0' && + (!has(oldObject.spec.claimRef.name) || + (has(object.spec.claimRef.name) && object.spec.claimRef.name == oldObject.spec.claimRef.name))) + message: "A World0 PersistentVolume keeps its World0 claim reference without the aevolution:world0-break-glass group" + reason: Forbidden --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicyBinding diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 35ee4fd..c98b05c 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -827,8 +827,9 @@ measurements (worker loss RPO 0 / RTO <= 60 s; host loss = explicit rewind to th last verified bundle); **World0 launch stays blocked** because no independent durable failure domain exists, enforced by the deploy policy check; host-loss restore in place with barrier fencing of the superseded database rehearsed -locally; promotion only of stage-pinned digests. **M8-dev/stage passed; -M8-World0 launch not passed.** +locally; promotion only of stage-pinned digests. **M8-World0 launch not passed** +(blocked by design). M8-dev/stage is implemented and locally qualified, not yet +passed: see "M8 gate status" below. M8.9 implemented and locally qualified (2026-09-30), under ADR-0076: accelerated clone runs of the current and candidate builds compared state-for-state and @@ -837,3 +838,16 @@ version barrier rehearsed with real hosts; `upgrade-barrier.sh` for dev/stage; stores/compute node labels with policy-checked placement and `scheduling-check.sh`. Rules changes stay refused until a converter ADR. In-cluster barrier and additional-node evidence wait on the maintainer. + +**M8 gate status (2026-09-30, after the M8.1–M8.9 review).** Implementation and +local qualification are complete; these live gates are still open, so M8-dev/stage +is **not passed**: +- M8.6: a valid dev database-outage run, and a re-run of full/empty with the new + image (positive quarantine evidence, RTO asserted against 60 s); +- M8.7: an in-cluster `cluster-backup.sh` bundle restore-tested off the host; the + destination, schedule, retention, encryption and WAL archiving (backlog); +- M8.5: an alert receiver and kubelet log-rotation check (backlog); +- M8.9: `upgrade-barrier.sh` on dev and `scheduling-check.sh` with a second labelled + node; +- M8.1/M8.3: storage re-measured after the RAID rework. +M8-World0 launch stays blocked by ADR-0075. diff --git a/docs/adr/0069-environment-isolation.md b/docs/adr/0069-environment-isolation.md index e4dbf6b..52c2fe4 100644 --- a/docs/adr/0069-environment-isolation.md +++ b/docs/adr/0069-environment-isolation.md @@ -130,3 +130,11 @@ What stays outside Argo: - Secrets, until a secret manager is chosen (docs/BACKLOG.md); - the store node label; - the one-time bootstrap of the platform project and Application. + +## Amendment: World0 volume claim references (M8 review, 2026-09-30) +The policy recognizes a World0 PersistentVolume by its claim reference. Clearing +or reassigning that reference first would have removed the protection. An UPDATE +that removes the reference, or changes its namespace or claim name, is now denied +without the break-glass group. The binder may still fill in uid and +resourceVersion. `verify-isolation.sh` tests this with a server-side dry run when +a World0 volume exists. diff --git a/docs/adr/0073-fault-drills.md b/docs/adr/0073-fault-drills.md index c3acefb..4161128 100644 --- a/docs/adr/0073-fault-drills.md +++ b/docs/adr/0073-fault-drills.md @@ -90,3 +90,17 @@ PostgreSQL: without a shared pipe that the image does not have. - **Partitioning with NetworkPolicy.** It does not cut established connections under Calico. + +## Amendment: drill verdicts (M8 review, 2026-09-30) +A missing observation no longer passes. The verdicts are in `tools/ops/lib/drill.sh` +and are fixture-tested by `tools/ci/ops-lib-check.sh`: +- Each kill must recover within `RTO_BUDGET_MS`, default 60 000 (ADR-0075). +- The empty scenario passes only on positive evidence: a `phase quarantined` line, + and exit 4 if the container ended. Any other exit fails. No evidence is + INCONCLUSIVE, which fails the drill. +- A dead or drained owner's acknowledgements are read from its log after the + stream has ended. + +Limit: the drill reads acknowledgements from logs. A line that was written but not +streamed before a forced kill is not observed, so the zero-lost-acknowledgement +check covers observed acknowledgements only. diff --git a/docs/adr/0074-world-backup-and-restore.md b/docs/adr/0074-world-backup-and-restore.md index 298f761..2e7b926 100644 --- a/docs/adr/0074-world-backup-and-restore.md +++ b/docs/adr/0074-world-backup-and-restore.md @@ -172,3 +172,22 @@ every PostgreSQL build: authority. Isolation goes through the clone path, which re-identifies the world. - **A new object-storage dependency (MinIO, S3 SDK).** The destination is not chosen; a directory and `tar` need no new dependency. + +## Amendment: restore ownership (M8 review, 2026-09-30) +The review reproduced a race. A restore checked that the target was empty, a +competing actor then created the schema, the restore's `pg_restore` failed, and +its cleanup dropped the other actor's schema. Restores now own what they touch: +- `--objects` is claimed by creating it (`mkdir`, never `-p`). An existing path is + refused and never touched. +- One psql session, held open for the whole restore, takes the restore claim + (advisory 71420004, try-lock, so a concurrent restore refuses at once) and the + upgrade lock (71420002, exclusive). Provisioning, acquisition and lease checks in + that database wait until the restore is verified or removed. +- Cleanup removes only what the invocation created: the schema only if its own + one-transaction `pg_restore` committed, and the directory only if its own `mkdir` + created it. + +`backup-check.sh` covers three cases: +- a competing schema created just before `pg_restore` survives; +- a concurrent restore refuses, and the first completes; +- a lease acquisition times out while the restore holds the database. diff --git a/docs/adr/0076-upgrade-and-growth.md b/docs/adr/0076-upgrade-and-growth.md index 3f67310..3a7cec1 100644 --- a/docs/adr/0076-upgrade-and-growth.md +++ b/docs/adr/0076-upgrade-and-growth.md @@ -150,3 +150,15 @@ In the cluster, still to be run by the maintainer: exactly the named risk. - **Retiring the old build automatically on every sync.** Retirement is irreversible. It stays a deliberate step, guarded by the checks above. + +## Amendment: barrier tool fixes (M8 review, 2026-09-30) +Two defects in `upgrade-barrier.sh` would have blocked the in-cluster barrier: +- Its Ready-pod template printed the list root instead of the pod name. Selection + now uses a plain jsonpath (`tools/ops/lib/pods.sh`) and requires exactly one + Ready pod that is not being deleted. Fixtures cover one pod, not Ready, being + deleted, no condition and two Ready pods. +- Its pod lacked `fsGroup`, so it could not read the 0440 Secret files. The pod + now follows the same contract as the other database-tool pods. + +Both script-generated pods (backup and barrier) render with `--print-manifest`, +and the deploy policy check inspects them. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 51bfaac..444e6a4 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -3034,3 +3034,56 @@ Fixes (this segment): - scheduling onto a second labelled node; - the Windows MSVC build (no Windows target changed; the server and PostgreSQL code are Linux-only). + +## M8.1–M8.9 review fixes - 2026-09-30 + +Review: `.cache/reviews/M8_REVIEW_680e7e5-5b482ad.md` (one P1, five P2). +- **P1 restore ownership.** `backup-check.sh` passed in the Release, analysis and + ASan/UBSan PostgreSQL builds, including three new cases: + - A schema created after the checks but before `pg_restore`: exit 4 ("pg_restore + failed; the database was not changed"). The competing table survived, and only + this restore's own objects directory was removed. + - A second concurrent restore of the same database: exit 4 ("another restore + holds this database"). It left no directory, and the first restore was + verified. + - While the first restore held the database, a lease acquisition there ended in + `lock timeout`. An occupied objects path was refused and left untouched. + - The reviewer's `restore-race.sh` now ends with `remaining_evolution_schemas=1` + (`schema=no, objects=yes`). +- **P2 barrier Ready-pod selection.** `tools/ops/lib/pods.sh` uses a plain + jsonpath and needs exactly one Ready pod that is not being deleted. + `ops-lib-check.sh` passed its fixtures: one pod, not Ready, being deleted, no + condition, Ready beside a deleting pod, two Ready pods, and no pods. Not run: + the jsonpath against a live API server (no kubectl or Go in this environment). +- **P2 barrier Secret access.** The generated pod has `fsGroup: 10001`. The deploy + policy check now renders the backup and barrier pods with `--print-manifest` and + checks: non-root 10001 with fsGroup 10001, no token, read-only root, no + escalation, Secret files 0440, and egress admission of the component. Not run: a + disposable pod reading the Secret in the cluster. +- **P2 World0 claimRef.** A new admission validation denies removing a World0 + volume's claim reference, or changing its namespace or name, without + break-glass. The policy check requires the rule, and `verify-isolation.sh` + dry-runs the removal when a World0 PV exists. Not run: no admission test cluster + here, and no World0 PV exists. +- **P2 CI change selection.** `ci-paths.sh` compares against the event's base (the + push's `before`, or the PR base, fetched if missing). It runs on manual + dispatch, local runs and any uncertainty, and the path lists now include the + check scripts. `ci-paths-check.sh` passed 11 fixtures, and the deploy suite runs + it first. The reviewer's `ci-range.sh` now runs in both cases. Not run: a real + Gitea push, so the event-context mapping is unverified. +- **P2 drill verdicts.** Every kill RTO is asserted against `RTO_BUDGET_MS` + (60 000). The empty scenario needs positive quarantine evidence; INCONCLUSIVE and + other exits fail. Acknowledgements are read from the dead or drained owner's + finalized log. The fixtures passed. Not run: the drill itself. +- Mutations, each caught: + - dropping a schema that was not installed: `backup-check.sh` failed; + - removing `fsGroup`: the policy check failed; + - accepting several Ready pods, or treating never-ran as PASS: the + `ops-lib-check.sh` fixtures failed; + - falling back to `HEAD^` when there is no base: the `ci-paths-check.sh` fixtures + failed. +- The host-loss and upgrade rehearsals, and the PostgreSQL migration and + permission suites, still pass in all three builds. The deploy policy check: 67 + objects, 0 failures. +- ROADMAP corrected: M8-dev/stage is implemented and locally qualified, but **not + passed** until the live gates listed there are run. diff --git a/tools/backup/cluster-backup.sh b/tools/backup/cluster-backup.sh index 40d63aa..70b3809 100644 --- a/tools/backup/cluster-backup.sh +++ b/tools/backup/cluster-backup.sh @@ -3,6 +3,7 @@ # (ADR-0074). # # tools/backup/cluster-backup.sh --to DIR [--verify WORLD] [--context NAME] +# tools/backup/cluster-backup.sh --print-manifest # # Runs one short-lived Pod in evolution- (image: the postgres-tools digest # pinned in deploy/kubernetes/stores/; migrator login; the aevolution-objects @@ -14,11 +15,12 @@ # hypervisor host. --verify WORLD then runs verify-backup.sh on this machine # (restore on a private cluster, rebuilt and served as isolated world WORLD); it # needs the PostgreSQL 18.6 binaries and evo-server/evo-world-clone. -# World0 is refused before anything touches the cluster. The Pod is an operator +# --print-manifest prints the Pod and exits without touching the cluster (the deploy +# policy check inspects it). World0 is refused before anything touches the cluster. The Pod is an operator # action, like fault-drill.sh; it is not part of any Argo application. A scheduled # backup (CronJob in Git, synced by Argo) follows once the destination exists. set -euo pipefail -usage() { echo "usage: $0 --to DIR [--verify WORLD] [--context NAME]" >&2; exit 2; } +usage() { echo "usage: $0 --to DIR [--verify WORLD] [--context NAME] | --print-manifest" >&2; exit 2; } [ $# -ge 1 ] || usage env="$1"; shift case "$env" in @@ -26,16 +28,19 @@ case "$env" in *) echo "refused: '$env' is not a disposable environment (only dev or stage; never world0)" >&2 exit 2 ;; esac -to="" verify="" k=(kubectl --request-timeout=30s) +to="" verify="" print=0 k=(kubectl --request-timeout=30s) while [ $# -gt 0 ]; do case "$1" in --to) to="${2:-}"; shift 2 ;; --verify) verify="${2:-}"; shift 2 ;; + --print-manifest) print=1; shift ;; --context) k+=(--context "${2:?}"); shift 2 ;; *) usage ;; esac done -[ -n "$to" ] && [ -d "$to" ] || { echo "--to must be an existing directory" >&2; exit 2; } +if [ "$print" != 1 ]; then + [ -n "$to" ] && [ -d "$to" ] || { echo "--to must be an existing directory" >&2; exit 2; } +fi root="$(cd "$(dirname "$0")/../.." && pwd)" ns="evolution-$env" image="$(awk '/name: aevolution-postgres-tools/ {f = 1} f && /newName:/ {n = $2} f && /digest:/ {print n "@" $2; exit}' \ @@ -45,15 +50,14 @@ stamp="$(date -u +%Y%m%dT%H%M%SZ)" pod="aevolution-backup-$(date -u +%Y%m%d%H%M%S)" dest="$to/$env-$stamp" say() { echo "$(date -u +%H:%M:%S) $*"; } -cleanup() { "${k[@]}" delete pod -n "$ns" "$pod" --wait=false > /dev/null 2>&1 || true; } -trap cleanup EXIT -say "starting $pod in $ns ($image)" -"${k[@]}" apply -n "$ns" -f - > /dev/null < /dev/null 2>&1 || true; } +trap cleanup EXIT +say "starting $pod in $ns ($image)" +manifest | "${k[@]}" apply -f - > /dev/null for _ in $(seq 1 1800); do phase="$("${k[@]}" get pod -n "$ns" "$pod" -o jsonpath='{.status.phase}' 2> /dev/null || true)" diff --git a/tools/backup/restore-world.sh b/tools/backup/restore-world.sh index b6717ed..12beb34 100644 --- a/tools/backup/restore-world.sh +++ b/tools/backup/restore-world.sh @@ -14,13 +14,23 @@ # Order, failing closed at each step: # 1. every bundle file matches SHA256SUMS, no file is unlisted, and every object's # name is its digest; nothing is touched before this passes; -# 2. the target has no `evolution` schema and runs the manifest's server version; +# 2. the targets are claimed for this invocation alone (below), then the target +# has no `evolution` schema and runs the manifest's server version; # 3. pg_restore in one transaction, then objects are copied; # 4. the restored heads equal the MANIFEST heads exactly; # 5. every region's full history is audited (evo-world-clone audit: each link's # manifest, payload, trace replay and outcomes, from genesis to the head). -# On a failure after step 2 the restored schema and the copied objects are removed, -# so an unverified restore can never be mistaken for a world. The result is a +# Ownership (review M8 P1): --objects is claimed by creating it (mkdir, never -p: an +# existing path is refused and never touched), and the database by one psql session +# held open for the whole restore. That session takes the restore claim (advisory +# 71420004, try-lock: a concurrent restore of the same database refuses at once) and +# the upgrade lock exclusively (71420002: provisioning, acquisition and lease checks +# in that database wait until this restore is verified or removed). On failure only +# what this invocation created is removed: the schema only if its own pg_restore +# committed, the objects directory only if its own mkdir created it. So an +# unverified restore can never be mistaken for a world, and a losing restore never +# deletes a winner's state. EVO_RESTORE_TEST_HOOK (tests only) is run by bash with +# $1 = before-restore (claimed, checked) and after-restore (installed, objects copied). The result is a # verbatim, owner-only copy with the source's world identities: it has no runtime # grants and is never served. To run it, clone it into an isolated world # (tools/backup/verify-backup.sh --isolate). @@ -89,32 +99,69 @@ if [ "$in_place" = 1 ]; then privileges=() fi -# 2. Target: empty, same server version. -[ ! -e "$objects" ] || refuse "objects target already exists: $objects" +target="${PGDATABASE:-}" +[ -n "${PGSERVICE:-}" ] && target="service=$PGSERVICE" +[ -n "$target" ] || { echo "set PGDATABASE or PGSERVICE for the target" >&2; exit 2; } + +# 2. Claim the targets, then check them under the claim. +created_objects=0 installed=0 restored=0 lock_pid="" claim="" +cleanup() { + local code=$? + if [ "$restored" != 1 ]; then + if [ "$installed" = 1 ]; then + psql -X --no-password -q -v ON_ERROR_STOP=1 -c 'DROP SCHEMA evolution CASCADE' > /dev/null 2>&1 || true + fi + [ "$created_objects" = 1 ] && rm -rf -- "$objects" + if [ "$installed" = 1 ] || [ "$created_objects" = 1 ]; then + echo "RESTORE FAILED: removed only what this restore created" \ + "(schema=$([ "$installed" = 1 ] && echo yes || echo no), objects=$([ "$created_objects" = 1 ] && echo yes || echo no))" >&2 + fi + fi + [ -n "$claim" ] && exec 7>&- # Ends the claim session: its locks are released. + [ -n "$lock_pid" ] && { wait "$lock_pid" 2> /dev/null || true; } + [ -n "$claim" ] && rm -rf -- "$claim" + return "$code" +} +trap cleanup EXIT +mkdir -m 700 -- "$objects" 2> /dev/null || refuse "objects target already exists or cannot be created: $objects" +created_objects=1 +claim="$(mktemp -d /tmp/evo-restore-claim.XXXXXXXX)" +mkfifo "$claim/in" +psql -X --no-password -q -A -t -v ON_ERROR_STOP=1 < "$claim/in" > "$claim/out" 2>&1 & +lock_pid=$! +exec 7> "$claim/in" +cat >&7 < /dev/null || break + sleep 0.1 +done +[ -e "$claim/busy" ] && refuse "another restore holds this database" +[ -e "$claim/held" ] || refuse "could not claim the target database ($(head -c 200 "$claim/out"))" version="$(psql -X --no-password -A -t -v ON_ERROR_STOP=1 -c "SELECT current_setting('server_version')")" grep -qx "database $version" "$bundle/MANIFEST" || refuse "target server $version differs from the manifest" existing="$(psql -X --no-password -A -t -v ON_ERROR_STOP=1 -c "SELECT count(*) FROM pg_namespace WHERE nspname = 'evolution'")" [ "$existing" = 0 ] || refuse "target database already has an evolution schema" -restored=0 -cleanup() { - [ "$restored" = 1 ] && return - psql -X --no-password -q -v ON_ERROR_STOP=1 -c 'DROP SCHEMA IF EXISTS evolution CASCADE' > /dev/null 2>&1 || true - rm -rf -- "$objects" - echo "RESTORE FAILED: the restored schema and objects were removed" >&2 -} -trap cleanup EXIT - -# 3. Database in one transaction, then objects. -target="${PGDATABASE:-}" -[ -n "${PGSERVICE:-}" ] && target="service=$PGSERVICE" -[ -n "$target" ] || { echo "set PGDATABASE or PGSERVICE for the target" >&2; exit 2; } +# 3. Database in one transaction, then objects. A failed pg_restore installed +# nothing (one transaction), so nothing of the database is removed after it. +hook() { [ -z "${EVO_RESTORE_TEST_HOOK:-}" ] || bash -c "$EVO_RESTORE_TEST_HOOK" hook "$1"; } +hook before-restore pg_restore --no-password --exit-on-error --single-transaction --no-owner "${privileges[@]}" \ - --dbname="$target" "$bundle/database.dump" -mkdir -m 700 -- "$objects" + --dbname="$target" "$bundle/database.dump" || refuse "pg_restore failed; the database was not changed" +installed=1 (cd "$bundle/objects" && find . -mindepth 2 -maxdepth 2 -type d -printf '%P\0' | xargs -0 -r -I{} mkdir -p -m 700 -- "$objects/{}") (cd "$bundle/objects" && find . -type f -printf '%P\0' | xargs -0 -r -I{} cp -- {} "$objects/{}") +hook after-restore # 4. Heads exactly as recorded. actual_heads="$(psql -X --no-password -A -t -F ' ' -v ON_ERROR_STOP=1 -c " diff --git a/tools/ci/backup-check.sh b/tools/ci/backup-check.sh index cab257d..3de1db0 100644 --- a/tools/ci/backup-check.sh +++ b/tools/ci/backup-check.sh @@ -90,4 +90,46 @@ PGDATABASE=evo_restore_twice bash "$root/tools/backup/restore-world.sh" --bundle [ "$code" = 4 ] && grep -q 'already has an evolution schema' "$work/twice.log" || fail "a second restore into one database" [ "$(PGDATABASE=evo_restore_twice psql -XAtc 'SELECT end_tick FROM evolution.region_heads WHERE world_id = 300')" = "$backup_tick" ] || fail "the refused second restore changed the first" +# Ownership races (review M8 P1). A competing actor creates the schema between this +# restore's checks and its pg_restore: the restore fails and leaves that schema alone. +createdb evo_restore_race +code=0 +PGDATABASE=evo_restore_race EVO_RESTORE_TEST_HOOK='[ "$1" = before-restore ] && psql -XAtq -c "CREATE SCHEMA evolution" -c "CREATE TABLE evolution.winner(x int)" || true' \ + bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" --objects "$work/race-a" \ + > "$work/race-a.log" 2>&1 || code=$? +[ "$code" = 4 ] && grep -q 'pg_restore failed; the database was not changed' "$work/race-a.log" || + { cat "$work/race-a.log" >&2; fail "a pre-empted restore: exit $code"; } +[ "$(PGDATABASE=evo_restore_race psql -XAtc "SELECT count(*) FROM pg_tables WHERE schemaname = 'evolution' AND tablename = 'winner'")" = 1 ] || + fail "a pre-empted restore deleted the competing schema" +[ ! -e "$work/race-a" ] || fail "a pre-empted restore left its own objects directory" +# Two restores of one database: the second refuses at once and touches nothing of +# the first, which completes. While the first holds the database (after its install), +# a lease acquisition there waits instead of running against unverified state. +createdb evo_restore_pair +PGDATABASE=evo_restore_pair EVO_RESTORE_TEST_HOOK=' + if [ "$1" = before-restore ]; then touch "'"$work"'/pair-claimed"; sleep 3; fi + if [ "$1" = after-restore ]; then + psql -XAt -c "SET lock_timeout = '"'"'1s'"'"'" -c "SELECT * FROM evolution.acquire_region_lease(300, 1, 20000, decode(repeat('"'"'b1'"'"', 32), '"'"'hex'"'"'))" > "'"$work"'/pair-acquire.log" 2>&1 || true + fi' bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" --objects "$work/pair-a" \ + > "$work/pair-a.log" 2>&1 & +pair_pid=$! +for _ in $(seq 1 100); do [ -e "$work/pair-claimed" ] && break; sleep 0.1; done +code=0 +PGDATABASE=evo_restore_pair bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" \ + --objects "$work/pair-b" > "$work/pair-b.log" 2>&1 || code=$? +[ "$code" = 4 ] && grep -q 'another restore holds this database' "$work/pair-b.log" || + { cat "$work/pair-b.log" >&2; fail "a concurrent restore: exit $code"; } +[ ! -e "$work/pair-b" ] || fail "the refused concurrent restore left its objects directory" +code=0; wait "$pair_pid" || code=$? +[ "$code" = 0 ] && grep -q '^RESTORE VERIFIED regions=1 ' "$work/pair-a.log" || + { cat "$work/pair-a.log" >&2; fail "the first of two restores: exit $code"; } +grep -q 'lock timeout' "$work/pair-acquire.log" || + { cat "$work/pair-acquire.log" >&2; fail "a lease acquisition ran during the restore"; } +# An existing objects path is never claimed or removed. +mkdir "$work/occupied"; echo keep > "$work/occupied/mine" +createdb evo_restore_occupied +code=0 +PGDATABASE=evo_restore_occupied bash "$root/tools/backup/restore-world.sh" --bundle "$work/bundle" \ + --objects "$work/occupied" > "$work/occupied.log" 2>&1 || code=$? +[ "$code" = 4 ] && [ "$(cat "$work/occupied/mine")" = keep ] || fail "an occupied objects path: exit $code" echo "backup and restore qualification passed (backup tick $backup_tick, acknowledged $acked_before, source $source_tick)" diff --git a/tools/ci/ci-paths-check.sh b/tools/ci/ci-paths-check.sh new file mode 100644 index 0000000..ed8bd4a --- /dev/null +++ b/tools/ci/ci-paths-check.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Offline regression for tools/ci/ci-paths.sh (review M8 P2) in a disposable Git +# repository: multi-commit pushes ending in documentation, missing and unknown bases, +# pull requests, manual dispatch, and changes to only a check script. Uncertainty +# must run validation. Run by the deploy suite before it decides anything. +set -euo pipefail +script="$(cd "$(dirname "$0")" && pwd)/ci-paths.sh" +repo="$(mktemp -d /tmp/evo-ci-paths.XXXXXXXX)" +trap 'rm -rf -- "$repo"' EXIT +cd "$repo" +git init -q +git config user.email ci@example.invalid +git config user.name ci +mkdir -p src docs tools/ci +echo base > src/a.cpp; echo base > docs/a.md; echo base > tools/ci/dr-check.sh +git add .; git commit -qm base +base="$(git rev-parse HEAD)" +echo change >> src/a.cpp; git commit -qam implementation +echo change >> docs/a.md; git commit -qam documentation +docs_only_base="$(git rev-parse HEAD~1)" +failures=0 +expect() { # expect