From 5512fdec54b1ca4646b23e0b57df3bf59d3504a1 Mon Sep 17 00:00:00 2001 From: dingo Date: Wed, 30 Sep 2026 15:51:04 -0400 Subject: [PATCH] feat(backup): M8.7 world backup bundles and verified restore (ADR-0074) Online, snapshot-consistent bundles (pg_dump at an exported snapshot, immutable objects verified by digest, MANIFEST of versions/builds/heads, SHA256SUMS); a fail-closed restore that audits every region's full history on independent storage; an isolated rebuild through clones that serves under load; an operator-run in-cluster backup streamed off the cluster. backup-check.sh runs in every PostgreSQL build, with corruption, missing-object, tampered-manifest and non-empty-target refusals. Destination, encryption, schedule and WAL archiving remain on the backlog. Co-Authored-By: Claude Opus 5.5 --- .gitea/workflows/build.yaml | 2 + deploy/docker/Dockerfile | 5 + .../stores/base/database-egress.yaml | 6 +- docs/BACKLOG.md | 4 +- docs/ROADMAP.md | 10 + docs/STATUS.md | 20 ++ docs/adr/0074-world-backup-and-restore.md | 172 ++++++++++++++++++ docs/adr/README.md | 1 + docs/engineering/VALIDATION.md | 36 ++++ docs/infrastructure/ENVIRONMENTS.md | 28 ++- docs/infrastructure/OPERATIONS.md | 13 +- tools/backup/backup-world.sh | 105 +++++++++++ tools/backup/cluster-backup.sh | 135 ++++++++++++++ tools/backup/restore-world.sh | 121 ++++++++++++ tools/backup/verify-backup.sh | 120 ++++++++++++ tools/ci/backup-check.sh | 93 ++++++++++ tools/ci/ci-paths.sh | 4 +- tools/ci/deploy-policy-check.py | 5 +- tools/ci/postgres-check.sh | 2 + 19 files changed, 869 insertions(+), 13 deletions(-) create mode 100644 docs/adr/0074-world-backup-and-restore.md create mode 100644 tools/backup/backup-world.sh create mode 100644 tools/backup/cluster-backup.sh create mode 100644 tools/backup/restore-world.sh create mode 100644 tools/backup/verify-backup.sh create mode 100644 tools/ci/backup-check.sh diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml index 01249d8..4df592d 100644 --- a/.gitea/workflows/build.yaml +++ b/.gitea/workflows/build.yaml @@ -86,6 +86,8 @@ jobs: bash tools/ci/docker-build.sh postgres-tools "$image" aevolution-postgres-cache docker run --rm --network none --read-only --cap-drop ALL \ --entrypoint /opt/postgres/bin/psql "$image" --version + docker run --rm --network none --read-only --cap-drop ALL \ + --entrypoint /opt/postgres/bin/pg_dump "$image" --version - name: Publish configured migration tools if: ${{ gitea.ref == 'refs/heads/main' && gitea.event_name != 'pull_request' && vars.HARBOR_POSTGRES_REPOSITORY != '' }} env: diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 3405a15..3e197e0 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -59,6 +59,7 @@ WORKDIR /source COPY deploy/ deploy/ COPY tools/ci/deploy-policy-check.py tools/ci/ COPY tools/ops/ tools/ops/ +COPY tools/backup/ tools/backup/ RUN python3 tools/ci/deploy-policy-check.py # M8.5: monitoring configuration and alert tests with the pinned images' own tools. COPY --from=quay.io/prometheus/prometheus@sha256:6976aa8a60fec930796ce5772b8d12da7a318a5daa8d40d69c5c7819a05eeed7 /bin/promtool /usr/local/bin/promtool @@ -109,9 +110,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends libssl3t64 zlib && rm -rf /var/lib/apt/lists/* \ && groupadd --gid 10001 evo && useradd --uid 10001 --gid 10001 --no-create-home evo COPY --from=postgres-toolchain /source/.cache/postgres/install/bin/psql /opt/postgres/bin/psql +# M8.7 backups (ADR-0074): the same pinned PostgreSQL build's dump tools. +COPY --from=postgres-toolchain /source/.cache/postgres/install/bin/pg_dump /opt/postgres/bin/pg_dump +COPY --from=postgres-toolchain /source/.cache/postgres/install/bin/pg_restore /opt/postgres/bin/pg_restore COPY --from=postgres-toolchain /source/.cache/postgres/install/lib/libpq.so* /opt/postgres/lib/ COPY --from=postgres-toolchain /source/.cache/postgres/postgresql-18.6/COPYRIGHT /usr/share/doc/evo-postgres/COPYRIGHT COPY tools/migrate-postgres.sh /opt/evo/tools/migrate-postgres.sh +COPY tools/backup/backup-world.sh /opt/evo/tools/backup-world.sh COPY migrations/ /opt/evo/migrations/ ENV PATH="/opt/postgres/bin:${PATH}" LD_LIBRARY_PATH=/opt/postgres/lib USER 10001:10001 diff --git a/deploy/kubernetes/stores/base/database-egress.yaml b/deploy/kubernetes/stores/base/database-egress.yaml index 79270f3..244c7d8 100644 --- a/deploy/kubernetes/stores/base/database-egress.yaml +++ b/deploy/kubernetes/stores/base/database-egress.yaml @@ -1,5 +1,5 @@ -# Egress from migrate, store-check and region pods to exactly the AEvolution -# PostgreSQL VM (pg-forge-01) and nothing else (ADR-0070). +# Egress from migrate, store-check, region and operator backup pods to exactly the +# AEvolution PostgreSQL VM (pg-forge-01) and nothing else (ADR-0070, ADR-0074). apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -13,7 +13,7 @@ spec: matchExpressions: - key: app.kubernetes.io/component operator: In - values: [migrate, store-check, region] + values: [migrate, store-check, region, backup] policyTypes: [Egress] egress: - to: diff --git a/docs/BACKLOG.md b/docs/BACKLOG.md index 57fbbaa..fe8946a 100644 --- a/docs/BACKLOG.md +++ b/docs/BACKLOG.md @@ -7,7 +7,9 @@ names it. Remove an item only when it is done, and record where it was done. | Item | Raised | Why deferred | Needed before | |---|---|---|---| | **World0 break-glass membership.** Decide who may act as `aevolution:world0-break-glass`, and how that group is granted: for example a short-lived client certificate with that organization, or an OIDC group. Decide also how its use is approved and recorded. The admission policy already enforces the group (ADR-0069). | M8.2, 2026-09-29 | Maintainer: figure out later | Any intentional deletion of World0 data or identity; M8.8 World0 launch | -| **Backup destination for dev/stage stores.** There are no backups yet (maintainer, 2026-09-29). The plan is to build backup tooling for the AEvolution PostgreSQL VM and each `aevolution-objects` volume up to a pluggable destination, then add the destination. It must be off the host for anything beyond disposable environments. | M8.3, 2026-09-29 | The maintainer will supply a destination later | M8.7 backup and restore; any World0 data | +| **Backup destination for dev/stage stores.** The tooling now exists up to the destination (ADR-0074): `tools/backup/cluster-backup.sh --to DIR` writes a verified bundle to any directory, and `verify-backup.sh` restore-tests it. Still to choose: the permanent off-host destination, and with it (1) bundle encryption and key custody, (2) a scheduled CronJob synced by Argo, (3) retention (proposal: 30 daily, 12 weekly, 12 monthly), (4) incremental object copies. | M8.3, 2026-09-29; M8.7, 2026-09-30 | The maintainer will supply a destination later | Scheduled backups; any World0 data; M8.8 | +| **WAL archiving on pg-forge-01.** Bundles are logical backups: the recovery point is the last bundle. Continuous archiving (`archive_command` or `pg_receivewal`) plus periodic base backups would allow point-in-time recovery. It needs the backup destination. | M8.7, 2026-09-30 | Needs the destination and VM configuration | M8.8 host-loss RPO decision | +| **Harbor retention of build images.** A bundle's MANIFEST names every build (region image digest) that wrote history; recovery and replay need those images. Confirm Harbor's retention/tag-immutability policy keeps them (or pin them in a protected project). | M8.7, 2026-09-30 | Not yet checked | M8.8; any upgrade barrier that retires a build | | **Secret manager for Argo-delivered configuration.** Secrets are still created by hand (ADR-0069 amendment). Choose how they reach the cluster through Git or Argo: for example Vault with the Agent Injector (as in the maintainer's other environment), or sealed or external secrets. Until then, rotate by hand. | M8.3, 2026-09-29 | The maintainer wants everything through Argo; a secret manager is not chosen | Stage and World0 credentials; M8.4 | | **Store Job values.** Supply the published registry digests for `aevolution-postgres-tools` and `aevolution-server` (from each CI Publish step's `Digest:` line), and the NOLOGIN runtime group role for dev (`aevolution`) and for stage (`aevolution_stage`). Then include `stores/` in each environment's path. | M8.3, 2026-09-29 | Values pending | M8.3 in-cluster evidence | | **Alert destination.** Alertmanager routes every alert to a null receiver (ADR-0072). Choose where alerts go (Discord, email, a webhook), and optionally a heartbeat receiver for the always-firing Watchdog. Receiver credentials go in a Secret. | M8.5, 2026-09-30 | The maintainer will decide later | Any claim that alerts reach a person; M8.6 alert evidence; World0 | diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 61c5ad8..8a5e20f 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -650,3 +650,13 @@ M8.6 implemented and locally qualified (2026-09-30), under ADR-0073: and an operator-driven database outage; - RTO and zero-lost-acknowledgement checks. The in-cluster drill results wait on the maintainer. + +M8.7 implemented and locally qualified (2026-09-30), under ADR-0074: +- online, snapshot-consistent world bundles (database dump, immutable objects, + MANIFEST of versions, builds and heads, checksums); +- a fail-closed restore that audits every region's full history on independent + storage, and an isolated rebuild through clones that serves under load; +- corruption, missing-object, tampered-manifest and non-empty-target refusals; +- an operator-run in-cluster backup to a directory off the cluster. +Pending the maintainer's destination: encryption, schedule, retention and WAL +archiving (docs/BACKLOG.md). In-cluster bundle evidence waits on a run. diff --git a/docs/STATUS.md b/docs/STATUS.md index d7e508a..fd407dd 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1105,3 +1105,23 @@ It restores the Git configuration on exit. Earlier (M8.4 follow-up): `REGION_RETAINED` keeps earlier builds recoverable across image upgrades. + +## M8.7 backup and restore (2026-09-30) + +ADR-0074 adds world backup bundles and a verified restore: +- `tools/backup/backup-world.sh`: an online backup cut by one exported database + snapshot (MANIFEST heads and `pg_dump` read the same snapshot), then the + immutable objects, each checked against its digest; `SHA256SUMS` over all of it. +- `tools/backup/restore-world.sh`: checksums first, then an owner-only restore into + an empty database, exact head comparison and a full history audit of every region; + any failure removes what it restored. +- `tools/backup/verify-backup.sh`: the restore on a private throwaway cluster with + fresh roles, then (`--isolate WORLD`) a clone into an isolated world that serves + under synthetic load and drains. +- `tools/backup/cluster-backup.sh --to DIR`: an operator Pod (read-only + objects volume, migrator login) whose bundle is streamed off the cluster. +- The postgres-tools image now carries `pg_dump` and `pg_restore` from the pinned + 18.6 build; the database egress policy admits the `backup` component. + +The destination, encryption, schedule, retention and WAL archiving wait on the +maintainer (docs/BACKLOG.md). diff --git a/docs/adr/0074-world-backup-and-restore.md b/docs/adr/0074-world-backup-and-restore.md new file mode 100644 index 0000000..3ae80ca --- /dev/null +++ b/docs/adr/0074-world-backup-and-restore.md @@ -0,0 +1,172 @@ +# ADR-0074: World backup bundles and verified restore + +## Status +Accepted for M8.7, 2026-09-30. It builds on: +- ADR-0061 (immutable, content-addressed objects; upload verified before commit); +- ADR-0067 (clones and history audit); +- ADR-0070 (dev/stage stores); +- ADR-0073 (synthetic load). + +The permanent backup destination, encryption, scheduling and WAL archiving wait +on the maintainer's choice of destination (docs/BACKLOG.md). Until then, this ADR +covers everything up to that destination. + +## Context +M8.7 requires: +- a coordinated manifest covering the database, the objects, the versions, the + binaries and any in-flight state; +- a restore onto independent storage; +- retention and GC that respect clone and backup pins. + +The evidence asked for is: +- rebuilding an isolated world from a backup; +- corruption and missing-object failures; +- credential and key recovery. + +The named risk is a backup that exists but cannot reconstruct causal state. + +docs/infrastructure/OPERATIONS.md proposes a coordinated admission barrier. Two +facts make a barrier unnecessary for a single database: +- a region's durable state is one PostgreSQL database plus immutable objects; +- every object a committed row references was verified durable before that row + committed. + +## Decision +**Bundle.** `tools/backup/backup-world.sh --objects DIR --out DIR` writes: + +| File | Content | +|---|---| +| `MANIFEST` | Format, time and label. The server version. Every schema migration and its checksum. Every retained build (image digest) and whether it is retired. Every region head (world, region, frame, sequence, tick, commit, digest, build). Row counts, and the object count and bytes | +| `database.dump` | `pg_dump` custom format of schema `evolution`, with no owners and no grants | +| `objects/w/r/` | Every object in the store, each checked against its name while it is copied | +| `SHA256SUMS` | Checksums of every other file | + +**The cut.** One psql session opens a `REPEATABLE READ, READ ONLY` transaction and +exports its snapshot. The session reads the heads for the MANIFEST, then runs +`pg_dump --snapshot` on that same snapshot while the session holds it. Objects are +copied afterwards. +- Objects are never overwritten or deleted, and no GC exists. So every object the + snapshot references is present and final when it is copied. +- Objects written after the snapshot are copied as harmless orphans. +- Nothing is written to the source, and admission never stops. The backup runs + online while the host commits. + +**In-flight state.** +- Every acknowledgement made before the snapshot is in the bundle, because an + acknowledgement follows its commit. +- A commit whose transaction completes after the snapshot is not in the bundle. + It was not acknowledged at the cut. +- Unacknowledged inputs are never durable, by design (ADR-0064). +- Lease rows are copied, but a restored copy is never served under them (see + below). +- There are no cross-region transfers yet. When they exist, they live in the same + database and are covered by the same snapshot. + +**Restore.** `tools/backup/restore-world.sh --bundle B --objects DIR` targets an +empty database owned by the restoring login, and fails closed in this order: +1. Every file matches `SHA256SUMS`, no file is unlisted, and every object's name + is its digest. Nothing is touched before this passes. +2. The target has no `evolution` schema and runs the manifest's server version. +3. `pg_restore` runs in one transaction, then the objects are copied. +4. The restored heads equal the MANIFEST heads exactly. +5. Every region's full history is audited with `evo-world-clone audit`. The audit + checks every link's manifest, payload, trace replay and outcomes, from genesis + to the head. + +A failure after step 2 removes the restored schema and objects, so an unverified +restore cannot be mistaken for a world. The result is a verbatim, owner-only copy +that keeps the source's world identities. It has no runtime grants and is never +served. + +**Isolated rebuild.** `tools/backup/verify-backup.sh BUNDLE --isolate WORLD`: +1. Starts a private, throwaway PostgreSQL cluster on local disk. +2. Restores the bundle as a fresh login that owns only the restore database. +3. Creates a separate database with its own owner, runtime group and login, and + migrates it with `migrate-postgres.sh`. +4. Clones every region of the source world into `WORLD` with + `evo-world-clone clone`, which audits the source history first. Provenance is + kept in `region_origins`. +5. Runs `evo-server` on each clone under synthetic load until it acknowledges 20 + ticks past the backup head, then drains it. + +This is OPERATIONS.md steps 3–5 for a clone. Restoring in place under the source's +own identity, with the old writers fenced, is host-loss recovery and belongs to +M8.8. + +**Credentials and keys.** +- A bundle carries no roles, grants, passwords or connection details. +- Restoring needs only a login that owns an empty database, and no source + credential. `backup-check.sh` and `verify-backup.sh` prove this with fresh, + unrelated roles. +- Bundles are not encrypted yet, so there is no key to lose. Encryption and its key + custody are chosen together with the destination. They are required before any + World0 data leaves the host. + +**Retention and GC.** +- Each bundle is a full, self-contained copy, and no bundle depends on another. + Deleting one never breaks another. +- Clones copy their payloads into the target namespace (ADR-0067), so a clone does + not depend on its source either. +- The object store has no delete, and history never expires. +- Any future GC must prove reachability from every live head, clone origin and + retained build. Bundles hold their own copies, so GC cannot damage them. +- Retention of bundles at the destination (the OPERATIONS proposal: 30 daily, 12 + weekly, 12 monthly) is applied when the destination exists. + +**Binaries.** The MANIFEST names every build identity that wrote history. These +are the region image digests that recovery and replay need. Harbor must keep those +images; its retention policy is not yet verified (backlog). + +**In the cluster.** `tools/backup/cluster-backup.sh --to DIR [--verify WORLD]` +is an operator action, like `fault-drill.sh`, and refuses World0 before kubectl +runs. It works as follows: +- It runs one short-lived Pod (component `backup`) with: + - the environment's pinned postgres-tools image, which now includes `pg_dump`, + `pg_restore` and `backup-world.sh`; + - the migrator login; + - the `aevolution-objects` volume mounted read-only on the store node; + - a 12 GiB emptyDir for the bundle. +- The database egress policy admits the `backup` component. +- The bundle is streamed out with `kubectl exec tar` and its checksums are checked + locally. Then the Pod is deleted, including on failure. + +A scheduled CronJob, synced by Argo, follows once the destination exists. + +**Qualification.** `tools/ci/backup-check.sh` runs inside `postgres-check.sh` for +every PostgreSQL build: +- It backs up a real `evo-server` while the server commits under synthetic load. + The backup head covers every acknowledgement made before the backup, and the + source then moves past it. +- It runs `verify-backup.sh --isolate` on a second cluster. +- It restores damaged bundles, each of which must be refused, leaving nothing + behind: a corrupt object, an unlisted file, a damaged dump, a missing payload + that has consistent checksums, and a MANIFEST head that disagrees with the + database. +- It restores a second time into a database that already holds a world, which + must be refused. + +## Consequences +- **RPO.** The recovery point is the last verified bundle. Without WAL archiving, + this is a logical backup and cannot recover to an arbitrary point in time. + Continuous archiving on pg-forge-01 needs the destination (backlog). M8.8 decides + the host-loss RPO for World0. +- **Full copies.** Every bundle copies the whole object store. That is fine at + dev/stage sizes (the objects volume is 10 GiB). Incremental object copies, which + are safe because objects are content-addressed, come with the destination. +- **Off host.** A bundle is off the host only if `--to` is on another machine. The + maintainer's workstation qualifies only if it is not the hypervisor host. +- **Restore test.** A scheduled job or a successful upload is not a restore test. + Only `verify-backup.sh` output counts as restore evidence. + +## Alternatives rejected +- **Physical base backups (`pg_basebackup`) as the only copy.** They restore the + whole server, with its roles, and cannot be audited per region or restored as an + isolated world without a second step. They remain the likely basis for PITR once + WAL archiving exists. +- **An admission barrier before the cut.** It is unnecessary given + upload-before-commit and immutable objects, and it would pause the world. +- **Restoring directly as a runnable world with fresh grants.** It would serve the + source's world identity from a second place, which could become competing + authority. Isolation goes through the clone path, which re-identifies the world. +- **A new object-storage dependency (MinIO, S3 SDK).** The destination is not + chosen; a directory and `tar` need no new dependency. diff --git a/docs/adr/README.md b/docs/adr/README.md index 7705435..1dc7520 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -84,3 +84,4 @@ design. No unadopted candidate is a link dependency of the proof. - [ADR-0071: Region host deployment in dev and stage](0071-region-host-deployment.md) - accepted for M8.4 single-replica Recreate host, digest-as-build identity, idempotent provisioning hook and the region drill; no director yet. - [ADR-0072: Observability, alerting and background delivery](0072-observability.md) - accepted for M8.5 pinned AEvolution Prometheus/Alertmanager, 16 tested alert rules, bounded logs, database outbox over any broker, cardinality/cost report. - [ADR-0073: Synthetic load and fault drills](0073-fault-drills.md) - accepted for M8.6 drill-only deterministic input driver, kill/full/empty/db-outage drill with RTO and zero-lost-acknowledgement checks. +- [ADR-0074: World backup bundles and verified restore](0074-world-backup-and-restore.md) - accepted for M8.7 snapshot-consistent online bundles (dump + immutable objects + manifest), fail-closed audited restore on independent storage, isolated rebuild through clones; destination, encryption, schedule and WAL archiving pending. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index 7b0363a..2a532e4 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -2916,3 +2916,39 @@ Fixes (this segment): MSVC CTest 59/59 (the server is not part of the Windows build). `fault-drill.sh` passed `bash -n`. - Pending: a new image, then re-running the full, empty and db-outage scenarios. + +## M8.7 backup and restore - 2026-09-30 + +- `tools/ci/backup-check.sh` (inside `postgres-check.sh`; real PostgreSQL 18.6, a + real `evo-server` under synthetic load) passed in the Release, analysis and + ASan/UBSan PostgreSQL builds: + - An online backup taken while the server committed recorded head tick 60, at or + past every acknowledgement before it (60). The source then moved on to tick 101. + - `verify-backup.sh --isolate 301` on a second, throwaway cluster, as fresh + roles: the restore audited 13 commits to tick 60 and matched the MANIFEST. The + clone into world 301 resumed at `epoch=1 committed=60`, acknowledged to tick + 82, and drained with exit 0. + - Refused bundles, each leaving no schema and no objects behind: + - corrupt object, unlisted file, damaged dump: exit 4 at the checksum step; + - missing payload object with consistent checksums: exit 3, + `audit failed missing_payload sequence=0`; + - MANIFEST head changed: exit 3, heads differ. + - A second restore into a database that already holds a world was refused + (exit 4), and the first restore was left unchanged. + - The MANIFEST names no credentials. +- Mutations, each caught by `backup-check.sh`: + - restore without the history audit: the missing object was accepted (exit 0); + - a failed restore that keeps its files: restored objects were left behind; + - `pg_dump` outside the exported snapshot (2 s later): the restored heads + differed from the MANIFEST, and verification failed. +- The PostgreSQL migration, rollback and permission suites passed in all three + builds. `deploy-policy-check.py`: 67 objects, 0 failures. + `cluster-backup.sh` refuses World0 before kubectl (policy rule). +- `bash -n` passed on every new script. shellcheck is not installed in WSL, so it + did not run. +- Not executed: + - the postgres-tools image build with `pg_dump` (CI builds it on push); + - `cluster-backup.sh` against dev or stage; + - any restore of in-cluster data. +- The permanent destination, encryption, schedule and WAL archiving are not + implemented; they are pending the maintainer's choice of destination. diff --git a/docs/infrastructure/ENVIRONMENTS.md b/docs/infrastructure/ENVIRONMENTS.md index cc77a50..9a5a64d 100644 --- a/docs/infrastructure/ENVIRONMENTS.md +++ b/docs/infrastructure/ENVIRONMENTS.md @@ -192,7 +192,7 @@ To delete World0 data or identity on purpose: 1. Record the operation and its approval. 2. Use a credential whose groups include `aevolution:world0-break-glass`, for example a short-lived client certificate with that organization. -3. Take and verify a backup first (M8.7). +3. Take and verify a backup first (M8.7; see "Backups" below). Removing the admission policy itself is the break-glass of last resort. It needs cluster-admin and must be recorded the same way. Never test by deleting World0: the @@ -206,3 +206,29 @@ verify script uses server-side dry runs. - **World0 volumes.** `local-path` provisions volumes with Delete. Patch each World0 PV to `Retain` when it is bound, and the policy then keeps it there. Volumes stay node-local until M8.3. + +## Backups (M8.7, ADR-0074) + +Dev and stage only; the tools refuse World0. From a machine with kubectl access +that is not the hypervisor host: + +```bash +# 1. Sync the environment's application once so the egress policy admits the +# backup component and the store overlay pins a postgres-tools image that +# contains pg_dump (any image built after ADR-0074). +# 2. Back up to a local directory (the destination until one is chosen): +tools/backup/cluster-backup.sh dev --to /path/to/backups +# 3. Restore-test it on independent storage and rebuild it as isolated world 900 +# (WSL: the pinned PostgreSQL build and the Linux PostgreSQL build of the tools). +PATH="$PWD/.cache/postgres/install/bin:$PATH" \ +LD_LIBRARY_PATH="$PWD/.cache/postgres/install/lib" \ +EVO_WORLD_CLONE_BIN="$PWD/build/postgres-linux/evo-world-clone" \ +EVO_SERVER_BIN="$PWD/build/postgres-linux/evo-server" \ + tools/backup/verify-backup.sh /path/to/backups/dev-