diff --git a/CMakeLists.txt b/CMakeLists.txt index 35c4d3a..c5d326d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -127,7 +127,7 @@ if(EVO_BUILD_POSTGRES) tests/region_checkpoint_store_tests.cpp tests/region_host_tests.cpp tests/region_server_process_tests.cpp tests/region_recovery_tests.cpp tests/region_ops_tests.cpp tests/region_clone_tests.cpp - tests/region_migration_tests.cpp) + tests/region_migration_tests.cpp tests/region_store_tests.cpp) target_link_libraries(evo-postgres-tests PRIVATE evo::region_host Threads::Threads) add_dependencies(evo-postgres-tests evo-server evo-world-clone) # Spawned by process tests. evo_owned_target(evo-postgres-tests) diff --git a/apps/evo-server/main.cpp b/apps/evo-server/main.cpp index a62993d..b7800e7 100644 --- a/apps/evo-server/main.cpp +++ b/apps/evo-server/main.cpp @@ -11,6 +11,10 @@ // stats max_queued= max_tick_debt= commits= store_failures= // Exit: 0 drained, 2 usage, 3 drain deadline passed, 4 quarantined. // +// --objects-reserve-mib (M8.3, default 1024) refuses new objects that would leave +// less free space on the object volume: node-local volumes do not enforce their +// requested size. A refused upload is a store failure; nothing is acknowledged. +// // --ops-listen (M7.8) serves GET /livez (the loop is alive; says // nothing about authority), /readyz (200 only with current authority) and /metrics // (Prometheus text). It prints "ops listening " once bound. @@ -54,6 +58,7 @@ struct Options { std::string crash_point; std::uint64_t crash_ordinal{}; std::string ops_listen; + std::uint64_t objects_reserve_mib{1024}; }; // Kills this process without any cleanup: the real SIGKILL, at a chosen boundary. @@ -148,6 +153,8 @@ std::optional parse(int argc, char** argv) { o.host.lease = {}; else if (flag == "--ops-listen") o.ops_listen = value; + else if (flag == "--objects-reserve-mib" && n && *n <= 1048576) + o.objects_reserve_mib = *n; else if (flag == "--crash-at") { const auto colon = value.find(':'); const auto ordinal = @@ -263,8 +270,9 @@ template int serve(Options& o) { }; } std::uint64_t ack_steps = 0; - auto objects = persistence::FilesystemObjectStore::open(o.objects, o.host.key, - persistence::max_lineage_payload_bytes); + auto objects = persistence::FilesystemObjectStore::open( + o.objects, o.host.key, persistence::max_lineage_payload_bytes, nullptr, + o.objects_reserve_mib * 1024 * 1024); if (!objects) { std::puts("error objects"); return 2; diff --git a/apps/evo-world-clone/main.cpp b/apps/evo-world-clone/main.cpp index ffa2a43..db0b3e4 100644 --- a/apps/evo-world-clone/main.cpp +++ b/apps/evo-world-clone/main.cpp @@ -7,6 +7,8 @@ // [--audit-max N] [--crash-at before-transaction] // evo-world-clone audit --world W --region R --pg-* ... --objects DIR --retained HEX... // [--max-commits N] +// evo-world-clone check-store --world W --region R --pg-* ... --objects DIR +// [--reserve-mib N] // // --*pg-* is host, hostaddr, port, database, user, passfile, ca. The source // connection needs only the runtime read functions and is switched to read-only @@ -18,8 +20,20 @@ // // Output: cloned source=W/R sequence=S tick=T target=W2/R2 outcomes=N digest=HEX // audited commits=N tick=T outcomes=N profile=P clone=0|1 -// Exit: 0 done, 2 usage, 3 source/history failure, 4 target refused, 5 target unknown. +// check-store (M8.3) qualifies an environment's stores from inside the cluster with +// the runtime identity. Objects: the volume must already exist (a missing or +// unmounted volume is refused), a fixed probe object is created once and verified by +// digest on every later run (so a restarted or rescheduled pod proves retention), +// and free space is reported against the reserve. Database: the adapter enforces +// server 18.6 and verify-full TLS off-host; the runtime login must reach the fenced +// functions but be refused table access. +// Output: objects available_mib=A reserve_mib=R / probe created|verified digest=HEX +// database version=V user=U ssl=S / database tables=denied functions=reachable +// +// Exit: 0 done, 2 usage, 3 source/history/database failure, 4 target/objects refused, +// 5 target unknown. #include "evo/persistence/region_clone.hpp" +#include "evo/persistence/sha256.hpp" #include #include @@ -49,7 +63,7 @@ struct Options { Side source, target; std::vector retained; std::optional commit; - std::uint64_t audit_max{}, max_commits{1u << 20U}; + std::uint64_t audit_max{}, max_commits{1u << 20U}, reserve_mib{1024}; bool crash{}; }; @@ -112,7 +126,7 @@ std::optional parse(int argc, char** argv) { return std::nullopt; Options o; o.command = argv[1]; - if (o.command != "clone" && o.command != "audit") + if (o.command != "clone" && o.command != "audit" && o.command != "check-store") return std::nullopt; const bool clone = o.command == "clone"; for (int i = 2; i < argc; ++i) { @@ -150,14 +164,18 @@ std::optional parse(int argc, char** argv) { o.audit_max = *n; else if (clone && flag == "--crash-at" && value == "before-transaction") o.crash = true; - else if (!clone && flag == "--max-commits" && n && *n > 0) + else if (o.command == "audit" && flag == "--max-commits" && n && *n > 0) o.max_commits = *n; + else if (o.command == "check-store" && flag == "--reserve-mib" && n && *n <= 1048576) + o.reserve_mib = *n; else return std::nullopt; } if (clone && o.target.key.region == 0) o.target.key.region = o.source.key.region; - if (!complete(o.source) || o.retained.empty() || (clone && !complete(o.target))) + const bool needs_builds = o.command != "check-store"; + if (!complete(o.source) || (needs_builds && o.retained.empty()) || + (clone && !complete(o.target))) return std::nullopt; return o; } @@ -236,6 +254,88 @@ std::unique_ptr connect(const Side& s) { return std::move(*db); } +// A fixed 64 KiB probe: the same bytes on every run, so its digest names it. +std::vector probe_bytes() { + std::vector out(65536); + const std::string_view marker = "aevolution-store-probe-v1"; + for (std::size_t i = 0; i < out.size(); ++i) + out[i] = i < marker.size() ? static_cast(marker[i]) + : static_cast((i * 131U) ^ (i >> 7U)); + return out; +} + +int check_store(const Options& o) { + auto store = persistence::FilesystemObjectStore::open(o.source.objects, o.source.key, + persistence::max_lineage_payload_bytes, + nullptr, o.reserve_mib * 1024 * 1024); + if (!store) { + std::printf("objects refused %s\n", store.error() == persistence::ObjectError::invalid + ? "missing-or-not-a-directory" + : "io"); + return 4; + } + const auto available = (*store)->available_bytes(); + std::printf( + "objects available_mib=%llu reserve_mib=%llu\n", + static_cast(available.value_or(0) / (std::uint64_t{1024} * 1024)), + static_cast(o.reserve_mib)); + const auto bytes = probe_bytes(); + const auto digest = persistence::sha256(bytes); + if (!digest) + return 4; + const persistence::VerifiedObject probe{*digest, bytes.size()}; + const char* state = "verified"; + if (auto existing = (*store)->get(probe); !existing) { + if (existing.error() != persistence::ObjectError::missing) { + std::printf("probe %s\n", existing.error() == persistence::ObjectError::corrupt + ? "corrupt" + : "unreadable"); + return 4; + } + auto stored = (*store)->put(bytes); + if (!stored) { + std::printf("probe refused %s\n", + stored.error() == persistence::ObjectError::full ? "full" : "io"); + return 4; + } + auto back = (*store)->get(probe); + if (!back || *back != bytes) + return 4; + state = "created"; + } + std::printf("probe %s digest=%s\n", state, text(*digest).c_str()); + std::fflush(stdout); + + auto db = persistence::Postgres::connect(o.source.pg, timeout); + if (!db) { + const auto kind = db.error().kind; + std::printf("database refused %s\n", kind == persistence::PgError::version ? "version" + : kind == persistence::PgError::config ? "config" + : kind == persistence::PgError::connect ? "connect" + : "other"); + return 3; + } + auto about = (*db)->query( + "SELECT pg_catalog.current_setting('server_version_num'), current_user::text, " + "coalesce((SELECT s.ssl::text || '/' || coalesce(s.version, '') FROM " + "pg_catalog.pg_stat_ssl s WHERE s.pid = pg_catalog.pg_backend_pid()), 'unknown')", + {}, timeout); + if (!about || about->size() != 1 || (*about)[0].size() != 3) + return 3; + const auto& row = (*about)[0]; + std::printf("database version=%s user=%s ssl=%s\n", row[0].value_or("?").c_str(), + row[1].value_or("?").c_str(), row[2].value_or("?").c_str()); + auto tables = (*db)->query("SELECT 1 FROM evolution.region_heads LIMIT 1", {}, timeout); + const bool denied = !tables && std::string_view(tables.error().sqlstate.data()) == "42501"; + auto head = persistence::load_region_head(**db, o.source.key, timeout); + const bool reachable = + head || head.error().kind == persistence::LeaseStoreError::not_provisioned; + std::printf("database tables=%s functions=%s head=%s\n", denied ? "denied" : "READABLE", + reachable ? "reachable" : "unreachable", + head ? std::to_string(head->sequence).c_str() : "none"); + return denied && reachable ? 0 : 3; +} + int audit(const Options& o) { auto db = connect(o.source); auto objects = open_objects(o.source); @@ -329,5 +429,7 @@ int main(int argc, char** argv) { stderr); return 2; } + if (options->command == "check-store") + return check_store(*options); return options->command == "clone" ? clone(*options) : audit(*options); } diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 86ee298..efbf777 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -100,6 +100,8 @@ ENTRYPOINT ["bash", "/opt/evo/tools/migrate-postgres.sh"] FROM ${UBUNTU_IMAGE} AS server-runtime RUN apt-get update && apt-get install -y --no-install-recommends libstdc++6 libgcc-s1 libssl3t64 zlib1g ca-certificates && rm -rf /var/lib/apt/lists/* && groupadd --gid 10001 evo && useradd --uid 10001 --gid 10001 --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin evo COPY --from=postgres-build /source/build/postgres/evo-server /usr/local/bin/evo-server +# M8.3 operations tool: check-store qualification, clone and history audit. +COPY --from=postgres-build /source/build/postgres/evo-world-clone /usr/local/bin/evo-world-clone COPY --from=postgres-build /source/.cache/postgres/install/lib/libpq.so* /opt/postgres/lib/ COPY --from=postgres-build /source/.cache/postgres/postgresql-18.6/COPYRIGHT /usr/share/doc/evo-postgres/COPYRIGHT ENV LD_LIBRARY_PATH=/opt/postgres/lib diff --git a/deploy/kubernetes/platform/dev/kustomization.yaml b/deploy/kubernetes/platform/dev/kustomization.yaml index 0bf27c0..ef38224 100644 --- a/deploy/kubernetes/platform/dev/kustomization.yaml +++ b/deploy/kubernetes/platform/dev/kustomization.yaml @@ -4,6 +4,7 @@ namespace: evolution-dev resources: - namespace.yaml - ../base + - objects-pvc.yaml labels: - pairs: aevolution.io/environment: dev diff --git a/deploy/kubernetes/platform/dev/objects-pvc.yaml b/deploy/kubernetes/platform/dev/objects-pvc.yaml new file mode 100644 index 0000000..3321229 --- /dev/null +++ b/deploy/kubernetes/platform/dev/objects-pvc.yaml @@ -0,0 +1,15 @@ +# M8.3 region object store volume (ADR-0070): durable data, so it belongs to the +# platform layer (applied by hand), never to an Argo Application. The volume is +# created on the node of its first consumer; every consumer selects +# aevolution.io/stores=true, so label exactly one worker first (ENVIRONMENTS.md). +# Retain: deleting this claim leaves the volume and its data Released, not deleted. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: aevolution-objects +spec: + accessModes: [ReadWriteOnce] + storageClassName: aevolution-local-retain + resources: + requests: + storage: 10Gi diff --git a/deploy/kubernetes/platform/stage/kustomization.yaml b/deploy/kubernetes/platform/stage/kustomization.yaml index 7bae733..e062092 100644 --- a/deploy/kubernetes/platform/stage/kustomization.yaml +++ b/deploy/kubernetes/platform/stage/kustomization.yaml @@ -4,6 +4,7 @@ namespace: evolution-stage resources: - namespace.yaml - ../base + - objects-pvc.yaml labels: - pairs: aevolution.io/environment: stage diff --git a/deploy/kubernetes/platform/stage/objects-pvc.yaml b/deploy/kubernetes/platform/stage/objects-pvc.yaml new file mode 100644 index 0000000..3321229 --- /dev/null +++ b/deploy/kubernetes/platform/stage/objects-pvc.yaml @@ -0,0 +1,15 @@ +# M8.3 region object store volume (ADR-0070): durable data, so it belongs to the +# platform layer (applied by hand), never to an Argo Application. The volume is +# created on the node of its first consumer; every consumer selects +# aevolution.io/stores=true, so label exactly one worker first (ENVIRONMENTS.md). +# Retain: deleting this claim leaves the volume and its data Released, not deleted. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: aevolution-objects +spec: + accessModes: [ReadWriteOnce] + storageClassName: aevolution-local-retain + resources: + requests: + storage: 10Gi diff --git a/deploy/kubernetes/platform/storage/storageclass.yaml b/deploy/kubernetes/platform/storage/storageclass.yaml new file mode 100644 index 0000000..600f553 --- /dev/null +++ b/deploy/kubernetes/platform/storage/storageclass.yaml @@ -0,0 +1,17 @@ +# M8.3 (ADR-0070). Cluster-scoped; a cluster administrator applies it by hand: +# kubectl apply -f deploy/kubernetes/platform/storage/storageclass.yaml +# Same provisioner as the cluster's only class (rancher.io/local-path: a directory on +# the node that first runs the consumer), but deleting a claim keeps the volume and +# its data (Released) instead of deleting it. Volumes are node-local: no replication, +# no failover, and the requested size is not enforced (the object store's free-space +# reserve and M8.5 alerts cover that). Never the default class. +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: aevolution-local-retain + labels: + app.kubernetes.io/part-of: aevolution +provisioner: rancher.io/local-path +reclaimPolicy: Retain +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: false diff --git a/deploy/kubernetes/stores/stores.yaml.example b/deploy/kubernetes/stores/stores.yaml.example new file mode 100644 index 0000000..8e8eb4c --- /dev/null +++ b/deploy/kubernetes/stores/stores.yaml.example @@ -0,0 +1,174 @@ +# M8.3 store Jobs for ONE disposable environment (dev or stage). Not an Argo path. +# Copy, then replace every REPLACE_* marker (the markers make this file unappliable +# as committed), and apply by hand: +# REPLACE_NAMESPACE evolution-dev | evolution-stage (never evolution-world0) +# REPLACE_DB_CIDR the database VM's address as a /32 (PGHOSTADDR) +# REPLACE_DB_PORT its port (PGPORT) +# REPLACE_RUNTIME_ROLE that environment's NOLOGIN runtime group role +# REPLACE_TOOLS_DIGEST aevolution-postgres-tools image digest (sha256:...) +# REPLACE_SERVER_DIGEST aevolution server-runtime image digest (sha256:...) +# Secret contract (docs/infrastructure/ENVIRONMENTS.md, never in Git): +# aevolution-db: PGHOST, PGHOSTADDR, PGPORT, PGDATABASE (only these +# keys are read; PGPASSWORD/PGUSER are never mounted) +# evolution-postgres-migrator: pg_service.conf, pgpass, ca.crt (migrate-postgres.sh) +# evolution-postgres-runtime: user, pgpass, ca.crt +# Order: apply the NetworkPolicy, run migrate, then check-store (twice: the second +# run must report "probe verified", proving the volume survived a new pod). +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: aevolution-database-egress + namespace: REPLACE_NAMESPACE + labels: + app.kubernetes.io/part-of: aevolution +spec: + podSelector: + matchExpressions: + - key: app.kubernetes.io/component + operator: In + values: [migrate, store-check, region] + policyTypes: [Egress] + egress: + - to: + - ipBlock: + cidr: REPLACE_DB_CIDR + ports: + - {protocol: TCP, port: REPLACE_DB_PORT} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: aevolution-db-migrate + namespace: REPLACE_NAMESPACE + labels: + app.kubernetes.io/part-of: aevolution + app.kubernetes.io/component: migrate +spec: + backoffLimit: 0 + activeDeadlineSeconds: 180 + ttlSecondsAfterFinished: 86400 + template: + metadata: + labels: + app.kubernetes.io/part-of: aevolution + app.kubernetes.io/component: migrate + spec: + serviceAccountName: aevolution-migrator + automountServiceAccountToken: false + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: {type: RuntimeDefault} + containers: + - name: migrate + image: registry.aeternalisgames.com/aeternalis/aevolution-postgres-tools@REPLACE_TOOLS_DIGEST + env: + - {name: PGSERVICE, value: evolution-migrator} + - {name: PGSERVICEFILE, value: /run/postgres/pg_service.conf} + - {name: EVO_PGPASS_SOURCE, value: /run/postgres/pgpass} + - {name: EVO_PG_RUNTIME_ROLE, value: REPLACE_RUNTIME_ROLE} + resources: + requests: {cpu: 100m, memory: 64Mi} + limits: {memory: 128Mi} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + volumeMounts: + - {name: connection, mountPath: /run/postgres, readOnly: true} + - {name: scratch, mountPath: /tmp} + volumes: + - name: connection + secret: + secretName: evolution-postgres-migrator + defaultMode: 0440 + items: + - {key: pg_service.conf, path: pg_service.conf} + - {key: pgpass, path: pgpass} + - {key: ca.crt, path: ca.crt} + - name: scratch + emptyDir: {medium: Memory, sizeLimit: 8Mi} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: aevolution-store-check + namespace: REPLACE_NAMESPACE + labels: + app.kubernetes.io/part-of: aevolution + app.kubernetes.io/component: store-check +spec: + backoffLimit: 0 + activeDeadlineSeconds: 120 + ttlSecondsAfterFinished: 86400 + template: + metadata: + labels: + app.kubernetes.io/part-of: aevolution + app.kubernetes.io/component: store-check + spec: + serviceAccountName: aevolution-region + automountServiceAccountToken: false + restartPolicy: Never + nodeSelector: + aevolution.io/stores: "true" + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: {type: RuntimeDefault} + containers: + - name: check-store + image: registry.aeternalisgames.com/aeternalis/aevolution-server@REPLACE_SERVER_DIGEST + # libpq ignores a group-readable password file, and Secret volumes are + # group-readable for a non-root fsGroup, so copy it to private memory first. + # w1/r1 is a reserved probe namespace, never a real world. + command: ["/bin/bash", "-c"] + args: + - | + set -eu + umask 077 + cp /run/postgres/pgpass /tmp/pgpass + exec /usr/local/bin/evo-world-clone check-store --world 1 --region 1 \ + --objects /objects --pg-host "$DB_HOST" --pg-hostaddr "$DB_HOSTADDR" \ + --pg-port "$DB_PORT" --pg-database "$DB_NAME" --pg-user "$DB_USER" \ + --pg-passfile /tmp/pgpass --pg-ca /run/postgres/ca.crt + env: + - name: DB_HOST + valueFrom: {secretKeyRef: {name: aevolution-db, key: PGHOST}} + - name: DB_HOSTADDR + valueFrom: {secretKeyRef: {name: aevolution-db, key: PGHOSTADDR}} + - name: DB_PORT + valueFrom: {secretKeyRef: {name: aevolution-db, key: PGPORT}} + - name: DB_NAME + valueFrom: {secretKeyRef: {name: aevolution-db, key: PGDATABASE}} + - name: DB_USER + valueFrom: {secretKeyRef: {name: evolution-postgres-runtime, key: user}} + resources: + requests: {cpu: 50m, memory: 64Mi} + limits: {memory: 256Mi} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + volumeMounts: + - {name: objects, mountPath: /objects} + - {name: connection, mountPath: /run/postgres, readOnly: true} + - {name: scratch, mountPath: /tmp} + volumes: + - name: objects + persistentVolumeClaim: + claimName: aevolution-objects + - name: scratch + emptyDir: {medium: Memory, sizeLimit: 1Mi} + - name: connection + secret: + secretName: evolution-postgres-runtime + defaultMode: 0440 + items: + - {key: pgpass, path: pgpass} + - {key: ca.crt, path: ca.crt} diff --git a/docs/BACKLOG.md b/docs/BACKLOG.md index 080127b..1f23a1c 100644 --- a/docs/BACKLOG.md +++ b/docs/BACKLOG.md @@ -7,6 +7,7 @@ names it. Remove an item only when it is done, and record where it was done. | Item | Raised | Why deferred | Needed before | |---|---|---|---| | **World0 break-glass membership.** Decide who may act as `aevolution:world0-break-glass`, and how that group is granted: for example a short-lived client certificate with that organization, or an OIDC group. Decide also how its use is approved and recorded. The admission policy already enforces the group (ADR-0069). | M8.2, 2026-09-29 | Maintainer: figure out later | Any intentional deletion of World0 data or identity; M8.8 World0 launch | +| **Backup destination for dev/stage stores.** Decide where backups go: the AEvolution PostgreSQL VM (WAL archive or base backups) and each environment's `aevolution-objects` volume. It should be off the host for any claim beyond disposable environments. | M8.3, 2026-09-29 | Needs the maintainer's choice of target | M8.7 backup and restore; any World0 data | | **Storage after the RAID/disk rework.** Re-run `deploy/kubernetes/inventory/disk-probe-local-path.yaml` (or a new class), and update INVENTORY.md. | M8.1, 2026-09-29 | Maintainer is reorganizing disks | M8.3's final storage decision | ## Accepted risks (maintainer decisions) diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 6e2bffe..0ad47ad 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -616,3 +616,11 @@ M8.2 designed and checked offline (2026-09-29), under ADR-0069: - an offline policy check in CI, and a live isolation check. The live checks run once the maintainer applies the platform layer. Next: M8.3, durable stores in dev and stage. + +M8.3 implemented and locally qualified (2026-09-29), under ADR-0070: +- each environment's database lives on the external PostgreSQL VM; +- object volumes are node-local and use the Retain class; +- a free-space reserve with a distinct full-volume refusal; +- `evo-world-clone check-store` for in-cluster qualification. +The in-cluster evidence and the backup destination wait on the maintainer +(docs/BACKLOG.md). diff --git a/docs/STATUS.md b/docs/STATUS.md index 0604d3d..1c4e050 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1015,3 +1015,24 @@ M8.2 applied and verified live (2026-09-29): 175 PASS and 2 SKIP. The only 3 FAI were a script defect in the project check, now fixed. The maintainer accepted leaving Argo's `default` project unrestricted. Break-glass membership is in the new docs/BACKLOG.md. Next: M8.3, durable stores in dev and stage. + +## M8.3 dev/stage stores (2026-09-29) + +ADR-0070 sets up the dev and stage stores: +- **Database:** each environment gets its own database and logins on the external + PostgreSQL VM. +- **Object store:** the filesystem store stays, on a node-local PVC in the new + Retain class `aevolution-local-retain`, pinned to one labelled worker. There is + no object server. +- **Capacity:** node-local volumes ignore their requested size, so the object + store now enforces a free-space reserve (`--objects-reserve-mib`, default 1024). + A full volume refuses new objects with a distinct error, and nothing is + acknowledged. +- **Qualification:** `evo-world-clone check-store`, now in the server image, + checks from inside the cluster: volume presence and retention, the reserve, + database version and TLS, and the runtime privilege boundary. + +Not yet done: +- the in-cluster run (it needs Secrets with the documented key names and + published image digests); +- choosing a backup destination (docs/BACKLOG.md). diff --git a/docs/adr/0070-dev-stage-stores.md b/docs/adr/0070-dev-stage-stores.md new file mode 100644 index 0000000..7a05465 --- /dev/null +++ b/docs/adr/0070-dev-stage-stores.md @@ -0,0 +1,102 @@ +# ADR-0070: Durable stores for dev and stage + +## Status +Accepted for M8.3, 2026-09-29. It builds on ADR-0059 (PostgreSQL), ADR-0061 +(object store), ADR-0069 (environments) and the M8.1 inventory. Storage decisions +are provisional until the planned RAID and disk rework (docs/BACKLOG.md). + +## Context +Inventory facts that drive this decision: +- PostgreSQL runs on a separate VM outside Kubernetes (maintainer). The adapter + already requires exactly 18.6 and `verify-full` TLS off-host, and the migration + audits the runtime role. +- The cluster's only StorageClass, `local-path`, is node-local, reclaims with + Delete and does not enforce requested sizes. +- Object publication on the workers measured 29–36 ms mean (73 ms max). Worker-3's + root filesystem is 87 % full. +- MinIO in the cluster is unpinned (`latest`) and belongs to another project. + +The roadmap asks for: +- a supported PostgreSQL and a maintained object store, with pinned versions; +- PV retention and a backup destination; +- evidence for pod restart, missing-volume refusal, object semantics, credential + rotation and full disk. + +The named risk is that a local PV binds failover to a node that has been lost. + +## Decision +**Database.** Use the maintainer's PostgreSQL VM. Each environment gets its own +database, and its own migrator login and NOLOGIN runtime group with a member login +(ADR-0060). The version pin (18.6), TLS verification and privilege audit are +enforced by code at connect and migrate time; they are not assumed. Migrations run +as a hand-applied Job using the pinned `postgres-tools` image. + +**Object store.** Keep the ADR-0061 filesystem store. There is **no object +server**: MinIO is ruled out as a new default (INFRASTRUCTURE_ARCHITECTURE), and an +S3 adapter would be a new dependency. Instead: +- The store lives on one PVC per environment, `aevolution-objects`, in the new + class `aevolution-local-retain` (the same provisioner, with **Retain** and + WaitForFirstConsumer; never the default class). +- The PVC is platform-owned (hand-applied, never Argo). +- Every consumer selects the node label `aevolution.io/stores=true`, which is put + on exactly one worker. worker-1 or worker-2 is recommended; never worker-3 or + the control plane. + +**Capacity guard.** `local-path` ignores the requested size, so +`FilesystemObjectStore` now takes a free-space reserve. `evo-server` exposes it as +`--objects-reserve-mib`, default 1024. +- A new object that would leave less free space is refused with a distinct + `ObjectError::full`, before anything is written. +- Identical objects that already exist stay reusable. +- The host treats a refused upload as a retryable store failure, so nothing is + published or acknowledged, and a drain that cannot publish ends without + committing anything (exit 3). +- A genuinely failing write (ENOSPC or EFBIG) is `io`, and leaves no temporary + file or object behind. + +**Qualification tool.** `evo-world-clone check-store` runs in the cluster as a Job +with the runtime identity. It: +- refuses a missing or unmounted object root; +- creates a fixed 64 KiB probe object once, then verifies it by digest on every + later run (proof of retention across pods); +- reports free space against the reserve; +- reports the server version, user and TLS; +- requires that the runtime login reaches the fenced functions but is denied + table access. + +The server image now includes the tool. + +**Secret contract.** +- `aevolution-db` supplies only `PGHOST`, `PGHOSTADDR`, `PGPORT` and `PGDATABASE`. +- `evolution-postgres-runtime` supplies `user`, `pgpass` and `ca.crt`. +- `evolution-postgres-migrator` supplies `pg_service.conf`, `pgpass` and `ca.crt`. + +Passwords are only ever mounted as files, and copied into private memory +(mode 0600) before use, because libpq ignores group-readable password files. The +policy check forbids `PGPASSWORD`/`PGUSER` references and whole-Secret `envFrom`. + +**Retention and backup.** +- Deleting the claim leaves the PV Released with its data. Recovery is an explicit + rebind by an administrator, verified by `check-store` ("probe verified"). +- The backup destination for the database VM and the object PVC is **not decided + yet**. It is a maintainer decision, recorded in docs/BACKLOG.md and required + before M8.7. Dev and stage run without a verified backup, which is acceptable + only because they are disposable. World0 remains blocked. + +## Consequences +- A lost worker takes its object volume with it. The dev and stage regions on that + worker are then unavailable (the server refuses a missing volume and quarantines + missing objects); they are never silently re-initialized. This is the named risk, + accepted for dev and stage only. World0 needs replicated or off-node storage + (M8.7/M8.8). +- The reserve and the kubelet's nodefs eviction threshold interact. Set the reserve + above the node's remaining headroom target, and alert on free space (M8.5). +- Re-probe and re-decide after the disk rework. + +## Alternatives rejected +- **MinIO or another object server.** It adds an unpinned or new dependency, a + second durability layer to back up, and network latency, for no gain on one + node. +- **Leaving `local-path`/Delete for durable data.** Deleting the claim or an Argo + prune would erase the store. +- **Trusting PVC sizes.** `local-path` does not enforce them. diff --git a/docs/adr/README.md b/docs/adr/README.md index 50dcfb2..b6ed52d 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -80,3 +80,4 @@ design. No unadopted candidate is a link dependency of the proof. - [ADR-0067: Region clones and retained-history audit](0067-region-clones.md) - accepted for M7.9a re-identified clone genesis naming its source cut, isolated clone databases and full-chain retention audit. - [ADR-0068: Upgrade barrier and durability qualification](0068-upgrade-barrier.md) - accepted for M7.9b build-bound authority, retirement barrier fencing old writers, explicit refusal of rule/format changes and the measured recovery fixture. - [ADR-0069: Environment isolation and World0 protection](0069-environment-isolation.md) - accepted for M8.2 per-environment guardrails, native World0 admission protection, scoped Argo projects and verification. +- [ADR-0070: Durable stores for dev and stage](0070-dev-stage-stores.md) - accepted for M8.3 external PostgreSQL per environment, Retain-class node-local object volumes, free-space reserve and in-cluster store qualification. diff --git a/docs/engineering/VALIDATION.md b/docs/engineering/VALIDATION.md index f642775..7ae61be 100644 --- a/docs/engineering/VALIDATION.md +++ b/docs/engineering/VALIDATION.md @@ -2689,3 +2689,42 @@ admission policy and the stage and World0 AppProjects, then ran `[]`. The corrected script has not been re-run on the cluster. - **WARN:** the `default` AppProject allows every namespace. The maintainer accepted this (docs/BACKLOG.md). + +## M8.3 dev/stage stores - 2026-09-29 + +- WSL Linux `checks` set: CTest 59/59 in each configuration. The format check + passed. +- PostgreSQL suites: Release and ASan/UBSan passed with the new + `region_store_tests`. Analysis passed after one clang-tidy fix (an int + multiplication widened in the MiB report). +- Store tests: + - An unreachable reserve refuses new objects with `full`, leaving no files. An + identical existing object is still reused. + - A lineage batch through a full store fails as `upload`/`full`, and the head is + unchanged. + - RLIMIT_FSIZE makes a real write fail with `io`, leaving no object or + temporary. + - `check-store`: + - its first run creates the probe and its second verifies it; + - a missing root exits 4 without creating it; + - a full volume exits 4; + - an owner login exits 3 (tables READABLE); + - the runtime login reports `version=180006 ... ssl=false` (local socket), and + `tables=denied functions=reachable`. + - An `evo-server` whose reserve cannot be met simulated but acknowledged + nothing. Its drain exited 3, and the head was unchanged. +- Mutations (restored and rebuilt), each caught: + - removing the reserve check; + - `check-store` always succeeding. +- `tools/ci/deploy-policy-check.py`: 41 objects, 0 failures. Four new-rule + mutations were caught: + - a Delete class; + - a platform claim on `local-path`; + - a `PGPASSWORD` key reference; + - a whole-Secret `envFrom`. +- Kustomize renders the dev and stage overlays with the `aevolution-objects` + claim; World0 has none. +- The server image now contains `evo-world-clone`, and its smoke script checks the + tool's usage exit. It has not been built locally, because Docker is not running. +- Not executed: anything in the cluster, the store Jobs, retention, rebind and + rotation evidence, and the image build. diff --git a/docs/infrastructure/ENVIRONMENTS.md b/docs/infrastructure/ENVIRONMENTS.md index 14d247c..c2f12c3 100644 --- a/docs/infrastructure/ENVIRONMENTS.md +++ b/docs/infrastructure/ENVIRONMENTS.md @@ -21,12 +21,12 @@ manual, and none is performed by Argo CD. Each namespace gets its own Secrets, with environment-specific values: - `harbor-pull`: a pull-only Harbor robot (docker-registry type). - `evolution-postgres-migrator`: the migrator/owner login for that environment's - database. + database. Keys: `pg_service.conf`, `pgpass`, `ca.crt`. - `evolution-postgres-runtime`: the runtime login, a member of that environment's - NOLOGIN runtime group (ADR-0060). -- `aevolution-db`: connection settings (host, address, port, database, TLS mode - and root certificate). Mount it as files. The adapter reads a password file, - never `PGPASSWORD`. + NOLOGIN runtime group (ADR-0060). Keys: `user`, `pgpass`, `ca.crt`. +- `aevolution-db`: connection settings. Only `PGHOST`, `PGHOSTADDR` (numeric), + `PGPORT` and `PGDATABASE` are ever read, as individual variables. `PGPASSWORD` + and `PGUSER` are never mounted; the policy check forbids it. A World0 credential must never exist in the dev or stage namespaces, and no dev or stage credential may reach the World0 database. Rotate by replacing the Secret and @@ -52,6 +52,48 @@ bash tools/ops/verify-isolation.sh Create the Secrets listed above in each namespace before any workload syncs. +## Stores (M8.3, ADR-0070) + +Run this once per disposable environment (`dev`, then `stage`). + +**1. Label the store node, and apply the storage class and claim.** +```sh +# Label exactly one worker (worker-1 or worker-2; never worker-3 or the control plane). +kubectl label node rke2-worker-1 aevolution.io/stores=true +kubectl apply -f deploy/kubernetes/platform/storage/storageclass.yaml +kubectl apply -k deploy/kubernetes/platform/dev # adds the aevolution-objects claim +``` + +**2. Prepare the store Jobs.** Copy `deploy/kubernetes/stores/stores.yaml.example`, +replace its `REPLACE_*` markers (namespace, database address and port, runtime +role, image digests), and create the Secrets above. + +**3. Run and qualify.** +```sh +kubectl apply -f stores-dev.yaml # policy + both Jobs +kubectl logs -n evolution-dev job/aevolution-db-migrate # schema and audit +kubectl logs -n evolution-dev job/aevolution-store-check # expect exit 0 and "probe created" +``` + +Evidence to record in docs/engineering/VALIDATION.md: +- **Pod restart:** delete the check Job and apply it again. It must print + `probe verified`, with the same digest. +- **Missing volume:** run the check with the `objects` volume removed. It must exit + 4 with `objects refused missing-or-not-a-directory`. +- **Full volume:** add `--reserve-mib 1048576` and use a new probe region, such as + `--region 2`. It must exit 4 with `probe refused full`. +- **Retention:** `kubectl delete pvc aevolution-objects -n evolution-dev`. The PV + must become `Released`, not deleted. To rebind: + - clear the PV's `spec.claimRef`; + - recreate the claim with `volumeName` set to that PV; + - rerun the check, which must print `probe verified`. +- **Credential rotation:** + - Change the runtime login's password on the database VM, and run the check + before updating the Secret. It must exit 3 with `database refused connect`. + - Update the Secret's `pgpass` and rerun. It must exit 0. +- **Privilege boundary:** every run must print + `database tables=denied functions=reachable`. + ## Break-glass How membership is granted is not decided yet; see docs/BACKLOG.md. Until it is, diff --git a/include/evo/persistence/object_store.hpp b/include/evo/persistence/object_store.hpp index b3bc98b..f816a51 100644 --- a/include/evo/persistence/object_store.hpp +++ b/include/evo/persistence/object_store.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -19,6 +20,7 @@ enum class ObjectError : std::uint8_t { corrupt, // Length or digest mismatch, or not a regular file: quarantine. bounds, // Larger than the configured maximum; rejected before allocation. io, // Write/sync/link failure; the object is absent or unverified. + full, // Writing would leave less free space than the reserve; nothing written. }; // Test-only fault injection and accounting for directory fsyncs. When passed to // open() it must outlive the store; production callers pass nothing. @@ -33,17 +35,22 @@ struct VerifiedObject { }; class FilesystemObjectStore { public: - // trusted_root must be an existing real directory (not a symlink). Namespace - // directories w/r are created 0700; their parents are synced on - // every open, so a creation whose sync failed earlier is made durable now. + // trusted_root must be an existing real directory (not a symlink): a missing or + // unmounted volume is refused, never created. Namespace directories + // w/r are created 0700; their parents are synced on every open, so + // a creation whose sync failed earlier is made durable now. reserve_bytes (M8.3): + // put() refuses new objects that would leave less free space than this on the + // volume, because node-local volumes do not enforce their requested size. [[nodiscard]] static std::expected, ObjectError> open(const std::filesystem::path& trusted_root, RegionKey key, std::size_t max_object_bytes, - ObjectStoreFaults* faults = nullptr); + ObjectStoreFaults* faults = nullptr, std::uint64_t reserve_bytes = 0); FilesystemObjectStore(const FilesystemObjectStore&) = delete; FilesystemObjectStore& operator=(const FilesystemObjectStore&) = delete; FilesystemObjectStore(FilesystemObjectStore&&) = delete; FilesystemObjectStore& operator=(FilesystemObjectStore&&) = delete; ~FilesystemObjectStore(); + // Refuses with `full`, before writing anything, when the volume's available + // space minus the object is below the reserve. // Writes a private temporary file, fsyncs it, links it to its key without // replacing anything, then fsyncs the directory. Identical bytes already stored // (an earlier attempt whose directory sync failed, or a concurrent installer) @@ -57,15 +64,21 @@ class FilesystemObjectStore { [[nodiscard]] const std::filesystem::path& directory() const noexcept { return directory_; } + // Bytes available to this process on the volume; nullopt if unknown. + [[nodiscard]] std::optional available_bytes() const noexcept; + [[nodiscard]] std::uint64_t reserve_bytes() const noexcept { + return reserve_; + } private: FilesystemObjectStore(std::filesystem::path directory, int descriptor, std::size_t maximum, - ObjectStoreFaults* faults); + ObjectStoreFaults* faults, std::uint64_t reserve); [[nodiscard]] std::expected durable_existing(const VerifiedObject& object) const; std::filesystem::path directory_; int descriptor_; std::size_t maximum_; ObjectStoreFaults* faults_; + std::uint64_t reserve_; }; } // namespace evo::persistence diff --git a/src/persistence/object_store.cpp b/src/persistence/object_store.cpp index 7b7449f..6dda841 100644 --- a/src/persistence/object_store.cpp +++ b/src/persistence/object_store.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include namespace evo::persistence { @@ -128,9 +129,17 @@ read_verified(int directory, const VerifiedObject& object, bool sync_file = fals } // namespace FilesystemObjectStore::FilesystemObjectStore(std::filesystem::path directory, int descriptor, - std::size_t maximum, ObjectStoreFaults* faults) - : directory_(std::move(directory)), descriptor_(descriptor), maximum_(maximum), - faults_(faults) {} + std::size_t maximum, ObjectStoreFaults* faults, + std::uint64_t reserve) + : directory_(std::move(directory)), descriptor_(descriptor), maximum_(maximum), faults_(faults), + reserve_(reserve) {} + +std::optional FilesystemObjectStore::available_bytes() const noexcept { + struct statvfs info{}; + if (::fstatvfs(descriptor_, &info) != 0) + return std::nullopt; + return static_cast(info.f_bavail) * static_cast(info.f_frsize); +} FilesystemObjectStore::~FilesystemObjectStore() { static_cast(::close(descriptor_)); @@ -138,7 +147,8 @@ FilesystemObjectStore::~FilesystemObjectStore() { std::expected, ObjectError> FilesystemObjectStore::open(const std::filesystem::path& trusted_root, RegionKey key, - std::size_t max_object_bytes, ObjectStoreFaults* faults) { + std::size_t max_object_bytes, ObjectStoreFaults* faults, + std::uint64_t reserve_bytes) { if (key.world == 0 || key.region == 0 || max_object_bytes == 0 || max_object_bytes > maximum_object_limit || trusted_root.empty()) return std::unexpected{ObjectError::invalid}; @@ -154,8 +164,9 @@ FilesystemObjectStore::open(const std::filesystem::path& trusted_root, RegionKey Descriptor region_fd{subdirectory(world_fd.get(), region, faults)}; if (region_fd.get() < 0) return std::unexpected{ObjectError::io}; - return std::unique_ptr(new FilesystemObjectStore( - trusted_root / world / region, region_fd.release(), max_object_bytes, faults)); + return std::unique_ptr( + new FilesystemObjectStore(trusted_root / world / region, region_fd.release(), + max_object_bytes, faults, reserve_bytes)); } std::expected @@ -171,6 +182,12 @@ FilesystemObjectStore::put(std::span bytes) const { // Immutable: identical content is reused (durably); anything else is corrupt. if (auto found = durable_existing(object); found || found.error() != ObjectError::missing) return found; + // Only new bytes need space; checked before anything is written. + const auto available = available_bytes(); + if (!available) + return std::unexpected{ObjectError::io}; + if (*available < reserve_ || *available - reserve_ < bytes.size()) + return std::unexpected{ObjectError::full}; const auto name = key_name(*digest); const auto temporary = ".tmp-" + name + "-" + std::to_string(::getpid()) + "-" + diff --git a/tests/postgres_tests.cpp b/tests/postgres_tests.cpp index f94cd38..5afebb6 100644 --- a/tests/postgres_tests.cpp +++ b/tests/postgres_tests.cpp @@ -11,6 +11,7 @@ void run_region_recovery_tests(); void run_region_ops_tests(); void run_region_clone_tests(); void run_region_migration_tests(); +void run_region_store_tests(); namespace { using namespace evo::persistence; using namespace std::chrono_literals; @@ -91,6 +92,8 @@ int main() { std::puts("region ops integration passed"); run_region_clone_tests(); std::puts("region clone and history integration passed"); + run_region_store_tests(); + std::puts("region store qualification integration passed"); run_region_migration_tests(); std::puts("region migration, barrier and qualification integration passed"); return 0; diff --git a/tests/region_store_tests.cpp b/tests/region_store_tests.cpp new file mode 100644 index 0000000..6ff1699 --- /dev/null +++ b/tests/region_store_tests.cpp @@ -0,0 +1,194 @@ +// M8.3 store qualification (ADR-0070) against real PostgreSQL and real processes: +// the object volume's free-space reserve, a genuinely failing write, the +// check-store probe (retention across runs, missing-volume refusal, privilege +// boundary) and a server whose volume is full. Run by tools/ci/postgres-check.sh. +#include "evo/persistence/checkpoint_store.hpp" +#include "fixtures/server_process.hpp" +#include +#include +#include + +void run_region_store_tests(); + +namespace { +using namespace evo::test::process; +using namespace evo::persistence; +void check(bool value, std::source_location at = std::source_location::current()) { + if (!value) + throw std::runtime_error("region store test line " + std::to_string(at.line())); +} +#define CHECK(...) check(static_cast(__VA_ARGS__)) +constexpr std::size_t budget = 8000000; +constexpr std::uint64_t unreachable_reserve = std::uint64_t{1} << 60U; // 1 EiB. +std::unique_ptr store(const fs::path& root, RegionKey key, + std::uint64_t reserve = 0) { + auto made = FilesystemObjectStore::open(root, key, max_lineage_payload_bytes, nullptr, reserve); + CHECK(made); + return std::move(*made); +} +fs::path fresh(const std::string& name) { + const auto root = fs::path(env("PGHOST")) / name; + fs::remove_all(root); + fs::create_directory(root); + fs::permissions(root, fs::perms::owner_all, fs::perm_options::replace); + return root; +} +std::size_t entries(const fs::path& dir) { + std::size_t n = 0; + for ([[maybe_unused]] const auto& entry : fs::directory_iterator(dir)) + ++n; + return n; +} +std::vector bytes_of(std::size_t size, std::uint8_t seed) { + std::vector out(size); + for (std::size_t i = 0; i < size; ++i) + out[i] = static_cast(seed + (i * 7U)); + return out; +} + +// A full volume refuses new objects before writing; identical objects already +// stored stay usable; a batch against a full volume publishes nothing. +void full_volumes_publish_nothing() { + const RegionKey key{7, 81}; + const auto root = fresh("store-full"); + const auto stored = bytes_of(4096, 0x10); + auto roomy = store(root, key); + CHECK(roomy->available_bytes().value_or(0) > 0 && roomy->reserve_bytes() == 0); + CHECK(roomy->put(stored)); + auto full = store(root, key, unreachable_reserve); + CHECK(full->reserve_bytes() == unreachable_reserve); + const auto before = entries(full->directory()); + auto refused = full->put(bytes_of(4096, 0x20)); + CHECK(!refused && refused.error() == ObjectError::full); + CHECK(entries(full->directory()) == before); // No temporary, no object. + CHECK(full->put(stored)); // Reuse needs no space. + + // A lineage batch cannot publish through a full volume; the head stays. + const auto r = provision(82, 0xDE); + auto db = owner(); + const auto head = load_region_checkpoint(*db, r.key, 5s); + CHECK(head); + auto runtime = Postgres::connect( + {env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("EVO_PG_RUNTIME_USER"), {}, {}}, + 5s); + CHECK(runtime); + Sha256Digest build{}; + build.fill(0xB1); + auto lease = acquire_region_lease(**runtime, r.key, build, 20000ms, 5s); + CHECK(lease); + auto state = evo::test::lineage_region(); + RegionBatch batch{*state}; + for (int i = 0; i < 10; ++i) + CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1))); + batch.seal(); + Uuid commit{}; + commit.fill(0xDF); + auto blocked = store(r.root, r.key, unreachable_reserve); + auto published = + publish_lineage_batch(**runtime, *blocked, *lease, *head, batch, commit, build, budget, 5s); + CHECK(!published && published.error().kind == PublicationError::upload && + published.error().object == ObjectError::full); + CHECK(load_region_checkpoint(*db, r.key, 5s)->head == head->head); + CHECK(release_region_lease(**runtime, r.key, *lease, 5s)); +} + +// A write the filesystem really refuses (RLIMIT_FSIZE stands in for a full disk: +// the kernel rejects the write mid-object) is an io error that leaves no object +// and no temporary behind. +void failing_writes_leave_nothing() { + const auto root = fresh("store-efbig"); + auto objects = store(root, {7, 83}); + rlimit old{}; + CHECK(::getrlimit(RLIMIT_FSIZE, &old) == 0); + const auto previous = std::signal(SIGXFSZ, SIG_IGN); + rlimit limited = old; + limited.rlim_cur = 8192; + CHECK(::setrlimit(RLIMIT_FSIZE, &limited) == 0); + auto refused = objects->put(bytes_of(65536, 0x30)); + CHECK(::setrlimit(RLIMIT_FSIZE, &old) == 0); + std::signal(SIGXFSZ, previous); + CHECK(!refused && refused.error() == ObjectError::io); + CHECK(entries(objects->directory()) == 0); + CHECK(objects->put(bytes_of(65536, 0x30))); // The same write succeeds with room. +} + +std::vector check_store(const fs::path& objects, const std::string& user, + std::vector extra, int& code) { + std::vector args{env("EVO_WORLD_CLONE_BIN"), + "check-store", + "--world", + "7", + "--region", + "84", + "--pg-host", + env("PGHOST"), + "--pg-port", + env("PGPORT"), + "--pg-database", + env("PGDATABASE"), + "--pg-user", + user, + "--objects", + objects.string()}; + args.insert(args.end(), extra.begin(), extra.end()); + Child tool{args}; + code = tool.wait_exit(30s); + return tool.lines(); +} +bool has(const std::vector& lines, std::string_view prefix) { + for (const auto& l : lines) + if (l.starts_with(prefix)) + return true; + return false; +} + +// The in-cluster qualification probe, exercised locally. +void check_store_qualifies_the_stores() { + const auto root = fresh("store-check"); + const auto runtime = env("EVO_PG_RUNTIME_USER"); + int code = -1; + auto first = check_store(root, runtime, {}, code); + CHECK(code == 0 && has(first, "objects available_mib=") && has(first, "probe created digest=")); + CHECK(has(first, "database version=180006 user=" + runtime + " ssl=false")); + CHECK(has(first, "database tables=denied functions=reachable head=none")); + // A later run (a restarted or rescheduled pod) verifies the retained probe. + auto second = check_store(root, runtime, {}, code); + CHECK(code == 0 && has(second, "probe verified digest=")); + // A missing or unmounted volume is refused, never created. + auto missing = check_store(root / "not-mounted", runtime, {}, code); + CHECK(code == 4 && has(missing, "objects refused missing-or-not-a-directory")); + CHECK(!fs::exists(root / "not-mounted")); + // A full volume refuses the probe (fresh namespace, reserve beyond any disk). + const auto other = fresh("store-check-full"); + auto full = check_store(other, runtime, {"--reserve-mib", "1048576"}, code); + CHECK(code == 4 && has(full, "probe refused full")); + // An owner login can read tables: not an acceptable runtime identity. + auto owner_run = check_store(root, env("PGUSER"), {}, code); + CHECK(code == 3 && has(owner_run, "database tables=READABLE")); +} + +// A real server on a full volume keeps simulating but never acknowledges, and its +// bounded drain gives up without committing anything. +void full_volume_server_acknowledges_nothing() { + const auto r = provision(85, 0xE7); + const auto before = head_of(r.key); + Child server{server_args(r, {"--objects-reserve-mib", "1048576", "--drain-ms", "1500", + "--ops-listen", "127.0.0.1:0"})}; + CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); + server.send(input_lines(1, 15, false)); + server.pump(3000ms); + CHECK(!server.printed("ack ")); + server.signal(SIGTERM); + CHECK(server.wait_exit(20s) == 3); // Drain deadline passed with the batch unpublished. + CHECK(!server.printed("ack ")); + const auto after = head_of(r.key); + CHECK(after.tick == before.tick && after.sequence == before.sequence); +} +} // namespace + +void run_region_store_tests() { + full_volumes_publish_nothing(); + failing_writes_leave_nothing(); + check_store_qualifies_the_stores(); + full_volume_server_acknowledges_nothing(); +} diff --git a/tools/ci/deploy-policy-check.py b/tools/ci/deploy-policy-check.py index de3ba60..995cc1e 100644 --- a/tools/ci/deploy-policy-check.py +++ b/tools/ci/deploy-policy-check.py @@ -145,6 +145,31 @@ for path, doc in objects: check(destination != 'evolution-world0' or name == 'aevolution-world0', f'{path}: only aevolution-world0 may target evolution-world0') +# M8.3 storage (ADR-0070): durable volumes retain their data; database passwords are +# only ever mounted as files, never placed in the environment. +classes = {d['metadata']['name']: d for p, d in objects if d['kind'] == 'StorageClass'} +retain = classes.get('aevolution-local-retain') +check(retain is not None and retain.get('reclaimPolicy') == 'Retain' + and retain.get('volumeBindingMode') == 'WaitForFirstConsumer' + and 'storageclass.kubernetes.io/is-default-class' not in retain['metadata'].get('annotations', {}), + 'aevolution-local-retain must be a non-default Retain class that binds on first consumer') +for path, doc in objects: + if doc['kind'] == 'PersistentVolumeClaim' and path.startswith(platform + '/'): + check(doc['spec'].get('storageClassName') == 'aevolution-local-retain', + f'{path}: platform volumes must use aevolution-local-retain') + if doc['kind'] in ('Job', 'Deployment', 'StatefulSet', 'Pod'): + pod = doc['spec'].get('template', {}).get('spec', doc['spec']) + for container in pod.get('containers', []) + pod.get('initContainers', []): + check(not any('secretRef' in source for source in container.get('envFrom', [])), + f'{path}: never import a whole Secret into the environment') + for variable in container.get('env', []): + ref = variable.get('valueFrom', {}).get('secretKeyRef', {}) + check(ref.get('key') not in ('PGPASSWORD', 'POSTGRES_PASSWORD', 'PGUSER', 'POSTGRES_USER'), + f'{path}: {variable.get("name")} must not come from {ref.get("key")}') + if pod.get('volumes') and 'store' in doc['metadata'].get('name', ''): + check(doc['metadata'].get('namespace') != 'evolution-world0', + f'{path}: store Jobs never target World0') + # The live Argo path. for path, doc in objects: if path.startswith('deploy/kubernetes/dev/'): diff --git a/tools/ci/server-image-smoke.sh b/tools/ci/server-image-smoke.sh index f1e88b2..be2d846 100644 --- a/tools/ci/server-image-smoke.sh +++ b/tools/ci/server-image-smoke.sh @@ -15,11 +15,19 @@ code=$? set -e test "$code" -eq 2 grep -q '^usage: evo-server ' <<<"$usage" +# The bundled operations tool refuses to run without a subcommand too (M8.3). +set +e +tool="$(run --entrypoint /usr/local/bin/evo-world-clone "$image" 2>&1)" +code=$? +set -e +test "$code" -eq 2 +grep -q '^usage: evo-world-clone ' <<<"$tool" # Non-root, every shared library resolved, no credential material in the image. run --entrypoint /bin/sh "$image" -c ' set -eu test "$(id -u)" = 10001 && test "$(id -g)" = 10001 ! ldd /usr/local/bin/evo-server | grep -q "not found" + ! ldd /usr/local/bin/evo-world-clone | grep -q "not found" test -z "$(find / -xdev \( -name "*pgpass*" -o -name "pg_service.conf" -o -name "*.pem" \ -o -name "*.key" \) -not -path "/etc/ssl/*" -not -path "/usr/share/*" -print 2>/dev/null)" '