Add M8.3 dev/stage stores: Retain volumes, free-space reserve, store check

Keep the filesystem object store on a node-local Retain-class PVC per
environment (ADR-0070), with the database on the external PostgreSQL VM.
Because local-path does not enforce sizes, the object store now refuses
new objects below a free-space reserve with a distinct full error, so a
full volume publishes and acknowledges nothing. evo-world-clone gains
check-store to qualify volume retention, reserve, database version/TLS
and the runtime privilege boundary from inside the cluster; it now ships
in the server image. Add the storage class, object claims, hand-applied
store Jobs, policy rules forbidding password env vars, and the runbook.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-29 13:51:15 -04:00
co-authored by Claude Opus 5.5
parent 9d9e00c307
commit 2d950060db
23 changed files with 833 additions and 24 deletions
+1 -1
View File
@@ -127,7 +127,7 @@ if(EVO_BUILD_POSTGRES)
tests/region_checkpoint_store_tests.cpp tests/region_host_tests.cpp
tests/region_server_process_tests.cpp tests/region_recovery_tests.cpp
tests/region_ops_tests.cpp tests/region_clone_tests.cpp
tests/region_migration_tests.cpp)
tests/region_migration_tests.cpp tests/region_store_tests.cpp)
target_link_libraries(evo-postgres-tests PRIVATE evo::region_host Threads::Threads)
add_dependencies(evo-postgres-tests evo-server evo-world-clone) # Spawned by process tests.
evo_owned_target(evo-postgres-tests)
+10 -2
View File
@@ -11,6 +11,10 @@
// stats max_queued=<n> max_tick_debt=<n> commits=<n> store_failures=<n>
// Exit: 0 drained, 2 usage, 3 drain deadline passed, 4 quarantined.
//
// --objects-reserve-mib <n> (M8.3, default 1024) refuses new objects that would leave
// less free space on the object volume: node-local volumes do not enforce their
// requested size. A refused upload is a store failure; nothing is acknowledged.
//
// --ops-listen <numeric address:port> (M7.8) serves GET /livez (the loop is alive; says
// nothing about authority), /readyz (200 only with current authority) and /metrics
// (Prometheus text). It prints "ops listening <port>" once bound.
@@ -54,6 +58,7 @@ struct Options {
std::string crash_point;
std::uint64_t crash_ordinal{};
std::string ops_listen;
std::uint64_t objects_reserve_mib{1024};
};
// Kills this process without any cleanup: the real SIGKILL, at a chosen boundary.
@@ -148,6 +153,8 @@ std::optional<Options> parse(int argc, char** argv) {
o.host.lease = {};
else if (flag == "--ops-listen")
o.ops_listen = value;
else if (flag == "--objects-reserve-mib" && n && *n <= 1048576)
o.objects_reserve_mib = *n;
else if (flag == "--crash-at") {
const auto colon = value.find(':');
const auto ordinal =
@@ -263,8 +270,9 @@ template <typename State> int serve(Options& o) {
};
}
std::uint64_t ack_steps = 0;
auto objects = persistence::FilesystemObjectStore::open(o.objects, o.host.key,
persistence::max_lineage_payload_bytes);
auto objects = persistence::FilesystemObjectStore::open(
o.objects, o.host.key, persistence::max_lineage_payload_bytes, nullptr,
o.objects_reserve_mib * 1024 * 1024);
if (!objects) {
std::puts("error objects");
return 2;
+107 -5
View File
@@ -7,6 +7,8 @@
// [--audit-max N] [--crash-at before-transaction]
// evo-world-clone audit --world W --region R --pg-* ... --objects DIR --retained HEX...
// [--max-commits N]
// evo-world-clone check-store --world W --region R --pg-* ... --objects DIR
// [--reserve-mib N]
//
// --*pg-* is host, hostaddr, port, database, user, passfile, ca. The source
// connection needs only the runtime read functions and is switched to read-only
@@ -18,8 +20,20 @@
//
// Output: cloned source=W/R sequence=S tick=T target=W2/R2 outcomes=N digest=HEX
// audited commits=N tick=T outcomes=N profile=P clone=0|1
// Exit: 0 done, 2 usage, 3 source/history failure, 4 target refused, 5 target unknown.
// check-store (M8.3) qualifies an environment's stores from inside the cluster with
// the runtime identity. Objects: the volume must already exist (a missing or
// unmounted volume is refused), a fixed probe object is created once and verified by
// digest on every later run (so a restarted or rescheduled pod proves retention),
// and free space is reported against the reserve. Database: the adapter enforces
// server 18.6 and verify-full TLS off-host; the runtime login must reach the fenced
// functions but be refused table access.
// Output: objects available_mib=A reserve_mib=R / probe created|verified digest=HEX
// database version=V user=U ssl=S / database tables=denied functions=reachable
//
// Exit: 0 done, 2 usage, 3 source/history/database failure, 4 target/objects refused,
// 5 target unknown.
#include "evo/persistence/region_clone.hpp"
#include "evo/persistence/sha256.hpp"
#include <array>
#include <charconv>
@@ -49,7 +63,7 @@ struct Options {
Side source, target;
std::vector<persistence::Sha256Digest> retained;
std::optional<persistence::Uuid> commit;
std::uint64_t audit_max{}, max_commits{1u << 20U};
std::uint64_t audit_max{}, max_commits{1u << 20U}, reserve_mib{1024};
bool crash{};
};
@@ -112,7 +126,7 @@ std::optional<Options> parse(int argc, char** argv) {
return std::nullopt;
Options o;
o.command = argv[1];
if (o.command != "clone" && o.command != "audit")
if (o.command != "clone" && o.command != "audit" && o.command != "check-store")
return std::nullopt;
const bool clone = o.command == "clone";
for (int i = 2; i < argc; ++i) {
@@ -150,14 +164,18 @@ std::optional<Options> parse(int argc, char** argv) {
o.audit_max = *n;
else if (clone && flag == "--crash-at" && value == "before-transaction")
o.crash = true;
else if (!clone && flag == "--max-commits" && n && *n > 0)
else if (o.command == "audit" && flag == "--max-commits" && n && *n > 0)
o.max_commits = *n;
else if (o.command == "check-store" && flag == "--reserve-mib" && n && *n <= 1048576)
o.reserve_mib = *n;
else
return std::nullopt;
}
if (clone && o.target.key.region == 0)
o.target.key.region = o.source.key.region;
if (!complete(o.source) || o.retained.empty() || (clone && !complete(o.target)))
const bool needs_builds = o.command != "check-store";
if (!complete(o.source) || (needs_builds && o.retained.empty()) ||
(clone && !complete(o.target)))
return std::nullopt;
return o;
}
@@ -236,6 +254,88 @@ std::unique_ptr<persistence::Postgres> connect(const Side& s) {
return std::move(*db);
}
// A fixed 64 KiB probe: the same bytes on every run, so its digest names it.
std::vector<std::uint8_t> probe_bytes() {
std::vector<std::uint8_t> out(65536);
const std::string_view marker = "aevolution-store-probe-v1";
for (std::size_t i = 0; i < out.size(); ++i)
out[i] = i < marker.size() ? static_cast<std::uint8_t>(marker[i])
: static_cast<std::uint8_t>((i * 131U) ^ (i >> 7U));
return out;
}
int check_store(const Options& o) {
auto store = persistence::FilesystemObjectStore::open(o.source.objects, o.source.key,
persistence::max_lineage_payload_bytes,
nullptr, o.reserve_mib * 1024 * 1024);
if (!store) {
std::printf("objects refused %s\n", store.error() == persistence::ObjectError::invalid
? "missing-or-not-a-directory"
: "io");
return 4;
}
const auto available = (*store)->available_bytes();
std::printf(
"objects available_mib=%llu reserve_mib=%llu\n",
static_cast<unsigned long long>(available.value_or(0) / (std::uint64_t{1024} * 1024)),
static_cast<unsigned long long>(o.reserve_mib));
const auto bytes = probe_bytes();
const auto digest = persistence::sha256(bytes);
if (!digest)
return 4;
const persistence::VerifiedObject probe{*digest, bytes.size()};
const char* state = "verified";
if (auto existing = (*store)->get(probe); !existing) {
if (existing.error() != persistence::ObjectError::missing) {
std::printf("probe %s\n", existing.error() == persistence::ObjectError::corrupt
? "corrupt"
: "unreadable");
return 4;
}
auto stored = (*store)->put(bytes);
if (!stored) {
std::printf("probe refused %s\n",
stored.error() == persistence::ObjectError::full ? "full" : "io");
return 4;
}
auto back = (*store)->get(probe);
if (!back || *back != bytes)
return 4;
state = "created";
}
std::printf("probe %s digest=%s\n", state, text(*digest).c_str());
std::fflush(stdout);
auto db = persistence::Postgres::connect(o.source.pg, timeout);
if (!db) {
const auto kind = db.error().kind;
std::printf("database refused %s\n", kind == persistence::PgError::version ? "version"
: kind == persistence::PgError::config ? "config"
: kind == persistence::PgError::connect ? "connect"
: "other");
return 3;
}
auto about = (*db)->query(
"SELECT pg_catalog.current_setting('server_version_num'), current_user::text, "
"coalesce((SELECT s.ssl::text || '/' || coalesce(s.version, '') FROM "
"pg_catalog.pg_stat_ssl s WHERE s.pid = pg_catalog.pg_backend_pid()), 'unknown')",
{}, timeout);
if (!about || about->size() != 1 || (*about)[0].size() != 3)
return 3;
const auto& row = (*about)[0];
std::printf("database version=%s user=%s ssl=%s\n", row[0].value_or("?").c_str(),
row[1].value_or("?").c_str(), row[2].value_or("?").c_str());
auto tables = (*db)->query("SELECT 1 FROM evolution.region_heads LIMIT 1", {}, timeout);
const bool denied = !tables && std::string_view(tables.error().sqlstate.data()) == "42501";
auto head = persistence::load_region_head(**db, o.source.key, timeout);
const bool reachable =
head || head.error().kind == persistence::LeaseStoreError::not_provisioned;
std::printf("database tables=%s functions=%s head=%s\n", denied ? "denied" : "READABLE",
reachable ? "reachable" : "unreachable",
head ? std::to_string(head->sequence).c_str() : "none");
return denied && reachable ? 0 : 3;
}
int audit(const Options& o) {
auto db = connect(o.source);
auto objects = open_objects(o.source);
@@ -329,5 +429,7 @@ int main(int argc, char** argv) {
stderr);
return 2;
}
if (options->command == "check-store")
return check_store(*options);
return options->command == "clone" ? clone(*options) : audit(*options);
}
+2
View File
@@ -100,6 +100,8 @@ ENTRYPOINT ["bash", "/opt/evo/tools/migrate-postgres.sh"]
FROM ${UBUNTU_IMAGE} AS server-runtime
RUN apt-get update && apt-get install -y --no-install-recommends libstdc++6 libgcc-s1 libssl3t64 zlib1g ca-certificates && rm -rf /var/lib/apt/lists/* && groupadd --gid 10001 evo && useradd --uid 10001 --gid 10001 --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin evo
COPY --from=postgres-build /source/build/postgres/evo-server /usr/local/bin/evo-server
# M8.3 operations tool: check-store qualification, clone and history audit.
COPY --from=postgres-build /source/build/postgres/evo-world-clone /usr/local/bin/evo-world-clone
COPY --from=postgres-build /source/.cache/postgres/install/lib/libpq.so* /opt/postgres/lib/
COPY --from=postgres-build /source/.cache/postgres/postgresql-18.6/COPYRIGHT /usr/share/doc/evo-postgres/COPYRIGHT
ENV LD_LIBRARY_PATH=/opt/postgres/lib
@@ -4,6 +4,7 @@ namespace: evolution-dev
resources:
- namespace.yaml
- ../base
- objects-pvc.yaml
labels:
- pairs:
aevolution.io/environment: dev
@@ -0,0 +1,15 @@
# M8.3 region object store volume (ADR-0070): durable data, so it belongs to the
# platform layer (applied by hand), never to an Argo Application. The volume is
# created on the node of its first consumer; every consumer selects
# aevolution.io/stores=true, so label exactly one worker first (ENVIRONMENTS.md).
# Retain: deleting this claim leaves the volume and its data Released, not deleted.
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: aevolution-objects
spec:
accessModes: [ReadWriteOnce]
storageClassName: aevolution-local-retain
resources:
requests:
storage: 10Gi
@@ -4,6 +4,7 @@ namespace: evolution-stage
resources:
- namespace.yaml
- ../base
- objects-pvc.yaml
labels:
- pairs:
aevolution.io/environment: stage
@@ -0,0 +1,15 @@
# M8.3 region object store volume (ADR-0070): durable data, so it belongs to the
# platform layer (applied by hand), never to an Argo Application. The volume is
# created on the node of its first consumer; every consumer selects
# aevolution.io/stores=true, so label exactly one worker first (ENVIRONMENTS.md).
# Retain: deleting this claim leaves the volume and its data Released, not deleted.
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: aevolution-objects
spec:
accessModes: [ReadWriteOnce]
storageClassName: aevolution-local-retain
resources:
requests:
storage: 10Gi
@@ -0,0 +1,17 @@
# M8.3 (ADR-0070). Cluster-scoped; a cluster administrator applies it by hand:
# kubectl apply -f deploy/kubernetes/platform/storage/storageclass.yaml
# Same provisioner as the cluster's only class (rancher.io/local-path: a directory on
# the node that first runs the consumer), but deleting a claim keeps the volume and
# its data (Released) instead of deleting it. Volumes are node-local: no replication,
# no failover, and the requested size is not enforced (the object store's free-space
# reserve and M8.5 alerts cover that). Never the default class.
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: aevolution-local-retain
labels:
app.kubernetes.io/part-of: aevolution
provisioner: rancher.io/local-path
reclaimPolicy: Retain
volumeBindingMode: WaitForFirstConsumer
allowVolumeExpansion: false
@@ -0,0 +1,174 @@
# M8.3 store Jobs for ONE disposable environment (dev or stage). Not an Argo path.
# Copy, then replace every REPLACE_* marker (the markers make this file unappliable
# as committed), and apply by hand:
# REPLACE_NAMESPACE evolution-dev | evolution-stage (never evolution-world0)
# REPLACE_DB_CIDR the database VM's address as a /32 (PGHOSTADDR)
# REPLACE_DB_PORT its port (PGPORT)
# REPLACE_RUNTIME_ROLE that environment's NOLOGIN runtime group role
# REPLACE_TOOLS_DIGEST aevolution-postgres-tools image digest (sha256:...)
# REPLACE_SERVER_DIGEST aevolution server-runtime image digest (sha256:...)
# Secret contract (docs/infrastructure/ENVIRONMENTS.md, never in Git):
# aevolution-db: PGHOST, PGHOSTADDR, PGPORT, PGDATABASE (only these
# keys are read; PGPASSWORD/PGUSER are never mounted)
# evolution-postgres-migrator: pg_service.conf, pgpass, ca.crt (migrate-postgres.sh)
# evolution-postgres-runtime: user, pgpass, ca.crt
# Order: apply the NetworkPolicy, run migrate, then check-store (twice: the second
# run must report "probe verified", proving the volume survived a new pod).
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: aevolution-database-egress
namespace: REPLACE_NAMESPACE
labels:
app.kubernetes.io/part-of: aevolution
spec:
podSelector:
matchExpressions:
- key: app.kubernetes.io/component
operator: In
values: [migrate, store-check, region]
policyTypes: [Egress]
egress:
- to:
- ipBlock:
cidr: REPLACE_DB_CIDR
ports:
- {protocol: TCP, port: REPLACE_DB_PORT}
---
apiVersion: batch/v1
kind: Job
metadata:
name: aevolution-db-migrate
namespace: REPLACE_NAMESPACE
labels:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: migrate
spec:
backoffLimit: 0
activeDeadlineSeconds: 180
ttlSecondsAfterFinished: 86400
template:
metadata:
labels:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: migrate
spec:
serviceAccountName: aevolution-migrator
automountServiceAccountToken: false
restartPolicy: Never
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile: {type: RuntimeDefault}
containers:
- name: migrate
image: registry.aeternalisgames.com/aeternalis/aevolution-postgres-tools@REPLACE_TOOLS_DIGEST
env:
- {name: PGSERVICE, value: evolution-migrator}
- {name: PGSERVICEFILE, value: /run/postgres/pg_service.conf}
- {name: EVO_PGPASS_SOURCE, value: /run/postgres/pgpass}
- {name: EVO_PG_RUNTIME_ROLE, value: REPLACE_RUNTIME_ROLE}
resources:
requests: {cpu: 100m, memory: 64Mi}
limits: {memory: 128Mi}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
volumeMounts:
- {name: connection, mountPath: /run/postgres, readOnly: true}
- {name: scratch, mountPath: /tmp}
volumes:
- name: connection
secret:
secretName: evolution-postgres-migrator
defaultMode: 0440
items:
- {key: pg_service.conf, path: pg_service.conf}
- {key: pgpass, path: pgpass}
- {key: ca.crt, path: ca.crt}
- name: scratch
emptyDir: {medium: Memory, sizeLimit: 8Mi}
---
apiVersion: batch/v1
kind: Job
metadata:
name: aevolution-store-check
namespace: REPLACE_NAMESPACE
labels:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: store-check
spec:
backoffLimit: 0
activeDeadlineSeconds: 120
ttlSecondsAfterFinished: 86400
template:
metadata:
labels:
app.kubernetes.io/part-of: aevolution
app.kubernetes.io/component: store-check
spec:
serviceAccountName: aevolution-region
automountServiceAccountToken: false
restartPolicy: Never
nodeSelector:
aevolution.io/stores: "true"
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
fsGroup: 10001
seccompProfile: {type: RuntimeDefault}
containers:
- name: check-store
image: registry.aeternalisgames.com/aeternalis/aevolution-server@REPLACE_SERVER_DIGEST
# libpq ignores a group-readable password file, and Secret volumes are
# group-readable for a non-root fsGroup, so copy it to private memory first.
# w1/r1 is a reserved probe namespace, never a real world.
command: ["/bin/bash", "-c"]
args:
- |
set -eu
umask 077
cp /run/postgres/pgpass /tmp/pgpass
exec /usr/local/bin/evo-world-clone check-store --world 1 --region 1 \
--objects /objects --pg-host "$DB_HOST" --pg-hostaddr "$DB_HOSTADDR" \
--pg-port "$DB_PORT" --pg-database "$DB_NAME" --pg-user "$DB_USER" \
--pg-passfile /tmp/pgpass --pg-ca /run/postgres/ca.crt
env:
- name: DB_HOST
valueFrom: {secretKeyRef: {name: aevolution-db, key: PGHOST}}
- name: DB_HOSTADDR
valueFrom: {secretKeyRef: {name: aevolution-db, key: PGHOSTADDR}}
- name: DB_PORT
valueFrom: {secretKeyRef: {name: aevolution-db, key: PGPORT}}
- name: DB_NAME
valueFrom: {secretKeyRef: {name: aevolution-db, key: PGDATABASE}}
- name: DB_USER
valueFrom: {secretKeyRef: {name: evolution-postgres-runtime, key: user}}
resources:
requests: {cpu: 50m, memory: 64Mi}
limits: {memory: 256Mi}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities: {drop: [ALL]}
volumeMounts:
- {name: objects, mountPath: /objects}
- {name: connection, mountPath: /run/postgres, readOnly: true}
- {name: scratch, mountPath: /tmp}
volumes:
- name: objects
persistentVolumeClaim:
claimName: aevolution-objects
- name: scratch
emptyDir: {medium: Memory, sizeLimit: 1Mi}
- name: connection
secret:
secretName: evolution-postgres-runtime
defaultMode: 0440
items:
- {key: pgpass, path: pgpass}
- {key: ca.crt, path: ca.crt}
+1
View File
@@ -7,6 +7,7 @@ names it. Remove an item only when it is done, and record where it was done.
| Item | Raised | Why deferred | Needed before |
|---|---|---|---|
| **World0 break-glass membership.** Decide who may act as `aevolution:world0-break-glass`, and how that group is granted: for example a short-lived client certificate with that organization, or an OIDC group. Decide also how its use is approved and recorded. The admission policy already enforces the group (ADR-0069). | M8.2, 2026-09-29 | Maintainer: figure out later | Any intentional deletion of World0 data or identity; M8.8 World0 launch |
| **Backup destination for dev/stage stores.** Decide where backups go: the AEvolution PostgreSQL VM (WAL archive or base backups) and each environment's `aevolution-objects` volume. It should be off the host for any claim beyond disposable environments. | M8.3, 2026-09-29 | Needs the maintainer's choice of target | M8.7 backup and restore; any World0 data |
| **Storage after the RAID/disk rework.** Re-run `deploy/kubernetes/inventory/disk-probe-local-path.yaml` (or a new class), and update INVENTORY.md. | M8.1, 2026-09-29 | Maintainer is reorganizing disks | M8.3's final storage decision |
## Accepted risks (maintainer decisions)
+8
View File
@@ -616,3 +616,11 @@ M8.2 designed and checked offline (2026-09-29), under ADR-0069:
- an offline policy check in CI, and a live isolation check.
The live checks run once the maintainer applies the platform layer. Next: M8.3,
durable stores in dev and stage.
M8.3 implemented and locally qualified (2026-09-29), under ADR-0070:
- each environment's database lives on the external PostgreSQL VM;
- object volumes are node-local and use the Retain class;
- a free-space reserve with a distinct full-volume refusal;
- `evo-world-clone check-store` for in-cluster qualification.
The in-cluster evidence and the backup destination wait on the maintainer
(docs/BACKLOG.md).
+21
View File
@@ -1015,3 +1015,24 @@ M8.2 applied and verified live (2026-09-29): 175 PASS and 2 SKIP. The only 3 FAI
were a script defect in the project check, now fixed. The maintainer accepted
leaving Argo's `default` project unrestricted. Break-glass membership is in the
new docs/BACKLOG.md. Next: M8.3, durable stores in dev and stage.
## M8.3 dev/stage stores (2026-09-29)
ADR-0070 sets up the dev and stage stores:
- **Database:** each environment gets its own database and logins on the external
PostgreSQL VM.
- **Object store:** the filesystem store stays, on a node-local PVC in the new
Retain class `aevolution-local-retain`, pinned to one labelled worker. There is
no object server.
- **Capacity:** node-local volumes ignore their requested size, so the object
store now enforces a free-space reserve (`--objects-reserve-mib`, default 1024).
A full volume refuses new objects with a distinct error, and nothing is
acknowledged.
- **Qualification:** `evo-world-clone check-store`, now in the server image,
checks from inside the cluster: volume presence and retention, the reserve,
database version and TLS, and the runtime privilege boundary.
Not yet done:
- the in-cluster run (it needs Secrets with the documented key names and
published image digests);
- choosing a backup destination (docs/BACKLOG.md).
+102
View File
@@ -0,0 +1,102 @@
# ADR-0070: Durable stores for dev and stage
## Status
Accepted for M8.3, 2026-09-29. It builds on ADR-0059 (PostgreSQL), ADR-0061
(object store), ADR-0069 (environments) and the M8.1 inventory. Storage decisions
are provisional until the planned RAID and disk rework (docs/BACKLOG.md).
## Context
Inventory facts that drive this decision:
- PostgreSQL runs on a separate VM outside Kubernetes (maintainer). The adapter
already requires exactly 18.6 and `verify-full` TLS off-host, and the migration
audits the runtime role.
- The cluster's only StorageClass, `local-path`, is node-local, reclaims with
Delete and does not enforce requested sizes.
- Object publication on the workers measured 29–36 ms mean (73 ms max). Worker-3's
root filesystem is 87 % full.
- MinIO in the cluster is unpinned (`latest`) and belongs to another project.
The roadmap asks for:
- a supported PostgreSQL and a maintained object store, with pinned versions;
- PV retention and a backup destination;
- evidence for pod restart, missing-volume refusal, object semantics, credential
rotation and full disk.
The named risk is that a local PV binds failover to a node that has been lost.
## Decision
**Database.** Use the maintainer's PostgreSQL VM. Each environment gets its own
database, and its own migrator login and NOLOGIN runtime group with a member login
(ADR-0060). The version pin (18.6), TLS verification and privilege audit are
enforced by code at connect and migrate time; they are not assumed. Migrations run
as a hand-applied Job using the pinned `postgres-tools` image.
**Object store.** Keep the ADR-0061 filesystem store. There is **no object
server**: MinIO is ruled out as a new default (INFRASTRUCTURE_ARCHITECTURE), and an
S3 adapter would be a new dependency. Instead:
- The store lives on one PVC per environment, `aevolution-objects`, in the new
class `aevolution-local-retain` (the same provisioner, with **Retain** and
WaitForFirstConsumer; never the default class).
- The PVC is platform-owned (hand-applied, never Argo).
- Every consumer selects the node label `aevolution.io/stores=true`, which is put
on exactly one worker. worker-1 or worker-2 is recommended; never worker-3 or
the control plane.
**Capacity guard.** `local-path` ignores the requested size, so
`FilesystemObjectStore` now takes a free-space reserve. `evo-server` exposes it as
`--objects-reserve-mib`, default 1024.
- A new object that would leave less free space is refused with a distinct
`ObjectError::full`, before anything is written.
- Identical objects that already exist stay reusable.
- The host treats a refused upload as a retryable store failure, so nothing is
published or acknowledged, and a drain that cannot publish ends without
committing anything (exit 3).
- A genuinely failing write (ENOSPC or EFBIG) is `io`, and leaves no temporary
file or object behind.
**Qualification tool.** `evo-world-clone check-store` runs in the cluster as a Job
with the runtime identity. It:
- refuses a missing or unmounted object root;
- creates a fixed 64 KiB probe object once, then verifies it by digest on every
later run (proof of retention across pods);
- reports free space against the reserve;
- reports the server version, user and TLS;
- requires that the runtime login reaches the fenced functions but is denied
table access.
The server image now includes the tool.
**Secret contract.**
- `aevolution-db` supplies only `PGHOST`, `PGHOSTADDR`, `PGPORT` and `PGDATABASE`.
- `evolution-postgres-runtime` supplies `user`, `pgpass` and `ca.crt`.
- `evolution-postgres-migrator` supplies `pg_service.conf`, `pgpass` and `ca.crt`.
Passwords are only ever mounted as files, and copied into private memory
(mode 0600) before use, because libpq ignores group-readable password files. The
policy check forbids `PGPASSWORD`/`PGUSER` references and whole-Secret `envFrom`.
**Retention and backup.**
- Deleting the claim leaves the PV Released with its data. Recovery is an explicit
rebind by an administrator, verified by `check-store` ("probe verified").
- The backup destination for the database VM and the object PVC is **not decided
yet**. It is a maintainer decision, recorded in docs/BACKLOG.md and required
before M8.7. Dev and stage run without a verified backup, which is acceptable
only because they are disposable. World0 remains blocked.
## Consequences
- A lost worker takes its object volume with it. The dev and stage regions on that
worker are then unavailable (the server refuses a missing volume and quarantines
missing objects); they are never silently re-initialized. This is the named risk,
accepted for dev and stage only. World0 needs replicated or off-node storage
(M8.7/M8.8).
- The reserve and the kubelet's nodefs eviction threshold interact. Set the reserve
above the node's remaining headroom target, and alert on free space (M8.5).
- Re-probe and re-decide after the disk rework.
## Alternatives rejected
- **MinIO or another object server.** It adds an unpinned or new dependency, a
second durability layer to back up, and network latency, for no gain on one
node.
- **Leaving `local-path`/Delete for durable data.** Deleting the claim or an Argo
prune would erase the store.
- **Trusting PVC sizes.** `local-path` does not enforce them.
+1
View File
@@ -80,3 +80,4 @@ design. No unadopted candidate is a link dependency of the proof.
- [ADR-0067: Region clones and retained-history audit](0067-region-clones.md) - accepted for M7.9a re-identified clone genesis naming its source cut, isolated clone databases and full-chain retention audit.
- [ADR-0068: Upgrade barrier and durability qualification](0068-upgrade-barrier.md) - accepted for M7.9b build-bound authority, retirement barrier fencing old writers, explicit refusal of rule/format changes and the measured recovery fixture.
- [ADR-0069: Environment isolation and World0 protection](0069-environment-isolation.md) - accepted for M8.2 per-environment guardrails, native World0 admission protection, scoped Argo projects and verification.
- [ADR-0070: Durable stores for dev and stage](0070-dev-stage-stores.md) - accepted for M8.3 external PostgreSQL per environment, Retain-class node-local object volumes, free-space reserve and in-cluster store qualification.
+39
View File
@@ -2689,3 +2689,42 @@ admission policy and the stage and World0 AppProjects, then ran
`[]`. The corrected script has not been re-run on the cluster.
- **WARN:** the `default` AppProject allows every namespace. The maintainer
accepted this (docs/BACKLOG.md).
## M8.3 dev/stage stores - 2026-09-29
- WSL Linux `checks` set: CTest 59/59 in each configuration. The format check
passed.
- PostgreSQL suites: Release and ASan/UBSan passed with the new
`region_store_tests`. Analysis passed after one clang-tidy fix (an int
multiplication widened in the MiB report).
- Store tests:
- An unreachable reserve refuses new objects with `full`, leaving no files. An
identical existing object is still reused.
- A lineage batch through a full store fails as `upload`/`full`, and the head is
unchanged.
- RLIMIT_FSIZE makes a real write fail with `io`, leaving no object or
temporary.
- `check-store`:
- its first run creates the probe and its second verifies it;
- a missing root exits 4 without creating it;
- a full volume exits 4;
- an owner login exits 3 (tables READABLE);
- the runtime login reports `version=180006 ... ssl=false` (local socket), and
`tables=denied functions=reachable`.
- An `evo-server` whose reserve cannot be met simulated but acknowledged
nothing. Its drain exited 3, and the head was unchanged.
- Mutations (restored and rebuilt), each caught:
- removing the reserve check;
- `check-store` always succeeding.
- `tools/ci/deploy-policy-check.py`: 41 objects, 0 failures. Four new-rule
mutations were caught:
- a Delete class;
- a platform claim on `local-path`;
- a `PGPASSWORD` key reference;
- a whole-Secret `envFrom`.
- Kustomize renders the dev and stage overlays with the `aevolution-objects`
claim; World0 has none.
- The server image now contains `evo-world-clone`, and its smoke script checks the
tool's usage exit. It has not been built locally, because Docker is not running.
- Not executed: anything in the cluster, the store Jobs, retention, rebind and
rotation evidence, and the image build.
+47 -5
View File
@@ -21,12 +21,12 @@ manual, and none is performed by Argo CD.
Each namespace gets its own Secrets, with environment-specific values:
- `harbor-pull`: a pull-only Harbor robot (docker-registry type).
- `evolution-postgres-migrator`: the migrator/owner login for that environment's
database.
database. Keys: `pg_service.conf`, `pgpass`, `ca.crt`.
- `evolution-postgres-runtime`: the runtime login, a member of that environment's
NOLOGIN runtime group (ADR-0060).
- `aevolution-db`: connection settings (host, address, port, database, TLS mode
and root certificate). Mount it as files. The adapter reads a password file,
never `PGPASSWORD`.
NOLOGIN runtime group (ADR-0060). Keys: `user`, `pgpass`, `ca.crt`.
- `aevolution-db`: connection settings. Only `PGHOST`, `PGHOSTADDR` (numeric),
`PGPORT` and `PGDATABASE` are ever read, as individual variables. `PGPASSWORD`
and `PGUSER` are never mounted; the policy check forbids it.
A World0 credential must never exist in the dev or stage namespaces, and no dev or
stage credential may reach the World0 database. Rotate by replacing the Secret and
@@ -52,6 +52,48 @@ bash tools/ops/verify-isolation.sh
Create the Secrets listed above in each namespace before any workload syncs.
## Stores (M8.3, ADR-0070)
Run this once per disposable environment (`dev`, then `stage`).
**1. Label the store node, and apply the storage class and claim.**
```sh
# Label exactly one worker (worker-1 or worker-2; never worker-3 or the control plane).
kubectl label node rke2-worker-1 aevolution.io/stores=true
kubectl apply -f deploy/kubernetes/platform/storage/storageclass.yaml
kubectl apply -k deploy/kubernetes/platform/dev # adds the aevolution-objects claim
```
**2. Prepare the store Jobs.** Copy `deploy/kubernetes/stores/stores.yaml.example`,
replace its `REPLACE_*` markers (namespace, database address and port, runtime
role, image digests), and create the Secrets above.
**3. Run and qualify.**
```sh
kubectl apply -f stores-dev.yaml # policy + both Jobs
kubectl logs -n evolution-dev job/aevolution-db-migrate # schema and audit
kubectl logs -n evolution-dev job/aevolution-store-check # expect exit 0 and "probe created"
```
Evidence to record in docs/engineering/VALIDATION.md:
- **Pod restart:** delete the check Job and apply it again. It must print
`probe verified`, with the same digest.
- **Missing volume:** run the check with the `objects` volume removed. It must exit
4 with `objects refused missing-or-not-a-directory`.
- **Full volume:** add `--reserve-mib 1048576` and use a new probe region, such as
`--region 2`. It must exit 4 with `probe refused full`.
- **Retention:** `kubectl delete pvc aevolution-objects -n evolution-dev`. The PV
must become `Released`, not deleted. To rebind:
- clear the PV's `spec.claimRef`;
- recreate the claim with `volumeName` set to that PV;
- rerun the check, which must print `probe verified`.
- **Credential rotation:**
- Change the runtime login's password on the database VM, and run the check
before updating the Secret. It must exit 3 with `database refused connect`.
- Update the Secret's `pgpass` and rerun. It must exit 0.
- **Privilege boundary:** every run must print
`database tables=denied functions=reachable`.
## Break-glass
How membership is granted is not decided yet; see docs/BACKLOG.md. Until it is,
+18 -5
View File
@@ -5,6 +5,7 @@
#include <expected>
#include <filesystem>
#include <memory>
#include <optional>
#include <span>
#include <vector>
@@ -19,6 +20,7 @@ enum class ObjectError : std::uint8_t {
corrupt, // Length or digest mismatch, or not a regular file: quarantine.
bounds, // Larger than the configured maximum; rejected before allocation.
io, // Write/sync/link failure; the object is absent or unverified.
full, // Writing would leave less free space than the reserve; nothing written.
};
// Test-only fault injection and accounting for directory fsyncs. When passed to
// open() it must outlive the store; production callers pass nothing.
@@ -33,17 +35,22 @@ struct VerifiedObject {
};
class FilesystemObjectStore {
public:
// trusted_root must be an existing real directory (not a symlink). Namespace
// directories w<world>/r<region> are created 0700; their parents are synced on
// every open, so a creation whose sync failed earlier is made durable now.
// trusted_root must be an existing real directory (not a symlink): a missing or
// unmounted volume is refused, never created. Namespace directories
// w<world>/r<region> are created 0700; their parents are synced on every open, so
// a creation whose sync failed earlier is made durable now. reserve_bytes (M8.3):
// put() refuses new objects that would leave less free space than this on the
// volume, because node-local volumes do not enforce their requested size.
[[nodiscard]] static std::expected<std::unique_ptr<FilesystemObjectStore>, ObjectError>
open(const std::filesystem::path& trusted_root, RegionKey key, std::size_t max_object_bytes,
ObjectStoreFaults* faults = nullptr);
ObjectStoreFaults* faults = nullptr, std::uint64_t reserve_bytes = 0);
FilesystemObjectStore(const FilesystemObjectStore&) = delete;
FilesystemObjectStore& operator=(const FilesystemObjectStore&) = delete;
FilesystemObjectStore(FilesystemObjectStore&&) = delete;
FilesystemObjectStore& operator=(FilesystemObjectStore&&) = delete;
~FilesystemObjectStore();
// Refuses with `full`, before writing anything, when the volume's available
// space minus the object is below the reserve.
// Writes a private temporary file, fsyncs it, links it to its key without
// replacing anything, then fsyncs the directory. Identical bytes already stored
// (an earlier attempt whose directory sync failed, or a concurrent installer)
@@ -57,15 +64,21 @@ class FilesystemObjectStore {
[[nodiscard]] const std::filesystem::path& directory() const noexcept {
return directory_;
}
// Bytes available to this process on the volume; nullopt if unknown.
[[nodiscard]] std::optional<std::uint64_t> available_bytes() const noexcept;
[[nodiscard]] std::uint64_t reserve_bytes() const noexcept {
return reserve_;
}
private:
FilesystemObjectStore(std::filesystem::path directory, int descriptor, std::size_t maximum,
ObjectStoreFaults* faults);
ObjectStoreFaults* faults, std::uint64_t reserve);
[[nodiscard]] std::expected<VerifiedObject, ObjectError>
durable_existing(const VerifiedObject& object) const;
std::filesystem::path directory_;
int descriptor_;
std::size_t maximum_;
ObjectStoreFaults* faults_;
std::uint64_t reserve_;
};
} // namespace evo::persistence
+23 -6
View File
@@ -5,6 +5,7 @@
#include <fcntl.h>
#include <string>
#include <sys/stat.h>
#include <sys/statvfs.h>
#include <unistd.h>
namespace evo::persistence {
@@ -128,9 +129,17 @@ read_verified(int directory, const VerifiedObject& object, bool sync_file = fals
} // namespace
FilesystemObjectStore::FilesystemObjectStore(std::filesystem::path directory, int descriptor,
std::size_t maximum, ObjectStoreFaults* faults)
: directory_(std::move(directory)), descriptor_(descriptor), maximum_(maximum),
faults_(faults) {}
std::size_t maximum, ObjectStoreFaults* faults,
std::uint64_t reserve)
: directory_(std::move(directory)), descriptor_(descriptor), maximum_(maximum), faults_(faults),
reserve_(reserve) {}
std::optional<std::uint64_t> FilesystemObjectStore::available_bytes() const noexcept {
struct statvfs info{};
if (::fstatvfs(descriptor_, &info) != 0)
return std::nullopt;
return static_cast<std::uint64_t>(info.f_bavail) * static_cast<std::uint64_t>(info.f_frsize);
}
FilesystemObjectStore::~FilesystemObjectStore() {
static_cast<void>(::close(descriptor_));
@@ -138,7 +147,8 @@ FilesystemObjectStore::~FilesystemObjectStore() {
std::expected<std::unique_ptr<FilesystemObjectStore>, ObjectError>
FilesystemObjectStore::open(const std::filesystem::path& trusted_root, RegionKey key,
std::size_t max_object_bytes, ObjectStoreFaults* faults) {
std::size_t max_object_bytes, ObjectStoreFaults* faults,
std::uint64_t reserve_bytes) {
if (key.world == 0 || key.region == 0 || max_object_bytes == 0 ||
max_object_bytes > maximum_object_limit || trusted_root.empty())
return std::unexpected{ObjectError::invalid};
@@ -154,8 +164,9 @@ FilesystemObjectStore::open(const std::filesystem::path& trusted_root, RegionKey
Descriptor region_fd{subdirectory(world_fd.get(), region, faults)};
if (region_fd.get() < 0)
return std::unexpected{ObjectError::io};
return std::unique_ptr<FilesystemObjectStore>(new FilesystemObjectStore(
trusted_root / world / region, region_fd.release(), max_object_bytes, faults));
return std::unique_ptr<FilesystemObjectStore>(
new FilesystemObjectStore(trusted_root / world / region, region_fd.release(),
max_object_bytes, faults, reserve_bytes));
}
std::expected<VerifiedObject, ObjectError>
@@ -171,6 +182,12 @@ FilesystemObjectStore::put(std::span<const std::uint8_t> bytes) const {
// Immutable: identical content is reused (durably); anything else is corrupt.
if (auto found = durable_existing(object); found || found.error() != ObjectError::missing)
return found;
// Only new bytes need space; checked before anything is written.
const auto available = available_bytes();
if (!available)
return std::unexpected{ObjectError::io};
if (*available < reserve_ || *available - reserve_ < bytes.size())
return std::unexpected{ObjectError::full};
const auto name = key_name(*digest);
const auto temporary = ".tmp-" + name + "-" + std::to_string(::getpid()) + "-" +
+3
View File
@@ -11,6 +11,7 @@ void run_region_recovery_tests();
void run_region_ops_tests();
void run_region_clone_tests();
void run_region_migration_tests();
void run_region_store_tests();
namespace {
using namespace evo::persistence;
using namespace std::chrono_literals;
@@ -91,6 +92,8 @@ int main() {
std::puts("region ops integration passed");
run_region_clone_tests();
std::puts("region clone and history integration passed");
run_region_store_tests();
std::puts("region store qualification integration passed");
run_region_migration_tests();
std::puts("region migration, barrier and qualification integration passed");
return 0;
+194
View File
@@ -0,0 +1,194 @@
// M8.3 store qualification (ADR-0070) against real PostgreSQL and real processes:
// the object volume's free-space reserve, a genuinely failing write, the
// check-store probe (retention across runs, missing-volume refusal, privilege
// boundary) and a server whose volume is full. Run by tools/ci/postgres-check.sh.
#include "evo/persistence/checkpoint_store.hpp"
#include "fixtures/server_process.hpp"
#include <csignal>
#include <source_location>
#include <sys/resource.h>
void run_region_store_tests();
namespace {
using namespace evo::test::process;
using namespace evo::persistence;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region store test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
constexpr std::size_t budget = 8000000;
constexpr std::uint64_t unreachable_reserve = std::uint64_t{1} << 60U; // 1 EiB.
std::unique_ptr<FilesystemObjectStore> store(const fs::path& root, RegionKey key,
std::uint64_t reserve = 0) {
auto made = FilesystemObjectStore::open(root, key, max_lineage_payload_bytes, nullptr, reserve);
CHECK(made);
return std::move(*made);
}
fs::path fresh(const std::string& name) {
const auto root = fs::path(env("PGHOST")) / name;
fs::remove_all(root);
fs::create_directory(root);
fs::permissions(root, fs::perms::owner_all, fs::perm_options::replace);
return root;
}
std::size_t entries(const fs::path& dir) {
std::size_t n = 0;
for ([[maybe_unused]] const auto& entry : fs::directory_iterator(dir))
++n;
return n;
}
std::vector<std::uint8_t> bytes_of(std::size_t size, std::uint8_t seed) {
std::vector<std::uint8_t> out(size);
for (std::size_t i = 0; i < size; ++i)
out[i] = static_cast<std::uint8_t>(seed + (i * 7U));
return out;
}
// A full volume refuses new objects before writing; identical objects already
// stored stay usable; a batch against a full volume publishes nothing.
void full_volumes_publish_nothing() {
const RegionKey key{7, 81};
const auto root = fresh("store-full");
const auto stored = bytes_of(4096, 0x10);
auto roomy = store(root, key);
CHECK(roomy->available_bytes().value_or(0) > 0 && roomy->reserve_bytes() == 0);
CHECK(roomy->put(stored));
auto full = store(root, key, unreachable_reserve);
CHECK(full->reserve_bytes() == unreachable_reserve);
const auto before = entries(full->directory());
auto refused = full->put(bytes_of(4096, 0x20));
CHECK(!refused && refused.error() == ObjectError::full);
CHECK(entries(full->directory()) == before); // No temporary, no object.
CHECK(full->put(stored)); // Reuse needs no space.
// A lineage batch cannot publish through a full volume; the head stays.
const auto r = provision(82, 0xDE);
auto db = owner();
const auto head = load_region_checkpoint(*db, r.key, 5s);
CHECK(head);
auto runtime = Postgres::connect(
{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("EVO_PG_RUNTIME_USER"), {}, {}},
5s);
CHECK(runtime);
Sha256Digest build{};
build.fill(0xB1);
auto lease = acquire_region_lease(**runtime, r.key, build, 20000ms, 5s);
CHECK(lease);
auto state = evo::test::lineage_region();
RegionBatch<evo::processes::LineageRegion> batch{*state};
for (int i = 0; i < 10; ++i)
CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1)));
batch.seal();
Uuid commit{};
commit.fill(0xDF);
auto blocked = store(r.root, r.key, unreachable_reserve);
auto published =
publish_lineage_batch(**runtime, *blocked, *lease, *head, batch, commit, build, budget, 5s);
CHECK(!published && published.error().kind == PublicationError::upload &&
published.error().object == ObjectError::full);
CHECK(load_region_checkpoint(*db, r.key, 5s)->head == head->head);
CHECK(release_region_lease(**runtime, r.key, *lease, 5s));
}
// A write the filesystem really refuses (RLIMIT_FSIZE stands in for a full disk:
// the kernel rejects the write mid-object) is an io error that leaves no object
// and no temporary behind.
void failing_writes_leave_nothing() {
const auto root = fresh("store-efbig");
auto objects = store(root, {7, 83});
rlimit old{};
CHECK(::getrlimit(RLIMIT_FSIZE, &old) == 0);
const auto previous = std::signal(SIGXFSZ, SIG_IGN);
rlimit limited = old;
limited.rlim_cur = 8192;
CHECK(::setrlimit(RLIMIT_FSIZE, &limited) == 0);
auto refused = objects->put(bytes_of(65536, 0x30));
CHECK(::setrlimit(RLIMIT_FSIZE, &old) == 0);
std::signal(SIGXFSZ, previous);
CHECK(!refused && refused.error() == ObjectError::io);
CHECK(entries(objects->directory()) == 0);
CHECK(objects->put(bytes_of(65536, 0x30))); // The same write succeeds with room.
}
std::vector<std::string> check_store(const fs::path& objects, const std::string& user,
std::vector<std::string> extra, int& code) {
std::vector<std::string> args{env("EVO_WORLD_CLONE_BIN"),
"check-store",
"--world",
"7",
"--region",
"84",
"--pg-host",
env("PGHOST"),
"--pg-port",
env("PGPORT"),
"--pg-database",
env("PGDATABASE"),
"--pg-user",
user,
"--objects",
objects.string()};
args.insert(args.end(), extra.begin(), extra.end());
Child tool{args};
code = tool.wait_exit(30s);
return tool.lines();
}
bool has(const std::vector<std::string>& lines, std::string_view prefix) {
for (const auto& l : lines)
if (l.starts_with(prefix))
return true;
return false;
}
// The in-cluster qualification probe, exercised locally.
void check_store_qualifies_the_stores() {
const auto root = fresh("store-check");
const auto runtime = env("EVO_PG_RUNTIME_USER");
int code = -1;
auto first = check_store(root, runtime, {}, code);
CHECK(code == 0 && has(first, "objects available_mib=") && has(first, "probe created digest="));
CHECK(has(first, "database version=180006 user=" + runtime + " ssl=false"));
CHECK(has(first, "database tables=denied functions=reachable head=none"));
// A later run (a restarted or rescheduled pod) verifies the retained probe.
auto second = check_store(root, runtime, {}, code);
CHECK(code == 0 && has(second, "probe verified digest="));
// A missing or unmounted volume is refused, never created.
auto missing = check_store(root / "not-mounted", runtime, {}, code);
CHECK(code == 4 && has(missing, "objects refused missing-or-not-a-directory"));
CHECK(!fs::exists(root / "not-mounted"));
// A full volume refuses the probe (fresh namespace, reserve beyond any disk).
const auto other = fresh("store-check-full");
auto full = check_store(other, runtime, {"--reserve-mib", "1048576"}, code);
CHECK(code == 4 && has(full, "probe refused full"));
// An owner login can read tables: not an acceptable runtime identity.
auto owner_run = check_store(root, env("PGUSER"), {}, code);
CHECK(code == 3 && has(owner_run, "database tables=READABLE"));
}
// A real server on a full volume keeps simulating but never acknowledges, and its
// bounded drain gives up without committing anything.
void full_volume_server_acknowledges_nothing() {
const auto r = provision(85, 0xE7);
const auto before = head_of(r.key);
Child server{server_args(r, {"--objects-reserve-mib", "1048576", "--drain-ms", "1500",
"--ops-listen", "127.0.0.1:0"})};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
server.send(input_lines(1, 15, false));
server.pump(3000ms);
CHECK(!server.printed("ack "));
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 3); // Drain deadline passed with the batch unpublished.
CHECK(!server.printed("ack "));
const auto after = head_of(r.key);
CHECK(after.tick == before.tick && after.sequence == before.sequence);
}
} // namespace
void run_region_store_tests() {
full_volumes_publish_nothing();
failing_writes_leave_nothing();
check_store_qualifies_the_stores();
full_volume_server_acknowledges_nothing();
}
+25
View File
@@ -145,6 +145,31 @@ for path, doc in objects:
check(destination != 'evolution-world0' or name == 'aevolution-world0',
f'{path}: only aevolution-world0 may target evolution-world0')
# M8.3 storage (ADR-0070): durable volumes retain their data; database passwords are
# only ever mounted as files, never placed in the environment.
classes = {d['metadata']['name']: d for p, d in objects if d['kind'] == 'StorageClass'}
retain = classes.get('aevolution-local-retain')
check(retain is not None and retain.get('reclaimPolicy') == 'Retain'
and retain.get('volumeBindingMode') == 'WaitForFirstConsumer'
and 'storageclass.kubernetes.io/is-default-class' not in retain['metadata'].get('annotations', {}),
'aevolution-local-retain must be a non-default Retain class that binds on first consumer')
for path, doc in objects:
if doc['kind'] == 'PersistentVolumeClaim' and path.startswith(platform + '/'):
check(doc['spec'].get('storageClassName') == 'aevolution-local-retain',
f'{path}: platform volumes must use aevolution-local-retain')
if doc['kind'] in ('Job', 'Deployment', 'StatefulSet', 'Pod'):
pod = doc['spec'].get('template', {}).get('spec', doc['spec'])
for container in pod.get('containers', []) + pod.get('initContainers', []):
check(not any('secretRef' in source for source in container.get('envFrom', [])),
f'{path}: never import a whole Secret into the environment')
for variable in container.get('env', []):
ref = variable.get('valueFrom', {}).get('secretKeyRef', {})
check(ref.get('key') not in ('PGPASSWORD', 'POSTGRES_PASSWORD', 'PGUSER', 'POSTGRES_USER'),
f'{path}: {variable.get("name")} must not come from {ref.get("key")}')
if pod.get('volumes') and 'store' in doc['metadata'].get('name', ''):
check(doc['metadata'].get('namespace') != 'evolution-world0',
f'{path}: store Jobs never target World0')
# The live Argo path.
for path, doc in objects:
if path.startswith('deploy/kubernetes/dev/'):
+8
View File
@@ -15,11 +15,19 @@ code=$?
set -e
test "$code" -eq 2
grep -q '^usage: evo-server ' <<<"$usage"
# The bundled operations tool refuses to run without a subcommand too (M8.3).
set +e
tool="$(run --entrypoint /usr/local/bin/evo-world-clone "$image" 2>&1)"
code=$?
set -e
test "$code" -eq 2
grep -q '^usage: evo-world-clone ' <<<"$tool"
# Non-root, every shared library resolved, no credential material in the image.
run --entrypoint /bin/sh "$image" -c '
set -eu
test "$(id -u)" = 10001 && test "$(id -g)" = 10001
! ldd /usr/local/bin/evo-server | grep -q "not found"
! ldd /usr/local/bin/evo-world-clone | grep -q "not found"
test -z "$(find / -xdev \( -name "*pgpass*" -o -name "pg_service.conf" -o -name "*.pem" \
-o -name "*.key" \) -not -path "/etc/ssl/*" -not -path "/usr/share/*" -print 2>/dev/null)"
'