Add M8.3 dev/stage stores: Retain volumes, free-space reserve, store check
Keep the filesystem object store on a node-local Retain-class PVC per environment (ADR-0070), with the database on the external PostgreSQL VM. Because local-path does not enforce sizes, the object store now refuses new objects below a free-space reserve with a distinct full error, so a full volume publishes and acknowledges nothing. evo-world-clone gains check-store to qualify volume retention, reserve, database version/TLS and the runtime privilege boundary from inside the cluster; it now ships in the server image. Add the storage class, object claims, hand-applied store Jobs, policy rules forbidding password env vars, and the runbook. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+1
-1
@@ -127,7 +127,7 @@ if(EVO_BUILD_POSTGRES)
|
||||
tests/region_checkpoint_store_tests.cpp tests/region_host_tests.cpp
|
||||
tests/region_server_process_tests.cpp tests/region_recovery_tests.cpp
|
||||
tests/region_ops_tests.cpp tests/region_clone_tests.cpp
|
||||
tests/region_migration_tests.cpp)
|
||||
tests/region_migration_tests.cpp tests/region_store_tests.cpp)
|
||||
target_link_libraries(evo-postgres-tests PRIVATE evo::region_host Threads::Threads)
|
||||
add_dependencies(evo-postgres-tests evo-server evo-world-clone) # Spawned by process tests.
|
||||
evo_owned_target(evo-postgres-tests)
|
||||
|
||||
@@ -11,6 +11,10 @@
|
||||
// stats max_queued=<n> max_tick_debt=<n> commits=<n> store_failures=<n>
|
||||
// Exit: 0 drained, 2 usage, 3 drain deadline passed, 4 quarantined.
|
||||
//
|
||||
// --objects-reserve-mib <n> (M8.3, default 1024) refuses new objects that would leave
|
||||
// less free space on the object volume: node-local volumes do not enforce their
|
||||
// requested size. A refused upload is a store failure; nothing is acknowledged.
|
||||
//
|
||||
// --ops-listen <numeric address:port> (M7.8) serves GET /livez (the loop is alive; says
|
||||
// nothing about authority), /readyz (200 only with current authority) and /metrics
|
||||
// (Prometheus text). It prints "ops listening <port>" once bound.
|
||||
@@ -54,6 +58,7 @@ struct Options {
|
||||
std::string crash_point;
|
||||
std::uint64_t crash_ordinal{};
|
||||
std::string ops_listen;
|
||||
std::uint64_t objects_reserve_mib{1024};
|
||||
};
|
||||
|
||||
// Kills this process without any cleanup: the real SIGKILL, at a chosen boundary.
|
||||
@@ -148,6 +153,8 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
o.host.lease = {};
|
||||
else if (flag == "--ops-listen")
|
||||
o.ops_listen = value;
|
||||
else if (flag == "--objects-reserve-mib" && n && *n <= 1048576)
|
||||
o.objects_reserve_mib = *n;
|
||||
else if (flag == "--crash-at") {
|
||||
const auto colon = value.find(':');
|
||||
const auto ordinal =
|
||||
@@ -263,8 +270,9 @@ template <typename State> int serve(Options& o) {
|
||||
};
|
||||
}
|
||||
std::uint64_t ack_steps = 0;
|
||||
auto objects = persistence::FilesystemObjectStore::open(o.objects, o.host.key,
|
||||
persistence::max_lineage_payload_bytes);
|
||||
auto objects = persistence::FilesystemObjectStore::open(
|
||||
o.objects, o.host.key, persistence::max_lineage_payload_bytes, nullptr,
|
||||
o.objects_reserve_mib * 1024 * 1024);
|
||||
if (!objects) {
|
||||
std::puts("error objects");
|
||||
return 2;
|
||||
|
||||
@@ -7,6 +7,8 @@
|
||||
// [--audit-max N] [--crash-at before-transaction]
|
||||
// evo-world-clone audit --world W --region R --pg-* ... --objects DIR --retained HEX...
|
||||
// [--max-commits N]
|
||||
// evo-world-clone check-store --world W --region R --pg-* ... --objects DIR
|
||||
// [--reserve-mib N]
|
||||
//
|
||||
// --*pg-* is host, hostaddr, port, database, user, passfile, ca. The source
|
||||
// connection needs only the runtime read functions and is switched to read-only
|
||||
@@ -18,8 +20,20 @@
|
||||
//
|
||||
// Output: cloned source=W/R sequence=S tick=T target=W2/R2 outcomes=N digest=HEX
|
||||
// audited commits=N tick=T outcomes=N profile=P clone=0|1
|
||||
// Exit: 0 done, 2 usage, 3 source/history failure, 4 target refused, 5 target unknown.
|
||||
// check-store (M8.3) qualifies an environment's stores from inside the cluster with
|
||||
// the runtime identity. Objects: the volume must already exist (a missing or
|
||||
// unmounted volume is refused), a fixed probe object is created once and verified by
|
||||
// digest on every later run (so a restarted or rescheduled pod proves retention),
|
||||
// and free space is reported against the reserve. Database: the adapter enforces
|
||||
// server 18.6 and verify-full TLS off-host; the runtime login must reach the fenced
|
||||
// functions but be refused table access.
|
||||
// Output: objects available_mib=A reserve_mib=R / probe created|verified digest=HEX
|
||||
// database version=V user=U ssl=S / database tables=denied functions=reachable
|
||||
//
|
||||
// Exit: 0 done, 2 usage, 3 source/history/database failure, 4 target/objects refused,
|
||||
// 5 target unknown.
|
||||
#include "evo/persistence/region_clone.hpp"
|
||||
#include "evo/persistence/sha256.hpp"
|
||||
|
||||
#include <array>
|
||||
#include <charconv>
|
||||
@@ -49,7 +63,7 @@ struct Options {
|
||||
Side source, target;
|
||||
std::vector<persistence::Sha256Digest> retained;
|
||||
std::optional<persistence::Uuid> commit;
|
||||
std::uint64_t audit_max{}, max_commits{1u << 20U};
|
||||
std::uint64_t audit_max{}, max_commits{1u << 20U}, reserve_mib{1024};
|
||||
bool crash{};
|
||||
};
|
||||
|
||||
@@ -112,7 +126,7 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
return std::nullopt;
|
||||
Options o;
|
||||
o.command = argv[1];
|
||||
if (o.command != "clone" && o.command != "audit")
|
||||
if (o.command != "clone" && o.command != "audit" && o.command != "check-store")
|
||||
return std::nullopt;
|
||||
const bool clone = o.command == "clone";
|
||||
for (int i = 2; i < argc; ++i) {
|
||||
@@ -150,14 +164,18 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
o.audit_max = *n;
|
||||
else if (clone && flag == "--crash-at" && value == "before-transaction")
|
||||
o.crash = true;
|
||||
else if (!clone && flag == "--max-commits" && n && *n > 0)
|
||||
else if (o.command == "audit" && flag == "--max-commits" && n && *n > 0)
|
||||
o.max_commits = *n;
|
||||
else if (o.command == "check-store" && flag == "--reserve-mib" && n && *n <= 1048576)
|
||||
o.reserve_mib = *n;
|
||||
else
|
||||
return std::nullopt;
|
||||
}
|
||||
if (clone && o.target.key.region == 0)
|
||||
o.target.key.region = o.source.key.region;
|
||||
if (!complete(o.source) || o.retained.empty() || (clone && !complete(o.target)))
|
||||
const bool needs_builds = o.command != "check-store";
|
||||
if (!complete(o.source) || (needs_builds && o.retained.empty()) ||
|
||||
(clone && !complete(o.target)))
|
||||
return std::nullopt;
|
||||
return o;
|
||||
}
|
||||
@@ -236,6 +254,88 @@ std::unique_ptr<persistence::Postgres> connect(const Side& s) {
|
||||
return std::move(*db);
|
||||
}
|
||||
|
||||
// A fixed 64 KiB probe: the same bytes on every run, so its digest names it.
|
||||
std::vector<std::uint8_t> probe_bytes() {
|
||||
std::vector<std::uint8_t> out(65536);
|
||||
const std::string_view marker = "aevolution-store-probe-v1";
|
||||
for (std::size_t i = 0; i < out.size(); ++i)
|
||||
out[i] = i < marker.size() ? static_cast<std::uint8_t>(marker[i])
|
||||
: static_cast<std::uint8_t>((i * 131U) ^ (i >> 7U));
|
||||
return out;
|
||||
}
|
||||
|
||||
int check_store(const Options& o) {
|
||||
auto store = persistence::FilesystemObjectStore::open(o.source.objects, o.source.key,
|
||||
persistence::max_lineage_payload_bytes,
|
||||
nullptr, o.reserve_mib * 1024 * 1024);
|
||||
if (!store) {
|
||||
std::printf("objects refused %s\n", store.error() == persistence::ObjectError::invalid
|
||||
? "missing-or-not-a-directory"
|
||||
: "io");
|
||||
return 4;
|
||||
}
|
||||
const auto available = (*store)->available_bytes();
|
||||
std::printf(
|
||||
"objects available_mib=%llu reserve_mib=%llu\n",
|
||||
static_cast<unsigned long long>(available.value_or(0) / (std::uint64_t{1024} * 1024)),
|
||||
static_cast<unsigned long long>(o.reserve_mib));
|
||||
const auto bytes = probe_bytes();
|
||||
const auto digest = persistence::sha256(bytes);
|
||||
if (!digest)
|
||||
return 4;
|
||||
const persistence::VerifiedObject probe{*digest, bytes.size()};
|
||||
const char* state = "verified";
|
||||
if (auto existing = (*store)->get(probe); !existing) {
|
||||
if (existing.error() != persistence::ObjectError::missing) {
|
||||
std::printf("probe %s\n", existing.error() == persistence::ObjectError::corrupt
|
||||
? "corrupt"
|
||||
: "unreadable");
|
||||
return 4;
|
||||
}
|
||||
auto stored = (*store)->put(bytes);
|
||||
if (!stored) {
|
||||
std::printf("probe refused %s\n",
|
||||
stored.error() == persistence::ObjectError::full ? "full" : "io");
|
||||
return 4;
|
||||
}
|
||||
auto back = (*store)->get(probe);
|
||||
if (!back || *back != bytes)
|
||||
return 4;
|
||||
state = "created";
|
||||
}
|
||||
std::printf("probe %s digest=%s\n", state, text(*digest).c_str());
|
||||
std::fflush(stdout);
|
||||
|
||||
auto db = persistence::Postgres::connect(o.source.pg, timeout);
|
||||
if (!db) {
|
||||
const auto kind = db.error().kind;
|
||||
std::printf("database refused %s\n", kind == persistence::PgError::version ? "version"
|
||||
: kind == persistence::PgError::config ? "config"
|
||||
: kind == persistence::PgError::connect ? "connect"
|
||||
: "other");
|
||||
return 3;
|
||||
}
|
||||
auto about = (*db)->query(
|
||||
"SELECT pg_catalog.current_setting('server_version_num'), current_user::text, "
|
||||
"coalesce((SELECT s.ssl::text || '/' || coalesce(s.version, '') FROM "
|
||||
"pg_catalog.pg_stat_ssl s WHERE s.pid = pg_catalog.pg_backend_pid()), 'unknown')",
|
||||
{}, timeout);
|
||||
if (!about || about->size() != 1 || (*about)[0].size() != 3)
|
||||
return 3;
|
||||
const auto& row = (*about)[0];
|
||||
std::printf("database version=%s user=%s ssl=%s\n", row[0].value_or("?").c_str(),
|
||||
row[1].value_or("?").c_str(), row[2].value_or("?").c_str());
|
||||
auto tables = (*db)->query("SELECT 1 FROM evolution.region_heads LIMIT 1", {}, timeout);
|
||||
const bool denied = !tables && std::string_view(tables.error().sqlstate.data()) == "42501";
|
||||
auto head = persistence::load_region_head(**db, o.source.key, timeout);
|
||||
const bool reachable =
|
||||
head || head.error().kind == persistence::LeaseStoreError::not_provisioned;
|
||||
std::printf("database tables=%s functions=%s head=%s\n", denied ? "denied" : "READABLE",
|
||||
reachable ? "reachable" : "unreachable",
|
||||
head ? std::to_string(head->sequence).c_str() : "none");
|
||||
return denied && reachable ? 0 : 3;
|
||||
}
|
||||
|
||||
int audit(const Options& o) {
|
||||
auto db = connect(o.source);
|
||||
auto objects = open_objects(o.source);
|
||||
@@ -329,5 +429,7 @@ int main(int argc, char** argv) {
|
||||
stderr);
|
||||
return 2;
|
||||
}
|
||||
if (options->command == "check-store")
|
||||
return check_store(*options);
|
||||
return options->command == "clone" ? clone(*options) : audit(*options);
|
||||
}
|
||||
|
||||
@@ -100,6 +100,8 @@ ENTRYPOINT ["bash", "/opt/evo/tools/migrate-postgres.sh"]
|
||||
FROM ${UBUNTU_IMAGE} AS server-runtime
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends libstdc++6 libgcc-s1 libssl3t64 zlib1g ca-certificates && rm -rf /var/lib/apt/lists/* && groupadd --gid 10001 evo && useradd --uid 10001 --gid 10001 --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin evo
|
||||
COPY --from=postgres-build /source/build/postgres/evo-server /usr/local/bin/evo-server
|
||||
# M8.3 operations tool: check-store qualification, clone and history audit.
|
||||
COPY --from=postgres-build /source/build/postgres/evo-world-clone /usr/local/bin/evo-world-clone
|
||||
COPY --from=postgres-build /source/.cache/postgres/install/lib/libpq.so* /opt/postgres/lib/
|
||||
COPY --from=postgres-build /source/.cache/postgres/postgresql-18.6/COPYRIGHT /usr/share/doc/evo-postgres/COPYRIGHT
|
||||
ENV LD_LIBRARY_PATH=/opt/postgres/lib
|
||||
|
||||
@@ -4,6 +4,7 @@ namespace: evolution-dev
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- ../base
|
||||
- objects-pvc.yaml
|
||||
labels:
|
||||
- pairs:
|
||||
aevolution.io/environment: dev
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
# M8.3 region object store volume (ADR-0070): durable data, so it belongs to the
|
||||
# platform layer (applied by hand), never to an Argo Application. The volume is
|
||||
# created on the node of its first consumer; every consumer selects
|
||||
# aevolution.io/stores=true, so label exactly one worker first (ENVIRONMENTS.md).
|
||||
# Retain: deleting this claim leaves the volume and its data Released, not deleted.
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: aevolution-objects
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: aevolution-local-retain
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
@@ -4,6 +4,7 @@ namespace: evolution-stage
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- ../base
|
||||
- objects-pvc.yaml
|
||||
labels:
|
||||
- pairs:
|
||||
aevolution.io/environment: stage
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
# M8.3 region object store volume (ADR-0070): durable data, so it belongs to the
|
||||
# platform layer (applied by hand), never to an Argo Application. The volume is
|
||||
# created on the node of its first consumer; every consumer selects
|
||||
# aevolution.io/stores=true, so label exactly one worker first (ENVIRONMENTS.md).
|
||||
# Retain: deleting this claim leaves the volume and its data Released, not deleted.
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: aevolution-objects
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: aevolution-local-retain
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
@@ -0,0 +1,17 @@
|
||||
# M8.3 (ADR-0070). Cluster-scoped; a cluster administrator applies it by hand:
|
||||
# kubectl apply -f deploy/kubernetes/platform/storage/storageclass.yaml
|
||||
# Same provisioner as the cluster's only class (rancher.io/local-path: a directory on
|
||||
# the node that first runs the consumer), but deleting a claim keeps the volume and
|
||||
# its data (Released) instead of deleting it. Volumes are node-local: no replication,
|
||||
# no failover, and the requested size is not enforced (the object store's free-space
|
||||
# reserve and M8.5 alerts cover that). Never the default class.
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: aevolution-local-retain
|
||||
labels:
|
||||
app.kubernetes.io/part-of: aevolution
|
||||
provisioner: rancher.io/local-path
|
||||
reclaimPolicy: Retain
|
||||
volumeBindingMode: WaitForFirstConsumer
|
||||
allowVolumeExpansion: false
|
||||
@@ -0,0 +1,174 @@
|
||||
# M8.3 store Jobs for ONE disposable environment (dev or stage). Not an Argo path.
|
||||
# Copy, then replace every REPLACE_* marker (the markers make this file unappliable
|
||||
# as committed), and apply by hand:
|
||||
# REPLACE_NAMESPACE evolution-dev | evolution-stage (never evolution-world0)
|
||||
# REPLACE_DB_CIDR the database VM's address as a /32 (PGHOSTADDR)
|
||||
# REPLACE_DB_PORT its port (PGPORT)
|
||||
# REPLACE_RUNTIME_ROLE that environment's NOLOGIN runtime group role
|
||||
# REPLACE_TOOLS_DIGEST aevolution-postgres-tools image digest (sha256:...)
|
||||
# REPLACE_SERVER_DIGEST aevolution server-runtime image digest (sha256:...)
|
||||
# Secret contract (docs/infrastructure/ENVIRONMENTS.md, never in Git):
|
||||
# aevolution-db: PGHOST, PGHOSTADDR, PGPORT, PGDATABASE (only these
|
||||
# keys are read; PGPASSWORD/PGUSER are never mounted)
|
||||
# evolution-postgres-migrator: pg_service.conf, pgpass, ca.crt (migrate-postgres.sh)
|
||||
# evolution-postgres-runtime: user, pgpass, ca.crt
|
||||
# Order: apply the NetworkPolicy, run migrate, then check-store (twice: the second
|
||||
# run must report "probe verified", proving the volume survived a new pod).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: aevolution-database-egress
|
||||
namespace: REPLACE_NAMESPACE
|
||||
labels:
|
||||
app.kubernetes.io/part-of: aevolution
|
||||
spec:
|
||||
podSelector:
|
||||
matchExpressions:
|
||||
- key: app.kubernetes.io/component
|
||||
operator: In
|
||||
values: [migrate, store-check, region]
|
||||
policyTypes: [Egress]
|
||||
egress:
|
||||
- to:
|
||||
- ipBlock:
|
||||
cidr: REPLACE_DB_CIDR
|
||||
ports:
|
||||
- {protocol: TCP, port: REPLACE_DB_PORT}
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: aevolution-db-migrate
|
||||
namespace: REPLACE_NAMESPACE
|
||||
labels:
|
||||
app.kubernetes.io/part-of: aevolution
|
||||
app.kubernetes.io/component: migrate
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
activeDeadlineSeconds: 180
|
||||
ttlSecondsAfterFinished: 86400
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/part-of: aevolution
|
||||
app.kubernetes.io/component: migrate
|
||||
spec:
|
||||
serviceAccountName: aevolution-migrator
|
||||
automountServiceAccountToken: false
|
||||
restartPolicy: Never
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
fsGroup: 10001
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
containers:
|
||||
- name: migrate
|
||||
image: registry.aeternalisgames.com/aeternalis/aevolution-postgres-tools@REPLACE_TOOLS_DIGEST
|
||||
env:
|
||||
- {name: PGSERVICE, value: evolution-migrator}
|
||||
- {name: PGSERVICEFILE, value: /run/postgres/pg_service.conf}
|
||||
- {name: EVO_PGPASS_SOURCE, value: /run/postgres/pgpass}
|
||||
- {name: EVO_PG_RUNTIME_ROLE, value: REPLACE_RUNTIME_ROLE}
|
||||
resources:
|
||||
requests: {cpu: 100m, memory: 64Mi}
|
||||
limits: {memory: 128Mi}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities: {drop: [ALL]}
|
||||
volumeMounts:
|
||||
- {name: connection, mountPath: /run/postgres, readOnly: true}
|
||||
- {name: scratch, mountPath: /tmp}
|
||||
volumes:
|
||||
- name: connection
|
||||
secret:
|
||||
secretName: evolution-postgres-migrator
|
||||
defaultMode: 0440
|
||||
items:
|
||||
- {key: pg_service.conf, path: pg_service.conf}
|
||||
- {key: pgpass, path: pgpass}
|
||||
- {key: ca.crt, path: ca.crt}
|
||||
- name: scratch
|
||||
emptyDir: {medium: Memory, sizeLimit: 8Mi}
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: aevolution-store-check
|
||||
namespace: REPLACE_NAMESPACE
|
||||
labels:
|
||||
app.kubernetes.io/part-of: aevolution
|
||||
app.kubernetes.io/component: store-check
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
activeDeadlineSeconds: 120
|
||||
ttlSecondsAfterFinished: 86400
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/part-of: aevolution
|
||||
app.kubernetes.io/component: store-check
|
||||
spec:
|
||||
serviceAccountName: aevolution-region
|
||||
automountServiceAccountToken: false
|
||||
restartPolicy: Never
|
||||
nodeSelector:
|
||||
aevolution.io/stores: "true"
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
fsGroup: 10001
|
||||
seccompProfile: {type: RuntimeDefault}
|
||||
containers:
|
||||
- name: check-store
|
||||
image: registry.aeternalisgames.com/aeternalis/aevolution-server@REPLACE_SERVER_DIGEST
|
||||
# libpq ignores a group-readable password file, and Secret volumes are
|
||||
# group-readable for a non-root fsGroup, so copy it to private memory first.
|
||||
# w1/r1 is a reserved probe namespace, never a real world.
|
||||
command: ["/bin/bash", "-c"]
|
||||
args:
|
||||
- |
|
||||
set -eu
|
||||
umask 077
|
||||
cp /run/postgres/pgpass /tmp/pgpass
|
||||
exec /usr/local/bin/evo-world-clone check-store --world 1 --region 1 \
|
||||
--objects /objects --pg-host "$DB_HOST" --pg-hostaddr "$DB_HOSTADDR" \
|
||||
--pg-port "$DB_PORT" --pg-database "$DB_NAME" --pg-user "$DB_USER" \
|
||||
--pg-passfile /tmp/pgpass --pg-ca /run/postgres/ca.crt
|
||||
env:
|
||||
- name: DB_HOST
|
||||
valueFrom: {secretKeyRef: {name: aevolution-db, key: PGHOST}}
|
||||
- name: DB_HOSTADDR
|
||||
valueFrom: {secretKeyRef: {name: aevolution-db, key: PGHOSTADDR}}
|
||||
- name: DB_PORT
|
||||
valueFrom: {secretKeyRef: {name: aevolution-db, key: PGPORT}}
|
||||
- name: DB_NAME
|
||||
valueFrom: {secretKeyRef: {name: aevolution-db, key: PGDATABASE}}
|
||||
- name: DB_USER
|
||||
valueFrom: {secretKeyRef: {name: evolution-postgres-runtime, key: user}}
|
||||
resources:
|
||||
requests: {cpu: 50m, memory: 64Mi}
|
||||
limits: {memory: 256Mi}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities: {drop: [ALL]}
|
||||
volumeMounts:
|
||||
- {name: objects, mountPath: /objects}
|
||||
- {name: connection, mountPath: /run/postgres, readOnly: true}
|
||||
- {name: scratch, mountPath: /tmp}
|
||||
volumes:
|
||||
- name: objects
|
||||
persistentVolumeClaim:
|
||||
claimName: aevolution-objects
|
||||
- name: scratch
|
||||
emptyDir: {medium: Memory, sizeLimit: 1Mi}
|
||||
- name: connection
|
||||
secret:
|
||||
secretName: evolution-postgres-runtime
|
||||
defaultMode: 0440
|
||||
items:
|
||||
- {key: pgpass, path: pgpass}
|
||||
- {key: ca.crt, path: ca.crt}
|
||||
@@ -7,6 +7,7 @@ names it. Remove an item only when it is done, and record where it was done.
|
||||
| Item | Raised | Why deferred | Needed before |
|
||||
|---|---|---|---|
|
||||
| **World0 break-glass membership.** Decide who may act as `aevolution:world0-break-glass`, and how that group is granted: for example a short-lived client certificate with that organization, or an OIDC group. Decide also how its use is approved and recorded. The admission policy already enforces the group (ADR-0069). | M8.2, 2026-09-29 | Maintainer: figure out later | Any intentional deletion of World0 data or identity; M8.8 World0 launch |
|
||||
| **Backup destination for dev/stage stores.** Decide where backups go: the AEvolution PostgreSQL VM (WAL archive or base backups) and each environment's `aevolution-objects` volume. It should be off the host for any claim beyond disposable environments. | M8.3, 2026-09-29 | Needs the maintainer's choice of target | M8.7 backup and restore; any World0 data |
|
||||
| **Storage after the RAID/disk rework.** Re-run `deploy/kubernetes/inventory/disk-probe-local-path.yaml` (or a new class), and update INVENTORY.md. | M8.1, 2026-09-29 | Maintainer is reorganizing disks | M8.3's final storage decision |
|
||||
|
||||
## Accepted risks (maintainer decisions)
|
||||
|
||||
@@ -616,3 +616,11 @@ M8.2 designed and checked offline (2026-09-29), under ADR-0069:
|
||||
- an offline policy check in CI, and a live isolation check.
|
||||
The live checks run once the maintainer applies the platform layer. Next: M8.3,
|
||||
durable stores in dev and stage.
|
||||
|
||||
M8.3 implemented and locally qualified (2026-09-29), under ADR-0070:
|
||||
- each environment's database lives on the external PostgreSQL VM;
|
||||
- object volumes are node-local and use the Retain class;
|
||||
- a free-space reserve with a distinct full-volume refusal;
|
||||
- `evo-world-clone check-store` for in-cluster qualification.
|
||||
The in-cluster evidence and the backup destination wait on the maintainer
|
||||
(docs/BACKLOG.md).
|
||||
|
||||
@@ -1015,3 +1015,24 @@ M8.2 applied and verified live (2026-09-29): 175 PASS and 2 SKIP. The only 3 FAI
|
||||
were a script defect in the project check, now fixed. The maintainer accepted
|
||||
leaving Argo's `default` project unrestricted. Break-glass membership is in the
|
||||
new docs/BACKLOG.md. Next: M8.3, durable stores in dev and stage.
|
||||
|
||||
## M8.3 dev/stage stores (2026-09-29)
|
||||
|
||||
ADR-0070 sets up the dev and stage stores:
|
||||
- **Database:** each environment gets its own database and logins on the external
|
||||
PostgreSQL VM.
|
||||
- **Object store:** the filesystem store stays, on a node-local PVC in the new
|
||||
Retain class `aevolution-local-retain`, pinned to one labelled worker. There is
|
||||
no object server.
|
||||
- **Capacity:** node-local volumes ignore their requested size, so the object
|
||||
store now enforces a free-space reserve (`--objects-reserve-mib`, default 1024).
|
||||
A full volume refuses new objects with a distinct error, and nothing is
|
||||
acknowledged.
|
||||
- **Qualification:** `evo-world-clone check-store`, now in the server image,
|
||||
checks from inside the cluster: volume presence and retention, the reserve,
|
||||
database version and TLS, and the runtime privilege boundary.
|
||||
|
||||
Not yet done:
|
||||
- the in-cluster run (it needs Secrets with the documented key names and
|
||||
published image digests);
|
||||
- choosing a backup destination (docs/BACKLOG.md).
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
# ADR-0070: Durable stores for dev and stage
|
||||
|
||||
## Status
|
||||
Accepted for M8.3, 2026-09-29. It builds on ADR-0059 (PostgreSQL), ADR-0061
|
||||
(object store), ADR-0069 (environments) and the M8.1 inventory. Storage decisions
|
||||
are provisional until the planned RAID and disk rework (docs/BACKLOG.md).
|
||||
|
||||
## Context
|
||||
Inventory facts that drive this decision:
|
||||
- PostgreSQL runs on a separate VM outside Kubernetes (maintainer). The adapter
|
||||
already requires exactly 18.6 and `verify-full` TLS off-host, and the migration
|
||||
audits the runtime role.
|
||||
- The cluster's only StorageClass, `local-path`, is node-local, reclaims with
|
||||
Delete and does not enforce requested sizes.
|
||||
- Object publication on the workers measured 29–36 ms mean (73 ms max). Worker-3's
|
||||
root filesystem is 87 % full.
|
||||
- MinIO in the cluster is unpinned (`latest`) and belongs to another project.
|
||||
|
||||
The roadmap asks for:
|
||||
- a supported PostgreSQL and a maintained object store, with pinned versions;
|
||||
- PV retention and a backup destination;
|
||||
- evidence for pod restart, missing-volume refusal, object semantics, credential
|
||||
rotation and full disk.
|
||||
|
||||
The named risk is that a local PV binds failover to a node that has been lost.
|
||||
|
||||
## Decision
|
||||
**Database.** Use the maintainer's PostgreSQL VM. Each environment gets its own
|
||||
database, and its own migrator login and NOLOGIN runtime group with a member login
|
||||
(ADR-0060). The version pin (18.6), TLS verification and privilege audit are
|
||||
enforced by code at connect and migrate time; they are not assumed. Migrations run
|
||||
as a hand-applied Job using the pinned `postgres-tools` image.
|
||||
|
||||
**Object store.** Keep the ADR-0061 filesystem store. There is **no object
|
||||
server**: MinIO is ruled out as a new default (INFRASTRUCTURE_ARCHITECTURE), and an
|
||||
S3 adapter would be a new dependency. Instead:
|
||||
- The store lives on one PVC per environment, `aevolution-objects`, in the new
|
||||
class `aevolution-local-retain` (the same provisioner, with **Retain** and
|
||||
WaitForFirstConsumer; never the default class).
|
||||
- The PVC is platform-owned (hand-applied, never Argo).
|
||||
- Every consumer selects the node label `aevolution.io/stores=true`, which is put
|
||||
on exactly one worker. worker-1 or worker-2 is recommended; never worker-3 or
|
||||
the control plane.
|
||||
|
||||
**Capacity guard.** `local-path` ignores the requested size, so
|
||||
`FilesystemObjectStore` now takes a free-space reserve. `evo-server` exposes it as
|
||||
`--objects-reserve-mib`, default 1024.
|
||||
- A new object that would leave less free space is refused with a distinct
|
||||
`ObjectError::full`, before anything is written.
|
||||
- Identical objects that already exist stay reusable.
|
||||
- The host treats a refused upload as a retryable store failure, so nothing is
|
||||
published or acknowledged, and a drain that cannot publish ends without
|
||||
committing anything (exit 3).
|
||||
- A genuinely failing write (ENOSPC or EFBIG) is `io`, and leaves no temporary
|
||||
file or object behind.
|
||||
|
||||
**Qualification tool.** `evo-world-clone check-store` runs in the cluster as a Job
|
||||
with the runtime identity. It:
|
||||
- refuses a missing or unmounted object root;
|
||||
- creates a fixed 64 KiB probe object once, then verifies it by digest on every
|
||||
later run (proof of retention across pods);
|
||||
- reports free space against the reserve;
|
||||
- reports the server version, user and TLS;
|
||||
- requires that the runtime login reaches the fenced functions but is denied
|
||||
table access.
|
||||
|
||||
The server image now includes the tool.
|
||||
|
||||
**Secret contract.**
|
||||
- `aevolution-db` supplies only `PGHOST`, `PGHOSTADDR`, `PGPORT` and `PGDATABASE`.
|
||||
- `evolution-postgres-runtime` supplies `user`, `pgpass` and `ca.crt`.
|
||||
- `evolution-postgres-migrator` supplies `pg_service.conf`, `pgpass` and `ca.crt`.
|
||||
|
||||
Passwords are only ever mounted as files, and copied into private memory
|
||||
(mode 0600) before use, because libpq ignores group-readable password files. The
|
||||
policy check forbids `PGPASSWORD`/`PGUSER` references and whole-Secret `envFrom`.
|
||||
|
||||
**Retention and backup.**
|
||||
- Deleting the claim leaves the PV Released with its data. Recovery is an explicit
|
||||
rebind by an administrator, verified by `check-store` ("probe verified").
|
||||
- The backup destination for the database VM and the object PVC is **not decided
|
||||
yet**. It is a maintainer decision, recorded in docs/BACKLOG.md and required
|
||||
before M8.7. Dev and stage run without a verified backup, which is acceptable
|
||||
only because they are disposable. World0 remains blocked.
|
||||
|
||||
## Consequences
|
||||
- A lost worker takes its object volume with it. The dev and stage regions on that
|
||||
worker are then unavailable (the server refuses a missing volume and quarantines
|
||||
missing objects); they are never silently re-initialized. This is the named risk,
|
||||
accepted for dev and stage only. World0 needs replicated or off-node storage
|
||||
(M8.7/M8.8).
|
||||
- The reserve and the kubelet's nodefs eviction threshold interact. Set the reserve
|
||||
above the node's remaining headroom target, and alert on free space (M8.5).
|
||||
- Re-probe and re-decide after the disk rework.
|
||||
|
||||
## Alternatives rejected
|
||||
- **MinIO or another object server.** It adds an unpinned or new dependency, a
|
||||
second durability layer to back up, and network latency, for no gain on one
|
||||
node.
|
||||
- **Leaving `local-path`/Delete for durable data.** Deleting the claim or an Argo
|
||||
prune would erase the store.
|
||||
- **Trusting PVC sizes.** `local-path` does not enforce them.
|
||||
@@ -80,3 +80,4 @@ design. No unadopted candidate is a link dependency of the proof.
|
||||
- [ADR-0067: Region clones and retained-history audit](0067-region-clones.md) - accepted for M7.9a re-identified clone genesis naming its source cut, isolated clone databases and full-chain retention audit.
|
||||
- [ADR-0068: Upgrade barrier and durability qualification](0068-upgrade-barrier.md) - accepted for M7.9b build-bound authority, retirement barrier fencing old writers, explicit refusal of rule/format changes and the measured recovery fixture.
|
||||
- [ADR-0069: Environment isolation and World0 protection](0069-environment-isolation.md) - accepted for M8.2 per-environment guardrails, native World0 admission protection, scoped Argo projects and verification.
|
||||
- [ADR-0070: Durable stores for dev and stage](0070-dev-stage-stores.md) - accepted for M8.3 external PostgreSQL per environment, Retain-class node-local object volumes, free-space reserve and in-cluster store qualification.
|
||||
|
||||
@@ -2689,3 +2689,42 @@ admission policy and the stage and World0 AppProjects, then ran
|
||||
`[]`. The corrected script has not been re-run on the cluster.
|
||||
- **WARN:** the `default` AppProject allows every namespace. The maintainer
|
||||
accepted this (docs/BACKLOG.md).
|
||||
|
||||
## M8.3 dev/stage stores - 2026-09-29
|
||||
|
||||
- WSL Linux `checks` set: CTest 59/59 in each configuration. The format check
|
||||
passed.
|
||||
- PostgreSQL suites: Release and ASan/UBSan passed with the new
|
||||
`region_store_tests`. Analysis passed after one clang-tidy fix (an int
|
||||
multiplication widened in the MiB report).
|
||||
- Store tests:
|
||||
- An unreachable reserve refuses new objects with `full`, leaving no files. An
|
||||
identical existing object is still reused.
|
||||
- A lineage batch through a full store fails as `upload`/`full`, and the head is
|
||||
unchanged.
|
||||
- RLIMIT_FSIZE makes a real write fail with `io`, leaving no object or
|
||||
temporary.
|
||||
- `check-store`:
|
||||
- its first run creates the probe and its second verifies it;
|
||||
- a missing root exits 4 without creating it;
|
||||
- a full volume exits 4;
|
||||
- an owner login exits 3 (tables READABLE);
|
||||
- the runtime login reports `version=180006 ... ssl=false` (local socket), and
|
||||
`tables=denied functions=reachable`.
|
||||
- An `evo-server` whose reserve cannot be met simulated but acknowledged
|
||||
nothing. Its drain exited 3, and the head was unchanged.
|
||||
- Mutations (restored and rebuilt), each caught:
|
||||
- removing the reserve check;
|
||||
- `check-store` always succeeding.
|
||||
- `tools/ci/deploy-policy-check.py`: 41 objects, 0 failures. Four new-rule
|
||||
mutations were caught:
|
||||
- a Delete class;
|
||||
- a platform claim on `local-path`;
|
||||
- a `PGPASSWORD` key reference;
|
||||
- a whole-Secret `envFrom`.
|
||||
- Kustomize renders the dev and stage overlays with the `aevolution-objects`
|
||||
claim; World0 has none.
|
||||
- The server image now contains `evo-world-clone`, and its smoke script checks the
|
||||
tool's usage exit. It has not been built locally, because Docker is not running.
|
||||
- Not executed: anything in the cluster, the store Jobs, retention, rebind and
|
||||
rotation evidence, and the image build.
|
||||
|
||||
@@ -21,12 +21,12 @@ manual, and none is performed by Argo CD.
|
||||
Each namespace gets its own Secrets, with environment-specific values:
|
||||
- `harbor-pull`: a pull-only Harbor robot (docker-registry type).
|
||||
- `evolution-postgres-migrator`: the migrator/owner login for that environment's
|
||||
database.
|
||||
database. Keys: `pg_service.conf`, `pgpass`, `ca.crt`.
|
||||
- `evolution-postgres-runtime`: the runtime login, a member of that environment's
|
||||
NOLOGIN runtime group (ADR-0060).
|
||||
- `aevolution-db`: connection settings (host, address, port, database, TLS mode
|
||||
and root certificate). Mount it as files. The adapter reads a password file,
|
||||
never `PGPASSWORD`.
|
||||
NOLOGIN runtime group (ADR-0060). Keys: `user`, `pgpass`, `ca.crt`.
|
||||
- `aevolution-db`: connection settings. Only `PGHOST`, `PGHOSTADDR` (numeric),
|
||||
`PGPORT` and `PGDATABASE` are ever read, as individual variables. `PGPASSWORD`
|
||||
and `PGUSER` are never mounted; the policy check forbids it.
|
||||
|
||||
A World0 credential must never exist in the dev or stage namespaces, and no dev or
|
||||
stage credential may reach the World0 database. Rotate by replacing the Secret and
|
||||
@@ -52,6 +52,48 @@ bash tools/ops/verify-isolation.sh
|
||||
|
||||
Create the Secrets listed above in each namespace before any workload syncs.
|
||||
|
||||
## Stores (M8.3, ADR-0070)
|
||||
|
||||
Run this once per disposable environment (`dev`, then `stage`).
|
||||
|
||||
**1. Label the store node, and apply the storage class and claim.**
|
||||
```sh
|
||||
# Label exactly one worker (worker-1 or worker-2; never worker-3 or the control plane).
|
||||
kubectl label node rke2-worker-1 aevolution.io/stores=true
|
||||
kubectl apply -f deploy/kubernetes/platform/storage/storageclass.yaml
|
||||
kubectl apply -k deploy/kubernetes/platform/dev # adds the aevolution-objects claim
|
||||
```
|
||||
|
||||
**2. Prepare the store Jobs.** Copy `deploy/kubernetes/stores/stores.yaml.example`,
|
||||
replace its `REPLACE_*` markers (namespace, database address and port, runtime
|
||||
role, image digests), and create the Secrets above.
|
||||
|
||||
**3. Run and qualify.**
|
||||
```sh
|
||||
kubectl apply -f stores-dev.yaml # policy + both Jobs
|
||||
kubectl logs -n evolution-dev job/aevolution-db-migrate # schema and audit
|
||||
kubectl logs -n evolution-dev job/aevolution-store-check # expect exit 0 and "probe created"
|
||||
```
|
||||
|
||||
Evidence to record in docs/engineering/VALIDATION.md:
|
||||
- **Pod restart:** delete the check Job and apply it again. It must print
|
||||
`probe verified`, with the same digest.
|
||||
- **Missing volume:** run the check with the `objects` volume removed. It must exit
|
||||
4 with `objects refused missing-or-not-a-directory`.
|
||||
- **Full volume:** add `--reserve-mib 1048576` and use a new probe region, such as
|
||||
`--region 2`. It must exit 4 with `probe refused full`.
|
||||
- **Retention:** `kubectl delete pvc aevolution-objects -n evolution-dev`. The PV
|
||||
must become `Released`, not deleted. To rebind:
|
||||
- clear the PV's `spec.claimRef`;
|
||||
- recreate the claim with `volumeName` set to that PV;
|
||||
- rerun the check, which must print `probe verified`.
|
||||
- **Credential rotation:**
|
||||
- Change the runtime login's password on the database VM, and run the check
|
||||
before updating the Secret. It must exit 3 with `database refused connect`.
|
||||
- Update the Secret's `pgpass` and rerun. It must exit 0.
|
||||
- **Privilege boundary:** every run must print
|
||||
`database tables=denied functions=reachable`.
|
||||
|
||||
## Break-glass
|
||||
|
||||
How membership is granted is not decided yet; see docs/BACKLOG.md. Until it is,
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#include <expected>
|
||||
#include <filesystem>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <span>
|
||||
#include <vector>
|
||||
|
||||
@@ -19,6 +20,7 @@ enum class ObjectError : std::uint8_t {
|
||||
corrupt, // Length or digest mismatch, or not a regular file: quarantine.
|
||||
bounds, // Larger than the configured maximum; rejected before allocation.
|
||||
io, // Write/sync/link failure; the object is absent or unverified.
|
||||
full, // Writing would leave less free space than the reserve; nothing written.
|
||||
};
|
||||
// Test-only fault injection and accounting for directory fsyncs. When passed to
|
||||
// open() it must outlive the store; production callers pass nothing.
|
||||
@@ -33,17 +35,22 @@ struct VerifiedObject {
|
||||
};
|
||||
class FilesystemObjectStore {
|
||||
public:
|
||||
// trusted_root must be an existing real directory (not a symlink). Namespace
|
||||
// directories w<world>/r<region> are created 0700; their parents are synced on
|
||||
// every open, so a creation whose sync failed earlier is made durable now.
|
||||
// trusted_root must be an existing real directory (not a symlink): a missing or
|
||||
// unmounted volume is refused, never created. Namespace directories
|
||||
// w<world>/r<region> are created 0700; their parents are synced on every open, so
|
||||
// a creation whose sync failed earlier is made durable now. reserve_bytes (M8.3):
|
||||
// put() refuses new objects that would leave less free space than this on the
|
||||
// volume, because node-local volumes do not enforce their requested size.
|
||||
[[nodiscard]] static std::expected<std::unique_ptr<FilesystemObjectStore>, ObjectError>
|
||||
open(const std::filesystem::path& trusted_root, RegionKey key, std::size_t max_object_bytes,
|
||||
ObjectStoreFaults* faults = nullptr);
|
||||
ObjectStoreFaults* faults = nullptr, std::uint64_t reserve_bytes = 0);
|
||||
FilesystemObjectStore(const FilesystemObjectStore&) = delete;
|
||||
FilesystemObjectStore& operator=(const FilesystemObjectStore&) = delete;
|
||||
FilesystemObjectStore(FilesystemObjectStore&&) = delete;
|
||||
FilesystemObjectStore& operator=(FilesystemObjectStore&&) = delete;
|
||||
~FilesystemObjectStore();
|
||||
// Refuses with `full`, before writing anything, when the volume's available
|
||||
// space minus the object is below the reserve.
|
||||
// Writes a private temporary file, fsyncs it, links it to its key without
|
||||
// replacing anything, then fsyncs the directory. Identical bytes already stored
|
||||
// (an earlier attempt whose directory sync failed, or a concurrent installer)
|
||||
@@ -57,15 +64,21 @@ class FilesystemObjectStore {
|
||||
[[nodiscard]] const std::filesystem::path& directory() const noexcept {
|
||||
return directory_;
|
||||
}
|
||||
// Bytes available to this process on the volume; nullopt if unknown.
|
||||
[[nodiscard]] std::optional<std::uint64_t> available_bytes() const noexcept;
|
||||
[[nodiscard]] std::uint64_t reserve_bytes() const noexcept {
|
||||
return reserve_;
|
||||
}
|
||||
|
||||
private:
|
||||
FilesystemObjectStore(std::filesystem::path directory, int descriptor, std::size_t maximum,
|
||||
ObjectStoreFaults* faults);
|
||||
ObjectStoreFaults* faults, std::uint64_t reserve);
|
||||
[[nodiscard]] std::expected<VerifiedObject, ObjectError>
|
||||
durable_existing(const VerifiedObject& object) const;
|
||||
std::filesystem::path directory_;
|
||||
int descriptor_;
|
||||
std::size_t maximum_;
|
||||
ObjectStoreFaults* faults_;
|
||||
std::uint64_t reserve_;
|
||||
};
|
||||
} // namespace evo::persistence
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#include <fcntl.h>
|
||||
#include <string>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/statvfs.h>
|
||||
#include <unistd.h>
|
||||
|
||||
namespace evo::persistence {
|
||||
@@ -128,9 +129,17 @@ read_verified(int directory, const VerifiedObject& object, bool sync_file = fals
|
||||
} // namespace
|
||||
|
||||
FilesystemObjectStore::FilesystemObjectStore(std::filesystem::path directory, int descriptor,
|
||||
std::size_t maximum, ObjectStoreFaults* faults)
|
||||
: directory_(std::move(directory)), descriptor_(descriptor), maximum_(maximum),
|
||||
faults_(faults) {}
|
||||
std::size_t maximum, ObjectStoreFaults* faults,
|
||||
std::uint64_t reserve)
|
||||
: directory_(std::move(directory)), descriptor_(descriptor), maximum_(maximum), faults_(faults),
|
||||
reserve_(reserve) {}
|
||||
|
||||
std::optional<std::uint64_t> FilesystemObjectStore::available_bytes() const noexcept {
|
||||
struct statvfs info{};
|
||||
if (::fstatvfs(descriptor_, &info) != 0)
|
||||
return std::nullopt;
|
||||
return static_cast<std::uint64_t>(info.f_bavail) * static_cast<std::uint64_t>(info.f_frsize);
|
||||
}
|
||||
|
||||
FilesystemObjectStore::~FilesystemObjectStore() {
|
||||
static_cast<void>(::close(descriptor_));
|
||||
@@ -138,7 +147,8 @@ FilesystemObjectStore::~FilesystemObjectStore() {
|
||||
|
||||
std::expected<std::unique_ptr<FilesystemObjectStore>, ObjectError>
|
||||
FilesystemObjectStore::open(const std::filesystem::path& trusted_root, RegionKey key,
|
||||
std::size_t max_object_bytes, ObjectStoreFaults* faults) {
|
||||
std::size_t max_object_bytes, ObjectStoreFaults* faults,
|
||||
std::uint64_t reserve_bytes) {
|
||||
if (key.world == 0 || key.region == 0 || max_object_bytes == 0 ||
|
||||
max_object_bytes > maximum_object_limit || trusted_root.empty())
|
||||
return std::unexpected{ObjectError::invalid};
|
||||
@@ -154,8 +164,9 @@ FilesystemObjectStore::open(const std::filesystem::path& trusted_root, RegionKey
|
||||
Descriptor region_fd{subdirectory(world_fd.get(), region, faults)};
|
||||
if (region_fd.get() < 0)
|
||||
return std::unexpected{ObjectError::io};
|
||||
return std::unique_ptr<FilesystemObjectStore>(new FilesystemObjectStore(
|
||||
trusted_root / world / region, region_fd.release(), max_object_bytes, faults));
|
||||
return std::unique_ptr<FilesystemObjectStore>(
|
||||
new FilesystemObjectStore(trusted_root / world / region, region_fd.release(),
|
||||
max_object_bytes, faults, reserve_bytes));
|
||||
}
|
||||
|
||||
std::expected<VerifiedObject, ObjectError>
|
||||
@@ -171,6 +182,12 @@ FilesystemObjectStore::put(std::span<const std::uint8_t> bytes) const {
|
||||
// Immutable: identical content is reused (durably); anything else is corrupt.
|
||||
if (auto found = durable_existing(object); found || found.error() != ObjectError::missing)
|
||||
return found;
|
||||
// Only new bytes need space; checked before anything is written.
|
||||
const auto available = available_bytes();
|
||||
if (!available)
|
||||
return std::unexpected{ObjectError::io};
|
||||
if (*available < reserve_ || *available - reserve_ < bytes.size())
|
||||
return std::unexpected{ObjectError::full};
|
||||
|
||||
const auto name = key_name(*digest);
|
||||
const auto temporary = ".tmp-" + name + "-" + std::to_string(::getpid()) + "-" +
|
||||
|
||||
@@ -11,6 +11,7 @@ void run_region_recovery_tests();
|
||||
void run_region_ops_tests();
|
||||
void run_region_clone_tests();
|
||||
void run_region_migration_tests();
|
||||
void run_region_store_tests();
|
||||
namespace {
|
||||
using namespace evo::persistence;
|
||||
using namespace std::chrono_literals;
|
||||
@@ -91,6 +92,8 @@ int main() {
|
||||
std::puts("region ops integration passed");
|
||||
run_region_clone_tests();
|
||||
std::puts("region clone and history integration passed");
|
||||
run_region_store_tests();
|
||||
std::puts("region store qualification integration passed");
|
||||
run_region_migration_tests();
|
||||
std::puts("region migration, barrier and qualification integration passed");
|
||||
return 0;
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
// M8.3 store qualification (ADR-0070) against real PostgreSQL and real processes:
|
||||
// the object volume's free-space reserve, a genuinely failing write, the
|
||||
// check-store probe (retention across runs, missing-volume refusal, privilege
|
||||
// boundary) and a server whose volume is full. Run by tools/ci/postgres-check.sh.
|
||||
#include "evo/persistence/checkpoint_store.hpp"
|
||||
#include "fixtures/server_process.hpp"
|
||||
#include <csignal>
|
||||
#include <source_location>
|
||||
#include <sys/resource.h>
|
||||
|
||||
void run_region_store_tests();
|
||||
|
||||
namespace {
|
||||
using namespace evo::test::process;
|
||||
using namespace evo::persistence;
|
||||
void check(bool value, std::source_location at = std::source_location::current()) {
|
||||
if (!value)
|
||||
throw std::runtime_error("region store test line " + std::to_string(at.line()));
|
||||
}
|
||||
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
||||
constexpr std::size_t budget = 8000000;
|
||||
constexpr std::uint64_t unreachable_reserve = std::uint64_t{1} << 60U; // 1 EiB.
|
||||
std::unique_ptr<FilesystemObjectStore> store(const fs::path& root, RegionKey key,
|
||||
std::uint64_t reserve = 0) {
|
||||
auto made = FilesystemObjectStore::open(root, key, max_lineage_payload_bytes, nullptr, reserve);
|
||||
CHECK(made);
|
||||
return std::move(*made);
|
||||
}
|
||||
fs::path fresh(const std::string& name) {
|
||||
const auto root = fs::path(env("PGHOST")) / name;
|
||||
fs::remove_all(root);
|
||||
fs::create_directory(root);
|
||||
fs::permissions(root, fs::perms::owner_all, fs::perm_options::replace);
|
||||
return root;
|
||||
}
|
||||
std::size_t entries(const fs::path& dir) {
|
||||
std::size_t n = 0;
|
||||
for ([[maybe_unused]] const auto& entry : fs::directory_iterator(dir))
|
||||
++n;
|
||||
return n;
|
||||
}
|
||||
std::vector<std::uint8_t> bytes_of(std::size_t size, std::uint8_t seed) {
|
||||
std::vector<std::uint8_t> out(size);
|
||||
for (std::size_t i = 0; i < size; ++i)
|
||||
out[i] = static_cast<std::uint8_t>(seed + (i * 7U));
|
||||
return out;
|
||||
}
|
||||
|
||||
// A full volume refuses new objects before writing; identical objects already
|
||||
// stored stay usable; a batch against a full volume publishes nothing.
|
||||
void full_volumes_publish_nothing() {
|
||||
const RegionKey key{7, 81};
|
||||
const auto root = fresh("store-full");
|
||||
const auto stored = bytes_of(4096, 0x10);
|
||||
auto roomy = store(root, key);
|
||||
CHECK(roomy->available_bytes().value_or(0) > 0 && roomy->reserve_bytes() == 0);
|
||||
CHECK(roomy->put(stored));
|
||||
auto full = store(root, key, unreachable_reserve);
|
||||
CHECK(full->reserve_bytes() == unreachable_reserve);
|
||||
const auto before = entries(full->directory());
|
||||
auto refused = full->put(bytes_of(4096, 0x20));
|
||||
CHECK(!refused && refused.error() == ObjectError::full);
|
||||
CHECK(entries(full->directory()) == before); // No temporary, no object.
|
||||
CHECK(full->put(stored)); // Reuse needs no space.
|
||||
|
||||
// A lineage batch cannot publish through a full volume; the head stays.
|
||||
const auto r = provision(82, 0xDE);
|
||||
auto db = owner();
|
||||
const auto head = load_region_checkpoint(*db, r.key, 5s);
|
||||
CHECK(head);
|
||||
auto runtime = Postgres::connect(
|
||||
{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("EVO_PG_RUNTIME_USER"), {}, {}},
|
||||
5s);
|
||||
CHECK(runtime);
|
||||
Sha256Digest build{};
|
||||
build.fill(0xB1);
|
||||
auto lease = acquire_region_lease(**runtime, r.key, build, 20000ms, 5s);
|
||||
CHECK(lease);
|
||||
auto state = evo::test::lineage_region();
|
||||
RegionBatch<evo::processes::LineageRegion> batch{*state};
|
||||
for (int i = 0; i < 10; ++i)
|
||||
CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1)));
|
||||
batch.seal();
|
||||
Uuid commit{};
|
||||
commit.fill(0xDF);
|
||||
auto blocked = store(r.root, r.key, unreachable_reserve);
|
||||
auto published =
|
||||
publish_lineage_batch(**runtime, *blocked, *lease, *head, batch, commit, build, budget, 5s);
|
||||
CHECK(!published && published.error().kind == PublicationError::upload &&
|
||||
published.error().object == ObjectError::full);
|
||||
CHECK(load_region_checkpoint(*db, r.key, 5s)->head == head->head);
|
||||
CHECK(release_region_lease(**runtime, r.key, *lease, 5s));
|
||||
}
|
||||
|
||||
// A write the filesystem really refuses (RLIMIT_FSIZE stands in for a full disk:
|
||||
// the kernel rejects the write mid-object) is an io error that leaves no object
|
||||
// and no temporary behind.
|
||||
void failing_writes_leave_nothing() {
|
||||
const auto root = fresh("store-efbig");
|
||||
auto objects = store(root, {7, 83});
|
||||
rlimit old{};
|
||||
CHECK(::getrlimit(RLIMIT_FSIZE, &old) == 0);
|
||||
const auto previous = std::signal(SIGXFSZ, SIG_IGN);
|
||||
rlimit limited = old;
|
||||
limited.rlim_cur = 8192;
|
||||
CHECK(::setrlimit(RLIMIT_FSIZE, &limited) == 0);
|
||||
auto refused = objects->put(bytes_of(65536, 0x30));
|
||||
CHECK(::setrlimit(RLIMIT_FSIZE, &old) == 0);
|
||||
std::signal(SIGXFSZ, previous);
|
||||
CHECK(!refused && refused.error() == ObjectError::io);
|
||||
CHECK(entries(objects->directory()) == 0);
|
||||
CHECK(objects->put(bytes_of(65536, 0x30))); // The same write succeeds with room.
|
||||
}
|
||||
|
||||
std::vector<std::string> check_store(const fs::path& objects, const std::string& user,
|
||||
std::vector<std::string> extra, int& code) {
|
||||
std::vector<std::string> args{env("EVO_WORLD_CLONE_BIN"),
|
||||
"check-store",
|
||||
"--world",
|
||||
"7",
|
||||
"--region",
|
||||
"84",
|
||||
"--pg-host",
|
||||
env("PGHOST"),
|
||||
"--pg-port",
|
||||
env("PGPORT"),
|
||||
"--pg-database",
|
||||
env("PGDATABASE"),
|
||||
"--pg-user",
|
||||
user,
|
||||
"--objects",
|
||||
objects.string()};
|
||||
args.insert(args.end(), extra.begin(), extra.end());
|
||||
Child tool{args};
|
||||
code = tool.wait_exit(30s);
|
||||
return tool.lines();
|
||||
}
|
||||
bool has(const std::vector<std::string>& lines, std::string_view prefix) {
|
||||
for (const auto& l : lines)
|
||||
if (l.starts_with(prefix))
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
// The in-cluster qualification probe, exercised locally.
|
||||
void check_store_qualifies_the_stores() {
|
||||
const auto root = fresh("store-check");
|
||||
const auto runtime = env("EVO_PG_RUNTIME_USER");
|
||||
int code = -1;
|
||||
auto first = check_store(root, runtime, {}, code);
|
||||
CHECK(code == 0 && has(first, "objects available_mib=") && has(first, "probe created digest="));
|
||||
CHECK(has(first, "database version=180006 user=" + runtime + " ssl=false"));
|
||||
CHECK(has(first, "database tables=denied functions=reachable head=none"));
|
||||
// A later run (a restarted or rescheduled pod) verifies the retained probe.
|
||||
auto second = check_store(root, runtime, {}, code);
|
||||
CHECK(code == 0 && has(second, "probe verified digest="));
|
||||
// A missing or unmounted volume is refused, never created.
|
||||
auto missing = check_store(root / "not-mounted", runtime, {}, code);
|
||||
CHECK(code == 4 && has(missing, "objects refused missing-or-not-a-directory"));
|
||||
CHECK(!fs::exists(root / "not-mounted"));
|
||||
// A full volume refuses the probe (fresh namespace, reserve beyond any disk).
|
||||
const auto other = fresh("store-check-full");
|
||||
auto full = check_store(other, runtime, {"--reserve-mib", "1048576"}, code);
|
||||
CHECK(code == 4 && has(full, "probe refused full"));
|
||||
// An owner login can read tables: not an acceptable runtime identity.
|
||||
auto owner_run = check_store(root, env("PGUSER"), {}, code);
|
||||
CHECK(code == 3 && has(owner_run, "database tables=READABLE"));
|
||||
}
|
||||
|
||||
// A real server on a full volume keeps simulating but never acknowledges, and its
|
||||
// bounded drain gives up without committing anything.
|
||||
void full_volume_server_acknowledges_nothing() {
|
||||
const auto r = provision(85, 0xE7);
|
||||
const auto before = head_of(r.key);
|
||||
Child server{server_args(r, {"--objects-reserve-mib", "1048576", "--drain-ms", "1500",
|
||||
"--ops-listen", "127.0.0.1:0"})};
|
||||
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
||||
server.send(input_lines(1, 15, false));
|
||||
server.pump(3000ms);
|
||||
CHECK(!server.printed("ack "));
|
||||
server.signal(SIGTERM);
|
||||
CHECK(server.wait_exit(20s) == 3); // Drain deadline passed with the batch unpublished.
|
||||
CHECK(!server.printed("ack "));
|
||||
const auto after = head_of(r.key);
|
||||
CHECK(after.tick == before.tick && after.sequence == before.sequence);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void run_region_store_tests() {
|
||||
full_volumes_publish_nothing();
|
||||
failing_writes_leave_nothing();
|
||||
check_store_qualifies_the_stores();
|
||||
full_volume_server_acknowledges_nothing();
|
||||
}
|
||||
@@ -145,6 +145,31 @@ for path, doc in objects:
|
||||
check(destination != 'evolution-world0' or name == 'aevolution-world0',
|
||||
f'{path}: only aevolution-world0 may target evolution-world0')
|
||||
|
||||
# M8.3 storage (ADR-0070): durable volumes retain their data; database passwords are
|
||||
# only ever mounted as files, never placed in the environment.
|
||||
classes = {d['metadata']['name']: d for p, d in objects if d['kind'] == 'StorageClass'}
|
||||
retain = classes.get('aevolution-local-retain')
|
||||
check(retain is not None and retain.get('reclaimPolicy') == 'Retain'
|
||||
and retain.get('volumeBindingMode') == 'WaitForFirstConsumer'
|
||||
and 'storageclass.kubernetes.io/is-default-class' not in retain['metadata'].get('annotations', {}),
|
||||
'aevolution-local-retain must be a non-default Retain class that binds on first consumer')
|
||||
for path, doc in objects:
|
||||
if doc['kind'] == 'PersistentVolumeClaim' and path.startswith(platform + '/'):
|
||||
check(doc['spec'].get('storageClassName') == 'aevolution-local-retain',
|
||||
f'{path}: platform volumes must use aevolution-local-retain')
|
||||
if doc['kind'] in ('Job', 'Deployment', 'StatefulSet', 'Pod'):
|
||||
pod = doc['spec'].get('template', {}).get('spec', doc['spec'])
|
||||
for container in pod.get('containers', []) + pod.get('initContainers', []):
|
||||
check(not any('secretRef' in source for source in container.get('envFrom', [])),
|
||||
f'{path}: never import a whole Secret into the environment')
|
||||
for variable in container.get('env', []):
|
||||
ref = variable.get('valueFrom', {}).get('secretKeyRef', {})
|
||||
check(ref.get('key') not in ('PGPASSWORD', 'POSTGRES_PASSWORD', 'PGUSER', 'POSTGRES_USER'),
|
||||
f'{path}: {variable.get("name")} must not come from {ref.get("key")}')
|
||||
if pod.get('volumes') and 'store' in doc['metadata'].get('name', ''):
|
||||
check(doc['metadata'].get('namespace') != 'evolution-world0',
|
||||
f'{path}: store Jobs never target World0')
|
||||
|
||||
# The live Argo path.
|
||||
for path, doc in objects:
|
||||
if path.startswith('deploy/kubernetes/dev/'):
|
||||
|
||||
@@ -15,11 +15,19 @@ code=$?
|
||||
set -e
|
||||
test "$code" -eq 2
|
||||
grep -q '^usage: evo-server ' <<<"$usage"
|
||||
# The bundled operations tool refuses to run without a subcommand too (M8.3).
|
||||
set +e
|
||||
tool="$(run --entrypoint /usr/local/bin/evo-world-clone "$image" 2>&1)"
|
||||
code=$?
|
||||
set -e
|
||||
test "$code" -eq 2
|
||||
grep -q '^usage: evo-world-clone ' <<<"$tool"
|
||||
# Non-root, every shared library resolved, no credential material in the image.
|
||||
run --entrypoint /bin/sh "$image" -c '
|
||||
set -eu
|
||||
test "$(id -u)" = 10001 && test "$(id -g)" = 10001
|
||||
! ldd /usr/local/bin/evo-server | grep -q "not found"
|
||||
! ldd /usr/local/bin/evo-world-clone | grep -q "not found"
|
||||
test -z "$(find / -xdev \( -name "*pgpass*" -o -name "pg_service.conf" -o -name "*.pem" \
|
||||
-o -name "*.key" \) -not -path "/etc/ssl/*" -not -path "/usr/share/*" -print 2>/dev/null)"
|
||||
'
|
||||
|
||||
Reference in New Issue
Block a user