Add M8.3 dev/stage stores: Retain volumes, free-space reserve, store check

Keep the filesystem object store on a node-local Retain-class PVC per
environment (ADR-0070), with the database on the external PostgreSQL VM.
Because local-path does not enforce sizes, the object store now refuses
new objects below a free-space reserve with a distinct full error, so a
full volume publishes and acknowledges nothing. evo-world-clone gains
check-store to qualify volume retention, reserve, database version/TLS
and the runtime privilege boundary from inside the cluster; it now ships
in the server image. Add the storage class, object claims, hand-applied
store Jobs, policy rules forbidding password env vars, and the runbook.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-09-29 13:51:15 -04:00
co-authored by Claude Opus 5.5
parent 9d9e00c307
commit 2d950060db
23 changed files with 833 additions and 24 deletions
+10 -2
View File
@@ -11,6 +11,10 @@
// stats max_queued=<n> max_tick_debt=<n> commits=<n> store_failures=<n>
// Exit: 0 drained, 2 usage, 3 drain deadline passed, 4 quarantined.
//
// --objects-reserve-mib <n> (M8.3, default 1024) refuses new objects that would leave
// less free space on the object volume: node-local volumes do not enforce their
// requested size. A refused upload is a store failure; nothing is acknowledged.
//
// --ops-listen <numeric address:port> (M7.8) serves GET /livez (the loop is alive; says
// nothing about authority), /readyz (200 only with current authority) and /metrics
// (Prometheus text). It prints "ops listening <port>" once bound.
@@ -54,6 +58,7 @@ struct Options {
std::string crash_point;
std::uint64_t crash_ordinal{};
std::string ops_listen;
std::uint64_t objects_reserve_mib{1024};
};
// Kills this process without any cleanup: the real SIGKILL, at a chosen boundary.
@@ -148,6 +153,8 @@ std::optional<Options> parse(int argc, char** argv) {
o.host.lease = {};
else if (flag == "--ops-listen")
o.ops_listen = value;
else if (flag == "--objects-reserve-mib" && n && *n <= 1048576)
o.objects_reserve_mib = *n;
else if (flag == "--crash-at") {
const auto colon = value.find(':');
const auto ordinal =
@@ -263,8 +270,9 @@ template <typename State> int serve(Options& o) {
};
}
std::uint64_t ack_steps = 0;
auto objects = persistence::FilesystemObjectStore::open(o.objects, o.host.key,
persistence::max_lineage_payload_bytes);
auto objects = persistence::FilesystemObjectStore::open(
o.objects, o.host.key, persistence::max_lineage_payload_bytes, nullptr,
o.objects_reserve_mib * 1024 * 1024);
if (!objects) {
std::puts("error objects");
return 2;
+107 -5
View File
@@ -7,6 +7,8 @@
// [--audit-max N] [--crash-at before-transaction]
// evo-world-clone audit --world W --region R --pg-* ... --objects DIR --retained HEX...
// [--max-commits N]
// evo-world-clone check-store --world W --region R --pg-* ... --objects DIR
// [--reserve-mib N]
//
// --*pg-* is host, hostaddr, port, database, user, passfile, ca. The source
// connection needs only the runtime read functions and is switched to read-only
@@ -18,8 +20,20 @@
//
// Output: cloned source=W/R sequence=S tick=T target=W2/R2 outcomes=N digest=HEX
// audited commits=N tick=T outcomes=N profile=P clone=0|1
// Exit: 0 done, 2 usage, 3 source/history failure, 4 target refused, 5 target unknown.
// check-store (M8.3) qualifies an environment's stores from inside the cluster with
// the runtime identity. Objects: the volume must already exist (a missing or
// unmounted volume is refused), a fixed probe object is created once and verified by
// digest on every later run (so a restarted or rescheduled pod proves retention),
// and free space is reported against the reserve. Database: the adapter enforces
// server 18.6 and verify-full TLS off-host; the runtime login must reach the fenced
// functions but be refused table access.
// Output: objects available_mib=A reserve_mib=R / probe created|verified digest=HEX
// database version=V user=U ssl=S / database tables=denied functions=reachable
//
// Exit: 0 done, 2 usage, 3 source/history/database failure, 4 target/objects refused,
// 5 target unknown.
#include "evo/persistence/region_clone.hpp"
#include "evo/persistence/sha256.hpp"
#include <array>
#include <charconv>
@@ -49,7 +63,7 @@ struct Options {
Side source, target;
std::vector<persistence::Sha256Digest> retained;
std::optional<persistence::Uuid> commit;
std::uint64_t audit_max{}, max_commits{1u << 20U};
std::uint64_t audit_max{}, max_commits{1u << 20U}, reserve_mib{1024};
bool crash{};
};
@@ -112,7 +126,7 @@ std::optional<Options> parse(int argc, char** argv) {
return std::nullopt;
Options o;
o.command = argv[1];
if (o.command != "clone" && o.command != "audit")
if (o.command != "clone" && o.command != "audit" && o.command != "check-store")
return std::nullopt;
const bool clone = o.command == "clone";
for (int i = 2; i < argc; ++i) {
@@ -150,14 +164,18 @@ std::optional<Options> parse(int argc, char** argv) {
o.audit_max = *n;
else if (clone && flag == "--crash-at" && value == "before-transaction")
o.crash = true;
else if (!clone && flag == "--max-commits" && n && *n > 0)
else if (o.command == "audit" && flag == "--max-commits" && n && *n > 0)
o.max_commits = *n;
else if (o.command == "check-store" && flag == "--reserve-mib" && n && *n <= 1048576)
o.reserve_mib = *n;
else
return std::nullopt;
}
if (clone && o.target.key.region == 0)
o.target.key.region = o.source.key.region;
if (!complete(o.source) || o.retained.empty() || (clone && !complete(o.target)))
const bool needs_builds = o.command != "check-store";
if (!complete(o.source) || (needs_builds && o.retained.empty()) ||
(clone && !complete(o.target)))
return std::nullopt;
return o;
}
@@ -236,6 +254,88 @@ std::unique_ptr<persistence::Postgres> connect(const Side& s) {
return std::move(*db);
}
// A fixed 64 KiB probe: the same bytes on every run, so its digest names it.
std::vector<std::uint8_t> probe_bytes() {
std::vector<std::uint8_t> out(65536);
const std::string_view marker = "aevolution-store-probe-v1";
for (std::size_t i = 0; i < out.size(); ++i)
out[i] = i < marker.size() ? static_cast<std::uint8_t>(marker[i])
: static_cast<std::uint8_t>((i * 131U) ^ (i >> 7U));
return out;
}
int check_store(const Options& o) {
auto store = persistence::FilesystemObjectStore::open(o.source.objects, o.source.key,
persistence::max_lineage_payload_bytes,
nullptr, o.reserve_mib * 1024 * 1024);
if (!store) {
std::printf("objects refused %s\n", store.error() == persistence::ObjectError::invalid
? "missing-or-not-a-directory"
: "io");
return 4;
}
const auto available = (*store)->available_bytes();
std::printf(
"objects available_mib=%llu reserve_mib=%llu\n",
static_cast<unsigned long long>(available.value_or(0) / (std::uint64_t{1024} * 1024)),
static_cast<unsigned long long>(o.reserve_mib));
const auto bytes = probe_bytes();
const auto digest = persistence::sha256(bytes);
if (!digest)
return 4;
const persistence::VerifiedObject probe{*digest, bytes.size()};
const char* state = "verified";
if (auto existing = (*store)->get(probe); !existing) {
if (existing.error() != persistence::ObjectError::missing) {
std::printf("probe %s\n", existing.error() == persistence::ObjectError::corrupt
? "corrupt"
: "unreadable");
return 4;
}
auto stored = (*store)->put(bytes);
if (!stored) {
std::printf("probe refused %s\n",
stored.error() == persistence::ObjectError::full ? "full" : "io");
return 4;
}
auto back = (*store)->get(probe);
if (!back || *back != bytes)
return 4;
state = "created";
}
std::printf("probe %s digest=%s\n", state, text(*digest).c_str());
std::fflush(stdout);
auto db = persistence::Postgres::connect(o.source.pg, timeout);
if (!db) {
const auto kind = db.error().kind;
std::printf("database refused %s\n", kind == persistence::PgError::version ? "version"
: kind == persistence::PgError::config ? "config"
: kind == persistence::PgError::connect ? "connect"
: "other");
return 3;
}
auto about = (*db)->query(
"SELECT pg_catalog.current_setting('server_version_num'), current_user::text, "
"coalesce((SELECT s.ssl::text || '/' || coalesce(s.version, '') FROM "
"pg_catalog.pg_stat_ssl s WHERE s.pid = pg_catalog.pg_backend_pid()), 'unknown')",
{}, timeout);
if (!about || about->size() != 1 || (*about)[0].size() != 3)
return 3;
const auto& row = (*about)[0];
std::printf("database version=%s user=%s ssl=%s\n", row[0].value_or("?").c_str(),
row[1].value_or("?").c_str(), row[2].value_or("?").c_str());
auto tables = (*db)->query("SELECT 1 FROM evolution.region_heads LIMIT 1", {}, timeout);
const bool denied = !tables && std::string_view(tables.error().sqlstate.data()) == "42501";
auto head = persistence::load_region_head(**db, o.source.key, timeout);
const bool reachable =
head || head.error().kind == persistence::LeaseStoreError::not_provisioned;
std::printf("database tables=%s functions=%s head=%s\n", denied ? "denied" : "READABLE",
reachable ? "reachable" : "unreachable",
head ? std::to_string(head->sequence).c_str() : "none");
return denied && reachable ? 0 : 3;
}
int audit(const Options& o) {
auto db = connect(o.source);
auto objects = open_objects(o.source);
@@ -329,5 +429,7 @@ int main(int argc, char** argv) {
stderr);
return 2;
}
if (options->command == "check-store")
return check_store(*options);
return options->command == "clone" ? clone(*options) : audit(*options);
}