Add M8.3 dev/stage stores: Retain volumes, free-space reserve, store check
Keep the filesystem object store on a node-local Retain-class PVC per environment (ADR-0070), with the database on the external PostgreSQL VM. Because local-path does not enforce sizes, the object store now refuses new objects below a free-space reserve with a distinct full error, so a full volume publishes and acknowledges nothing. evo-world-clone gains check-store to qualify volume retention, reserve, database version/TLS and the runtime privilege boundary from inside the cluster; it now ships in the server image. Add the storage class, object claims, hand-applied store Jobs, policy rules forbidding password env vars, and the runbook. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -11,6 +11,10 @@
|
||||
// stats max_queued=<n> max_tick_debt=<n> commits=<n> store_failures=<n>
|
||||
// Exit: 0 drained, 2 usage, 3 drain deadline passed, 4 quarantined.
|
||||
//
|
||||
// --objects-reserve-mib <n> (M8.3, default 1024) refuses new objects that would leave
|
||||
// less free space on the object volume: node-local volumes do not enforce their
|
||||
// requested size. A refused upload is a store failure; nothing is acknowledged.
|
||||
//
|
||||
// --ops-listen <numeric address:port> (M7.8) serves GET /livez (the loop is alive; says
|
||||
// nothing about authority), /readyz (200 only with current authority) and /metrics
|
||||
// (Prometheus text). It prints "ops listening <port>" once bound.
|
||||
@@ -54,6 +58,7 @@ struct Options {
|
||||
std::string crash_point;
|
||||
std::uint64_t crash_ordinal{};
|
||||
std::string ops_listen;
|
||||
std::uint64_t objects_reserve_mib{1024};
|
||||
};
|
||||
|
||||
// Kills this process without any cleanup: the real SIGKILL, at a chosen boundary.
|
||||
@@ -148,6 +153,8 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
o.host.lease = {};
|
||||
else if (flag == "--ops-listen")
|
||||
o.ops_listen = value;
|
||||
else if (flag == "--objects-reserve-mib" && n && *n <= 1048576)
|
||||
o.objects_reserve_mib = *n;
|
||||
else if (flag == "--crash-at") {
|
||||
const auto colon = value.find(':');
|
||||
const auto ordinal =
|
||||
@@ -263,8 +270,9 @@ template <typename State> int serve(Options& o) {
|
||||
};
|
||||
}
|
||||
std::uint64_t ack_steps = 0;
|
||||
auto objects = persistence::FilesystemObjectStore::open(o.objects, o.host.key,
|
||||
persistence::max_lineage_payload_bytes);
|
||||
auto objects = persistence::FilesystemObjectStore::open(
|
||||
o.objects, o.host.key, persistence::max_lineage_payload_bytes, nullptr,
|
||||
o.objects_reserve_mib * 1024 * 1024);
|
||||
if (!objects) {
|
||||
std::puts("error objects");
|
||||
return 2;
|
||||
|
||||
@@ -7,6 +7,8 @@
|
||||
// [--audit-max N] [--crash-at before-transaction]
|
||||
// evo-world-clone audit --world W --region R --pg-* ... --objects DIR --retained HEX...
|
||||
// [--max-commits N]
|
||||
// evo-world-clone check-store --world W --region R --pg-* ... --objects DIR
|
||||
// [--reserve-mib N]
|
||||
//
|
||||
// --*pg-* is host, hostaddr, port, database, user, passfile, ca. The source
|
||||
// connection needs only the runtime read functions and is switched to read-only
|
||||
@@ -18,8 +20,20 @@
|
||||
//
|
||||
// Output: cloned source=W/R sequence=S tick=T target=W2/R2 outcomes=N digest=HEX
|
||||
// audited commits=N tick=T outcomes=N profile=P clone=0|1
|
||||
// Exit: 0 done, 2 usage, 3 source/history failure, 4 target refused, 5 target unknown.
|
||||
// check-store (M8.3) qualifies an environment's stores from inside the cluster with
|
||||
// the runtime identity. Objects: the volume must already exist (a missing or
|
||||
// unmounted volume is refused), a fixed probe object is created once and verified by
|
||||
// digest on every later run (so a restarted or rescheduled pod proves retention),
|
||||
// and free space is reported against the reserve. Database: the adapter enforces
|
||||
// server 18.6 and verify-full TLS off-host; the runtime login must reach the fenced
|
||||
// functions but be refused table access.
|
||||
// Output: objects available_mib=A reserve_mib=R / probe created|verified digest=HEX
|
||||
// database version=V user=U ssl=S / database tables=denied functions=reachable
|
||||
//
|
||||
// Exit: 0 done, 2 usage, 3 source/history/database failure, 4 target/objects refused,
|
||||
// 5 target unknown.
|
||||
#include "evo/persistence/region_clone.hpp"
|
||||
#include "evo/persistence/sha256.hpp"
|
||||
|
||||
#include <array>
|
||||
#include <charconv>
|
||||
@@ -49,7 +63,7 @@ struct Options {
|
||||
Side source, target;
|
||||
std::vector<persistence::Sha256Digest> retained;
|
||||
std::optional<persistence::Uuid> commit;
|
||||
std::uint64_t audit_max{}, max_commits{1u << 20U};
|
||||
std::uint64_t audit_max{}, max_commits{1u << 20U}, reserve_mib{1024};
|
||||
bool crash{};
|
||||
};
|
||||
|
||||
@@ -112,7 +126,7 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
return std::nullopt;
|
||||
Options o;
|
||||
o.command = argv[1];
|
||||
if (o.command != "clone" && o.command != "audit")
|
||||
if (o.command != "clone" && o.command != "audit" && o.command != "check-store")
|
||||
return std::nullopt;
|
||||
const bool clone = o.command == "clone";
|
||||
for (int i = 2; i < argc; ++i) {
|
||||
@@ -150,14 +164,18 @@ std::optional<Options> parse(int argc, char** argv) {
|
||||
o.audit_max = *n;
|
||||
else if (clone && flag == "--crash-at" && value == "before-transaction")
|
||||
o.crash = true;
|
||||
else if (!clone && flag == "--max-commits" && n && *n > 0)
|
||||
else if (o.command == "audit" && flag == "--max-commits" && n && *n > 0)
|
||||
o.max_commits = *n;
|
||||
else if (o.command == "check-store" && flag == "--reserve-mib" && n && *n <= 1048576)
|
||||
o.reserve_mib = *n;
|
||||
else
|
||||
return std::nullopt;
|
||||
}
|
||||
if (clone && o.target.key.region == 0)
|
||||
o.target.key.region = o.source.key.region;
|
||||
if (!complete(o.source) || o.retained.empty() || (clone && !complete(o.target)))
|
||||
const bool needs_builds = o.command != "check-store";
|
||||
if (!complete(o.source) || (needs_builds && o.retained.empty()) ||
|
||||
(clone && !complete(o.target)))
|
||||
return std::nullopt;
|
||||
return o;
|
||||
}
|
||||
@@ -236,6 +254,88 @@ std::unique_ptr<persistence::Postgres> connect(const Side& s) {
|
||||
return std::move(*db);
|
||||
}
|
||||
|
||||
// A fixed 64 KiB probe: the same bytes on every run, so its digest names it.
|
||||
std::vector<std::uint8_t> probe_bytes() {
|
||||
std::vector<std::uint8_t> out(65536);
|
||||
const std::string_view marker = "aevolution-store-probe-v1";
|
||||
for (std::size_t i = 0; i < out.size(); ++i)
|
||||
out[i] = i < marker.size() ? static_cast<std::uint8_t>(marker[i])
|
||||
: static_cast<std::uint8_t>((i * 131U) ^ (i >> 7U));
|
||||
return out;
|
||||
}
|
||||
|
||||
int check_store(const Options& o) {
|
||||
auto store = persistence::FilesystemObjectStore::open(o.source.objects, o.source.key,
|
||||
persistence::max_lineage_payload_bytes,
|
||||
nullptr, o.reserve_mib * 1024 * 1024);
|
||||
if (!store) {
|
||||
std::printf("objects refused %s\n", store.error() == persistence::ObjectError::invalid
|
||||
? "missing-or-not-a-directory"
|
||||
: "io");
|
||||
return 4;
|
||||
}
|
||||
const auto available = (*store)->available_bytes();
|
||||
std::printf(
|
||||
"objects available_mib=%llu reserve_mib=%llu\n",
|
||||
static_cast<unsigned long long>(available.value_or(0) / (std::uint64_t{1024} * 1024)),
|
||||
static_cast<unsigned long long>(o.reserve_mib));
|
||||
const auto bytes = probe_bytes();
|
||||
const auto digest = persistence::sha256(bytes);
|
||||
if (!digest)
|
||||
return 4;
|
||||
const persistence::VerifiedObject probe{*digest, bytes.size()};
|
||||
const char* state = "verified";
|
||||
if (auto existing = (*store)->get(probe); !existing) {
|
||||
if (existing.error() != persistence::ObjectError::missing) {
|
||||
std::printf("probe %s\n", existing.error() == persistence::ObjectError::corrupt
|
||||
? "corrupt"
|
||||
: "unreadable");
|
||||
return 4;
|
||||
}
|
||||
auto stored = (*store)->put(bytes);
|
||||
if (!stored) {
|
||||
std::printf("probe refused %s\n",
|
||||
stored.error() == persistence::ObjectError::full ? "full" : "io");
|
||||
return 4;
|
||||
}
|
||||
auto back = (*store)->get(probe);
|
||||
if (!back || *back != bytes)
|
||||
return 4;
|
||||
state = "created";
|
||||
}
|
||||
std::printf("probe %s digest=%s\n", state, text(*digest).c_str());
|
||||
std::fflush(stdout);
|
||||
|
||||
auto db = persistence::Postgres::connect(o.source.pg, timeout);
|
||||
if (!db) {
|
||||
const auto kind = db.error().kind;
|
||||
std::printf("database refused %s\n", kind == persistence::PgError::version ? "version"
|
||||
: kind == persistence::PgError::config ? "config"
|
||||
: kind == persistence::PgError::connect ? "connect"
|
||||
: "other");
|
||||
return 3;
|
||||
}
|
||||
auto about = (*db)->query(
|
||||
"SELECT pg_catalog.current_setting('server_version_num'), current_user::text, "
|
||||
"coalesce((SELECT s.ssl::text || '/' || coalesce(s.version, '') FROM "
|
||||
"pg_catalog.pg_stat_ssl s WHERE s.pid = pg_catalog.pg_backend_pid()), 'unknown')",
|
||||
{}, timeout);
|
||||
if (!about || about->size() != 1 || (*about)[0].size() != 3)
|
||||
return 3;
|
||||
const auto& row = (*about)[0];
|
||||
std::printf("database version=%s user=%s ssl=%s\n", row[0].value_or("?").c_str(),
|
||||
row[1].value_or("?").c_str(), row[2].value_or("?").c_str());
|
||||
auto tables = (*db)->query("SELECT 1 FROM evolution.region_heads LIMIT 1", {}, timeout);
|
||||
const bool denied = !tables && std::string_view(tables.error().sqlstate.data()) == "42501";
|
||||
auto head = persistence::load_region_head(**db, o.source.key, timeout);
|
||||
const bool reachable =
|
||||
head || head.error().kind == persistence::LeaseStoreError::not_provisioned;
|
||||
std::printf("database tables=%s functions=%s head=%s\n", denied ? "denied" : "READABLE",
|
||||
reachable ? "reachable" : "unreachable",
|
||||
head ? std::to_string(head->sequence).c_str() : "none");
|
||||
return denied && reachable ? 0 : 3;
|
||||
}
|
||||
|
||||
int audit(const Options& o) {
|
||||
auto db = connect(o.source);
|
||||
auto objects = open_objects(o.source);
|
||||
@@ -329,5 +429,7 @@ int main(int argc, char** argv) {
|
||||
stderr);
|
||||
return 2;
|
||||
}
|
||||
if (options->command == "check-store")
|
||||
return check_store(*options);
|
||||
return options->command == "clone" ? clone(*options) : audit(*options);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user