Keep the filesystem object store on a node-local Retain-class PVC per environment (ADR-0070), with the database on the external PostgreSQL VM. Because local-path does not enforce sizes, the object store now refuses new objects below a free-space reserve with a distinct full error, so a full volume publishes and acknowledges nothing. evo-world-clone gains check-store to qualify volume retention, reserve, database version/TLS and the runtime privilege boundary from inside the cluster; it now ships in the server image. Add the storage class, object claims, hand-applied store Jobs, policy rules forbidding password env vars, and the runbook. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
436 lines
18 KiB
C++
436 lines
18 KiB
C++
// M7.9 region clone and history audit tool (ADR-0067). Linux only; requires the
|
|
// PostgreSQL build.
|
|
//
|
|
// evo-world-clone clone --source-world W --source-region R --target-world W2
|
|
// [--target-region R2] --source-pg-* ... --target-pg-* ... --source-objects DIR
|
|
// --target-objects DIR --retained HEX [--retained HEX]... [--commit HEX32]
|
|
// [--audit-max N] [--crash-at before-transaction]
|
|
// evo-world-clone audit --world W --region R --pg-* ... --objects DIR --retained HEX...
|
|
// [--max-commits N]
|
|
// evo-world-clone check-store --world W --region R --pg-* ... --objects DIR
|
|
// [--reserve-mib N]
|
|
//
|
|
// --*pg-* is host, hostaddr, port, database, user, passfile, ca. The source
|
|
// connection needs only the runtime read functions and is switched to read-only
|
|
// transactions; the target connection is the target database's owner. Source and
|
|
// target must be different databases reached with different users: a clone never
|
|
// shares its source's credentials or authority. --audit-max N first audits up to N
|
|
// links of the source history. --crash-at (tests only) SIGKILLs this process after
|
|
// the verified target upload, before the target transaction.
|
|
//
|
|
// Output: cloned source=W/R sequence=S tick=T target=W2/R2 outcomes=N digest=HEX
|
|
// audited commits=N tick=T outcomes=N profile=P clone=0|1
|
|
// check-store (M8.3) qualifies an environment's stores from inside the cluster with
|
|
// the runtime identity. Objects: the volume must already exist (a missing or
|
|
// unmounted volume is refused), a fixed probe object is created once and verified by
|
|
// digest on every later run (so a restarted or rescheduled pod proves retention),
|
|
// and free space is reported against the reserve. Database: the adapter enforces
|
|
// server 18.6 and verify-full TLS off-host; the runtime login must reach the fenced
|
|
// functions but be refused table access.
|
|
// Output: objects available_mib=A reserve_mib=R / probe created|verified digest=HEX
|
|
// database version=V user=U ssl=S / database tables=denied functions=reachable
|
|
//
|
|
// Exit: 0 done, 2 usage, 3 source/history/database failure, 4 target/objects refused,
|
|
// 5 target unknown.
|
|
#include "evo/persistence/region_clone.hpp"
|
|
#include "evo/persistence/sha256.hpp"
|
|
|
|
#include <array>
|
|
#include <charconv>
|
|
#include <csignal>
|
|
#include <cstdio>
|
|
#include <filesystem>
|
|
#include <optional>
|
|
#include <string>
|
|
#include <string_view>
|
|
#include <sys/random.h>
|
|
#include <unistd.h>
|
|
#include <vector>
|
|
|
|
namespace {
|
|
using namespace evo;
|
|
using namespace std::chrono_literals;
|
|
constexpr std::size_t restore_budget = 8000000;
|
|
constexpr auto timeout = 10s;
|
|
|
|
struct Side {
|
|
persistence::RegionKey key{};
|
|
persistence::PgOptions pg;
|
|
std::filesystem::path objects;
|
|
};
|
|
struct Options {
|
|
std::string command;
|
|
Side source, target;
|
|
std::vector<persistence::Sha256Digest> retained;
|
|
std::optional<persistence::Uuid> commit;
|
|
std::uint64_t audit_max{}, max_commits{1u << 20U}, reserve_mib{1024};
|
|
bool crash{};
|
|
};
|
|
|
|
std::optional<std::uint64_t> number(std::string_view text) {
|
|
std::uint64_t value{};
|
|
const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), value);
|
|
if (error != std::errc{} || end != text.data() + text.size())
|
|
return std::nullopt;
|
|
return value;
|
|
}
|
|
template <std::size_t N> std::optional<std::array<std::uint8_t, N>> hex(std::string_view text) {
|
|
if (text.size() != N * 2)
|
|
return std::nullopt;
|
|
std::array<std::uint8_t, N> out{};
|
|
for (std::size_t i = 0; i < N; ++i) {
|
|
unsigned value{};
|
|
const auto [end, error] =
|
|
std::from_chars(text.data() + (2 * i), text.data() + (2 * i) + 2, value, 16);
|
|
if (error != std::errc{} || end != text.data() + (2 * i) + 2)
|
|
return std::nullopt;
|
|
out[i] = static_cast<std::uint8_t>(value);
|
|
}
|
|
return out;
|
|
}
|
|
template <std::size_t N> std::string text(const std::array<std::uint8_t, N>& bytes) {
|
|
constexpr char digits[] = "0123456789abcdef";
|
|
std::string out;
|
|
for (const auto b : bytes) {
|
|
out.push_back(digits[b >> 4U]);
|
|
out.push_back(digits[b & 0x0FU]);
|
|
}
|
|
return out;
|
|
}
|
|
// Connection flags with the given prefix ("--pg-", "--source-pg-", "--target-pg-").
|
|
bool pg_flag(std::string_view flag, std::string_view prefix, std::string_view value,
|
|
persistence::PgOptions& pg) {
|
|
if (!flag.starts_with(prefix))
|
|
return false;
|
|
const auto name = flag.substr(prefix.size());
|
|
std::string* field = name == "host" ? &pg.host
|
|
: name == "hostaddr" ? &pg.hostaddr
|
|
: name == "port" ? &pg.port
|
|
: name == "database" ? &pg.database
|
|
: name == "user" ? &pg.user
|
|
: name == "passfile" ? &pg.password_file
|
|
: name == "ca" ? &pg.ca_file
|
|
: nullptr;
|
|
if (field == nullptr)
|
|
return false;
|
|
*field = value;
|
|
return true;
|
|
}
|
|
bool complete(const Side& s) {
|
|
return s.key.world != 0 && s.key.region != 0 && !s.pg.host.empty() && !s.pg.port.empty() &&
|
|
!s.pg.database.empty() && !s.pg.user.empty() && !s.objects.empty();
|
|
}
|
|
|
|
std::optional<Options> parse(int argc, char** argv) {
|
|
if (argc < 2)
|
|
return std::nullopt;
|
|
Options o;
|
|
o.command = argv[1];
|
|
if (o.command != "clone" && o.command != "audit" && o.command != "check-store")
|
|
return std::nullopt;
|
|
const bool clone = o.command == "clone";
|
|
for (int i = 2; i < argc; ++i) {
|
|
const std::string_view flag = argv[i];
|
|
if (i + 1 >= argc)
|
|
return std::nullopt;
|
|
const std::string_view value = argv[++i];
|
|
const auto n = number(value);
|
|
if (clone && pg_flag(flag, "--source-pg-", value, o.source.pg))
|
|
continue;
|
|
if (clone && pg_flag(flag, "--target-pg-", value, o.target.pg))
|
|
continue;
|
|
if (!clone && pg_flag(flag, "--pg-", value, o.source.pg))
|
|
continue;
|
|
if (flag == "--retained") {
|
|
const auto digest = hex<32>(value);
|
|
if (!digest)
|
|
return std::nullopt;
|
|
o.retained.push_back(*digest);
|
|
} else if (flag == (clone ? "--source-world" : "--world") && n)
|
|
o.source.key.world = *n;
|
|
else if (flag == (clone ? "--source-region" : "--region") && n)
|
|
o.source.key.region = *n;
|
|
else if (clone && flag == "--target-world" && n)
|
|
o.target.key.world = *n;
|
|
else if (clone && flag == "--target-region" && n)
|
|
o.target.key.region = *n;
|
|
else if (flag == (clone ? "--source-objects" : "--objects"))
|
|
o.source.objects = value;
|
|
else if (clone && flag == "--target-objects")
|
|
o.target.objects = value;
|
|
else if (clone && flag == "--commit" && hex<16>(value))
|
|
o.commit = hex<16>(value);
|
|
else if (clone && flag == "--audit-max" && n && *n > 0)
|
|
o.audit_max = *n;
|
|
else if (clone && flag == "--crash-at" && value == "before-transaction")
|
|
o.crash = true;
|
|
else if (o.command == "audit" && flag == "--max-commits" && n && *n > 0)
|
|
o.max_commits = *n;
|
|
else if (o.command == "check-store" && flag == "--reserve-mib" && n && *n <= 1048576)
|
|
o.reserve_mib = *n;
|
|
else
|
|
return std::nullopt;
|
|
}
|
|
if (clone && o.target.key.region == 0)
|
|
o.target.key.region = o.source.key.region;
|
|
const bool needs_builds = o.command != "check-store";
|
|
if (!complete(o.source) || (needs_builds && o.retained.empty()) ||
|
|
(clone && !complete(o.target)))
|
|
return std::nullopt;
|
|
return o;
|
|
}
|
|
|
|
[[noreturn]] void crash_now() {
|
|
std::fflush(stdout);
|
|
::kill(::getpid(), SIGKILL);
|
|
for (;;)
|
|
::pause();
|
|
}
|
|
|
|
const char* name(persistence::AuditError e) {
|
|
using enum persistence::AuditError;
|
|
switch (e) {
|
|
case store:
|
|
return "store";
|
|
case not_provisioned:
|
|
return "not_provisioned";
|
|
case bounds:
|
|
return "bounds";
|
|
case manifest:
|
|
return "manifest";
|
|
case chain:
|
|
return "chain";
|
|
case unsupported:
|
|
return "unsupported";
|
|
case missing_payload:
|
|
return "missing_payload";
|
|
case corrupt_payload:
|
|
return "corrupt_payload";
|
|
case payload:
|
|
return "payload";
|
|
case tail:
|
|
return "tail";
|
|
case history:
|
|
return "history";
|
|
}
|
|
return "unknown";
|
|
}
|
|
const char* name(persistence::CloneError e) {
|
|
using enum persistence::CloneError;
|
|
switch (e) {
|
|
case identity:
|
|
return "identity";
|
|
case session:
|
|
return "session";
|
|
case source:
|
|
return "source";
|
|
case candidate:
|
|
return "candidate";
|
|
case upload:
|
|
return "upload";
|
|
case verify:
|
|
return "verify";
|
|
case rejected:
|
|
return "rejected";
|
|
case unknown:
|
|
return "unknown";
|
|
case receipt:
|
|
return "receipt";
|
|
}
|
|
return "unknown";
|
|
}
|
|
|
|
std::unique_ptr<persistence::FilesystemObjectStore> open_objects(const Side& s) {
|
|
auto store = persistence::FilesystemObjectStore::open(s.objects, s.key,
|
|
persistence::max_lineage_payload_bytes);
|
|
if (!store)
|
|
return nullptr;
|
|
return std::move(*store);
|
|
}
|
|
std::unique_ptr<persistence::Postgres> connect(const Side& s) {
|
|
auto db = persistence::Postgres::connect(s.pg, timeout);
|
|
if (!db)
|
|
return nullptr;
|
|
return std::move(*db);
|
|
}
|
|
|
|
// A fixed 64 KiB probe: the same bytes on every run, so its digest names it.
|
|
std::vector<std::uint8_t> probe_bytes() {
|
|
std::vector<std::uint8_t> out(65536);
|
|
const std::string_view marker = "aevolution-store-probe-v1";
|
|
for (std::size_t i = 0; i < out.size(); ++i)
|
|
out[i] = i < marker.size() ? static_cast<std::uint8_t>(marker[i])
|
|
: static_cast<std::uint8_t>((i * 131U) ^ (i >> 7U));
|
|
return out;
|
|
}
|
|
|
|
int check_store(const Options& o) {
|
|
auto store = persistence::FilesystemObjectStore::open(o.source.objects, o.source.key,
|
|
persistence::max_lineage_payload_bytes,
|
|
nullptr, o.reserve_mib * 1024 * 1024);
|
|
if (!store) {
|
|
std::printf("objects refused %s\n", store.error() == persistence::ObjectError::invalid
|
|
? "missing-or-not-a-directory"
|
|
: "io");
|
|
return 4;
|
|
}
|
|
const auto available = (*store)->available_bytes();
|
|
std::printf(
|
|
"objects available_mib=%llu reserve_mib=%llu\n",
|
|
static_cast<unsigned long long>(available.value_or(0) / (std::uint64_t{1024} * 1024)),
|
|
static_cast<unsigned long long>(o.reserve_mib));
|
|
const auto bytes = probe_bytes();
|
|
const auto digest = persistence::sha256(bytes);
|
|
if (!digest)
|
|
return 4;
|
|
const persistence::VerifiedObject probe{*digest, bytes.size()};
|
|
const char* state = "verified";
|
|
if (auto existing = (*store)->get(probe); !existing) {
|
|
if (existing.error() != persistence::ObjectError::missing) {
|
|
std::printf("probe %s\n", existing.error() == persistence::ObjectError::corrupt
|
|
? "corrupt"
|
|
: "unreadable");
|
|
return 4;
|
|
}
|
|
auto stored = (*store)->put(bytes);
|
|
if (!stored) {
|
|
std::printf("probe refused %s\n",
|
|
stored.error() == persistence::ObjectError::full ? "full" : "io");
|
|
return 4;
|
|
}
|
|
auto back = (*store)->get(probe);
|
|
if (!back || *back != bytes)
|
|
return 4;
|
|
state = "created";
|
|
}
|
|
std::printf("probe %s digest=%s\n", state, text(*digest).c_str());
|
|
std::fflush(stdout);
|
|
|
|
auto db = persistence::Postgres::connect(o.source.pg, timeout);
|
|
if (!db) {
|
|
const auto kind = db.error().kind;
|
|
std::printf("database refused %s\n", kind == persistence::PgError::version ? "version"
|
|
: kind == persistence::PgError::config ? "config"
|
|
: kind == persistence::PgError::connect ? "connect"
|
|
: "other");
|
|
return 3;
|
|
}
|
|
auto about = (*db)->query(
|
|
"SELECT pg_catalog.current_setting('server_version_num'), current_user::text, "
|
|
"coalesce((SELECT s.ssl::text || '/' || coalesce(s.version, '') FROM "
|
|
"pg_catalog.pg_stat_ssl s WHERE s.pid = pg_catalog.pg_backend_pid()), 'unknown')",
|
|
{}, timeout);
|
|
if (!about || about->size() != 1 || (*about)[0].size() != 3)
|
|
return 3;
|
|
const auto& row = (*about)[0];
|
|
std::printf("database version=%s user=%s ssl=%s\n", row[0].value_or("?").c_str(),
|
|
row[1].value_or("?").c_str(), row[2].value_or("?").c_str());
|
|
auto tables = (*db)->query("SELECT 1 FROM evolution.region_heads LIMIT 1", {}, timeout);
|
|
const bool denied = !tables && std::string_view(tables.error().sqlstate.data()) == "42501";
|
|
auto head = persistence::load_region_head(**db, o.source.key, timeout);
|
|
const bool reachable =
|
|
head || head.error().kind == persistence::LeaseStoreError::not_provisioned;
|
|
std::printf("database tables=%s functions=%s head=%s\n", denied ? "denied" : "READABLE",
|
|
reachable ? "reachable" : "unreachable",
|
|
head ? std::to_string(head->sequence).c_str() : "none");
|
|
return denied && reachable ? 0 : 3;
|
|
}
|
|
|
|
int audit(const Options& o) {
|
|
auto db = connect(o.source);
|
|
auto objects = open_objects(o.source);
|
|
if (!db || !objects) {
|
|
std::fputs("audit failed store\n", stderr);
|
|
return 3;
|
|
}
|
|
auto result = persistence::audit_region_history(*db, *objects, o.source.key, o.retained,
|
|
restore_budget, o.max_commits, timeout);
|
|
if (!result) {
|
|
std::fprintf(stderr, "audit failed %s sequence=%llu\n", name(result.error().kind),
|
|
static_cast<unsigned long long>(result.error().sequence));
|
|
return 3;
|
|
}
|
|
std::printf("audited commits=%llu tick=%llu outcomes=%llu profile=%u clone=%d\n",
|
|
static_cast<unsigned long long>(result->commits),
|
|
static_cast<unsigned long long>(result->end_tick),
|
|
static_cast<unsigned long long>(result->outcomes),
|
|
static_cast<unsigned>(result->profile), result->clone ? 1 : 0);
|
|
return 0;
|
|
}
|
|
|
|
int clone(const Options& o) {
|
|
const auto& s = o.source.pg;
|
|
const auto& t = o.target.pg;
|
|
if ((s.host == t.host && s.hostaddr == t.hostaddr && s.port == t.port &&
|
|
s.database == t.database) ||
|
|
s.user == t.user) {
|
|
std::fputs("clone refused: source and target need separate databases and users\n", stderr);
|
|
return 2;
|
|
}
|
|
auto source = connect(o.source);
|
|
auto source_objects = open_objects(o.source);
|
|
if (!source || !source_objects) {
|
|
std::fputs("clone failed source store\n", stderr);
|
|
return 3;
|
|
}
|
|
if (o.audit_max > 0) {
|
|
auto audited =
|
|
persistence::audit_region_history(*source, *source_objects, o.source.key, o.retained,
|
|
restore_budget, o.audit_max, timeout);
|
|
if (!audited) {
|
|
std::fprintf(stderr, "clone failed audit %s sequence=%llu\n",
|
|
name(audited.error().kind),
|
|
static_cast<unsigned long long>(audited.error().sequence));
|
|
return 3;
|
|
}
|
|
}
|
|
auto target = connect(o.target);
|
|
auto target_objects = open_objects(o.target);
|
|
if (!target || !target_objects) {
|
|
std::fputs("clone failed target store\n", stderr);
|
|
return 4;
|
|
}
|
|
persistence::Uuid commit{};
|
|
if (o.commit)
|
|
commit = *o.commit;
|
|
else if (::getrandom(commit.data(), commit.size(), 0) != static_cast<ssize_t>(commit.size()))
|
|
return 4;
|
|
persistence::CloneHooks hooks;
|
|
if (o.crash)
|
|
hooks.before_transaction = [] { crash_now(); };
|
|
auto cloned =
|
|
persistence::clone_region(*source, *source_objects, o.source.key, *target, *target_objects,
|
|
o.target.key, commit, o.retained, restore_budget, timeout, hooks);
|
|
if (!cloned) {
|
|
const auto kind = cloned.error().kind;
|
|
std::fprintf(stderr, "clone failed %s\n", name(kind));
|
|
using persistence::CloneError;
|
|
return kind == CloneError::source || kind == CloneError::session ? 3
|
|
: kind == CloneError::unknown ? 5
|
|
: 4;
|
|
}
|
|
std::printf("cloned source=%llu/%llu sequence=%llu tick=%llu target=%llu/%llu outcomes=%zu "
|
|
"digest=%s\n",
|
|
static_cast<unsigned long long>(o.source.key.world),
|
|
static_cast<unsigned long long>(o.source.key.region),
|
|
static_cast<unsigned long long>(cloned->origin.sequence),
|
|
static_cast<unsigned long long>(cloned->origin.tick),
|
|
static_cast<unsigned long long>(o.target.key.world),
|
|
static_cast<unsigned long long>(o.target.key.region), cloned->inherited_outcomes,
|
|
text(cloned->genesis.head.digest).c_str());
|
|
return 0;
|
|
}
|
|
} // namespace
|
|
|
|
int main(int argc, char** argv) {
|
|
const auto options = parse(argc, argv);
|
|
if (!options) {
|
|
std::fputs("usage: evo-world-clone clone|audit ... (see apps/evo-world-clone/main.cpp)\n",
|
|
stderr);
|
|
return 2;
|
|
}
|
|
if (options->command == "check-store")
|
|
return check_store(*options);
|
|
return options->command == "clone" ? clone(*options) : audit(*options);
|
|
}
|