Files
aevolution/apps/evo-world-clone/main.cpp
T
dingoandClaude Opus 5.5 8ef69fab1d Add M8.4 region host deployment, provisioning hook and drill
Deploy evo-region per environment (ADR-0071): one replica with the
Recreate strategy (a rolling update would deadlock on the lease),
/readyz readiness, /livez liveness, no CPU limit and a drain inside the
grace period, with the image digest doubling as the build identity. A
new idempotent evo-world-clone provision subcommand creates genesis from
development_opening (byte-identical to the qualification fixture) as a
Sync hook after the migration. Dev and stage paths include their region;
a dev AppProject allows Deployment and Service; region-drill.sh times
rollout and forced-deletion recovery; the policy check enforces the
host's invariants.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-29 21:32:56 -04:00

524 lines
22 KiB
C++

// M7.9 region clone and history audit tool (ADR-0067). Linux only; requires the
// PostgreSQL build.
//
// evo-world-clone clone --source-world W --source-region R --target-world W2
// [--target-region R2] --source-pg-* ... --target-pg-* ... --source-objects DIR
// --target-objects DIR --retained HEX [--retained HEX]... [--commit HEX32]
// [--audit-max N] [--crash-at before-transaction]
// evo-world-clone audit --world W --region R --pg-* ... --objects DIR --retained HEX...
// [--max-commits N]
// evo-world-clone check-store --world W --region R --pg-* ... --objects DIR
// [--reserve-mib N]
// evo-world-clone provision --world W --region R --opening development-v1 --pg-* ...
// --objects DIR --build HEX [--retain-build]
//
// --*pg-* is host, hostaddr, port, database, user, passfile, ca, or service (a libpq
// service file whose [evolution] section supplies everything but the password file). The source
// connection needs only the runtime read functions and is switched to read-only
// transactions; the target connection is the target database's owner. Source and
// target must be different databases reached with different users: a clone never
// shares its source's credentials or authority. --audit-max N first audits up to N
// links of the source history. --crash-at (tests only) SIGKILLs this process after
// the verified target upload, before the target transaction.
//
// Output: cloned source=W/R sequence=S tick=T target=W2/R2 outcomes=N digest=HEX
// audited commits=N tick=T outcomes=N profile=P clone=0|1
// check-store (M8.3) qualifies an environment's stores from inside the cluster with
// the runtime identity. Objects: the volume must already exist (a missing or
// unmounted volume is refused), a fixed probe object is created once and verified by
// digest on every later run (so a restarted or rescheduled pod proves retention),
// and free space is reported against the reserve. Database: the adapter enforces
// server 18.6 and verify-full TLS off-host; the runtime login must reach the fenced
// functions but be refused table access.
// Output: objects available_mib=A reserve_mib=R / probe created|verified digest=HEX
// database version=V user=U ssl=S / database tables=denied functions=reachable
//
// provision (M8.4) creates a region's genesis with the owner (migrator) login, once:
// an already provisioned region is reported and never touched. --retain-build first
// records the build identity (the server image digest) in retained_builds. The only
// opening is development-v1 (evo/processes/development_opening.hpp), for dev and
// stage; World0's opening is a separate, reviewed decision.
// Output: provisioned|already-provisioned world=W region=R sequence=S tick=T
//
// Exit: 0 done, 2 usage, 3 source/history/database failure, 4 target/objects refused,
// 5 target unknown.
#include "evo/persistence/region_clone.hpp"
#include "evo/persistence/sha256.hpp"
#include "evo/processes/development_opening.hpp"
#include <array>
#include <charconv>
#include <csignal>
#include <cstdio>
#include <filesystem>
#include <optional>
#include <string>
#include <string_view>
#include <sys/random.h>
#include <unistd.h>
#include <vector>
namespace {
using namespace evo;
using namespace std::chrono_literals;
constexpr std::size_t restore_budget = 8000000;
constexpr auto timeout = 10s;
struct Side {
persistence::RegionKey key{};
persistence::PgOptions pg;
std::filesystem::path objects;
};
struct Options {
std::string command;
Side source, target;
std::vector<persistence::Sha256Digest> retained;
std::optional<persistence::Uuid> commit;
std::uint64_t audit_max{}, max_commits{1u << 20U}, reserve_mib{1024};
std::string opening;
std::optional<persistence::Sha256Digest> build;
bool retain_build{};
bool crash{};
};
std::optional<std::uint64_t> number(std::string_view text) {
std::uint64_t value{};
const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), value);
if (error != std::errc{} || end != text.data() + text.size())
return std::nullopt;
return value;
}
template <std::size_t N> std::optional<std::array<std::uint8_t, N>> hex(std::string_view text) {
if (text.size() != N * 2)
return std::nullopt;
std::array<std::uint8_t, N> out{};
for (std::size_t i = 0; i < N; ++i) {
unsigned value{};
const auto [end, error] =
std::from_chars(text.data() + (2 * i), text.data() + (2 * i) + 2, value, 16);
if (error != std::errc{} || end != text.data() + (2 * i) + 2)
return std::nullopt;
out[i] = static_cast<std::uint8_t>(value);
}
return out;
}
template <std::size_t N> std::string text(const std::array<std::uint8_t, N>& bytes) {
constexpr char digits[] = "0123456789abcdef";
std::string out;
for (const auto b : bytes) {
out.push_back(digits[b >> 4U]);
out.push_back(digits[b & 0x0FU]);
}
return out;
}
// Connection flags with the given prefix ("--pg-", "--source-pg-", "--target-pg-").
bool pg_flag(std::string_view flag, std::string_view prefix, std::string_view value,
persistence::PgOptions& pg) {
if (!flag.starts_with(prefix))
return false;
const auto name = flag.substr(prefix.size());
if (name == "service") {
// [evolution] section of a libpq service file; the password file stays separate.
auto service = persistence::read_pg_service(std::string(value), "evolution");
if (!service)
return false;
service->password_file = pg.password_file;
pg = *service;
return true;
}
std::string* field = name == "host" ? &pg.host
: name == "hostaddr" ? &pg.hostaddr
: name == "port" ? &pg.port
: name == "database" ? &pg.database
: name == "user" ? &pg.user
: name == "passfile" ? &pg.password_file
: name == "ca" ? &pg.ca_file
: nullptr;
if (field == nullptr)
return false;
*field = value;
return true;
}
bool complete(const Side& s) {
return s.key.world != 0 && s.key.region != 0 && !s.pg.host.empty() && !s.pg.port.empty() &&
!s.pg.database.empty() && !s.pg.user.empty() && !s.objects.empty();
}
std::optional<Options> parse(int argc, char** argv) {
if (argc < 2)
return std::nullopt;
Options o;
o.command = argv[1];
if (o.command != "clone" && o.command != "audit" && o.command != "check-store" &&
o.command != "provision")
return std::nullopt;
const bool clone = o.command == "clone";
for (int i = 2; i < argc; ++i) {
const std::string_view flag = argv[i];
if (i + 1 >= argc)
return std::nullopt;
const std::string_view value = argv[++i];
const auto n = number(value);
if (clone && pg_flag(flag, "--source-pg-", value, o.source.pg))
continue;
if (clone && pg_flag(flag, "--target-pg-", value, o.target.pg))
continue;
if (!clone && pg_flag(flag, "--pg-", value, o.source.pg))
continue;
if (flag == "--retained") {
const auto digest = hex<32>(value);
if (!digest)
return std::nullopt;
o.retained.push_back(*digest);
} else if (flag == (clone ? "--source-world" : "--world") && n)
o.source.key.world = *n;
else if (flag == (clone ? "--source-region" : "--region") && n)
o.source.key.region = *n;
else if (clone && flag == "--target-world" && n)
o.target.key.world = *n;
else if (clone && flag == "--target-region" && n)
o.target.key.region = *n;
else if (flag == (clone ? "--source-objects" : "--objects"))
o.source.objects = value;
else if (clone && flag == "--target-objects")
o.target.objects = value;
else if (clone && flag == "--commit" && hex<16>(value))
o.commit = hex<16>(value);
else if (clone && flag == "--audit-max" && n && *n > 0)
o.audit_max = *n;
else if (clone && flag == "--crash-at" && value == "before-transaction")
o.crash = true;
else if (o.command == "audit" && flag == "--max-commits" && n && *n > 0)
o.max_commits = *n;
else if (o.command == "check-store" && flag == "--reserve-mib" && n && *n <= 1048576)
o.reserve_mib = *n;
else if (o.command == "provision" && flag == "--opening" && value == "development-v1")
o.opening = value;
else if (o.command == "provision" && flag == "--build" && hex<32>(value))
o.build = hex<32>(value);
else if (o.command == "provision" && flag == "--retain-build" && value == "true")
o.retain_build = true;
else
return std::nullopt;
}
if (clone && o.target.key.region == 0)
o.target.key.region = o.source.key.region;
const bool needs_builds = o.command != "check-store" && o.command != "provision";
if (o.command == "provision" && (o.opening.empty() || !o.build))
return std::nullopt;
if (!complete(o.source) || (needs_builds && o.retained.empty()) ||
(clone && !complete(o.target)))
return std::nullopt;
return o;
}
[[noreturn]] void crash_now() {
std::fflush(stdout);
::kill(::getpid(), SIGKILL);
for (;;)
::pause();
}
const char* name(persistence::AuditError e) {
using enum persistence::AuditError;
switch (e) {
case store:
return "store";
case not_provisioned:
return "not_provisioned";
case bounds:
return "bounds";
case manifest:
return "manifest";
case chain:
return "chain";
case unsupported:
return "unsupported";
case missing_payload:
return "missing_payload";
case corrupt_payload:
return "corrupt_payload";
case payload:
return "payload";
case tail:
return "tail";
case history:
return "history";
}
return "unknown";
}
const char* name(persistence::CloneError e) {
using enum persistence::CloneError;
switch (e) {
case identity:
return "identity";
case session:
return "session";
case source:
return "source";
case candidate:
return "candidate";
case upload:
return "upload";
case verify:
return "verify";
case rejected:
return "rejected";
case unknown:
return "unknown";
case receipt:
return "receipt";
}
return "unknown";
}
std::unique_ptr<persistence::FilesystemObjectStore> open_objects(const Side& s) {
auto store = persistence::FilesystemObjectStore::open(s.objects, s.key,
persistence::max_lineage_payload_bytes);
if (!store)
return nullptr;
return std::move(*store);
}
std::unique_ptr<persistence::Postgres> connect(const Side& s) {
auto db = persistence::Postgres::connect(s.pg, timeout);
if (!db)
return nullptr;
return std::move(*db);
}
// A fixed 64 KiB probe: the same bytes on every run, so its digest names it.
std::vector<std::uint8_t> probe_bytes() {
std::vector<std::uint8_t> out(65536);
const std::string_view marker = "aevolution-store-probe-v1";
for (std::size_t i = 0; i < out.size(); ++i)
out[i] = i < marker.size() ? static_cast<std::uint8_t>(marker[i])
: static_cast<std::uint8_t>((i * 131U) ^ (i >> 7U));
return out;
}
int provision(const Options& o) {
auto store = open_objects(o.source);
if (!store) {
std::puts("objects refused");
return 4;
}
auto db = connect(o.source);
if (!db) {
std::puts("database refused");
return 3;
}
if (!o.build)
return 2; // Required by parse(); checked again for the reader.
const auto& build = *o.build;
if (o.retain_build) {
const std::array<std::string, 1> p{"\\x" + text(build)};
if (!db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES ($1::bytea) "
"ON CONFLICT DO NOTHING",
p, timeout)) {
std::puts("retain-build refused");
return 3;
}
}
const auto report = [&](const char* what, const persistence::RegionHead& head) {
std::printf("%s world=%llu region=%llu sequence=%llu tick=%llu\n", what,
static_cast<unsigned long long>(o.source.key.world),
static_cast<unsigned long long>(o.source.key.region),
static_cast<unsigned long long>(head.sequence),
static_cast<unsigned long long>(head.end_tick));
return 0;
};
// Never re-provision: an existing region, whatever its state, is left alone.
auto existing = persistence::load_region_head(*db, o.source.key, timeout);
if (existing)
return report("already-provisioned", *existing);
if (existing.error().kind != persistence::LeaseStoreError::not_provisioned) {
std::puts("database refused");
return 3;
}
auto opening = processes::development_opening(o.source.key.world);
persistence::Uuid commit{};
if (!opening ||
::getrandom(commit.data(), commit.size(), 0) != static_cast<ssize_t>(commit.size()))
return 4;
auto made = persistence::provision_lineage_region(
*db, *store, o.source.key, processes::development_opening_frame, **opening, commit, build,
restore_budget, timeout);
if (!made) {
std::printf("provision refused %u\n", static_cast<unsigned>(made.error().kind));
return 4;
}
return report("provisioned", made->head);
}
int check_store(const Options& o) {
auto store = persistence::FilesystemObjectStore::open(o.source.objects, o.source.key,
persistence::max_lineage_payload_bytes,
nullptr, o.reserve_mib * 1024 * 1024);
if (!store) {
std::printf("objects refused %s\n", store.error() == persistence::ObjectError::invalid
? "missing-or-not-a-directory"
: "io");
return 4;
}
const auto available = (*store)->available_bytes();
std::printf(
"objects available_mib=%llu reserve_mib=%llu\n",
static_cast<unsigned long long>(available.value_or(0) / (std::uint64_t{1024} * 1024)),
static_cast<unsigned long long>(o.reserve_mib));
const auto bytes = probe_bytes();
const auto digest = persistence::sha256(bytes);
if (!digest)
return 4;
const persistence::VerifiedObject probe{*digest, bytes.size()};
const char* state = "verified";
if (auto existing = (*store)->get(probe); !existing) {
if (existing.error() != persistence::ObjectError::missing) {
std::printf("probe %s\n", existing.error() == persistence::ObjectError::corrupt
? "corrupt"
: "unreadable");
return 4;
}
auto stored = (*store)->put(bytes);
if (!stored) {
std::printf("probe refused %s\n",
stored.error() == persistence::ObjectError::full ? "full" : "io");
return 4;
}
auto back = (*store)->get(probe);
if (!back || *back != bytes)
return 4;
state = "created";
}
std::printf("probe %s digest=%s\n", state, text(*digest).c_str());
std::fflush(stdout);
auto db = persistence::Postgres::connect(o.source.pg, timeout);
if (!db) {
const auto kind = db.error().kind;
std::printf("database refused %s\n", kind == persistence::PgError::version ? "version"
: kind == persistence::PgError::config ? "config"
: kind == persistence::PgError::connect ? "connect"
: "other");
return 3;
}
auto about = (*db)->query(
"SELECT pg_catalog.current_setting('server_version_num'), current_user::text, "
"coalesce((SELECT s.ssl::text || '/' || coalesce(s.version, '') FROM "
"pg_catalog.pg_stat_ssl s WHERE s.pid = pg_catalog.pg_backend_pid()), 'unknown')",
{}, timeout);
if (!about || about->size() != 1 || (*about)[0].size() != 3)
return 3;
const auto& row = (*about)[0];
std::printf("database version=%s user=%s ssl=%s\n", row[0].value_or("?").c_str(),
row[1].value_or("?").c_str(), row[2].value_or("?").c_str());
auto tables = (*db)->query("SELECT 1 FROM evolution.region_heads LIMIT 1", {}, timeout);
const bool denied = !tables && std::string_view(tables.error().sqlstate.data()) == "42501";
auto head = persistence::load_region_head(**db, o.source.key, timeout);
const bool reachable =
head || head.error().kind == persistence::LeaseStoreError::not_provisioned;
std::printf("database tables=%s functions=%s head=%s\n", denied ? "denied" : "READABLE",
reachable ? "reachable" : "unreachable",
head ? std::to_string(head->sequence).c_str() : "none");
return denied && reachable ? 0 : 3;
}
int audit(const Options& o) {
auto db = connect(o.source);
auto objects = open_objects(o.source);
if (!db || !objects) {
std::fputs("audit failed store\n", stderr);
return 3;
}
auto result = persistence::audit_region_history(*db, *objects, o.source.key, o.retained,
restore_budget, o.max_commits, timeout);
if (!result) {
std::fprintf(stderr, "audit failed %s sequence=%llu\n", name(result.error().kind),
static_cast<unsigned long long>(result.error().sequence));
return 3;
}
std::printf("audited commits=%llu tick=%llu outcomes=%llu profile=%u clone=%d\n",
static_cast<unsigned long long>(result->commits),
static_cast<unsigned long long>(result->end_tick),
static_cast<unsigned long long>(result->outcomes),
static_cast<unsigned>(result->profile), result->clone ? 1 : 0);
return 0;
}
int clone(const Options& o) {
const auto& s = o.source.pg;
const auto& t = o.target.pg;
if ((s.host == t.host && s.hostaddr == t.hostaddr && s.port == t.port &&
s.database == t.database) ||
s.user == t.user) {
std::fputs("clone refused: source and target need separate databases and users\n", stderr);
return 2;
}
auto source = connect(o.source);
auto source_objects = open_objects(o.source);
if (!source || !source_objects) {
std::fputs("clone failed source store\n", stderr);
return 3;
}
if (o.audit_max > 0) {
auto audited =
persistence::audit_region_history(*source, *source_objects, o.source.key, o.retained,
restore_budget, o.audit_max, timeout);
if (!audited) {
std::fprintf(stderr, "clone failed audit %s sequence=%llu\n",
name(audited.error().kind),
static_cast<unsigned long long>(audited.error().sequence));
return 3;
}
}
auto target = connect(o.target);
auto target_objects = open_objects(o.target);
if (!target || !target_objects) {
std::fputs("clone failed target store\n", stderr);
return 4;
}
persistence::Uuid commit{};
if (o.commit)
commit = *o.commit;
else if (::getrandom(commit.data(), commit.size(), 0) != static_cast<ssize_t>(commit.size()))
return 4;
persistence::CloneHooks hooks;
if (o.crash)
hooks.before_transaction = [] { crash_now(); };
auto cloned =
persistence::clone_region(*source, *source_objects, o.source.key, *target, *target_objects,
o.target.key, commit, o.retained, restore_budget, timeout, hooks);
if (!cloned) {
const auto kind = cloned.error().kind;
std::fprintf(stderr, "clone failed %s\n", name(kind));
using persistence::CloneError;
return kind == CloneError::source || kind == CloneError::session ? 3
: kind == CloneError::unknown ? 5
: 4;
}
std::printf("cloned source=%llu/%llu sequence=%llu tick=%llu target=%llu/%llu outcomes=%zu "
"digest=%s\n",
static_cast<unsigned long long>(o.source.key.world),
static_cast<unsigned long long>(o.source.key.region),
static_cast<unsigned long long>(cloned->origin.sequence),
static_cast<unsigned long long>(cloned->origin.tick),
static_cast<unsigned long long>(o.target.key.world),
static_cast<unsigned long long>(o.target.key.region), cloned->inherited_outcomes,
text(cloned->genesis.head.digest).c_str());
return 0;
}
} // namespace
int main(int argc, char** argv) {
const auto options = parse(argc, argv);
if (!options) {
std::fputs("usage: evo-world-clone clone|audit ... (see apps/evo-world-clone/main.cpp)\n",
stderr);
return 2;
}
if (options->command == "check-store")
return check_store(*options);
if (options->command == "provision")
return provision(*options);
return options->command == "clone" ? clone(*options) : audit(*options);
}