Files
aevolution/tests/region_store_tests.cpp
T
dingoandClaude Opus 5.5 2d950060db Add M8.3 dev/stage stores: Retain volumes, free-space reserve, store check
Keep the filesystem object store on a node-local Retain-class PVC per
environment (ADR-0070), with the database on the external PostgreSQL VM.
Because local-path does not enforce sizes, the object store now refuses
new objects below a free-space reserve with a distinct full error, so a
full volume publishes and acknowledges nothing. evo-world-clone gains
check-store to qualify volume retention, reserve, database version/TLS
and the runtime privilege boundary from inside the cluster; it now ships
in the server image. Add the storage class, object claims, hand-applied
store Jobs, policy rules forbidding password env vars, and the runbook.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-29 13:51:15 -04:00

195 lines
8.6 KiB
C++

// M8.3 store qualification (ADR-0070) against real PostgreSQL and real processes:
// the object volume's free-space reserve, a genuinely failing write, the
// check-store probe (retention across runs, missing-volume refusal, privilege
// boundary) and a server whose volume is full. Run by tools/ci/postgres-check.sh.
#include "evo/persistence/checkpoint_store.hpp"
#include "fixtures/server_process.hpp"
#include <csignal>
#include <source_location>
#include <sys/resource.h>
void run_region_store_tests();
namespace {
using namespace evo::test::process;
using namespace evo::persistence;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region store test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
constexpr std::size_t budget = 8000000;
constexpr std::uint64_t unreachable_reserve = std::uint64_t{1} << 60U; // 1 EiB.
std::unique_ptr<FilesystemObjectStore> store(const fs::path& root, RegionKey key,
std::uint64_t reserve = 0) {
auto made = FilesystemObjectStore::open(root, key, max_lineage_payload_bytes, nullptr, reserve);
CHECK(made);
return std::move(*made);
}
fs::path fresh(const std::string& name) {
const auto root = fs::path(env("PGHOST")) / name;
fs::remove_all(root);
fs::create_directory(root);
fs::permissions(root, fs::perms::owner_all, fs::perm_options::replace);
return root;
}
std::size_t entries(const fs::path& dir) {
std::size_t n = 0;
for ([[maybe_unused]] const auto& entry : fs::directory_iterator(dir))
++n;
return n;
}
std::vector<std::uint8_t> bytes_of(std::size_t size, std::uint8_t seed) {
std::vector<std::uint8_t> out(size);
for (std::size_t i = 0; i < size; ++i)
out[i] = static_cast<std::uint8_t>(seed + (i * 7U));
return out;
}
// A full volume refuses new objects before writing; identical objects already
// stored stay usable; a batch against a full volume publishes nothing.
void full_volumes_publish_nothing() {
const RegionKey key{7, 81};
const auto root = fresh("store-full");
const auto stored = bytes_of(4096, 0x10);
auto roomy = store(root, key);
CHECK(roomy->available_bytes().value_or(0) > 0 && roomy->reserve_bytes() == 0);
CHECK(roomy->put(stored));
auto full = store(root, key, unreachable_reserve);
CHECK(full->reserve_bytes() == unreachable_reserve);
const auto before = entries(full->directory());
auto refused = full->put(bytes_of(4096, 0x20));
CHECK(!refused && refused.error() == ObjectError::full);
CHECK(entries(full->directory()) == before); // No temporary, no object.
CHECK(full->put(stored)); // Reuse needs no space.
// A lineage batch cannot publish through a full volume; the head stays.
const auto r = provision(82, 0xDE);
auto db = owner();
const auto head = load_region_checkpoint(*db, r.key, 5s);
CHECK(head);
auto runtime = Postgres::connect(
{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("EVO_PG_RUNTIME_USER"), {}, {}},
5s);
CHECK(runtime);
Sha256Digest build{};
build.fill(0xB1);
auto lease = acquire_region_lease(**runtime, r.key, build, 20000ms, 5s);
CHECK(lease);
auto state = evo::test::lineage_region();
RegionBatch<evo::processes::LineageRegion> batch{*state};
for (int i = 0; i < 10; ++i)
CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1)));
batch.seal();
Uuid commit{};
commit.fill(0xDF);
auto blocked = store(r.root, r.key, unreachable_reserve);
auto published =
publish_lineage_batch(**runtime, *blocked, *lease, *head, batch, commit, build, budget, 5s);
CHECK(!published && published.error().kind == PublicationError::upload &&
published.error().object == ObjectError::full);
CHECK(load_region_checkpoint(*db, r.key, 5s)->head == head->head);
CHECK(release_region_lease(**runtime, r.key, *lease, 5s));
}
// A write the filesystem really refuses (RLIMIT_FSIZE stands in for a full disk:
// the kernel rejects the write mid-object) is an io error that leaves no object
// and no temporary behind.
void failing_writes_leave_nothing() {
const auto root = fresh("store-efbig");
auto objects = store(root, {7, 83});
rlimit old{};
CHECK(::getrlimit(RLIMIT_FSIZE, &old) == 0);
const auto previous = std::signal(SIGXFSZ, SIG_IGN);
rlimit limited = old;
limited.rlim_cur = 8192;
CHECK(::setrlimit(RLIMIT_FSIZE, &limited) == 0);
auto refused = objects->put(bytes_of(65536, 0x30));
CHECK(::setrlimit(RLIMIT_FSIZE, &old) == 0);
std::signal(SIGXFSZ, previous);
CHECK(!refused && refused.error() == ObjectError::io);
CHECK(entries(objects->directory()) == 0);
CHECK(objects->put(bytes_of(65536, 0x30))); // The same write succeeds with room.
}
std::vector<std::string> check_store(const fs::path& objects, const std::string& user,
std::vector<std::string> extra, int& code) {
std::vector<std::string> args{env("EVO_WORLD_CLONE_BIN"),
"check-store",
"--world",
"7",
"--region",
"84",
"--pg-host",
env("PGHOST"),
"--pg-port",
env("PGPORT"),
"--pg-database",
env("PGDATABASE"),
"--pg-user",
user,
"--objects",
objects.string()};
args.insert(args.end(), extra.begin(), extra.end());
Child tool{args};
code = tool.wait_exit(30s);
return tool.lines();
}
bool has(const std::vector<std::string>& lines, std::string_view prefix) {
for (const auto& l : lines)
if (l.starts_with(prefix))
return true;
return false;
}
// The in-cluster qualification probe, exercised locally.
void check_store_qualifies_the_stores() {
const auto root = fresh("store-check");
const auto runtime = env("EVO_PG_RUNTIME_USER");
int code = -1;
auto first = check_store(root, runtime, {}, code);
CHECK(code == 0 && has(first, "objects available_mib=") && has(first, "probe created digest="));
CHECK(has(first, "database version=180006 user=" + runtime + " ssl=false"));
CHECK(has(first, "database tables=denied functions=reachable head=none"));
// A later run (a restarted or rescheduled pod) verifies the retained probe.
auto second = check_store(root, runtime, {}, code);
CHECK(code == 0 && has(second, "probe verified digest="));
// A missing or unmounted volume is refused, never created.
auto missing = check_store(root / "not-mounted", runtime, {}, code);
CHECK(code == 4 && has(missing, "objects refused missing-or-not-a-directory"));
CHECK(!fs::exists(root / "not-mounted"));
// A full volume refuses the probe (fresh namespace, reserve beyond any disk).
const auto other = fresh("store-check-full");
auto full = check_store(other, runtime, {"--reserve-mib", "1048576"}, code);
CHECK(code == 4 && has(full, "probe refused full"));
// An owner login can read tables: not an acceptable runtime identity.
auto owner_run = check_store(root, env("PGUSER"), {}, code);
CHECK(code == 3 && has(owner_run, "database tables=READABLE"));
}
// A real server on a full volume keeps simulating but never acknowledges, and its
// bounded drain gives up without committing anything.
void full_volume_server_acknowledges_nothing() {
const auto r = provision(85, 0xE7);
const auto before = head_of(r.key);
Child server{server_args(r, {"--objects-reserve-mib", "1048576", "--drain-ms", "1500",
"--ops-listen", "127.0.0.1:0"})};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
server.send(input_lines(1, 15, false));
server.pump(3000ms);
CHECK(!server.printed("ack "));
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 3); // Drain deadline passed with the batch unpublished.
CHECK(!server.printed("ack "));
const auto after = head_of(r.key);
CHECK(after.tick == before.tick && after.sequence == before.sequence);
}
} // namespace
void run_region_store_tests() {
full_volumes_publish_nothing();
failing_writes_leave_nothing();
check_store_qualifies_the_stores();
full_volume_server_acknowledges_nothing();
}