Deploy evo-region per environment (ADR-0071): one replica with the Recreate strategy (a rolling update would deadlock on the lease), /readyz readiness, /livez liveness, no CPU limit and a drain inside the grace period, with the image digest doubling as the build identity. A new idempotent evo-world-clone provision subcommand creates genesis from development_opening (byte-identical to the qualification fixture) as a Sync hook after the migration. Dev and stage paths include their region; a dev AppProject allows Deployment and Service; region-drill.sh times rollout and forced-deletion recovery; the policy check enforces the host's invariants. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
257 lines
12 KiB
C++
257 lines
12 KiB
C++
// M8.3 store qualification (ADR-0070) against real PostgreSQL and real processes:
|
|
// the object volume's free-space reserve, a genuinely failing write, the
|
|
// check-store probe (retention across runs, missing-volume refusal, privilege
|
|
// boundary) and a server whose volume is full. Run by tools/ci/postgres-check.sh.
|
|
#include "evo/persistence/checkpoint_store.hpp"
|
|
#include "fixtures/server_process.hpp"
|
|
#include <csignal>
|
|
#include <source_location>
|
|
#include <sys/resource.h>
|
|
|
|
void run_region_store_tests();
|
|
|
|
namespace {
|
|
using namespace evo::test::process;
|
|
using namespace evo::persistence;
|
|
void check(bool value, std::source_location at = std::source_location::current()) {
|
|
if (!value)
|
|
throw std::runtime_error("region store test line " + std::to_string(at.line()));
|
|
}
|
|
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
|
constexpr std::size_t budget = 8000000;
|
|
constexpr std::uint64_t unreachable_reserve = std::uint64_t{1} << 60U; // 1 EiB.
|
|
std::unique_ptr<FilesystemObjectStore> store(const fs::path& root, RegionKey key,
|
|
std::uint64_t reserve = 0) {
|
|
auto made = FilesystemObjectStore::open(root, key, max_lineage_payload_bytes, nullptr, reserve);
|
|
CHECK(made);
|
|
return std::move(*made);
|
|
}
|
|
fs::path fresh(const std::string& name) {
|
|
const auto root = fs::path(env("PGHOST")) / name;
|
|
fs::remove_all(root);
|
|
fs::create_directory(root);
|
|
fs::permissions(root, fs::perms::owner_all, fs::perm_options::replace);
|
|
return root;
|
|
}
|
|
std::size_t entries(const fs::path& dir) {
|
|
std::size_t n = 0;
|
|
for ([[maybe_unused]] const auto& entry : fs::directory_iterator(dir))
|
|
++n;
|
|
return n;
|
|
}
|
|
std::vector<std::uint8_t> bytes_of(std::size_t size, std::uint8_t seed) {
|
|
std::vector<std::uint8_t> out(size);
|
|
for (std::size_t i = 0; i < size; ++i)
|
|
out[i] = static_cast<std::uint8_t>(seed + (i * 7U));
|
|
return out;
|
|
}
|
|
|
|
// A full volume refuses new objects before writing; identical objects already
|
|
// stored stay usable; a batch against a full volume publishes nothing.
|
|
void full_volumes_publish_nothing() {
|
|
const RegionKey key{7, 81};
|
|
const auto root = fresh("store-full");
|
|
const auto stored = bytes_of(4096, 0x10);
|
|
auto roomy = store(root, key);
|
|
CHECK(roomy->available_bytes().value_or(0) > 0 && roomy->reserve_bytes() == 0);
|
|
CHECK(roomy->put(stored));
|
|
auto full = store(root, key, unreachable_reserve);
|
|
CHECK(full->reserve_bytes() == unreachable_reserve);
|
|
const auto before = entries(full->directory());
|
|
auto refused = full->put(bytes_of(4096, 0x20));
|
|
CHECK(!refused && refused.error() == ObjectError::full);
|
|
CHECK(entries(full->directory()) == before); // No temporary, no object.
|
|
CHECK(full->put(stored)); // Reuse needs no space.
|
|
|
|
// A lineage batch cannot publish through a full volume; the head stays.
|
|
const auto r = provision(82, 0xDE);
|
|
auto db = owner();
|
|
const auto head = load_region_checkpoint(*db, r.key, 5s);
|
|
CHECK(head);
|
|
auto runtime = Postgres::connect(
|
|
{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("EVO_PG_RUNTIME_USER"), {}, {}},
|
|
5s);
|
|
CHECK(runtime);
|
|
Sha256Digest build{};
|
|
build.fill(0xB1);
|
|
auto lease = acquire_region_lease(**runtime, r.key, build, 20000ms, 5s);
|
|
CHECK(lease);
|
|
auto state = evo::test::lineage_region();
|
|
RegionBatch<evo::processes::LineageRegion> batch{*state};
|
|
for (int i = 0; i < 10; ++i)
|
|
CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1)));
|
|
batch.seal();
|
|
Uuid commit{};
|
|
commit.fill(0xDF);
|
|
auto blocked = store(r.root, r.key, unreachable_reserve);
|
|
auto published =
|
|
publish_lineage_batch(**runtime, *blocked, *lease, *head, batch, commit, build, budget, 5s);
|
|
CHECK(!published && published.error().kind == PublicationError::upload &&
|
|
published.error().object == ObjectError::full);
|
|
CHECK(load_region_checkpoint(*db, r.key, 5s)->head == head->head);
|
|
CHECK(release_region_lease(**runtime, r.key, *lease, 5s));
|
|
}
|
|
|
|
// A write the filesystem really refuses (RLIMIT_FSIZE stands in for a full disk:
|
|
// the kernel rejects the write mid-object) is an io error that leaves no object
|
|
// and no temporary behind.
|
|
void failing_writes_leave_nothing() {
|
|
const auto root = fresh("store-efbig");
|
|
auto objects = store(root, {7, 83});
|
|
rlimit old{};
|
|
CHECK(::getrlimit(RLIMIT_FSIZE, &old) == 0);
|
|
const auto previous = std::signal(SIGXFSZ, SIG_IGN);
|
|
rlimit limited = old;
|
|
limited.rlim_cur = 8192;
|
|
CHECK(::setrlimit(RLIMIT_FSIZE, &limited) == 0);
|
|
auto refused = objects->put(bytes_of(65536, 0x30));
|
|
CHECK(::setrlimit(RLIMIT_FSIZE, &old) == 0);
|
|
std::signal(SIGXFSZ, previous);
|
|
CHECK(!refused && refused.error() == ObjectError::io);
|
|
CHECK(entries(objects->directory()) == 0);
|
|
CHECK(objects->put(bytes_of(65536, 0x30))); // The same write succeeds with room.
|
|
}
|
|
|
|
std::vector<std::string> check_store(const fs::path& objects, const std::string& user,
|
|
std::vector<std::string> extra, int& code) {
|
|
std::vector<std::string> args{env("EVO_WORLD_CLONE_BIN"),
|
|
"check-store",
|
|
"--world",
|
|
"7",
|
|
"--region",
|
|
"84",
|
|
"--pg-host",
|
|
env("PGHOST"),
|
|
"--pg-port",
|
|
env("PGPORT"),
|
|
"--pg-database",
|
|
env("PGDATABASE"),
|
|
"--pg-user",
|
|
user,
|
|
"--objects",
|
|
objects.string()};
|
|
args.insert(args.end(), extra.begin(), extra.end());
|
|
Child tool{args};
|
|
code = tool.wait_exit(30s);
|
|
return tool.lines();
|
|
}
|
|
bool has(const std::vector<std::string>& lines, std::string_view prefix) {
|
|
for (const auto& l : lines)
|
|
if (l.starts_with(prefix))
|
|
return true;
|
|
return false;
|
|
}
|
|
|
|
// The in-cluster qualification probe, exercised locally.
|
|
void check_store_qualifies_the_stores() {
|
|
const auto root = fresh("store-check");
|
|
const auto runtime = env("EVO_PG_RUNTIME_USER");
|
|
int code = -1;
|
|
auto first = check_store(root, runtime, {}, code);
|
|
CHECK(code == 0 && has(first, "objects available_mib=") && has(first, "probe created digest="));
|
|
CHECK(has(first, "database version=180006 user=" + runtime + " ssl=false"));
|
|
CHECK(has(first, "database tables=denied functions=reachable head=none"));
|
|
// A later run (a restarted or rescheduled pod) verifies the retained probe.
|
|
auto second = check_store(root, runtime, {}, code);
|
|
CHECK(code == 0 && has(second, "probe verified digest="));
|
|
// A missing or unmounted volume is refused, never created.
|
|
auto missing = check_store(root / "not-mounted", runtime, {}, code);
|
|
CHECK(code == 4 && has(missing, "objects refused missing-or-not-a-directory"));
|
|
CHECK(!fs::exists(root / "not-mounted"));
|
|
// A full volume refuses the probe (fresh namespace, reserve beyond any disk).
|
|
const auto other = fresh("store-check-full");
|
|
auto full = check_store(other, runtime, {"--reserve-mib", "1048576"}, code);
|
|
CHECK(code == 4 && has(full, "probe refused full"));
|
|
// The same check through a libpq service file, as the cluster Jobs run it.
|
|
const auto service = root / "pg_service.conf";
|
|
{
|
|
std::FILE* f = std::fopen(service.c_str(), "wb");
|
|
CHECK(f != nullptr);
|
|
const auto text = "[evolution]\nhost=" + env("PGHOST") + "\nport=" + env("PGPORT") +
|
|
"\ndbname=" + env("PGDATABASE") + "\nuser=" + runtime +
|
|
"\nsslmode=disable\nconnect_timeout=5\n";
|
|
CHECK(std::fwrite(text.data(), 1, text.size(), f) == text.size());
|
|
CHECK(std::fclose(f) == 0);
|
|
}
|
|
Child via_service{{env("EVO_WORLD_CLONE_BIN"), "check-store", "--world", "7", "--region", "84",
|
|
"--pg-service", service.string(), "--objects", root.string()}};
|
|
CHECK(via_service.wait_exit(30s) == 0 && via_service.printed("probe verified digest=") &&
|
|
via_service.printed("database tables=denied functions=reachable"));
|
|
// An owner login can read tables: not an acceptable runtime identity.
|
|
auto owner_run = check_store(root, env("PGUSER"), {}, code);
|
|
CHECK(code == 3 && has(owner_run, "database tables=READABLE"));
|
|
}
|
|
|
|
// A real server on a full volume keeps simulating but never acknowledges, and its
|
|
// bounded drain gives up without committing anything.
|
|
void full_volume_server_acknowledges_nothing() {
|
|
const auto r = provision(85, 0xE7);
|
|
const auto before = head_of(r.key);
|
|
Child server{server_args(r, {"--objects-reserve-mib", "1048576", "--drain-ms", "1500",
|
|
"--ops-listen", "127.0.0.1:0"})};
|
|
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
|
server.send(input_lines(1, 15, false));
|
|
server.pump(3000ms);
|
|
CHECK(!server.printed("ack "));
|
|
server.signal(SIGTERM);
|
|
CHECK(server.wait_exit(20s) == 3); // Drain deadline passed with the batch unpublished.
|
|
CHECK(!server.printed("ack "));
|
|
const auto after = head_of(r.key);
|
|
CHECK(after.tick == before.tick && after.sequence == before.sequence);
|
|
}
|
|
// M8.4: the provisioning tool creates a region once from the development opening;
|
|
// a rerun (every Argo sync) leaves it untouched, and a server then serves it.
|
|
void provisioning_is_idempotent() {
|
|
const ServerRegion r{{7, 86}, fresh("provision-86")};
|
|
const auto run = [&](const std::string& user, int& code) {
|
|
Child tool{{env("EVO_WORLD_CLONE_BIN"),
|
|
"provision",
|
|
"--world",
|
|
"7",
|
|
"--region",
|
|
"86",
|
|
"--opening",
|
|
"development-v1",
|
|
"--pg-host",
|
|
env("PGHOST"),
|
|
"--pg-port",
|
|
env("PGPORT"),
|
|
"--pg-database",
|
|
env("PGDATABASE"),
|
|
"--pg-user",
|
|
user,
|
|
"--objects",
|
|
r.root.string(),
|
|
"--build",
|
|
"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1",
|
|
"--retain-build",
|
|
"true"}};
|
|
code = tool.wait_exit(30s);
|
|
return tool.lines();
|
|
};
|
|
int code = -1;
|
|
auto denied = run(env("EVO_PG_RUNTIME_USER"), code);
|
|
CHECK(code != 0 && !has(denied, "provisioned "));
|
|
auto first = run(env("PGUSER"), code);
|
|
CHECK(code == 0 && has(first, "provisioned world=7 region=86 sequence=0 tick=0"));
|
|
auto second = run(env("PGUSER"), code);
|
|
CHECK(code == 0 && has(second, "already-provisioned world=7 region=86 sequence=0 tick=0"));
|
|
Child server{server_args(r)};
|
|
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
|
server.send(input_lines(1, 5, false));
|
|
CHECK(server.wait_line("ack " + request_hex(5) + " 5 ", 15s));
|
|
server.signal(SIGTERM);
|
|
CHECK(server.wait_exit(20s) == 0);
|
|
auto after = run(env("PGUSER"), code);
|
|
CHECK(code == 0 && has(after, "already-provisioned world=7 region=86 sequence=1 tick=5"));
|
|
}
|
|
} // namespace
|
|
|
|
void run_region_store_tests() {
|
|
provisioning_is_idempotent();
|
|
full_volumes_publish_nothing();
|
|
failing_writes_leave_nothing();
|
|
check_store_qualifies_the_stores();
|
|
full_volume_server_acknowledges_nothing();
|
|
}
|