Files
aevolution/tests/region_store_tests.cpp
T
dingoandClaude Opus 5.5 b073ac7672
Foundation / checks (client) (push) Successful in 5s
Foundation / checks (network) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 1m56s
Foundation / checks (deploy) (push) Successful in 3m36s
Foundation / checks (tsan) (push) Successful in 8m50s
Foundation / checks (checks) (push) Successful in 37m4s
Foundation / checks (postgres) (push) Failing after 39m46s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
Give primitive provisioning fixture its own region
The primitive provisioning test provisioned world 7 region 87, which the
synthetic load suite provisions again later in the same disposable
cluster, failing with region_leases_pkey SQLSTATE 23505. Move it to the
unused region 89. Refresh the dev session window.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 13:02:44 -04:00

358 lines
16 KiB
C++

// M8.3 store qualification (ADR-0070) against real PostgreSQL and real processes:
// the object volume's free-space reserve, a genuinely failing write, the
// check-store probe (retention across runs, missing-volume refusal, privilege
// boundary) and a server whose volume is full. Run by tools/ci/postgres-check.sh.
#include "evo/persistence/checkpoint_store.hpp"
#include "evo/persistence/region_checkpoint.hpp"
#include "evo/server/lifecycle/primitive_lives.hpp"
#include "fixtures/server_process.hpp"
#include <algorithm>
#include <csignal>
#include <source_location>
#include <sys/resource.h>
void run_region_store_tests();
namespace {
using namespace evo::test::process;
using namespace evo::persistence;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region store test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
constexpr std::size_t budget = 8000000;
constexpr std::uint64_t unreachable_reserve = std::uint64_t{1} << 60U; // 1 EiB.
std::unique_ptr<FilesystemObjectStore> store(const fs::path& root, RegionKey key,
std::uint64_t reserve = 0) {
auto made = FilesystemObjectStore::open(root, key, max_lineage_payload_bytes, nullptr, reserve);
CHECK(made);
return std::move(*made);
}
fs::path fresh(const std::string& name) {
const auto root = fs::path(env("PGHOST")) / name;
fs::remove_all(root);
fs::create_directory(root);
fs::permissions(root, fs::perms::owner_all, fs::perm_options::replace);
return root;
}
std::size_t entries(const fs::path& dir) {
std::size_t n = 0;
for ([[maybe_unused]] const auto& entry : fs::directory_iterator(dir))
++n;
return n;
}
std::vector<std::uint8_t> bytes_of(std::size_t size, std::uint8_t seed) {
std::vector<std::uint8_t> out(size);
for (std::size_t i = 0; i < size; ++i)
out[i] = static_cast<std::uint8_t>(seed + (i * 7U));
return out;
}
// A full volume refuses new objects before writing; identical objects already
// stored stay usable; a batch against a full volume publishes nothing.
void full_volumes_publish_nothing() {
const RegionKey key{7, 81};
const auto root = fresh("store-full");
const auto stored = bytes_of(4096, 0x10);
auto roomy = store(root, key);
CHECK(roomy->available_bytes().value_or(0) > 0 && roomy->reserve_bytes() == 0);
CHECK(roomy->put(stored));
auto full = store(root, key, unreachable_reserve);
CHECK(full->reserve_bytes() == unreachable_reserve);
const auto before = entries(full->directory());
auto refused = full->put(bytes_of(4096, 0x20));
CHECK(!refused && refused.error() == ObjectError::full);
CHECK(entries(full->directory()) == before); // No temporary, no object.
CHECK(full->put(stored)); // Reuse needs no space.
// A lineage batch cannot publish through a full volume; the head stays.
const auto r = provision(82, 0xDE);
auto db = owner();
const auto head = load_region_checkpoint(*db, r.key, 5s);
CHECK(head);
auto runtime = Postgres::connect(
{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("EVO_PG_RUNTIME_USER"), {}, {}},
5s);
CHECK(runtime);
Sha256Digest build{};
build.fill(0xB1);
auto lease = acquire_region_lease(**runtime, r.key, build, 20000ms, 5s);
CHECK(lease);
auto state = evo::test::lineage_region();
RegionBatch<evo::processes::LineageRegion> batch{*state};
for (int i = 0; i < 10; ++i)
CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1)));
batch.seal();
Uuid commit{};
commit.fill(0xDF);
auto blocked = store(r.root, r.key, unreachable_reserve);
auto published =
publish_lineage_batch(**runtime, *blocked, *lease, *head, batch, commit, build, budget, 5s);
CHECK(!published && published.error().kind == PublicationError::upload &&
published.error().object == ObjectError::full);
CHECK(load_region_checkpoint(*db, r.key, 5s)->head == head->head);
CHECK(release_region_lease(**runtime, r.key, *lease, 5s));
}
// A write the filesystem really refuses (RLIMIT_FSIZE stands in for a full disk:
// the kernel rejects the write mid-object) is an io error that leaves no object
// and no temporary behind.
void failing_writes_leave_nothing() {
const auto root = fresh("store-efbig");
auto objects = store(root, {7, 83});
rlimit old{};
CHECK(::getrlimit(RLIMIT_FSIZE, &old) == 0);
const auto previous = std::signal(SIGXFSZ, SIG_IGN);
rlimit limited = old;
limited.rlim_cur = 8192;
CHECK(::setrlimit(RLIMIT_FSIZE, &limited) == 0);
auto refused = objects->put(bytes_of(65536, 0x30));
CHECK(::setrlimit(RLIMIT_FSIZE, &old) == 0);
std::signal(SIGXFSZ, previous);
CHECK(!refused && refused.error() == ObjectError::io);
CHECK(entries(objects->directory()) == 0);
CHECK(objects->put(bytes_of(65536, 0x30))); // The same write succeeds with room.
}
std::vector<std::string> check_store(const fs::path& objects, const std::string& user,
std::vector<std::string> extra, int& code) {
std::vector<std::string> args{env("EVO_WORLD_CLONE_BIN"),
"check-store",
"--world",
"7",
"--region",
"84",
"--pg-host",
env("PGHOST"),
"--pg-port",
env("PGPORT"),
"--pg-database",
env("PGDATABASE"),
"--pg-user",
user,
"--objects",
objects.string()};
args.insert(args.end(), extra.begin(), extra.end());
Child tool{args};
code = tool.wait_exit(30s);
return tool.lines();
}
bool has(const std::vector<std::string>& lines, std::string_view prefix) {
for (const auto& l : lines)
if (l.starts_with(prefix))
return true;
return false;
}
// The in-cluster qualification probe, exercised locally.
void check_store_qualifies_the_stores() {
const auto root = fresh("store-check");
const auto runtime = env("EVO_PG_RUNTIME_USER");
int code = -1;
auto first = check_store(root, runtime, {}, code);
CHECK(code == 0 && has(first, "objects available_mib=") && has(first, "probe created digest="));
CHECK(has(first, "database version=180006 user=" + runtime + " ssl=false"));
CHECK(has(first, "database tables=denied functions=reachable head=none"));
// A later run (a restarted or rescheduled pod) verifies the retained probe.
auto second = check_store(root, runtime, {}, code);
CHECK(code == 0 && has(second, "probe verified digest="));
// A missing or unmounted volume is refused, never created.
auto missing = check_store(root / "not-mounted", runtime, {}, code);
CHECK(code == 4 && has(missing, "objects refused missing-or-not-a-directory"));
CHECK(!fs::exists(root / "not-mounted"));
// A full volume refuses the probe (fresh namespace, reserve beyond any disk).
const auto other = fresh("store-check-full");
auto full = check_store(other, runtime, {"--reserve-mib", "1048576"}, code);
CHECK(code == 4 && has(full, "probe refused full"));
// The same check through a libpq service file, as the cluster Jobs run it.
const auto service = root / "pg_service.conf";
{
std::FILE* f = std::fopen(service.c_str(), "wb");
CHECK(f != nullptr);
const auto text = "[evolution]\nhost=" + env("PGHOST") + "\nport=" + env("PGPORT") +
"\ndbname=" + env("PGDATABASE") + "\nuser=" + runtime +
"\nsslmode=disable\nconnect_timeout=5\n";
CHECK(std::fwrite(text.data(), 1, text.size(), f) == text.size());
CHECK(std::fclose(f) == 0);
}
Child via_service{{env("EVO_WORLD_CLONE_BIN"), "check-store", "--world", "7", "--region", "84",
"--pg-service", service.string(), "--objects", root.string()}};
CHECK(via_service.wait_exit(30s) == 0 && via_service.printed("probe verified digest=") &&
via_service.printed("database tables=denied functions=reachable"));
// An owner login can read tables: not an acceptable runtime identity.
auto owner_run = check_store(root, env("PGUSER"), {}, code);
CHECK(code == 3 && has(owner_run, "database tables=READABLE"));
}
// A real server on a full volume keeps simulating but never acknowledges, and its
// bounded drain gives up without committing anything.
void full_volume_server_acknowledges_nothing() {
const auto r = provision(85, 0xE7);
const auto before = head_of(r.key);
Child server{server_args(r, {"--objects-reserve-mib", "1048576", "--drain-ms", "1500",
"--ops-listen", "127.0.0.1:0"})};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
server.send(input_lines(1, 15, false));
server.pump(3000ms);
CHECK(!server.printed("ack "));
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 3); // Drain deadline passed with the batch unpublished.
CHECK(!server.printed("ack "));
const auto after = head_of(r.key);
CHECK(after.tick == before.tick && after.sequence == before.sequence);
}
// M8.4: the provisioning tool creates a region once from the development opening;
// a rerun (every Argo sync) leaves it untouched, and a server then serves it.
void provisioning_is_idempotent() {
const ServerRegion r{{7, 86}, fresh("provision-86")};
const auto run = [&](const std::string& user, int& code) {
Child tool{{env("EVO_WORLD_CLONE_BIN"),
"provision",
"--world",
"7",
"--region",
"86",
"--opening",
"development-v1",
"--pg-host",
env("PGHOST"),
"--pg-port",
env("PGPORT"),
"--pg-database",
env("PGDATABASE"),
"--pg-user",
user,
"--objects",
r.root.string(),
"--build",
"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1",
"--retain-build",
"true"}};
code = tool.wait_exit(30s);
return tool.lines();
};
int code = -1;
auto denied = run(env("EVO_PG_RUNTIME_USER"), code);
CHECK(code != 0 && !has(denied, "provisioned "));
auto first = run(env("PGUSER"), code);
CHECK(code == 0 && has(first, "provisioned world=7 region=86 sequence=0 tick=0"));
auto second = run(env("PGUSER"), code);
CHECK(code == 0 && has(second, "already-provisioned world=7 region=86 sequence=0 tick=0"));
Child server{server_args(r)};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
server.send(input_lines(1, 5, false));
CHECK(server.wait_line("ack " + request_hex(5) + " 5 ", 15s));
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 0);
auto after = run(env("PGUSER"), code);
CHECK(code == 0 && has(after, "already-provisioned world=7 region=86 sequence=1 tick=5"));
}
// M11.2d-4: primitive genesis requires an explicit owner opening, and retries
// cannot mistake a different profile or frame for an idempotent success.
void primitive_provisioning_is_explicit_and_profile_safe() {
using namespace evo::server;
using evo::biology::primitive_endowment;
const ServerRegion r{{7, 89}, fresh("primitive-provision-89")};
const auto endowed = primitive_endowment();
CHECK(endowed);
evo::accounting::Inventory funding = endowed->body;
funding.energy.value += endowed->reserve.energy.value + endowed->construction.value;
const auto run = [&](const std::string& user, std::uint64_t frame, bool complete,
bool retain_build, int& code, std::uint64_t energy_delta = 0) {
std::vector<std::string> args{
env("EVO_WORLD_CLONE_BIN"),
"provision",
"--world",
"7",
"--region",
"89",
"--opening",
"primitive-v1",
"--pg-host",
env("PGHOST"),
"--pg-port",
env("PGPORT"),
"--pg-database",
env("PGDATABASE"),
"--pg-user",
user,
"--objects",
r.root.string(),
"--build",
"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2",
"--frame",
std::to_string(frame),
"--source-id",
"1",
"--funding-energy-pj",
std::to_string(funding.energy.value + energy_delta),
"--funding-carbon-fg",
std::to_string(funding.matter[0].value),
"--funding-nitrogen-fg",
std::to_string(funding.matter[1].value),
"--funding-phosphorus-fg",
std::to_string(funding.matter[2].value),
"--funding-other-fg",
std::to_string(funding.matter[3].value),
"--site",
"0,0",
"--site",
"10000000,0"};
if (retain_build) {
args.insert(args.end() - 6, {"--retain-build", "true"});
}
if (!complete) {
const auto field = std::find(args.begin(), args.end(), "--funding-other-fg");
args.erase(field, field + 2);
}
Child tool{args};
code = tool.wait_exit(30s);
return tool.lines();
};
auto db = owner();
CHECK(db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('c2',32),'hex')) ON CONFLICT DO NOTHING",
{}, 5s));
int code = -1;
const auto denied = run(env("EVO_PG_RUNTIME_USER"), 3, true, false, code);
CHECK(code != 0 && !has(denied, "provisioned "));
const auto first = run(env("PGUSER"), 3, true, true, code);
CHECK(code == 0 && has(first, "provisioned world=7 region=89 sequence=0 tick=0"));
const auto retry = run(env("PGUSER"), 3, true, true, code);
CHECK(code == 0 && has(retry, "already-provisioned world=7 region=89 sequence=0 tick=0"));
const auto mismatch = run(env("PGUSER"), 4, true, true, code);
CHECK(code == 4 && has(mismatch, "provision refused existing profile/frame mismatch"));
const auto changed_opening = run(env("PGUSER"), 3, true, true, code, 1);
CHECK(code == 4 && has(changed_opening, "provision refused existing opening mismatch"));
const auto incomplete = run(env("PGUSER"), 3, false, false, code);
CHECK(code == 2);
const auto committed = load_region_checkpoint(*db, r.key, 5s);
CHECK(committed);
const auto manifest = decode_manifest(committed->manifest);
CHECK(manifest && manifest->profile == CheckpointProfile::primitive_lives_v1 &&
manifest->frame == 3);
auto objects =
FilesystemObjectStore::open(r.root, r.key, evo::server::max_primitive_lives_bytes);
CHECK(objects);
Sha256Digest build{};
build.fill(0xC2);
const std::array<Sha256Digest, 1> retained{build};
const auto recovered = recover_primitive_checkpoint(*db, **objects, r.key, retained, 5s);
CHECK(recovered && recovered->state->config().source == 1 &&
recovered->state->config().funding == funding &&
recovered->state->config().sites == std::vector<SpawnSite>{{0, 0}, {10000000, 0}});
}
} // namespace
void run_region_store_tests() {
provisioning_is_idempotent();
primitive_provisioning_is_explicit_and_profile_safe();
full_volumes_publish_nothing();
failing_writes_leave_nothing();
check_store_qualifies_the_stores();
full_volume_server_acknowledges_nothing();
}