Files
aevolution/tests/region_store_tests.cpp
T
dingoandClaude Opus 5.5 5e45be3b6b
Foundation / checks (checks) (push) Successful in 40m17s
Foundation / checks (deploy) (push) Successful in 3m54s
Foundation / checks (ecs) (push) Successful in 2m24s
Foundation / checks (network) (push) Successful in 18m40s
Foundation / checks (postgres) (push) Successful in 42m11s
Foundation / checks (tsan) (push) Successful in 9m39s
Foundation / checks (client) (push) Successful in 13m33s
Foundation / postgres-tools (push) Successful in 1m13s
Foundation / image (push) Successful in 3m10s
Foundation / server-image (push) Successful in 3m16s
Resolve a cellular genesis under the build that wrote it
Promoting the dev server image changed --build, and the cellular
provision rerun re-encoded the genesis manifest with the new build, so it
refused region 100/2 as a different genesis. The check could also never
pass once the head moved past genesis. Match an unadvanced cellular
genesis under its recorded build, and accept profile and frame past
genesis as the other openings do. A different commit or opening is still
refused. Add a PostgreSQL process test and record the finding in ADR-0123.
Also refresh the dev session window (3 h).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 13:14:15 -04:00

523 lines
24 KiB
C++

// M8.3 store qualification (ADR-0070) against real PostgreSQL and real processes:
// the object volume's free-space reserve, a genuinely failing write, the
// check-store probe (retention across runs, missing-volume refusal, privilege
// boundary) and a server whose volume is full. Run by tools/ci/postgres-check.sh.
#include "evo/persistence/checkpoint_store.hpp"
#include "evo/persistence/region_checkpoint.hpp"
#include "evo/server/lifecycle/primitive_lives.hpp"
#include "fixtures/server_process.hpp"
#include <algorithm>
#include <csignal>
#include <source_location>
#include <sys/resource.h>
void run_region_store_tests();
namespace {
using namespace evo::test::process;
using namespace evo::persistence;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region store test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
constexpr std::size_t budget = 8000000;
constexpr std::uint64_t unreachable_reserve = std::uint64_t{1} << 60U; // 1 EiB.
std::unique_ptr<FilesystemObjectStore> store(const fs::path& root, RegionKey key,
std::uint64_t reserve = 0) {
auto made = FilesystemObjectStore::open(root, key, max_lineage_payload_bytes, nullptr, reserve);
CHECK(made);
return std::move(*made);
}
fs::path fresh(const std::string& name) {
const auto root = fs::path(env("PGHOST")) / name;
fs::remove_all(root);
fs::create_directory(root);
fs::permissions(root, fs::perms::owner_all, fs::perm_options::replace);
return root;
}
std::size_t entries(const fs::path& dir) {
std::size_t n = 0;
for ([[maybe_unused]] const auto& entry : fs::directory_iterator(dir))
++n;
return n;
}
std::vector<std::uint8_t> bytes_of(std::size_t size, std::uint8_t seed) {
std::vector<std::uint8_t> out(size);
for (std::size_t i = 0; i < size; ++i)
out[i] = static_cast<std::uint8_t>(seed + (i * 7U));
return out;
}
// A full volume refuses new objects before writing; identical objects already
// stored stay usable; a batch against a full volume publishes nothing.
void full_volumes_publish_nothing() {
const RegionKey key{7, 81};
const auto root = fresh("store-full");
const auto stored = bytes_of(4096, 0x10);
auto roomy = store(root, key);
CHECK(roomy->available_bytes().value_or(0) > 0 && roomy->reserve_bytes() == 0);
CHECK(roomy->put(stored));
auto full = store(root, key, unreachable_reserve);
CHECK(full->reserve_bytes() == unreachable_reserve);
const auto before = entries(full->directory());
auto refused = full->put(bytes_of(4096, 0x20));
CHECK(!refused && refused.error() == ObjectError::full);
CHECK(entries(full->directory()) == before); // No temporary, no object.
CHECK(full->put(stored)); // Reuse needs no space.
// A lineage batch cannot publish through a full volume; the head stays.
const auto r = provision(82, 0xDE);
auto db = owner();
const auto head = load_region_checkpoint(*db, r.key, 5s);
CHECK(head);
auto runtime = Postgres::connect(
{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("EVO_PG_RUNTIME_USER"), {}, {}},
5s);
CHECK(runtime);
Sha256Digest build{};
build.fill(0xB1);
auto lease = acquire_region_lease(**runtime, r.key, build, 20000ms, 5s);
CHECK(lease);
auto state = evo::test::lineage_region();
RegionBatch<evo::processes::LineageRegion> batch{*state};
for (int i = 0; i < 10; ++i)
CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1)));
batch.seal();
Uuid commit{};
commit.fill(0xDF);
auto blocked = store(r.root, r.key, unreachable_reserve);
auto published =
publish_lineage_batch(**runtime, *blocked, *lease, *head, batch, commit, build, budget, 5s);
CHECK(!published && published.error().kind == PublicationError::upload &&
published.error().object == ObjectError::full);
CHECK(load_region_checkpoint(*db, r.key, 5s)->head == head->head);
CHECK(release_region_lease(**runtime, r.key, *lease, 5s));
}
// A write the filesystem really refuses (RLIMIT_FSIZE stands in for a full disk:
// the kernel rejects the write mid-object) is an io error that leaves no object
// and no temporary behind.
void failing_writes_leave_nothing() {
const auto root = fresh("store-efbig");
auto objects = store(root, {7, 83});
rlimit old{};
CHECK(::getrlimit(RLIMIT_FSIZE, &old) == 0);
const auto previous = std::signal(SIGXFSZ, SIG_IGN);
rlimit limited = old;
limited.rlim_cur = 8192;
CHECK(::setrlimit(RLIMIT_FSIZE, &limited) == 0);
auto refused = objects->put(bytes_of(65536, 0x30));
CHECK(::setrlimit(RLIMIT_FSIZE, &old) == 0);
std::signal(SIGXFSZ, previous);
CHECK(!refused && refused.error() == ObjectError::io);
CHECK(entries(objects->directory()) == 0);
CHECK(objects->put(bytes_of(65536, 0x30))); // The same write succeeds with room.
}
std::vector<std::string> check_store(const fs::path& objects, const std::string& user,
std::vector<std::string> extra, int& code) {
std::vector<std::string> args{env("EVO_WORLD_CLONE_BIN"),
"check-store",
"--world",
"7",
"--region",
"84",
"--pg-host",
env("PGHOST"),
"--pg-port",
env("PGPORT"),
"--pg-database",
env("PGDATABASE"),
"--pg-user",
user,
"--objects",
objects.string()};
args.insert(args.end(), extra.begin(), extra.end());
Child tool{args};
code = tool.wait_exit(30s);
return tool.lines();
}
bool has(const std::vector<std::string>& lines, std::string_view prefix) {
for (const auto& l : lines)
if (l.starts_with(prefix))
return true;
return false;
}
// The in-cluster qualification probe, exercised locally.
void check_store_qualifies_the_stores() {
const auto root = fresh("store-check");
const auto runtime = env("EVO_PG_RUNTIME_USER");
int code = -1;
auto first = check_store(root, runtime, {}, code);
CHECK(code == 0 && has(first, "objects available_mib=") && has(first, "probe created digest="));
CHECK(has(first, "database version=180006 user=" + runtime + " ssl=false"));
CHECK(has(first, "database tables=denied functions=reachable head=none"));
// A later run (a restarted or rescheduled pod) verifies the retained probe.
auto second = check_store(root, runtime, {}, code);
CHECK(code == 0 && has(second, "probe verified digest="));
// A missing or unmounted volume is refused, never created.
auto missing = check_store(root / "not-mounted", runtime, {}, code);
CHECK(code == 4 && has(missing, "objects refused missing-or-not-a-directory"));
CHECK(!fs::exists(root / "not-mounted"));
// A full volume refuses the probe (fresh namespace, reserve beyond any disk).
const auto other = fresh("store-check-full");
auto full = check_store(other, runtime, {"--reserve-mib", "1048576"}, code);
CHECK(code == 4 && has(full, "probe refused full"));
// The same check through a libpq service file, as the cluster Jobs run it.
const auto service = root / "pg_service.conf";
{
std::FILE* f = std::fopen(service.c_str(), "wb");
CHECK(f != nullptr);
const auto text = "[evolution]\nhost=" + env("PGHOST") + "\nport=" + env("PGPORT") +
"\ndbname=" + env("PGDATABASE") + "\nuser=" + runtime +
"\nsslmode=disable\nconnect_timeout=5\n";
CHECK(std::fwrite(text.data(), 1, text.size(), f) == text.size());
CHECK(std::fclose(f) == 0);
}
Child via_service{{env("EVO_WORLD_CLONE_BIN"), "check-store", "--world", "7", "--region", "84",
"--pg-service", service.string(), "--objects", root.string()}};
CHECK(via_service.wait_exit(30s) == 0 && via_service.printed("probe verified digest=") &&
via_service.printed("database tables=denied functions=reachable"));
// An owner login can read tables: not an acceptable runtime identity.
auto owner_run = check_store(root, env("PGUSER"), {}, code);
CHECK(code == 3 && has(owner_run, "database tables=READABLE"));
}
// A real server on a full volume keeps simulating but never acknowledges, and its
// bounded drain gives up without committing anything.
void full_volume_server_acknowledges_nothing() {
const auto r = provision(85, 0xE7);
const auto before = head_of(r.key);
Child server{server_args(r, {"--objects-reserve-mib", "1048576", "--drain-ms", "1500",
"--ops-listen", "127.0.0.1:0"})};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
server.send(input_lines(1, 15, false));
server.pump(3000ms);
CHECK(!server.printed("ack "));
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 3); // Drain deadline passed with the batch unpublished.
CHECK(!server.printed("ack "));
const auto after = head_of(r.key);
CHECK(after.tick == before.tick && after.sequence == before.sequence);
}
// M8.4: the provisioning tool creates a region once from the development opening;
// a rerun (every Argo sync) leaves it untouched, and a server then serves it.
void provisioning_is_idempotent() {
const ServerRegion r{{7, 86}, fresh("provision-86")};
const auto run = [&](const std::string& user, int& code) {
Child tool{{env("EVO_WORLD_CLONE_BIN"),
"provision",
"--world",
"7",
"--region",
"86",
"--opening",
"development-v1",
"--pg-host",
env("PGHOST"),
"--pg-port",
env("PGPORT"),
"--pg-database",
env("PGDATABASE"),
"--pg-user",
user,
"--objects",
r.root.string(),
"--build",
"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1",
"--retain-build",
"true"}};
code = tool.wait_exit(30s);
return tool.lines();
};
int code = -1;
auto denied = run(env("EVO_PG_RUNTIME_USER"), code);
CHECK(code != 0 && !has(denied, "provisioned "));
auto first = run(env("PGUSER"), code);
CHECK(code == 0 && has(first, "provisioned world=7 region=86 sequence=0 tick=0"));
auto second = run(env("PGUSER"), code);
CHECK(code == 0 && has(second, "already-provisioned world=7 region=86 sequence=0 tick=0"));
Child server{server_args(r)};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
server.send(input_lines(1, 5, false));
CHECK(server.wait_line("ack " + request_hex(5) + " 5 ", 15s));
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 0);
auto after = run(env("PGUSER"), code);
CHECK(code == 0 && has(after, "already-provisioned world=7 region=86 sequence=1 tick=5"));
}
// M11.2d-4: primitive genesis requires an explicit owner opening, and retries
// cannot mistake a different profile or frame for an idempotent success.
void primitive_provisioning_is_explicit_and_profile_safe() {
using namespace evo::server;
using evo::biology::primitive_endowment;
const ServerRegion r{{7, 89}, fresh("primitive-provision-89")};
const auto endowed = primitive_endowment();
CHECK(endowed);
evo::accounting::Inventory funding = endowed->body;
funding.energy.value += endowed->reserve.energy.value + endowed->construction.value;
const auto run = [&](const std::string& user, std::uint64_t frame, bool complete,
bool retain_build, int& code, std::uint64_t energy_delta = 0) {
std::vector<std::string> args{
env("EVO_WORLD_CLONE_BIN"),
"provision",
"--world",
"7",
"--region",
"89",
"--opening",
"primitive-v1",
"--pg-host",
env("PGHOST"),
"--pg-port",
env("PGPORT"),
"--pg-database",
env("PGDATABASE"),
"--pg-user",
user,
"--objects",
r.root.string(),
"--build",
"c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2",
"--frame",
std::to_string(frame),
"--source-id",
"1",
"--funding-energy-pj",
std::to_string(funding.energy.value + energy_delta),
"--funding-carbon-fg",
std::to_string(funding.matter[0].value),
"--funding-nitrogen-fg",
std::to_string(funding.matter[1].value),
"--funding-phosphorus-fg",
std::to_string(funding.matter[2].value),
"--funding-other-fg",
std::to_string(funding.matter[3].value),
"--site",
"0,0",
"--site",
"10000000,0"};
if (retain_build) {
args.insert(args.end() - 6, {"--retain-build", "true"});
}
if (!complete) {
const auto field = std::find(args.begin(), args.end(), "--funding-other-fg");
args.erase(field, field + 2);
}
Child tool{args};
code = tool.wait_exit(30s);
return tool.lines();
};
auto db = owner();
CHECK(db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('c2',32),'hex')) ON CONFLICT DO NOTHING",
{}, 5s));
int code = -1;
const auto denied = run(env("EVO_PG_RUNTIME_USER"), 3, true, false, code);
CHECK(code != 0 && !has(denied, "provisioned "));
const auto first = run(env("PGUSER"), 3, true, true, code);
CHECK(code == 0 && has(first, "provisioned world=7 region=89 sequence=0 tick=0"));
const auto retry = run(env("PGUSER"), 3, true, true, code);
CHECK(code == 0 && has(retry, "already-provisioned world=7 region=89 sequence=0 tick=0"));
const auto mismatch = run(env("PGUSER"), 4, true, true, code);
CHECK(code == 4 && has(mismatch, "provision refused existing profile/frame mismatch"));
const auto changed_opening = run(env("PGUSER"), 3, true, true, code, 1);
CHECK(code == 4 && has(changed_opening, "provision refused existing opening mismatch"));
const auto incomplete = run(env("PGUSER"), 3, false, false, code);
CHECK(code == 2);
const auto committed = load_region_checkpoint(*db, r.key, 5s);
CHECK(committed);
const auto manifest = decode_manifest(committed->manifest);
CHECK(manifest && manifest->profile == CheckpointProfile::primitive_lives_v1 &&
manifest->frame == 3);
auto objects =
FilesystemObjectStore::open(r.root, r.key, evo::server::max_primitive_lives_bytes);
CHECK(objects);
Sha256Digest build{};
build.fill(0xC2);
const std::array<Sha256Digest, 1> retained{build};
const auto recovered = recover_primitive_checkpoint(*db, **objects, r.key, retained, 5s);
CHECK(recovered && recovered->state->config().source == 1 &&
recovered->state->config().funding == funding &&
recovered->state->config().sites == std::vector<SpawnSite>{{0, 0}, {10000000, 0}});
}
// ADR-0123: provisioning creates the world's life allocator once at the audited
// start; reruns leave it untouched, and a start above the existing counter refuses.
void provisioning_creates_the_life_allocator_once() {
using evo::biology::primitive_endowment;
const ServerRegion r{{9760, 1}, fresh("allocator-9760")};
const auto endowed = primitive_endowment();
CHECK(endowed);
evo::accounting::Inventory funding = endowed->body;
funding.energy.value += endowed->reserve.energy.value + endowed->construction.value;
const auto run = [&](const std::string& opening, std::uint64_t start, int& code) {
std::vector<std::string> args{
env("EVO_WORLD_CLONE_BIN"),
"provision",
"--world",
"9760",
"--region",
"1",
"--opening",
opening,
"--pg-host",
env("PGHOST"),
"--pg-port",
env("PGPORT"),
"--pg-database",
env("PGDATABASE"),
"--pg-user",
env("PGUSER"),
"--objects",
r.root.string(),
"--build",
"c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3",
"--retain-build",
"true",
"--life-allocator-start",
std::to_string(start)};
if (opening == "primitive-v1") {
const std::vector<std::string> primitive{"--frame",
"3",
"--source-id",
"1",
"--funding-energy-pj",
std::to_string(funding.energy.value),
"--funding-carbon-fg",
std::to_string(funding.matter[0].value),
"--funding-nitrogen-fg",
std::to_string(funding.matter[1].value),
"--funding-phosphorus-fg",
std::to_string(funding.matter[2].value),
"--funding-other-fg",
std::to_string(funding.matter[3].value),
"--site",
"0,0"};
args.insert(args.end(), primitive.begin(), primitive.end());
}
Child tool{args};
code = tool.wait_exit(30s);
return tool.lines();
};
const auto next_identity = [] {
auto db = owner();
auto rows = db->query(
"SELECT next_identity FROM evolution.life_allocators WHERE world_id = 9760", {}, 5s);
if (!rows || rows->size() != 1 || (*rows)[0].size() != 1)
return std::string{};
return (*rows)[0][0].value_or(std::string{});
};
int code = -1;
const auto development = run("development-v1", 5000, code);
CHECK(code == 2 && next_identity().empty()); // No lives, so no allocator.
const auto first = run("primitive-v1", 5000, code);
CHECK(code == 0 && has(first, "life-allocator world=9760 next=5000") &&
has(first, "provisioned world=9760 region=1 sequence=0 tick=0"));
const auto retry = run("primitive-v1", 5000, code);
CHECK(code == 0 && has(retry, "life-allocator world=9760 next=5000") &&
has(retry, "already-provisioned world=9760 region=1 sequence=0 tick=0"));
const auto lower = run("primitive-v1", 10, code);
CHECK(code == 0 && has(lower, "life-allocator world=9760 next=5000"));
const auto higher = run("primitive-v1", 6000, code);
CHECK(code == 4 && has(higher, "life-allocator refused existing below start"));
CHECK(next_identity() == "5000");
}
// ADR-0123: an image promotion changes --build, but a cellular genesis records the
// build that wrote it. A rerun under a new build still resolves the same genesis,
// while a different commit or opening is refused.
void cellular_provisioning_survives_a_build_promotion() {
using evo::biology::primitive_endowment;
const ServerRegion r{{9761, 1}, fresh("cellular-provision-9761")};
const auto endowed = primitive_endowment();
CHECK(endowed);
evo::accounting::Inventory funding = endowed->body;
funding.energy.value += endowed->reserve.energy.value + endowed->construction.value;
const auto run = [&](const std::string& build, const std::string& commit,
const std::string& patch, int& code) {
Child tool{{env("EVO_WORLD_CLONE_BIN"),
"provision",
"--world",
"9761",
"--region",
"1",
"--opening",
"cellular-v1",
"--pg-host",
env("PGHOST"),
"--pg-port",
env("PGPORT"),
"--pg-database",
env("PGDATABASE"),
"--pg-user",
env("PGUSER"),
"--objects",
r.root.string(),
"--build",
build,
"--retain-build",
"true",
"--commit",
commit,
"--frame",
"9",
"--source-id",
"1",
"--funding-energy-pj",
std::to_string(funding.energy.value),
"--funding-carbon-fg",
std::to_string(funding.matter[0].value),
"--funding-nitrogen-fg",
std::to_string(funding.matter[1].value),
"--funding-phosphorus-fg",
std::to_string(funding.matter[2].value),
"--funding-other-fg",
std::to_string(funding.matter[3].value),
"--site",
"2500,2500",
"--width-um",
"10000",
"--height-um",
"10000",
"--ambient-temperature-mk",
"293150",
"--patch",
patch}};
code = tool.wait_exit(30s);
return tool.lines();
};
const std::string first_build(64, 'c');
const std::string second_build = std::string(63, 'c') + "d";
const std::string commit(32, 'a');
const std::string patch = "3000,3000,2,1000,1000,1000,1000,1000";
int code = -1;
const auto first = run(first_build, commit, patch, code);
CHECK(code == 0 && has(first, "provisioned world=9761 region=1 sequence=0 tick=0"));
const auto promoted = run(second_build, commit, patch, code);
CHECK(code == 0 && has(promoted, "already-provisioned world=9761 region=1 sequence=0 tick=0"));
const auto other_commit = run(second_build, std::string(32, 'b'), patch, code);
CHECK(code == 4 && has(other_commit, "provision refused existing cellular region; use its "
"original --commit to resolve genesis"));
const auto other_opening =
run(second_build, commit, "4000,4000,2,1000,1000,1000,1000,1000", code);
CHECK(code == 4 && has(other_opening, "provision refused existing cellular region; use its "
"original --commit to resolve genesis"));
}
} // namespace
void run_region_store_tests() {
provisioning_is_idempotent();
primitive_provisioning_is_explicit_and_profile_safe();
provisioning_creates_the_life_allocator_once();
cellular_provisioning_survives_a_build_promotion();
full_volumes_publish_nothing();
failing_writes_leave_nothing();
check_store_qualifies_the_stores();
full_volume_server_acknowledges_nothing();
}