// M8.3 store qualification (ADR-0070) against real PostgreSQL and real processes: // the object volume's free-space reserve, a genuinely failing write, the // check-store probe (retention across runs, missing-volume refusal, privilege // boundary) and a server whose volume is full. Run by tools/ci/postgres-check.sh. #include "evo/persistence/checkpoint_store.hpp" #include "evo/persistence/region_checkpoint.hpp" #include "evo/server/lifecycle/primitive_lives.hpp" #include "fixtures/server_process.hpp" #include #include #include #include void run_region_store_tests(); namespace { using namespace evo::test::process; using namespace evo::persistence; void check(bool value, std::source_location at = std::source_location::current()) { if (!value) throw std::runtime_error("region store test line " + std::to_string(at.line())); } #define CHECK(...) check(static_cast(__VA_ARGS__)) constexpr std::size_t budget = 8000000; constexpr std::uint64_t unreachable_reserve = std::uint64_t{1} << 60U; // 1 EiB. std::unique_ptr store(const fs::path& root, RegionKey key, std::uint64_t reserve = 0) { auto made = FilesystemObjectStore::open(root, key, max_lineage_payload_bytes, nullptr, reserve); CHECK(made); return std::move(*made); } fs::path fresh(const std::string& name) { const auto root = fs::path(env("PGHOST")) / name; fs::remove_all(root); fs::create_directory(root); fs::permissions(root, fs::perms::owner_all, fs::perm_options::replace); return root; } std::size_t entries(const fs::path& dir) { std::size_t n = 0; for ([[maybe_unused]] const auto& entry : fs::directory_iterator(dir)) ++n; return n; } std::vector bytes_of(std::size_t size, std::uint8_t seed) { std::vector out(size); for (std::size_t i = 0; i < size; ++i) out[i] = static_cast(seed + (i * 7U)); return out; } // A full volume refuses new objects before writing; identical objects already // stored stay usable; a batch against a full volume publishes nothing. void full_volumes_publish_nothing() { const RegionKey key{7, 81}; const auto root = fresh("store-full"); const auto stored = bytes_of(4096, 0x10); auto roomy = store(root, key); CHECK(roomy->available_bytes().value_or(0) > 0 && roomy->reserve_bytes() == 0); CHECK(roomy->put(stored)); auto full = store(root, key, unreachable_reserve); CHECK(full->reserve_bytes() == unreachable_reserve); const auto before = entries(full->directory()); auto refused = full->put(bytes_of(4096, 0x20)); CHECK(!refused && refused.error() == ObjectError::full); CHECK(entries(full->directory()) == before); // No temporary, no object. CHECK(full->put(stored)); // Reuse needs no space. // A lineage batch cannot publish through a full volume; the head stays. const auto r = provision(82, 0xDE); auto db = owner(); const auto head = load_region_checkpoint(*db, r.key, 5s); CHECK(head); auto runtime = Postgres::connect( {env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("EVO_PG_RUNTIME_USER"), {}, {}}, 5s); CHECK(runtime); Sha256Digest build{}; build.fill(0xB1); auto lease = acquire_region_lease(**runtime, r.key, build, 20000ms, 5s); CHECK(lease); auto state = evo::test::lineage_region(); RegionBatch batch{*state}; for (int i = 0; i < 10; ++i) CHECK(batch.advance(evo::test::region_input(batch.end_tick() + 1))); batch.seal(); Uuid commit{}; commit.fill(0xDF); auto blocked = store(r.root, r.key, unreachable_reserve); auto published = publish_lineage_batch(**runtime, *blocked, *lease, *head, batch, commit, build, budget, 5s); CHECK(!published && published.error().kind == PublicationError::upload && published.error().object == ObjectError::full); CHECK(load_region_checkpoint(*db, r.key, 5s)->head == head->head); CHECK(release_region_lease(**runtime, r.key, *lease, 5s)); } // A write the filesystem really refuses (RLIMIT_FSIZE stands in for a full disk: // the kernel rejects the write mid-object) is an io error that leaves no object // and no temporary behind. void failing_writes_leave_nothing() { const auto root = fresh("store-efbig"); auto objects = store(root, {7, 83}); rlimit old{}; CHECK(::getrlimit(RLIMIT_FSIZE, &old) == 0); const auto previous = std::signal(SIGXFSZ, SIG_IGN); rlimit limited = old; limited.rlim_cur = 8192; CHECK(::setrlimit(RLIMIT_FSIZE, &limited) == 0); auto refused = objects->put(bytes_of(65536, 0x30)); CHECK(::setrlimit(RLIMIT_FSIZE, &old) == 0); std::signal(SIGXFSZ, previous); CHECK(!refused && refused.error() == ObjectError::io); CHECK(entries(objects->directory()) == 0); CHECK(objects->put(bytes_of(65536, 0x30))); // The same write succeeds with room. } std::vector check_store(const fs::path& objects, const std::string& user, std::vector extra, int& code) { std::vector args{env("EVO_WORLD_CLONE_BIN"), "check-store", "--world", "7", "--region", "84", "--pg-host", env("PGHOST"), "--pg-port", env("PGPORT"), "--pg-database", env("PGDATABASE"), "--pg-user", user, "--objects", objects.string()}; args.insert(args.end(), extra.begin(), extra.end()); Child tool{args}; code = tool.wait_exit(30s); return tool.lines(); } bool has(const std::vector& lines, std::string_view prefix) { for (const auto& l : lines) if (l.starts_with(prefix)) return true; return false; } // The in-cluster qualification probe, exercised locally. void check_store_qualifies_the_stores() { const auto root = fresh("store-check"); const auto runtime = env("EVO_PG_RUNTIME_USER"); int code = -1; auto first = check_store(root, runtime, {}, code); CHECK(code == 0 && has(first, "objects available_mib=") && has(first, "probe created digest=")); CHECK(has(first, "database version=180006 user=" + runtime + " ssl=false")); CHECK(has(first, "database tables=denied functions=reachable head=none")); // A later run (a restarted or rescheduled pod) verifies the retained probe. auto second = check_store(root, runtime, {}, code); CHECK(code == 0 && has(second, "probe verified digest=")); // A missing or unmounted volume is refused, never created. auto missing = check_store(root / "not-mounted", runtime, {}, code); CHECK(code == 4 && has(missing, "objects refused missing-or-not-a-directory")); CHECK(!fs::exists(root / "not-mounted")); // A full volume refuses the probe (fresh namespace, reserve beyond any disk). const auto other = fresh("store-check-full"); auto full = check_store(other, runtime, {"--reserve-mib", "1048576"}, code); CHECK(code == 4 && has(full, "probe refused full")); // The same check through a libpq service file, as the cluster Jobs run it. const auto service = root / "pg_service.conf"; { std::FILE* f = std::fopen(service.c_str(), "wb"); CHECK(f != nullptr); const auto text = "[evolution]\nhost=" + env("PGHOST") + "\nport=" + env("PGPORT") + "\ndbname=" + env("PGDATABASE") + "\nuser=" + runtime + "\nsslmode=disable\nconnect_timeout=5\n"; CHECK(std::fwrite(text.data(), 1, text.size(), f) == text.size()); CHECK(std::fclose(f) == 0); } Child via_service{{env("EVO_WORLD_CLONE_BIN"), "check-store", "--world", "7", "--region", "84", "--pg-service", service.string(), "--objects", root.string()}}; CHECK(via_service.wait_exit(30s) == 0 && via_service.printed("probe verified digest=") && via_service.printed("database tables=denied functions=reachable")); // An owner login can read tables: not an acceptable runtime identity. auto owner_run = check_store(root, env("PGUSER"), {}, code); CHECK(code == 3 && has(owner_run, "database tables=READABLE")); } // A real server on a full volume keeps simulating but never acknowledges, and its // bounded drain gives up without committing anything. void full_volume_server_acknowledges_nothing() { const auto r = provision(85, 0xE7); const auto before = head_of(r.key); Child server{server_args(r, {"--objects-reserve-mib", "1048576", "--drain-ms", "1500", "--ops-listen", "127.0.0.1:0"})}; CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); server.send(input_lines(1, 15, false)); server.pump(3000ms); CHECK(!server.printed("ack ")); server.signal(SIGTERM); CHECK(server.wait_exit(20s) == 3); // Drain deadline passed with the batch unpublished. CHECK(!server.printed("ack ")); const auto after = head_of(r.key); CHECK(after.tick == before.tick && after.sequence == before.sequence); } // M8.4: the provisioning tool creates a region once from the development opening; // a rerun (every Argo sync) leaves it untouched, and a server then serves it. void provisioning_is_idempotent() { const ServerRegion r{{7, 86}, fresh("provision-86")}; const auto run = [&](const std::string& user, int& code) { Child tool{{env("EVO_WORLD_CLONE_BIN"), "provision", "--world", "7", "--region", "86", "--opening", "development-v1", "--pg-host", env("PGHOST"), "--pg-port", env("PGPORT"), "--pg-database", env("PGDATABASE"), "--pg-user", user, "--objects", r.root.string(), "--build", "b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1", "--retain-build", "true"}}; code = tool.wait_exit(30s); return tool.lines(); }; int code = -1; auto denied = run(env("EVO_PG_RUNTIME_USER"), code); CHECK(code != 0 && !has(denied, "provisioned ")); auto first = run(env("PGUSER"), code); CHECK(code == 0 && has(first, "provisioned world=7 region=86 sequence=0 tick=0")); auto second = run(env("PGUSER"), code); CHECK(code == 0 && has(second, "already-provisioned world=7 region=86 sequence=0 tick=0")); Child server{server_args(r)}; CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); server.send(input_lines(1, 5, false)); CHECK(server.wait_line("ack " + request_hex(5) + " 5 ", 15s)); server.signal(SIGTERM); CHECK(server.wait_exit(20s) == 0); auto after = run(env("PGUSER"), code); CHECK(code == 0 && has(after, "already-provisioned world=7 region=86 sequence=1 tick=5")); } // M11.2d-4: primitive genesis requires an explicit owner opening, and retries // cannot mistake a different profile or frame for an idempotent success. void primitive_provisioning_is_explicit_and_profile_safe() { using namespace evo::server; using evo::biology::primitive_endowment; const ServerRegion r{{7, 89}, fresh("primitive-provision-89")}; const auto endowed = primitive_endowment(); CHECK(endowed); evo::accounting::Inventory funding = endowed->body; funding.energy.value += endowed->reserve.energy.value + endowed->construction.value; const auto run = [&](const std::string& user, std::uint64_t frame, bool complete, bool retain_build, int& code, std::uint64_t energy_delta = 0) { std::vector args{ env("EVO_WORLD_CLONE_BIN"), "provision", "--world", "7", "--region", "89", "--opening", "primitive-v1", "--pg-host", env("PGHOST"), "--pg-port", env("PGPORT"), "--pg-database", env("PGDATABASE"), "--pg-user", user, "--objects", r.root.string(), "--build", "c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2c2", "--frame", std::to_string(frame), "--source-id", "1", "--funding-energy-pj", std::to_string(funding.energy.value + energy_delta), "--funding-carbon-fg", std::to_string(funding.matter[0].value), "--funding-nitrogen-fg", std::to_string(funding.matter[1].value), "--funding-phosphorus-fg", std::to_string(funding.matter[2].value), "--funding-other-fg", std::to_string(funding.matter[3].value), "--site", "0,0", "--site", "10000000,0"}; if (retain_build) { args.insert(args.end() - 6, {"--retain-build", "true"}); } if (!complete) { const auto field = std::find(args.begin(), args.end(), "--funding-other-fg"); args.erase(field, field + 2); } Child tool{args}; code = tool.wait_exit(30s); return tool.lines(); }; auto db = owner(); CHECK(db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " "(decode(repeat('c2',32),'hex')) ON CONFLICT DO NOTHING", {}, 5s)); int code = -1; const auto denied = run(env("EVO_PG_RUNTIME_USER"), 3, true, false, code); CHECK(code != 0 && !has(denied, "provisioned ")); const auto first = run(env("PGUSER"), 3, true, true, code); CHECK(code == 0 && has(first, "provisioned world=7 region=89 sequence=0 tick=0")); const auto retry = run(env("PGUSER"), 3, true, true, code); CHECK(code == 0 && has(retry, "already-provisioned world=7 region=89 sequence=0 tick=0")); const auto mismatch = run(env("PGUSER"), 4, true, true, code); CHECK(code == 4 && has(mismatch, "provision refused existing profile/frame mismatch")); const auto changed_opening = run(env("PGUSER"), 3, true, true, code, 1); CHECK(code == 4 && has(changed_opening, "provision refused existing opening mismatch")); const auto incomplete = run(env("PGUSER"), 3, false, false, code); CHECK(code == 2); const auto committed = load_region_checkpoint(*db, r.key, 5s); CHECK(committed); const auto manifest = decode_manifest(committed->manifest); CHECK(manifest && manifest->profile == CheckpointProfile::primitive_lives_v1 && manifest->frame == 3); auto objects = FilesystemObjectStore::open(r.root, r.key, evo::server::max_primitive_lives_bytes); CHECK(objects); Sha256Digest build{}; build.fill(0xC2); const std::array retained{build}; const auto recovered = recover_primitive_checkpoint(*db, **objects, r.key, retained, 5s); CHECK(recovered && recovered->state->config().source == 1 && recovered->state->config().funding == funding && recovered->state->config().sites == std::vector{{0, 0}, {10000000, 0}}); } // ADR-0123: provisioning creates the world's life allocator once at the audited // start; reruns leave it untouched, and a start above the existing counter refuses. void provisioning_creates_the_life_allocator_once() { using evo::biology::primitive_endowment; const ServerRegion r{{9760, 1}, fresh("allocator-9760")}; const auto endowed = primitive_endowment(); CHECK(endowed); evo::accounting::Inventory funding = endowed->body; funding.energy.value += endowed->reserve.energy.value + endowed->construction.value; const auto run = [&](const std::string& opening, std::uint64_t start, int& code) { std::vector args{ env("EVO_WORLD_CLONE_BIN"), "provision", "--world", "9760", "--region", "1", "--opening", opening, "--pg-host", env("PGHOST"), "--pg-port", env("PGPORT"), "--pg-database", env("PGDATABASE"), "--pg-user", env("PGUSER"), "--objects", r.root.string(), "--build", "c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3", "--retain-build", "true", "--life-allocator-start", std::to_string(start)}; if (opening == "primitive-v1") { const std::vector primitive{"--frame", "3", "--source-id", "1", "--funding-energy-pj", std::to_string(funding.energy.value), "--funding-carbon-fg", std::to_string(funding.matter[0].value), "--funding-nitrogen-fg", std::to_string(funding.matter[1].value), "--funding-phosphorus-fg", std::to_string(funding.matter[2].value), "--funding-other-fg", std::to_string(funding.matter[3].value), "--site", "0,0"}; args.insert(args.end(), primitive.begin(), primitive.end()); } Child tool{args}; code = tool.wait_exit(30s); return tool.lines(); }; const auto next_identity = [] { auto db = owner(); auto rows = db->query( "SELECT next_identity FROM evolution.life_allocators WHERE world_id = 9760", {}, 5s); if (!rows || rows->size() != 1 || (*rows)[0].size() != 1) return std::string{}; return (*rows)[0][0].value_or(std::string{}); }; int code = -1; const auto development = run("development-v1", 5000, code); CHECK(code == 2 && next_identity().empty()); // No lives, so no allocator. const auto first = run("primitive-v1", 5000, code); CHECK(code == 0 && has(first, "life-allocator world=9760 next=5000") && has(first, "provisioned world=9760 region=1 sequence=0 tick=0")); const auto retry = run("primitive-v1", 5000, code); CHECK(code == 0 && has(retry, "life-allocator world=9760 next=5000") && has(retry, "already-provisioned world=9760 region=1 sequence=0 tick=0")); const auto lower = run("primitive-v1", 10, code); CHECK(code == 0 && has(lower, "life-allocator world=9760 next=5000")); const auto higher = run("primitive-v1", 6000, code); CHECK(code == 4 && has(higher, "life-allocator refused existing below start")); CHECK(next_identity() == "5000"); } // ADR-0123: an image promotion changes --build, but a cellular genesis records the // build that wrote it. A rerun under a new build still resolves the same genesis, // while a different commit or opening is refused. // One cellular provisioning run for region 1 of `world` with a one-founder opening. std::vector provision_cellular(const ServerRegion& r, const std::string& world, const std::string& opening, const std::string& build, const std::string& commit, const std::string& patch, int& code) { const auto endowed = evo::biology::primitive_endowment(); CHECK(endowed); evo::accounting::Inventory funding = endowed->body; funding.energy.value += endowed->reserve.energy.value + endowed->construction.value; Child tool{{env("EVO_WORLD_CLONE_BIN"), "provision", "--world", world, "--region", "1", "--opening", opening, "--pg-host", env("PGHOST"), "--pg-port", env("PGPORT"), "--pg-database", env("PGDATABASE"), "--pg-user", env("PGUSER"), "--objects", r.root.string(), "--build", build, "--retain-build", "true", "--commit", commit, "--frame", "9", "--source-id", "1", "--funding-energy-pj", std::to_string(funding.energy.value), "--funding-carbon-fg", std::to_string(funding.matter[0].value), "--funding-nitrogen-fg", std::to_string(funding.matter[1].value), "--funding-phosphorus-fg", std::to_string(funding.matter[2].value), "--funding-other-fg", std::to_string(funding.matter[3].value), "--site", "2500,2500", "--width-um", "10000", "--height-um", "10000", "--ambient-temperature-mk", "293150", "--patch", patch}}; code = tool.wait_exit(30s); return tool.lines(); } // ADR-0124: cellular-v2 provisions a rules-3 genesis under the v4 bundle (schema v34), // resolves it again on rerun, and refuses the same region asked for as cellular-v1. void cellular_v2_provisioning_uses_the_engulf_bundle() { const ServerRegion r{{9762, 1}, fresh("cellular-v2-provision-9762")}; const std::string build(64, 'e'); const std::string commit(32, 'f'); const std::string patch = "3000,3000,2,1000,1000,1000,1000,1000"; int code = -1; const auto first = provision_cellular(r, "9762", "cellular-v2", build, commit, patch, code); CHECK(code == 0 && has(first, "provisioned world=9762 region=1 sequence=0 tick=0")); const auto again = provision_cellular(r, "9762", "cellular-v2", build, commit, patch, code); CHECK(code == 0 && has(again, "already-provisioned world=9762 region=1 sequence=0 tick=0")); const auto older = provision_cellular(r, "9762", "cellular-v1", build, commit, patch, code); CHECK(code == 4); auto db = owner(); const auto head = load_region_checkpoint(*db, r.key, 5s); CHECK(head); const auto manifest = decode_manifest(head->manifest); CHECK(manifest && manifest->cellular_versions == cellular_v4_versions && manifest->history_profile == cellular_revision_history_profile); } // ADR-0125: cellular-v3 is the rules-4 budding opening on the v5 bundle. void cellular_v3_provisioning_uses_the_budding_bundle() { const ServerRegion r{{9766, 1}, fresh("cellular-v3-provision-9766")}; const std::string build(64, 'e'); const std::string commit(32, 'b'); const std::string patch = "3000,3000,2,1000,1000,1000,1000,1000"; int code = -1; const auto first = provision_cellular(r, "9766", "cellular-v3", build, commit, patch, code); CHECK(code == 0 && has(first, "provisioned world=9766 region=1 sequence=0 tick=0")); const auto again = provision_cellular(r, "9766", "cellular-v3", build, commit, patch, code); CHECK(code == 0 && has(again, "already-provisioned world=9766 region=1 sequence=0 tick=0")); const auto older = provision_cellular(r, "9766", "cellular-v2", build, commit, patch, code); CHECK(code == 4); auto db = owner(); const auto head = load_region_checkpoint(*db, r.key, 5s); CHECK(head); const auto manifest = decode_manifest(head->manifest); CHECK(manifest && manifest->cellular_versions == cellular_v5_versions && manifest->history_profile == cellular_revision_history_profile); } // ADR-0128: cellular-v4 is the rules-5 presence opening on the v6 bundle. void cellular_v4_provisioning_uses_the_presence_bundle() { const ServerRegion r{{9769, 1}, fresh("cellular-v4-provision-9769")}; const std::string build(64, 'e'); const std::string commit(32, 'd'); const std::string patch = "3000,3000,2,1000,1000,1000,1000,1000"; int code = -1; const auto first = provision_cellular(r, "9769", "cellular-v4", build, commit, patch, code); CHECK(code == 0 && has(first, "provisioned world=9769 region=1 sequence=0 tick=0")); const auto older = provision_cellular(r, "9769", "cellular-v3", build, commit, patch, code); CHECK(code == 4); auto db = owner(); const auto head = load_region_checkpoint(*db, r.key, 5s); CHECK(head); const auto manifest = decode_manifest(head->manifest); CHECK(manifest && manifest->cellular_versions == cellular_v6_versions && manifest->history_profile == cellular_revision_history_profile); // 11.9 (ADR-0130): the read-only summary reports account-free aggregates of the head. Child summary{{env("EVO_WORLD_CLONE_BIN"), "summary", "--world", "9769", "--region", "1", "--pg-host", env("PGHOST"), "--pg-port", env("PGPORT"), "--pg-database", env("PGDATABASE"), "--pg-user", env("PGUSER"), "--objects", r.root.string()}}; CHECK(summary.wait_exit(30s) == 0); const auto lines = summary.lines(); CHECK(has(lines, "cellular summary world=9769 region=1 sequence=0 tick=0") && has(lines, "living owned=0 wildlife=0 lingering=0 suspended=0") && has(lines, "deaths total=0 starved=0 engulfed=0 unrecorded=0 owned=0 wildlife=0") && has(lines, "evolution buds=0 mutations=0 living_genomes=0 deepest_lineage=0")); } void cellular_provisioning_survives_a_build_promotion() { const ServerRegion r{{9761, 1}, fresh("cellular-provision-9761")}; const auto run = [&](const std::string& build, const std::string& commit, const std::string& patch, int& code) { return provision_cellular(r, "9761", "cellular-v1", build, commit, patch, code); }; const std::string first_build(64, 'c'); const std::string second_build = std::string(63, 'c') + "d"; const std::string commit(32, 'a'); const std::string patch = "3000,3000,2,1000,1000,1000,1000,1000"; int code = -1; const auto first = run(first_build, commit, patch, code); CHECK(code == 0 && has(first, "provisioned world=9761 region=1 sequence=0 tick=0")); const auto promoted = run(second_build, commit, patch, code); CHECK(code == 0 && has(promoted, "already-provisioned world=9761 region=1 sequence=0 tick=0")); const auto other_commit = run(second_build, std::string(32, 'b'), patch, code); CHECK(code == 4 && has(other_commit, "provision refused existing cellular region; use its " "original --commit to resolve genesis")); const auto other_opening = run(second_build, commit, "4000,4000,2,1000,1000,1000,1000,1000", code); CHECK(code == 4 && has(other_opening, "provision refused existing cellular region; use its " "original --commit to resolve genesis")); } } // namespace void run_region_store_tests() { provisioning_is_idempotent(); primitive_provisioning_is_explicit_and_profile_safe(); provisioning_creates_the_life_allocator_once(); cellular_provisioning_survives_a_build_promotion(); cellular_v2_provisioning_uses_the_engulf_bundle(); cellular_v3_provisioning_uses_the_budding_bundle(); cellular_v4_provisioning_uses_the_presence_bundle(); full_volumes_publish_nothing(); failing_writes_leave_nothing(); check_store_qualifies_the_stores(); full_volume_server_acknowledges_nothing(); }