Files
aevolution/tests/region_server_process_tests.cpp
dingo 7da7ef2a6c
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 1m57s
Foundation / checks (postgres) (push) Canceled after 0s
Foundation / checks (tsan) (push) Canceled after 0s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (network) (push) Canceled after 6m34s
Foundation / checks (client) (push) Canceled after 8m38s
Foundation / checks (checks) (push) Canceled after 8m44s
Select the primitive region host explicitly
2026-10-10 20:37:58 -04:00

195 lines
8.7 KiB
C++

// Real process tests for evo-server (M7.6, ADR-0064): SIGTERM drain and SIGKILL
// with recovery by a successor process. Run by tools/ci/postgres-check.sh, which
// sets EVO_SERVER_BIN.
#include "evo/server/lifecycle/primitive_lives.hpp"
#include "fixtures/server_process.hpp"
#include <algorithm>
#include <source_location>
#include <thread>
void run_region_server_process_tests();
namespace {
using namespace evo::test::process;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region server process test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
// SIGTERM: stop admission, commit pending ticks, release the lease, exit 0. Every
// acknowledged tick is committed, the backlog never exceeds its bound, and the
// permanent outcome was announced only after its commit.
void sigterm_drains_and_releases() {
const auto r = provision(31, 0x42);
Child server{server_args(r)};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
CHECK(fs::exists(r.root / "ready"));
server.send(input_lines(1, 80, true)); // More than the 64-input backlog: stdin backpressures.
CHECK(server.wait_line("ack " + request_hex(80) + " 80 ", 30s));
server.signal(SIGTERM);
CHECK(server.wait_exit(20s) == 0);
const auto& lines = server.lines();
CHECK(std::ranges::find(lines, "exit 0") != lines.end() && server.printed("phase stopped"));
unsigned long long max_queued = 999;
for (const auto& l : lines)
std::sscanf(l.c_str(), "stats max_queued=%llu", &max_queued);
CHECK(max_queued <= 64);
const auto all = acks(server);
CHECK(all.size() == 80 && all.back().first == 80);
// The death at tick 20 is announced only with its commit: after the final
// acknowledgement of tick 20, which the host emits only after confirmation.
const auto outcome = std::ranges::find_if(
lines, [](const std::string& l) { return starts(l, "outcome 20 1 11 "); });
const auto ack20 = std::ranges::find_if(
lines, [](const std::string& l) { return starts(l, "ack " + request_hex(20) + " 20 "); });
CHECK(outcome != lines.end() && ack20 != lines.end() && ack20 < outcome);
const auto head = head_of(r.key);
CHECK(head.tick == 80 && head.released);
CHECK(!fs::exists(r.root / "ready"));
std::printf("server_sigterm head=%llu sequence=%llu acks=%zu max_queued=%llu\n",
static_cast<unsigned long long>(head.tick),
static_cast<unsigned long long>(head.sequence), all.size(), max_queued);
}
// SIGKILL: no drain. Nothing acknowledged is lost, nothing unacknowledged was
// claimed, and a successor becomes ready only after the database lease expires.
void sigkill_recovers_through_a_successor() {
const auto r = provision(32, 0x43);
std::uint64_t acked_max = 0;
{
Child server{server_args(r)};
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
server.send(input_lines(1, 60, false));
CHECK(server.wait_line("ack ", 15s));
std::this_thread::sleep_for(300ms); // Ticks run past the last commit.
server.signal(SIGKILL);
CHECK(server.wait_exit(10s) == 128 + SIGKILL);
acked_max = max_acked(server);
}
const auto committed = head_of(r.key);
CHECK(acked_max > 0 && committed.tick >= acked_max); // No acknowledged tick was lost.
CHECK(!committed.released); // The killed holder never released.
const auto started = Clock::now();
Child successor{server_args(r)};
CHECK(successor.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
const auto to_ready =
std::chrono::duration_cast<std::chrono::milliseconds>(Clock::now() - started).count();
const auto running = std::ranges::find_if(
successor.lines(), [](const std::string& l) { return starts(l, "phase running"); });
CHECK(running != successor.lines().end());
CHECK(running->find("committed=" + std::to_string(committed.tick) + " ") != std::string::npos);
CHECK(running->find("epoch=2 ") != std::string::npos);
successor.signal(SIGTERM);
CHECK(successor.wait_exit(20s) == 0);
const auto after = head_of(r.key);
CHECK(after.tick == committed.tick && after.released);
std::printf("server_sigkill acked_max=%llu committed=%llu successor_ready_ms=%lld\n",
static_cast<unsigned long long>(acked_max),
static_cast<unsigned long long>(committed.tick), static_cast<long long>(to_ready));
}
// Profile 3 has an explicit host selection. It becomes ready only after the
// primitive checkpoint is recovered and exits without advancing biology.
void primitive_profile_recovers_before_readiness() {
using namespace evo::persistence;
using namespace evo::server;
const RegionKey key{7, 33};
const fs::path root = fs::path(env("PGHOST")) / "server-primitive-33";
fs::remove_all(root);
fs::create_directory(root);
fs::permissions(root, fs::perms::owner_all, fs::perm_options::replace);
auto db = owner();
CHECK(db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('b1',32),'hex')) ON CONFLICT DO NOTHING",
{}, 5s));
const auto endowed = evo::biology::primitive_endowment();
CHECK(endowed);
auto funding = endowed->body;
funding.energy.value += endowed->reserve.energy.value + endowed->construction.value;
auto opening = PrimitiveLives::create({key.world, key.region, 1, 1, funding, {{0, 0}}});
CHECK(opening);
auto objects = FilesystemObjectStore::open(root, key, max_primitive_lives_bytes);
CHECK(objects);
Sha256Digest build{};
build.fill(0xB1);
Uuid commit{};
commit.fill(0x33);
CHECK(provision_primitive_region(*db, **objects, key, 1, **opening, commit, build, 5s));
CHECK(db->query("INSERT INTO evolution.life_allocators VALUES(7,1) ON CONFLICT DO NOTHING", {},
5s));
std::vector<std::string> args{
env("EVO_SERVER_BIN"),
"--profile",
"primitive",
"--world",
"7",
"--region",
"33",
"--frame",
"1",
"--pg-host",
env("PGHOST"),
"--pg-port",
env("PGPORT"),
"--pg-database",
env("PGDATABASE"),
"--pg-user",
env("EVO_PG_RUNTIME_USER"),
"--objects",
root.string(),
"--build",
"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1",
"--lease-profile",
"fast",
"--ready-file",
(root / "ready").string()};
Child host{args};
CHECK(host.wait_for([](const std::string& line) { return line == "ready 1"; }, 15s));
CHECK(fs::exists(root / "ready"));
CHECK(host.printed("phase running epoch=1 committed=0 simulated=0"));
host.signal(SIGTERM);
CHECK(host.wait_exit(20s) == 0);
CHECK(host.printed("exit 0") && host.printed("ready 0"));
CHECK(!fs::exists(root / "ready"));
const auto head = head_of(key);
CHECK(head.tick == 0 && head.sequence == 0 && head.released);
auto synthetic_args = args;
synthetic_args.emplace_back("--synthetic-inputs");
synthetic_args.emplace_back("development-v1");
Child synthetic{synthetic_args};
CHECK(synthetic.wait_exit(15s) == 2); // Profile 3 never accepts biological tick inputs.
const RegionKey missing_key{7, 34};
const fs::path missing_root = fs::path(env("PGHOST")) / "server-primitive-missing-34";
fs::remove_all(missing_root);
fs::create_directory(missing_root);
fs::permissions(missing_root, fs::perms::owner_all, fs::perm_options::replace);
auto missing_args = args;
const auto region_arg = std::find(missing_args.begin(), missing_args.end(), "--region");
*(region_arg + 1) = "34";
const auto objects_arg = std::find(missing_args.begin(), missing_args.end(), "--objects");
*(objects_arg + 1) = missing_root.string();
const auto ready_arg = std::find(missing_args.begin(), missing_args.end(), "--ready-file");
*(ready_arg + 1) = (missing_root / "ready").string();
Child absent{missing_args};
CHECK(absent.wait_for(
[](const std::string& line) {
return line == "phase quarantined epoch=0 committed=0 simulated=0";
},
15s));
CHECK(absent.wait_exit(15s) == 4 && !absent.printed("ready 1"));
const auto missing_head = load_region_head(*db, missing_key, 5s);
CHECK(!missing_head && missing_head.error().kind == LeaseStoreError::not_provisioned);
}
} // namespace
void run_region_server_process_tests() {
sigterm_drains_and_releases();
sigkill_recovers_through_a_successor();
primitive_profile_recovers_before_readiness();
}