// Real process tests for evo-server (M7.6, ADR-0064): SIGTERM drain and SIGKILL // with recovery by a successor process. Run by tools/ci/postgres-check.sh, which // sets EVO_SERVER_BIN. #include "evo/server/lifecycle/primitive_lives.hpp" #include "fixtures/server_process.hpp" #include #include #include void run_region_server_process_tests(); namespace { using namespace evo::test::process; void check(bool value, std::source_location at = std::source_location::current()) { if (!value) throw std::runtime_error("region server process test line " + std::to_string(at.line())); } #define CHECK(...) check(static_cast(__VA_ARGS__)) // SIGTERM: stop admission, commit pending ticks, release the lease, exit 0. Every // acknowledged tick is committed, the backlog never exceeds its bound, and the // permanent outcome was announced only after its commit. void sigterm_drains_and_releases() { const auto r = provision(31, 0x42); Child server{server_args(r)}; CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); CHECK(fs::exists(r.root / "ready")); server.send(input_lines(1, 80, true)); // More than the 64-input backlog: stdin backpressures. CHECK(server.wait_line("ack " + request_hex(80) + " 80 ", 30s)); server.signal(SIGTERM); CHECK(server.wait_exit(20s) == 0); const auto& lines = server.lines(); CHECK(std::ranges::find(lines, "exit 0") != lines.end() && server.printed("phase stopped")); unsigned long long max_queued = 999; for (const auto& l : lines) std::sscanf(l.c_str(), "stats max_queued=%llu", &max_queued); CHECK(max_queued <= 64); const auto all = acks(server); CHECK(all.size() == 80 && all.back().first == 80); // The death at tick 20 is announced only with its commit: after the final // acknowledgement of tick 20, which the host emits only after confirmation. const auto outcome = std::ranges::find_if( lines, [](const std::string& l) { return starts(l, "outcome 20 1 11 "); }); const auto ack20 = std::ranges::find_if( lines, [](const std::string& l) { return starts(l, "ack " + request_hex(20) + " 20 "); }); CHECK(outcome != lines.end() && ack20 != lines.end() && ack20 < outcome); const auto head = head_of(r.key); CHECK(head.tick == 80 && head.released); CHECK(!fs::exists(r.root / "ready")); std::printf("server_sigterm head=%llu sequence=%llu acks=%zu max_queued=%llu\n", static_cast(head.tick), static_cast(head.sequence), all.size(), max_queued); } // SIGKILL: no drain. Nothing acknowledged is lost, nothing unacknowledged was // claimed, and a successor becomes ready only after the database lease expires. void sigkill_recovers_through_a_successor() { const auto r = provision(32, 0x43); std::uint64_t acked_max = 0; { Child server{server_args(r)}; CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); server.send(input_lines(1, 60, false)); CHECK(server.wait_line("ack ", 15s)); std::this_thread::sleep_for(300ms); // Ticks run past the last commit. server.signal(SIGKILL); CHECK(server.wait_exit(10s) == 128 + SIGKILL); acked_max = max_acked(server); } const auto committed = head_of(r.key); CHECK(acked_max > 0 && committed.tick >= acked_max); // No acknowledged tick was lost. CHECK(!committed.released); // The killed holder never released. const auto started = Clock::now(); Child successor{server_args(r)}; CHECK(successor.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); const auto to_ready = std::chrono::duration_cast(Clock::now() - started).count(); const auto running = std::ranges::find_if( successor.lines(), [](const std::string& l) { return starts(l, "phase running"); }); CHECK(running != successor.lines().end()); CHECK(running->find("committed=" + std::to_string(committed.tick) + " ") != std::string::npos); CHECK(running->find("epoch=2 ") != std::string::npos); successor.signal(SIGTERM); CHECK(successor.wait_exit(20s) == 0); const auto after = head_of(r.key); CHECK(after.tick == committed.tick && after.released); std::printf("server_sigkill acked_max=%llu committed=%llu successor_ready_ms=%lld\n", static_cast(acked_max), static_cast(committed.tick), static_cast(to_ready)); } // Profile 3 has an explicit host selection. It becomes ready only after the // primitive checkpoint is recovered and exits without advancing biology. void primitive_profile_recovers_before_readiness() { using namespace evo::persistence; using namespace evo::server; const RegionKey key{7, 33}; const fs::path root = fs::path(env("PGHOST")) / "server-primitive-33"; fs::remove_all(root); fs::create_directory(root); fs::permissions(root, fs::perms::owner_all, fs::perm_options::replace); auto db = owner(); CHECK(db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " "(decode(repeat('b1',32),'hex')) ON CONFLICT DO NOTHING", {}, 5s)); const auto endowed = evo::biology::primitive_endowment(); CHECK(endowed); auto funding = endowed->body; funding.energy.value += endowed->reserve.energy.value + endowed->construction.value; auto opening = PrimitiveLives::create({key.world, key.region, 1, 1, funding, {{0, 0}}}); CHECK(opening); auto objects = FilesystemObjectStore::open(root, key, max_primitive_lives_bytes); CHECK(objects); Sha256Digest build{}; build.fill(0xB1); Uuid commit{}; commit.fill(0x33); CHECK(provision_primitive_region(*db, **objects, key, 1, **opening, commit, build, 5s)); CHECK(db->query("INSERT INTO evolution.life_allocators VALUES(7,1) ON CONFLICT DO NOTHING", {}, 5s)); std::vector args{ env("EVO_SERVER_BIN"), "--profile", "primitive", "--world", "7", "--region", "33", "--frame", "1", "--pg-host", env("PGHOST"), "--pg-port", env("PGPORT"), "--pg-database", env("PGDATABASE"), "--pg-user", env("EVO_PG_RUNTIME_USER"), "--objects", root.string(), "--build", "b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1", "--lease-profile", "fast", "--ready-file", (root / "ready").string()}; Child host{args}; CHECK(host.wait_for([](const std::string& line) { return line == "ready 1"; }, 15s)); CHECK(fs::exists(root / "ready")); CHECK(host.printed("phase running epoch=1 committed=0 simulated=0")); host.signal(SIGTERM); CHECK(host.wait_exit(20s) == 0); CHECK(host.printed("exit 0") && host.printed("ready 0")); CHECK(!fs::exists(root / "ready")); const auto head = head_of(key); CHECK(head.tick == 0 && head.sequence == 0 && head.released); auto synthetic_args = args; synthetic_args.emplace_back("--synthetic-inputs"); synthetic_args.emplace_back("development-v1"); Child synthetic{synthetic_args}; CHECK(synthetic.wait_exit(15s) == 2); // Profile 3 never accepts biological tick inputs. const RegionKey missing_key{7, 34}; const fs::path missing_root = fs::path(env("PGHOST")) / "server-primitive-missing-34"; fs::remove_all(missing_root); fs::create_directory(missing_root); fs::permissions(missing_root, fs::perms::owner_all, fs::perm_options::replace); auto missing_args = args; const auto region_arg = std::find(missing_args.begin(), missing_args.end(), "--region"); *(region_arg + 1) = "34"; const auto objects_arg = std::find(missing_args.begin(), missing_args.end(), "--objects"); *(objects_arg + 1) = missing_root.string(); const auto ready_arg = std::find(missing_args.begin(), missing_args.end(), "--ready-file"); *(ready_arg + 1) = (missing_root / "ready").string(); Child absent{missing_args}; CHECK(absent.wait_for( [](const std::string& line) { return line == "phase quarantined epoch=0 committed=0 simulated=0"; }, 15s)); CHECK(absent.wait_exit(15s) == 4 && !absent.printed("ready 1")); const auto missing_head = load_region_head(*db, missing_key, 5s); CHECK(!missing_head && missing_head.error().kind == LeaseStoreError::not_provisioned); } } // namespace void run_region_server_process_tests() { sigterm_drains_and_releases(); sigkill_recovers_through_a_successor(); primitive_profile_recovers_before_readiness(); }