// Real process tests for evo-server (M7.6, ADR-0064): SIGTERM drain and SIGKILL // with recovery by a successor process. Run by tools/ci/postgres-check.sh, which // sets EVO_SERVER_BIN. #include "evo/persistence/checkpoint_store.hpp" #include "fixtures/environmental_region.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include extern char** environ; void run_region_server_process_tests(); namespace { using namespace evo::persistence; using namespace std::chrono_literals; namespace fs = std::filesystem; using Clock = std::chrono::steady_clock; void check(bool value, std::source_location at = std::source_location::current()) { if (!value) throw std::runtime_error("region server process test line " + std::to_string(at.line())); } #define CHECK(...) check(static_cast(__VA_ARGS__)) std::string env(const char* key) { const auto* value = std::getenv(key); if (!value) throw std::runtime_error("missing test environment"); return value; } std::unique_ptr owner() { auto made = Postgres::connect( {env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("PGUSER"), {}, {}}, 5s); CHECK(made); return std::move(*made); } std::string hex32(std::uint64_t tick) { char out[33]; std::snprintf(out, sizeof out, "9b000000000000000000000000%06llx", static_cast(tick)); return out; } std::string input_line(std::uint64_t tick, bool consumer_dies) { const auto in = evo::test::region_input(tick, consumer_dies); std::string line = "input " + hex32(tick) + " " + std::to_string(in.temperature_mK) + " " + std::to_string(in.source_enabled); for (const auto b : in.bite) line += " " + std::to_string(b); for (const auto t : in.terminal) line += " " + std::to_string(t); return line + "\n"; } class Child { public: explicit Child(const std::vector& args) { int to_child[2], from_child[2]; CHECK(::pipe(to_child) == 0 && ::pipe(from_child) == 0); posix_spawn_file_actions_t actions; posix_spawn_file_actions_init(&actions); posix_spawn_file_actions_adddup2(&actions, to_child[0], STDIN_FILENO); posix_spawn_file_actions_adddup2(&actions, from_child[1], STDOUT_FILENO); posix_spawn_file_actions_addclose(&actions, to_child[1]); posix_spawn_file_actions_addclose(&actions, from_child[0]); std::vector argv; argv.reserve(args.size() + 1); for (const auto& a : args) argv.push_back(const_cast(a.c_str())); argv.push_back(nullptr); CHECK(posix_spawn(&pid_, argv[0], &actions, nullptr, argv.data(), environ) == 0); posix_spawn_file_actions_destroy(&actions); ::close(to_child[0]); ::close(from_child[1]); in_ = to_child[1]; out_ = from_child[0]; } Child(const Child&) = delete; Child& operator=(const Child&) = delete; ~Child() { if (pid_ > 0) { ::kill(pid_, SIGKILL); ::waitpid(pid_, nullptr, 0); } ::close(in_); ::close(out_); } void send(const std::string& text) { std::size_t done = 0; while (done < text.size()) { const auto n = ::write(in_, text.data() + done, text.size() - done); CHECK(n > 0 || errno == EINTR); if (n > 0) done += static_cast(n); } } // Reads lines until one satisfies `want`; every line is kept in lines(). template bool wait_for(Want want, std::chrono::milliseconds timeout) { const auto deadline = Clock::now() + timeout; for (std::size_t seen = 0;;) { for (; seen < lines_.size(); ++seen) if (want(lines_[seen])) return true; const auto left = std::chrono::duration_cast(deadline - Clock::now()); if (left.count() <= 0 || !read_some(static_cast(left.count()))) return false; } } void signal(int sig) { CHECK(::kill(pid_, sig) == 0); } // Exit code, or 128+signal; drains remaining output. int wait_exit(std::chrono::milliseconds timeout) { const auto deadline = Clock::now() + timeout; int status = 0; for (;;) { read_some(20); const auto r = ::waitpid(pid_, &status, WNOHANG); if (r == pid_) break; CHECK(Clock::now() < deadline); } while (read_some(20)) { } pid_ = -1; return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status); } [[nodiscard]] const std::vector& lines() const noexcept { return lines_; } private: bool read_some(int timeout_ms) { pollfd fd{out_, POLLIN, 0}; if (::poll(&fd, 1, timeout_ms) <= 0) return false; char chunk[4096]; const auto n = ::read(out_, chunk, sizeof chunk); if (n <= 0) return false; buffer_.append(chunk, static_cast(n)); for (auto end = buffer_.find('\n'); end != std::string::npos; end = buffer_.find('\n')) { lines_.push_back(buffer_.substr(0, end)); buffer_.erase(0, end + 1); } return true; } pid_t pid_{-1}; int in_{-1}, out_{-1}; std::string buffer_; std::vector lines_; }; struct Region { RegionKey key; fs::path root; }; Region provision(std::uint64_t region, std::uint8_t genesis) { Region r{{7, region}, fs::path(env("PGHOST")) / ("server-" + std::to_string(region))}; fs::remove_all(r.root); fs::create_directory(r.root); fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace); auto db = owner(); CHECK(db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " "(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING", {}, 5s)); auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes); CHECK(objects); Sha256Digest build{}; build.fill(0xB1); Uuid commit{}; commit.fill(genesis); CHECK(provision_lineage_region(*db, **objects, r.key, evo::test::region_identity.frame, *evo::test::lineage_region(), commit, build, 8000000, 5s)); return r; } std::vector server_args(const Region& r) { return {env("EVO_SERVER_BIN"), "--profile", "lineage", "--world", "7", "--region", std::to_string(r.key.region), "--frame", std::to_string(evo::test::region_identity.frame), "--pg-host", env("PGHOST"), "--pg-port", env("PGPORT"), "--pg-database", env("PGDATABASE"), "--pg-user", env("EVO_PG_RUNTIME_USER"), "--objects", r.root.string(), "--build", "b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1", "--flush-ms", "500", "--lease-profile", "fast", "--ready-file", (r.root / "ready").string()}; } // (tick, commit_sequence) of every ack line. std::vector> acks(const Child& c) { std::vector> out; for (const auto& line : c.lines()) { unsigned long long tick = 0, sequence = 0; char request[40]; if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &tick, &sequence) == 3) out.emplace_back(tick, sequence); } return out; } std::string head_of(RegionKey key) { const std::array p{std::to_string(key.region)}; auto rows = owner()->query("SELECT h.end_tick::text || ':' || h.commit_sequence::text || ':' " "|| coalesce(l.holder::text, 'released') FROM " "evolution.region_heads h JOIN evolution.region_leases l USING " "(world_id, region_id) WHERE world_id = 7 AND region_id = " "$1::evolution.u64", p, 5s); CHECK(rows && rows->size() == 1); const auto& value = (*rows)[0][0]; CHECK(value.has_value()); return value.value_or(""); } std::uint64_t head_tick(RegionKey key) { return std::stoull(head_of(key).substr(0, head_of(key).find(':'))); } bool starts(const std::string& line, std::string_view prefix) { return line.starts_with(prefix); } // SIGTERM: stop admission, commit pending ticks, release the lease, exit 0. Every // acknowledged tick is committed, the backlog never exceeds its bound, and the // permanent outcome was announced only after its commit. void sigterm_drains_and_releases() { const auto r = provision(31, 0x42); Child server{server_args(r)}; CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); CHECK(fs::exists(r.root / "ready")); std::string burst; for (std::uint64_t tick = 1; tick <= 80; ++tick) burst += input_line(tick, true); server.send(burst); // More than the 64-input backlog: stdin backpressures. CHECK(server.wait_for( [](const std::string& l) { return starts(l, "ack " + hex32(80) + " 80 "); }, 30s)); server.signal(SIGTERM); CHECK(server.wait_exit(20s) == 0); const auto& lines = server.lines(); CHECK(std::ranges::find(lines, "exit 0") != lines.end()); CHECK(std::ranges::find_if(lines, [](const std::string& l) { return starts(l, "phase stopped"); }) != lines.end()); unsigned long long max_queued = 999; for (const auto& l : lines) std::sscanf(l.c_str(), "stats max_queued=%llu", &max_queued); CHECK(max_queued <= 64); const auto all = acks(server); CHECK(all.size() == 80 && all.back().first == 80); // The death at tick 20 is announced only with its commit: after the final // acknowledgement of tick 20, which the host emits only after confirmation. const auto outcome = std::ranges::find_if( lines, [](const std::string& l) { return starts(l, "outcome 20 1 11 "); }); const auto ack20 = std::ranges::find_if( lines, [](const std::string& l) { return starts(l, "ack " + hex32(20) + " 20 "); }); CHECK(outcome != lines.end() && ack20 != lines.end() && ack20 < outcome); CHECK(head_of(r.key).starts_with("80:") && head_of(r.key).ends_with(":released")); CHECK(!fs::exists(r.root / "ready")); std::printf("server_sigterm commits=%s acks=%zu max_queued=%llu\n", head_of(r.key).c_str(), all.size(), max_queued); } // SIGKILL: no drain. Nothing acknowledged is lost, nothing unacknowledged was // claimed, and a successor becomes ready only after the database lease expires. void sigkill_recovers_through_a_successor() { const auto r = provision(32, 0x43); std::uint64_t acked_max = 0; { Child server{server_args(r)}; CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s)); std::string burst; for (std::uint64_t tick = 1; tick <= 60; ++tick) burst += input_line(tick, false); server.send(burst); CHECK(server.wait_for([](const std::string& l) { return starts(l, "ack "); }, 15s)); std::this_thread::sleep_for(300ms); // Ticks run past the last commit. server.signal(SIGKILL); CHECK(server.wait_exit(10s) == 128 + SIGKILL); for (const auto& [tick, sequence] : acks(server)) { static_cast(sequence); acked_max = std::max(acked_max, tick); } } const auto committed = head_tick(r.key); CHECK(acked_max > 0 && committed >= acked_max); // No acknowledged tick was lost. CHECK(!head_of(r.key).ends_with(":released")); // The killed holder never released. const auto started = Clock::now(); Child successor{server_args(r)}; CHECK(successor.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s)); const auto to_ready = std::chrono::duration_cast(Clock::now() - started).count(); const auto running = std::ranges::find_if( successor.lines(), [](const std::string& l) { return starts(l, "phase running"); }); CHECK(running != successor.lines().end()); CHECK(running->find("committed=" + std::to_string(committed) + " ") != std::string::npos); CHECK(running->find("epoch=2 ") != std::string::npos); successor.signal(SIGTERM); CHECK(successor.wait_exit(20s) == 0); CHECK(head_tick(r.key) == committed && head_of(r.key).ends_with(":released")); std::printf("server_sigkill acked_max=%llu committed=%llu successor_ready_ms=%lld\n", static_cast(acked_max), static_cast(committed), static_cast(to_ready)); } } // namespace void run_region_server_process_tests() { sigterm_drains_and_releases(); sigkill_recovers_through_a_successor(); }