Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 1m57s
Foundation / checks (postgres) (push) Canceled after 0s
Foundation / checks (tsan) (push) Canceled after 0s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (network) (push) Canceled after 6m34s
Foundation / checks (client) (push) Canceled after 8m38s
Foundation / checks (checks) (push) Canceled after 8m44s
195 lines
8.7 KiB
C++
195 lines
8.7 KiB
C++
// Real process tests for evo-server (M7.6, ADR-0064): SIGTERM drain and SIGKILL
|
|
// with recovery by a successor process. Run by tools/ci/postgres-check.sh, which
|
|
// sets EVO_SERVER_BIN.
|
|
#include "evo/server/lifecycle/primitive_lives.hpp"
|
|
#include "fixtures/server_process.hpp"
|
|
#include <algorithm>
|
|
#include <source_location>
|
|
#include <thread>
|
|
|
|
void run_region_server_process_tests();
|
|
|
|
namespace {
|
|
using namespace evo::test::process;
|
|
void check(bool value, std::source_location at = std::source_location::current()) {
|
|
if (!value)
|
|
throw std::runtime_error("region server process test line " + std::to_string(at.line()));
|
|
}
|
|
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
|
|
|
// SIGTERM: stop admission, commit pending ticks, release the lease, exit 0. Every
|
|
// acknowledged tick is committed, the backlog never exceeds its bound, and the
|
|
// permanent outcome was announced only after its commit.
|
|
void sigterm_drains_and_releases() {
|
|
const auto r = provision(31, 0x42);
|
|
Child server{server_args(r)};
|
|
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
|
CHECK(fs::exists(r.root / "ready"));
|
|
server.send(input_lines(1, 80, true)); // More than the 64-input backlog: stdin backpressures.
|
|
CHECK(server.wait_line("ack " + request_hex(80) + " 80 ", 30s));
|
|
server.signal(SIGTERM);
|
|
CHECK(server.wait_exit(20s) == 0);
|
|
const auto& lines = server.lines();
|
|
CHECK(std::ranges::find(lines, "exit 0") != lines.end() && server.printed("phase stopped"));
|
|
unsigned long long max_queued = 999;
|
|
for (const auto& l : lines)
|
|
std::sscanf(l.c_str(), "stats max_queued=%llu", &max_queued);
|
|
CHECK(max_queued <= 64);
|
|
const auto all = acks(server);
|
|
CHECK(all.size() == 80 && all.back().first == 80);
|
|
// The death at tick 20 is announced only with its commit: after the final
|
|
// acknowledgement of tick 20, which the host emits only after confirmation.
|
|
const auto outcome = std::ranges::find_if(
|
|
lines, [](const std::string& l) { return starts(l, "outcome 20 1 11 "); });
|
|
const auto ack20 = std::ranges::find_if(
|
|
lines, [](const std::string& l) { return starts(l, "ack " + request_hex(20) + " 20 "); });
|
|
CHECK(outcome != lines.end() && ack20 != lines.end() && ack20 < outcome);
|
|
const auto head = head_of(r.key);
|
|
CHECK(head.tick == 80 && head.released);
|
|
CHECK(!fs::exists(r.root / "ready"));
|
|
std::printf("server_sigterm head=%llu sequence=%llu acks=%zu max_queued=%llu\n",
|
|
static_cast<unsigned long long>(head.tick),
|
|
static_cast<unsigned long long>(head.sequence), all.size(), max_queued);
|
|
}
|
|
|
|
// SIGKILL: no drain. Nothing acknowledged is lost, nothing unacknowledged was
|
|
// claimed, and a successor becomes ready only after the database lease expires.
|
|
void sigkill_recovers_through_a_successor() {
|
|
const auto r = provision(32, 0x43);
|
|
std::uint64_t acked_max = 0;
|
|
{
|
|
Child server{server_args(r)};
|
|
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
|
server.send(input_lines(1, 60, false));
|
|
CHECK(server.wait_line("ack ", 15s));
|
|
std::this_thread::sleep_for(300ms); // Ticks run past the last commit.
|
|
server.signal(SIGKILL);
|
|
CHECK(server.wait_exit(10s) == 128 + SIGKILL);
|
|
acked_max = max_acked(server);
|
|
}
|
|
const auto committed = head_of(r.key);
|
|
CHECK(acked_max > 0 && committed.tick >= acked_max); // No acknowledged tick was lost.
|
|
CHECK(!committed.released); // The killed holder never released.
|
|
const auto started = Clock::now();
|
|
Child successor{server_args(r)};
|
|
CHECK(successor.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
|
const auto to_ready =
|
|
std::chrono::duration_cast<std::chrono::milliseconds>(Clock::now() - started).count();
|
|
const auto running = std::ranges::find_if(
|
|
successor.lines(), [](const std::string& l) { return starts(l, "phase running"); });
|
|
CHECK(running != successor.lines().end());
|
|
CHECK(running->find("committed=" + std::to_string(committed.tick) + " ") != std::string::npos);
|
|
CHECK(running->find("epoch=2 ") != std::string::npos);
|
|
successor.signal(SIGTERM);
|
|
CHECK(successor.wait_exit(20s) == 0);
|
|
const auto after = head_of(r.key);
|
|
CHECK(after.tick == committed.tick && after.released);
|
|
std::printf("server_sigkill acked_max=%llu committed=%llu successor_ready_ms=%lld\n",
|
|
static_cast<unsigned long long>(acked_max),
|
|
static_cast<unsigned long long>(committed.tick), static_cast<long long>(to_ready));
|
|
}
|
|
|
|
// Profile 3 has an explicit host selection. It becomes ready only after the
|
|
// primitive checkpoint is recovered and exits without advancing biology.
|
|
void primitive_profile_recovers_before_readiness() {
|
|
using namespace evo::persistence;
|
|
using namespace evo::server;
|
|
const RegionKey key{7, 33};
|
|
const fs::path root = fs::path(env("PGHOST")) / "server-primitive-33";
|
|
fs::remove_all(root);
|
|
fs::create_directory(root);
|
|
fs::permissions(root, fs::perms::owner_all, fs::perm_options::replace);
|
|
|
|
auto db = owner();
|
|
CHECK(db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
|
"(decode(repeat('b1',32),'hex')) ON CONFLICT DO NOTHING",
|
|
{}, 5s));
|
|
const auto endowed = evo::biology::primitive_endowment();
|
|
CHECK(endowed);
|
|
auto funding = endowed->body;
|
|
funding.energy.value += endowed->reserve.energy.value + endowed->construction.value;
|
|
auto opening = PrimitiveLives::create({key.world, key.region, 1, 1, funding, {{0, 0}}});
|
|
CHECK(opening);
|
|
auto objects = FilesystemObjectStore::open(root, key, max_primitive_lives_bytes);
|
|
CHECK(objects);
|
|
Sha256Digest build{};
|
|
build.fill(0xB1);
|
|
Uuid commit{};
|
|
commit.fill(0x33);
|
|
CHECK(provision_primitive_region(*db, **objects, key, 1, **opening, commit, build, 5s));
|
|
CHECK(db->query("INSERT INTO evolution.life_allocators VALUES(7,1) ON CONFLICT DO NOTHING", {},
|
|
5s));
|
|
|
|
std::vector<std::string> args{
|
|
env("EVO_SERVER_BIN"),
|
|
"--profile",
|
|
"primitive",
|
|
"--world",
|
|
"7",
|
|
"--region",
|
|
"33",
|
|
"--frame",
|
|
"1",
|
|
"--pg-host",
|
|
env("PGHOST"),
|
|
"--pg-port",
|
|
env("PGPORT"),
|
|
"--pg-database",
|
|
env("PGDATABASE"),
|
|
"--pg-user",
|
|
env("EVO_PG_RUNTIME_USER"),
|
|
"--objects",
|
|
root.string(),
|
|
"--build",
|
|
"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1",
|
|
"--lease-profile",
|
|
"fast",
|
|
"--ready-file",
|
|
(root / "ready").string()};
|
|
Child host{args};
|
|
CHECK(host.wait_for([](const std::string& line) { return line == "ready 1"; }, 15s));
|
|
CHECK(fs::exists(root / "ready"));
|
|
CHECK(host.printed("phase running epoch=1 committed=0 simulated=0"));
|
|
host.signal(SIGTERM);
|
|
CHECK(host.wait_exit(20s) == 0);
|
|
CHECK(host.printed("exit 0") && host.printed("ready 0"));
|
|
CHECK(!fs::exists(root / "ready"));
|
|
const auto head = head_of(key);
|
|
CHECK(head.tick == 0 && head.sequence == 0 && head.released);
|
|
|
|
auto synthetic_args = args;
|
|
synthetic_args.emplace_back("--synthetic-inputs");
|
|
synthetic_args.emplace_back("development-v1");
|
|
Child synthetic{synthetic_args};
|
|
CHECK(synthetic.wait_exit(15s) == 2); // Profile 3 never accepts biological tick inputs.
|
|
|
|
const RegionKey missing_key{7, 34};
|
|
const fs::path missing_root = fs::path(env("PGHOST")) / "server-primitive-missing-34";
|
|
fs::remove_all(missing_root);
|
|
fs::create_directory(missing_root);
|
|
fs::permissions(missing_root, fs::perms::owner_all, fs::perm_options::replace);
|
|
auto missing_args = args;
|
|
const auto region_arg = std::find(missing_args.begin(), missing_args.end(), "--region");
|
|
*(region_arg + 1) = "34";
|
|
const auto objects_arg = std::find(missing_args.begin(), missing_args.end(), "--objects");
|
|
*(objects_arg + 1) = missing_root.string();
|
|
const auto ready_arg = std::find(missing_args.begin(), missing_args.end(), "--ready-file");
|
|
*(ready_arg + 1) = (missing_root / "ready").string();
|
|
Child absent{missing_args};
|
|
CHECK(absent.wait_for(
|
|
[](const std::string& line) {
|
|
return line == "phase quarantined epoch=0 committed=0 simulated=0";
|
|
},
|
|
15s));
|
|
CHECK(absent.wait_exit(15s) == 4 && !absent.printed("ready 1"));
|
|
const auto missing_head = load_region_head(*db, missing_key, 5s);
|
|
CHECK(!missing_head && missing_head.error().kind == LeaseStoreError::not_provisioned);
|
|
}
|
|
} // namespace
|
|
|
|
void run_region_server_process_tests() {
|
|
sigterm_drains_and_releases();
|
|
sigkill_recovers_through_a_successor();
|
|
primitive_profile_recovers_before_readiness();
|
|
}
|