Test life owner routing across region restart
Foundation / checks (deploy) (push) Successful in 6s
Foundation / checks (checks) (push) Failing after 40s
Foundation / checks (postgres) (push) Canceled after 0s
Foundation / checks (tsan) (push) Canceled after 0s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (client) (push) Canceled after 1m12s
Foundation / checks (network) (push) Canceled after 37s
Foundation / checks (ecs) (push) Canceled after 1m9s

This commit is contained in:
dingo
2026-10-11 01:34:46 -04:00
parent a6b1c62168
commit e6ba9bd1a9
5 changed files with 129 additions and 49 deletions
+90 -41
View File
@@ -332,50 +332,58 @@ struct PrimitiveSetup {
claims.expires_at = claims.issued_at + 120;
return must(issuer).sign(claims).value();
}
std::vector<std::string> args(std::uint16_t port = 0) const {
return {env("EVO_SERVER_BIN"),
"--profile",
"primitive",
"--world",
std::to_string(region.key.world),
"--region",
std::to_string(region.key.region),
"--frame",
std::to_string(region_identity.frame),
"--pg-host",
env("PGHOST"),
"--pg-port",
env("PGPORT"),
"--pg-database",
env("PGDATABASE"),
"--pg-user",
env("EVO_PG_RUNTIME_USER"),
"--objects",
region.root.string(),
"--build",
"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1",
"--lease-profile",
"fast",
"--ready-file",
(region.root / "ready").string(),
"--primitive-listen",
"127.0.0.1:" + std::to_string(port),
"--primitive-cert",
(region.root / "server.pem").string(),
"--primitive-key",
(region.root / "server.key").string(),
"--primitive-issuer",
(region.root / "issuer.pub").string(),
"--primitive-environment",
"dev"};
std::vector<std::string> args(std::uint16_t port = 0,
const std::vector<std::string>& routes = {}) const {
auto result = std::vector<std::string>{
env("EVO_SERVER_BIN"),
"--profile",
"primitive",
"--world",
std::to_string(region.key.world),
"--region",
std::to_string(region.key.region),
"--frame",
std::to_string(region_identity.frame),
"--pg-host",
env("PGHOST"),
"--pg-port",
env("PGPORT"),
"--pg-database",
env("PGDATABASE"),
"--pg-user",
env("EVO_PG_RUNTIME_USER"),
"--objects",
region.root.string(),
"--build",
"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1",
"--lease-profile",
"fast",
"--ready-file",
(region.root / "ready").string(),
"--primitive-listen",
"127.0.0.1:" + std::to_string(port),
"--primitive-cert",
(region.root / "server.pem").string(),
"--primitive-key",
(region.root / "server.key").string(),
"--primitive-issuer",
(region.root / "issuer.pub").string(),
"--primitive-environment",
"dev"};
for (const auto& route : routes) {
result.push_back("--life-route");
result.push_back(route);
}
return result;
}
};
struct PrimitiveServer {
std::unique_ptr<Child> child;
std::uint16_t port{};
explicit PrimitiveServer(const PrimitiveSetup& setup, std::uint16_t fixed_port = 0)
: child{std::make_unique<Child>(setup.args(fixed_port))} {
explicit PrimitiveServer(const PrimitiveSetup& setup, std::uint16_t fixed_port = 0,
const std::vector<std::string>& routes = {})
: child{std::make_unique<Child>(setup.args(fixed_port, routes))} {
CHECK(child->wait_line("primitive listening ", 30s));
for (const auto& line : child->lines())
if (starts(line, "primitive listening "))
@@ -434,9 +442,37 @@ void primitive_life_v5_process_round_trip() {
outcomes.front().result == PrimitiveLifeResult::born &&
outcomes.front().life_identity != 0 && outcomes.front().commit_sequence != 0);
const auto born = outcomes.front();
client->close();
server->child->signal(SIGTERM);
CHECK(server->child->wait_exit(30s) == 0);
// A second region sees the durable life owner and redirects before its ledger can
// move account control. The route catalog includes both regions, independently of
// the probe-neighbour configuration.
PrimitiveSetup destination_setup{36};
const auto destination_port = free_port();
const std::vector<std::string> routes{
"35=127.0.0.1:" + std::to_string(server->port) + "=" + name,
"36=127.0.0.1:" + std::to_string(destination_port) + "=" + name};
auto destination =
std::make_unique<PrimitiveServer>(destination_setup, destination_port, routes);
auto redirected =
connect_primitive(destination_setup, *destination, destination_setup.grant(who, 2));
CHECK(until_primitive(*destination, *redirected,
[&] { return redirected->state() == RegionSessionState::closed; }));
CHECK(redirected->close_reason() == SessionCloseReason::moved && redirected->redirect());
CHECK(must(redirected->redirect()) ==
RegionAddress{35, "127.0.0.1:" + std::to_string(server->port), name});
const std::array<std::string, 2> account_key{"7", hex(who)};
const auto control = owner()->query(
"SELECT region_id FROM evolution.account_controls WHERE world_id=$1::evolution.u64 "
"AND account_id=$2::uuid",
account_key, 5s);
CHECK(control && control->size() == 1 && (*control)[0][0] == "35");
destination->child->signal(SIGTERM);
CHECK(destination->child->wait_exit(30s) == 0);
// Kill the owner process without cleanup. Its recovered reservation remains the route
// target after the lease expires and the region host restarts.
server->child->signal(SIGKILL);
CHECK(server->child->wait_exit(30s) == 128 + SIGKILL);
server = std::make_unique<PrimitiveServer>(setup, server->port);
auto resumed = connect_primitive(setup, *server, setup.grant(who, 2));
@@ -449,6 +485,19 @@ void primitive_life_v5_process_round_trip() {
return !outcomes.empty();
}));
CHECK(outcomes.front() == born);
destination = std::make_unique<PrimitiveServer>(destination_setup, destination_port, routes);
auto redirected_after_restart =
connect_primitive(destination_setup, *destination, destination_setup.grant(who, 3));
CHECK(until_primitive(*destination, *redirected_after_restart, [&] {
return redirected_after_restart->state() == RegionSessionState::closed;
}));
CHECK(redirected_after_restart->close_reason() == SessionCloseReason::moved &&
redirected_after_restart->redirect() &&
must(redirected_after_restart->redirect()).region == 35);
destination->child->signal(SIGTERM);
CHECK(destination->child->wait_exit(30s) == 0);
resumed->close();
server->child->signal(SIGTERM);
CHECK(server->child->wait_exit(30s) == 0);