Test life owner routing across region restart
Foundation / checks (deploy) (push) Successful in 6s
Foundation / checks (checks) (push) Failing after 40s
Foundation / checks (postgres) (push) Canceled after 0s
Foundation / checks (tsan) (push) Canceled after 0s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (client) (push) Canceled after 1m12s
Foundation / checks (network) (push) Canceled after 37s
Foundation / checks (ecs) (push) Canceled after 1m9s

This commit is contained in:
dingo
2026-10-11 01:34:46 -04:00
parent a6b1c62168
commit e6ba9bd1a9
5 changed files with 129 additions and 49 deletions
+5 -3
View File
@@ -66,11 +66,13 @@ struct M10Options {
// M11 owner routing, independent of --m10-neighbour:
// REGION=HOST:PORT=TLS_NAME, one entry per region in the world.
std::vector<std::string> life_routes;
[[nodiscard]] bool any() const noexcept {
[[nodiscard]] bool session_any() const noexcept {
return !listen.empty() || !certificate.empty() || !key.empty() || !issuer.empty() ||
!environment.empty() || !session_window.empty() || max_grant_lifetime != 0 ||
!area.empty() || !handoff_key.empty() || !neighbours.empty() ||
!life_routes.empty() || !load_governor;
!area.empty() || !handoff_key.empty() || !neighbours.empty() || !load_governor;
}
[[nodiscard]] bool any() const noexcept {
return session_any() || !life_routes.empty();
}
[[nodiscard]] bool complete() const noexcept {
return !listen.empty() && !certificate.empty() && !key.empty() && !issuer.empty() &&
+16 -5
View File
@@ -309,7 +309,11 @@ std::optional<Options> parse(int argc, char** argv) {
(!server_app::sessions_available || !o.session.complete() || !o.synthetic))
return std::nullopt;
// M10 sessions (ADR-0091/0092) need every setting and the QUIC build.
if (o.m10.any() && (!server_app::m10_available || !o.m10.complete()))
if (o.m10.session_any() && (!server_app::m10_available || !o.m10.complete()))
return std::nullopt;
if (!o.m10.life_routes.empty() && o.profile != "primitive" && o.profile != "cellular")
return std::nullopt;
if (!o.m10.life_routes.empty() && o.profile == "primitive" && !o.primitive_session.any())
return std::nullopt;
// Primitive profile serves only the creation-only host. Its optional listener
// uses a separate v5 contract and never enables the M9/M10 action path.
@@ -317,7 +321,7 @@ std::optional<Options> parse(int argc, char** argv) {
(!server_app::m10_available || !o.primitive_session.complete() || o.profile != "primitive"))
return std::nullopt;
if (o.profile == "primitive" &&
(o.synthetic || !o.crash_point.empty() || o.session.any() || o.m10.any() ||
(o.synthetic || !o.crash_point.empty() || o.session.any() || o.m10.session_any() ||
o.host.flush_interval != 1000ms || o.drain != 10000ms))
return std::nullopt;
if (o.profile == "cellular" &&
@@ -696,8 +700,9 @@ struct PrimitiveSessionBridge {
std::optional<PrimitiveSessionBridge>
open_primitive_sessions(const Options::PrimitiveSessionOptions& options,
server::PrimitiveRegionHost& host, persistence::RegionKey key,
std::uint64_t frame, const persistence::PgOptions& pg) {
const server_app::M10Options& routing, server::PrimitiveRegionHost& host,
persistence::RegionKey key, std::uint64_t frame,
const persistence::PgOptions& pg) {
PrimitiveSessionBridge bridge;
if (!options.any())
return bridge;
@@ -730,6 +735,12 @@ open_primitive_sessions(const Options::PrimitiveSessionOptions& options,
config.control.epoch = 1;
config.control.max_sessions = 512;
config.transport.limits.max_connections = 576;
const auto life_routes = server_app::parse_life_owner_routes(routing);
if (!life_routes) {
std::puts("error primitive-life-routes");
return std::nullopt;
}
config.life_owner_routes = *life_routes;
network::GrantPublicKey issuer_key{};
std::copy(issuer.begin(), issuer.end(), issuer_key.begin());
config.issuer_keys.push_back({1, issuer_key});
@@ -782,7 +793,7 @@ int serve_primitive(Options& o) {
auto& host = **made;
#ifdef EVO_SERVER_M10
auto primitive_sessions =
open_primitive_sessions(o.primitive_session, host, o.host.key, o.host.frame, o.pg);
open_primitive_sessions(o.primitive_session, o.m10, host, o.host.key, o.host.frame, o.pg);
if (!primitive_sessions)
return 2;
#endif
+5
View File
@@ -145,6 +145,11 @@ Local checks do not close ADR-0097's reference-worker qualification.
closed. QUIC tests cover redirect-before-claim and missing-route refusal.
PostgreSQL and migration CI, client switching, and process-kill qualification
remain pending. Do not claim 11.2 complete until redirect/restart gates pass.
- 11.2d-7c qualification: the PostgreSQL-backed region-session process test now
creates a life in region 35, logs the account into region 36 with an explicit
two-region catalog, checks the redirect and unchanged account-control owner,
kills region 35, restarts it, verifies the same life, and checks routing again.
Execution remains pending Linux/PostgreSQL CI.
No deployed world has been provisioned or changed by this work.
+13
View File
@@ -6574,3 +6574,16 @@ also compiled, and the Visual Studio LLVM `clang-format` check and
locally because this Windows environment lacks POSIX `poll.h`; the QUIC test
binary, PostgreSQL integration suite, and CI have not run. The database owner
lookup and route behavior still require CI/runtime qualification.
## 2026-10-11 — M11.2d-7c two-region restart qualification (pending CI)
Extended `tests/region_m10_session_tests.cpp` to use the production primitive
session bridge and PostgreSQL. The scenario creates a life in region 35, tries
login in region 36 with both routes configured, checks `SessionRedirect` and
that `account_controls.region_id` stays 35, SIGKILLs and restarts region 35,
queries the same durable life, and checks the redirect once more.
The process-test source cannot be syntax-compiled locally because the Windows
toolchain lacks POSIX `poll.h`, which its existing process fixture requires.
The app bridge header compile, formatting check, and `git diff --check` passed;
the PostgreSQL process test has not run and requires CI confirmation.
+90 -41
View File
@@ -332,50 +332,58 @@ struct PrimitiveSetup {
claims.expires_at = claims.issued_at + 120;
return must(issuer).sign(claims).value();
}
std::vector<std::string> args(std::uint16_t port = 0) const {
return {env("EVO_SERVER_BIN"),
"--profile",
"primitive",
"--world",
std::to_string(region.key.world),
"--region",
std::to_string(region.key.region),
"--frame",
std::to_string(region_identity.frame),
"--pg-host",
env("PGHOST"),
"--pg-port",
env("PGPORT"),
"--pg-database",
env("PGDATABASE"),
"--pg-user",
env("EVO_PG_RUNTIME_USER"),
"--objects",
region.root.string(),
"--build",
"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1",
"--lease-profile",
"fast",
"--ready-file",
(region.root / "ready").string(),
"--primitive-listen",
"127.0.0.1:" + std::to_string(port),
"--primitive-cert",
(region.root / "server.pem").string(),
"--primitive-key",
(region.root / "server.key").string(),
"--primitive-issuer",
(region.root / "issuer.pub").string(),
"--primitive-environment",
"dev"};
std::vector<std::string> args(std::uint16_t port = 0,
const std::vector<std::string>& routes = {}) const {
auto result = std::vector<std::string>{
env("EVO_SERVER_BIN"),
"--profile",
"primitive",
"--world",
std::to_string(region.key.world),
"--region",
std::to_string(region.key.region),
"--frame",
std::to_string(region_identity.frame),
"--pg-host",
env("PGHOST"),
"--pg-port",
env("PGPORT"),
"--pg-database",
env("PGDATABASE"),
"--pg-user",
env("EVO_PG_RUNTIME_USER"),
"--objects",
region.root.string(),
"--build",
"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1",
"--lease-profile",
"fast",
"--ready-file",
(region.root / "ready").string(),
"--primitive-listen",
"127.0.0.1:" + std::to_string(port),
"--primitive-cert",
(region.root / "server.pem").string(),
"--primitive-key",
(region.root / "server.key").string(),
"--primitive-issuer",
(region.root / "issuer.pub").string(),
"--primitive-environment",
"dev"};
for (const auto& route : routes) {
result.push_back("--life-route");
result.push_back(route);
}
return result;
}
};
struct PrimitiveServer {
std::unique_ptr<Child> child;
std::uint16_t port{};
explicit PrimitiveServer(const PrimitiveSetup& setup, std::uint16_t fixed_port = 0)
: child{std::make_unique<Child>(setup.args(fixed_port))} {
explicit PrimitiveServer(const PrimitiveSetup& setup, std::uint16_t fixed_port = 0,
const std::vector<std::string>& routes = {})
: child{std::make_unique<Child>(setup.args(fixed_port, routes))} {
CHECK(child->wait_line("primitive listening ", 30s));
for (const auto& line : child->lines())
if (starts(line, "primitive listening "))
@@ -434,9 +442,37 @@ void primitive_life_v5_process_round_trip() {
outcomes.front().result == PrimitiveLifeResult::born &&
outcomes.front().life_identity != 0 && outcomes.front().commit_sequence != 0);
const auto born = outcomes.front();
client->close();
server->child->signal(SIGTERM);
CHECK(server->child->wait_exit(30s) == 0);
// A second region sees the durable life owner and redirects before its ledger can
// move account control. The route catalog includes both regions, independently of
// the probe-neighbour configuration.
PrimitiveSetup destination_setup{36};
const auto destination_port = free_port();
const std::vector<std::string> routes{
"35=127.0.0.1:" + std::to_string(server->port) + "=" + name,
"36=127.0.0.1:" + std::to_string(destination_port) + "=" + name};
auto destination =
std::make_unique<PrimitiveServer>(destination_setup, destination_port, routes);
auto redirected =
connect_primitive(destination_setup, *destination, destination_setup.grant(who, 2));
CHECK(until_primitive(*destination, *redirected,
[&] { return redirected->state() == RegionSessionState::closed; }));
CHECK(redirected->close_reason() == SessionCloseReason::moved && redirected->redirect());
CHECK(must(redirected->redirect()) ==
RegionAddress{35, "127.0.0.1:" + std::to_string(server->port), name});
const std::array<std::string, 2> account_key{"7", hex(who)};
const auto control = owner()->query(
"SELECT region_id FROM evolution.account_controls WHERE world_id=$1::evolution.u64 "
"AND account_id=$2::uuid",
account_key, 5s);
CHECK(control && control->size() == 1 && (*control)[0][0] == "35");
destination->child->signal(SIGTERM);
CHECK(destination->child->wait_exit(30s) == 0);
// Kill the owner process without cleanup. Its recovered reservation remains the route
// target after the lease expires and the region host restarts.
server->child->signal(SIGKILL);
CHECK(server->child->wait_exit(30s) == 128 + SIGKILL);
server = std::make_unique<PrimitiveServer>(setup, server->port);
auto resumed = connect_primitive(setup, *server, setup.grant(who, 2));
@@ -449,6 +485,19 @@ void primitive_life_v5_process_round_trip() {
return !outcomes.empty();
}));
CHECK(outcomes.front() == born);
destination = std::make_unique<PrimitiveServer>(destination_setup, destination_port, routes);
auto redirected_after_restart =
connect_primitive(destination_setup, *destination, destination_setup.grant(who, 3));
CHECK(until_primitive(*destination, *redirected_after_restart, [&] {
return redirected_after_restart->state() == RegionSessionState::closed;
}));
CHECK(redirected_after_restart->close_reason() == SessionCloseReason::moved &&
redirected_after_restart->redirect() &&
must(redirected_after_restart->redirect()).region == 35);
destination->child->signal(SIGTERM);
CHECK(destination->child->wait_exit(30s) == 0);
resumed->close();
server->child->signal(SIGTERM);
CHECK(server->child->wait_exit(30s) == 0);