Foundation / checks (checks) (push) Failing after 56s
Foundation / checks (ecs) (push) Successful in 1m45s
Foundation / checks (postgres) (push) Canceled after 0s
Foundation / checks (tsan) (push) Canceled after 0s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (deploy) (push) Canceled after 3m3s
Foundation / checks (client) (push) Canceled after 3m9s
Foundation / checks (network) (push) Canceled after 28s
310 lines
14 KiB
C++
310 lines
14 KiB
C++
// Real process tests for M10.4c (ADR-0091, ADR-0092): evo-server with PostgreSQL, synthetic
|
|
// ticks and the QUIC session listener; in-process RegionSessionClients and the headless
|
|
// probe. Intentions get receipts and the simulated watermark; actions get durable outcomes
|
|
// that survive SIGKILL after their commit, and resolve as "no record" (then commit once on
|
|
// retry) after SIGKILL before it. Region 7/151. Run by tools/ci/postgres-check.sh
|
|
// when the build has EVO_BUILD_QUIC.
|
|
#include "evo/client/region_session.hpp"
|
|
#include "evo/network/dev_credentials.hpp"
|
|
#include "evo/network/session_grant.hpp"
|
|
#include "fixtures/server_process.hpp"
|
|
#include <chrono>
|
|
#include <fcntl.h>
|
|
#include <functional>
|
|
#include <optional>
|
|
#include <source_location>
|
|
#include <thread>
|
|
#include <unistd.h>
|
|
|
|
void run_region_m10_session_tests();
|
|
|
|
namespace {
|
|
using namespace evo::test::process;
|
|
using namespace evo;
|
|
using namespace evo::network;
|
|
using client::RegionSessionClient;
|
|
using client::RegionSessionState;
|
|
using evo::test::region_identity;
|
|
void check(bool value, std::source_location at = std::source_location::current()) {
|
|
if (!value)
|
|
throw std::runtime_error("region m10 session test line " + std::to_string(at.line()));
|
|
}
|
|
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
|
template <typename T>
|
|
const T& must(const std::optional<T>& o,
|
|
std::source_location at = std::source_location::current()) {
|
|
if (!o)
|
|
throw std::runtime_error("region m10 session missing value line " +
|
|
std::to_string(at.line()));
|
|
return *o;
|
|
}
|
|
constexpr const char* name = "dev-region.evolution.test";
|
|
|
|
void write_file(const fs::path& path, std::span<const std::uint8_t> bytes) {
|
|
const int fd = ::open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0600);
|
|
expect(fd >= 0 &&
|
|
::write(fd, bytes.data(), bytes.size()) == static_cast<ssize_t>(bytes.size()) &&
|
|
::close(fd) == 0,
|
|
"write file");
|
|
}
|
|
void write_text(const fs::path& path, const std::string& text) {
|
|
write_file(path, {reinterpret_cast<const std::uint8_t*>(text.data()), text.size()});
|
|
}
|
|
AccountId account(std::uint8_t tag) {
|
|
AccountId a;
|
|
a.bytes.fill(tag);
|
|
return a;
|
|
}
|
|
std::string hex(const AccountId& a) {
|
|
constexpr char digits[] = "0123456789abcdef";
|
|
std::string out;
|
|
for (const auto b : a.bytes) {
|
|
out.push_back(digits[b >> 4U]);
|
|
out.push_back(digits[b & 0xFU]);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// One region's credentials, issuer key and server process arguments.
|
|
struct Setup {
|
|
ServerRegion region;
|
|
std::string ca_pem;
|
|
std::optional<GrantSigningKey> issuer;
|
|
explicit Setup(std::uint64_t r, std::uint8_t genesis) : region{provision(r, genesis)} {
|
|
const auto made = generate_dev_credentials(DeploymentEnvironment::dev, name, 1,
|
|
std::chrono::system_clock::now());
|
|
expect(made.has_value(), "credentials");
|
|
ca_pem = made->authority_pem;
|
|
write_text(region.root / "ca.pem", made->authority_pem);
|
|
write_text(region.root / "server.pem", made->server_certificate_pem);
|
|
write_text(region.root / "server.key", made->server_key_pem);
|
|
for (int attempt = 0; attempt < 8 && !issuer; ++attempt) {
|
|
std::array<std::uint8_t, grant_private_key_bytes> scalar{};
|
|
expect(random_bytes(scalar).has_value(), "random");
|
|
if (auto key = GrantSigningKey::import(DeploymentEnvironment::dev, 1, scalar)) {
|
|
write_file(region.root / "issuer.key", scalar);
|
|
write_file(region.root / "issuer.pub", *key->public_key());
|
|
issuer.emplace(std::move(*key));
|
|
}
|
|
}
|
|
expect(issuer.has_value(), "issuer key");
|
|
}
|
|
[[nodiscard]] std::vector<std::string> args() const {
|
|
return server_args(region,
|
|
{"--synthetic-inputs", "development-v1", "--m10-listen", "127.0.0.1:0",
|
|
"--m10-cert", (region.root / "server.pem").string(), "--m10-key",
|
|
(region.root / "server.key").string(), "--m10-issuer",
|
|
(region.root / "issuer.pub").string(), "--m10-environment", "dev", "--session-window", "test-" + std::to_string(static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count()) + 1200)});
|
|
}
|
|
[[nodiscard]] WireGrant grant(const AccountId& who, std::uint64_t serial) const {
|
|
GrantClaims c;
|
|
c.grant_id = 9000 + serial;
|
|
c.account = who;
|
|
c.serial = serial;
|
|
c.world = {region.key.world, region_identity.frame};
|
|
c.region = region.key.region;
|
|
c.issued_at =
|
|
static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(
|
|
std::chrono::system_clock::now().time_since_epoch())
|
|
.count());
|
|
c.expires_at = c.issued_at + 120;
|
|
const auto g = must(issuer).sign(c);
|
|
expect(g.has_value(), "grant");
|
|
return *g;
|
|
}
|
|
};
|
|
|
|
// A running evo-server and its QUIC port, its output drained while waiting.
|
|
struct Server {
|
|
std::unique_ptr<Child> child;
|
|
std::uint16_t port{};
|
|
explicit Server(const Setup& s) : child{std::make_unique<Child>(s.args())} {
|
|
CHECK(child->wait_line("m10 listening ", 30s));
|
|
for (const auto& l : child->lines())
|
|
if (starts(l, "m10 listening "))
|
|
port = static_cast<std::uint16_t>(std::stoul(l.substr(14)));
|
|
CHECK(port != 0 && child->wait_line("ready 1", 30s));
|
|
}
|
|
void pump() {
|
|
child->pump(1ms);
|
|
}
|
|
void kill() {
|
|
child->signal(SIGKILL);
|
|
CHECK(child->wait_exit(30s) == 128 + SIGKILL);
|
|
}
|
|
};
|
|
|
|
std::unique_ptr<RegionSessionClient> connect(const Setup& s, Server& server,
|
|
const WireGrant& grant) {
|
|
client::RegionSessionClientConfig c;
|
|
c.transport.port = server.port;
|
|
c.transport.server_name = name;
|
|
c.transport.authority_pem = s.ca_pem;
|
|
c.transport.authority_pem_file = (s.region.root / "ca.pem").string();
|
|
c.grant = grant;
|
|
auto made = RegionSessionClient::connect(std::move(c), std::chrono::steady_clock::now());
|
|
CHECK(made.has_value());
|
|
return std::move(*made);
|
|
}
|
|
// Polls client and server until `done`, at most `limit`.
|
|
bool until(Server& server, RegionSessionClient& c, const std::function<bool()>& done,
|
|
std::chrono::milliseconds limit = 20s) {
|
|
const auto end = Clock::now() + limit;
|
|
while (Clock::now() < end) {
|
|
c.poll(Clock::now());
|
|
server.pump();
|
|
if (done())
|
|
return true;
|
|
}
|
|
return done();
|
|
}
|
|
bool established(Server& server, RegionSessionClient& c) {
|
|
return until(server, c,
|
|
[&] {
|
|
return c.state() == RegionSessionState::established ||
|
|
c.state() == RegionSessionState::closed;
|
|
}) &&
|
|
c.state() == RegionSessionState::established;
|
|
}
|
|
std::optional<ActionOutcome> outcome_of(Server& server, RegionSessionClient& c, std::uint64_t id,
|
|
bool final_only) {
|
|
std::optional<ActionOutcome> found;
|
|
until(server, c, [&] {
|
|
for (const auto& o : c.take_action_outcomes())
|
|
if (o.action_id == id &&
|
|
(!final_only || o.result == ActionResult::committed_no_effect ||
|
|
o.result == ActionResult::not_committed))
|
|
found = o;
|
|
return found.has_value();
|
|
});
|
|
return found;
|
|
}
|
|
std::optional<ActionReceipt> receipt_of(Server& server, RegionSessionClient& c, std::uint64_t id) {
|
|
std::optional<ActionReceipt> found;
|
|
until(server, c, [&] {
|
|
for (const auto& r : c.take_action_receipts())
|
|
if (r.action_id == id)
|
|
found = r;
|
|
return found.has_value();
|
|
});
|
|
return found;
|
|
}
|
|
// "tick:commit" of the durable record of (account, id) in world 7, or empty.
|
|
std::string record(const AccountId& who, std::uint64_t id) {
|
|
const std::array<std::string, 2> p{hex(who), std::to_string(id)};
|
|
auto rows =
|
|
owner()->query("SELECT tick || ':' || commit_sequence FROM evolution.region_actions "
|
|
"WHERE world_id = 7 AND account_id = $1::uuid AND "
|
|
"action_id = $2::evolution.u64",
|
|
p, 5s);
|
|
CHECK(rows.has_value());
|
|
return rows->empty() ? std::string{} : (*rows)[0][0].value_or("");
|
|
}
|
|
|
|
void end_to_end() {
|
|
Setup s{151, 0xC5};
|
|
const auto alice = account(0x51), bob = account(0x52);
|
|
auto server = std::make_unique<Server>(s);
|
|
auto a = connect(s, *server, s.grant(alice, 1));
|
|
CHECK(established(*server, *a));
|
|
// Intentions: receipts, then the watermark.
|
|
// Datagrams may be refused until the QUIC datagram extension is negotiated: retry.
|
|
std::optional<std::uint64_t> sequence;
|
|
CHECK(until(*server, *a, [&] {
|
|
return a->server_tick() > 0 &&
|
|
(sequence = a->send_intention(a->server_tick() + 2, 100, 0)).has_value();
|
|
}));
|
|
CHECK(sequence == 1u);
|
|
std::vector<IntentionReceipt> receipts;
|
|
CHECK(until(*server, *a, [&] { return !(receipts = a->take_intention_receipts()).empty(); }));
|
|
CHECK(receipts[0].sequence == 1 && receipts[0].status == AdmitStatus::admitted);
|
|
CHECK(until(*server, *a, [&] { return a->simulated() && a->simulated()->sequence >= 1; }));
|
|
// Action: receipt, then a final outcome naming the durable commit.
|
|
CHECK(a->send_action(1, ActionKind::probe, a->server_tick() + 3));
|
|
const auto r1 = receipt_of(*server, *a, 1);
|
|
CHECK(r1 && must(r1).status == AdmitStatus::admitted);
|
|
const auto o1 = outcome_of(*server, *a, 1, true);
|
|
CHECK(o1 && must(o1).result == ActionResult::committed_no_effect &&
|
|
must(o1).tick == must(r1).tick && must(o1).commit_sequence > 0);
|
|
CHECK(record(alice, 1) ==
|
|
std::to_string(must(o1).tick) + ":" + std::to_string(must(o1).commit_sequence));
|
|
|
|
// SIGKILL after the commit (the record exists), whether or not the outcome was sent.
|
|
CHECK(a->send_action(2, ActionKind::probe, a->server_tick() + 3));
|
|
const auto r2 = receipt_of(*server, *a, 2);
|
|
CHECK(r2 && must(r2).status == AdmitStatus::admitted);
|
|
const bool recorded = until(*server, *a, [&] { return !record(alice, 2).empty(); });
|
|
if (!recorded)
|
|
for (const auto& line : server->child->lines())
|
|
if (starts(line, "m10") || starts(line, "phase") || starts(line, "dropped"))
|
|
std::printf("server: %s\n", line.c_str());
|
|
CHECK(recorded);
|
|
server->kill();
|
|
server = std::make_unique<Server>(s); // Waits out the old lease.
|
|
auto b = connect(s, *server, s.grant(alice, 2));
|
|
CHECK(established(*server, *b));
|
|
CHECK(b->query_action(2));
|
|
const auto o2 = outcome_of(*server, *b, 2, false);
|
|
CHECK(o2 && must(o2).result == ActionResult::committed_no_effect &&
|
|
record(alice, 2) ==
|
|
std::to_string(must(o2).tick) + ":" + std::to_string(must(o2).commit_sequence));
|
|
// A retry of a committed action on the new process: duplicate, same outcome.
|
|
CHECK(b->send_action(1, ActionKind::probe, b->server_tick() + 3));
|
|
const auto d1 = receipt_of(*server, *b, 1);
|
|
CHECK(d1 && must(d1).status == AdmitStatus::duplicate && must(d1).tick == must(o1).tick);
|
|
CHECK(outcome_of(*server, *b, 1, true) == o1);
|
|
|
|
// SIGKILL before the commit: the action's tick has not run when the process dies.
|
|
CHECK(b->send_action(3, ActionKind::probe, b->server_tick() + 8));
|
|
const auto r3 = receipt_of(*server, *b, 3);
|
|
CHECK(r3 && must(r3).status == AdmitStatus::admitted);
|
|
server->kill();
|
|
CHECK(record(alice, 3).empty());
|
|
server = std::make_unique<Server>(s);
|
|
auto c = connect(s, *server, s.grant(alice, 3));
|
|
CHECK(established(*server, *c));
|
|
CHECK(c->query_action(3));
|
|
const auto unknown = outcome_of(*server, *c, 3, false);
|
|
CHECK(unknown && unknown->result == ActionResult::unknown); // No record: never committed.
|
|
CHECK(c->send_action(3, ActionKind::probe, c->server_tick() + 3)); // Safe to retry.
|
|
CHECK(must(receipt_of(*server, *c, 3)).status == AdmitStatus::admitted);
|
|
const auto o3 = outcome_of(*server, *c, 3, true);
|
|
CHECK(o3 && must(o3).result == ActionResult::committed_no_effect && !record(alice, 3).empty());
|
|
|
|
// The headless probe: login, intentions, actions and a query, then a clean exit.
|
|
const auto probe = fs::path(env("EVO_SERVER_BIN")).parent_path() / "evo-session-probe";
|
|
Child p{{probe.string(), "run",
|
|
"--connect", "127.0.0.1:" + std::to_string(server->port),
|
|
"--server-name", name,
|
|
"--ca", (s.region.root / "ca.pem").string(),
|
|
"--issuer-key", (s.region.root / "issuer.key").string(),
|
|
"--world", "7",
|
|
"--frame", std::to_string(region_identity.frame),
|
|
"--region", "151",
|
|
"--account", hex(bob),
|
|
"--serial", "1",
|
|
"--intents", "3",
|
|
"--actions", "2",
|
|
"--action-base", "10",
|
|
"--query", "10"}};
|
|
int code = -1;
|
|
for (const auto end = Clock::now() + 30s; Clock::now() < end && code < 0;) {
|
|
server->pump();
|
|
p.pump(5ms);
|
|
if (p.printed("done") || p.printed("timeout") || p.printed("closed"))
|
|
code = p.wait_exit(10s);
|
|
}
|
|
CHECK(code == 0 && p.printed("welcome ") && p.printed("intention 1 0") &&
|
|
p.printed("outcome 10 1 ") && p.printed("outcome 11 1 "));
|
|
CHECK(!record(bob, 10).empty() && !record(bob, 11).empty());
|
|
server->child->signal(SIGTERM);
|
|
CHECK(server->child->wait_exit(30s) == 0);
|
|
CHECK(server->child->printed("m10 opened ") && server->child->printed("m10 committed "));
|
|
}
|
|
} // namespace
|
|
|
|
void run_region_m10_session_tests() {
|
|
end_to_end();
|
|
std::puts("region m10 session integration passed");
|
|
}
|