Complete M7.6 bounded authoritative region host
ADR-0064: RegionHost<State> (environmental and lineage profiles) is a single-owner, single-threaded state machine advanced by step(now): acquire the fenced lease and recover the verified head, run, resolve ambiguous commits, drain and stop, or quarantine. Admission is provisional and bounded (64 inputs, 64 KiB) with backpressure; FinalAck and FinalOutcome are emitted only after a confirmed commit, and inputs that will never be acknowledged are reported with a reason. Ticks follow the caller's scheduling clock with bounded catch-up and measured tick debt, never invented inputs. A death seals the batch and commits in the same step, so no dependent tick uses an uncommitted outcome. Storage outages pause and retry, unknown outcomes resolve by exact bytes, fencing discards unconfirmed work and re-acquires; readiness fails closed. Drain commits pending ticks and releases the lease. evo-server hosts one region with a bounded stdin input feed, stdout event lines, a readiness file and SIGTERM drain (exit 0/3/4). Validated: Windows CTest 58/58; PostgreSQL suites under GCC Release, clang-tidy and ASan/UBSan including real-process SIGTERM drain and SIGKILL with successor recovery; four host mutations each caught. No gateway, metrics, Docker/Gitea CI or deployment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -5,6 +5,8 @@
|
||||
#include <stdexcept>
|
||||
void run_region_lease_store_tests();
|
||||
void run_region_checkpoint_store_tests();
|
||||
void run_region_host_tests();
|
||||
void run_region_server_process_tests();
|
||||
namespace {
|
||||
using namespace evo::persistence;
|
||||
using namespace std::chrono_literals;
|
||||
@@ -75,6 +77,10 @@ int main() {
|
||||
std::puts("region lease fencing integration passed");
|
||||
run_region_checkpoint_store_tests();
|
||||
std::puts("region checkpoint publication integration passed");
|
||||
run_region_host_tests();
|
||||
std::puts("region host integration passed");
|
||||
run_region_server_process_tests();
|
||||
std::puts("region server process integration passed");
|
||||
return 0;
|
||||
} catch (const std::exception& e) {
|
||||
std::fprintf(stderr, "%s\n", e.what());
|
||||
|
||||
@@ -0,0 +1,332 @@
|
||||
// Real PostgreSQL M7.6 region host behavior (ADR-0064). Run by tools/ci/postgres-check.sh.
|
||||
#include "evo/server/region_host.hpp"
|
||||
#include "fixtures/environmental_region.hpp"
|
||||
#include <algorithm>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <filesystem>
|
||||
#include <source_location>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
|
||||
void run_region_host_tests();
|
||||
|
||||
namespace {
|
||||
using namespace evo::persistence;
|
||||
using namespace evo::server;
|
||||
using namespace std::chrono_literals;
|
||||
using evo::processes::EnvironmentalCommunity;
|
||||
using evo::processes::LineageRegion;
|
||||
namespace fs = std::filesystem;
|
||||
void check(bool value, std::source_location at = std::source_location::current()) {
|
||||
if (!value)
|
||||
throw std::runtime_error("region host test line " + std::to_string(at.line()));
|
||||
}
|
||||
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
||||
std::string env(const char* key) {
|
||||
const auto* value = std::getenv(key);
|
||||
if (!value)
|
||||
throw std::runtime_error("missing test environment");
|
||||
return value;
|
||||
}
|
||||
PgOptions options(const std::string& user) {
|
||||
return {env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}};
|
||||
}
|
||||
std::unique_ptr<Postgres> connect_as(const std::string& user) {
|
||||
auto made = Postgres::connect(options(user), 5s);
|
||||
CHECK(made);
|
||||
return std::move(*made);
|
||||
}
|
||||
Connect runtime_connect() {
|
||||
return [] { return Postgres::connect(options(env("EVO_PG_RUNTIME_USER")), 2s); };
|
||||
}
|
||||
template <std::size_t N> std::array<std::uint8_t, N> filled(std::uint8_t value) {
|
||||
std::array<std::uint8_t, N> out{};
|
||||
out.fill(value);
|
||||
return out;
|
||||
}
|
||||
const Sha256Digest build = filled<32>(0xB1);
|
||||
// Fast lease for tests: 4 s, renew every 0.8 s, 0.4 s margin/renew deadline, 1 s commits.
|
||||
const LeasePolicy fast{4000ms, 800ms, 400ms, 400ms, 1000ms};
|
||||
TickInput input(std::uint64_t tick, bool consumer_dies = false) {
|
||||
Uuid request{};
|
||||
request[0] = 0x9A;
|
||||
for (unsigned i = 0; i < 8; ++i)
|
||||
request[8 + i] = static_cast<std::uint8_t>((tick >> (8U * i)) & 0xFFU);
|
||||
return {request, evo::test::region_input(tick, consumer_dies)};
|
||||
}
|
||||
|
||||
struct Region {
|
||||
RegionKey key{7, 21};
|
||||
fs::path root;
|
||||
};
|
||||
// Provisions a fresh region (owner) and returns its key and object root.
|
||||
template <typename State> Region provisioned(std::uint64_t region, std::uint8_t genesis) {
|
||||
Region r{{7, 20 + region}, fs::path(env("PGHOST")) / ("host-" + std::to_string(region))};
|
||||
fs::remove_all(r.root);
|
||||
fs::create_directory(r.root);
|
||||
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
|
||||
auto owner = connect_as(env("PGUSER"));
|
||||
CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
||||
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
|
||||
{}, 5s));
|
||||
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
|
||||
CHECK(objects);
|
||||
if constexpr (std::is_same_v<State, LineageRegion>)
|
||||
CHECK(provision_lineage_region(*owner, **objects, r.key, evo::test::region_identity.frame,
|
||||
*evo::test::lineage_region(), filled<16>(genesis), build,
|
||||
8000000, 5s));
|
||||
else
|
||||
CHECK(provision_environmental_region(
|
||||
*owner, **objects, r.key, evo::test::region_identity.frame,
|
||||
*evo::test::region_community(), filled<16>(genesis), build, 8000000, 5s));
|
||||
return r;
|
||||
}
|
||||
HostConfig config(const Region& r) {
|
||||
HostConfig c;
|
||||
c.key = r.key;
|
||||
c.frame = evo::test::region_identity.frame;
|
||||
c.lease = fast;
|
||||
c.build = build;
|
||||
c.retained_builds = {build};
|
||||
return c;
|
||||
}
|
||||
template <typename State>
|
||||
std::unique_ptr<RegionHost<State>> host(const Region& r, HostConfig c,
|
||||
ObjectStoreFaults* faults = nullptr) {
|
||||
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes, faults);
|
||||
CHECK(objects);
|
||||
auto made = RegionHost<State>::create(std::move(c), runtime_connect(), std::move(*objects));
|
||||
CHECK(made);
|
||||
return std::move(*made);
|
||||
}
|
||||
// Steps until `done` holds, advancing scheduling time by `advance` each step.
|
||||
template <typename Host, typename Done>
|
||||
const HostStatus& step_until(Host& h, SchedulerTime& t, std::chrono::microseconds advance,
|
||||
Done done, int limit = 400) {
|
||||
for (int i = 0; i < limit; ++i) {
|
||||
t += advance;
|
||||
const auto& s = h.step(t);
|
||||
if (done(s))
|
||||
return s;
|
||||
std::this_thread::sleep_for(10ms);
|
||||
}
|
||||
CHECK(false);
|
||||
return h.status();
|
||||
}
|
||||
|
||||
// Admission, provisional ticks, pause at a permanent outcome and final acknowledgement.
|
||||
void acknowledgements_follow_commits() {
|
||||
const auto r = provisioned<LineageRegion>(1, 0x3A);
|
||||
auto h = host<LineageRegion>(r, config(r));
|
||||
CHECK(h->submit(input(1)).error() == AdmissionError::not_ready);
|
||||
SchedulerTime t = std::chrono::steady_clock::now();
|
||||
auto s = h->step(t);
|
||||
CHECK(s.phase == HostPhase::running && s.ready && s.committed_tick == 0 && s.epoch == 1);
|
||||
for (std::uint64_t tick = 1; tick <= 64; ++tick)
|
||||
CHECK(h->submit(input(tick, true)));
|
||||
CHECK(h->submit(input(65)).error() == AdmissionError::full); // Backlog of 64.
|
||||
CHECK(h->submit(input(1)).error() == AdmissionError::duplicate);
|
||||
CHECK(h->submit({{}, {}}).error() == AdmissionError::invalid);
|
||||
CHECK(h->status().queued_bytes == 64 * tick_input_bytes);
|
||||
for (std::uint64_t tick = 1; tick <= 19; ++tick) {
|
||||
t += 50ms;
|
||||
s = h->step(t);
|
||||
// Simulated ahead of committed; nothing is final yet.
|
||||
CHECK(s.simulated_tick == tick && s.committed_tick == 0 && s.ready);
|
||||
CHECK(h->take_acks().empty() && h->take_outcomes().empty());
|
||||
}
|
||||
// Tick 20 kills the consumer: the region pauses and commits in the same step,
|
||||
// so no dependent tick ever runs on the uncommitted death.
|
||||
t += 50ms;
|
||||
s = h->step(t);
|
||||
CHECK(s.simulated_tick == 20 && s.committed_tick == 20 && s.committed_sequence == 1);
|
||||
const auto outcomes = h->take_outcomes();
|
||||
CHECK(outcomes.size() == 1 && outcomes[0].commit_sequence == 1 &&
|
||||
outcomes[0].decision == BatchDecision{20, Actor::consumer, Outcome::death, 11});
|
||||
auto acks = h->take_acks();
|
||||
CHECK(acks.size() == 20);
|
||||
for (std::uint64_t i = 0; i < 20; ++i)
|
||||
CHECK(acks[i] == FinalAck{input(i + 1).request, i + 1, 1});
|
||||
// The next batch flushes after the configured interval, not per tick.
|
||||
bool provisional = false;
|
||||
s = step_until(*h, t, 50ms, [&](const HostStatus& x) {
|
||||
provisional = provisional || (x.simulated_tick > x.committed_tick);
|
||||
return x.committed_sequence == 2;
|
||||
});
|
||||
CHECK(provisional && s.simulated_tick == s.committed_tick && s.committed_tick > 30);
|
||||
acks = h->take_acks();
|
||||
CHECK(!acks.empty() && acks.back().tick == s.committed_tick && acks.front().tick == 21);
|
||||
CHECK(std::ranges::all_of(acks, [](const FinalAck& a) { return a.commit_sequence == 2; }));
|
||||
CHECK(h->take_dropped().empty());
|
||||
std::printf("host_commits=%llu committed_tick=%llu\n",
|
||||
static_cast<unsigned long long>(s.commits),
|
||||
static_cast<unsigned long long>(s.committed_tick));
|
||||
}
|
||||
|
||||
// No inputs, no invented ticks: debt accrues and is repaid with bounded catch-up.
|
||||
void starvation_accrues_bounded_debt() {
|
||||
const auto r = provisioned<LineageRegion>(2, 0x3B);
|
||||
auto h = host<LineageRegion>(r, config(r));
|
||||
SchedulerTime t = std::chrono::steady_clock::now();
|
||||
CHECK(h->step(t).ready);
|
||||
for (int i = 0; i < 10; ++i) {
|
||||
t += 50ms;
|
||||
CHECK(h->step(t).simulated_tick == 0);
|
||||
}
|
||||
CHECK(h->status().tick_debt == 10);
|
||||
for (std::uint64_t tick = 1; tick <= 20; ++tick)
|
||||
CHECK(h->submit(input(tick)));
|
||||
auto s = h->step(t);
|
||||
CHECK(s.simulated_tick == 8 && s.tick_debt == 2); // At most 8 per step.
|
||||
s = h->step(t);
|
||||
CHECK(s.simulated_tick == 10 && s.tick_debt == 0 && s.max_tick_debt == 10);
|
||||
CHECK(s.queued == 10);
|
||||
}
|
||||
|
||||
// Object store outage: the sealed batch waits, nothing is acknowledged, and the
|
||||
// region resumes once storage is durable again.
|
||||
void storage_outage_pauses_the_region() {
|
||||
const auto r = provisioned<LineageRegion>(3, 0x41);
|
||||
ObjectStoreFaults faults;
|
||||
auto h = host<LineageRegion>(r, config(r), &faults);
|
||||
SchedulerTime t = std::chrono::steady_clock::now();
|
||||
CHECK(h->step(t).ready);
|
||||
for (std::uint64_t tick = 1; tick <= 30; ++tick)
|
||||
CHECK(h->submit(input(tick, true)));
|
||||
step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 19; });
|
||||
faults.fail_directory_syncs = 1000;
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
t += 50ms;
|
||||
const auto& s = h->step(t);
|
||||
CHECK(s.simulated_tick == 20 && s.committed_tick == 0); // Paused at the death.
|
||||
}
|
||||
CHECK(h->status().store_failures >= 5 && h->take_acks().empty() && h->take_outcomes().empty());
|
||||
CHECK(h->status().tick_debt == 4);
|
||||
faults.fail_directory_syncs = 0;
|
||||
t += 50ms;
|
||||
auto s = h->step(t);
|
||||
CHECK(s.committed_tick == 20 && h->take_acks().size() == 20 && h->take_outcomes().size() == 1);
|
||||
s = h->step(t); // Catch-up after the outage is bounded but continues.
|
||||
CHECK(s.simulated_tick > 20 && s.simulated_tick <= 28);
|
||||
}
|
||||
|
||||
// Holds the lease row lock to make database calls time out.
|
||||
struct RowLock {
|
||||
std::unique_ptr<Postgres> db = connect_as(env("PGUSER"));
|
||||
explicit RowLock(RegionKey key) {
|
||||
const std::array<std::string, 2> p{std::to_string(key.world), std::to_string(key.region)};
|
||||
CHECK(db->query("BEGIN", {}, 5s));
|
||||
CHECK(db->query("SELECT 1 FROM evolution.region_leases WHERE world_id = $1::evolution.u64 "
|
||||
"AND region_id = $2::evolution.u64 FOR UPDATE",
|
||||
p, 5s));
|
||||
}
|
||||
void release() {
|
||||
CHECK(db->query("ROLLBACK", {}, 5s));
|
||||
}
|
||||
};
|
||||
|
||||
// Unknown commit outcome: not ready, admission refused, resolved by exact bytes.
|
||||
void unknown_outcome_is_resolved_before_continuing() {
|
||||
const auto r = provisioned<LineageRegion>(4, 0x3D);
|
||||
auto h = host<LineageRegion>(r, config(r));
|
||||
SchedulerTime t = std::chrono::steady_clock::now();
|
||||
CHECK(h->step(t).ready);
|
||||
for (std::uint64_t tick = 1; tick <= 25; ++tick)
|
||||
CHECK(h->submit(input(tick, true)));
|
||||
step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 19; });
|
||||
RowLock lock{r.key};
|
||||
t += 50ms;
|
||||
auto s = h->step(t); // The death seals; the commit cannot finish in time.
|
||||
CHECK(s.phase == HostPhase::resolving && !s.ready && s.unknown_outcomes == 1);
|
||||
CHECK(h->submit(input(26)).error() == AdmissionError::not_ready);
|
||||
CHECK(h->take_outcomes().empty() && h->take_acks().empty());
|
||||
lock.release();
|
||||
s = step_until(*h, t, 0ms, [](const HostStatus& x) { return x.phase == HostPhase::running; });
|
||||
CHECK(s.committed_tick == 20 && s.ready);
|
||||
CHECK(h->take_outcomes().size() == 1 && h->take_acks().size() == 20);
|
||||
}
|
||||
|
||||
// Renewals that cannot reach the database fence the host locally; unconfirmed
|
||||
// work is reported as never final, and authority is re-acquired from the head.
|
||||
void renewal_failure_fences_and_recovers() {
|
||||
const auto r = provisioned<LineageRegion>(5, 0x3E);
|
||||
auto h = host<LineageRegion>(r, config(r));
|
||||
SchedulerTime t = std::chrono::steady_clock::now();
|
||||
CHECK(h->step(t).ready);
|
||||
for (std::uint64_t tick = 1; tick <= 8; ++tick)
|
||||
CHECK(h->submit(input(tick)));
|
||||
step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 5; });
|
||||
CHECK(h->status().committed_tick == 0);
|
||||
const auto epoch = h->status().epoch;
|
||||
std::optional<RowLock> lock{std::in_place, r.key};
|
||||
step_until(*h, t, 0ms, [](const HostStatus& x) { return x.phase != HostPhase::running; }, 1000);
|
||||
CHECK(!h->status().ready && h->status().fences == 1 && h->state() == nullptr);
|
||||
const auto dropped = h->take_dropped();
|
||||
CHECK(dropped.size() == 8 && std::ranges::all_of(dropped, [](const Dropped& d) {
|
||||
return d.reason == Unacknowledged::fenced;
|
||||
}));
|
||||
CHECK(h->take_acks().empty());
|
||||
lock->release();
|
||||
lock.reset();
|
||||
const auto& s = step_until(
|
||||
*h, t, 0ms, [](const HostStatus& x) { return x.phase == HostPhase::running; }, 1000);
|
||||
CHECK(s.epoch > epoch && s.committed_tick == 0 && s.simulated_tick == 0 && s.ready);
|
||||
}
|
||||
|
||||
// Drain: backlog dropped, pending ticks committed, lease released for an
|
||||
// immediate successor (environmental-v1 profile).
|
||||
void drain_commits_and_hands_over() {
|
||||
const auto r = provisioned<EnvironmentalCommunity>(6, 0x3F);
|
||||
auto h = host<EnvironmentalCommunity>(r, config(r));
|
||||
SchedulerTime t = std::chrono::steady_clock::now();
|
||||
CHECK(h->step(t).ready);
|
||||
for (std::uint64_t tick = 1; tick <= 9; ++tick)
|
||||
CHECK(h->submit(input(tick)));
|
||||
step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 6; });
|
||||
h->drain();
|
||||
CHECK(!h->status().ready && h->submit(input(10)).error() == AdmissionError::draining);
|
||||
auto dropped = h->take_dropped();
|
||||
CHECK(dropped.size() == 3 && dropped[0].reason == Unacknowledged::drained);
|
||||
const auto& s = h->step(t);
|
||||
CHECK(s.phase == HostPhase::stopped && s.committed_tick == 6);
|
||||
CHECK(h->take_acks().size() == 6);
|
||||
auto successor = host<EnvironmentalCommunity>(r, config(r));
|
||||
const auto& next = successor->step(t);
|
||||
CHECK(next.phase == HostPhase::running && next.committed_tick == 6 && next.epoch == 2);
|
||||
successor->drain();
|
||||
CHECK(successor->step(t).phase == HostPhase::stopped);
|
||||
}
|
||||
|
||||
// A committed head this host cannot restore quarantines; it is never replaced.
|
||||
void unrestorable_head_quarantines() {
|
||||
const auto r = provisioned<LineageRegion>(7, 0x40);
|
||||
auto c = config(r);
|
||||
c.build = filled<32>(0xB2);
|
||||
c.retained_builds = {c.build}; // Not the build that produced the head.
|
||||
auto h = host<LineageRegion>(r, c);
|
||||
const auto& s = h->step(std::chrono::steady_clock::now());
|
||||
CHECK(s.phase == HostPhase::quarantined && !s.ready && h->state() == nullptr);
|
||||
CHECK(h->submit(input(1)).error() == AdmissionError::not_ready);
|
||||
// Wrong frame and invalid configuration are refused too.
|
||||
auto wrong = config(r);
|
||||
wrong.frame = 10;
|
||||
CHECK(host<LineageRegion>(r, wrong)->step(std::chrono::steady_clock::now()).phase ==
|
||||
HostPhase::quarantined);
|
||||
auto invalid = config(r);
|
||||
invalid.retained_builds = {};
|
||||
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
|
||||
CHECK(!RegionHost<LineageRegion>::create(invalid, runtime_connect(), std::move(*objects)));
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void run_region_host_tests() {
|
||||
acknowledgements_follow_commits();
|
||||
starvation_accrues_bounded_debt();
|
||||
storage_outage_pauses_the_region();
|
||||
unknown_outcome_is_resolved_before_continuing();
|
||||
renewal_failure_fences_and_recovers();
|
||||
drain_commits_and_hands_over();
|
||||
unrestorable_head_quarantines();
|
||||
}
|
||||
@@ -0,0 +1,334 @@
|
||||
// Real process tests for evo-server (M7.6, ADR-0064): SIGTERM drain and SIGKILL
|
||||
// with recovery by a successor process. Run by tools/ci/postgres-check.sh, which
|
||||
// sets EVO_SERVER_BIN.
|
||||
#include "evo/persistence/checkpoint_store.hpp"
|
||||
#include "fixtures/environmental_region.hpp"
|
||||
#include <algorithm>
|
||||
#include <cerrno>
|
||||
#include <chrono>
|
||||
#include <csignal>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <filesystem>
|
||||
#include <poll.h>
|
||||
#include <source_location>
|
||||
#include <spawn.h>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <sys/wait.h>
|
||||
#include <thread>
|
||||
#include <unistd.h>
|
||||
#include <vector>
|
||||
|
||||
extern char** environ;
|
||||
void run_region_server_process_tests();
|
||||
|
||||
namespace {
|
||||
using namespace evo::persistence;
|
||||
using namespace std::chrono_literals;
|
||||
namespace fs = std::filesystem;
|
||||
using Clock = std::chrono::steady_clock;
|
||||
void check(bool value, std::source_location at = std::source_location::current()) {
|
||||
if (!value)
|
||||
throw std::runtime_error("region server process test line " + std::to_string(at.line()));
|
||||
}
|
||||
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
||||
std::string env(const char* key) {
|
||||
const auto* value = std::getenv(key);
|
||||
if (!value)
|
||||
throw std::runtime_error("missing test environment");
|
||||
return value;
|
||||
}
|
||||
std::unique_ptr<Postgres> owner() {
|
||||
auto made = Postgres::connect(
|
||||
{env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("PGUSER"), {}, {}}, 5s);
|
||||
CHECK(made);
|
||||
return std::move(*made);
|
||||
}
|
||||
std::string hex32(std::uint64_t tick) {
|
||||
char out[33];
|
||||
std::snprintf(out, sizeof out, "9b000000000000000000000000%06llx",
|
||||
static_cast<unsigned long long>(tick));
|
||||
return out;
|
||||
}
|
||||
std::string input_line(std::uint64_t tick, bool consumer_dies) {
|
||||
const auto in = evo::test::region_input(tick, consumer_dies);
|
||||
std::string line = "input " + hex32(tick) + " " + std::to_string(in.temperature_mK) + " " +
|
||||
std::to_string(in.source_enabled);
|
||||
for (const auto b : in.bite)
|
||||
line += " " + std::to_string(b);
|
||||
for (const auto t : in.terminal)
|
||||
line += " " + std::to_string(t);
|
||||
return line + "\n";
|
||||
}
|
||||
|
||||
class Child {
|
||||
public:
|
||||
explicit Child(const std::vector<std::string>& args) {
|
||||
int to_child[2], from_child[2];
|
||||
CHECK(::pipe(to_child) == 0 && ::pipe(from_child) == 0);
|
||||
posix_spawn_file_actions_t actions;
|
||||
posix_spawn_file_actions_init(&actions);
|
||||
posix_spawn_file_actions_adddup2(&actions, to_child[0], STDIN_FILENO);
|
||||
posix_spawn_file_actions_adddup2(&actions, from_child[1], STDOUT_FILENO);
|
||||
posix_spawn_file_actions_addclose(&actions, to_child[1]);
|
||||
posix_spawn_file_actions_addclose(&actions, from_child[0]);
|
||||
std::vector<char*> argv;
|
||||
argv.reserve(args.size() + 1);
|
||||
for (const auto& a : args)
|
||||
argv.push_back(const_cast<char*>(a.c_str()));
|
||||
argv.push_back(nullptr);
|
||||
CHECK(posix_spawn(&pid_, argv[0], &actions, nullptr, argv.data(), environ) == 0);
|
||||
posix_spawn_file_actions_destroy(&actions);
|
||||
::close(to_child[0]);
|
||||
::close(from_child[1]);
|
||||
in_ = to_child[1];
|
||||
out_ = from_child[0];
|
||||
}
|
||||
Child(const Child&) = delete;
|
||||
Child& operator=(const Child&) = delete;
|
||||
~Child() {
|
||||
if (pid_ > 0) {
|
||||
::kill(pid_, SIGKILL);
|
||||
::waitpid(pid_, nullptr, 0);
|
||||
}
|
||||
::close(in_);
|
||||
::close(out_);
|
||||
}
|
||||
void send(const std::string& text) {
|
||||
std::size_t done = 0;
|
||||
while (done < text.size()) {
|
||||
const auto n = ::write(in_, text.data() + done, text.size() - done);
|
||||
CHECK(n > 0 || errno == EINTR);
|
||||
if (n > 0)
|
||||
done += static_cast<std::size_t>(n);
|
||||
}
|
||||
}
|
||||
// Reads lines until one satisfies `want`; every line is kept in lines().
|
||||
template <typename Want> bool wait_for(Want want, std::chrono::milliseconds timeout) {
|
||||
const auto deadline = Clock::now() + timeout;
|
||||
for (std::size_t seen = 0;;) {
|
||||
for (; seen < lines_.size(); ++seen)
|
||||
if (want(lines_[seen]))
|
||||
return true;
|
||||
const auto left =
|
||||
std::chrono::duration_cast<std::chrono::milliseconds>(deadline - Clock::now());
|
||||
if (left.count() <= 0 || !read_some(static_cast<int>(left.count())))
|
||||
return false;
|
||||
}
|
||||
}
|
||||
void signal(int sig) {
|
||||
CHECK(::kill(pid_, sig) == 0);
|
||||
}
|
||||
// Exit code, or 128+signal; drains remaining output.
|
||||
int wait_exit(std::chrono::milliseconds timeout) {
|
||||
const auto deadline = Clock::now() + timeout;
|
||||
int status = 0;
|
||||
for (;;) {
|
||||
read_some(20);
|
||||
const auto r = ::waitpid(pid_, &status, WNOHANG);
|
||||
if (r == pid_)
|
||||
break;
|
||||
CHECK(Clock::now() < deadline);
|
||||
}
|
||||
while (read_some(20)) {
|
||||
}
|
||||
pid_ = -1;
|
||||
return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status);
|
||||
}
|
||||
[[nodiscard]] const std::vector<std::string>& lines() const noexcept {
|
||||
return lines_;
|
||||
}
|
||||
|
||||
private:
|
||||
bool read_some(int timeout_ms) {
|
||||
pollfd fd{out_, POLLIN, 0};
|
||||
if (::poll(&fd, 1, timeout_ms) <= 0)
|
||||
return false;
|
||||
char chunk[4096];
|
||||
const auto n = ::read(out_, chunk, sizeof chunk);
|
||||
if (n <= 0)
|
||||
return false;
|
||||
buffer_.append(chunk, static_cast<std::size_t>(n));
|
||||
for (auto end = buffer_.find('\n'); end != std::string::npos; end = buffer_.find('\n')) {
|
||||
lines_.push_back(buffer_.substr(0, end));
|
||||
buffer_.erase(0, end + 1);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
pid_t pid_{-1};
|
||||
int in_{-1}, out_{-1};
|
||||
std::string buffer_;
|
||||
std::vector<std::string> lines_;
|
||||
};
|
||||
|
||||
struct Region {
|
||||
RegionKey key;
|
||||
fs::path root;
|
||||
};
|
||||
Region provision(std::uint64_t region, std::uint8_t genesis) {
|
||||
Region r{{7, region}, fs::path(env("PGHOST")) / ("server-" + std::to_string(region))};
|
||||
fs::remove_all(r.root);
|
||||
fs::create_directory(r.root);
|
||||
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
|
||||
auto db = owner();
|
||||
CHECK(db->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
||||
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
|
||||
{}, 5s));
|
||||
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
|
||||
CHECK(objects);
|
||||
Sha256Digest build{};
|
||||
build.fill(0xB1);
|
||||
Uuid commit{};
|
||||
commit.fill(genesis);
|
||||
CHECK(provision_lineage_region(*db, **objects, r.key, evo::test::region_identity.frame,
|
||||
*evo::test::lineage_region(), commit, build, 8000000, 5s));
|
||||
return r;
|
||||
}
|
||||
std::vector<std::string> server_args(const Region& r) {
|
||||
return {env("EVO_SERVER_BIN"),
|
||||
"--profile",
|
||||
"lineage",
|
||||
"--world",
|
||||
"7",
|
||||
"--region",
|
||||
std::to_string(r.key.region),
|
||||
"--frame",
|
||||
std::to_string(evo::test::region_identity.frame),
|
||||
"--pg-host",
|
||||
env("PGHOST"),
|
||||
"--pg-port",
|
||||
env("PGPORT"),
|
||||
"--pg-database",
|
||||
env("PGDATABASE"),
|
||||
"--pg-user",
|
||||
env("EVO_PG_RUNTIME_USER"),
|
||||
"--objects",
|
||||
r.root.string(),
|
||||
"--build",
|
||||
"b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1",
|
||||
"--flush-ms",
|
||||
"500",
|
||||
"--lease-profile",
|
||||
"fast",
|
||||
"--ready-file",
|
||||
(r.root / "ready").string()};
|
||||
}
|
||||
// (tick, commit_sequence) of every ack line.
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> acks(const Child& c) {
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> out;
|
||||
for (const auto& line : c.lines()) {
|
||||
unsigned long long tick = 0, sequence = 0;
|
||||
char request[40];
|
||||
if (std::sscanf(line.c_str(), "ack %39s %llu %llu", request, &tick, &sequence) == 3)
|
||||
out.emplace_back(tick, sequence);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
std::string head_of(RegionKey key) {
|
||||
const std::array<std::string, 1> p{std::to_string(key.region)};
|
||||
auto rows = owner()->query("SELECT h.end_tick::text || ':' || h.commit_sequence::text || ':' "
|
||||
"|| coalesce(l.holder::text, 'released') FROM "
|
||||
"evolution.region_heads h JOIN evolution.region_leases l USING "
|
||||
"(world_id, region_id) WHERE world_id = 7 AND region_id = "
|
||||
"$1::evolution.u64",
|
||||
p, 5s);
|
||||
CHECK(rows && rows->size() == 1);
|
||||
const auto& value = (*rows)[0][0];
|
||||
CHECK(value.has_value());
|
||||
return value.value_or("");
|
||||
}
|
||||
std::uint64_t head_tick(RegionKey key) {
|
||||
return std::stoull(head_of(key).substr(0, head_of(key).find(':')));
|
||||
}
|
||||
bool starts(const std::string& line, std::string_view prefix) {
|
||||
return line.starts_with(prefix);
|
||||
}
|
||||
|
||||
// SIGTERM: stop admission, commit pending ticks, release the lease, exit 0. Every
|
||||
// acknowledged tick is committed, the backlog never exceeds its bound, and the
|
||||
// permanent outcome was announced only after its commit.
|
||||
void sigterm_drains_and_releases() {
|
||||
const auto r = provision(31, 0x42);
|
||||
Child server{server_args(r)};
|
||||
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
||||
CHECK(fs::exists(r.root / "ready"));
|
||||
std::string burst;
|
||||
for (std::uint64_t tick = 1; tick <= 80; ++tick)
|
||||
burst += input_line(tick, true);
|
||||
server.send(burst); // More than the 64-input backlog: stdin backpressures.
|
||||
CHECK(server.wait_for(
|
||||
[](const std::string& l) { return starts(l, "ack " + hex32(80) + " 80 "); }, 30s));
|
||||
server.signal(SIGTERM);
|
||||
CHECK(server.wait_exit(20s) == 0);
|
||||
const auto& lines = server.lines();
|
||||
CHECK(std::ranges::find(lines, "exit 0") != lines.end());
|
||||
CHECK(std::ranges::find_if(lines, [](const std::string& l) {
|
||||
return starts(l, "phase stopped");
|
||||
}) != lines.end());
|
||||
unsigned long long max_queued = 999;
|
||||
for (const auto& l : lines)
|
||||
std::sscanf(l.c_str(), "stats max_queued=%llu", &max_queued);
|
||||
CHECK(max_queued <= 64);
|
||||
const auto all = acks(server);
|
||||
CHECK(all.size() == 80 && all.back().first == 80);
|
||||
// The death at tick 20 is announced only with its commit: after the final
|
||||
// acknowledgement of tick 20, which the host emits only after confirmation.
|
||||
const auto outcome = std::ranges::find_if(
|
||||
lines, [](const std::string& l) { return starts(l, "outcome 20 1 11 "); });
|
||||
const auto ack20 = std::ranges::find_if(
|
||||
lines, [](const std::string& l) { return starts(l, "ack " + hex32(20) + " 20 "); });
|
||||
CHECK(outcome != lines.end() && ack20 != lines.end() && ack20 < outcome);
|
||||
CHECK(head_of(r.key).starts_with("80:") && head_of(r.key).ends_with(":released"));
|
||||
CHECK(!fs::exists(r.root / "ready"));
|
||||
std::printf("server_sigterm commits=%s acks=%zu max_queued=%llu\n", head_of(r.key).c_str(),
|
||||
all.size(), max_queued);
|
||||
}
|
||||
|
||||
// SIGKILL: no drain. Nothing acknowledged is lost, nothing unacknowledged was
|
||||
// claimed, and a successor becomes ready only after the database lease expires.
|
||||
void sigkill_recovers_through_a_successor() {
|
||||
const auto r = provision(32, 0x43);
|
||||
std::uint64_t acked_max = 0;
|
||||
{
|
||||
Child server{server_args(r)};
|
||||
CHECK(server.wait_for([](const std::string& l) { return l == "ready 1"; }, 15s));
|
||||
std::string burst;
|
||||
for (std::uint64_t tick = 1; tick <= 60; ++tick)
|
||||
burst += input_line(tick, false);
|
||||
server.send(burst);
|
||||
CHECK(server.wait_for([](const std::string& l) { return starts(l, "ack "); }, 15s));
|
||||
std::this_thread::sleep_for(300ms); // Ticks run past the last commit.
|
||||
server.signal(SIGKILL);
|
||||
CHECK(server.wait_exit(10s) == 128 + SIGKILL);
|
||||
for (const auto& [tick, sequence] : acks(server)) {
|
||||
static_cast<void>(sequence);
|
||||
acked_max = std::max(acked_max, tick);
|
||||
}
|
||||
}
|
||||
const auto committed = head_tick(r.key);
|
||||
CHECK(acked_max > 0 && committed >= acked_max); // No acknowledged tick was lost.
|
||||
CHECK(!head_of(r.key).ends_with(":released")); // The killed holder never released.
|
||||
const auto started = Clock::now();
|
||||
Child successor{server_args(r)};
|
||||
CHECK(successor.wait_for([](const std::string& l) { return l == "ready 1"; }, 20s));
|
||||
const auto to_ready =
|
||||
std::chrono::duration_cast<std::chrono::milliseconds>(Clock::now() - started).count();
|
||||
const auto running = std::ranges::find_if(
|
||||
successor.lines(), [](const std::string& l) { return starts(l, "phase running"); });
|
||||
CHECK(running != successor.lines().end());
|
||||
CHECK(running->find("committed=" + std::to_string(committed) + " ") != std::string::npos);
|
||||
CHECK(running->find("epoch=2 ") != std::string::npos);
|
||||
successor.signal(SIGTERM);
|
||||
CHECK(successor.wait_exit(20s) == 0);
|
||||
CHECK(head_tick(r.key) == committed && head_of(r.key).ends_with(":released"));
|
||||
std::printf("server_sigkill acked_max=%llu committed=%llu successor_ready_ms=%lld\n",
|
||||
static_cast<unsigned long long>(acked_max),
|
||||
static_cast<unsigned long long>(committed), static_cast<long long>(to_ready));
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void run_region_server_process_tests() {
|
||||
sigterm_drains_and_releases();
|
||||
sigkill_recovers_through_a_successor();
|
||||
}
|
||||
Reference in New Issue
Block a user