Files
aevolution/tests/region_host_tests.cpp
T
dingoandClaude Opus 5.5 d6d0408aae Complete M7.6 bounded authoritative region host
ADR-0064: RegionHost<State> (environmental and lineage profiles) is a
single-owner, single-threaded state machine advanced by step(now):
acquire the fenced lease and recover the verified head, run, resolve
ambiguous commits, drain and stop, or quarantine. Admission is
provisional and bounded (64 inputs, 64 KiB) with backpressure; FinalAck
and FinalOutcome are emitted only after a confirmed commit, and inputs
that will never be acknowledged are reported with a reason. Ticks follow
the caller's scheduling clock with bounded catch-up and measured tick
debt, never invented inputs. A death seals the batch and commits in the
same step, so no dependent tick uses an uncommitted outcome. Storage
outages pause and retry, unknown outcomes resolve by exact bytes,
fencing discards unconfirmed work and re-acquires; readiness fails
closed. Drain commits pending ticks and releases the lease.

evo-server hosts one region with a bounded stdin input feed, stdout
event lines, a readiness file and SIGTERM drain (exit 0/3/4).

Validated: Windows CTest 58/58; PostgreSQL suites under GCC Release,
clang-tidy and ASan/UBSan including real-process SIGTERM drain and
SIGKILL with successor recovery; four host mutations each caught. No
gateway, metrics, Docker/Gitea CI or deployment.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-28 20:53:11 -04:00

333 lines
15 KiB
C++

// Real PostgreSQL M7.6 region host behavior (ADR-0064). Run by tools/ci/postgres-check.sh.
#include "evo/server/region_host.hpp"
#include "fixtures/environmental_region.hpp"
#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <filesystem>
#include <source_location>
#include <stdexcept>
#include <string>
#include <thread>
void run_region_host_tests();
namespace {
using namespace evo::persistence;
using namespace evo::server;
using namespace std::chrono_literals;
using evo::processes::EnvironmentalCommunity;
using evo::processes::LineageRegion;
namespace fs = std::filesystem;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region host test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
std::string env(const char* key) {
const auto* value = std::getenv(key);
if (!value)
throw std::runtime_error("missing test environment");
return value;
}
PgOptions options(const std::string& user) {
return {env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}};
}
std::unique_ptr<Postgres> connect_as(const std::string& user) {
auto made = Postgres::connect(options(user), 5s);
CHECK(made);
return std::move(*made);
}
Connect runtime_connect() {
return [] { return Postgres::connect(options(env("EVO_PG_RUNTIME_USER")), 2s); };
}
template <std::size_t N> std::array<std::uint8_t, N> filled(std::uint8_t value) {
std::array<std::uint8_t, N> out{};
out.fill(value);
return out;
}
const Sha256Digest build = filled<32>(0xB1);
// Fast lease for tests: 4 s, renew every 0.8 s, 0.4 s margin/renew deadline, 1 s commits.
const LeasePolicy fast{4000ms, 800ms, 400ms, 400ms, 1000ms};
TickInput input(std::uint64_t tick, bool consumer_dies = false) {
Uuid request{};
request[0] = 0x9A;
for (unsigned i = 0; i < 8; ++i)
request[8 + i] = static_cast<std::uint8_t>((tick >> (8U * i)) & 0xFFU);
return {request, evo::test::region_input(tick, consumer_dies)};
}
struct Region {
RegionKey key{7, 21};
fs::path root;
};
// Provisions a fresh region (owner) and returns its key and object root.
template <typename State> Region provisioned(std::uint64_t region, std::uint8_t genesis) {
Region r{{7, 20 + region}, fs::path(env("PGHOST")) / ("host-" + std::to_string(region))};
fs::remove_all(r.root);
fs::create_directory(r.root);
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
auto owner = connect_as(env("PGUSER"));
CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
{}, 5s));
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
CHECK(objects);
if constexpr (std::is_same_v<State, LineageRegion>)
CHECK(provision_lineage_region(*owner, **objects, r.key, evo::test::region_identity.frame,
*evo::test::lineage_region(), filled<16>(genesis), build,
8000000, 5s));
else
CHECK(provision_environmental_region(
*owner, **objects, r.key, evo::test::region_identity.frame,
*evo::test::region_community(), filled<16>(genesis), build, 8000000, 5s));
return r;
}
HostConfig config(const Region& r) {
HostConfig c;
c.key = r.key;
c.frame = evo::test::region_identity.frame;
c.lease = fast;
c.build = build;
c.retained_builds = {build};
return c;
}
template <typename State>
std::unique_ptr<RegionHost<State>> host(const Region& r, HostConfig c,
ObjectStoreFaults* faults = nullptr) {
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes, faults);
CHECK(objects);
auto made = RegionHost<State>::create(std::move(c), runtime_connect(), std::move(*objects));
CHECK(made);
return std::move(*made);
}
// Steps until `done` holds, advancing scheduling time by `advance` each step.
template <typename Host, typename Done>
const HostStatus& step_until(Host& h, SchedulerTime& t, std::chrono::microseconds advance,
Done done, int limit = 400) {
for (int i = 0; i < limit; ++i) {
t += advance;
const auto& s = h.step(t);
if (done(s))
return s;
std::this_thread::sleep_for(10ms);
}
CHECK(false);
return h.status();
}
// Admission, provisional ticks, pause at a permanent outcome and final acknowledgement.
void acknowledgements_follow_commits() {
const auto r = provisioned<LineageRegion>(1, 0x3A);
auto h = host<LineageRegion>(r, config(r));
CHECK(h->submit(input(1)).error() == AdmissionError::not_ready);
SchedulerTime t = std::chrono::steady_clock::now();
auto s = h->step(t);
CHECK(s.phase == HostPhase::running && s.ready && s.committed_tick == 0 && s.epoch == 1);
for (std::uint64_t tick = 1; tick <= 64; ++tick)
CHECK(h->submit(input(tick, true)));
CHECK(h->submit(input(65)).error() == AdmissionError::full); // Backlog of 64.
CHECK(h->submit(input(1)).error() == AdmissionError::duplicate);
CHECK(h->submit({{}, {}}).error() == AdmissionError::invalid);
CHECK(h->status().queued_bytes == 64 * tick_input_bytes);
for (std::uint64_t tick = 1; tick <= 19; ++tick) {
t += 50ms;
s = h->step(t);
// Simulated ahead of committed; nothing is final yet.
CHECK(s.simulated_tick == tick && s.committed_tick == 0 && s.ready);
CHECK(h->take_acks().empty() && h->take_outcomes().empty());
}
// Tick 20 kills the consumer: the region pauses and commits in the same step,
// so no dependent tick ever runs on the uncommitted death.
t += 50ms;
s = h->step(t);
CHECK(s.simulated_tick == 20 && s.committed_tick == 20 && s.committed_sequence == 1);
const auto outcomes = h->take_outcomes();
CHECK(outcomes.size() == 1 && outcomes[0].commit_sequence == 1 &&
outcomes[0].decision == BatchDecision{20, Actor::consumer, Outcome::death, 11});
auto acks = h->take_acks();
CHECK(acks.size() == 20);
for (std::uint64_t i = 0; i < 20; ++i)
CHECK(acks[i] == FinalAck{input(i + 1).request, i + 1, 1});
// The next batch flushes after the configured interval, not per tick.
bool provisional = false;
s = step_until(*h, t, 50ms, [&](const HostStatus& x) {
provisional = provisional || (x.simulated_tick > x.committed_tick);
return x.committed_sequence == 2;
});
CHECK(provisional && s.simulated_tick == s.committed_tick && s.committed_tick > 30);
acks = h->take_acks();
CHECK(!acks.empty() && acks.back().tick == s.committed_tick && acks.front().tick == 21);
CHECK(std::ranges::all_of(acks, [](const FinalAck& a) { return a.commit_sequence == 2; }));
CHECK(h->take_dropped().empty());
std::printf("host_commits=%llu committed_tick=%llu\n",
static_cast<unsigned long long>(s.commits),
static_cast<unsigned long long>(s.committed_tick));
}
// No inputs, no invented ticks: debt accrues and is repaid with bounded catch-up.
void starvation_accrues_bounded_debt() {
const auto r = provisioned<LineageRegion>(2, 0x3B);
auto h = host<LineageRegion>(r, config(r));
SchedulerTime t = std::chrono::steady_clock::now();
CHECK(h->step(t).ready);
for (int i = 0; i < 10; ++i) {
t += 50ms;
CHECK(h->step(t).simulated_tick == 0);
}
CHECK(h->status().tick_debt == 10);
for (std::uint64_t tick = 1; tick <= 20; ++tick)
CHECK(h->submit(input(tick)));
auto s = h->step(t);
CHECK(s.simulated_tick == 8 && s.tick_debt == 2); // At most 8 per step.
s = h->step(t);
CHECK(s.simulated_tick == 10 && s.tick_debt == 0 && s.max_tick_debt == 10);
CHECK(s.queued == 10);
}
// Object store outage: the sealed batch waits, nothing is acknowledged, and the
// region resumes once storage is durable again.
void storage_outage_pauses_the_region() {
const auto r = provisioned<LineageRegion>(3, 0x41);
ObjectStoreFaults faults;
auto h = host<LineageRegion>(r, config(r), &faults);
SchedulerTime t = std::chrono::steady_clock::now();
CHECK(h->step(t).ready);
for (std::uint64_t tick = 1; tick <= 30; ++tick)
CHECK(h->submit(input(tick, true)));
step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 19; });
faults.fail_directory_syncs = 1000;
for (int i = 0; i < 5; ++i) {
t += 50ms;
const auto& s = h->step(t);
CHECK(s.simulated_tick == 20 && s.committed_tick == 0); // Paused at the death.
}
CHECK(h->status().store_failures >= 5 && h->take_acks().empty() && h->take_outcomes().empty());
CHECK(h->status().tick_debt == 4);
faults.fail_directory_syncs = 0;
t += 50ms;
auto s = h->step(t);
CHECK(s.committed_tick == 20 && h->take_acks().size() == 20 && h->take_outcomes().size() == 1);
s = h->step(t); // Catch-up after the outage is bounded but continues.
CHECK(s.simulated_tick > 20 && s.simulated_tick <= 28);
}
// Holds the lease row lock to make database calls time out.
struct RowLock {
std::unique_ptr<Postgres> db = connect_as(env("PGUSER"));
explicit RowLock(RegionKey key) {
const std::array<std::string, 2> p{std::to_string(key.world), std::to_string(key.region)};
CHECK(db->query("BEGIN", {}, 5s));
CHECK(db->query("SELECT 1 FROM evolution.region_leases WHERE world_id = $1::evolution.u64 "
"AND region_id = $2::evolution.u64 FOR UPDATE",
p, 5s));
}
void release() {
CHECK(db->query("ROLLBACK", {}, 5s));
}
};
// Unknown commit outcome: not ready, admission refused, resolved by exact bytes.
void unknown_outcome_is_resolved_before_continuing() {
const auto r = provisioned<LineageRegion>(4, 0x3D);
auto h = host<LineageRegion>(r, config(r));
SchedulerTime t = std::chrono::steady_clock::now();
CHECK(h->step(t).ready);
for (std::uint64_t tick = 1; tick <= 25; ++tick)
CHECK(h->submit(input(tick, true)));
step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 19; });
RowLock lock{r.key};
t += 50ms;
auto s = h->step(t); // The death seals; the commit cannot finish in time.
CHECK(s.phase == HostPhase::resolving && !s.ready && s.unknown_outcomes == 1);
CHECK(h->submit(input(26)).error() == AdmissionError::not_ready);
CHECK(h->take_outcomes().empty() && h->take_acks().empty());
lock.release();
s = step_until(*h, t, 0ms, [](const HostStatus& x) { return x.phase == HostPhase::running; });
CHECK(s.committed_tick == 20 && s.ready);
CHECK(h->take_outcomes().size() == 1 && h->take_acks().size() == 20);
}
// Renewals that cannot reach the database fence the host locally; unconfirmed
// work is reported as never final, and authority is re-acquired from the head.
void renewal_failure_fences_and_recovers() {
const auto r = provisioned<LineageRegion>(5, 0x3E);
auto h = host<LineageRegion>(r, config(r));
SchedulerTime t = std::chrono::steady_clock::now();
CHECK(h->step(t).ready);
for (std::uint64_t tick = 1; tick <= 8; ++tick)
CHECK(h->submit(input(tick)));
step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 5; });
CHECK(h->status().committed_tick == 0);
const auto epoch = h->status().epoch;
std::optional<RowLock> lock{std::in_place, r.key};
step_until(*h, t, 0ms, [](const HostStatus& x) { return x.phase != HostPhase::running; }, 1000);
CHECK(!h->status().ready && h->status().fences == 1 && h->state() == nullptr);
const auto dropped = h->take_dropped();
CHECK(dropped.size() == 8 && std::ranges::all_of(dropped, [](const Dropped& d) {
return d.reason == Unacknowledged::fenced;
}));
CHECK(h->take_acks().empty());
lock->release();
lock.reset();
const auto& s = step_until(
*h, t, 0ms, [](const HostStatus& x) { return x.phase == HostPhase::running; }, 1000);
CHECK(s.epoch > epoch && s.committed_tick == 0 && s.simulated_tick == 0 && s.ready);
}
// Drain: backlog dropped, pending ticks committed, lease released for an
// immediate successor (environmental-v1 profile).
void drain_commits_and_hands_over() {
const auto r = provisioned<EnvironmentalCommunity>(6, 0x3F);
auto h = host<EnvironmentalCommunity>(r, config(r));
SchedulerTime t = std::chrono::steady_clock::now();
CHECK(h->step(t).ready);
for (std::uint64_t tick = 1; tick <= 9; ++tick)
CHECK(h->submit(input(tick)));
step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 6; });
h->drain();
CHECK(!h->status().ready && h->submit(input(10)).error() == AdmissionError::draining);
auto dropped = h->take_dropped();
CHECK(dropped.size() == 3 && dropped[0].reason == Unacknowledged::drained);
const auto& s = h->step(t);
CHECK(s.phase == HostPhase::stopped && s.committed_tick == 6);
CHECK(h->take_acks().size() == 6);
auto successor = host<EnvironmentalCommunity>(r, config(r));
const auto& next = successor->step(t);
CHECK(next.phase == HostPhase::running && next.committed_tick == 6 && next.epoch == 2);
successor->drain();
CHECK(successor->step(t).phase == HostPhase::stopped);
}
// A committed head this host cannot restore quarantines; it is never replaced.
void unrestorable_head_quarantines() {
const auto r = provisioned<LineageRegion>(7, 0x40);
auto c = config(r);
c.build = filled<32>(0xB2);
c.retained_builds = {c.build}; // Not the build that produced the head.
auto h = host<LineageRegion>(r, c);
const auto& s = h->step(std::chrono::steady_clock::now());
CHECK(s.phase == HostPhase::quarantined && !s.ready && h->state() == nullptr);
CHECK(h->submit(input(1)).error() == AdmissionError::not_ready);
// Wrong frame and invalid configuration are refused too.
auto wrong = config(r);
wrong.frame = 10;
CHECK(host<LineageRegion>(r, wrong)->step(std::chrono::steady_clock::now()).phase ==
HostPhase::quarantined);
auto invalid = config(r);
invalid.retained_builds = {};
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
CHECK(!RegionHost<LineageRegion>::create(invalid, runtime_connect(), std::move(*objects)));
}
} // namespace
void run_region_host_tests() {
acknowledgements_follow_commits();
starvation_accrues_bounded_debt();
storage_outage_pauses_the_region();
unknown_outcome_is_resolved_before_continuing();
renewal_failure_fences_and_recovers();
drain_commits_and_hands_over();
unrestorable_head_quarantines();
}