// Real PostgreSQL M7.6 region host behavior (ADR-0064). Run by tools/ci/postgres-check.sh. #include "evo/server/region_host.hpp" #include "fixtures/environmental_region.hpp" #include #include #include #include #include #include #include #include void run_region_host_tests(); namespace { using namespace evo::persistence; using namespace evo::server; using namespace std::chrono_literals; using evo::processes::EnvironmentalCommunity; using evo::processes::LineageRegion; namespace fs = std::filesystem; void check(bool value, std::source_location at = std::source_location::current()) { if (!value) throw std::runtime_error("region host test line " + std::to_string(at.line())); } #define CHECK(...) check(static_cast(__VA_ARGS__)) std::string env(const char* key) { const auto* value = std::getenv(key); if (!value) throw std::runtime_error("missing test environment"); return value; } PgOptions options(const std::string& user) { return {env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}}; } std::unique_ptr connect_as(const std::string& user) { auto made = Postgres::connect(options(user), 5s); CHECK(made); return std::move(*made); } Connect runtime_connect() { return [] { return Postgres::connect(options(env("EVO_PG_RUNTIME_USER")), 2s); }; } template std::array filled(std::uint8_t value) { std::array out{}; out.fill(value); return out; } const Sha256Digest build = filled<32>(0xB1); // Fast lease for tests: 4 s, renew every 0.8 s, 0.4 s margin/renew deadline, 1 s commits. const LeasePolicy fast{4000ms, 800ms, 400ms, 400ms, 1000ms}; TickInput input(std::uint64_t tick, bool consumer_dies = false) { Uuid request{}; request[0] = 0x9A; for (unsigned i = 0; i < 8; ++i) request[8 + i] = static_cast((tick >> (8U * i)) & 0xFFU); return {request, evo::test::region_input(tick, consumer_dies)}; } struct Region { RegionKey key{7, 21}; fs::path root; }; // Provisions a fresh region (owner) and returns its key and object root. template Region provisioned(std::uint64_t region, std::uint8_t genesis) { Region r{{7, 20 + region}, fs::path(env("PGHOST")) / ("host-" + std::to_string(region))}; fs::remove_all(r.root); fs::create_directory(r.root); fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace); auto owner = connect_as(env("PGUSER")); CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " "(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING", {}, 5s)); auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes); CHECK(objects); if constexpr (std::is_same_v) CHECK(provision_lineage_region(*owner, **objects, r.key, evo::test::region_identity.frame, *evo::test::lineage_region(), filled<16>(genesis), build, 8000000, 5s)); else CHECK(provision_environmental_region( *owner, **objects, r.key, evo::test::region_identity.frame, *evo::test::region_community(), filled<16>(genesis), build, 8000000, 5s)); return r; } HostConfig config(const Region& r) { HostConfig c; c.key = r.key; c.frame = evo::test::region_identity.frame; c.lease = fast; c.build = build; c.retained_builds = {build}; return c; } template std::unique_ptr> host(const Region& r, HostConfig c, ObjectStoreFaults* faults = nullptr) { auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes, faults); CHECK(objects); auto made = RegionHost::create(std::move(c), runtime_connect(), std::move(*objects)); CHECK(made); return std::move(*made); } // Steps until `done` holds, advancing scheduling time by `advance` each step. template const HostStatus& step_until(Host& h, SchedulerTime& t, std::chrono::microseconds advance, Done done, int limit = 400) { for (int i = 0; i < limit; ++i) { t += advance; const auto& s = h.step(t); if (done(s)) return s; std::this_thread::sleep_for(10ms); } CHECK(false); return h.status(); } // Admission, provisional ticks, pause at a permanent outcome and final acknowledgement. void acknowledgements_follow_commits() { const auto r = provisioned(1, 0x3A); auto h = host(r, config(r)); CHECK(h->submit(input(1)).error() == AdmissionError::not_ready); SchedulerTime t = std::chrono::steady_clock::now(); auto s = h->step(t); CHECK(s.phase == HostPhase::running && s.ready && s.committed_tick == 0 && s.epoch == 1); for (std::uint64_t tick = 1; tick <= 64; ++tick) CHECK(h->submit(input(tick, true))); CHECK(h->submit(input(65)).error() == AdmissionError::full); // Backlog of 64. CHECK(h->submit(input(1)).error() == AdmissionError::duplicate); CHECK(h->submit({{}, {}}).error() == AdmissionError::invalid); CHECK(h->status().queued_bytes == 64 * tick_input_bytes); for (std::uint64_t tick = 1; tick <= 19; ++tick) { t += 50ms; s = h->step(t); // Simulated ahead of committed; nothing is final yet. CHECK(s.simulated_tick == tick && s.committed_tick == 0 && s.ready); CHECK(h->take_acks().empty() && h->take_outcomes().empty()); } // Tick 20 kills the consumer: the region pauses and commits in the same step, // so no dependent tick ever runs on the uncommitted death. t += 50ms; s = h->step(t); CHECK(s.simulated_tick == 20 && s.committed_tick == 20 && s.committed_sequence == 1); const auto outcomes = h->take_outcomes(); CHECK(outcomes.size() == 1 && outcomes[0].commit_sequence == 1 && outcomes[0].decision == BatchDecision{20, Actor::consumer, Outcome::death, 11}); auto acks = h->take_acks(); CHECK(acks.size() == 20); for (std::uint64_t i = 0; i < 20; ++i) CHECK(acks[i] == FinalAck{input(i + 1).request, i + 1, 1}); // The next batch flushes after the configured interval, not per tick. bool provisional = false; s = step_until(*h, t, 50ms, [&](const HostStatus& x) { provisional = provisional || (x.simulated_tick > x.committed_tick); return x.committed_sequence == 2; }); CHECK(provisional && s.simulated_tick == s.committed_tick && s.committed_tick > 30); acks = h->take_acks(); CHECK(!acks.empty() && acks.back().tick == s.committed_tick && acks.front().tick == 21); CHECK(std::ranges::all_of(acks, [](const FinalAck& a) { return a.commit_sequence == 2; })); CHECK(h->take_dropped().empty()); std::printf("host_commits=%llu committed_tick=%llu\n", static_cast(s.commits), static_cast(s.committed_tick)); } // No inputs, no invented ticks: debt accrues and is repaid with bounded catch-up. void starvation_accrues_bounded_debt() { const auto r = provisioned(2, 0x3B); auto h = host(r, config(r)); SchedulerTime t = std::chrono::steady_clock::now(); CHECK(h->step(t).ready); for (int i = 0; i < 10; ++i) { t += 50ms; CHECK(h->step(t).simulated_tick == 0); } CHECK(h->status().tick_debt == 10); for (std::uint64_t tick = 1; tick <= 20; ++tick) CHECK(h->submit(input(tick))); auto s = h->step(t); CHECK(s.simulated_tick == 8 && s.tick_debt == 2); // At most 8 per step. s = h->step(t); CHECK(s.simulated_tick == 10 && s.tick_debt == 0 && s.max_tick_debt == 10); CHECK(s.queued == 10); } // Object store outage: the sealed batch waits, nothing is acknowledged, and the // region resumes once storage is durable again. void storage_outage_pauses_the_region() { const auto r = provisioned(3, 0x41); ObjectStoreFaults faults; auto h = host(r, config(r), &faults); SchedulerTime t = std::chrono::steady_clock::now(); CHECK(h->step(t).ready); for (std::uint64_t tick = 1; tick <= 30; ++tick) CHECK(h->submit(input(tick, true))); step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 19; }); faults.fail_directory_syncs = 1000; for (int i = 0; i < 5; ++i) { t += 50ms; const auto& s = h->step(t); CHECK(s.simulated_tick == 20 && s.committed_tick == 0); // Paused at the death. } CHECK(h->status().store_failures >= 5 && h->take_acks().empty() && h->take_outcomes().empty()); CHECK(h->status().tick_debt == 4); faults.fail_directory_syncs = 0; t += 50ms; auto s = h->step(t); CHECK(s.committed_tick == 20 && h->take_acks().size() == 20 && h->take_outcomes().size() == 1); s = h->step(t); // Catch-up after the outage is bounded but continues. CHECK(s.simulated_tick > 20 && s.simulated_tick <= 28); } // Holds the lease row lock to make database calls time out. struct RowLock { std::unique_ptr db = connect_as(env("PGUSER")); explicit RowLock(RegionKey key) { const std::array p{std::to_string(key.world), std::to_string(key.region)}; CHECK(db->query("BEGIN", {}, 5s)); CHECK(db->query("SELECT 1 FROM evolution.region_leases WHERE world_id = $1::evolution.u64 " "AND region_id = $2::evolution.u64 FOR UPDATE", p, 5s)); } void release() { CHECK(db->query("ROLLBACK", {}, 5s)); } }; // Unknown commit outcome: not ready, admission refused, resolved by exact bytes. void unknown_outcome_is_resolved_before_continuing() { const auto r = provisioned(4, 0x3D); auto h = host(r, config(r)); SchedulerTime t = std::chrono::steady_clock::now(); CHECK(h->step(t).ready); for (std::uint64_t tick = 1; tick <= 25; ++tick) CHECK(h->submit(input(tick, true))); step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 19; }); RowLock lock{r.key}; t += 50ms; auto s = h->step(t); // The death seals; the commit cannot finish in time. CHECK(s.phase == HostPhase::resolving && !s.ready && s.unknown_outcomes == 1); CHECK(h->submit(input(26)).error() == AdmissionError::not_ready); CHECK(h->take_outcomes().empty() && h->take_acks().empty()); lock.release(); s = step_until(*h, t, 0ms, [](const HostStatus& x) { return x.phase == HostPhase::running; }); CHECK(s.committed_tick == 20 && s.ready); CHECK(h->take_outcomes().size() == 1 && h->take_acks().size() == 20); } // Renewals that cannot reach the database fence the host locally; unconfirmed // work is reported as never final, and authority is re-acquired from the head. void renewal_failure_fences_and_recovers() { const auto r = provisioned(5, 0x3E); auto h = host(r, config(r)); SchedulerTime t = std::chrono::steady_clock::now(); CHECK(h->step(t).ready); for (std::uint64_t tick = 1; tick <= 8; ++tick) CHECK(h->submit(input(tick))); step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 5; }); CHECK(h->status().committed_tick == 0); const auto epoch = h->status().epoch; std::optional lock{std::in_place, r.key}; step_until(*h, t, 0ms, [](const HostStatus& x) { return x.phase != HostPhase::running; }, 1000); CHECK(!h->status().ready && h->status().fences == 1 && h->state() == nullptr); const auto dropped = h->take_dropped(); CHECK(dropped.size() == 8 && std::ranges::all_of(dropped, [](const Dropped& d) { return d.reason == Unacknowledged::fenced; })); CHECK(h->take_acks().empty()); lock->release(); lock.reset(); const auto& s = step_until( *h, t, 0ms, [](const HostStatus& x) { return x.phase == HostPhase::running; }, 1000); CHECK(s.epoch > epoch && s.committed_tick == 0 && s.simulated_tick == 0 && s.ready); } // Drain: backlog dropped, pending ticks committed, lease released for an // immediate successor (environmental-v1 profile). void drain_commits_and_hands_over() { const auto r = provisioned(6, 0x3F); auto h = host(r, config(r)); SchedulerTime t = std::chrono::steady_clock::now(); CHECK(h->step(t).ready); for (std::uint64_t tick = 1; tick <= 9; ++tick) CHECK(h->submit(input(tick))); step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 6; }); h->drain(); CHECK(!h->status().ready && h->submit(input(10)).error() == AdmissionError::draining); auto dropped = h->take_dropped(); CHECK(dropped.size() == 3 && dropped[0].reason == Unacknowledged::drained); const auto& s = h->step(t); CHECK(s.phase == HostPhase::stopped && s.committed_tick == 6); CHECK(h->take_acks().size() == 6); auto successor = host(r, config(r)); const auto& next = successor->step(t); CHECK(next.phase == HostPhase::running && next.committed_tick == 6 && next.epoch == 2); successor->drain(); CHECK(successor->step(t).phase == HostPhase::stopped); } // A committed head this host cannot restore quarantines; it is never replaced. void unrestorable_head_quarantines() { const auto r = provisioned(7, 0x40); auto c = config(r); c.build = filled<32>(0xB2); c.retained_builds = {c.build}; // Not the build that produced the head. // A current build in the database (so acquisition succeeds, ADR-0068): the // quarantine comes from recovery refusing the head, not from the lease. CHECK(connect_as(env("PGUSER")) ->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " "(decode(repeat('b2', 32), 'hex')) ON CONFLICT DO NOTHING", {}, 5s)); auto h = host(r, c); const auto& s = h->step(std::chrono::steady_clock::now()); CHECK(s.phase == HostPhase::quarantined && !s.ready && h->state() == nullptr); CHECK(h->submit(input(1)).error() == AdmissionError::not_ready); // Wrong frame and invalid configuration are refused too. auto wrong = config(r); wrong.frame = 10; CHECK(host(r, wrong)->step(std::chrono::steady_clock::now()).phase == HostPhase::quarantined); auto invalid = config(r); invalid.retained_builds = {}; auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes); CHECK(!RegionHost::create(invalid, runtime_connect(), std::move(*objects))); } } // namespace void run_region_host_tests() { acknowledgements_follow_commits(); starvation_accrues_bounded_debt(); storage_outage_pauses_the_region(); unknown_outcome_is_resolved_before_continuing(); renewal_failure_fences_and_recovers(); drain_commits_and_hands_over(); unrestorable_head_quarantines(); }