The first dev drill showed a full volume driving ~140 refused publications a second. Commit retries now back off 100 ms doubling to a 2 s cap, reset on the next confirmed commit. fault-drill.sh reports the empty-volume pod's exit code, restarts and logs, and prints node-specific iptables rules for db-outage. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
350 lines
16 KiB
C++
350 lines
16 KiB
C++
// Real PostgreSQL M7.6 region host behavior (ADR-0064). Run by tools/ci/postgres-check.sh.
|
|
#include "evo/server/region_host.hpp"
|
|
#include "fixtures/environmental_region.hpp"
|
|
#include <algorithm>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <filesystem>
|
|
#include <source_location>
|
|
#include <stdexcept>
|
|
#include <string>
|
|
#include <thread>
|
|
|
|
void run_region_host_tests();
|
|
|
|
namespace {
|
|
using namespace evo::persistence;
|
|
using namespace evo::server;
|
|
using namespace std::chrono_literals;
|
|
using evo::processes::EnvironmentalCommunity;
|
|
using evo::processes::LineageRegion;
|
|
namespace fs = std::filesystem;
|
|
void check(bool value, std::source_location at = std::source_location::current()) {
|
|
if (!value)
|
|
throw std::runtime_error("region host test line " + std::to_string(at.line()));
|
|
}
|
|
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
|
std::string env(const char* key) {
|
|
const auto* value = std::getenv(key);
|
|
if (!value)
|
|
throw std::runtime_error("missing test environment");
|
|
return value;
|
|
}
|
|
PgOptions options(const std::string& user) {
|
|
return {env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}};
|
|
}
|
|
std::unique_ptr<Postgres> connect_as(const std::string& user) {
|
|
auto made = Postgres::connect(options(user), 5s);
|
|
CHECK(made);
|
|
return std::move(*made);
|
|
}
|
|
Connect runtime_connect() {
|
|
return [] { return Postgres::connect(options(env("EVO_PG_RUNTIME_USER")), 2s); };
|
|
}
|
|
template <std::size_t N> std::array<std::uint8_t, N> filled(std::uint8_t value) {
|
|
std::array<std::uint8_t, N> out{};
|
|
out.fill(value);
|
|
return out;
|
|
}
|
|
const Sha256Digest build = filled<32>(0xB1);
|
|
// Fast lease for tests: 4 s, renew every 0.8 s, 0.4 s margin/renew deadline, 1 s commits.
|
|
const LeasePolicy fast{4000ms, 800ms, 400ms, 400ms, 1000ms};
|
|
TickInput input(std::uint64_t tick, bool consumer_dies = false) {
|
|
Uuid request{};
|
|
request[0] = 0x9A;
|
|
for (unsigned i = 0; i < 8; ++i)
|
|
request[8 + i] = static_cast<std::uint8_t>((tick >> (8U * i)) & 0xFFU);
|
|
return {request, evo::test::region_input(tick, consumer_dies)};
|
|
}
|
|
|
|
struct Region {
|
|
RegionKey key{7, 21};
|
|
fs::path root;
|
|
};
|
|
// Provisions a fresh region (owner) and returns its key and object root.
|
|
template <typename State> Region provisioned(std::uint64_t region, std::uint8_t genesis) {
|
|
Region r{{7, 20 + region}, fs::path(env("PGHOST")) / ("host-" + std::to_string(region))};
|
|
fs::remove_all(r.root);
|
|
fs::create_directory(r.root);
|
|
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
|
|
auto owner = connect_as(env("PGUSER"));
|
|
CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
|
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
|
|
{}, 5s));
|
|
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
|
|
CHECK(objects);
|
|
if constexpr (std::is_same_v<State, LineageRegion>)
|
|
CHECK(provision_lineage_region(*owner, **objects, r.key, evo::test::region_identity.frame,
|
|
*evo::test::lineage_region(), filled<16>(genesis), build,
|
|
8000000, 5s));
|
|
else
|
|
CHECK(provision_environmental_region(
|
|
*owner, **objects, r.key, evo::test::region_identity.frame,
|
|
*evo::test::region_community(), filled<16>(genesis), build, 8000000, 5s));
|
|
return r;
|
|
}
|
|
HostConfig config(const Region& r) {
|
|
HostConfig c;
|
|
c.key = r.key;
|
|
c.frame = evo::test::region_identity.frame;
|
|
c.lease = fast;
|
|
c.build = build;
|
|
c.retained_builds = {build};
|
|
return c;
|
|
}
|
|
template <typename State>
|
|
std::unique_ptr<RegionHost<State>> host(const Region& r, HostConfig c,
|
|
ObjectStoreFaults* faults = nullptr) {
|
|
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes, faults);
|
|
CHECK(objects);
|
|
auto made = RegionHost<State>::create(std::move(c), runtime_connect(), std::move(*objects));
|
|
CHECK(made);
|
|
return std::move(*made);
|
|
}
|
|
// Steps until `done` holds, advancing scheduling time by `advance` each step.
|
|
template <typename Host, typename Done>
|
|
const HostStatus& step_until(Host& h, SchedulerTime& t, std::chrono::microseconds advance,
|
|
Done done, int limit = 400) {
|
|
for (int i = 0; i < limit; ++i) {
|
|
t += advance;
|
|
const auto& s = h.step(t);
|
|
if (done(s))
|
|
return s;
|
|
std::this_thread::sleep_for(10ms);
|
|
}
|
|
CHECK(false);
|
|
return h.status();
|
|
}
|
|
|
|
// Admission, provisional ticks, pause at a permanent outcome and final acknowledgement.
|
|
void acknowledgements_follow_commits() {
|
|
const auto r = provisioned<LineageRegion>(1, 0x3A);
|
|
auto h = host<LineageRegion>(r, config(r));
|
|
CHECK(h->submit(input(1)).error() == AdmissionError::not_ready);
|
|
SchedulerTime t = std::chrono::steady_clock::now();
|
|
auto s = h->step(t);
|
|
CHECK(s.phase == HostPhase::running && s.ready && s.committed_tick == 0 && s.epoch == 1);
|
|
for (std::uint64_t tick = 1; tick <= 64; ++tick)
|
|
CHECK(h->submit(input(tick, true)));
|
|
CHECK(h->submit(input(65)).error() == AdmissionError::full); // Backlog of 64.
|
|
CHECK(h->submit(input(1)).error() == AdmissionError::duplicate);
|
|
CHECK(h->submit({{}, {}}).error() == AdmissionError::invalid);
|
|
CHECK(h->status().queued_bytes == 64 * tick_input_bytes);
|
|
for (std::uint64_t tick = 1; tick <= 19; ++tick) {
|
|
t += 50ms;
|
|
s = h->step(t);
|
|
// Simulated ahead of committed; nothing is final yet.
|
|
CHECK(s.simulated_tick == tick && s.committed_tick == 0 && s.ready);
|
|
CHECK(h->take_acks().empty() && h->take_outcomes().empty());
|
|
}
|
|
// Tick 20 kills the consumer: the region pauses and commits in the same step,
|
|
// so no dependent tick ever runs on the uncommitted death.
|
|
t += 50ms;
|
|
s = h->step(t);
|
|
CHECK(s.simulated_tick == 20 && s.committed_tick == 20 && s.committed_sequence == 1);
|
|
const auto outcomes = h->take_outcomes();
|
|
CHECK(outcomes.size() == 1 && outcomes[0].commit_sequence == 1 &&
|
|
outcomes[0].decision == BatchDecision{20, Actor::consumer, Outcome::death, 11});
|
|
auto acks = h->take_acks();
|
|
CHECK(acks.size() == 20);
|
|
for (std::uint64_t i = 0; i < 20; ++i)
|
|
CHECK(acks[i] == FinalAck{input(i + 1).request, i + 1, 1});
|
|
// The next batch flushes after the configured interval, not per tick.
|
|
bool provisional = false;
|
|
s = step_until(*h, t, 50ms, [&](const HostStatus& x) {
|
|
provisional = provisional || (x.simulated_tick > x.committed_tick);
|
|
return x.committed_sequence == 2;
|
|
});
|
|
CHECK(provisional && s.simulated_tick == s.committed_tick && s.committed_tick > 30);
|
|
acks = h->take_acks();
|
|
CHECK(!acks.empty() && acks.back().tick == s.committed_tick && acks.front().tick == 21);
|
|
CHECK(std::ranges::all_of(acks, [](const FinalAck& a) { return a.commit_sequence == 2; }));
|
|
CHECK(h->take_dropped().empty());
|
|
std::printf("host_commits=%llu committed_tick=%llu\n",
|
|
static_cast<unsigned long long>(s.commits),
|
|
static_cast<unsigned long long>(s.committed_tick));
|
|
}
|
|
|
|
// No inputs, no invented ticks: debt accrues and is repaid with bounded catch-up.
|
|
void starvation_accrues_bounded_debt() {
|
|
const auto r = provisioned<LineageRegion>(2, 0x3B);
|
|
auto h = host<LineageRegion>(r, config(r));
|
|
SchedulerTime t = std::chrono::steady_clock::now();
|
|
CHECK(h->step(t).ready);
|
|
for (int i = 0; i < 10; ++i) {
|
|
t += 50ms;
|
|
CHECK(h->step(t).simulated_tick == 0);
|
|
}
|
|
CHECK(h->status().tick_debt == 10);
|
|
for (std::uint64_t tick = 1; tick <= 20; ++tick)
|
|
CHECK(h->submit(input(tick)));
|
|
auto s = h->step(t);
|
|
CHECK(s.simulated_tick == 8 && s.tick_debt == 2); // At most 8 per step.
|
|
s = h->step(t);
|
|
CHECK(s.simulated_tick == 10 && s.tick_debt == 0 && s.max_tick_debt == 10);
|
|
CHECK(s.queued == 10);
|
|
}
|
|
|
|
// Object store outage: the sealed batch waits, nothing is acknowledged, retries
|
|
// back off (100 ms doubling, not one upload per step), and the region resumes once
|
|
// storage is durable again.
|
|
void storage_outage_pauses_the_region() {
|
|
const auto r = provisioned<LineageRegion>(3, 0x41);
|
|
ObjectStoreFaults faults;
|
|
auto h = host<LineageRegion>(r, config(r), &faults);
|
|
SchedulerTime t = std::chrono::steady_clock::now();
|
|
CHECK(h->step(t).ready);
|
|
for (std::uint64_t tick = 1; tick <= 30; ++tick)
|
|
CHECK(h->submit(input(tick, true)));
|
|
step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 19; });
|
|
faults.fail_directory_syncs = 1000;
|
|
for (int i = 0; i < 5; ++i) {
|
|
t += 50ms;
|
|
const auto& s = h->step(t);
|
|
CHECK(s.simulated_tick == 20 && s.committed_tick == 0); // Paused at the death.
|
|
}
|
|
// Attempts at +0 and +100 ms; the next waits until +300 ms.
|
|
CHECK(h->status().store_failures == 2 && h->take_acks().empty() && h->take_outcomes().empty());
|
|
CHECK(h->status().tick_debt == 4);
|
|
for (int i = 5; i < 40; ++i) { // Two seconds of 50 ms steps: +300, +700, +1500 only.
|
|
t += 50ms;
|
|
CHECK(h->step(t).committed_tick == 0);
|
|
}
|
|
CHECK(h->status().store_failures == 5 && h->take_acks().empty());
|
|
faults.fail_directory_syncs = 0;
|
|
int waited = 0; // The next attempt (+3100 ms, delay capped at 2 s) succeeds.
|
|
auto s = step_until(*h, t, 50ms, [&](const HostStatus& x) {
|
|
++waited;
|
|
return x.committed_tick == 20;
|
|
});
|
|
CHECK(waited <= 23 && s.store_failures == 5);
|
|
CHECK(h->take_acks().size() == 20 && h->take_outcomes().size() == 1);
|
|
s = h->step(t); // Catch-up after the outage is bounded but continues.
|
|
CHECK(s.simulated_tick > 20 && s.simulated_tick <= 28);
|
|
}
|
|
|
|
// Holds the lease row lock to make database calls time out.
|
|
struct RowLock {
|
|
std::unique_ptr<Postgres> db = connect_as(env("PGUSER"));
|
|
explicit RowLock(RegionKey key) {
|
|
const std::array<std::string, 2> p{std::to_string(key.world), std::to_string(key.region)};
|
|
CHECK(db->query("BEGIN", {}, 5s));
|
|
CHECK(db->query("SELECT 1 FROM evolution.region_leases WHERE world_id = $1::evolution.u64 "
|
|
"AND region_id = $2::evolution.u64 FOR UPDATE",
|
|
p, 5s));
|
|
}
|
|
void release() {
|
|
CHECK(db->query("ROLLBACK", {}, 5s));
|
|
}
|
|
};
|
|
|
|
// Unknown commit outcome: not ready, admission refused, resolved by exact bytes.
|
|
void unknown_outcome_is_resolved_before_continuing() {
|
|
const auto r = provisioned<LineageRegion>(4, 0x3D);
|
|
auto h = host<LineageRegion>(r, config(r));
|
|
SchedulerTime t = std::chrono::steady_clock::now();
|
|
CHECK(h->step(t).ready);
|
|
for (std::uint64_t tick = 1; tick <= 25; ++tick)
|
|
CHECK(h->submit(input(tick, true)));
|
|
step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 19; });
|
|
RowLock lock{r.key};
|
|
t += 50ms;
|
|
auto s = h->step(t); // The death seals; the commit cannot finish in time.
|
|
CHECK(s.phase == HostPhase::resolving && !s.ready && s.unknown_outcomes == 1);
|
|
CHECK(h->submit(input(26)).error() == AdmissionError::not_ready);
|
|
CHECK(h->take_outcomes().empty() && h->take_acks().empty());
|
|
lock.release();
|
|
s = step_until(*h, t, 0ms, [](const HostStatus& x) { return x.phase == HostPhase::running; });
|
|
CHECK(s.committed_tick == 20 && s.ready);
|
|
CHECK(h->take_outcomes().size() == 1 && h->take_acks().size() == 20);
|
|
}
|
|
|
|
// Renewals that cannot reach the database fence the host locally; unconfirmed
|
|
// work is reported as never final, and authority is re-acquired from the head.
|
|
void renewal_failure_fences_and_recovers() {
|
|
const auto r = provisioned<LineageRegion>(5, 0x3E);
|
|
auto h = host<LineageRegion>(r, config(r));
|
|
SchedulerTime t = std::chrono::steady_clock::now();
|
|
CHECK(h->step(t).ready);
|
|
for (std::uint64_t tick = 1; tick <= 8; ++tick)
|
|
CHECK(h->submit(input(tick)));
|
|
step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 5; });
|
|
CHECK(h->status().committed_tick == 0);
|
|
const auto epoch = h->status().epoch;
|
|
std::optional<RowLock> lock{std::in_place, r.key};
|
|
step_until(*h, t, 0ms, [](const HostStatus& x) { return x.phase != HostPhase::running; }, 1000);
|
|
CHECK(!h->status().ready && h->status().fences == 1 && h->state() == nullptr);
|
|
const auto dropped = h->take_dropped();
|
|
CHECK(dropped.size() == 8 && std::ranges::all_of(dropped, [](const Dropped& d) {
|
|
return d.reason == Unacknowledged::fenced;
|
|
}));
|
|
CHECK(h->take_acks().empty());
|
|
lock->release();
|
|
lock.reset();
|
|
const auto& s = step_until(
|
|
*h, t, 0ms, [](const HostStatus& x) { return x.phase == HostPhase::running; }, 1000);
|
|
CHECK(s.epoch > epoch && s.committed_tick == 0 && s.simulated_tick == 0 && s.ready);
|
|
}
|
|
|
|
// Drain: backlog dropped, pending ticks committed, lease released for an
|
|
// immediate successor (environmental-v1 profile).
|
|
void drain_commits_and_hands_over() {
|
|
const auto r = provisioned<EnvironmentalCommunity>(6, 0x3F);
|
|
auto h = host<EnvironmentalCommunity>(r, config(r));
|
|
SchedulerTime t = std::chrono::steady_clock::now();
|
|
CHECK(h->step(t).ready);
|
|
for (std::uint64_t tick = 1; tick <= 9; ++tick)
|
|
CHECK(h->submit(input(tick)));
|
|
step_until(*h, t, 50ms, [](const HostStatus& x) { return x.simulated_tick == 6; });
|
|
h->drain();
|
|
CHECK(!h->status().ready && h->submit(input(10)).error() == AdmissionError::draining);
|
|
auto dropped = h->take_dropped();
|
|
CHECK(dropped.size() == 3 && dropped[0].reason == Unacknowledged::drained);
|
|
const auto& s = h->step(t);
|
|
CHECK(s.phase == HostPhase::stopped && s.committed_tick == 6);
|
|
CHECK(h->take_acks().size() == 6);
|
|
auto successor = host<EnvironmentalCommunity>(r, config(r));
|
|
const auto& next = successor->step(t);
|
|
CHECK(next.phase == HostPhase::running && next.committed_tick == 6 && next.epoch == 2);
|
|
successor->drain();
|
|
CHECK(successor->step(t).phase == HostPhase::stopped);
|
|
}
|
|
|
|
// A committed head this host cannot restore quarantines; it is never replaced.
|
|
void unrestorable_head_quarantines() {
|
|
const auto r = provisioned<LineageRegion>(7, 0x40);
|
|
auto c = config(r);
|
|
c.build = filled<32>(0xB2);
|
|
c.retained_builds = {c.build}; // Not the build that produced the head.
|
|
// A current build in the database (so acquisition succeeds, ADR-0068): the
|
|
// quarantine comes from recovery refusing the head, not from the lease.
|
|
CHECK(connect_as(env("PGUSER"))
|
|
->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
|
"(decode(repeat('b2', 32), 'hex')) ON CONFLICT DO NOTHING",
|
|
{}, 5s));
|
|
auto h = host<LineageRegion>(r, c);
|
|
const auto& s = h->step(std::chrono::steady_clock::now());
|
|
CHECK(s.phase == HostPhase::quarantined && !s.ready && h->state() == nullptr);
|
|
CHECK(h->submit(input(1)).error() == AdmissionError::not_ready);
|
|
// Wrong frame and invalid configuration are refused too.
|
|
auto wrong = config(r);
|
|
wrong.frame = 10;
|
|
CHECK(host<LineageRegion>(r, wrong)->step(std::chrono::steady_clock::now()).phase ==
|
|
HostPhase::quarantined);
|
|
auto invalid = config(r);
|
|
invalid.retained_builds = {};
|
|
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
|
|
CHECK(!RegionHost<LineageRegion>::create(invalid, runtime_connect(), std::move(*objects)));
|
|
}
|
|
} // namespace
|
|
|
|
void run_region_host_tests() {
|
|
acknowledgements_follow_commits();
|
|
starvation_accrues_bounded_debt();
|
|
storage_outage_pauses_the_region();
|
|
unknown_outcome_is_resolved_before_continuing();
|
|
renewal_failure_fences_and_recovers();
|
|
drain_commits_and_hands_over();
|
|
unrestorable_head_quarantines();
|
|
}
|