// Real PostgreSQL M10.4b durable action records (ADR-0092, schema v11): actions committed in // the same statement as their batch's checkpoint, exact retries and resolution including // the action block, invalid blocks, an action id never committed twice, and host crashes // before and after the commit. Regions 7/131..135 and commit ids tagged 0xA7 are used only // here. Run by // tools/ci/postgres-check.sh. #include "evo/persistence/checkpoint_store.hpp" #include "evo/persistence/region_lease_store.hpp" #include "evo/server/region_host.hpp" #include "fixtures/environmental_region.hpp" #ifdef EVO_POSTGRES_SESSIONS #include "evo/server/postgres_action_ledger.hpp" #endif #include #include #include #include #include #include #include #include #include void run_region_action_tests(); namespace { using namespace evo::persistence; using namespace evo::server; using namespace std::chrono_literals; using evo::processes::LineageRegion; namespace fs = std::filesystem; void check(bool value, std::source_location at = std::source_location::current()) { if (!value) throw std::runtime_error("region action test line " + std::to_string(at.line())); } #define CHECK(...) check(static_cast(__VA_ARGS__)) template bool holds(const std::expected& e, const T& v) { return e.has_value() && *e == v; } constexpr std::uint64_t world = 7; // The fixtures' world. constexpr std::uint64_t base_region = 130; constexpr std::size_t budget = 8000000; std::string env(const char* key) { const auto* value = std::getenv(key); if (!value) throw std::runtime_error("missing test environment"); return value; } PgOptions options(const std::string& user) { return {env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}}; } std::unique_ptr connect_as(const std::string& user) { auto made = Postgres::connect(options(user), 5s); CHECK(made); return std::move(*made); } template std::array filled(std::uint8_t value) { std::array out{}; out.fill(value); return out; } const Sha256Digest build = filled<32>(0xB1); const Uuid alice = filled<16>(0xA1), bob = filled<16>(0xB0); const Uuid carol = filled<16>(0xC1), dave = filled<16>(0xD0); // Host tests only. // Commit ids are unique across the database: tag this file's. Uuid tagged(std::uint8_t n) { Uuid id = filled<16>(0xA7); id.back() = n; return id; } struct Region { RegionKey key; fs::path root; CommittedCheckpoint genesis; }; Region provisioned(std::uint64_t region) { region += base_region; Region r{{world, region}, fs::path(env("PGHOST")) / ("actions-" + std::to_string(region)), {}}; fs::remove_all(r.root); fs::create_directory(r.root); fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace); auto owner = connect_as(env("PGUSER")); CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES " "(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING", {}, 5s)); auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes); CHECK(objects); auto genesis = provision_lineage_region( *owner, **objects, r.key, evo::test::region_identity.frame, *evo::test::lineage_region(), tagged(static_cast(region)), build, budget, 5s); if (!genesis) std::fprintf(stderr, "provision failed: kind %d sqlstate %s\n", static_cast(genesis.error().kind), genesis.error().store.cause.sqlstate.data()); CHECK(genesis); r.genesis = *genesis; return r; } std::vector record(const Uuid& account, std::uint64_t id, std::uint64_t tick, std::uint16_t kind = 1) { std::vector out(account.begin(), account.end()); for (unsigned b = 0; b < 8; ++b) out.push_back(static_cast(id >> (8U * b))); for (unsigned b = 0; b < 8; ++b) out.push_back(static_cast(tick >> (8U * b))); out.push_back(static_cast(kind)); out.push_back(static_cast(kind >> 8U)); return out; } std::vector join(std::initializer_list> parts) { std::vector out; for (const auto& p : parts) out.insert(out.end(), p.begin(), p.end()); return out; } // Owner read of the durable record: (region, tick, commit) or empty. std::string stored(Postgres& owner, const Uuid& account, std::uint64_t id) { const std::array p{std::string{"\\x"} + [&] { std::string h; constexpr char digits[] = "0123456789abcdef"; for (const auto b : account) { h.push_back(digits[b >> 4U]); h.push_back(digits[b & 0xFU]); } return h; }(), std::to_string(id)}; auto rows = owner.query("SELECT region_id || ':' || tick || ':' || commit_sequence FROM " "evolution.region_actions WHERE world_id = 7 AND account_id = " "encode($1::bytea, 'hex')::uuid AND action_id = $2::evolution.u64", p, 5s); CHECK(rows); return rows->empty() ? std::string{} : (*rows)[0][0].value_or(""); } void store_rules() { auto r = provisioned(1); auto db = connect_as(env("EVO_PG_RUNTIME_USER")); auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes); CHECK(objects); auto lease = acquire_region_lease(*db, r.key, build, 20000ms, 5s); CHECK(lease); auto state = evo::test::lineage_region(); RegionBatch batch{*state}; for (std::uint64_t tick = 1; tick <= 10; ++tick) CHECK(batch.advance(evo::test::region_input(tick, false))); batch.seal(); const std::vector actions{{alice, 2, 10, 1}, {bob, 1, 3, 1}, {alice, 1, 3, 1}}; auto first = publish_lineage_batch(*db, **objects, *lease, r.genesis, batch, tagged(0x01), build, budget, 5s, {}, actions); if (!first) std::fprintf(stderr, "publish failed: kind %d store %d sqlstate %s\n", static_cast(first.error().kind), static_cast(first.error().store.kind), first.error().store.cause.sqlstate.data()); CHECK(first && first->committed.head.sequence == 1); CHECK(batch.confirm(10)); auto found = find_region_action(*db, r.key, *lease, alice, 1, 5s); CHECK(found && *found == CommittedAction{131, 3, 1}); CHECK(holds(find_region_action(*db, r.key, *lease, bob, 1, 5s), std::optional{CommittedAction{131, 3, 1}})); CHECK(holds(find_region_action(*db, r.key, *lease, alice, 2, 5s), std::optional{CommittedAction{131, 10, 1}})); CHECK(holds(find_region_action(*db, r.key, *lease, alice, 3, 5s), std::optional{})); // Exact retry and resolution include the action block. const auto block = encode_actions(actions, 0, 10); CHECK(block); PreparedCommit prepared{r.key, tagged(0x01), first->committed.manifest, {}, *block}; CHECK(retry_prepared(*db, prepared, 5s) == first->committed.head); auto resolved = resolve_prepared(*db, prepared, 5s); CHECK(resolved && resolved->status == Resolution::committed); auto changed = prepared; changed.actions = join({record(alice, 1, 3), record(alice, 2, 10)}); // One missing. auto retried = retry_prepared(*db, changed, 5s); CHECK(!retried && retried.error().kind == LeaseStoreError::commit_conflict); resolved = resolve_prepared(*db, changed, 5s); CHECK(resolved && resolved->status == Resolution::conflict); // Every invalid block is refused before anything else. for (const auto& bad : {record(alice, 9, 11), record(alice, 9, 0), record(alice, 9, 5, 2), record(Uuid{}, 9, 5), record(alice, 0, 5), join({record(bob, 9, 5), record(alice, 9, 5)}), join({record(alice, 9, 5), record(alice, 9, 5)}), join({record(alice, 9, 6), record(alice, 9, 5)}), join({record(alice, 9, 5), record(alice, 9, 6)}), // One id at two ticks. std::vector(33, 1)}) { auto p = prepared; p.actions = bad; retried = retry_prepared(*db, p, 5s); CHECK(!retried && retried.error().kind == LeaseStoreError::invalid); } // The local encoder refuses the same blocks it would send. CHECK(!encode_actions(std::vector{{alice, 9, 11, 1}}, 0, 10)); CHECK(!encode_actions(std::vector{{alice, 9, 5, 1}, {alice, 9, 6, 1}}, 0, 10)); // An action id is recorded once in a world: a later batch carrying it commits, and the // first record stands. RegionBatch next{*state}; for (std::uint64_t tick = 11; tick <= 20; ++tick) CHECK(next.advance(evo::test::region_input(tick, false))); next.seal(); const std::vector repeat{{alice, 1, 12, 1}, {alice, 3, 12, 1}}; auto second = publish_lineage_batch(*db, **objects, *lease, first->committed, next, tagged(0x03), build, budget, 5s, {}, repeat); CHECK(second && second->committed.head.sequence == 2); CHECK(holds(find_region_action(*db, r.key, *lease, alice, 1, 5s), std::optional{CommittedAction{131, 3, 1}})); // A batch without actions still publishes as before v11. CHECK(next.confirm(20)); RegionBatch plain{*state}; for (std::uint64_t tick = 21; tick <= 25; ++tick) CHECK(plain.advance(evo::test::region_input(tick, false))); plain.seal(); CHECK(publish_lineage_batch(*db, **objects, *lease, second->committed, plain, tagged(0x04), build, budget, 5s)); // Lookups are fenced like everything else under the lease. CHECK(release_region_lease(*db, r.key, *lease, 5s)); auto later = acquire_region_lease(*db, r.key, build, 20000ms, 5s); CHECK(later); auto fenced = find_region_action(*db, r.key, *lease, alice, 1, 5s); CHECK(!fenced && fenced.error().kind == LeaseStoreError::fenced); CHECK(holds(find_region_action(*db, r.key, *later, alice, 3, 5s), std::optional{CommittedAction{131, 12, 2}})); #ifdef EVO_POSTGRES_SESSIONS // The session layer's ledger over the same lookup. std::optional current = *later; PostgresActionLedger ledger{*db, r.key, [&] { return current; }}; evo::network::AccountId who; who.bytes = alice; CHECK(holds(ledger.find(who, 3), std::optional{evo::network::ActionOutcome{ 3, evo::network::ActionResult::committed_no_effect, 12, 2}})); CHECK(holds(ledger.find(who, 99), std::optional{})); current.reset(); CHECK(!ledger.find(who, 3)); #endif CHECK(release_region_lease(*db, r.key, *later, 5s)); } // Host-level: fast lease (4 s), 1 s commits, as in the host tests. const LeasePolicy fast{4000ms, 800ms, 400ms, 400ms, 1000ms}; struct Crash {}; HostConfig config(const Region& r) { HostConfig c; c.key = r.key; c.frame = evo::test::region_identity.frame; c.lease = fast; c.build = build; c.retained_builds = {build}; c.max_queued_actions = 3; return c; } std::unique_ptr> host(const Region& r, HostConfig c) { auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes); CHECK(objects); auto made = RegionHost::create( std::move(c), [] { return Postgres::connect(options(env("EVO_PG_RUNTIME_USER")), 2s); }, std::move(*objects)); CHECK(made); return std::move(*made); } TickInput input(std::uint64_t tick, std::uint8_t tag) { Uuid request{}; request[0] = tag; for (unsigned i = 0; i < 8; ++i) request[8 + i] = static_cast((tick >> (8U * i)) & 0xFFU); return {request, evo::test::region_input(tick, false)}; } template const HostStatus& step_until(RegionHost& h, SchedulerTime& t, Done done, int limit = 1000) { for (int i = 0; i < limit; ++i) { t += 50ms; const auto& s = h.step(t); if (done(s)) return s; std::this_thread::sleep_for(10ms); } CHECK(false); return h.status(); } void host_actions() { auto owner = connect_as(env("PGUSER")); { // Normal path: acknowledged with the tick each rode; late actions ride the next tick. const auto r = provisioned(2); auto h = host(r, config(r)); CHECK(h->submit_action({carol, 1, 3, 1}).error() == AdmissionError::not_ready); SchedulerTime t = std::chrono::steady_clock::now(); CHECK(h->step(t).ready); CHECK(h->submit_action({carol, 1, 3, 1})); CHECK(h->submit_action({carol, 1, 4, 1}).error() == AdmissionError::duplicate); CHECK(h->submit_action({carol, 0, 4, 1}).error() == AdmissionError::invalid); CHECK(h->submit_action({carol, 5, 4, 2}).error() == AdmissionError::invalid); CHECK(h->submit_action({dave, 1, 1, 1})); CHECK(h->submit_action({dave, 2, 2, 1})); CHECK(h->submit_action({dave, 3, 2, 1}).error() == AdmissionError::full); for (std::uint64_t tick = 1; tick <= 30; ++tick) CHECK(h->submit(input(tick, 0x71))); step_until(*h, t, [](const HostStatus& s) { return s.committed_sequence >= 1; }); auto acks = h->take_action_acks(); CHECK(acks.size() == 3); for (const auto& a : acks) CHECK(a.commit_sequence == 1 && a.record.tick == (a.record.account == dave ? a.record.action_id : 3u)); CHECK(stored(*owner, carol, 1) == "132:3:1"); // Submitted after tick 20 ran for tick 5: rides the first tick simulated next. const auto ran = h->status().simulated_tick; CHECK(h->submit_action({carol, 7, 5, 1})); step_until(*h, t, [](const HostStatus& s) { return s.committed_sequence >= 2; }); acks = h->take_action_acks(); CHECK(acks.size() == 1 && acks[0].record.tick == ran + 1); CHECK(h->take_dropped_actions().empty()); } { // Crash before the transaction: never committed; a successor finds no record. const auto r = provisioned(3); bool armed = false; auto c = config(r); c.crash_hook = [&](CrashPoint p) { if (armed && p == CrashPoint::before_transaction) throw Crash{}; }; auto h = host(r, c); SchedulerTime t = std::chrono::steady_clock::now(); CHECK(h->step(t).ready); for (std::uint64_t tick = 1; tick <= 30; ++tick) CHECK(h->submit(input(tick, 0x72))); CHECK(h->submit_action({carol, 10, 5, 1})); armed = true; bool crashed = false; try { step_until(*h, t, [](const HostStatus& s) { return s.committed_sequence >= 1; }); } catch (const Crash&) { crashed = true; } CHECK(crashed); h.reset(); auto successor = host(r, config(r)); step_until(*successor, t, [](const HostStatus& s) { return s.ready; }); CHECK(successor->status().committed_sequence == 0 && stored(*owner, carol, 10).empty()); } { // Crash after the commit, before any acknowledgement: the record is durable. const auto r = provisioned(4); bool armed = false; auto c = config(r); c.crash_hook = [&](CrashPoint p) { if (armed && p == CrashPoint::after_commit) throw Crash{}; }; auto h = host(r, c); SchedulerTime t = std::chrono::steady_clock::now(); CHECK(h->step(t).ready); for (std::uint64_t tick = 1; tick <= 30; ++tick) CHECK(h->submit(input(tick, 0x73))); CHECK(h->submit_action({carol, 11, 5, 1})); armed = true; bool crashed = false; try { step_until(*h, t, [](const HostStatus& s) { return s.committed_sequence >= 1; }); } catch (const Crash&) { crashed = true; } CHECK(crashed && h->take_action_acks().empty()); h.reset(); CHECK(stored(*owner, carol, 11) == "134:5:1"); auto successor = host(r, config(r)); step_until(*successor, t, [](const HostStatus& s) { return s.ready; }); CHECK(successor->status().committed_sequence == 1); CHECK(successor->submit_action({carol, 11, 40, 1})); // The host does not deduplicate // across processes: admission does (find_region_action), and the database refuses a // second commit of the id (store_rules). } { // Unknown publication outcome: resolved by the exact bytes, actions included. const auto r = provisioned(5); auto h = host(r, config(r)); SchedulerTime t = std::chrono::steady_clock::now(); CHECK(h->step(t).ready); for (std::uint64_t tick = 1; tick <= 30; ++tick) CHECK(h->submit(input(tick, 0x74))); CHECK(h->submit_action({dave, 12, 2, 1})); step_until(*h, t, [](const HostStatus& s) { return s.simulated_tick >= 2; }); auto lock = connect_as(env("PGUSER")); const std::array key{std::to_string(r.key.world), std::to_string(r.key.region)}; CHECK(lock->query("BEGIN", {}, 5s)); CHECK(lock->query("SELECT 1 FROM evolution.region_leases WHERE world_id = " "$1::evolution.u64 AND region_id = $2::evolution.u64 FOR UPDATE", key, 5s)); step_until(*h, t, [](const HostStatus& s) { return s.phase == HostPhase::resolving; }); CHECK(h->take_action_acks().empty() && h->take_dropped_actions().empty()); CHECK(lock->query("ROLLBACK", {}, 5s)); step_until(*h, t, [](const HostStatus& s) { return s.phase == HostPhase::running; }); const auto acks = h->take_action_acks(); CHECK(acks.size() == 1 && acks[0].record.action_id == 12 && acks[0].commit_sequence == 1); CHECK(stored(*owner, dave, 12) == "135:2:1"); } } } // namespace void run_region_action_tests() { store_rules(); host_actions(); std::puts("region action integration passed"); }