Foundation / checks (deploy) (push) Successful in 3m27s
Foundation / checks (ecs) (push) Successful in 1m57s
Foundation / checks (client) (push) Successful in 12m41s
Foundation / checks (network) (push) Successful in 13m33s
Foundation / checks (tsan) (push) Successful in 7m54s
Foundation / checks (checks) (push) Successful in 32m56s
Foundation / checks (postgres) (push) Successful in 46m10s
Foundation / postgres-tools (push) Successful in 18s
Foundation / image (push) Successful in 2m33s
Foundation / server-image (push) Successful in 4m24s
The fourth dev drill showed a synchronous PostgreSQL round trip per action on the simulation thread: at admission (uniqueness) and again at commit (first-record rule). About 12 lookups landed on each commit tick (commit ticks 58-70 ms, session p99 30-39 ms). - ActionLedger::find_all; persistence::find_region_actions applies the schema v11 find_region_action to many pairs in one statement (no migration). - IntentAdmission::admit/verify: actions only the ledger can refuse wait for one verify per poll; receipts still go out in the same poll. committed() asks once per commit. - Tests: deferred_actions (network_control), find_region_actions and find_all (Postgres). - ADR-0097 fourth run, ADR-0092 amendment, VALIDATION; refresh the dev session window. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
432 lines
20 KiB
C++
432 lines
20 KiB
C++
// Real PostgreSQL M10.4b durable action records (ADR-0092, schema v11): actions committed in
|
|
// the same statement as their batch's checkpoint, exact retries and resolution including
|
|
// the action block, invalid blocks, an action id never committed twice, and host crashes
|
|
// before and after the commit. Regions 7/131..135 and commit ids tagged 0xA7 are used only
|
|
// here. Run by
|
|
// tools/ci/postgres-check.sh.
|
|
#include "evo/persistence/checkpoint_store.hpp"
|
|
#include "evo/persistence/region_lease_store.hpp"
|
|
#include "evo/server/region_host.hpp"
|
|
#include "fixtures/environmental_region.hpp"
|
|
#ifdef EVO_POSTGRES_SESSIONS
|
|
#include "evo/server/postgres_action_ledger.hpp"
|
|
#endif
|
|
#include <array>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <filesystem>
|
|
#include <source_location>
|
|
#include <stdexcept>
|
|
#include <string>
|
|
#include <thread>
|
|
#include <vector>
|
|
|
|
void run_region_action_tests();
|
|
|
|
namespace {
|
|
using namespace evo::persistence;
|
|
using namespace evo::server;
|
|
using namespace std::chrono_literals;
|
|
using evo::processes::LineageRegion;
|
|
namespace fs = std::filesystem;
|
|
void check(bool value, std::source_location at = std::source_location::current()) {
|
|
if (!value)
|
|
throw std::runtime_error("region action test line " + std::to_string(at.line()));
|
|
}
|
|
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
|
template <typename T, typename E> bool holds(const std::expected<T, E>& e, const T& v) {
|
|
return e.has_value() && *e == v;
|
|
}
|
|
constexpr std::uint64_t world = 7; // The fixtures' world.
|
|
constexpr std::uint64_t base_region = 130;
|
|
constexpr std::size_t budget = 8000000;
|
|
|
|
std::string env(const char* key) {
|
|
const auto* value = std::getenv(key);
|
|
if (!value)
|
|
throw std::runtime_error("missing test environment");
|
|
return value;
|
|
}
|
|
PgOptions options(const std::string& user) {
|
|
return {env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}};
|
|
}
|
|
std::unique_ptr<Postgres> connect_as(const std::string& user) {
|
|
auto made = Postgres::connect(options(user), 5s);
|
|
CHECK(made);
|
|
return std::move(*made);
|
|
}
|
|
template <std::size_t N> std::array<std::uint8_t, N> filled(std::uint8_t value) {
|
|
std::array<std::uint8_t, N> out{};
|
|
out.fill(value);
|
|
return out;
|
|
}
|
|
const Sha256Digest build = filled<32>(0xB1);
|
|
const Uuid alice = filled<16>(0xA1), bob = filled<16>(0xB0);
|
|
const Uuid carol = filled<16>(0xC1), dave = filled<16>(0xD0); // Host tests only.
|
|
// Commit ids are unique across the database: tag this file's.
|
|
Uuid tagged(std::uint8_t n) {
|
|
Uuid id = filled<16>(0xA7);
|
|
id.back() = n;
|
|
return id;
|
|
}
|
|
|
|
struct Region {
|
|
RegionKey key;
|
|
fs::path root;
|
|
CommittedCheckpoint genesis;
|
|
};
|
|
Region provisioned(std::uint64_t region) {
|
|
region += base_region;
|
|
Region r{{world, region}, fs::path(env("PGHOST")) / ("actions-" + std::to_string(region)), {}};
|
|
fs::remove_all(r.root);
|
|
fs::create_directory(r.root);
|
|
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
|
|
auto owner = connect_as(env("PGUSER"));
|
|
CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
|
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
|
|
{}, 5s));
|
|
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
|
|
CHECK(objects);
|
|
auto genesis = provision_lineage_region(
|
|
*owner, **objects, r.key, evo::test::region_identity.frame, *evo::test::lineage_region(),
|
|
tagged(static_cast<std::uint8_t>(region)), build, budget, 5s);
|
|
if (!genesis)
|
|
std::fprintf(stderr, "provision failed: kind %d sqlstate %s\n",
|
|
static_cast<int>(genesis.error().kind),
|
|
genesis.error().store.cause.sqlstate.data());
|
|
CHECK(genesis);
|
|
r.genesis = *genesis;
|
|
return r;
|
|
}
|
|
std::vector<std::uint8_t> record(const Uuid& account, std::uint64_t id, std::uint64_t tick,
|
|
std::uint16_t kind = 1) {
|
|
std::vector<std::uint8_t> out(account.begin(), account.end());
|
|
for (unsigned b = 0; b < 8; ++b)
|
|
out.push_back(static_cast<std::uint8_t>(id >> (8U * b)));
|
|
for (unsigned b = 0; b < 8; ++b)
|
|
out.push_back(static_cast<std::uint8_t>(tick >> (8U * b)));
|
|
out.push_back(static_cast<std::uint8_t>(kind));
|
|
out.push_back(static_cast<std::uint8_t>(kind >> 8U));
|
|
return out;
|
|
}
|
|
std::vector<std::uint8_t> join(std::initializer_list<std::vector<std::uint8_t>> parts) {
|
|
std::vector<std::uint8_t> out;
|
|
for (const auto& p : parts)
|
|
out.insert(out.end(), p.begin(), p.end());
|
|
return out;
|
|
}
|
|
// Owner read of the durable record: (region, tick, commit) or empty.
|
|
std::string stored(Postgres& owner, const Uuid& account, std::uint64_t id) {
|
|
const std::array<std::string, 2> p{std::string{"\\x"} +
|
|
[&] {
|
|
std::string h;
|
|
constexpr char digits[] = "0123456789abcdef";
|
|
for (const auto b : account) {
|
|
h.push_back(digits[b >> 4U]);
|
|
h.push_back(digits[b & 0xFU]);
|
|
}
|
|
return h;
|
|
}(),
|
|
std::to_string(id)};
|
|
auto rows = owner.query("SELECT region_id || ':' || tick || ':' || commit_sequence FROM "
|
|
"evolution.region_actions WHERE world_id = 7 AND account_id = "
|
|
"encode($1::bytea, 'hex')::uuid AND action_id = $2::evolution.u64",
|
|
p, 5s);
|
|
CHECK(rows);
|
|
return rows->empty() ? std::string{} : (*rows)[0][0].value_or("");
|
|
}
|
|
|
|
void store_rules() {
|
|
auto r = provisioned(1);
|
|
auto db = connect_as(env("EVO_PG_RUNTIME_USER"));
|
|
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
|
|
CHECK(objects);
|
|
auto lease = acquire_region_lease(*db, r.key, build, 20000ms, 5s);
|
|
CHECK(lease);
|
|
auto state = evo::test::lineage_region();
|
|
RegionBatch<LineageRegion> batch{*state};
|
|
for (std::uint64_t tick = 1; tick <= 10; ++tick)
|
|
CHECK(batch.advance(evo::test::region_input(tick, false)));
|
|
batch.seal();
|
|
const std::vector<ActionRecord> actions{{alice, 2, 10, 1}, {bob, 1, 3, 1}, {alice, 1, 3, 1}};
|
|
auto first = publish_lineage_batch(*db, **objects, *lease, r.genesis, batch, tagged(0x01),
|
|
build, budget, 5s, {}, actions);
|
|
if (!first)
|
|
std::fprintf(stderr, "publish failed: kind %d store %d sqlstate %s\n",
|
|
static_cast<int>(first.error().kind),
|
|
static_cast<int>(first.error().store.kind),
|
|
first.error().store.cause.sqlstate.data());
|
|
CHECK(first && first->committed.head.sequence == 1);
|
|
CHECK(batch.confirm(10));
|
|
auto found = find_region_action(*db, r.key, *lease, alice, 1, 5s);
|
|
CHECK(found && *found == CommittedAction{131, 3, 1});
|
|
CHECK(holds(find_region_action(*db, r.key, *lease, bob, 1, 5s),
|
|
std::optional{CommittedAction{131, 3, 1}}));
|
|
CHECK(holds(find_region_action(*db, r.key, *lease, alice, 2, 5s),
|
|
std::optional{CommittedAction{131, 10, 1}}));
|
|
CHECK(holds(find_region_action(*db, r.key, *lease, alice, 3, 5s),
|
|
std::optional<CommittedAction>{}));
|
|
// M10.9d: the same answers for several pairs in one statement, in the order asked.
|
|
const std::vector<ActionRef> asked{{alice, 3}, {bob, 1}, {alice, 2}, {alice, 1}, {bob, 7}};
|
|
auto many = find_region_actions(*db, r.key, *lease, asked, 5s);
|
|
CHECK(many &&
|
|
*many == std::vector<std::optional<CommittedAction>>{
|
|
std::nullopt, CommittedAction{131, 3, 1}, CommittedAction{131, 10, 1},
|
|
CommittedAction{131, 3, 1}, std::nullopt});
|
|
CHECK(holds(find_region_actions(*db, r.key, *lease, {}, 5s),
|
|
std::vector<std::optional<CommittedAction>>{}));
|
|
|
|
// Exact retry and resolution include the action block.
|
|
const auto block = encode_actions(actions, 0, 10);
|
|
CHECK(block);
|
|
PreparedCommit prepared{r.key, tagged(0x01), first->committed.manifest, {}, *block};
|
|
CHECK(retry_prepared(*db, prepared, 5s) == first->committed.head);
|
|
auto resolved = resolve_prepared(*db, prepared, 5s);
|
|
CHECK(resolved && resolved->status == Resolution::committed);
|
|
auto changed = prepared;
|
|
changed.actions = join({record(alice, 1, 3), record(alice, 2, 10)}); // One missing.
|
|
auto retried = retry_prepared(*db, changed, 5s);
|
|
CHECK(!retried && retried.error().kind == LeaseStoreError::commit_conflict);
|
|
resolved = resolve_prepared(*db, changed, 5s);
|
|
CHECK(resolved && resolved->status == Resolution::conflict);
|
|
// Every invalid block is refused before anything else.
|
|
for (const auto& bad :
|
|
{record(alice, 9, 11), record(alice, 9, 0), record(alice, 9, 5, 2), record(Uuid{}, 9, 5),
|
|
record(alice, 0, 5), join({record(bob, 9, 5), record(alice, 9, 5)}),
|
|
join({record(alice, 9, 5), record(alice, 9, 5)}),
|
|
join({record(alice, 9, 6), record(alice, 9, 5)}),
|
|
join({record(alice, 9, 5), record(alice, 9, 6)}), // One id at two ticks.
|
|
std::vector<std::uint8_t>(33, 1)}) {
|
|
auto p = prepared;
|
|
p.actions = bad;
|
|
retried = retry_prepared(*db, p, 5s);
|
|
CHECK(!retried && retried.error().kind == LeaseStoreError::invalid);
|
|
}
|
|
// The local encoder refuses the same blocks it would send.
|
|
CHECK(!encode_actions(std::vector<ActionRecord>{{alice, 9, 11, 1}}, 0, 10));
|
|
CHECK(!encode_actions(std::vector<ActionRecord>{{alice, 9, 5, 1}, {alice, 9, 6, 1}}, 0, 10));
|
|
|
|
// An action id is recorded once in a world: a later batch carrying it commits, and the
|
|
// first record stands.
|
|
RegionBatch<LineageRegion> next{*state};
|
|
for (std::uint64_t tick = 11; tick <= 20; ++tick)
|
|
CHECK(next.advance(evo::test::region_input(tick, false)));
|
|
next.seal();
|
|
const std::vector<ActionRecord> repeat{{alice, 1, 12, 1}, {alice, 3, 12, 1}};
|
|
auto second = publish_lineage_batch(*db, **objects, *lease, first->committed, next,
|
|
tagged(0x03), build, budget, 5s, {}, repeat);
|
|
CHECK(second && second->committed.head.sequence == 2);
|
|
CHECK(holds(find_region_action(*db, r.key, *lease, alice, 1, 5s),
|
|
std::optional{CommittedAction{131, 3, 1}}));
|
|
// A batch without actions still publishes as before v11.
|
|
CHECK(next.confirm(20));
|
|
RegionBatch<LineageRegion> plain{*state};
|
|
for (std::uint64_t tick = 21; tick <= 25; ++tick)
|
|
CHECK(plain.advance(evo::test::region_input(tick, false)));
|
|
plain.seal();
|
|
CHECK(publish_lineage_batch(*db, **objects, *lease, second->committed, plain, tagged(0x04),
|
|
build, budget, 5s));
|
|
// Lookups are fenced like everything else under the lease.
|
|
CHECK(release_region_lease(*db, r.key, *lease, 5s));
|
|
auto later = acquire_region_lease(*db, r.key, build, 20000ms, 5s);
|
|
CHECK(later);
|
|
auto fenced = find_region_action(*db, r.key, *lease, alice, 1, 5s);
|
|
CHECK(!fenced && fenced.error().kind == LeaseStoreError::fenced);
|
|
const std::vector<ActionRef> one{{alice, 1}};
|
|
auto fenced_many = find_region_actions(*db, r.key, *lease, one, 5s);
|
|
CHECK(!fenced_many && fenced_many.error().kind == LeaseStoreError::fenced);
|
|
CHECK(holds(find_region_action(*db, r.key, *later, alice, 3, 5s),
|
|
std::optional{CommittedAction{131, 12, 2}}));
|
|
#ifdef EVO_POSTGRES_SESSIONS
|
|
// The session layer's ledger over the same lookup.
|
|
std::optional<LeaseGrant> current = *later;
|
|
PgSlot slot{options(env("EVO_PG_RUNTIME_USER")), 5s};
|
|
PostgresActionLedger ledger{slot, r.key, [&] { return current; }};
|
|
evo::network::AccountId who;
|
|
who.bytes = alice;
|
|
CHECK(holds(ledger.find(who, 3),
|
|
std::optional{evo::network::ActionOutcome{
|
|
3, evo::network::ActionResult::committed_no_effect, 12, 2}}));
|
|
CHECK(holds(ledger.find(who, 99), std::optional<evo::network::ActionOutcome>{}));
|
|
const std::vector<evo::network::ActionKey> keys{{who, 99}, {who, 3}};
|
|
CHECK(holds(ledger.find_all(keys),
|
|
std::vector<std::optional<evo::network::ActionOutcome>>{
|
|
std::nullopt, evo::network::ActionOutcome{
|
|
3, evo::network::ActionResult::committed_no_effect, 12, 2}}));
|
|
current.reset();
|
|
CHECK(!ledger.find(who, 3));
|
|
CHECK(!ledger.find_all(keys));
|
|
#endif
|
|
CHECK(release_region_lease(*db, r.key, *later, 5s));
|
|
}
|
|
|
|
// Host-level: fast lease (4 s), 1 s commits, as in the host tests.
|
|
const LeasePolicy fast{4000ms, 800ms, 400ms, 400ms, 1000ms};
|
|
struct Crash {};
|
|
HostConfig config(const Region& r) {
|
|
HostConfig c;
|
|
c.key = r.key;
|
|
c.frame = evo::test::region_identity.frame;
|
|
c.lease = fast;
|
|
c.build = build;
|
|
c.retained_builds = {build};
|
|
c.max_queued_actions = 3;
|
|
return c;
|
|
}
|
|
std::unique_ptr<RegionHost<LineageRegion>> host(const Region& r, HostConfig c) {
|
|
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
|
|
CHECK(objects);
|
|
auto made = RegionHost<LineageRegion>::create(
|
|
std::move(c), [] { return Postgres::connect(options(env("EVO_PG_RUNTIME_USER")), 2s); },
|
|
std::move(*objects));
|
|
CHECK(made);
|
|
return std::move(*made);
|
|
}
|
|
TickInput input(std::uint64_t tick, std::uint8_t tag) {
|
|
Uuid request{};
|
|
request[0] = tag;
|
|
for (unsigned i = 0; i < 8; ++i)
|
|
request[8 + i] = static_cast<std::uint8_t>((tick >> (8U * i)) & 0xFFU);
|
|
return {request, evo::test::region_input(tick, false)};
|
|
}
|
|
template <typename Done>
|
|
const HostStatus& step_until(RegionHost<LineageRegion>& h, SchedulerTime& t, Done done,
|
|
int limit = 1000) {
|
|
for (int i = 0; i < limit; ++i) {
|
|
t += 50ms;
|
|
const auto& s = h.step(t);
|
|
if (done(s))
|
|
return s;
|
|
std::this_thread::sleep_for(10ms);
|
|
}
|
|
CHECK(false);
|
|
return h.status();
|
|
}
|
|
|
|
void host_actions() {
|
|
auto owner = connect_as(env("PGUSER"));
|
|
{
|
|
// Normal path: acknowledged with the tick each rode; late actions ride the next tick.
|
|
const auto r = provisioned(2);
|
|
auto h = host(r, config(r));
|
|
CHECK(h->submit_action({carol, 1, 3, 1}).error() == AdmissionError::not_ready);
|
|
SchedulerTime t = std::chrono::steady_clock::now();
|
|
CHECK(h->step(t).ready);
|
|
CHECK(h->submit_action({carol, 1, 3, 1}));
|
|
CHECK(h->submit_action({carol, 1, 4, 1}).error() == AdmissionError::duplicate);
|
|
CHECK(h->submit_action({carol, 0, 4, 1}).error() == AdmissionError::invalid);
|
|
CHECK(h->submit_action({carol, 5, 4, 2}).error() == AdmissionError::invalid);
|
|
CHECK(h->submit_action({dave, 1, 1, 1}));
|
|
CHECK(h->submit_action({dave, 2, 2, 1}));
|
|
CHECK(h->submit_action({dave, 3, 2, 1}).error() == AdmissionError::full);
|
|
for (std::uint64_t tick = 1; tick <= 30; ++tick)
|
|
CHECK(h->submit(input(tick, 0x71)));
|
|
step_until(*h, t, [](const HostStatus& s) { return s.committed_sequence >= 1; });
|
|
auto acks = h->take_action_acks();
|
|
CHECK(acks.size() == 3);
|
|
for (const auto& a : acks)
|
|
CHECK(a.commit_sequence == 1 &&
|
|
a.record.tick == (a.record.account == dave ? a.record.action_id : 3u));
|
|
CHECK(stored(*owner, carol, 1) == "132:3:1");
|
|
// Submitted after tick 20 ran for tick 5: rides the first tick simulated next.
|
|
const auto ran = h->status().simulated_tick;
|
|
CHECK(h->submit_action({carol, 7, 5, 1}));
|
|
step_until(*h, t, [](const HostStatus& s) { return s.committed_sequence >= 2; });
|
|
acks = h->take_action_acks();
|
|
CHECK(acks.size() == 1 && acks[0].record.tick == ran + 1);
|
|
CHECK(h->take_dropped_actions().empty());
|
|
}
|
|
{
|
|
// Crash before the transaction: never committed; a successor finds no record.
|
|
const auto r = provisioned(3);
|
|
bool armed = false;
|
|
auto c = config(r);
|
|
c.crash_hook = [&](CrashPoint p) {
|
|
if (armed && p == CrashPoint::before_transaction)
|
|
throw Crash{};
|
|
};
|
|
auto h = host(r, c);
|
|
SchedulerTime t = std::chrono::steady_clock::now();
|
|
CHECK(h->step(t).ready);
|
|
for (std::uint64_t tick = 1; tick <= 30; ++tick)
|
|
CHECK(h->submit(input(tick, 0x72)));
|
|
CHECK(h->submit_action({carol, 10, 5, 1}));
|
|
armed = true;
|
|
bool crashed = false;
|
|
try {
|
|
step_until(*h, t, [](const HostStatus& s) { return s.committed_sequence >= 1; });
|
|
} catch (const Crash&) {
|
|
crashed = true;
|
|
}
|
|
CHECK(crashed);
|
|
h.reset();
|
|
auto successor = host(r, config(r));
|
|
step_until(*successor, t, [](const HostStatus& s) { return s.ready; });
|
|
CHECK(successor->status().committed_sequence == 0 && stored(*owner, carol, 10).empty());
|
|
}
|
|
{
|
|
// Crash after the commit, before any acknowledgement: the record is durable.
|
|
const auto r = provisioned(4);
|
|
bool armed = false;
|
|
auto c = config(r);
|
|
c.crash_hook = [&](CrashPoint p) {
|
|
if (armed && p == CrashPoint::after_commit)
|
|
throw Crash{};
|
|
};
|
|
auto h = host(r, c);
|
|
SchedulerTime t = std::chrono::steady_clock::now();
|
|
CHECK(h->step(t).ready);
|
|
for (std::uint64_t tick = 1; tick <= 30; ++tick)
|
|
CHECK(h->submit(input(tick, 0x73)));
|
|
CHECK(h->submit_action({carol, 11, 5, 1}));
|
|
armed = true;
|
|
bool crashed = false;
|
|
try {
|
|
step_until(*h, t, [](const HostStatus& s) { return s.committed_sequence >= 1; });
|
|
} catch (const Crash&) {
|
|
crashed = true;
|
|
}
|
|
CHECK(crashed && h->take_action_acks().empty());
|
|
h.reset();
|
|
CHECK(stored(*owner, carol, 11) == "134:5:1");
|
|
auto successor = host(r, config(r));
|
|
step_until(*successor, t, [](const HostStatus& s) { return s.ready; });
|
|
CHECK(successor->status().committed_sequence == 1);
|
|
CHECK(successor->submit_action({carol, 11, 40, 1})); // The host does not deduplicate
|
|
// across processes: admission does (find_region_action), and the database refuses a
|
|
// second commit of the id (store_rules).
|
|
}
|
|
{
|
|
// Unknown publication outcome: resolved by the exact bytes, actions included.
|
|
const auto r = provisioned(5);
|
|
auto h = host(r, config(r));
|
|
SchedulerTime t = std::chrono::steady_clock::now();
|
|
CHECK(h->step(t).ready);
|
|
for (std::uint64_t tick = 1; tick <= 30; ++tick)
|
|
CHECK(h->submit(input(tick, 0x74)));
|
|
CHECK(h->submit_action({dave, 12, 2, 1}));
|
|
step_until(*h, t, [](const HostStatus& s) { return s.simulated_tick >= 2; });
|
|
auto lock = connect_as(env("PGUSER"));
|
|
const std::array<std::string, 2> key{std::to_string(r.key.world),
|
|
std::to_string(r.key.region)};
|
|
CHECK(lock->query("BEGIN", {}, 5s));
|
|
CHECK(lock->query("SELECT 1 FROM evolution.region_leases WHERE world_id = "
|
|
"$1::evolution.u64 AND region_id = $2::evolution.u64 FOR UPDATE",
|
|
key, 5s));
|
|
step_until(*h, t, [](const HostStatus& s) { return s.phase == HostPhase::resolving; });
|
|
CHECK(h->take_action_acks().empty() && h->take_dropped_actions().empty());
|
|
CHECK(lock->query("ROLLBACK", {}, 5s));
|
|
step_until(*h, t, [](const HostStatus& s) { return s.phase == HostPhase::running; });
|
|
const auto acks = h->take_action_acks();
|
|
CHECK(acks.size() == 1 && acks[0].record.action_id == 12 && acks[0].commit_sequence == 1);
|
|
CHECK(stored(*owner, dave, 12) == "135:2:1");
|
|
}
|
|
}
|
|
} // namespace
|
|
|
|
void run_region_action_tests() {
|
|
store_rules();
|
|
host_actions();
|
|
std::puts("region action integration passed");
|
|
}
|