Add M10.4b durable action records (schema v11)
Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 1m47s
Foundation / checks (network) (push) Successful in 9m30s
Foundation / checks (client) (push) Successful in 12m19s
Foundation / checks (tsan) (push) Successful in 7m43s
Foundation / checks (checks) (push) Successful in 31m29s
Foundation / checks (postgres) (push) Successful in 36m39s
Foundation / postgres-tools (push) Successful in 19s
Foundation / image (push) Successful in 2m15s
Foundation / server-image (push) Successful in 20m37s

Each batch's actions are recorded in the same statement as its checkpoint: one
region_actions row per (world, account, action id) and the block in an append-only
table, compared on exact retries and resolution. The region host attaches actions to
the ticks they ride and reports acknowledgements and drops, keeping unknown-outcome
batches unresolved. Outcomes come from the durable record via ActionLedger, so
crashes before or after a commit, unknown publications and cross-region moves all
resolve to one answer; a repeated id keeps its first record instead of stalling a
region. ADR-0092.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dingo
2026-10-03 22:53:38 -04:00
co-authored by Claude Opus 5.5
parent e2f6208433
commit 69f3ad7cf8
32 changed files with 1289 additions and 75 deletions
+93 -14
View File
@@ -412,6 +412,85 @@ void session_codec() {
CHECK(refused(decode_session(encode_session(w)), SessionCodecError::invalid_value));
}
}
// Scripted durable action records (10.4b).
struct FakeActions final : ActionLedger {
std::expected<std::optional<ActionOutcome>, ActionLedgerError>
find(const AccountId& who, std::uint64_t action_id) override {
++lookups;
if (unavailable)
return std::unexpected(ActionLedgerError::unavailable);
for (const auto& [account, outcome] : records)
if (account == who && outcome.action_id == action_id)
return std::optional{outcome};
return std::optional<ActionOutcome>{};
}
std::vector<std::pair<AccountId, ActionOutcome>> records;
bool unavailable{};
int lookups{};
};
void durable_actions() {
auto a = *IntentAdmission::create(IntentLimits{});
FakeActions ledger;
a.attach(&ledger);
const auto alice = account(1);
CHECK(a.open(1, alice));
AdmissionFacts f{true, 5, 100};
const auto action = [&](std::uint64_t id, std::uint64_t tick) {
return a.action(1, SessionAction{1, 5, id, 1, tick}, f);
};
// Committed by an earlier epoch, process or region: a duplicate with its outcome.
const ActionOutcome old{9, ActionResult::committed_no_effect, 40, 3};
ledger.records.emplace_back(alice, old);
auto r = action(9, 101);
CHECK(r.status == AdmitStatus::duplicate && r.tick == 40);
CHECK(a.query(1, {9}, f) == old);
// Uniqueness that cannot be confirmed is not admitted.
ledger.unavailable = true;
CHECK(action(10, 101).status == AdmitStatus::not_ready);
ledger.unavailable = false;
const int before = ledger.lookups;
CHECK(action(10, 101).status == AdmitStatus::admitted && ledger.lookups == before + 1);
// A refused attempt never costs a lookup.
CHECK(action(11, 150).status == AdmitStatus::outside_window && ledger.lookups == before + 1);
// Exact commit: the tick reported is the tick the action rode.
auto riding = a.carry(101);
CHECK(riding.size() == 1);
riding[0].tick = 102;
auto outcomes = a.committed(riding, 6);
CHECK(outcomes.size() == 1 &&
outcomes[0].outcome == ActionOutcome{10, ActionResult::committed_no_effect, 102, 6});
// A final drop is not_committed; an unknown one stays pending until its record decides.
f.server_tick = 108;
CHECK(action(12, 109).status == AdmitStatus::admitted);
CHECK(action(13, 109).status == AdmitStatus::admitted);
riding = a.carry(109);
CHECK(riding.size() == 2);
outcomes = a.dropped(std::span{riding}.first(1), true);
CHECK(outcomes.size() == 1 && outcomes[0].outcome.result == ActionResult::not_committed);
CHECK(a.dropped(std::span{riding}.subspan(1), false).empty());
f.ready = false; // Recovering: nothing is decided yet.
CHECK(a.query(1, {13}, f).result == ActionResult::pending);
f.ready = true;
ledger.records.emplace_back(alice,
ActionOutcome{13, ActionResult::committed_no_effect, 109, 7});
CHECK(a.query(1, {13}, f) == ActionOutcome{13, ActionResult::committed_no_effect, 109, 7});
// The same for an action whose record is absent once the new authority is ready.
f.server_tick = 116;
CHECK(action(14, 117).status == AdmitStatus::admitted);
riding = a.carry(117);
CHECK(a.dropped(riding, false).empty());
CHECK(a.query(1, {14}, f) == ActionOutcome{14, ActionResult::not_committed, 0, 0});
// Outcomes evicted from memory are found again on record.
auto small = IntentLimits{};
small.max_finals = 1;
auto b = *IntentAdmission::create(small);
b.attach(&ledger);
CHECK(b.open(1, alice));
CHECK(b.query(1, {13}, f).result == ActionResult::committed_no_effect);
CHECK(b.query(1, {99}, f).result == ActionResult::unknown);
}
void intent_admission() {
IntentLimits limits;
limits.max_unresolved = 6;
@@ -465,7 +544,7 @@ void intent_admission() {
f.server_tick = 200;
auto r = action(1, 10, 203);
CHECK(r.status == AdmitStatus::admitted && r.tick == 203 && r.server_tick == 200);
CHECK(a.query(1, {10}).result == ActionResult::pending);
CHECK(a.query(1, {10}, f).result == ActionResult::pending);
CHECK(action(1, 0, 203).status == AdmitStatus::invalid_action);
CHECK(action(1, 11, 203, 9).status == AdmitStatus::invalid_action);
{ // Wire kind 257 must not truncate to a known kind.
@@ -481,23 +560,23 @@ void intent_admission() {
const auto carried = a.carry(203);
CHECK(carried.size() == 1 && carried[0] == CarriedAction{alice, 10, 203, ActionKind::probe});
CHECK(a.carry(203).empty()); // Each action is carried once.
CHECK(a.committed(202, 7).empty());
auto outcomes = a.committed(203, 8);
CHECK(a.committed({}, 7).empty());
auto outcomes = a.committed(carried, 8);
CHECK(outcomes.size() == 1 && outcomes[0].account == alice &&
outcomes[0].outcome == ActionOutcome{10, ActionResult::committed_no_effect, 203, 8});
CHECK(a.query(1, {10}) == outcomes[0].outcome);
CHECK(a.query(2, {10}).result == ActionResult::unknown); // Bob's account has no action 10.
CHECK(a.query(1, {10}, f) == outcomes[0].outcome);
CHECK(a.query(2, {10}, f).result == ActionResult::unknown); // Bob's account has no action 10.
// A reconnect of the same account (new session) sees the same outcome.
a.close(1);
CHECK(a.query(1, {10}).result == ActionResult::unknown); // Closed session asks nothing.
CHECK(a.query(1, {10}, f).result == ActionResult::unknown); // Closed session asks nothing.
CHECK(a.open(3, alice) && a.session_of(alice) == 3u);
CHECK(a.query(3, {10}) == outcomes[0].outcome);
CHECK(a.query(3, {10}, f) == outcomes[0].outcome);
f.server_tick = 300;
CHECK(action(3, 10, 301).status == AdmitStatus::duplicate); // Never admitted twice.
// Unresolved cap counts intentions and actions; abandon finalizes only uncarried actions.
CHECK(action(3, 20, 301).status == AdmitStatus::admitted);
CHECK(action(3, 21, 308).status == AdmitStatus::admitted);
(void)a.carry(301);
const auto riding = a.carry(301);
for (std::uint64_t i = 0; i < 4; ++i)
CHECK(a.intention(3, SessionIntention{3, 5, i + 1, 302 + i, 0, 0}, f).status ==
AdmitStatus::admitted);
@@ -507,18 +586,17 @@ void intent_admission() {
outcomes = a.abandon();
CHECK(outcomes.size() == 1 &&
outcomes[0].outcome == ActionOutcome{21, ActionResult::not_committed, 0, 0});
CHECK(a.query(3, {20}).result == ActionResult::pending); // Carried: awaits its record.
CHECK(a.query(3, {20}, f).result == ActionResult::pending); // Carried: awaits its record.
CHECK(a.unresolved(3) == 1);
CHECK(a.committed(301, 9).size() == 1); // Its batch turned out durable.
CHECK(a.committed(riding, 9).size() == 1); // Its batch turned out durable.
// Final outcomes are kept within a bound; older ones become unknown here.
for (std::uint64_t id = 30; id < 34; ++id) {
f.server_tick += 8;
CHECK(action(3, id, f.server_tick + 1).status == AdmitStatus::admitted);
(void)a.carry(f.server_tick + 1);
CHECK(a.committed(f.server_tick + 1, id).size() == 1);
CHECK(a.committed(a.carry(f.server_tick + 1), id).size() == 1);
}
CHECK(a.query(3, {10}).result == ActionResult::unknown);
CHECK(a.query(3, {33}).result == ActionResult::committed_no_effect);
CHECK(a.query(3, {10}, f).result == ActionResult::unknown);
CHECK(a.query(3, {33}, f).result == ActionResult::committed_no_effect);
}
} // namespace
@@ -528,5 +606,6 @@ void control_tests() {
control_rules();
session_codec();
intent_admission();
durable_actions();
}
} // namespace evo::network_test
+2
View File
@@ -5,6 +5,7 @@
#include <stdexcept>
void run_region_lease_store_tests();
void run_account_control_store_tests();
void run_region_action_tests();
void run_region_checkpoint_store_tests();
void run_region_host_tests();
void run_region_server_process_tests();
@@ -118,6 +119,7 @@ int main() {
std::puts("postgres adapter integration passed");
run_region_lease_store_tests();
run_account_control_store_tests();
run_region_action_tests();
std::puts("region lease fencing integration passed");
run_region_checkpoint_store_tests();
std::puts("region checkpoint publication integration passed");
+1 -1
View File
@@ -406,7 +406,7 @@ void intentions_and_actions(const Files& f) {
CHECK(h.server->carry(102).empty());
const auto carried = h.server->carry(103);
CHECK(carried.size() == 1 && carried[0] == CarriedAction{alice, 7, 103, ActionKind::probe});
h.server->committed(103, 4);
h.server->committed(carried, 4);
CHECK(h.until([&] { return !(outcomes = a->take_action_outcomes()).empty(); }));
CHECK(outcomes[0] == ActionOutcome{7, ActionResult::committed_no_effect, 103, 4});
// A retry gets a duplicate receipt and the same final outcome.
+412
View File
@@ -0,0 +1,412 @@
// Real PostgreSQL M10.4b durable action records (ADR-0092, schema v11): actions committed in
// the same statement as their batch's checkpoint, exact retries and resolution including
// the action block, invalid blocks, an action id never committed twice, and host crashes
// before and after the commit. Regions 7/131..135 and commit ids tagged 0xA7 are used only
// here. Run by
// tools/ci/postgres-check.sh.
#include "evo/persistence/checkpoint_store.hpp"
#include "evo/persistence/region_lease_store.hpp"
#include "evo/server/region_host.hpp"
#include "fixtures/environmental_region.hpp"
#ifdef EVO_POSTGRES_SESSIONS
#include "evo/server/postgres_action_ledger.hpp"
#endif
#include <array>
#include <cstdio>
#include <cstdlib>
#include <filesystem>
#include <source_location>
#include <stdexcept>
#include <string>
#include <thread>
#include <vector>
void run_region_action_tests();
namespace {
using namespace evo::persistence;
using namespace evo::server;
using namespace std::chrono_literals;
using evo::processes::LineageRegion;
namespace fs = std::filesystem;
void check(bool value, std::source_location at = std::source_location::current()) {
if (!value)
throw std::runtime_error("region action test line " + std::to_string(at.line()));
}
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
template <typename T, typename E> bool holds(const std::expected<T, E>& e, const T& v) {
return e.has_value() && *e == v;
}
constexpr std::uint64_t world = 7; // The fixtures' world.
constexpr std::uint64_t base_region = 130;
constexpr std::size_t budget = 8000000;
std::string env(const char* key) {
const auto* value = std::getenv(key);
if (!value)
throw std::runtime_error("missing test environment");
return value;
}
PgOptions options(const std::string& user) {
return {env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}};
}
std::unique_ptr<Postgres> connect_as(const std::string& user) {
auto made = Postgres::connect(options(user), 5s);
CHECK(made);
return std::move(*made);
}
template <std::size_t N> std::array<std::uint8_t, N> filled(std::uint8_t value) {
std::array<std::uint8_t, N> out{};
out.fill(value);
return out;
}
const Sha256Digest build = filled<32>(0xB1);
const Uuid alice = filled<16>(0xA1), bob = filled<16>(0xB0);
const Uuid carol = filled<16>(0xC1), dave = filled<16>(0xD0); // Host tests only.
// Commit ids are unique across the database: tag this file's.
Uuid tagged(std::uint8_t n) {
Uuid id = filled<16>(0xA7);
id.back() = n;
return id;
}
struct Region {
RegionKey key;
fs::path root;
CommittedCheckpoint genesis;
};
Region provisioned(std::uint64_t region) {
region += base_region;
Region r{{world, region}, fs::path(env("PGHOST")) / ("actions-" + std::to_string(region)), {}};
fs::remove_all(r.root);
fs::create_directory(r.root);
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
auto owner = connect_as(env("PGUSER"));
CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
{}, 5s));
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
CHECK(objects);
auto genesis = provision_lineage_region(
*owner, **objects, r.key, evo::test::region_identity.frame, *evo::test::lineage_region(),
tagged(static_cast<std::uint8_t>(region)), build, budget, 5s);
if (!genesis)
std::fprintf(stderr, "provision failed: kind %d sqlstate %s\n",
static_cast<int>(genesis.error().kind),
genesis.error().store.cause.sqlstate.data());
CHECK(genesis);
r.genesis = *genesis;
return r;
}
std::vector<std::uint8_t> record(const Uuid& account, std::uint64_t id, std::uint64_t tick,
std::uint16_t kind = 1) {
std::vector<std::uint8_t> out(account.begin(), account.end());
for (unsigned b = 0; b < 8; ++b)
out.push_back(static_cast<std::uint8_t>(id >> (8U * b)));
for (unsigned b = 0; b < 8; ++b)
out.push_back(static_cast<std::uint8_t>(tick >> (8U * b)));
out.push_back(static_cast<std::uint8_t>(kind));
out.push_back(static_cast<std::uint8_t>(kind >> 8U));
return out;
}
std::vector<std::uint8_t> join(std::initializer_list<std::vector<std::uint8_t>> parts) {
std::vector<std::uint8_t> out;
for (const auto& p : parts)
out.insert(out.end(), p.begin(), p.end());
return out;
}
// Owner read of the durable record: (region, tick, commit) or empty.
std::string stored(Postgres& owner, const Uuid& account, std::uint64_t id) {
const std::array<std::string, 2> p{std::string{"\\x"} +
[&] {
std::string h;
constexpr char digits[] = "0123456789abcdef";
for (const auto b : account) {
h.push_back(digits[b >> 4U]);
h.push_back(digits[b & 0xFU]);
}
return h;
}(),
std::to_string(id)};
auto rows = owner.query("SELECT region_id || ':' || tick || ':' || commit_sequence FROM "
"evolution.region_actions WHERE world_id = 7 AND account_id = "
"encode($1::bytea, 'hex')::uuid AND action_id = $2::evolution.u64",
p, 5s);
CHECK(rows);
return rows->empty() ? std::string{} : (*rows)[0][0].value_or("");
}
void store_rules() {
auto r = provisioned(1);
auto db = connect_as(env("EVO_PG_RUNTIME_USER"));
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
CHECK(objects);
auto lease = acquire_region_lease(*db, r.key, build, 20000ms, 5s);
CHECK(lease);
auto state = evo::test::lineage_region();
RegionBatch<LineageRegion> batch{*state};
for (std::uint64_t tick = 1; tick <= 10; ++tick)
CHECK(batch.advance(evo::test::region_input(tick, false)));
batch.seal();
const std::vector<ActionRecord> actions{{alice, 2, 10, 1}, {bob, 1, 3, 1}, {alice, 1, 3, 1}};
auto first = publish_lineage_batch(*db, **objects, *lease, r.genesis, batch, tagged(0x01),
build, budget, 5s, {}, actions);
if (!first)
std::fprintf(stderr, "publish failed: kind %d store %d sqlstate %s\n",
static_cast<int>(first.error().kind),
static_cast<int>(first.error().store.kind),
first.error().store.cause.sqlstate.data());
CHECK(first && first->committed.head.sequence == 1);
CHECK(batch.confirm(10));
auto found = find_region_action(*db, r.key, *lease, alice, 1, 5s);
CHECK(found && *found == CommittedAction{131, 3, 1});
CHECK(holds(find_region_action(*db, r.key, *lease, bob, 1, 5s),
std::optional{CommittedAction{131, 3, 1}}));
CHECK(holds(find_region_action(*db, r.key, *lease, alice, 2, 5s),
std::optional{CommittedAction{131, 10, 1}}));
CHECK(holds(find_region_action(*db, r.key, *lease, alice, 3, 5s),
std::optional<CommittedAction>{}));
// Exact retry and resolution include the action block.
const auto block = encode_actions(actions, 0, 10);
CHECK(block);
PreparedCommit prepared{r.key, tagged(0x01), first->committed.manifest, {}, *block};
CHECK(retry_prepared(*db, prepared, 5s) == first->committed.head);
auto resolved = resolve_prepared(*db, prepared, 5s);
CHECK(resolved && resolved->status == Resolution::committed);
auto changed = prepared;
changed.actions = join({record(alice, 1, 3), record(alice, 2, 10)}); // One missing.
auto retried = retry_prepared(*db, changed, 5s);
CHECK(!retried && retried.error().kind == LeaseStoreError::commit_conflict);
resolved = resolve_prepared(*db, changed, 5s);
CHECK(resolved && resolved->status == Resolution::conflict);
// Every invalid block is refused before anything else.
for (const auto& bad :
{record(alice, 9, 11), record(alice, 9, 0), record(alice, 9, 5, 2), record(Uuid{}, 9, 5),
record(alice, 0, 5), join({record(bob, 9, 5), record(alice, 9, 5)}),
join({record(alice, 9, 5), record(alice, 9, 5)}),
join({record(alice, 9, 6), record(alice, 9, 5)}),
join({record(alice, 9, 5), record(alice, 9, 6)}), // One id at two ticks.
std::vector<std::uint8_t>(33, 1)}) {
auto p = prepared;
p.actions = bad;
retried = retry_prepared(*db, p, 5s);
CHECK(!retried && retried.error().kind == LeaseStoreError::invalid);
}
// The local encoder refuses the same blocks it would send.
CHECK(!encode_actions(std::vector<ActionRecord>{{alice, 9, 11, 1}}, 0, 10));
CHECK(!encode_actions(std::vector<ActionRecord>{{alice, 9, 5, 1}, {alice, 9, 6, 1}}, 0, 10));
// An action id is recorded once in a world: a later batch carrying it commits, and the
// first record stands.
RegionBatch<LineageRegion> next{*state};
for (std::uint64_t tick = 11; tick <= 20; ++tick)
CHECK(next.advance(evo::test::region_input(tick, false)));
next.seal();
const std::vector<ActionRecord> repeat{{alice, 1, 12, 1}, {alice, 3, 12, 1}};
auto second = publish_lineage_batch(*db, **objects, *lease, first->committed, next,
tagged(0x03), build, budget, 5s, {}, repeat);
CHECK(second && second->committed.head.sequence == 2);
CHECK(holds(find_region_action(*db, r.key, *lease, alice, 1, 5s),
std::optional{CommittedAction{131, 3, 1}}));
// A batch without actions still publishes as before v11.
CHECK(next.confirm(20));
RegionBatch<LineageRegion> plain{*state};
for (std::uint64_t tick = 21; tick <= 25; ++tick)
CHECK(plain.advance(evo::test::region_input(tick, false)));
plain.seal();
CHECK(publish_lineage_batch(*db, **objects, *lease, second->committed, plain, tagged(0x04),
build, budget, 5s));
// Lookups are fenced like everything else under the lease.
CHECK(release_region_lease(*db, r.key, *lease, 5s));
auto later = acquire_region_lease(*db, r.key, build, 20000ms, 5s);
CHECK(later);
auto fenced = find_region_action(*db, r.key, *lease, alice, 1, 5s);
CHECK(!fenced && fenced.error().kind == LeaseStoreError::fenced);
CHECK(holds(find_region_action(*db, r.key, *later, alice, 3, 5s),
std::optional{CommittedAction{131, 12, 2}}));
#ifdef EVO_POSTGRES_SESSIONS
// The session layer's ledger over the same lookup.
std::optional<LeaseGrant> current = *later;
PostgresActionLedger ledger{*db, r.key, [&] { return current; }};
evo::network::AccountId who;
who.bytes = alice;
CHECK(holds(ledger.find(who, 3),
std::optional{evo::network::ActionOutcome{
3, evo::network::ActionResult::committed_no_effect, 12, 2}}));
CHECK(holds(ledger.find(who, 99), std::optional<evo::network::ActionOutcome>{}));
current.reset();
CHECK(!ledger.find(who, 3));
#endif
CHECK(release_region_lease(*db, r.key, *later, 5s));
}
// Host-level: fast lease (4 s), 1 s commits, as in the host tests.
const LeasePolicy fast{4000ms, 800ms, 400ms, 400ms, 1000ms};
struct Crash {};
HostConfig config(const Region& r) {
HostConfig c;
c.key = r.key;
c.frame = evo::test::region_identity.frame;
c.lease = fast;
c.build = build;
c.retained_builds = {build};
c.max_queued_actions = 3;
return c;
}
std::unique_ptr<RegionHost<LineageRegion>> host(const Region& r, HostConfig c) {
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
CHECK(objects);
auto made = RegionHost<LineageRegion>::create(
std::move(c), [] { return Postgres::connect(options(env("EVO_PG_RUNTIME_USER")), 2s); },
std::move(*objects));
CHECK(made);
return std::move(*made);
}
TickInput input(std::uint64_t tick, std::uint8_t tag) {
Uuid request{};
request[0] = tag;
for (unsigned i = 0; i < 8; ++i)
request[8 + i] = static_cast<std::uint8_t>((tick >> (8U * i)) & 0xFFU);
return {request, evo::test::region_input(tick, false)};
}
template <typename Done>
const HostStatus& step_until(RegionHost<LineageRegion>& h, SchedulerTime& t, Done done,
int limit = 1000) {
for (int i = 0; i < limit; ++i) {
t += 50ms;
const auto& s = h.step(t);
if (done(s))
return s;
std::this_thread::sleep_for(10ms);
}
CHECK(false);
return h.status();
}
void host_actions() {
auto owner = connect_as(env("PGUSER"));
{
// Normal path: acknowledged with the tick each rode; late actions ride the next tick.
const auto r = provisioned(2);
auto h = host(r, config(r));
CHECK(h->submit_action({carol, 1, 3, 1}).error() == AdmissionError::not_ready);
SchedulerTime t = std::chrono::steady_clock::now();
CHECK(h->step(t).ready);
CHECK(h->submit_action({carol, 1, 3, 1}));
CHECK(h->submit_action({carol, 1, 4, 1}).error() == AdmissionError::duplicate);
CHECK(h->submit_action({carol, 0, 4, 1}).error() == AdmissionError::invalid);
CHECK(h->submit_action({carol, 5, 4, 2}).error() == AdmissionError::invalid);
CHECK(h->submit_action({dave, 1, 1, 1}));
CHECK(h->submit_action({dave, 2, 2, 1}));
CHECK(h->submit_action({dave, 3, 2, 1}).error() == AdmissionError::full);
for (std::uint64_t tick = 1; tick <= 30; ++tick)
CHECK(h->submit(input(tick, 0x71)));
step_until(*h, t, [](const HostStatus& s) { return s.committed_sequence >= 1; });
auto acks = h->take_action_acks();
CHECK(acks.size() == 3);
for (const auto& a : acks)
CHECK(a.commit_sequence == 1 &&
a.record.tick == (a.record.account == dave ? a.record.action_id : 3u));
CHECK(stored(*owner, carol, 1) == "132:3:1");
// Submitted after tick 20 ran for tick 5: rides the first tick simulated next.
const auto ran = h->status().simulated_tick;
CHECK(h->submit_action({carol, 7, 5, 1}));
step_until(*h, t, [](const HostStatus& s) { return s.committed_sequence >= 2; });
acks = h->take_action_acks();
CHECK(acks.size() == 1 && acks[0].record.tick == ran + 1);
CHECK(h->take_dropped_actions().empty());
}
{
// Crash before the transaction: never committed; a successor finds no record.
const auto r = provisioned(3);
bool armed = false;
auto c = config(r);
c.crash_hook = [&](CrashPoint p) {
if (armed && p == CrashPoint::before_transaction)
throw Crash{};
};
auto h = host(r, c);
SchedulerTime t = std::chrono::steady_clock::now();
CHECK(h->step(t).ready);
for (std::uint64_t tick = 1; tick <= 30; ++tick)
CHECK(h->submit(input(tick, 0x72)));
CHECK(h->submit_action({carol, 10, 5, 1}));
armed = true;
bool crashed = false;
try {
step_until(*h, t, [](const HostStatus& s) { return s.committed_sequence >= 1; });
} catch (const Crash&) {
crashed = true;
}
CHECK(crashed);
h.reset();
auto successor = host(r, config(r));
step_until(*successor, t, [](const HostStatus& s) { return s.ready; });
CHECK(successor->status().committed_sequence == 0 && stored(*owner, carol, 10).empty());
}
{
// Crash after the commit, before any acknowledgement: the record is durable.
const auto r = provisioned(4);
bool armed = false;
auto c = config(r);
c.crash_hook = [&](CrashPoint p) {
if (armed && p == CrashPoint::after_commit)
throw Crash{};
};
auto h = host(r, c);
SchedulerTime t = std::chrono::steady_clock::now();
CHECK(h->step(t).ready);
for (std::uint64_t tick = 1; tick <= 30; ++tick)
CHECK(h->submit(input(tick, 0x73)));
CHECK(h->submit_action({carol, 11, 5, 1}));
armed = true;
bool crashed = false;
try {
step_until(*h, t, [](const HostStatus& s) { return s.committed_sequence >= 1; });
} catch (const Crash&) {
crashed = true;
}
CHECK(crashed && h->take_action_acks().empty());
h.reset();
CHECK(stored(*owner, carol, 11) == "134:5:1");
auto successor = host(r, config(r));
step_until(*successor, t, [](const HostStatus& s) { return s.ready; });
CHECK(successor->status().committed_sequence == 1);
CHECK(successor->submit_action({carol, 11, 40, 1})); // The host does not deduplicate
// across processes: admission does (find_region_action), and the database refuses a
// second commit of the id (store_rules).
}
{
// Unknown publication outcome: resolved by the exact bytes, actions included.
const auto r = provisioned(5);
auto h = host(r, config(r));
SchedulerTime t = std::chrono::steady_clock::now();
CHECK(h->step(t).ready);
for (std::uint64_t tick = 1; tick <= 30; ++tick)
CHECK(h->submit(input(tick, 0x74)));
CHECK(h->submit_action({dave, 12, 2, 1}));
step_until(*h, t, [](const HostStatus& s) { return s.simulated_tick >= 2; });
auto lock = connect_as(env("PGUSER"));
const std::array<std::string, 2> key{std::to_string(r.key.world),
std::to_string(r.key.region)};
CHECK(lock->query("BEGIN", {}, 5s));
CHECK(lock->query("SELECT 1 FROM evolution.region_leases WHERE world_id = "
"$1::evolution.u64 AND region_id = $2::evolution.u64 FOR UPDATE",
key, 5s));
step_until(*h, t, [](const HostStatus& s) { return s.phase == HostPhase::resolving; });
CHECK(h->take_action_acks().empty() && h->take_dropped_actions().empty());
CHECK(lock->query("ROLLBACK", {}, 5s));
step_until(*h, t, [](const HostStatus& s) { return s.phase == HostPhase::running; });
const auto acks = h->take_action_acks();
CHECK(acks.size() == 1 && acks[0].record.action_id == 12 && acks[0].commit_sequence == 1);
CHECK(stored(*owner, dave, 12) == "135:2:1");
}
}
} // namespace
void run_region_action_tests() {
store_rules();
host_actions();
std::puts("region action integration passed");
}
+1 -1
View File
@@ -582,7 +582,7 @@ void durable_outcomes_and_ambiguity() {
// receipt and records nothing twice; resolution confirms it under any epoch.
auto death_block = encode_decisions(died->decisions, 0, 20);
CHECK(death_block);
const PreparedCommit lost{r.key, filled<16>(0xF1), died->committed.manifest, *death_block};
const PreparedCommit lost{r.key, filled<16>(0xF1), died->committed.manifest, *death_block, {}};
auto retried = retry_prepared(*r.runtime, lost, 5s);
CHECK(retried && *retried == died->committed.head);
auto resolved = resolve_prepared(*r.runtime, lost, 5s);
+2 -1
View File
@@ -176,6 +176,7 @@ void upgrade_across_migration_and_barrier() {
" -v migration_sha_7=$(s 0007_region_clones.sql) -v migration_sha_8=invalid"
" -v migration_sha_9=$(s 0009_serialized_authority.sql)"
" -v migration_sha_10=$(s 0010_account_control.sql)"
" -v migration_sha_11=$(s 0011_region_actions.sql)"
" -v runtime_role=evo_runtime_test -f migrations/apply.sql"
" > /dev/null 2>&1") != 0);
CHECK(history(*owner_db) == "1,2,3,4,5,6,7");
@@ -191,7 +192,7 @@ void upgrade_across_migration_and_barrier() {
CHECK(shell("PGDATABASE=" + database +
" EVO_PG_RUNTIME_ROLE=evo_runtime_test bash \"$EVO_SOURCE_ROOT/"
"tools/migrate-postgres.sh\" > /dev/null") == 0);
CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8,9,10");
CHECK(history(*owner_db) == "1,2,3,4,5,6,7,8,9,10,11");
CHECK(renew_region_lease(*runtime, r.key, *lease, 20000ms, 5s));
confirm(publish_next(20)); // 41..60, now v8.
CHECK(head.head.end_tick == 60);