Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 1m46s
Foundation / checks (postgres) (push) Canceled after 0s
Foundation / checks (tsan) (push) Canceled after 0s
Foundation / postgres-tools (push) Canceled after 0s
Foundation / server-image (push) Canceled after 0s
Foundation / image (push) Canceled after 0s
Foundation / checks (checks) (push) Canceled after 3m44s
Foundation / checks (client) (push) Canceled after 3m49s
Foundation / checks (network) (push) Canceled after 1m59s
evo-server gains an opt-in M10 listener (--m10-*, QUIC builds only): RegionSessions with the PostgreSQL control and action ledgers on their own connection under the host's lease, carried actions submitted to the host and its acknowledgements and drops turned into outcomes. evo-session-probe is a dev-only headless client. Real process tests cover intentions, durable action outcomes, SIGKILL before and after an action's commit, duplicate retries and the probe. The simulated watermark now reaches every session every tick, so clients always know the server tick. ADR-0092. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
414 lines
19 KiB
C++
414 lines
19 KiB
C++
// Real PostgreSQL M10.4b durable action records (ADR-0092, schema v11): actions committed in
|
|
// the same statement as their batch's checkpoint, exact retries and resolution including
|
|
// the action block, invalid blocks, an action id never committed twice, and host crashes
|
|
// before and after the commit. Regions 7/131..135 and commit ids tagged 0xA7 are used only
|
|
// here. Run by
|
|
// tools/ci/postgres-check.sh.
|
|
#include "evo/persistence/checkpoint_store.hpp"
|
|
#include "evo/persistence/region_lease_store.hpp"
|
|
#include "evo/server/region_host.hpp"
|
|
#include "fixtures/environmental_region.hpp"
|
|
#ifdef EVO_POSTGRES_SESSIONS
|
|
#include "evo/server/postgres_action_ledger.hpp"
|
|
#endif
|
|
#include <array>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <filesystem>
|
|
#include <source_location>
|
|
#include <stdexcept>
|
|
#include <string>
|
|
#include <thread>
|
|
#include <vector>
|
|
|
|
void run_region_action_tests();
|
|
|
|
namespace {
|
|
using namespace evo::persistence;
|
|
using namespace evo::server;
|
|
using namespace std::chrono_literals;
|
|
using evo::processes::LineageRegion;
|
|
namespace fs = std::filesystem;
|
|
void check(bool value, std::source_location at = std::source_location::current()) {
|
|
if (!value)
|
|
throw std::runtime_error("region action test line " + std::to_string(at.line()));
|
|
}
|
|
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
|
template <typename T, typename E> bool holds(const std::expected<T, E>& e, const T& v) {
|
|
return e.has_value() && *e == v;
|
|
}
|
|
constexpr std::uint64_t world = 7; // The fixtures' world.
|
|
constexpr std::uint64_t base_region = 130;
|
|
constexpr std::size_t budget = 8000000;
|
|
|
|
std::string env(const char* key) {
|
|
const auto* value = std::getenv(key);
|
|
if (!value)
|
|
throw std::runtime_error("missing test environment");
|
|
return value;
|
|
}
|
|
PgOptions options(const std::string& user) {
|
|
return {env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}};
|
|
}
|
|
std::unique_ptr<Postgres> connect_as(const std::string& user) {
|
|
auto made = Postgres::connect(options(user), 5s);
|
|
CHECK(made);
|
|
return std::move(*made);
|
|
}
|
|
template <std::size_t N> std::array<std::uint8_t, N> filled(std::uint8_t value) {
|
|
std::array<std::uint8_t, N> out{};
|
|
out.fill(value);
|
|
return out;
|
|
}
|
|
const Sha256Digest build = filled<32>(0xB1);
|
|
const Uuid alice = filled<16>(0xA1), bob = filled<16>(0xB0);
|
|
const Uuid carol = filled<16>(0xC1), dave = filled<16>(0xD0); // Host tests only.
|
|
// Commit ids are unique across the database: tag this file's.
|
|
Uuid tagged(std::uint8_t n) {
|
|
Uuid id = filled<16>(0xA7);
|
|
id.back() = n;
|
|
return id;
|
|
}
|
|
|
|
struct Region {
|
|
RegionKey key;
|
|
fs::path root;
|
|
CommittedCheckpoint genesis;
|
|
};
|
|
Region provisioned(std::uint64_t region) {
|
|
region += base_region;
|
|
Region r{{world, region}, fs::path(env("PGHOST")) / ("actions-" + std::to_string(region)), {}};
|
|
fs::remove_all(r.root);
|
|
fs::create_directory(r.root);
|
|
fs::permissions(r.root, fs::perms::owner_all, fs::perm_options::replace);
|
|
auto owner = connect_as(env("PGUSER"));
|
|
CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
|
"(decode(repeat('b1', 32), 'hex')) ON CONFLICT DO NOTHING",
|
|
{}, 5s));
|
|
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
|
|
CHECK(objects);
|
|
auto genesis = provision_lineage_region(
|
|
*owner, **objects, r.key, evo::test::region_identity.frame, *evo::test::lineage_region(),
|
|
tagged(static_cast<std::uint8_t>(region)), build, budget, 5s);
|
|
if (!genesis)
|
|
std::fprintf(stderr, "provision failed: kind %d sqlstate %s\n",
|
|
static_cast<int>(genesis.error().kind),
|
|
genesis.error().store.cause.sqlstate.data());
|
|
CHECK(genesis);
|
|
r.genesis = *genesis;
|
|
return r;
|
|
}
|
|
std::vector<std::uint8_t> record(const Uuid& account, std::uint64_t id, std::uint64_t tick,
|
|
std::uint16_t kind = 1) {
|
|
std::vector<std::uint8_t> out(account.begin(), account.end());
|
|
for (unsigned b = 0; b < 8; ++b)
|
|
out.push_back(static_cast<std::uint8_t>(id >> (8U * b)));
|
|
for (unsigned b = 0; b < 8; ++b)
|
|
out.push_back(static_cast<std::uint8_t>(tick >> (8U * b)));
|
|
out.push_back(static_cast<std::uint8_t>(kind));
|
|
out.push_back(static_cast<std::uint8_t>(kind >> 8U));
|
|
return out;
|
|
}
|
|
std::vector<std::uint8_t> join(std::initializer_list<std::vector<std::uint8_t>> parts) {
|
|
std::vector<std::uint8_t> out;
|
|
for (const auto& p : parts)
|
|
out.insert(out.end(), p.begin(), p.end());
|
|
return out;
|
|
}
|
|
// Owner read of the durable record: (region, tick, commit) or empty.
|
|
std::string stored(Postgres& owner, const Uuid& account, std::uint64_t id) {
|
|
const std::array<std::string, 2> p{std::string{"\\x"} +
|
|
[&] {
|
|
std::string h;
|
|
constexpr char digits[] = "0123456789abcdef";
|
|
for (const auto b : account) {
|
|
h.push_back(digits[b >> 4U]);
|
|
h.push_back(digits[b & 0xFU]);
|
|
}
|
|
return h;
|
|
}(),
|
|
std::to_string(id)};
|
|
auto rows = owner.query("SELECT region_id || ':' || tick || ':' || commit_sequence FROM "
|
|
"evolution.region_actions WHERE world_id = 7 AND account_id = "
|
|
"encode($1::bytea, 'hex')::uuid AND action_id = $2::evolution.u64",
|
|
p, 5s);
|
|
CHECK(rows);
|
|
return rows->empty() ? std::string{} : (*rows)[0][0].value_or("");
|
|
}
|
|
|
|
void store_rules() {
|
|
auto r = provisioned(1);
|
|
auto db = connect_as(env("EVO_PG_RUNTIME_USER"));
|
|
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
|
|
CHECK(objects);
|
|
auto lease = acquire_region_lease(*db, r.key, build, 20000ms, 5s);
|
|
CHECK(lease);
|
|
auto state = evo::test::lineage_region();
|
|
RegionBatch<LineageRegion> batch{*state};
|
|
for (std::uint64_t tick = 1; tick <= 10; ++tick)
|
|
CHECK(batch.advance(evo::test::region_input(tick, false)));
|
|
batch.seal();
|
|
const std::vector<ActionRecord> actions{{alice, 2, 10, 1}, {bob, 1, 3, 1}, {alice, 1, 3, 1}};
|
|
auto first = publish_lineage_batch(*db, **objects, *lease, r.genesis, batch, tagged(0x01),
|
|
build, budget, 5s, {}, actions);
|
|
if (!first)
|
|
std::fprintf(stderr, "publish failed: kind %d store %d sqlstate %s\n",
|
|
static_cast<int>(first.error().kind),
|
|
static_cast<int>(first.error().store.kind),
|
|
first.error().store.cause.sqlstate.data());
|
|
CHECK(first && first->committed.head.sequence == 1);
|
|
CHECK(batch.confirm(10));
|
|
auto found = find_region_action(*db, r.key, *lease, alice, 1, 5s);
|
|
CHECK(found && *found == CommittedAction{131, 3, 1});
|
|
CHECK(holds(find_region_action(*db, r.key, *lease, bob, 1, 5s),
|
|
std::optional{CommittedAction{131, 3, 1}}));
|
|
CHECK(holds(find_region_action(*db, r.key, *lease, alice, 2, 5s),
|
|
std::optional{CommittedAction{131, 10, 1}}));
|
|
CHECK(holds(find_region_action(*db, r.key, *lease, alice, 3, 5s),
|
|
std::optional<CommittedAction>{}));
|
|
|
|
// Exact retry and resolution include the action block.
|
|
const auto block = encode_actions(actions, 0, 10);
|
|
CHECK(block);
|
|
PreparedCommit prepared{r.key, tagged(0x01), first->committed.manifest, {}, *block};
|
|
CHECK(retry_prepared(*db, prepared, 5s) == first->committed.head);
|
|
auto resolved = resolve_prepared(*db, prepared, 5s);
|
|
CHECK(resolved && resolved->status == Resolution::committed);
|
|
auto changed = prepared;
|
|
changed.actions = join({record(alice, 1, 3), record(alice, 2, 10)}); // One missing.
|
|
auto retried = retry_prepared(*db, changed, 5s);
|
|
CHECK(!retried && retried.error().kind == LeaseStoreError::commit_conflict);
|
|
resolved = resolve_prepared(*db, changed, 5s);
|
|
CHECK(resolved && resolved->status == Resolution::conflict);
|
|
// Every invalid block is refused before anything else.
|
|
for (const auto& bad :
|
|
{record(alice, 9, 11), record(alice, 9, 0), record(alice, 9, 5, 2), record(Uuid{}, 9, 5),
|
|
record(alice, 0, 5), join({record(bob, 9, 5), record(alice, 9, 5)}),
|
|
join({record(alice, 9, 5), record(alice, 9, 5)}),
|
|
join({record(alice, 9, 6), record(alice, 9, 5)}),
|
|
join({record(alice, 9, 5), record(alice, 9, 6)}), // One id at two ticks.
|
|
std::vector<std::uint8_t>(33, 1)}) {
|
|
auto p = prepared;
|
|
p.actions = bad;
|
|
retried = retry_prepared(*db, p, 5s);
|
|
CHECK(!retried && retried.error().kind == LeaseStoreError::invalid);
|
|
}
|
|
// The local encoder refuses the same blocks it would send.
|
|
CHECK(!encode_actions(std::vector<ActionRecord>{{alice, 9, 11, 1}}, 0, 10));
|
|
CHECK(!encode_actions(std::vector<ActionRecord>{{alice, 9, 5, 1}, {alice, 9, 6, 1}}, 0, 10));
|
|
|
|
// An action id is recorded once in a world: a later batch carrying it commits, and the
|
|
// first record stands.
|
|
RegionBatch<LineageRegion> next{*state};
|
|
for (std::uint64_t tick = 11; tick <= 20; ++tick)
|
|
CHECK(next.advance(evo::test::region_input(tick, false)));
|
|
next.seal();
|
|
const std::vector<ActionRecord> repeat{{alice, 1, 12, 1}, {alice, 3, 12, 1}};
|
|
auto second = publish_lineage_batch(*db, **objects, *lease, first->committed, next,
|
|
tagged(0x03), build, budget, 5s, {}, repeat);
|
|
CHECK(second && second->committed.head.sequence == 2);
|
|
CHECK(holds(find_region_action(*db, r.key, *lease, alice, 1, 5s),
|
|
std::optional{CommittedAction{131, 3, 1}}));
|
|
// A batch without actions still publishes as before v11.
|
|
CHECK(next.confirm(20));
|
|
RegionBatch<LineageRegion> plain{*state};
|
|
for (std::uint64_t tick = 21; tick <= 25; ++tick)
|
|
CHECK(plain.advance(evo::test::region_input(tick, false)));
|
|
plain.seal();
|
|
CHECK(publish_lineage_batch(*db, **objects, *lease, second->committed, plain, tagged(0x04),
|
|
build, budget, 5s));
|
|
// Lookups are fenced like everything else under the lease.
|
|
CHECK(release_region_lease(*db, r.key, *lease, 5s));
|
|
auto later = acquire_region_lease(*db, r.key, build, 20000ms, 5s);
|
|
CHECK(later);
|
|
auto fenced = find_region_action(*db, r.key, *lease, alice, 1, 5s);
|
|
CHECK(!fenced && fenced.error().kind == LeaseStoreError::fenced);
|
|
CHECK(holds(find_region_action(*db, r.key, *later, alice, 3, 5s),
|
|
std::optional{CommittedAction{131, 12, 2}}));
|
|
#ifdef EVO_POSTGRES_SESSIONS
|
|
// The session layer's ledger over the same lookup.
|
|
std::optional<LeaseGrant> current = *later;
|
|
PgSlot slot{options(env("EVO_PG_RUNTIME_USER")), 5s};
|
|
PostgresActionLedger ledger{slot, r.key, [&] { return current; }};
|
|
evo::network::AccountId who;
|
|
who.bytes = alice;
|
|
CHECK(holds(ledger.find(who, 3),
|
|
std::optional{evo::network::ActionOutcome{
|
|
3, evo::network::ActionResult::committed_no_effect, 12, 2}}));
|
|
CHECK(holds(ledger.find(who, 99), std::optional<evo::network::ActionOutcome>{}));
|
|
current.reset();
|
|
CHECK(!ledger.find(who, 3));
|
|
#endif
|
|
CHECK(release_region_lease(*db, r.key, *later, 5s));
|
|
}
|
|
|
|
// Host-level: fast lease (4 s), 1 s commits, as in the host tests.
|
|
const LeasePolicy fast{4000ms, 800ms, 400ms, 400ms, 1000ms};
|
|
struct Crash {};
|
|
HostConfig config(const Region& r) {
|
|
HostConfig c;
|
|
c.key = r.key;
|
|
c.frame = evo::test::region_identity.frame;
|
|
c.lease = fast;
|
|
c.build = build;
|
|
c.retained_builds = {build};
|
|
c.max_queued_actions = 3;
|
|
return c;
|
|
}
|
|
std::unique_ptr<RegionHost<LineageRegion>> host(const Region& r, HostConfig c) {
|
|
auto objects = FilesystemObjectStore::open(r.root, r.key, max_lineage_payload_bytes);
|
|
CHECK(objects);
|
|
auto made = RegionHost<LineageRegion>::create(
|
|
std::move(c), [] { return Postgres::connect(options(env("EVO_PG_RUNTIME_USER")), 2s); },
|
|
std::move(*objects));
|
|
CHECK(made);
|
|
return std::move(*made);
|
|
}
|
|
TickInput input(std::uint64_t tick, std::uint8_t tag) {
|
|
Uuid request{};
|
|
request[0] = tag;
|
|
for (unsigned i = 0; i < 8; ++i)
|
|
request[8 + i] = static_cast<std::uint8_t>((tick >> (8U * i)) & 0xFFU);
|
|
return {request, evo::test::region_input(tick, false)};
|
|
}
|
|
template <typename Done>
|
|
const HostStatus& step_until(RegionHost<LineageRegion>& h, SchedulerTime& t, Done done,
|
|
int limit = 1000) {
|
|
for (int i = 0; i < limit; ++i) {
|
|
t += 50ms;
|
|
const auto& s = h.step(t);
|
|
if (done(s))
|
|
return s;
|
|
std::this_thread::sleep_for(10ms);
|
|
}
|
|
CHECK(false);
|
|
return h.status();
|
|
}
|
|
|
|
void host_actions() {
|
|
auto owner = connect_as(env("PGUSER"));
|
|
{
|
|
// Normal path: acknowledged with the tick each rode; late actions ride the next tick.
|
|
const auto r = provisioned(2);
|
|
auto h = host(r, config(r));
|
|
CHECK(h->submit_action({carol, 1, 3, 1}).error() == AdmissionError::not_ready);
|
|
SchedulerTime t = std::chrono::steady_clock::now();
|
|
CHECK(h->step(t).ready);
|
|
CHECK(h->submit_action({carol, 1, 3, 1}));
|
|
CHECK(h->submit_action({carol, 1, 4, 1}).error() == AdmissionError::duplicate);
|
|
CHECK(h->submit_action({carol, 0, 4, 1}).error() == AdmissionError::invalid);
|
|
CHECK(h->submit_action({carol, 5, 4, 2}).error() == AdmissionError::invalid);
|
|
CHECK(h->submit_action({dave, 1, 1, 1}));
|
|
CHECK(h->submit_action({dave, 2, 2, 1}));
|
|
CHECK(h->submit_action({dave, 3, 2, 1}).error() == AdmissionError::full);
|
|
for (std::uint64_t tick = 1; tick <= 30; ++tick)
|
|
CHECK(h->submit(input(tick, 0x71)));
|
|
step_until(*h, t, [](const HostStatus& s) { return s.committed_sequence >= 1; });
|
|
auto acks = h->take_action_acks();
|
|
CHECK(acks.size() == 3);
|
|
for (const auto& a : acks)
|
|
CHECK(a.commit_sequence == 1 &&
|
|
a.record.tick == (a.record.account == dave ? a.record.action_id : 3u));
|
|
CHECK(stored(*owner, carol, 1) == "132:3:1");
|
|
// Submitted after tick 20 ran for tick 5: rides the first tick simulated next.
|
|
const auto ran = h->status().simulated_tick;
|
|
CHECK(h->submit_action({carol, 7, 5, 1}));
|
|
step_until(*h, t, [](const HostStatus& s) { return s.committed_sequence >= 2; });
|
|
acks = h->take_action_acks();
|
|
CHECK(acks.size() == 1 && acks[0].record.tick == ran + 1);
|
|
CHECK(h->take_dropped_actions().empty());
|
|
}
|
|
{
|
|
// Crash before the transaction: never committed; a successor finds no record.
|
|
const auto r = provisioned(3);
|
|
bool armed = false;
|
|
auto c = config(r);
|
|
c.crash_hook = [&](CrashPoint p) {
|
|
if (armed && p == CrashPoint::before_transaction)
|
|
throw Crash{};
|
|
};
|
|
auto h = host(r, c);
|
|
SchedulerTime t = std::chrono::steady_clock::now();
|
|
CHECK(h->step(t).ready);
|
|
for (std::uint64_t tick = 1; tick <= 30; ++tick)
|
|
CHECK(h->submit(input(tick, 0x72)));
|
|
CHECK(h->submit_action({carol, 10, 5, 1}));
|
|
armed = true;
|
|
bool crashed = false;
|
|
try {
|
|
step_until(*h, t, [](const HostStatus& s) { return s.committed_sequence >= 1; });
|
|
} catch (const Crash&) {
|
|
crashed = true;
|
|
}
|
|
CHECK(crashed);
|
|
h.reset();
|
|
auto successor = host(r, config(r));
|
|
step_until(*successor, t, [](const HostStatus& s) { return s.ready; });
|
|
CHECK(successor->status().committed_sequence == 0 && stored(*owner, carol, 10).empty());
|
|
}
|
|
{
|
|
// Crash after the commit, before any acknowledgement: the record is durable.
|
|
const auto r = provisioned(4);
|
|
bool armed = false;
|
|
auto c = config(r);
|
|
c.crash_hook = [&](CrashPoint p) {
|
|
if (armed && p == CrashPoint::after_commit)
|
|
throw Crash{};
|
|
};
|
|
auto h = host(r, c);
|
|
SchedulerTime t = std::chrono::steady_clock::now();
|
|
CHECK(h->step(t).ready);
|
|
for (std::uint64_t tick = 1; tick <= 30; ++tick)
|
|
CHECK(h->submit(input(tick, 0x73)));
|
|
CHECK(h->submit_action({carol, 11, 5, 1}));
|
|
armed = true;
|
|
bool crashed = false;
|
|
try {
|
|
step_until(*h, t, [](const HostStatus& s) { return s.committed_sequence >= 1; });
|
|
} catch (const Crash&) {
|
|
crashed = true;
|
|
}
|
|
CHECK(crashed && h->take_action_acks().empty());
|
|
h.reset();
|
|
CHECK(stored(*owner, carol, 11) == "134:5:1");
|
|
auto successor = host(r, config(r));
|
|
step_until(*successor, t, [](const HostStatus& s) { return s.ready; });
|
|
CHECK(successor->status().committed_sequence == 1);
|
|
CHECK(successor->submit_action({carol, 11, 40, 1})); // The host does not deduplicate
|
|
// across processes: admission does (find_region_action), and the database refuses a
|
|
// second commit of the id (store_rules).
|
|
}
|
|
{
|
|
// Unknown publication outcome: resolved by the exact bytes, actions included.
|
|
const auto r = provisioned(5);
|
|
auto h = host(r, config(r));
|
|
SchedulerTime t = std::chrono::steady_clock::now();
|
|
CHECK(h->step(t).ready);
|
|
for (std::uint64_t tick = 1; tick <= 30; ++tick)
|
|
CHECK(h->submit(input(tick, 0x74)));
|
|
CHECK(h->submit_action({dave, 12, 2, 1}));
|
|
step_until(*h, t, [](const HostStatus& s) { return s.simulated_tick >= 2; });
|
|
auto lock = connect_as(env("PGUSER"));
|
|
const std::array<std::string, 2> key{std::to_string(r.key.world),
|
|
std::to_string(r.key.region)};
|
|
CHECK(lock->query("BEGIN", {}, 5s));
|
|
CHECK(lock->query("SELECT 1 FROM evolution.region_leases WHERE world_id = "
|
|
"$1::evolution.u64 AND region_id = $2::evolution.u64 FOR UPDATE",
|
|
key, 5s));
|
|
step_until(*h, t, [](const HostStatus& s) { return s.phase == HostPhase::resolving; });
|
|
CHECK(h->take_action_acks().empty() && h->take_dropped_actions().empty());
|
|
CHECK(lock->query("ROLLBACK", {}, 5s));
|
|
step_until(*h, t, [](const HostStatus& s) { return s.phase == HostPhase::running; });
|
|
const auto acks = h->take_action_acks();
|
|
CHECK(acks.size() == 1 && acks[0].record.action_id == 12 && acks[0].commit_sequence == 1);
|
|
CHECK(stored(*owner, dave, 12) == "135:2:1");
|
|
}
|
|
}
|
|
} // namespace
|
|
|
|
void run_region_action_tests() {
|
|
store_rules();
|
|
host_actions();
|
|
std::puts("region action integration passed");
|
|
}
|