Foundation / checks (deploy) (push) Successful in 5s
Foundation / checks (ecs) (push) Successful in 1m56s
Foundation / checks (client) (push) Successful in 12m38s
Foundation / checks (network) (push) Successful in 14m7s
Foundation / checks (tsan) (push) Successful in 8m11s
Foundation / checks (postgres) (push) Failing after 18m3s
Foundation / checks (checks) (push) Successful in 33m8s
Foundation / postgres-tools (push) Skipped
Foundation / server-image (push) Skipped
Foundation / image (push) Skipped
339 lines
17 KiB
C++
339 lines
17 KiB
C++
// Real database publication checks using the portable payload and trace codecs.
|
|
#include "evo/persistence/account_control_store.hpp"
|
|
#include "evo/persistence/checkpoint_store.hpp"
|
|
#include "evo/persistence/life_reservation_store.hpp"
|
|
#include "evo/persistence/region_checkpoint.hpp"
|
|
#include "evo/persistence/region_lease_store.hpp"
|
|
#include "evo/persistence/sha256.hpp"
|
|
#include "evo/server/primitive_region_host.hpp"
|
|
#include <cstdlib>
|
|
#include <source_location>
|
|
#include <stdexcept>
|
|
#include <thread>
|
|
|
|
void run_primitive_publication_tests();
|
|
namespace {
|
|
using namespace evo::persistence;
|
|
using namespace evo::server;
|
|
using namespace std::chrono_literals;
|
|
void check(bool value, std::source_location at = std::source_location::current()) {
|
|
if (!value)
|
|
throw std::runtime_error("primitive publication test line " + std::to_string(at.line()));
|
|
}
|
|
#define CHECK(...) check(static_cast<bool>(__VA_ARGS__))
|
|
std::string env(const char* name) {
|
|
const auto* value = std::getenv(name);
|
|
if (!value)
|
|
throw std::runtime_error("missing PostgreSQL test environment");
|
|
return value;
|
|
}
|
|
auto connect_as(const std::string& user) {
|
|
auto db =
|
|
Postgres::connect({env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), user, {}, {}}, 5s);
|
|
CHECK(db);
|
|
return std::move(*db);
|
|
}
|
|
std::string hex(std::span<const std::uint8_t> bytes) {
|
|
constexpr char digits[] = "0123456789abcdef";
|
|
std::string out = "\\x";
|
|
for (auto b : bytes) {
|
|
out.push_back(digits[b >> 4U]);
|
|
out.push_back(digits[b & 15U]);
|
|
}
|
|
return out;
|
|
}
|
|
template <std::size_t N> auto id(std::uint8_t b) {
|
|
std::array<std::uint8_t, N> out{};
|
|
out.fill(b);
|
|
return out;
|
|
}
|
|
Uuid test_commit(std::uint8_t suffix) {
|
|
Uuid out{};
|
|
out[0] = 0xEF;
|
|
out[15] = suffix;
|
|
return out;
|
|
}
|
|
bool state_is(const auto& result, std::string_view state) {
|
|
return !result && std::string_view(result.error().sqlstate.data()) == state;
|
|
}
|
|
void adapter_publication_and_recovery() {
|
|
auto owner = connect_as(env("PGUSER"));
|
|
auto db = connect_as(env("EVO_PG_RUNTIME_USER"));
|
|
const RegionKey key{9754, 1};
|
|
const auto build = id<32>(0xD7);
|
|
CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
|
"(decode(repeat('d7',32),'hex')) ON CONFLICT DO NOTHING",
|
|
{}, 5s));
|
|
auto endowed = evo::biology::primitive_endowment();
|
|
CHECK(endowed);
|
|
auto funding = endowed->body;
|
|
funding.energy.value = endowed->reserve.energy.value + endowed->construction.value;
|
|
auto base = PrimitiveLives::create({9754, 1, 1, 1, funding, {{0, 0}}});
|
|
CHECK(base);
|
|
auto objects = FilesystemObjectStore::open(env("PGHOST"), key, max_primitive_lives_bytes);
|
|
CHECK(objects);
|
|
const auto genesis =
|
|
provision_primitive_region(*owner, **objects, key, 1, **base, test_commit(10), build, 5s);
|
|
CHECK(genesis);
|
|
CHECK(owner->query("INSERT INTO evolution.life_allocators VALUES(9754,1)", {}, 5s));
|
|
auto lease = acquire_region_lease(*db, key, build, 20000ms, 5s);
|
|
CHECK(lease);
|
|
const auto account = id<16>(0x72);
|
|
const auto control = claim_account_control(*db, key, *lease, account, 1, 1, 20000ms, 5s);
|
|
CHECK(control);
|
|
const auto reserved =
|
|
reserve_primitive_life(*db, key, *lease, account, control->generation, 1, 1, 5s);
|
|
CHECK(reserved && reserved->identity == 1);
|
|
const auto predecessor_digest = sha256(genesis->manifest);
|
|
CHECK(predecessor_digest);
|
|
PrimitiveBatch batch{
|
|
{key.world, key.region, 1, genesis->head.commit, *predecessor_digest, 0, 1},
|
|
{{{account, 1}, reserved->identity, control->generation, {1}}}};
|
|
auto candidate = replay_primitive_batch(**base, batch);
|
|
CHECK(candidate);
|
|
const auto published = publish_primitive_batch(*db, **objects, *lease, *genesis, batch,
|
|
**candidate, test_commit(11), build, 5s);
|
|
CHECK(published && published->committed.head.sequence == 1);
|
|
CHECK(release_region_lease(*db, key, *lease, 5s));
|
|
const auto retry = publish_primitive_batch(*db, **objects, *lease, *genesis, batch, **candidate,
|
|
test_commit(11), build, 5s);
|
|
CHECK(retry && retry->committed.manifest == published->committed.manifest &&
|
|
retry->committed.head == published->committed.head);
|
|
const auto successor_lease = acquire_region_lease(*db, key, build, 20000ms, 5s);
|
|
CHECK(successor_lease && successor_lease->epoch > lease->epoch);
|
|
const std::array<Sha256Digest, 1> retained{build};
|
|
auto recovered = recover_primitive_checkpoint(*db, **objects, key, retained, 5s);
|
|
CHECK(recovered && recovered->bindings.size() == 1 &&
|
|
recovered->state->snapshot() == (*candidate)->snapshot());
|
|
CHECK(release_region_lease(*db, key, *successor_lease, 5s));
|
|
}
|
|
|
|
void primitive_host_fenced_spawn() {
|
|
auto owner = connect_as(env("PGUSER"));
|
|
const RegionKey key{9755, 1};
|
|
const auto build = id<32>(0xD8);
|
|
CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
|
"(decode(repeat('d8',32),'hex')) ON CONFLICT DO NOTHING",
|
|
{}, 5s));
|
|
const auto endowed = evo::biology::primitive_endowment();
|
|
CHECK(endowed);
|
|
auto funding = endowed->body;
|
|
funding.energy.value = endowed->reserve.energy.value + endowed->construction.value;
|
|
auto opening = PrimitiveLives::create({key.world, key.region, 1, 1, funding, {{0, 0}}});
|
|
CHECK(opening);
|
|
auto objects = FilesystemObjectStore::open(env("PGHOST"), key, max_primitive_lives_bytes);
|
|
CHECK(objects);
|
|
CHECK(provision_primitive_region(*owner, **objects, key, 1, **opening, test_commit(20), build,
|
|
5s));
|
|
CHECK(owner->query("INSERT INTO evolution.life_allocators VALUES(9755,1)", {}, 5s));
|
|
|
|
auto host_objects = FilesystemObjectStore::open(env("PGHOST"), key, max_primitive_lives_bytes);
|
|
CHECK(host_objects);
|
|
const auto pg = PgOptions{
|
|
env("PGHOST"), {}, env("PGPORT"), env("PGDATABASE"), env("EVO_PG_RUNTIME_USER"), {}, {}};
|
|
auto made = PrimitiveRegionHost::create(
|
|
{key, 1, {}, build, {build}}, [pg] { return Postgres::connect(pg, 5s); },
|
|
std::move(*host_objects));
|
|
CHECK(made && (*made)->step().ready);
|
|
auto runtime = connect_as(env("EVO_PG_RUNTIME_USER"));
|
|
const auto account = id<16>(0x73);
|
|
auto control =
|
|
claim_account_control(*runtime, key, *(*made)->authority(), account, 1, 1, 20000ms, 5s);
|
|
CHECK(control);
|
|
const auto born = (*made)->new_life(account, control->generation, 1, 1);
|
|
CHECK(born.kind == PrimitiveHostReplyKind::born && born.published &&
|
|
born.published->identity == 1 && born.published->born == 1 &&
|
|
born.published->commit_sequence == 1);
|
|
CHECK((*made)->status().ready && (*made)->state()->lives().size() == 1 &&
|
|
(*made)->status().committed_tick == 1);
|
|
const auto exact = (*made)->new_life(account, control->generation, 1, 1);
|
|
CHECK(exact == born);
|
|
const auto second = (*made)->new_life(account, control->generation, 2, 1);
|
|
CHECK(second.kind == PrimitiveHostReplyKind::refused &&
|
|
second.reservation.refusal == LifeRefusal::existing_life);
|
|
(*made)->drain();
|
|
CHECK((*made)->status().phase == HostPhase::stopped && !(*made)->status().ready);
|
|
}
|
|
} // namespace
|
|
|
|
void run_primitive_publication_tests() {
|
|
static_assert(max_primitive_lives_bytes == 3266800); // SQL cap must match C++.
|
|
auto owner = connect_as(env("PGUSER"));
|
|
auto db = connect_as(env("EVO_PG_RUNTIME_USER"));
|
|
const RegionKey key{9753, 1};
|
|
const auto build = id<32>(0xD7), payload_digest = id<32>(0xA7);
|
|
CHECK(owner->query("INSERT INTO evolution.retained_builds(build_identity) VALUES "
|
|
"(decode(repeat('d7',32),'hex'))",
|
|
{}, 5s));
|
|
auto endowed = evo::biology::primitive_endowment();
|
|
CHECK(endowed);
|
|
auto funding = endowed->body;
|
|
funding.energy.value = endowed->reserve.energy.value + endowed->construction.value;
|
|
auto base = PrimitiveLives::create({9753, 1, 1, 1, funding, {{0, 0}}});
|
|
CHECK(base);
|
|
auto opening = (*base)->snapshot();
|
|
const auto initial_digest = sha256(opening);
|
|
CHECK(initial_digest);
|
|
auto genesis = genesis_manifest(key, 1, test_commit(1), 50000, build, opening.size(),
|
|
*initial_digest, CheckpointProfile::primitive_lives_v1);
|
|
CHECK(genesis);
|
|
auto initial = encode_manifest(*genesis);
|
|
CHECK(initial);
|
|
const auto provisioned = owner->query("SELECT evolution.provision_region_checkpoint($1::bytea)",
|
|
std::array{hex(*initial)}, 5s);
|
|
if (!provisioned)
|
|
throw std::runtime_error("primitive genesis SQLSTATE " +
|
|
std::string{provisioned.error().sqlstate.data(), 5});
|
|
CHECK(owner->query("INSERT INTO evolution.life_allocators VALUES(9753,1)", {}, 5s));
|
|
auto lease = acquire_region_lease(*db, key, build, 20000ms, 5s);
|
|
CHECK(lease);
|
|
const auto alice = id<16>(0x71);
|
|
auto control = claim_account_control(*db, key, *lease, alice, 1, 1, 20000ms, 5s);
|
|
CHECK(control);
|
|
auto reserved = reserve_primitive_life(*db, key, *lease, alice, control->generation, 1, 1, 5s);
|
|
CHECK(reserved && reserved->identity == 1);
|
|
const auto predecessor_digest = sha256(*initial);
|
|
CHECK(predecessor_digest);
|
|
PrimitiveBatch batch{{9753, 1, 1, genesis->commit, *predecessor_digest, 0, 1},
|
|
{{{alice, 1}, 1, control->generation, {1}}}};
|
|
auto replayed = replay_primitive_batch(**base, batch);
|
|
CHECK(replayed);
|
|
const auto payload = (*replayed)->snapshot();
|
|
auto trace = encode_primitive_batch(batch);
|
|
CHECK(trace);
|
|
auto trace_digest = sha256(*trace), actual_payload_digest = sha256(payload);
|
|
CHECK(trace_digest && actual_payload_digest);
|
|
auto next = successor_manifest(*genesis, *predecessor_digest, test_commit(2), *lease, 1, build,
|
|
payload.size(), *actual_payload_digest,
|
|
{trace->size(), *trace_digest, 0, {}});
|
|
CHECK(next);
|
|
auto manifest = encode_manifest(*next);
|
|
CHECK(manifest);
|
|
const auto publish = [&](const auto& m, const auto& t) {
|
|
return db->query(
|
|
"SELECT * FROM evolution.publish_primitive_checkpoint($1::bytea,$2::bytea)",
|
|
std::array{hex(m), hex(t)}, 5s);
|
|
};
|
|
// Old runtime publication routes cannot insert primitive commits.
|
|
auto bypass = publish_region_checkpoint(*db, *manifest, {}, 5s);
|
|
CHECK(!bypass && bypass.error().kind == LeaseStoreError::unsupported);
|
|
auto bypass_actions = publish_region_checkpoint(*db, *manifest, {}, {}, 5s);
|
|
CHECK(!bypass_actions && bypass_actions.error().kind == LeaseStoreError::unsupported);
|
|
CHECK(!db->query("SELECT * FROM evolution.primitive_publications", {}, 5s));
|
|
CHECK(!db->query("SELECT * FROM evolution.primitive_life_bindings", {}, 5s));
|
|
const auto refuses = [&](auto change, std::string_view sqlstate) {
|
|
auto altered = batch;
|
|
change(altered);
|
|
auto altered_trace = encode_primitive_batch(altered);
|
|
CHECK(altered_trace);
|
|
auto altered_manifest = *next;
|
|
altered_manifest.batch.trace_sha256 = *sha256(*altered_trace);
|
|
auto bytes = encode_manifest(altered_manifest);
|
|
CHECK(bytes);
|
|
CHECK(state_is(publish(*bytes, *altered_trace), sqlstate));
|
|
const auto head = load_region_head(*db, key, 5s);
|
|
CHECK(head && head->sequence == 0);
|
|
};
|
|
refuses([](auto& b) { b.births[0].generation = 99; }, "EVL02");
|
|
refuses([](auto& b) { b.births[0].identity = 99; }, "EVL16");
|
|
refuses([](auto& b) { b.births[0].request.action = 99; }, "EVL16");
|
|
refuses([](auto& b) { b.header.frame = 99; }, "22023");
|
|
const auto committed = publish(*manifest, *trace);
|
|
CHECK(committed);
|
|
db = connect_as(env("EVO_PG_RUNTIME_USER"));
|
|
const auto retry = publish(*manifest, *trace);
|
|
CHECK(retry && *retry == *committed);
|
|
CHECK(load_region_head(*db, key, 5s)->sequence == 1);
|
|
auto bindings = db->query("SELECT * FROM evolution.load_primitive_lives(9753,1)", {}, 5s);
|
|
CHECK(bindings && bindings->size() == 1 && (*bindings)[0][2] == "1" &&
|
|
(*bindings)[0][5] == "1");
|
|
const auto durable_outcome =
|
|
query_primitive_life(*db, key, *lease, alice, control->generation, 1, 5s);
|
|
CHECK(durable_outcome && *durable_outcome && (**durable_outcome).published &&
|
|
(**durable_outcome).published->identity == 1 &&
|
|
(**durable_outcome).published->commit_sequence == 1);
|
|
auto reject = reject_primitive_life(*db, key, *lease, alice, control->generation, 1,
|
|
LifeRefusal::cancelled, 5s);
|
|
CHECK(!reject && reject.error().kind == LifeReservationError::existing_life);
|
|
auto changed = *next;
|
|
changed.payload_sha256 = payload_digest;
|
|
CHECK(state_is(publish(*encode_manifest(changed), *trace), "EVL07"));
|
|
// Same birth under a later commit is rejected, not first-result-wins.
|
|
auto repeated = batch;
|
|
repeated.header.predecessor = next->commit;
|
|
repeated.header.predecessor_digest = *sha256(*manifest);
|
|
repeated.header.begin_tick = 1;
|
|
repeated.header.end_tick = 2;
|
|
repeated.births[0].born.value = 2;
|
|
auto repeated_trace = encode_primitive_batch(repeated);
|
|
CHECK(repeated_trace);
|
|
auto later = successor_manifest(*next, *sha256(*manifest), test_commit(3), *lease, 2, build,
|
|
payload.size(), *actual_payload_digest,
|
|
{repeated_trace->size(), *sha256(*repeated_trace), 0, {}});
|
|
CHECK(later && state_is(publish(*encode_manifest(*later), *repeated_trace), "EVL16"));
|
|
repeated.births.clear();
|
|
auto idle_trace = encode_primitive_batch(repeated);
|
|
CHECK(idle_trace);
|
|
later->batch = {idle_trace->size(), *sha256(*idle_trace), 0, {}};
|
|
auto idle_manifest = encode_manifest(*later);
|
|
CHECK(idle_manifest);
|
|
CHECK(db->query("BEGIN", {}, 5s));
|
|
CHECK(publish(*idle_manifest, *idle_trace));
|
|
CHECK(db->query("ROLLBACK", {}, 5s));
|
|
CHECK(load_region_head(*db, key, 5s)->sequence == 1);
|
|
CHECK(publish(*idle_manifest, *idle_trace));
|
|
CHECK(load_region_head(*db, key, 5s)->sequence == 2);
|
|
// Both exact retries enter before either can commit. The loser must recheck
|
|
// evidence after the lease lock wait, rather than refuse a duplicate birth.
|
|
repeated.header.predecessor = later->commit;
|
|
repeated.header.predecessor_digest = *sha256(*idle_manifest);
|
|
repeated.header.begin_tick = 2;
|
|
repeated.header.end_tick = 3;
|
|
auto concurrent_trace = encode_primitive_batch(repeated);
|
|
CHECK(concurrent_trace);
|
|
auto concurrent = successor_manifest(
|
|
*later, *sha256(*idle_manifest), test_commit(4), *lease, 3, build, payload.size(),
|
|
*actual_payload_digest, {concurrent_trace->size(), *sha256(*concurrent_trace), 0, {}});
|
|
CHECK(concurrent);
|
|
const auto concurrent_manifest = encode_manifest(*concurrent);
|
|
CHECK(concurrent_manifest);
|
|
auto first_db = connect_as(env("EVO_PG_RUNTIME_USER"));
|
|
auto second_db = connect_as(env("EVO_PG_RUNTIME_USER"));
|
|
CHECK(owner->query("BEGIN", {}, 5s));
|
|
CHECK(owner->query(
|
|
"SELECT * FROM evolution.region_leases WHERE world_id=9753 AND region_id=1 FOR UPDATE", {},
|
|
5s));
|
|
std::optional<std::expected<PgRows, PgFailure>> first_result, second_result;
|
|
const std::array parameters{hex(*concurrent_manifest), hex(*concurrent_trace)};
|
|
const std::string sql =
|
|
"SELECT * FROM evolution.publish_primitive_checkpoint($1::bytea,$2::bytea)";
|
|
std::jthread first([&] { first_result = first_db->query(sql, parameters, 5s); });
|
|
std::jthread second([&] { second_result = second_db->query(sql, parameters, 5s); });
|
|
bool waiting = false;
|
|
for (int attempt = 0; attempt < 200; ++attempt) {
|
|
CHECK(owner->query("SELECT pg_stat_clear_snapshot()", {}, 5s));
|
|
auto active =
|
|
owner->query("SELECT count(*) FROM pg_stat_activity WHERE wait_event_type='Lock' "
|
|
"AND query LIKE 'SELECT * FROM evolution.publish_primitive_checkpoint%'",
|
|
{}, 5s);
|
|
CHECK(active);
|
|
if ((*active)[0][0] == "2") {
|
|
waiting = true;
|
|
break;
|
|
}
|
|
std::this_thread::sleep_for(10ms);
|
|
}
|
|
CHECK(owner->query("COMMIT", {}, 5s));
|
|
first.join();
|
|
second.join();
|
|
if (!waiting || !first_result || !second_result || !*first_result || !*second_result)
|
|
throw std::runtime_error("concurrent primitive publication did not converge");
|
|
CHECK(**first_result == **second_result);
|
|
CHECK(load_region_head(*db, key, 5s)->sequence == 3);
|
|
CHECK(release_region_lease(*db, key, *lease, 5s));
|
|
// Exact committed retry survives loss of authority, but cannot originate work.
|
|
CHECK(publish(*manifest, *trace));
|
|
adapter_publication_and_recovery();
|
|
primitive_host_fenced_spawn();
|
|
}
|